jesse-uxandGrok (agent loop)
9455fbfc4f
Expired daily files are pruned before a new backup, a verify older than 36 hours raises an alert, and the host monitors call sudo with the production env file. The Linux 0600 check is on the staging quick list. Co-Authored-By: Grok (agent loop)
283 lines
7.6 KiB
Bash
283 lines
7.6 KiB
Bash
#!/usr/bin/env bash
|
|
# Probe the public health URL, or the host disk. Install the health mode on
|
|
# the Gitea runner, not on the production host. Install disk mode on the host.
|
|
set -euo pipefail
|
|
set +x
|
|
umask 077
|
|
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=notify-feishu.sh
|
|
source "$script_dir/notify-feishu.sh"
|
|
|
|
mode="health"
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--mode)
|
|
mode="${2:-}"
|
|
shift 2
|
|
;;
|
|
--mode=*)
|
|
mode="${1#--mode=}"
|
|
shift
|
|
;;
|
|
*)
|
|
echo "unknown argument" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
case "$mode" in
|
|
health|disk|all) ;;
|
|
*)
|
|
echo "unknown mode" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
|
ensure_private_dir "$state_dir"
|
|
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
|
load_monitor_env "$env_file"
|
|
|
|
health_url="${JYOTISHA_HEALTH_URL:-https://jyotisha.chat/api/health}"
|
|
timeout="${JYOTISHA_HEALTH_TIMEOUT:-10}"
|
|
[[ "$timeout" =~ ^[1-9][0-9]?$ ]] && (( timeout <= 30 )) || {
|
|
echo "health timeout must be 1 to 30 seconds" >&2
|
|
exit 2
|
|
}
|
|
if [ "$mode" != "disk" ]; then
|
|
[[ "$health_url" == "https://jyotisha.chat/api/health" \
|
|
|| "$health_url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/health$ \
|
|
|| "$health_url" =~ ^http://localhost:[0-9]{1,5}/health$ ]] || {
|
|
echo "unexpected health url" >&2
|
|
exit 2
|
|
}
|
|
fi
|
|
|
|
state_file="$state_dir/monitor.state"
|
|
consecutive_failures=0
|
|
alert_open=0
|
|
last_commit=""
|
|
commit_alerted=""
|
|
disk_alert_open=0
|
|
backup_stale_open=0
|
|
|
|
read_state() {
|
|
local line value
|
|
[ -f "$state_file" ] && [ ! -L "$state_file" ] || return 0
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
consecutive_failures=[0-9]*)
|
|
consecutive_failures="${line#consecutive_failures=}"
|
|
;;
|
|
alert_open=[01])
|
|
alert_open="${line#alert_open=}"
|
|
;;
|
|
last_commit=)
|
|
last_commit=""
|
|
;;
|
|
last_commit=*)
|
|
value="${line#last_commit=}"
|
|
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && last_commit="$value"
|
|
;;
|
|
commit_alerted=)
|
|
commit_alerted=""
|
|
;;
|
|
commit_alerted=*)
|
|
value="${line#commit_alerted=}"
|
|
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && commit_alerted="$value"
|
|
;;
|
|
disk_alert_open=[01])
|
|
disk_alert_open="${line#disk_alert_open=}"
|
|
;;
|
|
backup_stale_open=[01])
|
|
backup_stale_open="${line#backup_stale_open=}"
|
|
;;
|
|
esac
|
|
done <"$state_file"
|
|
}
|
|
|
|
write_state() {
|
|
local tmp
|
|
tmp="$state_file.tmp.$$"
|
|
printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\nbackup_stale_open=%s\n' \
|
|
"$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" "$backup_stale_open" >"$tmp"
|
|
if [ "$(uname -s)" = "Linux" ]; then
|
|
chmod 600 "$tmp"
|
|
fi
|
|
mv "$tmp" "$state_file"
|
|
}
|
|
|
|
stamp() {
|
|
date -u +%Y-%m-%dT%H:%M:%SZ
|
|
}
|
|
|
|
send_alert() {
|
|
local service="$1"
|
|
local route="$2"
|
|
local error_code="$3"
|
|
local count="$4"
|
|
local request_id="${5:--}"
|
|
post_monitor_alert "service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_id} time=$(stamp)"
|
|
}
|
|
|
|
release_window_open() {
|
|
local marker="$state_dir/release-window"
|
|
local mtime now age
|
|
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
|
|
mtime="$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker")"
|
|
now="$(date +%s)"
|
|
age=$((now - mtime))
|
|
[ "$age" -ge 0 ] && [ "$age" -le 2700 ]
|
|
}
|
|
|
|
check_health() {
|
|
local body_file http_code curl_status status_line commit_line status commit healthy py
|
|
body_file="$(mktemp)"
|
|
http_code="000"
|
|
curl_status=0
|
|
http_code="$(curl --silent --show-error --connect-timeout "$timeout" --max-time "$timeout" \
|
|
-o "$body_file" -w '%{http_code}' "$health_url")" || curl_status=$?
|
|
status=""
|
|
commit=""
|
|
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ]; then
|
|
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
|
py="$JYOTISHA_PYTHON"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
py="python3"
|
|
else
|
|
py="python"
|
|
fi
|
|
status_line="$("$py" - "$body_file" <<'PY'
|
|
import json
|
|
import sys
|
|
try:
|
|
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
except Exception:
|
|
print("")
|
|
print("")
|
|
raise SystemExit(0)
|
|
if not isinstance(payload, dict):
|
|
print("")
|
|
print("")
|
|
raise SystemExit(0)
|
|
status = payload.get("status")
|
|
deployment = payload.get("deployment")
|
|
commit = ""
|
|
if isinstance(deployment, dict):
|
|
raw = deployment.get("gitCommit")
|
|
if isinstance(raw, str):
|
|
commit = raw
|
|
print(status if isinstance(status, str) else "")
|
|
print(commit)
|
|
PY
|
|
)"
|
|
status="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==1 {print}')"
|
|
commit="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==2 {print}')"
|
|
fi
|
|
rm -f -- "$body_file"
|
|
|
|
healthy=0
|
|
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ] && [ "$status" = "ok" ]; then
|
|
healthy=1
|
|
fi
|
|
|
|
if [ "$healthy" -eq 1 ]; then
|
|
if [ "$alert_open" -eq 1 ]; then
|
|
send_alert site /api/health recovered 0 -
|
|
alert_open=0
|
|
fi
|
|
consecutive_failures=0
|
|
if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then
|
|
if [ -n "$last_commit" ] && [ "$commit" != "$last_commit" ] && [ "$commit_alerted" != "$commit" ]; then
|
|
if ! release_window_open; then
|
|
send_alert site /api/health commit_changed 1 -
|
|
commit_alerted="$commit"
|
|
fi
|
|
fi
|
|
last_commit="$commit"
|
|
fi
|
|
else
|
|
consecutive_failures=$((consecutive_failures + 1))
|
|
if [ "$consecutive_failures" -ge 3 ] && [ "$alert_open" -eq 0 ]; then
|
|
send_alert site /api/health health_down "$consecutive_failures" -
|
|
alert_open=1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
disk_percent() {
|
|
df -Pk "${JYOTISHA_DISK_PATH:-/}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}'
|
|
}
|
|
|
|
check_disk() {
|
|
local percent
|
|
percent="$(disk_percent)"
|
|
[[ "$percent" =~ ^[0-9]+$ ]] && [ "$percent" -le 100 ] || {
|
|
echo "disk usage is unreadable" >&2
|
|
exit 1
|
|
}
|
|
if [ "$percent" -ge 80 ]; then
|
|
if [ "$disk_alert_open" -eq 0 ]; then
|
|
send_alert host - disk_high "$percent" -
|
|
disk_alert_open=1
|
|
fi
|
|
elif [ "$disk_alert_open" -eq 1 ]; then
|
|
send_alert host - recovered 0 -
|
|
disk_alert_open=0
|
|
fi
|
|
}
|
|
|
|
check_daily_backup() {
|
|
local record now_text now_epoch verified verified_epoch age stale hours
|
|
record="${JYOTISHA_DAILY_BACKUP_RECORD:-/opt/jyotisha-production/.state/daily-backup-latest.env}"
|
|
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
|
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || {
|
|
echo "clock is unsafe" >&2
|
|
exit 2
|
|
}
|
|
now_epoch="$(date -u -d "$now_text" +%s)"
|
|
[[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2
|
|
stale=1
|
|
hours=36
|
|
if [ -f "$record" ] && [ ! -L "$record" ]; then
|
|
verified="$(awk -F= '$1 == "VERIFIED_AT" { print $2 }' "$record" | tr -d '\r')"
|
|
if [[ "$verified" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
|
|
verified_epoch="$(date -u -d "$verified" +%s)"
|
|
age=$((now_epoch - verified_epoch))
|
|
if [ "$age" -le 129600 ]; then
|
|
stale=0
|
|
else
|
|
hours=$((age / 3600))
|
|
if [ "$hours" -lt 36 ]; then
|
|
hours=36
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$stale" -eq 1 ]; then
|
|
if [ "$backup_stale_open" -eq 0 ]; then
|
|
send_alert host - backup_stale "$hours" -
|
|
backup_stale_open=1
|
|
fi
|
|
elif [ "$backup_stale_open" -eq 1 ]; then
|
|
send_alert host - recovered 0 -
|
|
backup_stale_open=0
|
|
fi
|
|
}
|
|
|
|
read_state
|
|
case "$mode" in
|
|
health) check_health ;;
|
|
disk)
|
|
check_disk
|
|
check_daily_backup
|
|
;;
|
|
all)
|
|
check_health
|
|
check_disk
|
|
check_daily_backup
|
|
;;
|
|
esac
|
|
write_state
|