给内测补上每日加密备份、宕机告警和日志上限。

主机安装和一次恢复演练留给产品。页面没有变化。

Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
2026-10-15 12:58:34 +08:00
co-authored by Grok (agent loop)
parent 71f7fac3a1
commit 53d8df257d
19 changed files with 1761 additions and 2 deletions
+160
View File
@@ -0,0 +1,160 @@
"""Classify fictional or container log lines into alert rows.
Outbound fields are service, route, errorCode, count, and request ids.
Lines with non-ASCII text, an email address, or a second calendar date are
dropped so conversation text and birth dates cannot leave the host.
"""
from __future__ import annotations
import json
import re
import sys
from collections import OrderedDict
from datetime import datetime, timedelta, timezone
EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
UUID = re.compile(
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
)
ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$")
ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$")
STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b")
FIXTURE = re.compile(
r"^(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?P<svc>web|api) (?P<rest>.*)$"
)
DOCKER = re.compile(
r"^(?P<svc>web|api)-\d+\s+\|\s+"
r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P<rest>.*)$"
)
CST = timezone(timedelta(hours=8))
def unsafe(line: str) -> bool:
if any(ord(char) > 126 for char in line):
return True
lowered = line.lower()
if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered:
return True
return len(DATE.findall(line)) >= 2
def parse_line(line: str) -> tuple[str, str, str] | None:
match = FIXTURE.match(line) or DOCKER.match(line)
if not match:
return None
return match.group("ts"), match.group("svc"), match.group("rest")
def timestamp_cst_date(timestamp: str):
stamp = timestamp
if "." in stamp:
stamp = stamp.split(".", 1)[0] + "Z"
moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
return moment.astimezone(CST).date()
def extract_fields(rest: str) -> tuple[str, str, str, int | None]:
route = "-"
error_code = ""
request_id = ""
status: int | None = None
marker = "[agent-observability]"
if marker in rest:
raw = rest.split(marker, 1)[1].strip()
try:
payload = json.loads(raw)
except json.JSONDecodeError:
payload = None
if isinstance(payload, dict):
code = payload.get("errorCode")
if isinstance(code, str) and ERROR_CODE.match(code):
error_code = code
candidate = payload.get("requestId")
if isinstance(candidate, str) and UUID.match(candidate):
request_id = candidate.lower()
route_value = payload.get("route")
if isinstance(route_value, str) and ROUTE.match(route_value):
route = route_value
status_match = STATUS.search(rest)
if status_match:
status = int(status_match.group(1))
return route, error_code, request_id, status
def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]:
now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
yesterday = now.astimezone(CST).date() - timedelta(days=1)
counts: OrderedDict[tuple[str, str, str], int] = OrderedDict()
request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict()
fivexx_counts: OrderedDict[str, int] = OrderedDict()
fivexx_ids: OrderedDict[str, list[str]] = OrderedDict()
def add(key: tuple[str, str, str], request_id: str) -> None:
counts[key] = counts.get(key, 0) + 1
bucket = request_ids.setdefault(key, [])
if request_id and request_id not in bucket and len(bucket) < 5:
bucket.append(request_id)
for raw in lines:
line = raw.rstrip("\r\n")
if not line or unsafe(line):
continue
parsed = parse_line(line)
if parsed is None:
continue
timestamp, service, rest = parsed
if mode == "daily" and timestamp_cst_date(timestamp) != yesterday:
continue
route, error_code, request_id, status = extract_fields(rest)
if error_code:
add((service, route, error_code), request_id)
elif status is not None:
add((service, route, f"http_{status}"), request_id)
if status is not None and 500 <= status <= 599:
fivexx_counts[service] = fivexx_counts.get(service, 0) + 1
bucket = fivexx_ids.setdefault(service, [])
if request_id and request_id not in bucket and len(bucket) < 5:
bucket.append(request_id)
rows: list[str] = []
def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None:
shown = ",".join(ids) if ids else "-"
rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}")
for key, count in counts.items():
if mode == "window" and count < error_threshold:
continue
if count <= 0:
continue
emit(*key, count, request_ids.get(key, []))
for service, count in fivexx_counts.items():
if mode == "window" and count < fivexx_threshold:
continue
if count <= 0:
continue
emit(service, "-", "http_5xx", count, fivexx_ids.get(service, []))
return rows
def main(argv: list[str]) -> int:
if len(argv) != 5:
print("usage: classify_error_log.py <fixture> <window|daily> <now> <error-threshold> <5xx-threshold>", file=sys.stderr)
return 2
fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv
if mode not in {"window", "daily"}:
print("unknown mode", file=sys.stderr)
return 2
error_threshold = int(error_threshold_text)
fivexx_threshold = int(fivexx_threshold_text)
with open(fixture, encoding="utf-8") as handle:
lines = handle.readlines()
for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold):
print(row)
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env bash
# Count recent web/api errors and alert without sending log text.
set -euo pipefail
set +x
umask 077
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=notify-feishu.sh
source "$script_dir/notify-feishu.sh"
mode="window"
case "${1:-}" in
""|--window) mode="window" ;;
--daily) mode="daily" ;;
*)
echo "unknown argument" >&2
exit 2
;;
esac
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
ensure_private_dir "$state_dir"
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
load_monitor_env "$env_file"
error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}"
fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}"
[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
py="$JYOTISHA_PYTHON"
elif command -v python3 >/dev/null 2>&1; then
py="python3"
else
py="python"
fi
fixture="${JYOTISHA_LOG_FIXTURE:-}"
cleanup() {
if [ -n "${generated_fixture:-}" ]; then
rm -f -- "$generated_fixture"
fi
}
trap cleanup EXIT
generated_fixture=""
if [ -z "$fixture" ]; then
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
since="5m"
[ "$mode" = "daily" ] && since="48h"
generated_fixture="$(mktemp)"
docker compose -p jyotisha-production \
--env-file "$compose_dir/.env.production" \
-f "$compose_dir/deploy/docker-compose.server.yml" \
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
-f "$compose_dir/deploy/docker-compose.production.yml" \
logs --since "$since" --no-color --timestamps web api >"$generated_fixture"
fixture="$generated_fixture"
fi
[ -f "$fixture" ] && [ ! -L "$fixture" ] || {
echo "log fixture is missing" >&2
exit 1
}
if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then
echo "could not classify logs" >&2
exit 1
fi
stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
text=""
if [ -n "$rows" ]; then
while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do
[ -n "${service:-}" ] || continue
line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}"
text="${text}${line}"$'\n'
done <<<"$rows" || true
elif [ "$mode" = "daily" ]; then
text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n'
fi
if [ -n "$text" ]; then
post_monitor_alert "${text%$'\n'}"
fi
+236
View File
@@ -0,0 +1,236 @@
#!/usr/bin/env bash
# Probe the public health URL, or the host disk. Install the health mode on
# the Gitea runner, not on the production host. Install disk mode on the host.
set -euo pipefail
set +x
umask 077
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=notify-feishu.sh
source "$script_dir/notify-feishu.sh"
mode="health"
while [ $# -gt 0 ]; do
case "$1" in
--mode)
mode="${2:-}"
shift 2
;;
--mode=*)
mode="${1#--mode=}"
shift
;;
*)
echo "unknown argument" >&2
exit 2
;;
esac
done
case "$mode" in
health|disk|all) ;;
*)
echo "unknown mode" >&2
exit 2
;;
esac
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
ensure_private_dir "$state_dir"
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
load_monitor_env "$env_file"
health_url="${JYOTISHA_HEALTH_URL:-https://jyotisha.chat/api/health}"
timeout="${JYOTISHA_HEALTH_TIMEOUT:-10}"
[[ "$timeout" =~ ^[1-9][0-9]?$ ]] && (( timeout <= 30 )) || {
echo "health timeout must be 1 to 30 seconds" >&2
exit 2
}
if [ "$mode" != "disk" ]; then
[[ "$health_url" == "https://jyotisha.chat/api/health" \
|| "$health_url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/health$ \
|| "$health_url" =~ ^http://localhost:[0-9]{1,5}/health$ ]] || {
echo "unexpected health url" >&2
exit 2
}
fi
state_file="$state_dir/monitor.state"
consecutive_failures=0
alert_open=0
last_commit=""
commit_alerted=""
disk_alert_open=0
read_state() {
local line value
[ -f "$state_file" ] && [ ! -L "$state_file" ] || return 0
while IFS= read -r line || [ -n "$line" ]; do
case "$line" in
consecutive_failures=[0-9]*)
consecutive_failures="${line#consecutive_failures=}"
;;
alert_open=[01])
alert_open="${line#alert_open=}"
;;
last_commit=)
last_commit=""
;;
last_commit=*)
value="${line#last_commit=}"
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && last_commit="$value"
;;
commit_alerted=)
commit_alerted=""
;;
commit_alerted=*)
value="${line#commit_alerted=}"
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && commit_alerted="$value"
;;
disk_alert_open=[01])
disk_alert_open="${line#disk_alert_open=}"
;;
esac
done <"$state_file"
}
write_state() {
local tmp
tmp="$state_file.tmp.$$"
printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\n' \
"$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" >"$tmp"
if [ "$(uname -s)" = "Linux" ]; then
chmod 600 "$tmp"
fi
mv "$tmp" "$state_file"
}
stamp() {
date -u +%Y-%m-%dT%H:%M:%SZ
}
send_alert() {
local service="$1"
local route="$2"
local error_code="$3"
local count="$4"
local request_id="${5:--}"
post_monitor_alert "service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_id} time=$(stamp)"
}
release_window_open() {
local marker="$state_dir/release-window"
local mtime now age
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
mtime="$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker")"
now="$(date +%s)"
age=$((now - mtime))
[ "$age" -ge 0 ] && [ "$age" -le 2700 ]
}
check_health() {
local body_file http_code curl_status status_line commit_line status commit healthy py
body_file="$(mktemp)"
http_code="000"
curl_status=0
http_code="$(curl --silent --show-error --connect-timeout "$timeout" --max-time "$timeout" \
-o "$body_file" -w '%{http_code}' "$health_url")" || curl_status=$?
status=""
commit=""
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ]; then
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
py="$JYOTISHA_PYTHON"
elif command -v python3 >/dev/null 2>&1; then
py="python3"
else
py="python"
fi
status_line="$("$py" - "$body_file" <<'PY'
import json
import sys
try:
payload = json.load(open(sys.argv[1], encoding="utf-8"))
except Exception:
print("")
print("")
raise SystemExit(0)
if not isinstance(payload, dict):
print("")
print("")
raise SystemExit(0)
status = payload.get("status")
deployment = payload.get("deployment")
commit = ""
if isinstance(deployment, dict):
raw = deployment.get("gitCommit")
if isinstance(raw, str):
commit = raw
print(status if isinstance(status, str) else "")
print(commit)
PY
)"
status="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==1 {print}')"
commit="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==2 {print}')"
fi
rm -f -- "$body_file"
healthy=0
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ] && [ "$status" = "ok" ]; then
healthy=1
fi
if [ "$healthy" -eq 1 ]; then
if [ "$alert_open" -eq 1 ]; then
send_alert site /api/health recovered 0 -
alert_open=0
fi
consecutive_failures=0
if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then
if [ -n "$last_commit" ] && [ "$commit" != "$last_commit" ] && [ "$commit_alerted" != "$commit" ]; then
if ! release_window_open; then
send_alert site /api/health commit_changed 1 -
commit_alerted="$commit"
fi
fi
last_commit="$commit"
fi
else
consecutive_failures=$((consecutive_failures + 1))
if [ "$consecutive_failures" -ge 3 ] && [ "$alert_open" -eq 0 ]; then
send_alert site /api/health health_down "$consecutive_failures" -
alert_open=1
fi
fi
}
disk_percent() {
df -Pk "${JYOTISHA_DISK_PATH:-/}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}'
}
check_disk() {
local percent
percent="$(disk_percent)"
[[ "$percent" =~ ^[0-9]+$ ]] && [ "$percent" -le 100 ] || {
echo "disk usage is unreadable" >&2
exit 1
}
if [ "$percent" -ge 80 ]; then
if [ "$disk_alert_open" -eq 0 ]; then
send_alert host - disk_high "$percent" -
disk_alert_open=1
fi
elif [ "$disk_alert_open" -eq 1 ]; then
send_alert host - recovered 0 -
disk_alert_open=0
fi
}
read_state
case "$mode" in
health) check_health ;;
disk) check_disk ;;
all)
check_health
check_disk
;;
esac
write_state
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env bash
# Shared alert sender. Source this file; do not execute it.
# Outbound text is limited to service, route, errorCode, count, requestId, time.
ensure_private_dir() {
local dir="$1"
mkdir -p -- "$dir"
if [ "$(uname -s)" = "Linux" ]; then
chmod 700 "$dir"
fi
}
load_monitor_env() {
local file="$1"
local line key value mode
[ -n "$file" ] || return 0
[ -f "$file" ] && [ ! -L "$file" ] || return 0
if [ "$(uname -s)" = "Linux" ]; then
mode="$(stat -c %a "$file")"
[ "$mode" = "600" ] || {
echo "monitor env must be mode 0600" >&2
exit 1
}
fi
while IFS= read -r line || [ -n "$line" ]; do
case "$line" in
""|\#*) continue ;;
FEISHU_WEBHOOK_URL=*|FEISHU_SIGN_SECRET=*)
key="${line%%=*}"
value="${line#*=}"
case "$value" in
*[$'\n\r`$']*)
echo "monitor env value is unsafe" >&2
exit 1
;;
esac
printf -v "$key" '%s' "$value"
export "$key"
;;
esac
done <"$file"
}
feishu_sign() {
local timestamp="$1"
local secret="$2"
local string_to_sign
string_to_sign="$(printf '%s\n%s' "$timestamp" "$secret")"
printf '' | openssl dgst -sha256 -hmac "$string_to_sign" -binary | openssl base64 -A
}
webhook_allowed() {
local url="$1"
[[ "$url" =~ ^https://open\.(feishu\.cn|larksuite\.com|larkoffice\.com)/open-apis/bot/v2/hook/[A-Za-z0-9-]+$ ]] && return 0
[[ "$url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/hook$ ]] && return 0
return 1
}
alert_line_allowed() {
local line="$1"
[[ "$line" =~ ^service=[a-z0-9_-]+[[:space:]]route=(/|/[A-Za-z0-9/_.-]+|-)[[:space:]]errorCode=[a-z][a-z0-9._-]*[[:space:]]count=[0-9]+[[:space:]]requestId=(-|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}){0,4})[[:space:]]time=[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
}
post_monitor_alert() {
local text="$1"
local state_dir line timestamp sign payload_file
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
ensure_private_dir "$state_dir"
while IFS= read -r line || [ -n "$line" ]; do
[ -n "$line" ] || continue
alert_line_allowed "$line" || {
echo "alert text has a field that is not allowed" >&2
exit 1
}
printf '%s\n' "$line" >>"$state_dir/alerts.log"
done <<<"$text"
[ -n "${FEISHU_WEBHOOK_URL:-}" ] || return 0
webhook_allowed "$FEISHU_WEBHOOK_URL" || {
printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log"
return 0
}
payload_file="$(mktemp)"
if [ "$(uname -s)" = "Linux" ]; then
chmod 600 "$payload_file"
fi
if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then
timestamp="$(date +%s)"
sign="$(feishu_sign "$timestamp" "$FEISHU_SIGN_SECRET")"
FEISHU_ALERT_TEXT="$text" FEISHU_ALERT_TIMESTAMP="$timestamp" FEISHU_ALERT_SIGN="$sign" \
python_write_payload "$payload_file" signed
else
FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain
fi
curl --silent --show-error --connect-timeout 10 --max-time 10 \
-H 'Content-Type: application/json' \
-H 'Connection: close' \
--data-binary @"$payload_file" \
"$FEISHU_WEBHOOK_URL" >/dev/null
rm -f -- "$payload_file"
}
python_write_payload() {
local destination="$1"
local mode="$2"
local py
export FEISHU_ALERT_TEXT FEISHU_ALERT_TIMESTAMP FEISHU_ALERT_SIGN
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
py="$JYOTISHA_PYTHON"
elif command -v python3 >/dev/null 2>&1; then
py="python3"
else
py="python"
fi
"$py" - "$destination" "$mode" <<'PY'
import json
import os
import sys
destination, mode = sys.argv[1], sys.argv[2]
text = os.environ["FEISHU_ALERT_TEXT"]
payload = {"msg_type": "text", "content": {"text": text}}
if mode == "signed":
payload["timestamp"] = os.environ["FEISHU_ALERT_TIMESTAMP"]
payload["sign"] = os.environ["FEISHU_ALERT_SIGN"]
with open(destination, "w", encoding="utf-8", newline="\n") as handle:
json.dump(payload, handle, ensure_ascii=True, separators=(",", ":"))
PY
}