From 53d8df257ddef8918034acef36df0b4b7daade2c Mon Sep 17 00:00:00 2001 From: jesse-ux Date: Thu, 15 Oct 2026 12:58:34 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=99=E5=86=85=E6=B5=8B=E8=A1=A5=E4=B8=8A?= =?UTF-8?q?=E6=AF=8F=E6=97=A5=E5=8A=A0=E5=AF=86=E5=A4=87=E4=BB=BD=E3=80=81?= =?UTF-8?q?=E5=AE=95=E6=9C=BA=E5=91=8A=E8=AD=A6=E5=92=8C=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E4=B8=8A=E9=99=90=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 主机安装和一次恢复演练留给产品。页面没有变化。 Co-Authored-By: Grok (agent loop) --- .gitea/workflows/backup-production-daily.yml | 183 ++++++ BLOCKED.md | 8 + CHANGELOG.md | 4 + deploy/README.md | 42 ++ deploy/docker-compose.postgres.yml | 15 + deploy/docker-compose.server.yml | 15 + deploy/find-request-log.sh | 62 ++ deploy/monitor/classify_error_log.py | 160 +++++ deploy/monitor/error-digest.sh | 85 +++ deploy/monitor/healthcheck-alert.sh | 236 +++++++ deploy/monitor/notify-feishu.sh | 130 ++++ deploy/prune-production-daily-backups.sh | 63 ++ deploy/run-production-recovery.sh | 14 +- docs/BUG_HISTORY.md | 45 ++ .../production-restore-drill-20261014.md | 63 ++ .../PROGRESS-beta-safety-net-20261014.md | 34 + docs/tasks/README.md | 2 +- .../production-restore-drill-20261014.md | 18 + tests/test_beta_safety_net.py | 584 ++++++++++++++++++ 19 files changed, 1761 insertions(+), 2 deletions(-) create mode 100644 .gitea/workflows/backup-production-daily.yml create mode 100644 deploy/find-request-log.sh create mode 100644 deploy/monitor/classify_error_log.py create mode 100644 deploy/monitor/error-digest.sh create mode 100644 deploy/monitor/healthcheck-alert.sh create mode 100644 deploy/monitor/notify-feishu.sh create mode 100644 deploy/prune-production-daily-backups.sh create mode 100644 docs/operations/production-restore-drill-20261014.md create mode 100644 docs/tasks/PROGRESS-beta-safety-net-20261014.md create mode 100644 docs/testing/production-restore-drill-20261014.md create mode 100644 tests/test_beta_safety_net.py diff --git a/.gitea/workflows/backup-production-daily.yml b/.gitea/workflows/backup-production-daily.yml new file mode 100644 index 00000000..53415803 --- /dev/null +++ b/.gitea/workflows/backup-production-daily.yml @@ -0,0 +1,183 @@ +name: Backup Production Daily + +on: + schedule: + # 04:00 Asia/Shanghai (UTC+8). Gitea cron is UTC. + - cron: "0 20 * * *" + workflow_dispatch: + inputs: + artifact_retention_days: + description: Days to keep the encrypted artifact on Gitea + required: false + default: "14" + type: string + local_retention_days: + description: Days to keep this workflow's encrypted files on the host + required: false + default: "7" + type: string + +permissions: + contents: read + actions: write + +concurrency: + group: production-mutation + cancel-in-progress: false + queue: max + +jobs: + backup: + if: gitea.ref == 'refs/heads/main' + runs-on: xiaoxin + timeout-minutes: 30 + env: + GITEA_SHA: ${{ gitea.sha }} + GITEA_REF: ${{ gitea.ref }} + GITEA_API_URL: ${{ gitea.api_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }} + DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }} + DEPLOY_USER: ${{ vars.PRODUCTION_USER }} + DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }} + PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }} + ARTIFACT_RETENTION_DAYS: ${{ inputs.artifact_retention_days }} + LOCAL_RETENTION_DAYS: ${{ inputs.local_retention_days }} + steps: + - name: Create, restore-verify, and retrieve the daily encrypted backup + id: backup + env: + SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }} + run: | + set -euo pipefail + set +x + [[ "$GITEA_REF" == "refs/heads/main" ]] + [[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]] + artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}" + [ -n "$artifact_retention" ] || artifact_retention=14 + local_retention="${LOCAL_RETENTION_DAYS:-7}" + [ -n "$local_retention" ] || local_retention=7 + [[ "$artifact_retention" =~ ^[1-9][0-9]{0,3}$ ]] + [[ "$local_retention" =~ ^[1-9][0-9]{0,3}$ ]] + [[ "$DEPLOY_HOST" == "118.194.235.34" ]] + [[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 )) + [[ "$DEPLOY_USER" == "deploy" ]] + [[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]] + [[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]] + test -n "$PRODUCTION_KNOWN_HOSTS" + echo "artifact_retention_days=$artifact_retention" >>"$GITHUB_OUTPUT" + echo "local_retention_days=$local_retention" >>"$GITHUB_OUTPUT" + ssh_root="${RUNNER_TEMP}/production-daily-ssh" + key_path="$ssh_root/id_ed25519" + known_hosts_path="$ssh_root/known_hosts" + artifact_directory="artifacts/production-daily" + install -m 700 -d "$ssh_root" "$artifact_directory" + test -n "$SSH_PRIVATE_KEY_BASE64" + printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode >"$key_path" + printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' >"$known_hosts_path" + chmod 600 "$key_path" "$known_hosts_path" + ssh-keygen -y -f "$key_path" >/dev/null + ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path") + scp_options=(-i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path") + remote="$DEPLOY_USER@$DEPLOY_HOST" + incoming="$(ssh "${ssh_options[@]}" "$remote" 'mktemp -d /tmp/jyotisha-production-daily.XXXXXXXXXX')" + [[ "$incoming" == /tmp/jyotisha-production-daily.* ]] + cleanup() { + ssh "${ssh_options[@]}" "$remote" "rm -rf -- '$incoming'" >/dev/null 2>&1 || true + rm -rf -- "$ssh_root" + } + trap cleanup EXIT + git init . + git remote remove origin 2>/dev/null || true + git remote add origin https://git.copse.top/root/Jyotisha.git + git fetch --no-tags origin "$GITEA_SHA" + git checkout --detach --force "$GITEA_SHA" + [[ "$(git rev-parse HEAD)" == "$GITEA_SHA" ]] + git show "$GITEA_SHA:deploy/run-production-recovery.sh" >"$ssh_root/run-production-recovery.sh" + git show "$GITEA_SHA:deploy/prune-production-daily-backups.sh" >"$ssh_root/prune-production-daily-backups.sh" + chmod 700 "$ssh_root/run-production-recovery.sh" "$ssh_root/prune-production-daily-backups.sh" + scp "${scp_options[@]}" \ + "$ssh_root/run-production-recovery.sh" \ + "$ssh_root/prune-production-daily-backups.sh" \ + "$remote:$incoming/" + ssh "${ssh_options[@]}" "$remote" \ + "DEPLOY_PATH='$DEPLOY_PATH' RECOVERY_RUN_ID='$GITHUB_RUN_ID' RECOVERY_BASENAME_PREFIX='production-daily' bash '$incoming/run-production-recovery.sh'" \ + >"$ssh_root/recovery-output.env" + [[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d ' ')" == 7 ]] + recovery_reference="$(awk -F= '$1 == "RECOVERY_REFERENCE" {print $2}' "$ssh_root/recovery-output.env")" + recovery_created_at="$(awk -F= '$1 == "RECOVERY_CREATED_AT" {print $2}' "$ssh_root/recovery-output.env")" + recovery_verified_at="$(awk -F= '$1 == "RECOVERY_VERIFIED_AT" {print $2}' "$ssh_root/recovery-output.env")" + recovery_sha256="$(awk -F= '$1 == "RECOVERY_SHA256" {print $2}' "$ssh_root/recovery-output.env")" + restore_verified="$(awk -F= '$1 == "RESTORE_VERIFIED" {print $2}' "$ssh_root/recovery-output.env")" + backup_basename="$(awk -F= '$1 == "BACKUP_BASENAME" {print $2}' "$ssh_root/recovery-output.env")" + verify_basename="$(awk -F= '$1 == "VERIFY_BASENAME" {print $2}' "$ssh_root/recovery-output.env")" + [[ "$backup_basename" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]] + [[ "$verify_basename" == "${backup_basename%.dump.enc}-restore-verify.json" ]] + [[ "$recovery_reference" == "gitea-actions-run-${GITHUB_RUN_ID}/${backup_basename}" ]] + [[ "$recovery_created_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] + [[ "$recovery_verified_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] + [[ "$recovery_sha256" =~ ^[0-9a-f]{64}$ ]] + [[ "$restore_verified" == "true" ]] + scp "${scp_options[@]}" \ + "$remote:$DEPLOY_PATH/backups/$backup_basename" \ + "$remote:$DEPLOY_PATH/backups/$verify_basename" \ + "$artifact_directory/" + test -s "$artifact_directory/$backup_basename" + test -s "$artifact_directory/$verify_basename" + printf '%s %s\n' "$recovery_sha256" "$artifact_directory/$backup_basename" | sha256sum --check --status + jq -e \ + --arg backup "$backup_basename" \ + --arg sha "$recovery_sha256" \ + --arg created "$recovery_created_at" \ + --arg verified "$recovery_verified_at" ' + .mode == "restore_verify" and .ok == true and + .backup == $backup and .sha256 == $sha and + .created_at == $created and .verified_at == $verified and + .restore_database_removed == true and + ([.checks.identity_users, .checks.profiles, .checks.credit_transactions, .checks.public_tables] | + all(type == "number" and . >= 0 and floor == .)) + ' "$artifact_directory/$verify_basename" >/dev/null + printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \ + "$recovery_reference" "$recovery_created_at" "$recovery_verified_at" "$recovery_sha256" \ + "$backup_basename" "$verify_basename" >"$artifact_directory/attestation.env" + chmod 600 "$artifact_directory"/* + ssh "${ssh_options[@]}" "$remote" \ + "bash '$incoming/prune-production-daily-backups.sh' '$DEPLOY_PATH/backups' '$local_retention'" + { + echo "recovery_reference=$recovery_reference" + echo "recovery_created_at=$recovery_created_at" + echo "recovery_verified_at=$recovery_verified_at" + echo "recovery_sha256=$recovery_sha256" + } >>"$GITHUB_OUTPUT" + echo "Daily backup restore-verified: reference=$recovery_reference created_at=$recovery_created_at verified_at=$recovery_verified_at" + + - name: Upload encrypted off-site daily backup + env: + ARTIFACT_RETENTION_DAYS: ${{ steps.backup.outputs.artifact_retention_days }} + run: | + set -euo pipefail + set +x + [[ "${ARTIFACT_RETENTION_DAYS:-}" =~ ^[1-9][0-9]{0,3}$ ]] + test -n "${ACTIONS_RUNTIME_TOKEN:-}" + test -n "${ACTIONS_RESULTS_URL:-}" + test -n "${GITHUB_RUN_ID:-}" + test -n "${GITHUB_REPOSITORY:-}" + workdir="$(pwd -P)" + docker run --rm \ + --user "$(id -u):$(id -g)" \ + --volume "$workdir:$workdir" \ + --workdir "$workdir" \ + --env HOME=/tmp \ + --env "INPUT_NAME=production-daily-$GITHUB_RUN_ID" \ + --env INPUT_PATH=artifacts/production-daily/ \ + --env INPUT_OVERWRITE=false \ + --env ACTIONS_RUNTIME_TOKEN \ + --env ACTIONS_RESULTS_URL \ + --env GITHUB_RUN_ID \ + --env GITHUB_REPOSITORY \ + --env "GITHUB_SHA=$GITEA_SHA" \ + --env "GITHUB_WORKSPACE=$workdir" \ + --env ARTIFACT_RETENTION_DAYS \ + node:22-bookworm-slim \ + node -e 'process.env["INPUT_IF-NO-FILES-FOUND"]="error"; process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS; process.env["INPUT_COMPRESSION-LEVEL"]="0"; require("./.gitea/actions/upload-artifact/dist/index.js")' diff --git a/BLOCKED.md b/BLOCKED.md index 27e7d7c9..831cf41c 100644 --- a/BLOCKED.md +++ b/BLOCKED.md @@ -1,5 +1,13 @@ # BLOCKED +## TASK-beta-safety-net-20261014：主机安装与一次恢复演练还没做（2026-10-15） + +- **每日备份**：工作流要等这支分支进入 `main` 之后才会在北京时间 04:00 跑。本机没有生产主机，没有手动触发生产工作流。 +- **探活**：应装在 Gitea runner 上，不要装在生产主机。磁盘检查和错误汇总装在生产主机。飞书 Webhook 由产品写进模式 `0600` 的环境文件，不进仓库。没配之前只写本机日志。 +- **日志轮转**：编排已写上单文件 20 MB、最多 10 个文件。要等生产容器按新编排重建后才生效。本机没有对生产主机执行。 +- **恢复演练**：内测开放前按 `docs/operations/production-restore-drill-20261014.md` 做一次一次性库还原，结果填 `docs/testing/production-restore-drill-20261014.md`。模板现在是空的。 +- 解除：上面四件事做完，并把演练模板里的「是否通过」填上。不要把 Webhook、密钥或用户内容写进仓库。 + ## TASK-consult-prep-parallel-20261013：审核真实延迟、Postgres 中位数、多领域镜像（2026-10-14） - **阿里云审核**：本机没有审核密钥。1500 毫秒挡首帧的结论来自测试里的假等待，不是真实供应商。开关 `CONSULT_PREP_PARALLEL` 保持关闭。 diff --git a/CHANGELOG.md b/CHANGELOG.md index 41cf6150..e3c2f28a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # 印度占星 Skill 更新日志 +## 2026-10-15 — 内测前补上每日备份、宕机告警和日志上限（待合入 staging） + +- 生产可以每天自动做一次加密备份。站点连续三次打不开、磁盘过满，或短时间错误变多时，可以通知到飞书群。容器日志有大小上限。可以按一轮请求的编号在服务器日志里查找。这些要等产品在主机上装好之后才生效。页面没有变化。Skill 版本不变（BUG-1354、BUG-1355、BUG-1356）。 + ## 2026-10-15 — 星盘页上升行补上星宿，表头对齐（待合入 staging） - 本命盘和月亮盘的上升行现在有星宿、宿主和 pada。状态写「不适用」，不再是空横杠。分盘等表里居中的那几列，表头也居中。当前大运不再画左侧那条线，浅色底还在。已经打开过的星盘会重新计算。Skill 版本不变（BUG-1352、BUG-1353）。 diff --git a/deploy/README.md b/deploy/README.md index ffc9ab62..b2a21423 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -394,6 +394,48 @@ smoke in `docs/operations/personal-report-staging.md` (section “Post-deploy report list / detail / create smoke”): `GET /api/reports` 200, owned `GET /api/reports/` 200, and one new `POST /api/reports` through ready. +## 运维与告警 + +这些步骤要等本分支进入 `main` 之后，由产品在主机上安装。仓库里没有 Webhook 地址。告警正文只允许服务名、路由、错误码、次数、requestId 和时间。 + +### 备份 + +- 工作流：`.gitea/workflows/backup-production-daily.yml`。北京时间每天 04:00，也可手动触发。只从 `main` 运行，调用 `deploy/run-production-recovery.sh`，不另写一套导出。 +- 每日文件前缀是 `production-daily-`。手动恢复点仍是 `production-pre-migration-`。清理脚本 `deploy/prune-production-daily-backups.sh` 只删超过 7 天的每日文件和对应的校验文件。 +- Gitea 产物默认保留 14 天。手动恢复点工作流仍是 30 天，没有改。 +- 还原步骤：`docs/operations/production-restore-drill-20261014.md`。 + +### 告警装在哪 + +- 探活不要装在生产主机上。装在 Gitea runner 上，每分钟跑 `deploy/monitor/healthcheck-alert.sh --mode health`。它访问公网 `https://jyotisha.chat/api/health`，超时 10 秒。连续 3 次失败才告警，恢复只发一次。 +- 发布窗口：在 runner 上执行 `touch /var/lib/jyotisha-monitor/release-window`。45 分钟内，版本号变化不告警。 +- 磁盘检查装在生产主机上，每分钟跑同一个脚本的 `--mode disk`。使用率达到 80% 告警一次，降下来再发一次已恢复。 +- 错误汇总装在生产主机上。每 5 分钟 `deploy/monitor/error-digest.sh --window`。每天北京时间 09:00（UTC 01:00）跑 `--daily`，汇总前一天。同类错误 5 分钟内至少 10 次，或 5xx 至少 20 次，才发窗口告警。消息里最多 5 个 requestId。含邮箱、第二个日期或非 ASCII 文本的日志行整行丢掉。 +- Webhook 写在模式 `0600` 的环境文件里，建议路径 `/opt/jyotisha-production/.monitor.env`（runner 上用 `JYOTISHA_MONITOR_ENV` 指向自己的文件）。只放这两行，不要提交： + +```text +FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/ +FEISHU_SIGN_SECRET=<只有开了签名校验才填> +``` + +没配 Webhook 时只写本机 `alerts.log`，不外发。飞书消息是 `msg_type: text`。配了签名就带 `timestamp` 和 `sign`。 + +### 日志轮转 + +`deploy/docker-compose.server.yml` 和 `deploy/docker-compose.postgres.yml` 的每个服务都是 json-file，单文件 20 MB，最多 10 个文件。镜像、环境变量、端口和健康检查没有改。要等生产容器按新编排重建后才生效。 + +看守护进程默认值：`docker info --format '{{.LoggingDriver}}'`，以及主机上的 `/etc/docker/daemon.json`。编排里的 `logging` 会盖过守护进程对这几个服务的默认值。 + +### 按 requestId 查日志 + +在生产主机上，只读： + +```bash +deploy/find-request-log.sh --since 24h +``` + +`` 必须是 UUID。其它写法会拒绝，避免被当成命令。`--since` 只接受类似 `24h`、`30m` 的形式。输出按时间排序，来自 web、api、caddy。 + ## Common operations ```bash diff --git a/deploy/docker-compose.postgres.yml b/deploy/docker-compose.postgres.yml index 7d26d900..df5b0f58 100644 --- a/deploy/docker-compose.postgres.yml +++ b/deploy/docker-compose.postgres.yml @@ -2,6 +2,11 @@ services: postgres: image: postgres:17-alpine restart: unless-stopped + logging: + driver: json-file + options: + max-size: "20m" + max-file: "10" shm_size: 128mb env_file: - ${DATABASE_ENV_FILE:-../.env.staging.database} @@ -30,6 +35,11 @@ services: image: ${WEB_IMAGE:-jyotisha-web:local} profiles: ["migration"] restart: "no" + logging: + driver: json-file + options: + max-size: "20m" + max-file: "10" env_file: - ${DATABASE_ENV_FILE:-../.env.staging.database} working_dir: /app/frontend @@ -43,6 +53,11 @@ services: image: ${WEB_IMAGE:-jyotisha-web:local} profiles: ["migration-check"] restart: "no" + logging: + driver: json-file + options: + max-size: "20m" + max-file: "10" env_file: - ${DATABASE_ENV_FILE:-../.env.staging.database} working_dir: /app/frontend diff --git a/deploy/docker-compose.server.yml b/deploy/docker-compose.server.yml index dc6c5da7..bee076d3 100644 --- a/deploy/docker-compose.server.yml +++ b/deploy/docker-compose.server.yml @@ -5,6 +5,11 @@ services: context: .. dockerfile: deploy/railway-api.Dockerfile restart: unless-stopped + logging: + driver: json-file + options: + max-size: "20m" + max-file: "10" env_file: - ${APP_ENV_FILE:-../.env.production} environment: @@ -35,6 +40,11 @@ services: NEXT_PUBLIC_SUPABASE_URL: ${NEXT_PUBLIC_SUPABASE_URL:-} NEXT_PUBLIC_SUPABASE_ANON_KEY: ${NEXT_PUBLIC_SUPABASE_ANON_KEY:-} restart: unless-stopped + logging: + driver: json-file + options: + max-size: "20m" + max-file: "10" env_file: - ${APP_ENV_FILE:-../.env.production} environment: @@ -57,6 +67,11 @@ services: caddy: image: caddy:2-alpine restart: unless-stopped + logging: + driver: json-file + options: + max-size: "20m" + max-file: "10" environment: SITE_ADDRESS: ${SITE_ADDRESS:-https://jyotisha.chat} ports: diff --git a/deploy/find-request-log.sh b/deploy/find-request-log.sh new file mode 100644 index 00000000..c4f655fc --- /dev/null +++ b/deploy/find-request-log.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Read-only search for one request id in web, api, and caddy logs. +set -euo pipefail +set +x + +if [ $# -lt 1 ] || [ $# -gt 3 ]; then + echo "usage: find-request-log.sh [--since 24h]" >&2 + exit 2 +fi +request_id="$1" +shift +since="24h" +while [ $# -gt 0 ]; do + case "$1" in + --since) + since="${2:-}" + shift 2 + ;; + *) + echo "unknown argument" >&2 + exit 2 + ;; + esac +done + +[[ "$request_id" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] || { + echo "requestId must be a UUID" >&2 + exit 2 +} +[[ "$since" =~ ^[1-9][0-9]{0,3}[smhd]$ ]] || { + echo "since must look like 24h" >&2 + exit 2 +} + +log_dir="${JYOTISHA_LOG_DIR:-}" +if [ -n "$log_dir" ]; then + [ -d "$log_dir" ] && [ ! -L "$log_dir" ] || { + echo "log directory is unsafe" >&2 + exit 1 + } + tmp="$(mktemp)" + for service in web api caddy; do + file="$log_dir/$service.log" + [ -f "$file" ] && [ ! -L "$file" ] || continue + grep -F -- "$request_id" "$file" >>"$tmp" || true + done + if [ -s "$tmp" ]; then + grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' "$tmp" | sort + fi + rm -f -- "$tmp" + exit 0 +fi + +compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}" +docker compose -p jyotisha-production \ + --env-file "$compose_dir/.env.production" \ + -f "$compose_dir/deploy/docker-compose.server.yml" \ + -f "$compose_dir/deploy/docker-compose.postgres.yml" \ + -f "$compose_dir/deploy/docker-compose.production.yml" \ + logs --since "$since" --no-color --timestamps web api caddy | + grep -F -- "$request_id" | + sort diff --git a/deploy/monitor/classify_error_log.py b/deploy/monitor/classify_error_log.py new file mode 100644 index 00000000..6328993d --- /dev/null +++ b/deploy/monitor/classify_error_log.py @@ -0,0 +1,160 @@ +"""Classify fictional or container log lines into alert rows. + +Outbound fields are service, route, errorCode, count, and request ids. +Lines with non-ASCII text, an email address, or a second calendar date are +dropped so conversation text and birth dates cannot leave the host. +""" + +from __future__ import annotations + +import json +import re +import sys +from collections import OrderedDict +from datetime import datetime, timedelta, timezone + +EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") +DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b") +UUID = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$") +ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$") +STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b") +FIXTURE = re.compile( + r"^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?Pweb|api) (?P.*)$" +) +DOCKER = re.compile( + r"^(?Pweb|api)-\d+\s+\|\s+" + r"(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P.*)$" +) +CST = timezone(timedelta(hours=8)) + + +def unsafe(line: str) -> bool: + if any(ord(char) > 126 for char in line): + return True + lowered = line.lower() + if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered: + return True + return len(DATE.findall(line)) >= 2 + + +def parse_line(line: str) -> tuple[str, str, str] | None: + match = FIXTURE.match(line) or DOCKER.match(line) + if not match: + return None + return match.group("ts"), match.group("svc"), match.group("rest") + + +def timestamp_cst_date(timestamp: str): + stamp = timestamp + if "." in stamp: + stamp = stamp.split(".", 1)[0] + "Z" + moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + return moment.astimezone(CST).date() + + +def extract_fields(rest: str) -> tuple[str, str, str, int | None]: + route = "-" + error_code = "" + request_id = "" + status: int | None = None + marker = "[agent-observability]" + if marker in rest: + raw = rest.split(marker, 1)[1].strip() + try: + payload = json.loads(raw) + except json.JSONDecodeError: + payload = None + if isinstance(payload, dict): + code = payload.get("errorCode") + if isinstance(code, str) and ERROR_CODE.match(code): + error_code = code + candidate = payload.get("requestId") + if isinstance(candidate, str) and UUID.match(candidate): + request_id = candidate.lower() + route_value = payload.get("route") + if isinstance(route_value, str) and ROUTE.match(route_value): + route = route_value + status_match = STATUS.search(rest) + if status_match: + status = int(status_match.group(1)) + return route, error_code, request_id, status + + +def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]: + now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + yesterday = now.astimezone(CST).date() - timedelta(days=1) + counts: OrderedDict[tuple[str, str, str], int] = OrderedDict() + request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict() + fivexx_counts: OrderedDict[str, int] = OrderedDict() + fivexx_ids: OrderedDict[str, list[str]] = OrderedDict() + + def add(key: tuple[str, str, str], request_id: str) -> None: + counts[key] = counts.get(key, 0) + 1 + bucket = request_ids.setdefault(key, []) + if request_id and request_id not in bucket and len(bucket) < 5: + bucket.append(request_id) + + for raw in lines: + line = raw.rstrip("\r\n") + if not line or unsafe(line): + continue + parsed = parse_line(line) + if parsed is None: + continue + timestamp, service, rest = parsed + if mode == "daily" and timestamp_cst_date(timestamp) != yesterday: + continue + route, error_code, request_id, status = extract_fields(rest) + if error_code: + add((service, route, error_code), request_id) + elif status is not None: + add((service, route, f"http_{status}"), request_id) + if status is not None and 500 <= status <= 599: + fivexx_counts[service] = fivexx_counts.get(service, 0) + 1 + bucket = fivexx_ids.setdefault(service, []) + if request_id and request_id not in bucket and len(bucket) < 5: + bucket.append(request_id) + + rows: list[str] = [] + + def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None: + shown = ",".join(ids) if ids else "-" + rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}") + + for key, count in counts.items(): + if mode == "window" and count < error_threshold: + continue + if count <= 0: + continue + emit(*key, count, request_ids.get(key, [])) + for service, count in fivexx_counts.items(): + if mode == "window" and count < fivexx_threshold: + continue + if count <= 0: + continue + emit(service, "-", "http_5xx", count, fivexx_ids.get(service, [])) + return rows + + +def main(argv: list[str]) -> int: + if len(argv) != 5: + print("usage: classify_error_log.py <5xx-threshold>", file=sys.stderr) + return 2 + fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv + if mode not in {"window", "daily"}: + print("unknown mode", file=sys.stderr) + return 2 + error_threshold = int(error_threshold_text) + fivexx_threshold = int(fivexx_threshold_text) + with open(fixture, encoding="utf-8") as handle: + lines = handle.readlines() + for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold): + print(row) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/deploy/monitor/error-digest.sh b/deploy/monitor/error-digest.sh new file mode 100644 index 00000000..7893557f --- /dev/null +++ b/deploy/monitor/error-digest.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Count recent web/api errors and alert without sending log text. +set -euo pipefail +set +x +umask 077 + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=notify-feishu.sh +source "$script_dir/notify-feishu.sh" + +mode="window" +case "${1:-}" in + ""|--window) mode="window" ;; + --daily) mode="daily" ;; + *) + echo "unknown argument" >&2 + exit 2 + ;; +esac + +state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}" +ensure_private_dir "$state_dir" +env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}" +load_monitor_env "$env_file" + +error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}" +fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}" +[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2 +[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2 +now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" +[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2 + +if [ -n "${JYOTISHA_PYTHON:-}" ]; then + py="$JYOTISHA_PYTHON" +elif command -v python3 >/dev/null 2>&1; then + py="python3" +else + py="python" +fi + +fixture="${JYOTISHA_LOG_FIXTURE:-}" +cleanup() { + if [ -n "${generated_fixture:-}" ]; then + rm -f -- "$generated_fixture" + fi +} +trap cleanup EXIT +generated_fixture="" +if [ -z "$fixture" ]; then + compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}" + since="5m" + [ "$mode" = "daily" ] && since="48h" + generated_fixture="$(mktemp)" + docker compose -p jyotisha-production \ + --env-file "$compose_dir/.env.production" \ + -f "$compose_dir/deploy/docker-compose.server.yml" \ + -f "$compose_dir/deploy/docker-compose.postgres.yml" \ + -f "$compose_dir/deploy/docker-compose.production.yml" \ + logs --since "$since" --no-color --timestamps web api >"$generated_fixture" + fixture="$generated_fixture" +fi +[ -f "$fixture" ] && [ ! -L "$fixture" ] || { + echo "log fixture is missing" >&2 + exit 1 +} + +if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then + echo "could not classify logs" >&2 + exit 1 +fi +stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +text="" +if [ -n "$rows" ]; then + while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do + [ -n "${service:-}" ] || continue + line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}" + text="${text}${line}"$'\n' + done <<<"$rows" || true +elif [ "$mode" = "daily" ]; then + text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n' +fi + +if [ -n "$text" ]; then + post_monitor_alert "${text%$'\n'}" +fi diff --git a/deploy/monitor/healthcheck-alert.sh b/deploy/monitor/healthcheck-alert.sh new file mode 100644 index 00000000..1e7c307a --- /dev/null +++ b/deploy/monitor/healthcheck-alert.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# Probe the public health URL, or the host disk. Install the health mode on +# the Gitea runner, not on the production host. Install disk mode on the host. +set -euo pipefail +set +x +umask 077 + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=notify-feishu.sh +source "$script_dir/notify-feishu.sh" + +mode="health" +while [ $# -gt 0 ]; do + case "$1" in + --mode) + mode="${2:-}" + shift 2 + ;; + --mode=*) + mode="${1#--mode=}" + shift + ;; + *) + echo "unknown argument" >&2 + exit 2 + ;; + esac +done +case "$mode" in + health|disk|all) ;; + *) + echo "unknown mode" >&2 + exit 2 + ;; +esac + +state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}" +ensure_private_dir "$state_dir" +env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}" +load_monitor_env "$env_file" + +health_url="${JYOTISHA_HEALTH_URL:-https://jyotisha.chat/api/health}" +timeout="${JYOTISHA_HEALTH_TIMEOUT:-10}" +[[ "$timeout" =~ ^[1-9][0-9]?$ ]] && (( timeout <= 30 )) || { + echo "health timeout must be 1 to 30 seconds" >&2 + exit 2 +} +if [ "$mode" != "disk" ]; then + [[ "$health_url" == "https://jyotisha.chat/api/health" \ + || "$health_url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/health$ \ + || "$health_url" =~ ^http://localhost:[0-9]{1,5}/health$ ]] || { + echo "unexpected health url" >&2 + exit 2 + } +fi + +state_file="$state_dir/monitor.state" +consecutive_failures=0 +alert_open=0 +last_commit="" +commit_alerted="" +disk_alert_open=0 + +read_state() { + local line value + [ -f "$state_file" ] && [ ! -L "$state_file" ] || return 0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + consecutive_failures=[0-9]*) + consecutive_failures="${line#consecutive_failures=}" + ;; + alert_open=[01]) + alert_open="${line#alert_open=}" + ;; + last_commit=) + last_commit="" + ;; + last_commit=*) + value="${line#last_commit=}" + [[ "$value" =~ ^[0-9a-f]{40}$ ]] && last_commit="$value" + ;; + commit_alerted=) + commit_alerted="" + ;; + commit_alerted=*) + value="${line#commit_alerted=}" + [[ "$value" =~ ^[0-9a-f]{40}$ ]] && commit_alerted="$value" + ;; + disk_alert_open=[01]) + disk_alert_open="${line#disk_alert_open=}" + ;; + esac + done <"$state_file" +} + +write_state() { + local tmp + tmp="$state_file.tmp.$$" + printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\n' \ + "$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" >"$tmp" + if [ "$(uname -s)" = "Linux" ]; then + chmod 600 "$tmp" + fi + mv "$tmp" "$state_file" +} + +stamp() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +send_alert() { + local service="$1" + local route="$2" + local error_code="$3" + local count="$4" + local request_id="${5:--}" + post_monitor_alert "service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_id} time=$(stamp)" +} + +release_window_open() { + local marker="$state_dir/release-window" + local mtime now age + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + mtime="$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker")" + now="$(date +%s)" + age=$((now - mtime)) + [ "$age" -ge 0 ] && [ "$age" -le 2700 ] +} + +check_health() { + local body_file http_code curl_status status_line commit_line status commit healthy py + body_file="$(mktemp)" + http_code="000" + curl_status=0 + http_code="$(curl --silent --show-error --connect-timeout "$timeout" --max-time "$timeout" \ + -o "$body_file" -w '%{http_code}' "$health_url")" || curl_status=$? + status="" + commit="" + if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ]; then + if [ -n "${JYOTISHA_PYTHON:-}" ]; then + py="$JYOTISHA_PYTHON" + elif command -v python3 >/dev/null 2>&1; then + py="python3" + else + py="python" + fi + status_line="$("$py" - "$body_file" <<'PY' +import json +import sys +try: + payload = json.load(open(sys.argv[1], encoding="utf-8")) +except Exception: + print("") + print("") + raise SystemExit(0) +if not isinstance(payload, dict): + print("") + print("") + raise SystemExit(0) +status = payload.get("status") +deployment = payload.get("deployment") +commit = "" +if isinstance(deployment, dict): + raw = deployment.get("gitCommit") + if isinstance(raw, str): + commit = raw +print(status if isinstance(status, str) else "") +print(commit) +PY +)" + status="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==1 {print}')" + commit="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==2 {print}')" + fi + rm -f -- "$body_file" + + healthy=0 + if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ] && [ "$status" = "ok" ]; then + healthy=1 + fi + + if [ "$healthy" -eq 1 ]; then + if [ "$alert_open" -eq 1 ]; then + send_alert site /api/health recovered 0 - + alert_open=0 + fi + consecutive_failures=0 + if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then + if [ -n "$last_commit" ] && [ "$commit" != "$last_commit" ] && [ "$commit_alerted" != "$commit" ]; then + if ! release_window_open; then + send_alert site /api/health commit_changed 1 - + commit_alerted="$commit" + fi + fi + last_commit="$commit" + fi + else + consecutive_failures=$((consecutive_failures + 1)) + if [ "$consecutive_failures" -ge 3 ] && [ "$alert_open" -eq 0 ]; then + send_alert site /api/health health_down "$consecutive_failures" - + alert_open=1 + fi + fi +} + +disk_percent() { + df -Pk "${JYOTISHA_DISK_PATH:-/}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}' +} + +check_disk() { + local percent + percent="$(disk_percent)" + [[ "$percent" =~ ^[0-9]+$ ]] && [ "$percent" -le 100 ] || { + echo "disk usage is unreadable" >&2 + exit 1 + } + if [ "$percent" -ge 80 ]; then + if [ "$disk_alert_open" -eq 0 ]; then + send_alert host - disk_high "$percent" - + disk_alert_open=1 + fi + elif [ "$disk_alert_open" -eq 1 ]; then + send_alert host - recovered 0 - + disk_alert_open=0 + fi +} + +read_state +case "$mode" in + health) check_health ;; + disk) check_disk ;; + all) + check_health + check_disk + ;; +esac +write_state diff --git a/deploy/monitor/notify-feishu.sh b/deploy/monitor/notify-feishu.sh new file mode 100644 index 00000000..4382ef7f --- /dev/null +++ b/deploy/monitor/notify-feishu.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Shared alert sender. Source this file; do not execute it. +# Outbound text is limited to service, route, errorCode, count, requestId, time. + +ensure_private_dir() { + local dir="$1" + mkdir -p -- "$dir" + if [ "$(uname -s)" = "Linux" ]; then + chmod 700 "$dir" + fi +} + +load_monitor_env() { + local file="$1" + local line key value mode + [ -n "$file" ] || return 0 + [ -f "$file" ] && [ ! -L "$file" ] || return 0 + if [ "$(uname -s)" = "Linux" ]; then + mode="$(stat -c %a "$file")" + [ "$mode" = "600" ] || { + echo "monitor env must be mode 0600" >&2 + exit 1 + } + fi + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ""|\#*) continue ;; + FEISHU_WEBHOOK_URL=*|FEISHU_SIGN_SECRET=*) + key="${line%%=*}" + value="${line#*=}" + case "$value" in + *[$'\n\r`$']*) + echo "monitor env value is unsafe" >&2 + exit 1 + ;; + esac + printf -v "$key" '%s' "$value" + export "$key" + ;; + esac + done <"$file" +} + +feishu_sign() { + local timestamp="$1" + local secret="$2" + local string_to_sign + string_to_sign="$(printf '%s\n%s' "$timestamp" "$secret")" + printf '' | openssl dgst -sha256 -hmac "$string_to_sign" -binary | openssl base64 -A +} + +webhook_allowed() { + local url="$1" + [[ "$url" =~ ^https://open\.(feishu\.cn|larksuite\.com|larkoffice\.com)/open-apis/bot/v2/hook/[A-Za-z0-9-]+$ ]] && return 0 + [[ "$url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/hook$ ]] && return 0 + return 1 +} + +alert_line_allowed() { + local line="$1" + [[ "$line" =~ ^service=[a-z0-9_-]+[[:space:]]route=(/|/[A-Za-z0-9/_.-]+|-)[[:space:]]errorCode=[a-z][a-z0-9._-]*[[:space:]]count=[0-9]+[[:space:]]requestId=(-|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}){0,4})[[:space:]]time=[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] +} + +post_monitor_alert() { + local text="$1" + local state_dir line timestamp sign payload_file + state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}" + ensure_private_dir "$state_dir" + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + alert_line_allowed "$line" || { + echo "alert text has a field that is not allowed" >&2 + exit 1 + } + printf '%s\n' "$line" >>"$state_dir/alerts.log" + done <<<"$text" + + [ -n "${FEISHU_WEBHOOK_URL:-}" ] || return 0 + webhook_allowed "$FEISHU_WEBHOOK_URL" || { + printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log" + return 0 + } + + payload_file="$(mktemp)" + if [ "$(uname -s)" = "Linux" ]; then + chmod 600 "$payload_file" + fi + if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then + timestamp="$(date +%s)" + sign="$(feishu_sign "$timestamp" "$FEISHU_SIGN_SECRET")" + FEISHU_ALERT_TEXT="$text" FEISHU_ALERT_TIMESTAMP="$timestamp" FEISHU_ALERT_SIGN="$sign" \ + python_write_payload "$payload_file" signed + else + FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain + fi + curl --silent --show-error --connect-timeout 10 --max-time 10 \ + -H 'Content-Type: application/json' \ + -H 'Connection: close' \ + --data-binary @"$payload_file" \ + "$FEISHU_WEBHOOK_URL" >/dev/null + rm -f -- "$payload_file" +} + +python_write_payload() { + local destination="$1" + local mode="$2" + local py + export FEISHU_ALERT_TEXT FEISHU_ALERT_TIMESTAMP FEISHU_ALERT_SIGN + if [ -n "${JYOTISHA_PYTHON:-}" ]; then + py="$JYOTISHA_PYTHON" + elif command -v python3 >/dev/null 2>&1; then + py="python3" + else + py="python" + fi + "$py" - "$destination" "$mode" <<'PY' +import json +import os +import sys + +destination, mode = sys.argv[1], sys.argv[2] +text = os.environ["FEISHU_ALERT_TEXT"] +payload = {"msg_type": "text", "content": {"text": text}} +if mode == "signed": + payload["timestamp"] = os.environ["FEISHU_ALERT_TIMESTAMP"] + payload["sign"] = os.environ["FEISHU_ALERT_SIGN"] +with open(destination, "w", encoding="utf-8", newline="\n") as handle: + json.dump(payload, handle, ensure_ascii=True, separators=(",", ":")) +PY +} diff --git a/deploy/prune-production-daily-backups.sh b/deploy/prune-production-daily-backups.sh new file mode 100644 index 00000000..db73222c --- /dev/null +++ b/deploy/prune-production-daily-backups.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Delete only expired production-daily backup files. The prefix is fixed so a +# caller cannot aim this at manual recovery points or other files. +set -euo pipefail + +prune_production_daily_backups() { + local directory="$1" + local retention_days="$2" + local now_epoch="${3:-}" + local dump name mtime age limit base verify verify_name + local -a doomed=() + + [ -n "$directory" ] && [ -d "$directory" ] && [ ! -L "$directory" ] || { + echo "backup directory is unsafe" >&2 + exit 1 + } + [[ "$retention_days" =~ ^[1-9][0-9]{0,3}$ ]] || { + echo "retention days must be a positive integer" >&2 + exit 1 + } + if [ -z "$now_epoch" ]; then + now_epoch="$(date +%s)" + fi + [[ "$now_epoch" =~ ^[0-9]+$ ]] || { + echo "clock is unsafe" >&2 + exit 1 + } + limit=$((retention_days * 86400)) + + shopt -s nullglob + for dump in "$directory"/production-daily-*.dump.enc; do + name="$(basename -- "$dump")" + [[ "$name" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]] || continue + [ -f "$dump" ] && [ ! -L "$dump" ] || continue + mtime="$(stat -c %Y "$dump" 2>/dev/null || stat -f %m "$dump")" + [[ "$mtime" =~ ^[0-9]+$ ]] || continue + age=$((now_epoch - mtime)) + if (( age > limit )); then + doomed+=("$name") + fi + done + shopt -u nullglob + + if ((${#doomed[@]} == 0)); then + return 0 + fi + for name in "${doomed[@]}"; do + base="${name%.dump.enc}" + verify="$directory/${base}-restore-verify.json" + rm -f -- "$directory/$name" + if [ -e "$verify" ] && [ -f "$verify" ] && [ ! -L "$verify" ]; then + verify_name="$(basename -- "$verify")" + if [[ "$verify_name" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z-restore-verify\.json$ ]]; then + rm -f -- "$verify" + fi + fi + printf '%s\n' "$name" + done +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + prune_production_daily_backups "${1:-}" "${2:-}" "${3:-}" +fi diff --git a/deploy/run-production-recovery.sh b/deploy/run-production-recovery.sh index f552bcac..ba3a1ffa 100755 --- a/deploy/run-production-recovery.sh +++ b/deploy/run-production-recovery.sh @@ -138,7 +138,19 @@ usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); p created_compact="$(date -u +%Y%m%dT%H%M%SZ)" created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -backup_basename="production-pre-migration-${created_compact}.dump.enc" +# Daily backups set RECOVERY_BASENAME_PREFIX=production-daily so retention can +# tell them from manual recovery points. Unset keeps the original name. +backup_prefix="production-pre-migration" +if [ -n "${RECOVERY_BASENAME_PREFIX:-}" ]; then + case "$RECOVERY_BASENAME_PREFIX" in + production-daily) backup_prefix="production-daily" ;; + *) + echo "unexpected recovery basename prefix" >&2 + exit 1 + ;; + esac +fi +backup_basename="${backup_prefix}-${created_compact}.dump.enc" backup_partial="$backup_directory/.${backup_basename}.$$.partial" backup_file="$backup_directory/$backup_basename" restore_database="restore_verify_${created_compact,,}" diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 30a95ab1..481957cd 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -36182,3 +36182,48 @@ - 复发自：无 - 修复版本：分支 `codex/chart-ascendant-row-20261015`。 +## BUG-1354 | 生产没有每日自动备份 + +- 状态：代码已交，主机安装待产品 +- 首次发现 / 最近更新：2026-10-14 / 2026-10-15 +- 来源：产品 10-14 内测体检。任务书 `docs/tasks/TASK-beta-safety-net-20261014.md`。 +- 现象：生产备份只在有人手动点恢复点时才做。仓库里没有按天跑的备份，也没有定时器文件。 +- 触发条件：内测开始写入真实用户数据之后，若两次手动备份之间出错或磁盘故障，中间的数据没有恢复点。 +- 根因：备份是按「发布前做一次」设计的，没有持续运行的每日任务。 +- 修复：新增只从 `main` 运行的每日备份工作流，复用现有加密与还原校验脚本。每日文件用单独前缀，过期清理不会碰到手动恢复点。工作流要等进入 `main` 后才会在北京时间 04:00 跑。本机没有生产主机，没有手动触发。 +- 验证：合同测试锁定有每日时刻、只在 `main` 上跑、调用同一个恢复脚本、清理只删过期的每日前缀。虚构目录里，手动恢复点和其他文件都还在。 +- 防复发：上述合同测试。清理函数不接受外部传入的前缀。 +- 相关记录：BUG-1350、BUG-1355、BUG-1356。 +- 复发自：无。 +- 修复版本：分支 `codex/beta-safety-net-20261014`。开工时本文件最大号是 BUG-1353。 + +## BUG-1355 | 站点打不开或磁盘过满时没有告警 + +- 状态：代码已交，主机安装待产品 +- 首次发现 / 最近更新：2026-10-14 / 2026-10-15 +- 来源：同 BUG-1354。 +- 现象：容器自己的健康检查只会尝试重启，不会通知人。磁盘变满也没有通知。 +- 触发条件：公网健康检查连续失败，或生产主机磁盘使用率达到 80%。 +- 根因：上线时没有持续探活，也没有把告警发到人用的渠道。 +- 修复：探活脚本连续 3 次失败才发一条告警，恢复只发一次。磁盘检查单独在生产主机上跑。Webhook 从环境文件读取，没有配置就只写本机日志。外发正文不含用户内容。脚本还没装到 runner 或生产主机上。 +- 验证：本地桩服务覆盖正常、500、超时和缺少字段。连续 3 次才告警，恢复只出现一次。签名测试里的密钥没有出现在发出的正文里。 +- 防复发：上述脚本测试。告警行若含不允许的字段，脚本拒绝发送。 +- 相关记录：BUG-1354、BUG-1356。 +- 复发自：无。 +- 修复版本：分支 `codex/beta-safety-net-20261014`。编号接在 BUG-1354 之后。 + +## BUG-1356 | 容器日志不轮转，也不能按一轮请求查找 + +- 状态：代码已交，主机安装待产品 +- 首次发现 / 最近更新：2026-10-14 / 2026-10-15 +- 来源：同 BUG-1354。BUG-1306 取证时日志只在容器控制台，重建后查不到。 +- 现象：编排没有日志大小上限。知道一轮请求的编号时，没有只读的查找命令。 +- 触发条件：日志持续写入，或需要按 requestId 回看 web、api、caddy。 +- 根因：日志沿用守护进程的未知默认值，也没有按编号检索的入口。 +- 修复：每个服务的日志改为单文件 20 MB、最多 10 个文件。新增只读查找脚本，只接受 UUID。错误汇总只计数和最多 5 个编号，丢掉含邮箱、额外日期或非 ASCII 文本的行。要等生产容器按新编排重建后，轮转才生效。 +- 验证：两个编排文件里每个服务的日志段一致。虚构日志能按时间排序找到编号；命令注入写法被拒绝，旁边的标记文件还在。含邮箱和汉字的样本没有进入告警文本。 +- 防复发：上述合同测试和查找脚本的格式检查。 +- 相关记录：BUG-1306、BUG-1354、BUG-1355。 +- 复发自：无。 +- 修复版本：分支 `codex/beta-safety-net-20261014`。编号接在 BUG-1355 之后。 + diff --git a/docs/operations/production-restore-drill-20261014.md b/docs/operations/production-restore-drill-20261014.md new file mode 100644 index 00000000..ab49b52d --- /dev/null +++ b/docs/operations/production-restore-drill-20261014.md @@ -0,0 +1,63 @@ +# 生产备份恢复演练（2026-10-14） + +给产品负责人照着做。这一次演练还原到一次性库，不碰正在服务的主库，网站不用停。真正整库还原会停写，放在文末，只有确认要覆盖现网数据时才做。 + +不要把备份文件、密钥、数据库口令、出生资料、姓名、邮箱或会话内容写进仓库、聊天或这份记录。 + +## 备份在哪里 + +- 每天北京时间 04:00，从 `main` 自动跑「Backup Production Daily」。也可以在 Gitea 里手动点一次。它只在 `main` 上跑。 +- 加密文件在生产主机 `/opt/jyotisha-production/backups/`，文件名是 `production-daily-日期T时间Z.dump.enc`。旁边有同名的 `-restore-verify.json`，里面只有行数和权限是否一致，没有用户正文。 +- 手动恢复点仍叫 `production-pre-migration-...`。每日清理只删过期的 `production-daily-` 文件，不会删手动恢复点。 +- Gitea 上的加密产物默认留 14 天。主机上的每日文件默认留 7 天。 +- 加密方式与现有恢复点相同：AES-256-CBC、PBKDF2。密钥在主机的数据库环境文件里，不要打印。 + +## 一次性库演练（不停机） + +在生产主机上用 `deploy` 用户做。下面的库名换成当天的日期，不要用正在服务的那个库名。 + +1. 选定一个 `production-daily-*.dump.enc`。确认旁边的校验文件写着 `"ok": true` 和 `"restore_database_removed": true`。 +2. 进入 `/opt/jyotisha-production`。数据库口令和加密密钥已经在 `.env.production.database` 里。只在这台机器上读取，不要回显。 +3. 确认生产 Postgres 容器只有一个。容器名用 `docker ps` 看，下面记成 ``。 +4. 建一次性库，名字必须不是主库名，例如 `restore_drill_20261015`： + +```bash +sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" \ + psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c 'CREATE DATABASE "restore_drill_20261015";' +``` + +5. 解密并还原进这个一次性库。不要把输出重定向到主库。 + +```bash +openssl enc -d -aes-256-cbc -pbkdf2 \ + -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY \ + -in /opt/jyotisha-production/backups/production-daily-日期T时间Z.dump.enc | +sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" \ + pg_restore -U "$POSTGRES_USER" -d restore_drill_20261015 --exit-on-error +``` + +6. 只记四个数字：`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表张数。不要把表里的内容抄出来。 +7. 删掉一次性库，并确认它已经不在： + +```bash +sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" \ + psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c 'DROP DATABASE "restore_drill_20261015" WITH (FORCE);' +``` + +8. 把四个数字、文件名（不要附文件）、一次性库是否已删除，填进 `docs/testing/production-restore-drill-20261014.md`。 + +通过：还原命令退出码是 0，四个数字都是非负整数，一次性库已删除，主库没有被这条命令点名。 + +失败：停在失败的那一步。不要为了演练去改主库。把退出码交给验收，不要附带环境文件。 + +## 整库还原（会停机，平时不要做） + +只有在确认现网库要被这份备份覆盖时才做。预留 30 分钟停写。这不是实测时间，数据变大就再加。 + +1. 停掉 `web` 和 `api`，让没有新的写入。Postgres 先留着。 +2. 用上面同一条解密和 `pg_restore`，目标改成主库名。这一步会覆盖主库里的现有数据。 +3. 核对四个行数，并对照校验文件里的权限三项是否为真。格式版本低于 2 的旧备份不要拿来做这件事。 +4. 再启动 `web` 和 `api`。打开 `https://jyotisha.chat/api/health`，`status` 应为 `ok`。 +5. 把实际停了多久填进演练模板。不要写用户内容。 diff --git a/docs/tasks/PROGRESS-beta-safety-net-20261014.md b/docs/tasks/PROGRESS-beta-safety-net-20261014.md new file mode 100644 index 00000000..b174979b --- /dev/null +++ b/docs/tasks/PROGRESS-beta-safety-net-20261014.md @@ -0,0 +1,34 @@ +# 进度 · 内测安全网（2026-10-15） + +- 分支：`codex/beta-safety-net-20261014` +- 基线：开工时 `origin/staging` 为 `71f7fac3a13f817f91755ff01b25b8706380ae19` +- 前置：`09906cdc`（发布通路）和 `4c3485bc`、`3893783e`（#58）都已是这个基线的祖先。`deploy/docker-compose.server.yml` 在加日志段之前没有 #58 留下的未合入冲突。 +- 编号：本文件最大号是 BUG-1353。本单用 BUG-1354（没有每日备份）、BUG-1355（没有探活和磁盘告警）、BUG-1356（日志不轮转、不能按请求编号查）。 + +## 做了什么 + +- 新增每日备份工作流。北京时间 04:00，只在 `main` 上跑，调用现有的 `deploy/run-production-recovery.sh`。产物默认留 14 天。主机上只清理超过 7 天、且文件名严格是 `production-daily-` 的加密备份和对应校验文件。 +- 恢复脚本增加可选前缀。不设置时文件名仍是 `production-pre-migration-`。每日任务把它设为 `production-daily`。加密、还原校验、锁和磁盘 70% 拒绝没有改。 +- 探活、磁盘、错误汇总、飞书文本消息（可选签名）。没有 Webhook 时只写本机日志。 +- 两个编排文件的每个服务加上相同的日志上限：json-file，20 MB，10 个文件。镜像、环境、端口、健康检查没改。 +- 按 UUID 查 web / api / caddy 日志的只读脚本。拒绝其它写法。 +- 恢复演练步骤和空白结果模板。安装步骤写在 `deploy/README.md` 和 `BLOCKED.md`。 + +## 跑过的检查 + +| 检查 | 命令 | 结果 | +| --- | --- | --- | +| 安全网测试 | `.venv\Scripts\python.exe -m pytest tests/test_beta_safety_net.py -q` | 13 通过，退出码 0 | +| 隐私标记 | `.venv\Scripts\python.exe -m pytest tests/test_repo_privacy_markers.py -q` | 退出码 0 | +| 编排能否解析 | `docker compose -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml config --quiet` | 退出码 0。本机没有真实环境文件，检查时用了临时空文件，没有提交 | +| shellcheck | 无 | 本机没有这条命令 | +| 前端 tsc / lint / 全量测试 | 未跑 | 没有改网页 | +| Python 快速门 | 未跑 | 按本轮约定不跑 | +| 数据库测试 | 未跑 | 没有改数据库 | + +## 偏离 + +- 没有改现有的四个生产工作流，也没有把它们的 SSH 步骤抽成共用脚本。新工作流自己复制了同一套 SSH，并调用同一个恢复脚本。 +- 没有往网页或接口日志里加新字段。现有观测记录已经有 requestId 和 errorCode。错误汇总会丢掉含有汉字、邮箱或第二个日期的整行，避免把对话带出去。因此中文报错不会进入计数。 +- 探活外发不带提交号，只带错误码 `commit_changed`。提交号留在 runner 本机状态里。 +- 本机没有生产主机，没有登录 runner，没有手动触发生产工作流。告警和定时备份要等产品安装，见 `BLOCKED.md`。 diff --git a/docs/tasks/README.md b/docs/tasks/README.md index 6a5ab0fe..baf4927e 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -338,7 +338,7 @@ | `TASK-release-path-production-20261014.md` | `PROGRESS-release-path-production-20261014.md` | **发布通路 + 生产首发**：产品 10-14 授权改生产工作流（发布 SHA = 最近门禁成功提交 + 其后纯文档）；63 个迁移清单与演练脚本、首发 runbook；生产拒绝测试验证码。Grok 不 dispatch 生产工作流 | 第 2 次打回后续修 | `codex/release-path-production-20261014`（BUG-1348、BUG-1349） | | `TASK-release-path-production-fix-20261014.md` | `PROGRESS-release-path-production-fix-20261014.md` | **#74 第 1 次打回**：发布规则成立；测试写死 Windows 路径（Linux 全红）、主机演练依赖主机 node 且会留下生产数据副本、docs-only 头时回滚被拒；另补 CHECK_RELEASE_GATE 断言与 OTP 守卫规范化 | 已修，第 2 次仍未过 | `codex/release-path-production-20261014` | | `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2（产品选 A）**：恢复点要带属主与权限，旧格式不得回退；演练按备份里的授权比对，并在真实 PostgreSQL 17 上跑通一次 | 待验收 | `codex/release-path-production-20261014` | -| `TASK-beta-safety-net-20261014.md` | — | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。**等发布通路单合入、#58 合入后开工** | 待前置 | — | +| `TASK-beta-safety-net-20261014.md` | `PROGRESS-beta-safety-net-20261014.md` | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。发布通路与 #58 已在开工时的 `origin/staging` 上。主机安装和一次演练留给产品 | 待验收 | `codex/beta-safety-net-20261014`（BUG-1354～1356） | | `TASK-beta-capacity-20261014.md` | — | **内测容量**：按用户限流、重计算短排队与对话重试、兑换码当邀请码、staging 10/20 并发压测。**等 #58 合入后开工，压测等 #57 合入** | 待前置 | — | | `TASK-special-lagnas-native-20261008.md` | `PROGRESS-special-lagnas-native-20261008.md` | **特殊上升点原生计算（线上缺陷）**：sync6 U6 运行时调 PyJHora，生产镜像没有 PyJHora → staging 上 Hora/Ghati/Sree/Indu/Pranapada 全 blocked；原生实现、PyJHora 只作冻结对照、快速门禁止运行时 import jhora；最先做 | 已验收（已部署 staging `795b8845`，BUG-1273） | `codex/special-lagnas-native-20261008`（未上线；部署以 `/api/health` 为准） | | `TASK-ephemeris-nutation-20261008.md` | `PROGRESS-ephemeris-nutation-20261008.md` | **恒星黄道少扣章动（BUG-1275）**：行星 = 含章动热带视位置 − 平岁差，全体偏 Δψ（≤17″），大运日期差 2–3 天；改标准算法、PyJHora 冻结对照、scoring-12、v5 重跑 | 已验收（已部署 staging `db0134f9`，BUG-1275） | `codex/ephemeris-nutation-20261008`（未推送，未部署） | diff --git a/docs/testing/production-restore-drill-20261014.md b/docs/testing/production-restore-drill-20261014.md new file mode 100644 index 00000000..2c833eae --- /dev/null +++ b/docs/testing/production-restore-drill-20261014.md @@ -0,0 +1,18 @@ +# 生产恢复演练结果（2026-10-14） + +内测开放前由产品做一次。步骤见 `docs/operations/production-restore-drill-20261014.md`。不要填写姓名、出生资料、邮箱、会话或密钥。 + +| 项 | 结果 | +| --- | --- | +| 日期 | | +| 用的备份文件名（只要 `production-daily-` 开头的文件名） | | +| 还原目标 | 一次性库 / 整库（圈一个） | +| 一次性库是否已删除 | | +| identity.users 行数 | | +| public.profiles 行数 | | +| public.credit_transactions 行数 | | +| public 表张数 | | +| 网站是否停过 | 一次性库应为否 | +| 停写大约多久 | 一次性库填「未停」 | +| 是否通过 | | +| 备注（不要写用户内容） | | diff --git a/tests/test_beta_safety_net.py b/tests/test_beta_safety_net.py new file mode 100644 index 00000000..8c72f92d --- /dev/null +++ b/tests/test_beta_safety_net.py @@ -0,0 +1,584 @@ +"""Contracts for the closed-beta safety net. Fixtures are fictional.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".gitea" / "workflows" / "backup-production-daily.yml" +RECOVERY_WORKFLOW = ROOT / ".gitea" / "workflows" / "create-production-recovery.yml" +RECOVERY_SCRIPT = ROOT / "deploy" / "run-production-recovery.sh" +PRUNE_SCRIPT = ROOT / "deploy" / "prune-production-daily-backups.sh" +HEALTH_SCRIPT = ROOT / "deploy" / "monitor" / "healthcheck-alert.sh" +DIGEST_SCRIPT = ROOT / "deploy" / "monitor" / "error-digest.sh" +FIND_SCRIPT = ROOT / "deploy" / "find-request-log.sh" +NOTIFY_SCRIPT = ROOT / "deploy" / "monitor" / "notify-feishu.sh" +SERVER_COMPOSE = ROOT / "deploy" / "docker-compose.server.yml" +POSTGRES_COMPOSE = ROOT / "deploy" / "docker-compose.postgres.yml" + + +def resolve_bash() -> str: + found = shutil.which("bash") + git_bash = Path(r"D:\Program Files\Git\bin\bash.exe") + if os.name == "nt" and git_bash.exists(): + lowered = (found or "").lower() + if not found or "system32" in lowered or "windowsapps" in lowered: + return str(git_bash) + if found: + return found + if git_bash.exists(): + return str(git_bash) + return "bash" + + +BASH = resolve_bash() + + +def bash_path(path: Path) -> str: + if os.name != "nt": + return str(path) + completed = subprocess.run( + [BASH, "-lc", 'cygpath -u "$1"', "cygpath", str(path)], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + ) + return completed.stdout.strip() + + +def run_bash(script: Path, extra: dict[str, str], args: list[str] | None = None) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.pop("FEISHU_WEBHOOK_URL", None) + env.pop("FEISHU_SIGN_SECRET", None) + env.update(extra) + env["JYOTISHA_PYTHON"] = bash_path(Path(sys.executable)) + return subprocess.run( + [BASH, bash_path(script), *(args or [])], + cwd=ROOT, + env=env, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + + +def write_text(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8", newline="\n") + + +class HealthHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path != "/health": + self.send_response(404) + self.end_headers() + return + kind = getattr(self.server, "kind", "ok") + if kind == "timeout": + time.sleep(3) + commit = getattr(self.server, "commit", "a" * 40) + if kind == "500": + status, body = 500, b'{"status":"ok"}' + elif kind == "missing": + status, body = 200, b"{}" + elif kind == "degraded": + status, body = 200, json.dumps({"status": "degraded", "deployment": {"gitCommit": commit}}).encode() + else: + status, body = 200, json.dumps({"status": "ok", "deployment": {"gitCommit": commit}}).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt: str, *args: object) -> None: + return + + +class HookHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + self.server.bodies.append(body) + self.close_connection = True + self.send_response(200) + self.send_header("Content-Length", "2") + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, fmt: str, *args: object) -> None: + return + + +def serve(handler: type[BaseHTTPRequestHandler]) -> tuple[ThreadingHTTPServer, threading.Thread]: + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread + + +def alert_lines(state: Path) -> list[str]: + log = state / "alerts.log" + if not log.exists(): + return [] + return [line for line in log.read_text(encoding="utf-8").splitlines() if line] + + +def test_daily_workflow_is_scheduled_from_main_and_reuses_recovery() -> None: + text = WORKFLOW.read_text(encoding="utf-8") + assert "\tif:" not in text + assert "if: gitea.ref == 'refs/heads/main'" in text + assert 'cron: "0 20 * * *"' in text + assert "workflow_dispatch:" in text + assert 'git show "$GITEA_SHA:deploy/run-production-recovery.sh"' in text + assert "RECOVERY_BASENAME_PREFIX='production-daily'" in text + assert "prune-production-daily-backups.sh" in text + assert "production-daily-" in text + assert 'artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"' in text + assert 'local_retention="${LOCAL_RETENTION_DAYS:-7}"' in text + assert 'process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS' in text + assert "set +x" in text + assert "pg_dump" not in text + assert "CHECK_RELEASE_GATE" not in text + assert "observed_staging_sha" not in text + assert "STAGING_URL" not in text + assert "echo \"$SSH_PRIVATE_KEY" not in text + assert "PRODUCTION_BACKUP_ENCRYPTION_KEY" not in text + recovery = RECOVERY_WORKFLOW.read_text(encoding="utf-8") + assert "observed_staging_sha" in recovery + assert 'process.env["INPUT_RETENTION-DAYS"]="30"' in recovery + + +def test_recovery_script_keeps_manual_names_unless_daily_prefix() -> None: + text = RECOVERY_SCRIPT.read_text(encoding="utf-8") + assert 'backup_prefix="production-pre-migration"' in text + assert 'production-daily) backup_prefix="production-daily"' in text + assert "openssl enc -aes-256-cbc -salt -pbkdf2" in text + assert "usage_percent < 70" in text + assert "flock -n 9" in text + + +def test_prune_deletes_only_expired_daily_files(tmp_path: Path) -> None: + backup = tmp_path / "backups" + nested = backup / "nested" + nested.mkdir(parents=True) + now = 1_700_000_000 + limit = 7 * 86400 + names = { + "production-daily-20261001T000000Z.dump.enc": now - limit - 60, + "production-daily-20261001T000000Z-restore-verify.json": now - limit - 60, + "production-daily-20261007T000000Z.dump.enc": now - limit, + "production-daily-20261007T000000Z-restore-verify.json": now - limit, + "production-daily-20261014T000000Z.dump.enc": now - 3600, + "production-daily-20261014T000000Z-restore-verify.json": now - 3600, + "production-pre-migration-20261001T000000Z.dump.enc": now - limit - 3600, + "production-pre-migration-20261001T000000Z-restore-verify.json": now - limit - 3600, + "production-daily-notes.txt": now - limit - 3600, + "production-daily-20261001T000000Z.dump.enc.bak": now - limit - 3600, + "unrelated.txt": now - limit - 3600, + } + for name, epoch in names.items(): + path = backup / name + write_text(path, "fictional\n") + subprocess.run( + [BASH, "-lc", 'touch -d "@$1" "$2"', "touch", str(epoch), bash_path(path)], + check=True, + ) + write_text(nested / "production-daily-20261001T000000Z.dump.enc", "nested\n") + completed = run_bash(PRUNE_SCRIPT, {}, [bash_path(backup), "7", str(now)]) + assert completed.returncode == 0, completed.stderr + deleted = completed.stdout.splitlines() + assert deleted == ["production-daily-20261001T000000Z.dump.enc"] + assert not (backup / "production-daily-20261001T000000Z.dump.enc").exists() + assert not (backup / "production-daily-20261001T000000Z-restore-verify.json").exists() + for name in names: + if name.startswith("production-daily-20261001T000000Z.dump") or name.startswith( + "production-daily-20261001T000000Z-restore" + ): + continue + assert (backup / name).exists(), name + assert (nested / "production-daily-20261001T000000Z.dump.enc").exists() + + +def test_health_alerts_after_three_failures_and_recovery_once(tmp_path: Path) -> None: + server, _thread = serve(HealthHandler) + server.kind = "500" + state = tmp_path / "state" + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health", + "JYOTISHA_HEALTH_TIMEOUT": "2", + } + try: + for _ in range(2): + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + assert alert_lines(state) == [] + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + lines = alert_lines(state) + assert len(lines) == 1 + assert "errorCode=health_down" in lines[0] + assert "count=3" in lines[0] + assert "route=/api/health" in lines[0] + server.kind = "ok" + server.commit = "b" * 40 + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + lines = alert_lines(state) + assert len(lines) == 2 + assert "errorCode=recovered" in lines[1] + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + assert len(alert_lines(state)) == 2 + finally: + server.shutdown() + + +def test_health_timeout_and_missing_fields_count_as_failures(tmp_path: Path) -> None: + server, _thread = serve(HealthHandler) + server.kind = "timeout" + state = tmp_path / "state" + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health", + "JYOTISHA_HEALTH_TIMEOUT": "1", + } + try: + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + assert alert_lines(state) == [] + server.kind = "missing" + run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + assert any("errorCode=health_down" in line for line in alert_lines(state)) + finally: + server.shutdown() + + +def test_commit_change_alerts_once_outside_release_window(tmp_path: Path) -> None: + server, _thread = serve(HealthHandler) + server.kind = "ok" + server.commit = "a" * 40 + state = tmp_path / "state" + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health", + "JYOTISHA_HEALTH_TIMEOUT": "2", + } + try: + assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 + assert alert_lines(state) == [] + server.commit = "b" * 40 + assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 + lines = alert_lines(state) + assert len(lines) == 1 + assert "errorCode=commit_changed" in lines[0] + assert "b" * 40 not in lines[0] + assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 + assert len(alert_lines(state)) == 1 + marker = state / "release-window" + state.mkdir(parents=True, exist_ok=True) + write_text(marker, "open\n") + server.commit = "c" * 40 + assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 + assert len(alert_lines(state)) == 1 + finally: + server.shutdown() + + +def test_disk_alerts_once_and_no_webhook_stays_local(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + write_text( + bin_dir / "df", + "#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 90 10 ${FAKE_DISK_PERCENT:?}% /\"\n", + ) + subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True) + state = tmp_path / "state" + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "FAKE_DISK_PERCENT": "81", + } + env.pop("FEISHU_WEBHOOK_URL", None) + env.pop("FEISHU_SIGN_SECRET", None) + completed = subprocess.run( + [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], + cwd=ROOT, + env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + assert completed.returncode == 0, completed.stderr + assert "errorCode=disk_high" in alert_lines(state)[0] + assert "count=81" in alert_lines(state)[0] + env["FAKE_DISK_PERCENT"] = "81" + again = subprocess.run( + [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], + cwd=ROOT, + env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + assert again.returncode == 0, again.stderr + assert len(alert_lines(state)) == 1 + env["FAKE_DISK_PERCENT"] = "10" + recovered = subprocess.run( + [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], + cwd=ROOT, + env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + assert recovered.returncode == 0, recovered.stderr + assert len(alert_lines(state)) == 2 + assert "errorCode=recovered" in alert_lines(state)[1] + + +def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None: + hook, _thread = serve(HookHandler) + hook.bodies = [] + health, _thread = serve(HealthHandler) + health.kind = "500" + state = tmp_path / "state" + secret = "fixture-sign-secret" + env_file = tmp_path / "monitor.env" + write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\nFEISHU_SIGN_SECRET={secret}\n") + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(env_file), + "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health", + "JYOTISHA_HEALTH_TIMEOUT": "2", + } + try: + for _ in range(3): + completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert completed.returncode == 0, completed.stderr + assert len(hook.bodies) == 1 + payload = json.loads(hook.bodies[0].decode("utf-8")) + assert set(payload) == {"timestamp", "sign", "msg_type", "content"} + assert payload["msg_type"] == "text" + text = payload["content"]["text"] + assert secret not in text + assert secret not in hook.bodies[0].decode("utf-8") + assert "errorCode=health_down" in text + assert "@" not in text + expected = subprocess.run( + [ + sys.executable, + "-c", + "import base64,hashlib,hmac,os,sys; ts=sys.argv[1]; secret=sys.argv[2]; key=f'{ts}\\n{secret}'.encode(); print(base64.b64encode(hmac.new(key, digestmod=hashlib.sha256).digest()).decode())", + payload["timestamp"], + secret, + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + assert payload["sign"] == expected + finally: + hook.shutdown() + health.shutdown() + + +def uuid(index: int) -> str: + return f"00000000-0000-4000-8000-{index:012d}" + + +def test_error_digest_drops_personal_text_and_respects_thresholds(tmp_path: Path) -> None: + safe = [ + f"2026-10-15T00:00:{index:02d}Z web [agent-observability] {json.dumps({'requestId': uuid(index), 'errorCode': 'settlement_failed'})}" + for index in range(1, 11) + ] + unsafe = [ + "2026-10-15T00:01:00Z web person@example.com errorCode=should_drop", + "2026-10-15T00:01:01Z web note 1900-01-01 birth_date", + "2026-10-15T00:01:02Z web 对话内容 今天工作不顺", + ] + fixture = tmp_path / "logs.txt" + write_text(fixture, "\n".join([*safe, *unsafe]) + "\n") + state = tmp_path / "state" + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_LOG_FIXTURE": bash_path(fixture), + "JYOTISHA_NOW": "2026-10-15T01:00:00Z", + } + below = tmp_path / "below.txt" + write_text(below, "\n".join(safe[:9]) + "\n") + quiet = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_LOG_FIXTURE": bash_path(below)}, ["--window"]) + assert quiet.returncode == 0, quiet.stderr + assert alert_lines(state) == [] + completed = run_bash(DIGEST_SCRIPT, env, ["--window"]) + assert completed.returncode == 0, completed.stderr + lines = alert_lines(state) + assert len(lines) == 1 + assert "errorCode=settlement_failed" in lines[0] + assert "count=10" in lines[0] + assert lines[0].count(",") == 4 + assert uuid(6) not in lines[0] + assert "person@example.com" not in lines[0] + assert "1900-01-01" not in lines[0] + assert "今天工作不顺" not in lines[0] + assert "should_drop" not in lines[0] + + fivexx = tmp_path / "fivexx.txt" + write_text(fivexx, "\n".join(f"2026-10-15T00:02:{index:02d}Z api status=502" for index in range(20)) + "\n") + state_5xx = tmp_path / "state5" + completed = run_bash( + DIGEST_SCRIPT, + {**env, "JYOTISHA_MONITOR_STATE_DIR": bash_path(state_5xx), "JYOTISHA_LOG_FIXTURE": bash_path(fivexx)}, + ["--window"], + ) + assert completed.returncode == 0, completed.stderr + joined = "\n".join(alert_lines(state_5xx)) + assert "errorCode=http_5xx" in joined + assert "count=20" in joined + + +def test_daily_digest_summarizes_previous_shanghai_day(tmp_path: Path) -> None: + fixture = tmp_path / "daily.txt" + write_text( + fixture, + "\n".join( + [ + '2026-10-14T02:00:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000001","errorCode":"settlement_failed"}', + '2026-10-15T01:30:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000002","errorCode":"settlement_failed"}', + ] + ) + + "\n", + ) + state = tmp_path / "state" + completed = run_bash( + DIGEST_SCRIPT, + { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_LOG_FIXTURE": bash_path(fixture), + "JYOTISHA_NOW": "2026-10-15T01:00:00Z", + }, + ["--daily"], + ) + assert completed.returncode == 0, completed.stderr + lines = alert_lines(state) + assert len(lines) == 1 + assert "count=1" in lines[0] + assert "00000000-0000-4000-8000-000000000001" in lines[0] + assert "00000000-0000-4000-8000-000000000002" not in lines[0] + + +def test_find_request_log_sorts_and_rejects_injection(tmp_path: Path) -> None: + log_dir = tmp_path / "logs" + log_dir.mkdir() + request_id = "11111111-2222-4333-8444-555555555555" + write_text(log_dir / "caddy.log", f"2026-10-15T03:00:00Z caddy {request_id}\n") + write_text(log_dir / "web.log", f"2026-10-15T01:00:00Z web {request_id}\nignore\n") + write_text(log_dir / "api.log", f"2026-10-15T02:00:00Z api {request_id}\n") + canary = tmp_path / "canary.txt" + write_text(canary, "keep\n") + found = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h"]) + assert found.returncode == 0, found.stderr + assert found.stdout.splitlines() == [ + f"2026-10-15T01:00:00Z web {request_id}", + f"2026-10-15T02:00:00Z api {request_id}", + f"2026-10-15T03:00:00Z caddy {request_id}", + ] + rejected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["; rm -rf /", "--since", "24h"]) + assert rejected.returncode == 2 + assert canary.read_text(encoding="utf-8") == "keep\n" + injected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["$(rm canary.txt)", "--since", "1h"]) + assert injected.returncode == 2 + assert canary.exists() + bad_since = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h;rm"]) + assert bad_since.returncode == 2 + + +def service_logging_blocks(text: str) -> dict[str, str]: + blocks: dict[str, str] = {} + service: str | None = None + in_services = False + capturing = False + buf: list[str] = [] + for line in text.splitlines(): + if line.startswith("services:"): + in_services = True + continue + if in_services and line and not line.startswith(" "): + in_services = False + if not in_services: + continue + if line.startswith(" ") and not line.startswith(" ") and line.endswith(":"): + if capturing and service is not None: + blocks[service] = "\n".join(buf) + service = line.strip()[:-1] + capturing = False + buf = [] + continue + if service and line == " logging:": + capturing = True + buf = [line] + continue + if capturing: + if line.startswith(" ") and not line.startswith(" "): + blocks[service] = "\n".join(buf) + capturing = False + buf = [] + else: + buf.append(line) + if capturing and service is not None: + blocks[service] = "\n".join(buf) + return blocks + + +def test_compose_logging_is_present_and_identical() -> None: + expected = "\n".join( + [ + " logging:", + " driver: json-file", + " options:", + ' max-size: "20m"', + ' max-file: "10"', + ] + ) + expected_names = { + SERVER_COMPOSE: {"api", "web", "caddy"}, + POSTGRES_COMPOSE: {"postgres", "migrator", "migration-checker"}, + } + for path, names in expected_names.items(): + blocks = service_logging_blocks(path.read_text(encoding="utf-8")) + assert set(blocks) == names + for name in names: + assert blocks[name] == expected, name + + +def test_shell_scripts_pass_bash_n() -> None: + for script in (PRUNE_SCRIPT, HEALTH_SCRIPT, DIGEST_SCRIPT, FIND_SCRIPT, NOTIFY_SCRIPT, RECOVERY_SCRIPT): + completed = subprocess.run( + [BASH, "-n", bash_path(script)], + capture_output=True, + text=True, + encoding="utf-8", + ) + assert completed.returncode == 0, script.name + completed.stderr