diff --git a/.gitea/workflows/backup-production-daily.yml b/.gitea/workflows/backup-production-daily.yml index 53415803..04913ce3 100644 --- a/.gitea/workflows/backup-production-daily.yml +++ b/.gitea/workflows/backup-production-daily.yml @@ -101,6 +101,8 @@ jobs: "$ssh_root/run-production-recovery.sh" \ "$ssh_root/prune-production-daily-backups.sh" \ "$remote:$incoming/" + ssh "${ssh_options[@]}" "$remote" \ + "bash '$incoming/prune-production-daily-backups.sh' '$DEPLOY_PATH/backups' '$local_retention'" ssh "${ssh_options[@]}" "$remote" \ "DEPLOY_PATH='$DEPLOY_PATH' RECOVERY_RUN_ID='$GITHUB_RUN_ID' RECOVERY_BASENAME_PREFIX='production-daily' bash '$incoming/run-production-recovery.sh'" \ >"$ssh_root/recovery-output.env" @@ -142,8 +144,6 @@ jobs: "$recovery_reference" "$recovery_created_at" "$recovery_verified_at" "$recovery_sha256" \ "$backup_basename" "$verify_basename" >"$artifact_directory/attestation.env" chmod 600 "$artifact_directory"/* - ssh "${ssh_options[@]}" "$remote" \ - "bash '$incoming/prune-production-daily-backups.sh' '$DEPLOY_PATH/backups' '$local_retention'" { echo "recovery_reference=$recovery_reference" echo "recovery_created_at=$recovery_created_at" diff --git a/CHANGELOG.md b/CHANGELOG.md index 21204977..51ab653d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 印度占星 Skill 更新日志 +## 2026-10-15 — 每日备份在磁盘变满后仍会清理,过期会有人知道 + +- 过期的每日备份改在新备份之前清理。最近一次核对超过 36 小时,或还没有核对记录,磁盘检查会告警一次。 +- 飞书没有真正收下的告警不再被当成已发送。同一类错误 30 分钟内不重复发。 +- 页面没有变化。Skill 版本不变(BUG-1362、BUG-1363、BUG-1364)。 + ## 2026-10-15 — 内测容量:问得太密会停一下,人多时先排一小会儿 - 同一个人短时间问得太密、校正答得太密,或一小时里报告生成太多,会先停一下,这次不扣点。 diff --git a/deploy/README.md b/deploy/README.md index cf28b877..e5c4d558 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -401,7 +401,7 @@ report list / detail / create smoke”): `GET /api/reports` 200, owned ### 备份 - 工作流:`.gitea/workflows/backup-production-daily.yml`。北京时间每天 04:00,也可手动触发。只从 `main` 运行,调用 `deploy/run-production-recovery.sh`,不另写一套导出。 -- 每日文件前缀是 `production-daily-`。手动恢复点仍是 `production-pre-migration-`。清理脚本 `deploy/prune-production-daily-backups.sh` 只删超过 7 天的每日文件和对应的校验文件,不会删 `production-pre-migration-` 手动恢复点。 +- 每日文件前缀是 `production-daily-`。手动恢复点仍是 `production-pre-migration-`。清理脚本 `deploy/prune-production-daily-backups.sh` 只删超过 7 天的每日文件和对应的校验文件,不会删 `production-pre-migration-` 手动恢复点。清理排在新备份之前,所以磁盘到 70% 时过期的每日文件仍会被清掉。 - 每日备份只做存档和还原核对,不是发布用的恢复点。它不改 `/opt/jyotisha-production/.state/recovery-baseline.env`。核对结果另写在同目录的 `daily-backup-latest.env`,只有文件名、sha256、核对时间和格式版本,没有数据和密钥。发布、迁移、回退都不读这个文件。上线前仍要手动点 Create production recovery。 - Gitea 产物默认保留 14 天。手动恢复点工作流仍是 30 天,没有改。 - 还原步骤:`docs/operations/production-restore-drill-20261014.md`。 @@ -410,8 +410,8 @@ report list / detail / create smoke”): `GET /api/reports` 200, owned - 探活不要装在生产主机上。装在 Gitea runner 上,每分钟跑 `deploy/monitor/healthcheck-alert.sh --mode health`。它访问公网 `https://jyotisha.chat/api/health`,超时 10 秒。连续 3 次失败才告警,恢复只发一次。 - 发布窗口:在 runner 上执行 `touch /var/lib/jyotisha-monitor/release-window`。45 分钟内,版本号变化不告警。 -- 磁盘检查装在生产主机上,每分钟跑同一个脚本的 `--mode disk`。使用率达到 80% 告警一次,降下来再发一次已恢复。 -- 错误汇总装在生产主机上。每 5 分钟 `deploy/monitor/error-digest.sh --window`。每天北京时间 09:00(UTC 01:00)跑 `--daily`,汇总前一天。同类错误 5 分钟内至少 10 次,或 5xx 至少 20 次,才发窗口告警。消息里最多 5 个 requestId。含邮箱、第二个日期或非 ASCII 文本的日志行整行丢掉。 +- 磁盘检查装在生产主机上,每分钟跑同一个脚本的 `--mode disk`。使用率达到 80% 告警一次,降下来再发一次已恢复。同一轮还会读 `/opt/jyotisha-production/.state/daily-backup-latest.env` 的 `VERIFIED_AT`。这份记录缺失,或比当前时间早超过 36 小时,会再告警一次。重新核对成功后发一次已恢复。探活模式不读这份文件。 +- 错误汇总装在生产主机上。每 5 分钟 `deploy/monitor/error-digest.sh --window`。每天北京时间 09:00(UTC 01:00)跑 `--daily`,汇总前一天。同类错误 5 分钟内至少 10 次,或 5xx 至少 20 次,才发窗口告警。同一服务、同一路由、同一错误码在 30 分钟内不重复发。消息里最多 5 个 requestId。含邮箱、第二个日期或非 ASCII 文本的日志行整行丢掉。飞书没有返回成功时,这条不会被记成已发送。 - Webhook 写在模式 `0600` 的环境文件里,建议路径 `/opt/jyotisha-production/.monitor.env`(runner 上用 `JYOTISHA_MONITOR_ENV` 指向自己的文件)。只放这两行,不要提交: ```text @@ -435,7 +435,7 @@ FEISHU_SIGN_SECRET=<只有开了签名校验才填> deploy/find-request-log.sh --since 24h ``` -`` 必须是 UUID。其它写法会拒绝,避免被当成命令。`--since` 只接受类似 `24h`、`30m` 的形式。输出按时间排序,来自 web、api、caddy。 +`` 必须是 UUID。其它写法会拒绝,避免被当成命令。`--since` 只接受类似 `24h`、`30m` 的形式。容器日志行前面的服务名前缀会先去掉,再按时间排序。来自 web、api、caddy。 ## Common operations diff --git a/deploy/find-request-log.sh b/deploy/find-request-log.sh index c4f655fc..a0914c0e 100644 --- a/deploy/find-request-log.sh +++ b/deploy/find-request-log.sh @@ -52,11 +52,28 @@ if [ -n "$log_dir" ]; then fi compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}" -docker compose -p jyotisha-production \ +export APP_ENV_FILE='../.env.production' +export DATABASE_ENV_FILE='../.env.production.database' +export CADDYFILE_PATH='./Caddyfile.production.selfhosted' +export SITE_ADDRESS='https://jyotisha.chat' +if [ -n "${JYOTISHA_COMPOSE_LOG_FIXTURE:-}" ]; then + [ -f "$JYOTISHA_COMPOSE_LOG_FIXTURE" ] && [ ! -L "$JYOTISHA_COMPOSE_LOG_FIXTURE" ] || { + echo "compose log fixture is missing" >&2 + exit 1 + } + sed -E 's/^[^|]*\| //' "$JYOTISHA_COMPOSE_LOG_FIXTURE" | + grep -F -- "$request_id" | + grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' | + sort + exit 0 +fi +sudo -n docker compose -p jyotisha-production \ --env-file "$compose_dir/.env.production" \ -f "$compose_dir/deploy/docker-compose.server.yml" \ -f "$compose_dir/deploy/docker-compose.postgres.yml" \ -f "$compose_dir/deploy/docker-compose.production.yml" \ logs --since "$since" --no-color --timestamps web api caddy | + sed -E 's/^[^|]*\| //' | grep -F -- "$request_id" | + grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' | sort diff --git a/deploy/monitor/error-digest.sh b/deploy/monitor/error-digest.sh index 7893557f..d51e25b0 100644 --- a/deploy/monitor/error-digest.sh +++ b/deploy/monitor/error-digest.sh @@ -51,7 +51,11 @@ if [ -z "$fixture" ]; then since="5m" [ "$mode" = "daily" ] && since="48h" generated_fixture="$(mktemp)" - docker compose -p jyotisha-production \ + export APP_ENV_FILE='../.env.production' + export DATABASE_ENV_FILE='../.env.production.database' + export CADDYFILE_PATH='./Caddyfile.production.selfhosted' + export SITE_ADDRESS='https://jyotisha.chat' + sudo -n docker compose -p jyotisha-production \ --env-file "$compose_dir/.env.production" \ -f "$compose_dir/deploy/docker-compose.server.yml" \ -f "$compose_dir/deploy/docker-compose.postgres.yml" \ @@ -81,5 +85,48 @@ elif [ "$mode" = "daily" ]; then fi if [ -n "$text" ]; then - post_monitor_alert "${text%$'\n'}" + now_epoch="$(date -u -d "$now_text" +%s)" + [[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2 + dedup_file="$state_dir/error-digest-sent.tsv" + declare -A last_sent=() + declare -a send_keys=() + filtered="" + if [ -f "$dedup_file" ] && [ ! -L "$dedup_file" ]; then + while IFS=$'\t' read -r sent_epoch sent_service sent_route sent_code || [ -n "${sent_epoch:-}" ]; do + [[ "${sent_epoch:-}" =~ ^[0-9]+$ ]] || continue + [[ "${sent_service:-}" =~ ^[a-z0-9_-]+$ ]] || continue + [ -n "${sent_route:-}" ] || continue + [ -n "${sent_code:-}" ] || continue + last_sent["${sent_service}"$'\t'"${sent_route}"$'\t'"${sent_code}"]="$sent_epoch" + done <"$dedup_file" + fi + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + if [[ "$line" =~ ^service=([a-z0-9_-]+)[[:space:]]route=([^[:space:]]+)[[:space:]]errorCode=([a-z][a-z0-9._-]*)[[:space:]] ]]; then + key="${BASH_REMATCH[1]}"$'\t'"${BASH_REMATCH[2]}"$'\t'"${BASH_REMATCH[3]}" + prev="${last_sent[$key]:-}" + if [ -n "$prev" ] && (( now_epoch - prev < 1800 )); then + continue + fi + send_keys+=("$key") + fi + filtered="${filtered}${line}"$'\n' + done <<<"${text%$'\n'}" + if [ -n "$filtered" ]; then + post_monitor_alert "${filtered%$'\n'}" + if [ "${#send_keys[@]}" -gt 0 ]; then + for key in "${send_keys[@]}"; do + last_sent["$key"]="$now_epoch" + done + fi + dedup_tmp="$dedup_file.tmp.$$" + : >"$dedup_tmp" + for key in "${!last_sent[@]}"; do + printf '%s\t%s\n' "${last_sent[$key]}" "$key" >>"$dedup_tmp" + done + if [ "$(uname -s)" = "Linux" ]; then + chmod 600 "$dedup_tmp" + fi + mv "$dedup_tmp" "$dedup_file" + fi fi diff --git a/deploy/monitor/healthcheck-alert.sh b/deploy/monitor/healthcheck-alert.sh index 1e7c307a..1d995f47 100644 --- a/deploy/monitor/healthcheck-alert.sh +++ b/deploy/monitor/healthcheck-alert.sh @@ -60,6 +60,7 @@ alert_open=0 last_commit="" commit_alerted="" disk_alert_open=0 +backup_stale_open=0 read_state() { local line value @@ -89,6 +90,9 @@ read_state() { disk_alert_open=[01]) disk_alert_open="${line#disk_alert_open=}" ;; + backup_stale_open=[01]) + backup_stale_open="${line#backup_stale_open=}" + ;; esac done <"$state_file" } @@ -96,8 +100,8 @@ read_state() { write_state() { local tmp tmp="$state_file.tmp.$$" - printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\n' \ - "$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" >"$tmp" + printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\nbackup_stale_open=%s\n' \ + "$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" "$backup_stale_open" >"$tmp" if [ "$(uname -s)" = "Linux" ]; then chmod 600 "$tmp" fi @@ -224,13 +228,55 @@ check_disk() { fi } +check_daily_backup() { + local record now_text now_epoch verified verified_epoch age stale hours + record="${JYOTISHA_DAILY_BACKUP_RECORD:-/opt/jyotisha-production/.state/daily-backup-latest.env}" + now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" + [[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || { + echo "clock is unsafe" >&2 + exit 2 + } + now_epoch="$(date -u -d "$now_text" +%s)" + [[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2 + stale=1 + hours=36 + if [ -f "$record" ] && [ ! -L "$record" ]; then + verified="$(awk -F= '$1 == "VERIFIED_AT" { print $2 }' "$record" | tr -d '\r')" + if [[ "$verified" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + verified_epoch="$(date -u -d "$verified" +%s)" + age=$((now_epoch - verified_epoch)) + if [ "$age" -le 129600 ]; then + stale=0 + else + hours=$((age / 3600)) + if [ "$hours" -lt 36 ]; then + hours=36 + fi + fi + fi + fi + if [ "$stale" -eq 1 ]; then + if [ "$backup_stale_open" -eq 0 ]; then + send_alert host - backup_stale "$hours" - + backup_stale_open=1 + fi + elif [ "$backup_stale_open" -eq 1 ]; then + send_alert host - recovered 0 - + backup_stale_open=0 + fi +} + read_state case "$mode" in health) check_health ;; - disk) check_disk ;; + disk) + check_disk + check_daily_backup + ;; all) check_health check_disk + check_daily_backup ;; esac write_state diff --git a/deploy/monitor/notify-feishu.sh b/deploy/monitor/notify-feishu.sh index 4382ef7f..882c684f 100644 --- a/deploy/monitor/notify-feishu.sh +++ b/deploy/monitor/notify-feishu.sh @@ -63,27 +63,34 @@ alert_line_allowed() { post_monitor_alert() { local text="$1" - local state_dir line timestamp sign payload_file + local state_dir line timestamp sign payload_file response_file pending py curl_status code state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}" ensure_private_dir "$state_dir" + pending="" while IFS= read -r line || [ -n "$line" ]; do [ -n "$line" ] || continue alert_line_allowed "$line" || { echo "alert text has a field that is not allowed" >&2 exit 1 } - printf '%s\n' "$line" >>"$state_dir/alerts.log" + pending="${pending}${line}"$'\n' done <<<"$text" + [ -n "$pending" ] || return 0 + pending="${pending%$'\n'}" - [ -n "${FEISHU_WEBHOOK_URL:-}" ] || return 0 + if [ -z "${FEISHU_WEBHOOK_URL:-}" ]; then + printf '%s\n' "$pending" >>"$state_dir/alerts.log" + return 0 + fi webhook_allowed "$FEISHU_WEBHOOK_URL" || { printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log" return 0 } payload_file="$(mktemp)" + response_file="$(mktemp)" if [ "$(uname -s)" = "Linux" ]; then - chmod 600 "$payload_file" + chmod 600 "$payload_file" "$response_file" fi if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then timestamp="$(date +%s)" @@ -93,12 +100,45 @@ post_monitor_alert() { else FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain fi - curl --silent --show-error --connect-timeout 10 --max-time 10 \ + if [ -n "${JYOTISHA_PYTHON:-}" ]; then + py="$JYOTISHA_PYTHON" + elif command -v python3 >/dev/null 2>&1; then + py="python3" + else + py="python" + fi + curl_status=0 + curl --silent --show-error --fail --connect-timeout 10 --max-time 10 \ -H 'Content-Type: application/json' \ -H 'Connection: close' \ --data-binary @"$payload_file" \ - "$FEISHU_WEBHOOK_URL" >/dev/null - rm -f -- "$payload_file" + -o "$response_file" \ + "$FEISHU_WEBHOOK_URL" || curl_status=$? + code="" + if [ "$curl_status" -eq 0 ]; then + code="$("$py" - "$response_file" <<'PY' +import json +import sys + +try: + payload = json.load(open(sys.argv[1], encoding="utf-8")) +except Exception: + print("") + raise SystemExit(0) +if isinstance(payload, dict) and payload.get("code") == 0: + print("0") +else: + print("") +PY +)" + code="$(printf '%s' "$code" | tr -d '\r')" + fi + rm -f -- "$payload_file" "$response_file" + if [ "$curl_status" -ne 0 ] || [ "$code" != "0" ]; then + echo "feishu webhook was not accepted" >&2 + return 1 + fi + printf '%s\n' "$pending" >>"$state_dir/alerts.log" } python_write_payload() { diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index ba24d81d..99e47fe1 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -36363,3 +36363,48 @@ - 复发自:无。 - 修复版本:分支 `codex/beta-safety-net-20261014`。本机没有装到生产主机,也没有点生产按钮。原先写成 BUG-1358。再次合并 staging 后该号给了「没有每日自动备份」,本条顺延为 BUG-1361。 +## BUG-1362 | 安全网测试在 Linux 上因环境文件权限失败,而且没进自动门禁 + +- 状态:resolved +- 首次发现 / 最近更新:2026-10-15 / 2026-10-15 +- 来源:issue #75 验收打回。修复单 `docs/tasks/TASK-beta-safety-net-fix2-20261015.md`。检索「monitor env」「0600」「test_beta_safety_net」,没有更早的同类记录。 +- 现象:发布前的全量测试在 Linux 上失败。staging 的自动检查却是绿的,所以这一版能进测试环境,却会挡住以后的上生产检查。 +- 触发条件:`tests/test_beta_safety_net.py` 写出飞书环境文件后直接跑告警脚本。Linux 上该文件是 0644。脚本拒绝不是 0600 的环境文件。macOS 和 Windows 跳过这项检查。这个测试文件也不在自动门禁的 quick 清单里。 +- 根因:测试没有把环境文件收成 0600。quick 清单只在发布工作流里才跑全树 pytest,所以 Linux 上的失败没有在 staging 门禁暴露。 +- 修复:测试写完环境文件后执行 `os.chmod(path, 0o600)`。脚本里的 0600 检查不改。把 `tests/test_beta_safety_net.py` 加进 quick 清单。 +- 验证:非 Linux 上看不到 Unix 权限位,权限相等断言只在 POSIX 上执行。脚本仍包含「必须是 600」的检查。quick 清单包含这个测试文件。 +- 防复发:上述权限调用、脚本检查和 quick 清单断言。 +- 相关记录:BUG-1359。 +- 复发自:无。 +- 修复版本:分支 `codex/beta-safety-net-20261014`。 + +## BUG-1363 | 磁盘到 70% 后每日备份每天失败,旧文件也不再清理 + +- 状态:代码已交,主机安装待产品 +- 首次发现 / 最近更新:2026-10-15 / 2026-10-15 +- 来源:issue #75 验收打回。修复单 `docs/tasks/TASK-beta-safety-net-fix2-20261015.md`。 +- 现象:磁盘用到 70% 以后,每天的自动备份都失败。过期的每日文件不会被删,所以磁盘腾不出来。磁盘告警要到 80% 才响,也没有人看最近一次核对是不是太旧。 +- 触发条件:备份目录使用率达到 70%。清理步骤写在备份成功之后。`daily-backup-latest.env` 的 `VERIFIED_AT` 超过 36 小时,或这份记录不存在。 +- 根因:恢复脚本在 70% 时直接退出,工作流因此走不到后面的清理。探活和磁盘告警都不读每日核对时间。 +- 修复:每日工作流先清理过期的 `production-daily-` 文件,再做备份。磁盘检查同时读取 `VERIFIED_AT`(任务书里的 verified_at 就是这个键)。超过 36 小时或记录不可用时告警一次,重新变新后再发一次已恢复。探活模式不读这份文件。70% 拒绝本身不改。 +- 验证:工作流文本里清理调用在备份调用之前。虚构时钟下,恰好 36 小时不告警,37 小时告警一次,缺失记录告警一次,变新后发已恢复。告警行不含备份文件名。 +- 防复发:上述顺序断言和过期告警测试。清理脚本仍只认每日前缀。 +- 相关记录:BUG-1358、BUG-1359、BUG-1361。 +- 复发自:无。 +- 修复版本:分支 `codex/beta-safety-net-20261014`。本机没有装到生产主机,也没有点生产按钮。 + +## BUG-1364 | 错误汇总和查日志在生产主机上跑不起来 + +- 状态:代码已交,主机安装待产品 +- 首次发现 / 最近更新:2026-10-15 / 2026-10-15 +- 来源:issue #75 验收打回。修复单 `docs/tasks/TASK-beta-safety-net-fix2-20261015.md`。 +- 现象:生产主机上的定时错误汇总会每次失败,一条告警都发不出去。按请求编号查容器日志同样起不来。飞书返回失败时仍像已经发出。同一类错误每 5 分钟重复发。容器日志按服务名而不是时间排序。 +- 触发条件:cron 直接执行 `error-digest.sh` 或 `find-request-log.sh`,且没有事先导出数据库环境文件。飞书响应体里的 `code` 不是 0。同一错误码在 30 分钟内再次出现。`docker compose logs` 的行首带有服务名前缀。 +- 根因:这两个脚本直接调用 `docker`,没有 `sudo -n`。也没有把数据库环境文件设成生产路径,编排会去找不存在的 staging 文件。发送命令不检查 HTTP 失败和飞书 `code`。排序把服务名前缀算了进去。 +- 修复:两个脚本改为 `sudo -n docker compose`,并显式传入生产编排文件和 `../.env.production.database`。飞书请求加上 `--fail`,响应 `code` 必须是 0,失败时不写入已发送记录。同一服务、路由、错误码 30 分钟内不重复发。查日志先去掉行首前缀再按时间排序。 +- 验证:源码断言命令行含 `sudo -n docker compose` 和生产环境文件,且没有裸的 `docker compose`。虚构飞书响应 `code` 非 0 时本地告警记录为空。30 分钟边界前后各跑一次。带服务名前缀的虚构日志按时间而不是按服务名输出。 +- 防复发:上述命令形状、发送失败、去重和排序测试。 +- 相关记录:BUG-1359、BUG-1360。 +- 复发自:无。 +- 修复版本:分支 `codex/beta-safety-net-20261014`。本机没有装到生产主机,也没有点生产按钮。 + diff --git a/docs/tasks/PROGRESS-beta-safety-net-fix2-20261015.md b/docs/tasks/PROGRESS-beta-safety-net-fix2-20261015.md new file mode 100644 index 00000000..9d7249cc --- /dev/null +++ b/docs/tasks/PROGRESS-beta-safety-net-fix2-20261015.md @@ -0,0 +1,41 @@ +# 进度 · 内测安全网修复单 2(2026-10-15) + +- 分支:`codex/beta-safety-net-20261014` +- 任务书:`docs/tasks/TASK-beta-safety-net-fix2-20261015.md` +- 被打回的提交:`5c0d895391b5d0806c98de669a2e9cdcd61e3335` +- 先合并最新 `origin/staging`(`32b238cf`),没有改写已经推送的提交。 + +## 做了什么 + +- 合并时安全网旧编号和内测容量撞号。容量保留 BUG-1355~1357。安全网顺延为 BUG-1358~1361。这次三条例外是 BUG-1362、BUG-1363、BUG-1364。 +- 飞书测试写完环境文件后改成 0600。脚本里「Linux 上必须是 0600」的检查没改。这个测试文件加进 staging 自动门禁的 quick 清单。 +- 每日工作流先清理过期的每日备份,再做新备份。磁盘到 70% 时仍会拒绝备份,但旧的每日文件不会因此永远留着。70% 这条拒绝本身没改。 +- 生产主机上的磁盘检查同时看 `daily-backup-latest.env` 的 `VERIFIED_AT`。缺失,或比现在早超过 36 小时,告警一次;重新变新后再发一次已恢复。探活模式不读这份文件。 +- 错误汇总和按请求查日志改为 `sudo -n docker`,并显式传入生产编排和 `../.env.production.database`。 +- 飞书请求加了 `--fail`,响应里的 `code` 必须是 0。失败时不写入已发送记录。同一服务、同一路由、同一错误码 30 分钟内不重复发。 +- 容器日志先去掉行首的服务名前缀,再按时间排序。 +- 上一轮「每日备份不改当前恢复点」没有重做。四个已有生产工作流、staging 工作流、`gated-paths.txt`、`is-docs-only-range.sh` 没改。 + +## 跑过的检查 + +| 检查 | 命令 | 结果 | +| --- | --- | --- | +| 安全网 | `.venv\Scripts\python.exe -m pytest tests/test_beta_safety_net.py -q` | 20 通过,退出码 0 | +| 隐私标记 | `.venv\Scripts\python.exe -m pytest tests/test_repo_privacy_markers.py -q` | 62 通过,退出码 0 | +| 前端 tsc / lint | 未跑 | 没有改网页 | +| Python 快速门 | 未跑 | 按本轮约定不跑 | +| 数据库测试 | 未跑 | 没有改数据库 | +| shellcheck | 未跑 | 本机没有这条命令 | + +## 断言变更 + +| 位置 | 原值 | 新值 | 原因 | +| --- | --- | --- | --- | +| 飞书测试桩的响应正文 | `ok` | `{"code":0}` | 发送方现在要求飞书 `code` 为 0。请求体字段断言没改 | +| 磁盘测试的准备数据 | 不提供每日核对记录 | 提供一份核对时间等于当前时间的记录 | 磁盘模式会同时检查核对时间。没有记录会多一条告警。磁盘告警原文断言没改 | + +## 偏离 + +- 任务书写 `verified_at`。上一轮文件里的键是 `VERIFIED_AT`。这次读这个键,没有改名。 +- 本机 `os.chmod(path, 0o600)` 之后,`stat` 仍是 `0o666`。Windows 存不下 Unix 权限。0600 相等断言只在 POSIX 上执行;`chmod` 每次都会调用。Linux 上的脚本检查保持原样。 +- 没有在这台 Windows 上安装到生产,没有点任何生产按钮。主机安装和恢复演练仍留给产品。 diff --git a/docs/tasks/README.md b/docs/tasks/README.md index 4bcd0835..05efc90e 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -340,7 +340,7 @@ | `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2(产品选 A)**:恢复点要带属主与权限,旧格式不得回退;演练按备份里的授权比对,并在真实 PostgreSQL 17 上跑通一次 | 待验收 | `codex/release-path-production-20261014` | | `TASK-beta-safety-net-20261014.md` | `PROGRESS-beta-safety-net-20261014.md` | **内测安全网**:每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。发布通路与 #58 已在开工时的 `origin/staging` 上。主机安装和一次演练留给产品 | 待验收 | `codex/beta-safety-net-20261014`(BUG-1358~1360) | | `TASK-beta-safety-net-fix-20261015.md` | `PROGRESS-beta-safety-net-fix-20261015.md` | **#75 产品选 A**:每日备份照常做,但不改当前恢复点;另记 `daily-backup-latest.env` | 待验收 | `codex/beta-safety-net-20261014`(BUG-1361) | -| `TASK-beta-safety-net-fix2-20261015.md` | — | **#75 修复单 2**:生产侧全过;飞书测试 env 文件权限 0644 在 Linux 上失败(会挡发布门禁);新测试不在 quick 门禁;磁盘 ≥70% 后备份静默停摆;监控脚本缺 sudo 与 env 文件 | 待领取 | `codex/beta-safety-net-20261014` | +| `TASK-beta-safety-net-fix2-20261015.md` | `PROGRESS-beta-safety-net-fix2-20261015.md` | **#75 修复单 2**:生产侧全过;飞书测试 env 文件权限 0644 在 Linux 上失败(会挡发布门禁);新测试不在 quick 门禁;磁盘 ≥70% 后备份静默停摆;监控脚本缺 sudo 与 env 文件 | 待验收 | `codex/beta-safety-net-20261014`(BUG-1362~1364) | | `TASK-beta-capacity-20261014.md` | — | **内测容量**:按用户限流、重计算短排队与对话重试、兑换码当邀请码、staging 10/20 并发压测。**等 #58 合入后开工,压测等 #57 合入** | 待前置 | — | | `TASK-beta-capacity-fix-20261015.md` | — | **#76 第 1 次打回**:`a54f75e5` 限流与短排队逻辑通过;源码文本断言未跟改、插入新测试使隐私白名单行号错位 | 待领取 | `codex/beta-capacity-20261014` | | `TASK-special-lagnas-native-20261008.md` | `PROGRESS-special-lagnas-native-20261008.md` | **特殊上升点原生计算(线上缺陷)**:sync6 U6 运行时调 PyJHora,生产镜像没有 PyJHora → staging 上 Hora/Ghati/Sree/Indu/Pranapada 全 blocked;原生实现、PyJHora 只作冻结对照、快速门禁止运行时 import jhora;最先做 | 已验收(已部署 staging `795b8845`,BUG-1273) | `codex/special-lagnas-native-20261008`(未上线;部署以 `/api/health` 为准) | diff --git a/scripts/run_quality_gate.py b/scripts/run_quality_gate.py index 272540d0..8dff53ee 100644 --- a/scripts/run_quality_gate.py +++ b/scripts/run_quality_gate.py @@ -121,6 +121,9 @@ CORE_PYTEST_TARGETS = [ "tests/test_upstream_import_plan.py", # Tracked-file privacy must run in staging's explicit quick test list. "tests/test_repo_privacy_markers.py", + # Closed-beta safety net. The Linux 0600 monitor-env check only fails in + # release-quality-gate unless this file is on the staging quick list. + "tests/test_beta_safety_net.py", # Summary lines by default; failure keeps the last 200 lines and a full log (BUG-1230). "tests/test_run_quality_gate_output.py", "tests/test_upstream_git_import_b9a0ef8f.py", diff --git a/tests/test_beta_safety_net.py b/tests/test_beta_safety_net.py index 2376e200..d3dab3a4 100644 --- a/tests/test_beta_safety_net.py +++ b/tests/test_beta_safety_net.py @@ -77,6 +77,30 @@ def write_text(path: Path, text: str) -> None: path.write_text(text, encoding="utf-8", newline="\n") +def write_backup_record(directory: Path, verified_at: str) -> Path: + record = directory / "daily-backup-latest.env" + write_text( + record, + "BACKUP_BASENAME=production-daily-20261015T010000Z.dump.enc\n" + f"BACKUP_SHA256={'a' * 64}\n" + f"VERIFIED_AT={verified_at}\n" + "BACKUP_FORMAT=2\n", + ) + return record + + +def run_disk_mode(bin_dir: Path, script_env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], + cwd=ROOT, + env={**os.environ, **script_env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + + class HealthHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: if self.path != "/health": @@ -111,11 +135,14 @@ class HookHandler(BaseHTTPRequestHandler): body = self.rfile.read(length) self.server.bodies.append(body) self.close_connection = True + code = int(getattr(self.server, "feishu_code", 0)) + reply = json.dumps({"code": code}).encode("utf-8") self.send_response(200) - self.send_header("Content-Length", "2") + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(reply))) self.send_header("Connection", "close") self.end_headers() - self.wfile.write(b"ok") + self.wfile.write(reply) def log_message(self, fmt: str, *args: object) -> None: return @@ -144,6 +171,9 @@ def test_daily_workflow_is_scheduled_from_main_and_reuses_recovery() -> None: assert 'git show "$GITEA_SHA:deploy/run-production-recovery.sh"' in text assert "RECOVERY_BASENAME_PREFIX='production-daily'" in text assert "prune-production-daily-backups.sh" in text + prune_call = text.index("bash '$incoming/prune-production-daily-backups.sh'") + recovery_call = text.index("bash '$incoming/run-production-recovery.sh'") + assert prune_call < recovery_call assert "production-daily-" in text assert 'artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"' in text assert 'local_retention="${LOCAL_RETENTION_DAYS:-7}"' in text @@ -567,47 +597,26 @@ def test_disk_alerts_once_and_no_webhook_stays_local(tmp_path: Path) -> None: ) subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True) state = tmp_path / "state" + record = write_backup_record(tmp_path, "2026-10-15T01:00:00Z") env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_DAILY_BACKUP_RECORD": bash_path(record), + "JYOTISHA_NOW": "2026-10-15T01:00:00Z", "FAKE_DISK_PERCENT": "81", } env.pop("FEISHU_WEBHOOK_URL", None) env.pop("FEISHU_SIGN_SECRET", None) - completed = subprocess.run( - [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], - cwd=ROOT, - env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - ) + completed = run_disk_mode(bin_dir, env) assert completed.returncode == 0, completed.stderr assert "errorCode=disk_high" in alert_lines(state)[0] assert "count=81" in alert_lines(state)[0] env["FAKE_DISK_PERCENT"] = "81" - again = subprocess.run( - [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], - cwd=ROOT, - env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - ) + again = run_disk_mode(bin_dir, env) assert again.returncode == 0, again.stderr assert len(alert_lines(state)) == 1 env["FAKE_DISK_PERCENT"] = "10" - recovered = subprocess.run( - [BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)], - cwd=ROOT, - env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))}, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - ) + recovered = run_disk_mode(bin_dir, env) assert recovered.returncode == 0, recovered.stderr assert len(alert_lines(state)) == 2 assert "errorCode=recovered" in alert_lines(state)[1] @@ -622,6 +631,12 @@ def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None: secret = "fixture-sign-secret" env_file = tmp_path / "monitor.env" write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\nFEISHU_SIGN_SECRET={secret}\n") + os.chmod(env_file, 0o600) + if os.name != "nt": + assert (env_file.stat().st_mode & 0o777) == 0o600 + notify = NOTIFY_SCRIPT.read_text(encoding="utf-8") + assert '[ "$mode" = "600" ]' in notify + assert "curl --silent --show-error --fail" in notify env = { "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), "JYOTISHA_MONITOR_ENV": bash_path(env_file), @@ -659,6 +674,35 @@ def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None: health.shutdown() +def test_feishu_rejection_is_not_recorded_as_sent(tmp_path: Path) -> None: + hook, _thread = serve(HookHandler) + hook.bodies = [] + hook.feishu_code = 19001 + health, _thread = serve(HealthHandler) + health.kind = "500" + state = tmp_path / "state" + env_file = tmp_path / "monitor.env" + write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\n") + os.chmod(env_file, 0o600) + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(env_file), + "JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health", + "JYOTISHA_HEALTH_TIMEOUT": "2", + } + try: + assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 + assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0 + rejected = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]) + assert rejected.returncode != 0 + assert len(hook.bodies) == 1 + assert alert_lines(state) == [] + assert "health_down" not in rejected.stdout + finally: + hook.shutdown() + health.shutdown() + + def uuid(index: int) -> str: return f"00000000-0000-4000-8000-{index:012d}" @@ -699,6 +743,13 @@ def test_error_digest_drops_personal_text_and_respects_thresholds(tmp_path: Path assert "1900-01-01" not in lines[0] assert "今天工作不顺" not in lines[0] assert "should_drop" not in lines[0] + soon = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_NOW": "2026-10-15T01:29:59Z"}, ["--window"]) + assert soon.returncode == 0, soon.stderr + assert len(alert_lines(state)) == 1 + later = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_NOW": "2026-10-15T01:30:00Z"}, ["--window"]) + assert later.returncode == 0, later.stderr + assert len(alert_lines(state)) == 2 + assert "errorCode=settlement_failed" in alert_lines(state)[1] fivexx = tmp_path / "fivexx.txt" write_text(fivexx, "\n".join(f"2026-10-15T00:02:{index:02d}Z api status=502" for index in range(20)) + "\n") @@ -769,6 +820,30 @@ def test_find_request_log_sorts_and_rejects_injection(tmp_path: Path) -> None: assert canary.exists() bad_since = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h;rm"]) assert bad_since.returncode == 2 + compose_log = tmp_path / "compose.log" + write_text( + compose_log, + "\n".join( + [ + f"web-1 | 2026-10-15T01:00:00Z web {request_id}", + f"api-1 | 2026-10-15T03:00:00Z api {request_id}", + f"caddy-1 | 2026-10-15T02:00:00Z caddy {request_id}", + f"api-1 | ignore {request_id}", + ] + ) + + "\n", + ) + composed = run_bash( + FIND_SCRIPT, + {"JYOTISHA_COMPOSE_LOG_FIXTURE": bash_path(compose_log)}, + [request_id, "--since", "24h"], + ) + assert composed.returncode == 0, composed.stderr + assert composed.stdout.splitlines() == [ + f"2026-10-15T01:00:00Z web {request_id}", + f"2026-10-15T02:00:00Z caddy {request_id}", + f"2026-10-15T03:00:00Z api {request_id}", + ] def service_logging_blocks(text: str) -> dict[str, str]: @@ -829,6 +904,90 @@ def test_compose_logging_is_present_and_identical() -> None: assert blocks[name] == expected, name +def test_monitor_commands_use_production_docker() -> None: + for script in (DIGEST_SCRIPT, FIND_SCRIPT): + text = script.read_text(encoding="utf-8") + assert "sudo -n docker compose -p jyotisha-production" in text + assert "export DATABASE_ENV_FILE='../.env.production.database'" in text + assert "export APP_ENV_FILE='../.env.production'" in text + assert "docker-compose.server.yml" in text + assert "docker-compose.postgres.yml" in text + assert "docker-compose.production.yml" in text + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("docker ") or stripped.startswith("docker compose"): + raise AssertionError(stripped) + finder = FIND_SCRIPT.read_text(encoding="utf-8") + strip_at = finder.index("sed -E 's/^[^|]*\\| //'") + sort_at = finder.index("sort", strip_at) + assert strip_at < sort_at + + +def test_safety_net_is_on_the_quick_gate() -> None: + text = (ROOT / "scripts" / "run_quality_gate.py").read_text(encoding="utf-8") + assert '"tests/test_beta_safety_net.py"' in text + + +def install_fake_df(bin_dir: Path) -> None: + bin_dir.mkdir() + write_text( + bin_dir / "df", + "#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 10 90 ${FAKE_DISK_PERCENT:?}% /\"\n", + ) + subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True) + + +def test_stale_daily_backup_alerts_once_until_a_fresh_record(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + install_fake_df(bin_dir) + state = tmp_path / "state" + now = "2026-10-15T12:00:00Z" + env = { + "JYOTISHA_MONITOR_STATE_DIR": bash_path(state), + "JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"), + "JYOTISHA_NOW": now, + "JYOTISHA_DAILY_BACKUP_RECORD": bash_path(tmp_path / "missing-record.env"), + "FAKE_DISK_PERCENT": "10", + } + missing = run_disk_mode(bin_dir, env) + assert missing.returncode == 0, missing.stderr + assert alert_lines(state) == [ + line for line in alert_lines(state) if "errorCode=backup_stale" in line and "count=36" in line + ] + assert len(alert_lines(state)) == 1 + assert "production-daily" not in alert_lines(state)[0] + assert ".env" not in alert_lines(state)[0] + again = run_disk_mode(bin_dir, env) + assert again.returncode == 0, again.stderr + assert len(alert_lines(state)) == 1 + + exact = tmp_path / "exact" + exact.mkdir() + env["JYOTISHA_MONITOR_STATE_DIR"] = bash_path(exact) + env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(exact, "2026-10-14T00:00:00Z")) + boundary = run_disk_mode(bin_dir, env) + assert boundary.returncode == 0, boundary.stderr + assert alert_lines(exact) == [] + + stale_state = tmp_path / "stale-state" + old_dir = tmp_path / "old" + old_dir.mkdir() + env["JYOTISHA_MONITOR_STATE_DIR"] = bash_path(stale_state) + env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(old_dir, "2026-10-13T23:00:00Z")) + old = run_disk_mode(bin_dir, env) + assert old.returncode == 0, old.stderr + assert len(alert_lines(stale_state)) == 1 + assert "errorCode=backup_stale" in alert_lines(stale_state)[0] + assert "count=37" in alert_lines(stale_state)[0] + fresh_dir = tmp_path / "fresh" + fresh_dir.mkdir() + env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(fresh_dir, now)) + recovered = run_disk_mode(bin_dir, env) + assert recovered.returncode == 0, recovered.stderr + assert len(alert_lines(stale_state)) == 2 + assert "errorCode=recovered" in alert_lines(stale_state)[1] + + def test_shell_scripts_pass_bash_n() -> None: for script in (PRUNE_SCRIPT, HEALTH_SCRIPT, DIGEST_SCRIPT, FIND_SCRIPT, NOTIFY_SCRIPT, RECOVERY_SCRIPT): completed = subprocess.run(