Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jesse-uxandGrok (agent loop)
|
9455fbfc4f | ||
|
jesse-uxandGrok (agent loop)
|
14897f0bf3 | ||
|
Jesse_ChenandClaude Opus 5.5
|
32b238cf86 | ||
|
jesse-uxandGrok (agent loop)
|
5c0d895391 | ||
|
jesse-uxandGrok (agent loop)
|
6ba63470a9 | ||
|
Claude
|
168e4dde19 | ||
|
Jesse_ChenandClaude Opus 5.5
|
2d41338897 | ||
|
jesse-uxandGrok (agent loop)
|
53d8df257d |
@@ -0,0 +1,183 @@
|
||||
name: Backup Production Daily
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 04:00 Asia/Shanghai (UTC+8). Gitea cron is UTC.
|
||||
- cron: "0 20 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
artifact_retention_days:
|
||||
description: Days to keep the encrypted artifact on Gitea
|
||||
required: false
|
||||
default: "14"
|
||||
type: string
|
||||
local_retention_days:
|
||||
description: Days to keep this workflow's encrypted files on the host
|
||||
required: false
|
||||
default: "7"
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
concurrency:
|
||||
group: production-mutation
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
backup:
|
||||
if: gitea.ref == 'refs/heads/main'
|
||||
runs-on: xiaoxin
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
GITEA_SHA: ${{ gitea.sha }}
|
||||
GITEA_REF: ${{ gitea.ref }}
|
||||
GITEA_API_URL: ${{ gitea.api_url }}
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }}
|
||||
DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }}
|
||||
DEPLOY_USER: ${{ vars.PRODUCTION_USER }}
|
||||
DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }}
|
||||
PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }}
|
||||
ARTIFACT_RETENTION_DAYS: ${{ inputs.artifact_retention_days }}
|
||||
LOCAL_RETENTION_DAYS: ${{ inputs.local_retention_days }}
|
||||
steps:
|
||||
- name: Create, restore-verify, and retrieve the daily encrypted backup
|
||||
id: backup
|
||||
env:
|
||||
SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "$GITEA_REF" == "refs/heads/main" ]]
|
||||
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"
|
||||
[ -n "$artifact_retention" ] || artifact_retention=14
|
||||
local_retention="${LOCAL_RETENTION_DAYS:-7}"
|
||||
[ -n "$local_retention" ] || local_retention=7
|
||||
[[ "$artifact_retention" =~ ^[1-9][0-9]{0,3}$ ]]
|
||||
[[ "$local_retention" =~ ^[1-9][0-9]{0,3}$ ]]
|
||||
[[ "$DEPLOY_HOST" == "118.194.235.34" ]]
|
||||
[[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 ))
|
||||
[[ "$DEPLOY_USER" == "deploy" ]]
|
||||
[[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]]
|
||||
[[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]]
|
||||
test -n "$PRODUCTION_KNOWN_HOSTS"
|
||||
echo "artifact_retention_days=$artifact_retention" >>"$GITHUB_OUTPUT"
|
||||
echo "local_retention_days=$local_retention" >>"$GITHUB_OUTPUT"
|
||||
ssh_root="${RUNNER_TEMP}/production-daily-ssh"
|
||||
key_path="$ssh_root/id_ed25519"
|
||||
known_hosts_path="$ssh_root/known_hosts"
|
||||
artifact_directory="artifacts/production-daily"
|
||||
install -m 700 -d "$ssh_root" "$artifact_directory"
|
||||
test -n "$SSH_PRIVATE_KEY_BASE64"
|
||||
printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode >"$key_path"
|
||||
printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' >"$known_hosts_path"
|
||||
chmod 600 "$key_path" "$known_hosts_path"
|
||||
ssh-keygen -y -f "$key_path" >/dev/null
|
||||
ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
|
||||
scp_options=(-i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
|
||||
remote="$DEPLOY_USER@$DEPLOY_HOST"
|
||||
incoming="$(ssh "${ssh_options[@]}" "$remote" 'mktemp -d /tmp/jyotisha-production-daily.XXXXXXXXXX')"
|
||||
[[ "$incoming" == /tmp/jyotisha-production-daily.* ]]
|
||||
cleanup() {
|
||||
ssh "${ssh_options[@]}" "$remote" "rm -rf -- '$incoming'" >/dev/null 2>&1 || true
|
||||
rm -rf -- "$ssh_root"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
git init .
|
||||
git remote remove origin 2>/dev/null || true
|
||||
git remote add origin https://git.copse.top/root/Jyotisha.git
|
||||
git fetch --no-tags origin "$GITEA_SHA"
|
||||
git checkout --detach --force "$GITEA_SHA"
|
||||
[[ "$(git rev-parse HEAD)" == "$GITEA_SHA" ]]
|
||||
git show "$GITEA_SHA:deploy/run-production-recovery.sh" >"$ssh_root/run-production-recovery.sh"
|
||||
git show "$GITEA_SHA:deploy/prune-production-daily-backups.sh" >"$ssh_root/prune-production-daily-backups.sh"
|
||||
chmod 700 "$ssh_root/run-production-recovery.sh" "$ssh_root/prune-production-daily-backups.sh"
|
||||
scp "${scp_options[@]}" \
|
||||
"$ssh_root/run-production-recovery.sh" \
|
||||
"$ssh_root/prune-production-daily-backups.sh" \
|
||||
"$remote:$incoming/"
|
||||
ssh "${ssh_options[@]}" "$remote" \
|
||||
"bash '$incoming/prune-production-daily-backups.sh' '$DEPLOY_PATH/backups' '$local_retention'"
|
||||
ssh "${ssh_options[@]}" "$remote" \
|
||||
"DEPLOY_PATH='$DEPLOY_PATH' RECOVERY_RUN_ID='$GITHUB_RUN_ID' RECOVERY_BASENAME_PREFIX='production-daily' bash '$incoming/run-production-recovery.sh'" \
|
||||
>"$ssh_root/recovery-output.env"
|
||||
[[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d ' ')" == 7 ]]
|
||||
recovery_reference="$(awk -F= '$1 == "RECOVERY_REFERENCE" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
recovery_created_at="$(awk -F= '$1 == "RECOVERY_CREATED_AT" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
recovery_verified_at="$(awk -F= '$1 == "RECOVERY_VERIFIED_AT" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
recovery_sha256="$(awk -F= '$1 == "RECOVERY_SHA256" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
restore_verified="$(awk -F= '$1 == "RESTORE_VERIFIED" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
backup_basename="$(awk -F= '$1 == "BACKUP_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
verify_basename="$(awk -F= '$1 == "VERIFY_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
[[ "$backup_basename" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]]
|
||||
[[ "$verify_basename" == "${backup_basename%.dump.enc}-restore-verify.json" ]]
|
||||
[[ "$recovery_reference" == "gitea-actions-run-${GITHUB_RUN_ID}/${backup_basename}" ]]
|
||||
[[ "$recovery_created_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
[[ "$recovery_verified_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
[[ "$recovery_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
[[ "$restore_verified" == "true" ]]
|
||||
scp "${scp_options[@]}" \
|
||||
"$remote:$DEPLOY_PATH/backups/$backup_basename" \
|
||||
"$remote:$DEPLOY_PATH/backups/$verify_basename" \
|
||||
"$artifact_directory/"
|
||||
test -s "$artifact_directory/$backup_basename"
|
||||
test -s "$artifact_directory/$verify_basename"
|
||||
printf '%s %s\n' "$recovery_sha256" "$artifact_directory/$backup_basename" | sha256sum --check --status
|
||||
jq -e \
|
||||
--arg backup "$backup_basename" \
|
||||
--arg sha "$recovery_sha256" \
|
||||
--arg created "$recovery_created_at" \
|
||||
--arg verified "$recovery_verified_at" '
|
||||
.mode == "restore_verify" and .ok == true and
|
||||
.backup == $backup and .sha256 == $sha and
|
||||
.created_at == $created and .verified_at == $verified and
|
||||
.restore_database_removed == true and
|
||||
([.checks.identity_users, .checks.profiles, .checks.credit_transactions, .checks.public_tables] |
|
||||
all(type == "number" and . >= 0 and floor == .))
|
||||
' "$artifact_directory/$verify_basename" >/dev/null
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
|
||||
"$recovery_reference" "$recovery_created_at" "$recovery_verified_at" "$recovery_sha256" \
|
||||
"$backup_basename" "$verify_basename" >"$artifact_directory/attestation.env"
|
||||
chmod 600 "$artifact_directory"/*
|
||||
{
|
||||
echo "recovery_reference=$recovery_reference"
|
||||
echo "recovery_created_at=$recovery_created_at"
|
||||
echo "recovery_verified_at=$recovery_verified_at"
|
||||
echo "recovery_sha256=$recovery_sha256"
|
||||
} >>"$GITHUB_OUTPUT"
|
||||
echo "Daily backup restore-verified: reference=$recovery_reference created_at=$recovery_created_at verified_at=$recovery_verified_at"
|
||||
|
||||
- name: Upload encrypted off-site daily backup
|
||||
env:
|
||||
ARTIFACT_RETENTION_DAYS: ${{ steps.backup.outputs.artifact_retention_days }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "${ARTIFACT_RETENTION_DAYS:-}" =~ ^[1-9][0-9]{0,3}$ ]]
|
||||
test -n "${ACTIONS_RUNTIME_TOKEN:-}"
|
||||
test -n "${ACTIONS_RESULTS_URL:-}"
|
||||
test -n "${GITHUB_RUN_ID:-}"
|
||||
test -n "${GITHUB_REPOSITORY:-}"
|
||||
workdir="$(pwd -P)"
|
||||
docker run --rm \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
--volume "$workdir:$workdir" \
|
||||
--workdir "$workdir" \
|
||||
--env HOME=/tmp \
|
||||
--env "INPUT_NAME=production-daily-$GITHUB_RUN_ID" \
|
||||
--env INPUT_PATH=artifacts/production-daily/ \
|
||||
--env INPUT_OVERWRITE=false \
|
||||
--env ACTIONS_RUNTIME_TOKEN \
|
||||
--env ACTIONS_RESULTS_URL \
|
||||
--env GITHUB_RUN_ID \
|
||||
--env GITHUB_REPOSITORY \
|
||||
--env "GITHUB_SHA=$GITEA_SHA" \
|
||||
--env "GITHUB_WORKSPACE=$workdir" \
|
||||
--env ARTIFACT_RETENTION_DAYS \
|
||||
node:22-bookworm-slim \
|
||||
node -e 'process.env["INPUT_IF-NO-FILES-FOUND"]="error"; process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS; process.env["INPUT_COMPRESSION-LEVEL"]="0"; require("./.gitea/actions/upload-artifact/dist/index.js")'
|
||||
@@ -1,5 +1,13 @@
|
||||
# BLOCKED
|
||||
|
||||
## TASK-beta-safety-net-20261014：主机安装与一次恢复演练还没做（2026-10-15）
|
||||
|
||||
- **每日备份**：工作流要等这支分支进入 `main` 之后才会在北京时间 04:00 跑。本机没有生产主机，没有手动触发生产工作流。
|
||||
- **探活**：应装在 Gitea runner 上，不要装在生产主机。磁盘检查和错误汇总装在生产主机。飞书 Webhook 由产品写进模式 `0600` 的环境文件，不进仓库。没配之前只写本机日志。
|
||||
- **日志轮转**：编排已写上单文件 20 MB、最多 10 个文件。要等生产容器按新编排重建后才生效。本机没有对生产主机执行。
|
||||
- **恢复演练**：内测开放前按 `docs/operations/production-restore-drill-20261014.md` 做一次一次性库还原，结果填 `docs/testing/production-restore-drill-20261014.md`。模板现在是空的。
|
||||
- 解除：上面四件事做完，并把演练模板里的「是否通过」填上。不要把 Webhook、密钥或用户内容写进仓库。
|
||||
|
||||
## TASK-answer-effort-ab-20261013：答题步推理强度的真实 A/B（2026-10-15）
|
||||
|
||||
- **A/B 未跑：缺模型 key。** 本机没有 `DS_KEY` 或 `BACKTEST_MODEL_KEY`。供应商是否接受 `low` / `medium` / `high` / `max`、推理 token 和耗时有没有下降，都没测。不得把请求体拦截写成「A/B 通过」。
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
# 印度占星 Skill 更新日志
|
||||
|
||||
## 2026-10-15 — 每日备份在磁盘变满后仍会清理，过期会有人知道
|
||||
|
||||
- 过期的每日备份改在新备份之前清理。最近一次核对超过 36 小时，或还没有核对记录，磁盘检查会告警一次。
|
||||
- 飞书没有真正收下的告警不再被当成已发送。同一类错误 30 分钟内不重复发。
|
||||
- 页面没有变化。Skill 版本不变（BUG-1362、BUG-1363、BUG-1364）。
|
||||
|
||||
## 2026-10-15 — 内测容量：问得太密会停一下，人多时先排一小会儿
|
||||
|
||||
- 同一个人短时间问得太密、校正答得太密，或一小时里报告生成太多，会先停一下，这次不扣点。
|
||||
- 同时算的人多过机器能接的两路时，先排一小会儿。排上了继续；排不上会说人有点多，并把这一轮预留的点数退回。
|
||||
- Skill 版本不变（BUG-1355、BUG-1356）。
|
||||
|
||||
## 2026-10-15 — 内测前补上每日备份、宕机告警和日志上限（待合入 staging）
|
||||
|
||||
- 生产可以每天自动做一次加密备份。站点连续三次打不开、磁盘过满，或短时间错误变多时，可以通知到飞书群。容器日志有大小上限。可以按一轮请求的编号在服务器日志里查找。这些要等产品在主机上装好之后才生效。页面没有变化。Skill 版本不变（BUG-1358、BUG-1359、BUG-1360）。
|
||||
|
||||
## 2026-10-15 — 合盘更快（待合入 staging）
|
||||
|
||||
- 合盘排两张盘时关掉用不上的外部概览，两张盘一起排。个人报告在没装参照引擎时不再为注定失败的回放反复起进程。轻重任务要不要分开限流，这一单只给了模拟表，还没改规则。
|
||||
|
||||
@@ -394,6 +394,49 @@ smoke in `docs/operations/personal-report-staging.md` (section “Post-deploy
|
||||
report list / detail / create smoke”): `GET /api/reports` 200, owned
|
||||
`GET /api/reports/<id>` 200, and one new `POST /api/reports` through ready.
|
||||
|
||||
## 运维与告警
|
||||
|
||||
这些步骤要等本分支进入 `main` 之后，由产品在主机上安装。仓库里没有 Webhook 地址。告警正文只允许服务名、路由、错误码、次数、requestId 和时间。
|
||||
|
||||
### 备份
|
||||
|
||||
- 工作流：`.gitea/workflows/backup-production-daily.yml`。北京时间每天 04:00，也可手动触发。只从 `main` 运行，调用 `deploy/run-production-recovery.sh`，不另写一套导出。
|
||||
- 每日文件前缀是 `production-daily-`。手动恢复点仍是 `production-pre-migration-`。清理脚本 `deploy/prune-production-daily-backups.sh` 只删超过 7 天的每日文件和对应的校验文件，不会删 `production-pre-migration-` 手动恢复点。清理排在新备份之前，所以磁盘到 70% 时过期的每日文件仍会被清掉。
|
||||
- 每日备份只做存档和还原核对，不是发布用的恢复点。它不改 `/opt/jyotisha-production/.state/recovery-baseline.env`。核对结果另写在同目录的 `daily-backup-latest.env`，只有文件名、sha256、核对时间和格式版本，没有数据和密钥。发布、迁移、回退都不读这个文件。上线前仍要手动点 Create production recovery。
|
||||
- Gitea 产物默认保留 14 天。手动恢复点工作流仍是 30 天，没有改。
|
||||
- 还原步骤：`docs/operations/production-restore-drill-20261014.md`。
|
||||
|
||||
### 告警装在哪
|
||||
|
||||
- 探活不要装在生产主机上。装在 Gitea runner 上，每分钟跑 `deploy/monitor/healthcheck-alert.sh --mode health`。它访问公网 `https://jyotisha.chat/api/health`，超时 10 秒。连续 3 次失败才告警，恢复只发一次。
|
||||
- 发布窗口：在 runner 上执行 `touch /var/lib/jyotisha-monitor/release-window`。45 分钟内，版本号变化不告警。
|
||||
- 磁盘检查装在生产主机上，每分钟跑同一个脚本的 `--mode disk`。使用率达到 80% 告警一次，降下来再发一次已恢复。同一轮还会读 `/opt/jyotisha-production/.state/daily-backup-latest.env` 的 `VERIFIED_AT`。这份记录缺失，或比当前时间早超过 36 小时，会再告警一次。重新核对成功后发一次已恢复。探活模式不读这份文件。
|
||||
- 错误汇总装在生产主机上。每 5 分钟 `deploy/monitor/error-digest.sh --window`。每天北京时间 09:00（UTC 01:00）跑 `--daily`，汇总前一天。同类错误 5 分钟内至少 10 次，或 5xx 至少 20 次，才发窗口告警。同一服务、同一路由、同一错误码在 30 分钟内不重复发。消息里最多 5 个 requestId。含邮箱、第二个日期或非 ASCII 文本的日志行整行丢掉。飞书没有返回成功时，这条不会被记成已发送。
|
||||
- Webhook 写在模式 `0600` 的环境文件里，建议路径 `/opt/jyotisha-production/.monitor.env`（runner 上用 `JYOTISHA_MONITOR_ENV` 指向自己的文件）。只放这两行，不要提交：
|
||||
|
||||
```text
|
||||
FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/<token>
|
||||
FEISHU_SIGN_SECRET=<只有开了签名校验才填>
|
||||
```
|
||||
|
||||
没配 Webhook 时只写本机 `alerts.log`，不外发。飞书消息是 `msg_type: text`。配了签名就带 `timestamp` 和 `sign`。
|
||||
|
||||
### 日志轮转
|
||||
|
||||
`deploy/docker-compose.server.yml` 和 `deploy/docker-compose.postgres.yml` 的每个服务都是 json-file，单文件 20 MB，最多 10 个文件。镜像、环境变量、端口和健康检查没有改。要等生产容器按新编排重建后才生效。
|
||||
|
||||
看守护进程默认值：`docker info --format '{{.LoggingDriver}}'`，以及主机上的 `/etc/docker/daemon.json`。编排里的 `logging` 会盖过守护进程对这几个服务的默认值。
|
||||
|
||||
### 按 requestId 查日志
|
||||
|
||||
在生产主机上，只读：
|
||||
|
||||
```bash
|
||||
deploy/find-request-log.sh <requestId> --since 24h
|
||||
```
|
||||
|
||||
`<requestId>` 必须是 UUID。其它写法会拒绝，避免被当成命令。`--since` 只接受类似 `24h`、`30m` 的形式。容器日志行前面的服务名前缀会先去掉，再按时间排序。来自 web、api、caddy。
|
||||
|
||||
## Common operations
|
||||
|
||||
```bash
|
||||
|
||||
@@ -2,6 +2,11 @@ services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
shm_size: 128mb
|
||||
env_file:
|
||||
- ${DATABASE_ENV_FILE:-../.env.staging.database}
|
||||
@@ -30,6 +35,11 @@ services:
|
||||
image: ${WEB_IMAGE:-jyotisha-web:local}
|
||||
profiles: ["migration"]
|
||||
restart: "no"
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${DATABASE_ENV_FILE:-../.env.staging.database}
|
||||
working_dir: /app/frontend
|
||||
@@ -43,6 +53,11 @@ services:
|
||||
image: ${WEB_IMAGE:-jyotisha-web:local}
|
||||
profiles: ["migration-check"]
|
||||
restart: "no"
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${DATABASE_ENV_FILE:-../.env.staging.database}
|
||||
working_dir: /app/frontend
|
||||
|
||||
@@ -5,6 +5,11 @@ services:
|
||||
context: ..
|
||||
dockerfile: deploy/railway-api.Dockerfile
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${APP_ENV_FILE:-../.env.production}
|
||||
environment:
|
||||
@@ -35,6 +40,11 @@ services:
|
||||
NEXT_PUBLIC_SUPABASE_URL: ${NEXT_PUBLIC_SUPABASE_URL:-}
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${NEXT_PUBLIC_SUPABASE_ANON_KEY:-}
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${APP_ENV_FILE:-../.env.production}
|
||||
environment:
|
||||
@@ -57,6 +67,11 @@ services:
|
||||
caddy:
|
||||
image: caddy:2-alpine
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
environment:
|
||||
SITE_ADDRESS: ${SITE_ADDRESS:-https://jyotisha.chat}
|
||||
ports:
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
# Read-only search for one request id in web, api, and caddy logs.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
|
||||
if [ $# -lt 1 ] || [ $# -gt 3 ]; then
|
||||
echo "usage: find-request-log.sh <requestId> [--since 24h]" >&2
|
||||
exit 2
|
||||
fi
|
||||
request_id="$1"
|
||||
shift
|
||||
since="24h"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--since)
|
||||
since="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ "$request_id" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] || {
|
||||
echo "requestId must be a UUID" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ "$since" =~ ^[1-9][0-9]{0,3}[smhd]$ ]] || {
|
||||
echo "since must look like 24h" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
log_dir="${JYOTISHA_LOG_DIR:-}"
|
||||
if [ -n "$log_dir" ]; then
|
||||
[ -d "$log_dir" ] && [ ! -L "$log_dir" ] || {
|
||||
echo "log directory is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
tmp="$(mktemp)"
|
||||
for service in web api caddy; do
|
||||
file="$log_dir/$service.log"
|
||||
[ -f "$file" ] && [ ! -L "$file" ] || continue
|
||||
grep -F -- "$request_id" "$file" >>"$tmp" || true
|
||||
done
|
||||
if [ -s "$tmp" ]; then
|
||||
grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' "$tmp" | sort
|
||||
fi
|
||||
rm -f -- "$tmp"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
|
||||
export APP_ENV_FILE='../.env.production'
|
||||
export DATABASE_ENV_FILE='../.env.production.database'
|
||||
export CADDYFILE_PATH='./Caddyfile.production.selfhosted'
|
||||
export SITE_ADDRESS='https://jyotisha.chat'
|
||||
if [ -n "${JYOTISHA_COMPOSE_LOG_FIXTURE:-}" ]; then
|
||||
[ -f "$JYOTISHA_COMPOSE_LOG_FIXTURE" ] && [ ! -L "$JYOTISHA_COMPOSE_LOG_FIXTURE" ] || {
|
||||
echo "compose log fixture is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
sed -E 's/^[^|]*\| //' "$JYOTISHA_COMPOSE_LOG_FIXTURE" |
|
||||
grep -F -- "$request_id" |
|
||||
grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' |
|
||||
sort
|
||||
exit 0
|
||||
fi
|
||||
sudo -n docker compose -p jyotisha-production \
|
||||
--env-file "$compose_dir/.env.production" \
|
||||
-f "$compose_dir/deploy/docker-compose.server.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.production.yml" \
|
||||
logs --since "$since" --no-color --timestamps web api caddy |
|
||||
sed -E 's/^[^|]*\| //' |
|
||||
grep -F -- "$request_id" |
|
||||
grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' |
|
||||
sort
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Classify fictional or container log lines into alert rows.
|
||||
|
||||
Outbound fields are service, route, errorCode, count, and request ids.
|
||||
Lines with non-ASCII text, an email address, or a second calendar date are
|
||||
dropped so conversation text and birth dates cannot leave the host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
|
||||
UUID = re.compile(
|
||||
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
|
||||
)
|
||||
ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$")
|
||||
ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$")
|
||||
STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b")
|
||||
FIXTURE = re.compile(
|
||||
r"^(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?P<svc>web|api) (?P<rest>.*)$"
|
||||
)
|
||||
DOCKER = re.compile(
|
||||
r"^(?P<svc>web|api)-\d+\s+\|\s+"
|
||||
r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P<rest>.*)$"
|
||||
)
|
||||
CST = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def unsafe(line: str) -> bool:
|
||||
if any(ord(char) > 126 for char in line):
|
||||
return True
|
||||
lowered = line.lower()
|
||||
if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered:
|
||||
return True
|
||||
return len(DATE.findall(line)) >= 2
|
||||
|
||||
|
||||
def parse_line(line: str) -> tuple[str, str, str] | None:
|
||||
match = FIXTURE.match(line) or DOCKER.match(line)
|
||||
if not match:
|
||||
return None
|
||||
return match.group("ts"), match.group("svc"), match.group("rest")
|
||||
|
||||
|
||||
def timestamp_cst_date(timestamp: str):
|
||||
stamp = timestamp
|
||||
if "." in stamp:
|
||||
stamp = stamp.split(".", 1)[0] + "Z"
|
||||
moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
return moment.astimezone(CST).date()
|
||||
|
||||
|
||||
def extract_fields(rest: str) -> tuple[str, str, str, int | None]:
|
||||
route = "-"
|
||||
error_code = ""
|
||||
request_id = ""
|
||||
status: int | None = None
|
||||
marker = "[agent-observability]"
|
||||
if marker in rest:
|
||||
raw = rest.split(marker, 1)[1].strip()
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, dict):
|
||||
code = payload.get("errorCode")
|
||||
if isinstance(code, str) and ERROR_CODE.match(code):
|
||||
error_code = code
|
||||
candidate = payload.get("requestId")
|
||||
if isinstance(candidate, str) and UUID.match(candidate):
|
||||
request_id = candidate.lower()
|
||||
route_value = payload.get("route")
|
||||
if isinstance(route_value, str) and ROUTE.match(route_value):
|
||||
route = route_value
|
||||
status_match = STATUS.search(rest)
|
||||
if status_match:
|
||||
status = int(status_match.group(1))
|
||||
return route, error_code, request_id, status
|
||||
|
||||
|
||||
def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]:
|
||||
now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
yesterday = now.astimezone(CST).date() - timedelta(days=1)
|
||||
counts: OrderedDict[tuple[str, str, str], int] = OrderedDict()
|
||||
request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict()
|
||||
fivexx_counts: OrderedDict[str, int] = OrderedDict()
|
||||
fivexx_ids: OrderedDict[str, list[str]] = OrderedDict()
|
||||
|
||||
def add(key: tuple[str, str, str], request_id: str) -> None:
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
bucket = request_ids.setdefault(key, [])
|
||||
if request_id and request_id not in bucket and len(bucket) < 5:
|
||||
bucket.append(request_id)
|
||||
|
||||
for raw in lines:
|
||||
line = raw.rstrip("\r\n")
|
||||
if not line or unsafe(line):
|
||||
continue
|
||||
parsed = parse_line(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
timestamp, service, rest = parsed
|
||||
if mode == "daily" and timestamp_cst_date(timestamp) != yesterday:
|
||||
continue
|
||||
route, error_code, request_id, status = extract_fields(rest)
|
||||
if error_code:
|
||||
add((service, route, error_code), request_id)
|
||||
elif status is not None:
|
||||
add((service, route, f"http_{status}"), request_id)
|
||||
if status is not None and 500 <= status <= 599:
|
||||
fivexx_counts[service] = fivexx_counts.get(service, 0) + 1
|
||||
bucket = fivexx_ids.setdefault(service, [])
|
||||
if request_id and request_id not in bucket and len(bucket) < 5:
|
||||
bucket.append(request_id)
|
||||
|
||||
rows: list[str] = []
|
||||
|
||||
def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None:
|
||||
shown = ",".join(ids) if ids else "-"
|
||||
rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}")
|
||||
|
||||
for key, count in counts.items():
|
||||
if mode == "window" and count < error_threshold:
|
||||
continue
|
||||
if count <= 0:
|
||||
continue
|
||||
emit(*key, count, request_ids.get(key, []))
|
||||
for service, count in fivexx_counts.items():
|
||||
if mode == "window" and count < fivexx_threshold:
|
||||
continue
|
||||
if count <= 0:
|
||||
continue
|
||||
emit(service, "-", "http_5xx", count, fivexx_ids.get(service, []))
|
||||
return rows
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 5:
|
||||
print("usage: classify_error_log.py <fixture> <window|daily> <now> <error-threshold> <5xx-threshold>", file=sys.stderr)
|
||||
return 2
|
||||
fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv
|
||||
if mode not in {"window", "daily"}:
|
||||
print("unknown mode", file=sys.stderr)
|
||||
return 2
|
||||
error_threshold = int(error_threshold_text)
|
||||
fivexx_threshold = int(fivexx_threshold_text)
|
||||
with open(fixture, encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold):
|
||||
print(row)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# Count recent web/api errors and alert without sending log text.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=notify-feishu.sh
|
||||
source "$script_dir/notify-feishu.sh"
|
||||
|
||||
mode="window"
|
||||
case "${1:-}" in
|
||||
""|--window) mode="window" ;;
|
||||
--daily) mode="daily" ;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
||||
load_monitor_env "$env_file"
|
||||
|
||||
error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}"
|
||||
fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}"
|
||||
[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
||||
[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
||||
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2
|
||||
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
|
||||
fixture="${JYOTISHA_LOG_FIXTURE:-}"
|
||||
cleanup() {
|
||||
if [ -n "${generated_fixture:-}" ]; then
|
||||
rm -f -- "$generated_fixture"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
generated_fixture=""
|
||||
if [ -z "$fixture" ]; then
|
||||
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
|
||||
since="5m"
|
||||
[ "$mode" = "daily" ] && since="48h"
|
||||
generated_fixture="$(mktemp)"
|
||||
export APP_ENV_FILE='../.env.production'
|
||||
export DATABASE_ENV_FILE='../.env.production.database'
|
||||
export CADDYFILE_PATH='./Caddyfile.production.selfhosted'
|
||||
export SITE_ADDRESS='https://jyotisha.chat'
|
||||
sudo -n docker compose -p jyotisha-production \
|
||||
--env-file "$compose_dir/.env.production" \
|
||||
-f "$compose_dir/deploy/docker-compose.server.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.production.yml" \
|
||||
logs --since "$since" --no-color --timestamps web api >"$generated_fixture"
|
||||
fixture="$generated_fixture"
|
||||
fi
|
||||
[ -f "$fixture" ] && [ ! -L "$fixture" ] || {
|
||||
echo "log fixture is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then
|
||||
echo "could not classify logs" >&2
|
||||
exit 1
|
||||
fi
|
||||
stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
text=""
|
||||
if [ -n "$rows" ]; then
|
||||
while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do
|
||||
[ -n "${service:-}" ] || continue
|
||||
line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}"
|
||||
text="${text}${line}"$'\n'
|
||||
done <<<"$rows" || true
|
||||
elif [ "$mode" = "daily" ]; then
|
||||
text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n'
|
||||
fi
|
||||
|
||||
if [ -n "$text" ]; then
|
||||
now_epoch="$(date -u -d "$now_text" +%s)"
|
||||
[[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2
|
||||
dedup_file="$state_dir/error-digest-sent.tsv"
|
||||
declare -A last_sent=()
|
||||
declare -a send_keys=()
|
||||
filtered=""
|
||||
if [ -f "$dedup_file" ] && [ ! -L "$dedup_file" ]; then
|
||||
while IFS=$'\t' read -r sent_epoch sent_service sent_route sent_code || [ -n "${sent_epoch:-}" ]; do
|
||||
[[ "${sent_epoch:-}" =~ ^[0-9]+$ ]] || continue
|
||||
[[ "${sent_service:-}" =~ ^[a-z0-9_-]+$ ]] || continue
|
||||
[ -n "${sent_route:-}" ] || continue
|
||||
[ -n "${sent_code:-}" ] || continue
|
||||
last_sent["${sent_service}"$'\t'"${sent_route}"$'\t'"${sent_code}"]="$sent_epoch"
|
||||
done <"$dedup_file"
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
[ -n "$line" ] || continue
|
||||
if [[ "$line" =~ ^service=([a-z0-9_-]+)[[:space:]]route=([^[:space:]]+)[[:space:]]errorCode=([a-z][a-z0-9._-]*)[[:space:]] ]]; then
|
||||
key="${BASH_REMATCH[1]}"$'\t'"${BASH_REMATCH[2]}"$'\t'"${BASH_REMATCH[3]}"
|
||||
prev="${last_sent[$key]:-}"
|
||||
if [ -n "$prev" ] && (( now_epoch - prev < 1800 )); then
|
||||
continue
|
||||
fi
|
||||
send_keys+=("$key")
|
||||
fi
|
||||
filtered="${filtered}${line}"$'\n'
|
||||
done <<<"${text%$'\n'}"
|
||||
if [ -n "$filtered" ]; then
|
||||
post_monitor_alert "${filtered%$'\n'}"
|
||||
if [ "${#send_keys[@]}" -gt 0 ]; then
|
||||
for key in "${send_keys[@]}"; do
|
||||
last_sent["$key"]="$now_epoch"
|
||||
done
|
||||
fi
|
||||
dedup_tmp="$dedup_file.tmp.$$"
|
||||
: >"$dedup_tmp"
|
||||
for key in "${!last_sent[@]}"; do
|
||||
printf '%s\t%s\n' "${last_sent[$key]}" "$key" >>"$dedup_tmp"
|
||||
done
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$dedup_tmp"
|
||||
fi
|
||||
mv "$dedup_tmp" "$dedup_file"
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,282 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe the public health URL, or the host disk. Install the health mode on
|
||||
# the Gitea runner, not on the production host. Install disk mode on the host.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=notify-feishu.sh
|
||||
source "$script_dir/notify-feishu.sh"
|
||||
|
||||
mode="health"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
mode="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--mode=*)
|
||||
mode="${1#--mode=}"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
case "$mode" in
|
||||
health|disk|all) ;;
|
||||
*)
|
||||
echo "unknown mode" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
||||
load_monitor_env "$env_file"
|
||||
|
||||
health_url="${JYOTISHA_HEALTH_URL:-https://jyotisha.chat/api/health}"
|
||||
timeout="${JYOTISHA_HEALTH_TIMEOUT:-10}"
|
||||
[[ "$timeout" =~ ^[1-9][0-9]?$ ]] && (( timeout <= 30 )) || {
|
||||
echo "health timeout must be 1 to 30 seconds" >&2
|
||||
exit 2
|
||||
}
|
||||
if [ "$mode" != "disk" ]; then
|
||||
[[ "$health_url" == "https://jyotisha.chat/api/health" \
|
||||
|| "$health_url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/health$ \
|
||||
|| "$health_url" =~ ^http://localhost:[0-9]{1,5}/health$ ]] || {
|
||||
echo "unexpected health url" >&2
|
||||
exit 2
|
||||
}
|
||||
fi
|
||||
|
||||
state_file="$state_dir/monitor.state"
|
||||
consecutive_failures=0
|
||||
alert_open=0
|
||||
last_commit=""
|
||||
commit_alerted=""
|
||||
disk_alert_open=0
|
||||
backup_stale_open=0
|
||||
|
||||
read_state() {
|
||||
local line value
|
||||
[ -f "$state_file" ] && [ ! -L "$state_file" ] || return 0
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
consecutive_failures=[0-9]*)
|
||||
consecutive_failures="${line#consecutive_failures=}"
|
||||
;;
|
||||
alert_open=[01])
|
||||
alert_open="${line#alert_open=}"
|
||||
;;
|
||||
last_commit=)
|
||||
last_commit=""
|
||||
;;
|
||||
last_commit=*)
|
||||
value="${line#last_commit=}"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && last_commit="$value"
|
||||
;;
|
||||
commit_alerted=)
|
||||
commit_alerted=""
|
||||
;;
|
||||
commit_alerted=*)
|
||||
value="${line#commit_alerted=}"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && commit_alerted="$value"
|
||||
;;
|
||||
disk_alert_open=[01])
|
||||
disk_alert_open="${line#disk_alert_open=}"
|
||||
;;
|
||||
backup_stale_open=[01])
|
||||
backup_stale_open="${line#backup_stale_open=}"
|
||||
;;
|
||||
esac
|
||||
done <"$state_file"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
local tmp
|
||||
tmp="$state_file.tmp.$$"
|
||||
printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\nbackup_stale_open=%s\n' \
|
||||
"$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" "$backup_stale_open" >"$tmp"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$tmp"
|
||||
fi
|
||||
mv "$tmp" "$state_file"
|
||||
}
|
||||
|
||||
stamp() {
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ
|
||||
}
|
||||
|
||||
send_alert() {
|
||||
local service="$1"
|
||||
local route="$2"
|
||||
local error_code="$3"
|
||||
local count="$4"
|
||||
local request_id="${5:--}"
|
||||
post_monitor_alert "service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_id} time=$(stamp)"
|
||||
}
|
||||
|
||||
release_window_open() {
|
||||
local marker="$state_dir/release-window"
|
||||
local mtime now age
|
||||
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
|
||||
mtime="$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker")"
|
||||
now="$(date +%s)"
|
||||
age=$((now - mtime))
|
||||
[ "$age" -ge 0 ] && [ "$age" -le 2700 ]
|
||||
}
|
||||
|
||||
check_health() {
|
||||
local body_file http_code curl_status status_line commit_line status commit healthy py
|
||||
body_file="$(mktemp)"
|
||||
http_code="000"
|
||||
curl_status=0
|
||||
http_code="$(curl --silent --show-error --connect-timeout "$timeout" --max-time "$timeout" \
|
||||
-o "$body_file" -w '%{http_code}' "$health_url")" || curl_status=$?
|
||||
status=""
|
||||
commit=""
|
||||
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ]; then
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
status_line="$("$py" - "$body_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
try:
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
except Exception:
|
||||
print("")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
if not isinstance(payload, dict):
|
||||
print("")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
status = payload.get("status")
|
||||
deployment = payload.get("deployment")
|
||||
commit = ""
|
||||
if isinstance(deployment, dict):
|
||||
raw = deployment.get("gitCommit")
|
||||
if isinstance(raw, str):
|
||||
commit = raw
|
||||
print(status if isinstance(status, str) else "")
|
||||
print(commit)
|
||||
PY
|
||||
)"
|
||||
status="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==1 {print}')"
|
||||
commit="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==2 {print}')"
|
||||
fi
|
||||
rm -f -- "$body_file"
|
||||
|
||||
healthy=0
|
||||
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ] && [ "$status" = "ok" ]; then
|
||||
healthy=1
|
||||
fi
|
||||
|
||||
if [ "$healthy" -eq 1 ]; then
|
||||
if [ "$alert_open" -eq 1 ]; then
|
||||
send_alert site /api/health recovered 0 -
|
||||
alert_open=0
|
||||
fi
|
||||
consecutive_failures=0
|
||||
if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
if [ -n "$last_commit" ] && [ "$commit" != "$last_commit" ] && [ "$commit_alerted" != "$commit" ]; then
|
||||
if ! release_window_open; then
|
||||
send_alert site /api/health commit_changed 1 -
|
||||
commit_alerted="$commit"
|
||||
fi
|
||||
fi
|
||||
last_commit="$commit"
|
||||
fi
|
||||
else
|
||||
consecutive_failures=$((consecutive_failures + 1))
|
||||
if [ "$consecutive_failures" -ge 3 ] && [ "$alert_open" -eq 0 ]; then
|
||||
send_alert site /api/health health_down "$consecutive_failures" -
|
||||
alert_open=1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
disk_percent() {
|
||||
df -Pk "${JYOTISHA_DISK_PATH:-/}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}'
|
||||
}
|
||||
|
||||
check_disk() {
|
||||
local percent
|
||||
percent="$(disk_percent)"
|
||||
[[ "$percent" =~ ^[0-9]+$ ]] && [ "$percent" -le 100 ] || {
|
||||
echo "disk usage is unreadable" >&2
|
||||
exit 1
|
||||
}
|
||||
if [ "$percent" -ge 80 ]; then
|
||||
if [ "$disk_alert_open" -eq 0 ]; then
|
||||
send_alert host - disk_high "$percent" -
|
||||
disk_alert_open=1
|
||||
fi
|
||||
elif [ "$disk_alert_open" -eq 1 ]; then
|
||||
send_alert host - recovered 0 -
|
||||
disk_alert_open=0
|
||||
fi
|
||||
}
|
||||
|
||||
check_daily_backup() {
|
||||
local record now_text now_epoch verified verified_epoch age stale hours
|
||||
record="${JYOTISHA_DAILY_BACKUP_RECORD:-/opt/jyotisha-production/.state/daily-backup-latest.env}"
|
||||
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || {
|
||||
echo "clock is unsafe" >&2
|
||||
exit 2
|
||||
}
|
||||
now_epoch="$(date -u -d "$now_text" +%s)"
|
||||
[[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2
|
||||
stale=1
|
||||
hours=36
|
||||
if [ -f "$record" ] && [ ! -L "$record" ]; then
|
||||
verified="$(awk -F= '$1 == "VERIFIED_AT" { print $2 }' "$record" | tr -d '\r')"
|
||||
if [[ "$verified" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
|
||||
verified_epoch="$(date -u -d "$verified" +%s)"
|
||||
age=$((now_epoch - verified_epoch))
|
||||
if [ "$age" -le 129600 ]; then
|
||||
stale=0
|
||||
else
|
||||
hours=$((age / 3600))
|
||||
if [ "$hours" -lt 36 ]; then
|
||||
hours=36
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if [ "$stale" -eq 1 ]; then
|
||||
if [ "$backup_stale_open" -eq 0 ]; then
|
||||
send_alert host - backup_stale "$hours" -
|
||||
backup_stale_open=1
|
||||
fi
|
||||
elif [ "$backup_stale_open" -eq 1 ]; then
|
||||
send_alert host - recovered 0 -
|
||||
backup_stale_open=0
|
||||
fi
|
||||
}
|
||||
|
||||
read_state
|
||||
case "$mode" in
|
||||
health) check_health ;;
|
||||
disk)
|
||||
check_disk
|
||||
check_daily_backup
|
||||
;;
|
||||
all)
|
||||
check_health
|
||||
check_disk
|
||||
check_daily_backup
|
||||
;;
|
||||
esac
|
||||
write_state
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared alert sender. Source this file; do not execute it.
|
||||
# Outbound text is limited to service, route, errorCode, count, requestId, time.
|
||||
|
||||
ensure_private_dir() {
|
||||
local dir="$1"
|
||||
mkdir -p -- "$dir"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 700 "$dir"
|
||||
fi
|
||||
}
|
||||
|
||||
load_monitor_env() {
|
||||
local file="$1"
|
||||
local line key value mode
|
||||
[ -n "$file" ] || return 0
|
||||
[ -f "$file" ] && [ ! -L "$file" ] || return 0
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
mode="$(stat -c %a "$file")"
|
||||
[ "$mode" = "600" ] || {
|
||||
echo "monitor env must be mode 0600" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
""|\#*) continue ;;
|
||||
FEISHU_WEBHOOK_URL=*|FEISHU_SIGN_SECRET=*)
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
case "$value" in
|
||||
*[$'\n\r`$']*)
|
||||
echo "monitor env value is unsafe" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf -v "$key" '%s' "$value"
|
||||
export "$key"
|
||||
;;
|
||||
esac
|
||||
done <"$file"
|
||||
}
|
||||
|
||||
feishu_sign() {
|
||||
local timestamp="$1"
|
||||
local secret="$2"
|
||||
local string_to_sign
|
||||
string_to_sign="$(printf '%s\n%s' "$timestamp" "$secret")"
|
||||
printf '' | openssl dgst -sha256 -hmac "$string_to_sign" -binary | openssl base64 -A
|
||||
}
|
||||
|
||||
webhook_allowed() {
|
||||
local url="$1"
|
||||
[[ "$url" =~ ^https://open\.(feishu\.cn|larksuite\.com|larkoffice\.com)/open-apis/bot/v2/hook/[A-Za-z0-9-]+$ ]] && return 0
|
||||
[[ "$url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/hook$ ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
alert_line_allowed() {
|
||||
local line="$1"
|
||||
[[ "$line" =~ ^service=[a-z0-9_-]+[[:space:]]route=(/|/[A-Za-z0-9/_.-]+|-)[[:space:]]errorCode=[a-z][a-z0-9._-]*[[:space:]]count=[0-9]+[[:space:]]requestId=(-|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}){0,4})[[:space:]]time=[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
}
|
||||
|
||||
post_monitor_alert() {
|
||||
local text="$1"
|
||||
local state_dir line timestamp sign payload_file response_file pending py curl_status code
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
pending=""
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
[ -n "$line" ] || continue
|
||||
alert_line_allowed "$line" || {
|
||||
echo "alert text has a field that is not allowed" >&2
|
||||
exit 1
|
||||
}
|
||||
pending="${pending}${line}"$'\n'
|
||||
done <<<"$text"
|
||||
[ -n "$pending" ] || return 0
|
||||
pending="${pending%$'\n'}"
|
||||
|
||||
if [ -z "${FEISHU_WEBHOOK_URL:-}" ]; then
|
||||
printf '%s\n' "$pending" >>"$state_dir/alerts.log"
|
||||
return 0
|
||||
fi
|
||||
webhook_allowed "$FEISHU_WEBHOOK_URL" || {
|
||||
printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log"
|
||||
return 0
|
||||
}
|
||||
|
||||
payload_file="$(mktemp)"
|
||||
response_file="$(mktemp)"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$payload_file" "$response_file"
|
||||
fi
|
||||
if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then
|
||||
timestamp="$(date +%s)"
|
||||
sign="$(feishu_sign "$timestamp" "$FEISHU_SIGN_SECRET")"
|
||||
FEISHU_ALERT_TEXT="$text" FEISHU_ALERT_TIMESTAMP="$timestamp" FEISHU_ALERT_SIGN="$sign" \
|
||||
python_write_payload "$payload_file" signed
|
||||
else
|
||||
FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain
|
||||
fi
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
curl_status=0
|
||||
curl --silent --show-error --fail --connect-timeout 10 --max-time 10 \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Connection: close' \
|
||||
--data-binary @"$payload_file" \
|
||||
-o "$response_file" \
|
||||
"$FEISHU_WEBHOOK_URL" || curl_status=$?
|
||||
code=""
|
||||
if [ "$curl_status" -eq 0 ]; then
|
||||
code="$("$py" - "$response_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
try:
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
except Exception:
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
if isinstance(payload, dict) and payload.get("code") == 0:
|
||||
print("0")
|
||||
else:
|
||||
print("")
|
||||
PY
|
||||
)"
|
||||
code="$(printf '%s' "$code" | tr -d '\r')"
|
||||
fi
|
||||
rm -f -- "$payload_file" "$response_file"
|
||||
if [ "$curl_status" -ne 0 ] || [ "$code" != "0" ]; then
|
||||
echo "feishu webhook was not accepted" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$pending" >>"$state_dir/alerts.log"
|
||||
}
|
||||
|
||||
python_write_payload() {
|
||||
local destination="$1"
|
||||
local mode="$2"
|
||||
local py
|
||||
export FEISHU_ALERT_TEXT FEISHU_ALERT_TIMESTAMP FEISHU_ALERT_SIGN
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
"$py" - "$destination" "$mode" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
destination, mode = sys.argv[1], sys.argv[2]
|
||||
text = os.environ["FEISHU_ALERT_TEXT"]
|
||||
payload = {"msg_type": "text", "content": {"text": text}}
|
||||
if mode == "signed":
|
||||
payload["timestamp"] = os.environ["FEISHU_ALERT_TIMESTAMP"]
|
||||
payload["sign"] = os.environ["FEISHU_ALERT_SIGN"]
|
||||
with open(destination, "w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, separators=(",", ":"))
|
||||
PY
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
# Delete only expired production-daily backup files. The prefix is fixed so a
|
||||
# caller cannot aim this at manual recovery points or other files.
|
||||
set -euo pipefail
|
||||
|
||||
prune_production_daily_backups() {
|
||||
local directory="$1"
|
||||
local retention_days="$2"
|
||||
local now_epoch="${3:-}"
|
||||
local dump name mtime age limit base verify verify_name
|
||||
local -a doomed=()
|
||||
|
||||
[ -n "$directory" ] && [ -d "$directory" ] && [ ! -L "$directory" ] || {
|
||||
echo "backup directory is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "$retention_days" =~ ^[1-9][0-9]{0,3}$ ]] || {
|
||||
echo "retention days must be a positive integer" >&2
|
||||
exit 1
|
||||
}
|
||||
if [ -z "$now_epoch" ]; then
|
||||
now_epoch="$(date +%s)"
|
||||
fi
|
||||
[[ "$now_epoch" =~ ^[0-9]+$ ]] || {
|
||||
echo "clock is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
limit=$((retention_days * 86400))
|
||||
|
||||
shopt -s nullglob
|
||||
for dump in "$directory"/production-daily-*.dump.enc; do
|
||||
name="$(basename -- "$dump")"
|
||||
[[ "$name" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]] || continue
|
||||
[ -f "$dump" ] && [ ! -L "$dump" ] || continue
|
||||
mtime="$(stat -c %Y "$dump" 2>/dev/null || stat -f %m "$dump")"
|
||||
[[ "$mtime" =~ ^[0-9]+$ ]] || continue
|
||||
age=$((now_epoch - mtime))
|
||||
if (( age > limit )); then
|
||||
doomed+=("$name")
|
||||
fi
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
if ((${#doomed[@]} == 0)); then
|
||||
return 0
|
||||
fi
|
||||
for name in "${doomed[@]}"; do
|
||||
base="${name%.dump.enc}"
|
||||
verify="$directory/${base}-restore-verify.json"
|
||||
rm -f -- "$directory/$name"
|
||||
if [ -e "$verify" ] && [ -f "$verify" ] && [ ! -L "$verify" ]; then
|
||||
verify_name="$(basename -- "$verify")"
|
||||
if [[ "$verify_name" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z-restore-verify\.json$ ]]; then
|
||||
rm -f -- "$verify"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' "$name"
|
||||
done
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
prune_production_daily_backups "${1:-}" "${2:-}" "${3:-}"
|
||||
fi
|
||||
@@ -6,10 +6,26 @@ umask 077
|
||||
: "${DEPLOY_PATH:?DEPLOY_PATH is required}"
|
||||
: "${RECOVERY_RUN_ID:?RECOVERY_RUN_ID is required}"
|
||||
|
||||
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
|
||||
echo "unexpected production path" >&2
|
||||
exit 1
|
||||
}
|
||||
# Tests set JYOTISHA_RECOVERY_FIXTURE=1 and point DEPLOY_PATH at a temp directory.
|
||||
# Production workflows never set it, so the live path check stays closed.
|
||||
if [ "${JYOTISHA_RECOVERY_FIXTURE:-}" = "1" ]; then
|
||||
[ -d "$DEPLOY_PATH" ] && [ ! -L "$DEPLOY_PATH" ] || {
|
||||
echo "unexpected production path" >&2
|
||||
exit 1
|
||||
}
|
||||
case "$DEPLOY_PATH" in
|
||||
/*) ;;
|
||||
*)
|
||||
echo "unexpected production path" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
else
|
||||
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
|
||||
echo "unexpected production path" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
[[ "$RECOVERY_RUN_ID" =~ ^[0-9]+$ ]] || {
|
||||
echo "recovery run id must be numeric" >&2
|
||||
exit 1
|
||||
@@ -138,7 +154,21 @@ usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); p
|
||||
|
||||
created_compact="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
backup_basename="production-pre-migration-${created_compact}.dump.enc"
|
||||
# Daily backups set RECOVERY_BASENAME_PREFIX=production-daily so retention can
|
||||
# tell them from manual recovery points. Unset keeps the original name.
|
||||
# That same prefix is the switch that skips recovery-baseline.env. Daily runs
|
||||
# still export, encrypt, and restore-verify, then record daily-backup-latest.env.
|
||||
backup_prefix="production-pre-migration"
|
||||
if [ -n "${RECOVERY_BASENAME_PREFIX:-}" ]; then
|
||||
case "$RECOVERY_BASENAME_PREFIX" in
|
||||
production-daily) backup_prefix="production-daily" ;;
|
||||
*)
|
||||
echo "unexpected recovery basename prefix" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
backup_basename="${backup_prefix}-${created_compact}.dump.enc"
|
||||
backup_partial="$backup_directory/.${backup_basename}.$$.partial"
|
||||
backup_file="$backup_directory/$backup_basename"
|
||||
restore_database="restore_verify_${created_compact,,}"
|
||||
@@ -214,11 +244,21 @@ printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha
|
||||
chmod 600 "$verify_file"
|
||||
|
||||
recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}"
|
||||
state_tmp="$state_directory/recovery-baseline.env.tmp.$$"
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nRECOVERY_FORMAT=2\nVERIFY_FILE=%s\n' \
|
||||
"$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp"
|
||||
chmod 600 "$state_tmp"
|
||||
mv "$state_tmp" "$state_directory/recovery-baseline.env"
|
||||
if [ "$backup_prefix" = "production-daily" ]; then
|
||||
daily_tmp="$state_directory/daily-backup-latest.env.tmp.$$"
|
||||
printf 'BACKUP_BASENAME=%s\nBACKUP_SHA256=%s\nVERIFIED_AT=%s\nBACKUP_FORMAT=2\n' \
|
||||
"$backup_basename" "$backup_sha256" "$verified_at" >"$daily_tmp"
|
||||
chmod 600 "$daily_tmp"
|
||||
mv "$daily_tmp" "$state_directory/daily-backup-latest.env"
|
||||
echo "baseline_updated=false" >&2
|
||||
else
|
||||
state_tmp="$state_directory/recovery-baseline.env.tmp.$$"
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nRECOVERY_FORMAT=2\nVERIFY_FILE=%s\n' \
|
||||
"$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp"
|
||||
chmod 600 "$state_tmp"
|
||||
mv "$state_tmp" "$state_directory/recovery-baseline.env"
|
||||
echo "baseline_updated=true" >&2
|
||||
fi
|
||||
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
|
||||
"$recovery_reference" "$created_at" "$verified_at" "$backup_sha256" \
|
||||
|
||||
@@ -36303,4 +36303,108 @@
|
||||
- 复发自：BUG-933（源码合同没在同一轮跟上）
|
||||
- 修复版本：分支 `codex/beta-capacity-20261014`。
|
||||
|
||||
## BUG-1358 | 生产没有每日自动备份
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-15
|
||||
- 来源：产品 10-14 内测体检。任务书 `docs/tasks/TASK-beta-safety-net-20261014.md`。
|
||||
- 现象：生产备份只在有人手动点恢复点时才做。仓库里没有按天跑的备份，也没有定时器文件。
|
||||
- 触发条件：内测开始写入真实用户数据之后，若两次手动备份之间出错或磁盘故障，中间的数据没有恢复点。
|
||||
- 根因：备份是按「发布前做一次」设计的，没有持续运行的每日任务。
|
||||
- 修复：新增只从 `main` 运行的每日备份工作流，复用现有加密与还原校验脚本。每日文件用单独前缀，过期清理不会碰到手动恢复点。工作流要等进入 `main` 后才会在北京时间 04:00 跑。本机没有生产主机，没有手动触发。
|
||||
- 验证：合同测试锁定有每日时刻、只在 `main` 上跑、调用同一个恢复脚本、清理只删过期的每日前缀。虚构目录里，手动恢复点和其他文件都还在。
|
||||
- 防复发：上述合同测试。清理函数不接受外部传入的前缀。
|
||||
- 相关记录：BUG-1350、BUG-1359、BUG-1360、BUG-1361。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。开工时本文件最大号是 BUG-1353，当时写成 BUG-1354。第一次合并 staging 后该号已被答题强度记录占用，中间号是 BUG-1355。再次合并后 BUG-1355～1357 已被内测容量占用，本条顺延为 BUG-1358。每日备份曾会改写当前恢复点，已由 BUG-1361 分开。
|
||||
|
||||
## BUG-1359 | 站点打不开或磁盘过满时没有告警
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-15
|
||||
- 来源：同 BUG-1358。
|
||||
- 现象：容器自己的健康检查只会尝试重启，不会通知人。磁盘变满也没有通知。
|
||||
- 触发条件：公网健康检查连续失败，或生产主机磁盘使用率达到 80%。
|
||||
- 根因：上线时没有持续探活，也没有把告警发到人用的渠道。
|
||||
- 修复：探活脚本连续 3 次失败才发一条告警，恢复只发一次。磁盘检查单独在生产主机上跑。Webhook 从环境文件读取，没有配置就只写本机日志。外发正文不含用户内容。脚本还没装到 runner 或生产主机上。
|
||||
- 验证：本地桩服务覆盖正常、500、超时和缺少字段。连续 3 次才告警，恢复只出现一次。签名测试里的密钥没有出现在发出的正文里。
|
||||
- 防复发：上述脚本测试。告警行若含不允许的字段，脚本拒绝发送。
|
||||
- 相关记录：BUG-1358、BUG-1360。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。原先写成 BUG-1355，第一次合并 staging 后为 BUG-1356。再次合并后顺延为 BUG-1359。
|
||||
|
||||
## BUG-1360 | 容器日志不轮转，也不能按一轮请求查找
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-15
|
||||
- 来源：同 BUG-1358。BUG-1306 取证时日志只在容器控制台，重建后查不到。
|
||||
- 现象：编排没有日志大小上限。知道一轮请求的编号时，没有只读的查找命令。
|
||||
- 触发条件：日志持续写入，或需要按 requestId 回看 web、api、caddy。
|
||||
- 根因：日志沿用守护进程的未知默认值，也没有按编号检索的入口。
|
||||
- 修复：每个服务的日志改为单文件 20 MB、最多 10 个文件。新增只读查找脚本，只接受 UUID。错误汇总只计数和最多 5 个编号，丢掉含邮箱、额外日期或非 ASCII 文本的行。要等生产容器按新编排重建后，轮转才生效。
|
||||
- 验证：两个编排文件里每个服务的日志段一致。虚构日志能按时间排序找到编号；命令注入写法被拒绝，旁边的标记文件还在。含邮箱和汉字的样本没有进入告警文本。
|
||||
- 防复发：上述合同测试和查找脚本的格式检查。
|
||||
- 相关记录：BUG-1306、BUG-1358、BUG-1359。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。原先写成 BUG-1356，第一次合并后为 BUG-1357。再次合并后顺延为 BUG-1360。
|
||||
|
||||
## BUG-1361 | 每日自动备份改写了当前恢复点
|
||||
|
||||
- 状态：resolved
|
||||
- 首次发现 / 最近更新：2026-10-15 / 2026-10-15
|
||||
- 来源：issue #75。产品选 A。修复单 `docs/tasks/TASK-beta-safety-net-fix-20261015.md`。检索本文件的「recovery-baseline」和「每日备份」，没有更早的同类记录。
|
||||
- 现象：每天凌晨的自动备份会把「当前恢复点」换成当天的自动备份。上线前手动建的恢复点，隔一夜就没了。
|
||||
- 触发条件：每日备份调用 `deploy/run-production-recovery.sh`，前缀是 `production-daily`。脚本结尾无条件写 `.state/recovery-baseline.env`。发布、迁移、回退都读这份文件。
|
||||
- 根因：每日备份复用了手动恢复点脚本，脚本不区分「只存档」和「更新当前恢复点」。
|
||||
- 修复：前缀为 `production-daily` 时仍完成导出、加密和还原核对（含表权限、列权限、属主），并写校验文件，但跳过 `recovery-baseline.env`。另写 `daily-backup-latest.env`，只有文件名、sha256、核对时间和格式版本。手动创建恢复点的路径不变。
|
||||
- 验证：临时目录加假 docker。已有恢复点的内容和修改时间不变；手动路径仍写出 `RECOVERY_FORMAT=2`。发布、迁移、回退所读的文件不引用 `daily-backup-latest.env`。清理仍只删过期的 `production-daily-` 文件。
|
||||
- 防复发：上述合同测试。生产工作流不设置测试用的路径开关。
|
||||
- 相关记录：BUG-1358。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。本机没有装到生产主机，也没有点生产按钮。原先写成 BUG-1358。再次合并 staging 后该号给了「没有每日自动备份」，本条顺延为 BUG-1361。
|
||||
|
||||
## BUG-1362 | 安全网测试在 Linux 上因环境文件权限失败，而且没进自动门禁
|
||||
|
||||
- 状态：resolved
|
||||
- 首次发现 / 最近更新：2026-10-15 / 2026-10-15
|
||||
- 来源：issue #75 验收打回。修复单 `docs/tasks/TASK-beta-safety-net-fix2-20261015.md`。检索「monitor env」「0600」「test_beta_safety_net」，没有更早的同类记录。
|
||||
- 现象：发布前的全量测试在 Linux 上失败。staging 的自动检查却是绿的，所以这一版能进测试环境，却会挡住以后的上生产检查。
|
||||
- 触发条件：`tests/test_beta_safety_net.py` 写出飞书环境文件后直接跑告警脚本。Linux 上该文件是 0644。脚本拒绝不是 0600 的环境文件。macOS 和 Windows 跳过这项检查。这个测试文件也不在自动门禁的 quick 清单里。
|
||||
- 根因：测试没有把环境文件收成 0600。quick 清单只在发布工作流里才跑全树 pytest，所以 Linux 上的失败没有在 staging 门禁暴露。
|
||||
- 修复：测试写完环境文件后执行 `os.chmod(path, 0o600)`。脚本里的 0600 检查不改。把 `tests/test_beta_safety_net.py` 加进 quick 清单。
|
||||
- 验证：非 Linux 上看不到 Unix 权限位，权限相等断言只在 POSIX 上执行。脚本仍包含「必须是 600」的检查。quick 清单包含这个测试文件。
|
||||
- 防复发：上述权限调用、脚本检查和 quick 清单断言。
|
||||
- 相关记录：BUG-1359。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。
|
||||
|
||||
## BUG-1363 | 磁盘到 70% 后每日备份每天失败，旧文件也不再清理
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-15 / 2026-10-15
|
||||
- 来源：issue #75 验收打回。修复单 `docs/tasks/TASK-beta-safety-net-fix2-20261015.md`。
|
||||
- 现象：磁盘用到 70% 以后，每天的自动备份都失败。过期的每日文件不会被删，所以磁盘腾不出来。磁盘告警要到 80% 才响，也没有人看最近一次核对是不是太旧。
|
||||
- 触发条件：备份目录使用率达到 70%。清理步骤写在备份成功之后。`daily-backup-latest.env` 的 `VERIFIED_AT` 超过 36 小时，或这份记录不存在。
|
||||
- 根因：恢复脚本在 70% 时直接退出，工作流因此走不到后面的清理。探活和磁盘告警都不读每日核对时间。
|
||||
- 修复：每日工作流先清理过期的 `production-daily-` 文件，再做备份。磁盘检查同时读取 `VERIFIED_AT`（任务书里的 verified_at 就是这个键）。超过 36 小时或记录不可用时告警一次，重新变新后再发一次已恢复。探活模式不读这份文件。70% 拒绝本身不改。
|
||||
- 验证：工作流文本里清理调用在备份调用之前。虚构时钟下，恰好 36 小时不告警，37 小时告警一次，缺失记录告警一次，变新后发已恢复。告警行不含备份文件名。
|
||||
- 防复发：上述顺序断言和过期告警测试。清理脚本仍只认每日前缀。
|
||||
- 相关记录：BUG-1358、BUG-1359、BUG-1361。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。本机没有装到生产主机，也没有点生产按钮。
|
||||
|
||||
## BUG-1364 | 错误汇总和查日志在生产主机上跑不起来
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-15 / 2026-10-15
|
||||
- 来源：issue #75 验收打回。修复单 `docs/tasks/TASK-beta-safety-net-fix2-20261015.md`。
|
||||
- 现象：生产主机上的定时错误汇总会每次失败，一条告警都发不出去。按请求编号查容器日志同样起不来。飞书返回失败时仍像已经发出。同一类错误每 5 分钟重复发。容器日志按服务名而不是时间排序。
|
||||
- 触发条件：cron 直接执行 `error-digest.sh` 或 `find-request-log.sh`，且没有事先导出数据库环境文件。飞书响应体里的 `code` 不是 0。同一错误码在 30 分钟内再次出现。`docker compose logs` 的行首带有服务名前缀。
|
||||
- 根因：这两个脚本直接调用 `docker`，没有 `sudo -n`。也没有把数据库环境文件设成生产路径，编排会去找不存在的 staging 文件。发送命令不检查 HTTP 失败和飞书 `code`。排序把服务名前缀算了进去。
|
||||
- 修复：两个脚本改为 `sudo -n docker compose`，并显式传入生产编排文件和 `../.env.production.database`。飞书请求加上 `--fail`，响应 `code` 必须是 0，失败时不写入已发送记录。同一服务、路由、错误码 30 分钟内不重复发。查日志先去掉行首前缀再按时间排序。
|
||||
- 验证：源码断言命令行含 `sudo -n docker compose` 和生产环境文件，且没有裸的 `docker compose`。虚构飞书响应 `code` 非 0 时本地告警记录为空。30 分钟边界前后各跑一次。带服务名前缀的虚构日志按时间而不是按服务名输出。
|
||||
- 防复发：上述命令形状、发送失败、去重和排序测试。
|
||||
- 相关记录：BUG-1359、BUG-1360。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。本机没有装到生产主机，也没有点生产按钮。
|
||||
|
||||
|
||||
@@ -39,6 +39,8 @@ Gitea → Create Production Recovery Point。Use workflow from：`main`。`deplo
|
||||
|
||||
通过：成功结束，记下三样东西：`recovery_reference`、`recovery_created_at`、以及它已经写明还原验证通过。主机上的 `/opt/jyotisha-production/.state/recovery-baseline.env` 里必须有 `RECOVERY_FORMAT=2`。没有这一行，或者数字小于 2，都不要往下走：先再点一次创建恢复点。8 月以来不带权限的旧备份不能再拿来回退。加密备份在生产主机的备份目录里，不要打开、不要把密钥贴出来。
|
||||
|
||||
每日自动备份不是这一步的恢复点。它每天只把加密文件存档并做还原核对，不会改 `recovery-baseline.env`。隔夜的自动备份不会换掉你上线前手动建的恢复点。上线前仍然要点 Create production recovery。
|
||||
|
||||
失败：停。不要迁移。
|
||||
|
||||
## 5. 在主机上演练迁移
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# 生产备份恢复演练（2026-10-14）
|
||||
|
||||
给产品负责人照着做。这一次演练还原到一次性库，不碰正在服务的主库，网站不用停。真正整库还原会停写，放在文末，只有确认要覆盖现网数据时才做。
|
||||
|
||||
不要把备份文件、密钥、数据库口令、出生资料、姓名、邮箱或会话内容写进仓库、聊天或这份记录。
|
||||
|
||||
## 备份在哪里
|
||||
|
||||
- 每天北京时间 04:00，从 `main` 自动跑「Backup Production Daily」。也可以在 Gitea 里手动点一次。它只在 `main` 上跑。
|
||||
- 加密文件在生产主机 `/opt/jyotisha-production/backups/`，文件名是 `production-daily-日期T时间Z.dump.enc`。旁边有同名的 `-restore-verify.json`，里面只有行数和权限是否一致，没有用户正文。
|
||||
- 手动恢复点仍叫 `production-pre-migration-...`。每日清理只删过期的 `production-daily-` 文件，不会删手动恢复点。
|
||||
- Gitea 上的加密产物默认留 14 天。主机上的每日文件默认留 7 天。
|
||||
- 加密方式与现有恢复点相同：AES-256-CBC、PBKDF2。密钥在主机的数据库环境文件里，不要打印。
|
||||
|
||||
## 一次性库演练（不停机）
|
||||
|
||||
在生产主机上用 `deploy` 用户做。下面的库名换成当天的日期，不要用正在服务的那个库名。
|
||||
|
||||
1. 选定一个 `production-daily-*.dump.enc`。确认旁边的校验文件写着 `"ok": true` 和 `"restore_database_removed": true`。
|
||||
2. 进入 `/opt/jyotisha-production`。数据库口令和加密密钥已经在 `.env.production.database` 里。只在这台机器上读取，不要回显。
|
||||
3. 确认生产 Postgres 容器只有一个。容器名用 `docker ps` 看，下面记成 `<postgres>`。
|
||||
4. 建一次性库，名字必须不是主库名，例如 `restore_drill_20261015`：
|
||||
|
||||
```bash
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres> \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c 'CREATE DATABASE "restore_drill_20261015";'
|
||||
```
|
||||
|
||||
5. 解密并还原进这个一次性库。不要把输出重定向到主库。
|
||||
|
||||
```bash
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 \
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY \
|
||||
-in /opt/jyotisha-production/backups/production-daily-日期T时间Z.dump.enc |
|
||||
sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres> \
|
||||
pg_restore -U "$POSTGRES_USER" -d restore_drill_20261015 --exit-on-error
|
||||
```
|
||||
|
||||
6. 只记四个数字：`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表张数。不要把表里的内容抄出来。
|
||||
7. 删掉一次性库，并确认它已经不在：
|
||||
|
||||
```bash
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres> \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c 'DROP DATABASE "restore_drill_20261015" WITH (FORCE);'
|
||||
```
|
||||
|
||||
8. 把四个数字、文件名（不要附文件）、一次性库是否已删除，填进 `docs/testing/production-restore-drill-20261014.md`。
|
||||
|
||||
通过：还原命令退出码是 0，四个数字都是非负整数，一次性库已删除，主库没有被这条命令点名。
|
||||
|
||||
失败：停在失败的那一步。不要为了演练去改主库。把退出码交给验收，不要附带环境文件。
|
||||
|
||||
## 整库还原（会停机，平时不要做）
|
||||
|
||||
只有在确认现网库要被这份备份覆盖时才做。预留 30 分钟停写。这不是实测时间，数据变大就再加。
|
||||
|
||||
1. 停掉 `web` 和 `api`，让没有新的写入。Postgres 先留着。
|
||||
2. 用上面同一条解密和 `pg_restore`，目标改成主库名。这一步会覆盖主库里的现有数据。
|
||||
3. 核对四个行数，并对照校验文件里的权限三项是否为真。格式版本低于 2 的旧备份不要拿来做这件事。
|
||||
4. 再启动 `web` 和 `api`。打开 `https://jyotisha.chat/api/health`，`status` 应为 `ok`。
|
||||
5. 把实际停了多久填进演练模板。不要写用户内容。
|
||||
@@ -0,0 +1,34 @@
|
||||
# 进度 · 内测安全网（2026-10-15）
|
||||
|
||||
- 分支：`codex/beta-safety-net-20261014`
|
||||
- 基线：开工时 `origin/staging` 为 `71f7fac3a13f817f91755ff01b25b8706380ae19`
|
||||
- 前置：`09906cdc`（发布通路）和 `4c3485bc`、`3893783e`（#58）都已是这个基线的祖先。`deploy/docker-compose.server.yml` 在加日志段之前没有 #58 留下的未合入冲突。
|
||||
- 编号：本文件最大号是 BUG-1353。本单当时写成 BUG-1354（没有每日备份）、BUG-1355（没有探活和磁盘告警）、BUG-1356（日志不轮转、不能按请求编号查）。合并 staging 后 BUG-1354 已被更早的答题强度记录占用，三条顺延为 BUG-1355、BUG-1356、BUG-1357。2026-10-15 再次合并 staging 时，这三个号已被内测容量占用，最终编号是 BUG-1358、BUG-1359、BUG-1360。
|
||||
|
||||
## 做了什么
|
||||
|
||||
- 新增每日备份工作流。北京时间 04:00，只在 `main` 上跑，调用现有的 `deploy/run-production-recovery.sh`。产物默认留 14 天。主机上只清理超过 7 天、且文件名严格是 `production-daily-` 的加密备份和对应校验文件。
|
||||
- 恢复脚本增加可选前缀。不设置时文件名仍是 `production-pre-migration-`。每日任务把它设为 `production-daily`。加密、还原校验、锁和磁盘 70% 拒绝没有改。
|
||||
- 探活、磁盘、错误汇总、飞书文本消息（可选签名）。没有 Webhook 时只写本机日志。
|
||||
- 两个编排文件的每个服务加上相同的日志上限：json-file，20 MB，10 个文件。镜像、环境、端口、健康检查没改。
|
||||
- 按 UUID 查 web / api / caddy 日志的只读脚本。拒绝其它写法。
|
||||
- 恢复演练步骤和空白结果模板。安装步骤写在 `deploy/README.md` 和 `BLOCKED.md`。
|
||||
|
||||
## 跑过的检查
|
||||
|
||||
| 检查 | 命令 | 结果 |
|
||||
| --- | --- | --- |
|
||||
| 安全网测试 | `.venv\Scripts\python.exe -m pytest tests/test_beta_safety_net.py -q` | 13 通过，退出码 0 |
|
||||
| 隐私标记 | `.venv\Scripts\python.exe -m pytest tests/test_repo_privacy_markers.py -q` | 退出码 0 |
|
||||
| 编排能否解析 | `docker compose -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml config --quiet` | 退出码 0。本机没有真实环境文件，检查时用了临时空文件，没有提交 |
|
||||
| shellcheck | 无 | 本机没有这条命令 |
|
||||
| 前端 tsc / lint / 全量测试 | 未跑 | 没有改网页 |
|
||||
| Python 快速门 | 未跑 | 按本轮约定不跑 |
|
||||
| 数据库测试 | 未跑 | 没有改数据库 |
|
||||
|
||||
## 偏离
|
||||
|
||||
- 没有改现有的四个生产工作流，也没有把它们的 SSH 步骤抽成共用脚本。新工作流自己复制了同一套 SSH，并调用同一个恢复脚本。
|
||||
- 没有往网页或接口日志里加新字段。现有观测记录已经有 requestId 和 errorCode。错误汇总会丢掉含有汉字、邮箱或第二个日期的整行，避免把对话带出去。因此中文报错不会进入计数。
|
||||
- 探活外发不带提交号，只带错误码 `commit_changed`。提交号留在 runner 本机状态里。
|
||||
- 本机没有生产主机，没有登录 runner，没有手动触发生产工作流。告警和定时备份要等产品安装，见 `BLOCKED.md`。
|
||||
@@ -0,0 +1,34 @@
|
||||
# 进度 · 内测安全网修复（2026-10-15）
|
||||
|
||||
- 分支：`codex/beta-safety-net-20261014`
|
||||
- 任务书：`docs/tasks/TASK-beta-safety-net-fix-20261015.md`
|
||||
- 上一轮已推送：`53d8df257ddef8918034acef36df0b4b7daade2c`
|
||||
- 产品选 A：每天的自动备份照做，但不改当前恢复点。
|
||||
|
||||
## 做了什么
|
||||
|
||||
- 先合并最新 `origin/staging`，没有改写已经推送的提交。合并时本分支的 BUG-1354 和更早的答题强度记录撞号，安全网三条顺延为 BUG-1355、BUG-1356、BUG-1357。这次的修复当时写成 BUG-1358。2026-10-15 再次合并 staging 后，BUG-1355～1357 归内测容量，本条修复最终是 BUG-1361。
|
||||
- 用已经存在的 `RECOVERY_BASENAME_PREFIX=production-daily` 当开关，没有另加模式参数。这个前缀下，脚本仍完成导出、加密、还原核对（含表权限、列权限、属主）并写校验文件，但跳过 `recovery-baseline.env`。标准错误打印 `baseline_updated=false`。另写 `.state/daily-backup-latest.env`，只有文件名、sha256、核对时间、`BACKUP_FORMAT=2`。
|
||||
- 不设这个前缀时，仍写 `recovery-baseline.env`，里面有 `RECOVERY_FORMAT=2`，标准错误打印 `baseline_updated=true`，不改已经存在的每日记录。
|
||||
- 标准输出仍是原来的 7 行。因此没有改 `.gitea` 里「恰好 7 行」的检查，也没有改四个已有生产工作流。
|
||||
- 再确认清理脚本只删严格匹配的过期 `production-daily-` 文件。既有测试里的 `production-pre-migration-` 手动恢复点仍被保留。
|
||||
- `docs/operations/production-release-runbook-20261014.md` 和 `deploy/README.md` 写明：每日备份只是存档和核对，上线前仍要手动点 Create production recovery。
|
||||
|
||||
## 跑过的检查
|
||||
|
||||
| 检查 | 命令 | 结果 |
|
||||
| --- | --- | --- |
|
||||
| 安全网与隐私 | `.venv\Scripts\python.exe -m pytest tests/test_beta_safety_net.py tests/test_repo_privacy_markers.py -q` | 78 通过（安全网 16，隐私 62），退出码 0 |
|
||||
| 前端合同 | `node node_modules/tsx/dist/cli.mjs --test tests/runtime-sources-no-test-imports.test.ts tests/staging-backend-workflows.test.ts` | 46 项：41 通过，5 失败，0 取消。失败是本机缺口，见下方偏离。恢复点工作流的文本检查通过 |
|
||||
| shellcheck | 无 | 本机没有这条命令 |
|
||||
| 前端 tsc / lint / 全量测试 | 未跑 | 没有改网页 |
|
||||
| Python 快速门 | 未跑 | 按本轮约定不跑 |
|
||||
| 数据库测试 | 未跑 | 没有改数据库 |
|
||||
|
||||
## 偏离
|
||||
|
||||
- `baseline_updated` 打在标准错误，避免把每日备份和手动恢复点工作流对标准输出「恰好 7 行」的检查撑破。没有改 `.gitea/**`。
|
||||
- 测试要在临时目录里跑脚本。生产路径检查保持原样；只有 `JYOTISHA_RECOVERY_FIXTURE=1` 时才允许别的目录。会调用恢复脚本的两个工作流都没有设置这个变量。
|
||||
- 本机 Git Bash 没有 `flock`，pytest 的临时目录也不能改 Unix 权限。测试只在自己的 PATH 里放了假的 `docker`、`sudo`、`df`、`install`、`chmod`、`flock`。生产脚本仍调用真命令。
|
||||
- 没有在这台 Windows 上安装到生产，没有点任何生产按钮。主机安装和一次恢复演练仍留给产品。
|
||||
- 前端合同 5 项失败都不是这次改的脚本：本机 Python 没有 yaml 模块（直接跑时退出码 9009）、没有 rsync（退出码 127）、临时目录不能改权限（install 报 Permission denied）、文档范围脚本退出码 49。这四类脚本本次都没改。恢复点脚本的语法检查和文本合同是通过的。
|
||||
@@ -0,0 +1,41 @@
|
||||
# 进度 · 内测安全网修复单 2（2026-10-15）
|
||||
|
||||
- 分支：`codex/beta-safety-net-20261014`
|
||||
- 任务书：`docs/tasks/TASK-beta-safety-net-fix2-20261015.md`
|
||||
- 被打回的提交：`5c0d895391b5d0806c98de669a2e9cdcd61e3335`
|
||||
- 先合并最新 `origin/staging`（`32b238cf`），没有改写已经推送的提交。
|
||||
|
||||
## 做了什么
|
||||
|
||||
- 合并时安全网旧编号和内测容量撞号。容量保留 BUG-1355～1357。安全网顺延为 BUG-1358～1361。这次三条例外是 BUG-1362、BUG-1363、BUG-1364。
|
||||
- 飞书测试写完环境文件后改成 0600。脚本里「Linux 上必须是 0600」的检查没改。这个测试文件加进 staging 自动门禁的 quick 清单。
|
||||
- 每日工作流先清理过期的每日备份，再做新备份。磁盘到 70% 时仍会拒绝备份，但旧的每日文件不会因此永远留着。70% 这条拒绝本身没改。
|
||||
- 生产主机上的磁盘检查同时看 `daily-backup-latest.env` 的 `VERIFIED_AT`。缺失，或比现在早超过 36 小时，告警一次；重新变新后再发一次已恢复。探活模式不读这份文件。
|
||||
- 错误汇总和按请求查日志改为 `sudo -n docker`，并显式传入生产编排和 `../.env.production.database`。
|
||||
- 飞书请求加了 `--fail`，响应里的 `code` 必须是 0。失败时不写入已发送记录。同一服务、同一路由、同一错误码 30 分钟内不重复发。
|
||||
- 容器日志先去掉行首的服务名前缀，再按时间排序。
|
||||
- 上一轮「每日备份不改当前恢复点」没有重做。四个已有生产工作流、staging 工作流、`gated-paths.txt`、`is-docs-only-range.sh` 没改。
|
||||
|
||||
## 跑过的检查
|
||||
|
||||
| 检查 | 命令 | 结果 |
|
||||
| --- | --- | --- |
|
||||
| 安全网 | `.venv\Scripts\python.exe -m pytest tests/test_beta_safety_net.py -q` | 20 通过，退出码 0 |
|
||||
| 隐私标记 | `.venv\Scripts\python.exe -m pytest tests/test_repo_privacy_markers.py -q` | 62 通过，退出码 0 |
|
||||
| 前端 tsc / lint | 未跑 | 没有改网页 |
|
||||
| Python 快速门 | 未跑 | 按本轮约定不跑 |
|
||||
| 数据库测试 | 未跑 | 没有改数据库 |
|
||||
| shellcheck | 未跑 | 本机没有这条命令 |
|
||||
|
||||
## 断言变更
|
||||
|
||||
| 位置 | 原值 | 新值 | 原因 |
|
||||
| --- | --- | --- | --- |
|
||||
| 飞书测试桩的响应正文 | `ok` | `{"code":0}` | 发送方现在要求飞书 `code` 为 0。请求体字段断言没改 |
|
||||
| 磁盘测试的准备数据 | 不提供每日核对记录 | 提供一份核对时间等于当前时间的记录 | 磁盘模式会同时检查核对时间。没有记录会多一条告警。磁盘告警原文断言没改 |
|
||||
|
||||
## 偏离
|
||||
|
||||
- 任务书写 `verified_at`。上一轮文件里的键是 `VERIFIED_AT`。这次读这个键，没有改名。
|
||||
- 本机 `os.chmod(path, 0o600)` 之后，`stat` 仍是 `0o666`。Windows 存不下 Unix 权限。0600 相等断言只在 POSIX 上执行；`chmod` 每次都会调用。Linux 上的脚本检查保持原样。
|
||||
- 没有在这台 Windows 上安装到生产，没有点任何生产按钮。主机安装和恢复演练仍留给产品。
|
||||
@@ -338,7 +338,9 @@
|
||||
| `TASK-release-path-production-20261014.md` | `PROGRESS-release-path-production-20261014.md` | **发布通路 + 生产首发**：产品 10-14 授权改生产工作流（发布 SHA = 最近门禁成功提交 + 其后纯文档）；63 个迁移清单与演练脚本、首发 runbook；生产拒绝测试验证码。Grok 不 dispatch 生产工作流 | 第 2 次打回后续修 | `codex/release-path-production-20261014`（BUG-1348、BUG-1349） |
|
||||
| `TASK-release-path-production-fix-20261014.md` | `PROGRESS-release-path-production-fix-20261014.md` | **#74 第 1 次打回**：发布规则成立；测试写死 Windows 路径（Linux 全红）、主机演练依赖主机 node 且会留下生产数据副本、docs-only 头时回滚被拒；另补 CHECK_RELEASE_GATE 断言与 OTP 守卫规范化 | 已修，第 2 次仍未过 | `codex/release-path-production-20261014` |
|
||||
| `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2（产品选 A）**：恢复点要带属主与权限，旧格式不得回退；演练按备份里的授权比对，并在真实 PostgreSQL 17 上跑通一次 | 待验收 | `codex/release-path-production-20261014` |
|
||||
| `TASK-beta-safety-net-20261014.md` | — | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。**等发布通路单合入、#58 合入后开工** | 待前置 | — |
|
||||
| `TASK-beta-safety-net-20261014.md` | `PROGRESS-beta-safety-net-20261014.md` | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。发布通路与 #58 已在开工时的 `origin/staging` 上。主机安装和一次演练留给产品 | 待验收 | `codex/beta-safety-net-20261014`（BUG-1358～1360） |
|
||||
| `TASK-beta-safety-net-fix-20261015.md` | `PROGRESS-beta-safety-net-fix-20261015.md` | **#75 产品选 A**：每日备份照常做，但不改当前恢复点；另记 `daily-backup-latest.env` | 待验收 | `codex/beta-safety-net-20261014`（BUG-1361） |
|
||||
| `TASK-beta-safety-net-fix2-20261015.md` | `PROGRESS-beta-safety-net-fix2-20261015.md` | **#75 修复单 2**：生产侧全过；飞书测试 env 文件权限 0644 在 Linux 上失败（会挡发布门禁）；新测试不在 quick 门禁；磁盘 ≥70% 后备份静默停摆；监控脚本缺 sudo 与 env 文件 | 待验收 | `codex/beta-safety-net-20261014`（BUG-1362～1364） |
|
||||
| `TASK-beta-capacity-20261014.md` | — | **内测容量**：按用户限流、重计算短排队与对话重试、兑换码当邀请码、staging 10/20 并发压测。**等 #58 合入后开工，压测等 #57 合入** | 待前置 | — |
|
||||
| `TASK-beta-capacity-fix-20261015.md` | — | **#76 第 1 次打回**：`a54f75e5` 限流与短排队逻辑通过；源码文本断言未跟改、插入新测试使隐私白名单行号错位 | 待领取 | `codex/beta-capacity-20261014` |
|
||||
| `TASK-special-lagnas-native-20261008.md` | `PROGRESS-special-lagnas-native-20261008.md` | **特殊上升点原生计算（线上缺陷）**：sync6 U6 运行时调 PyJHora，生产镜像没有 PyJHora → staging 上 Hora/Ghati/Sree/Indu/Pranapada 全 blocked；原生实现、PyJHora 只作冻结对照、快速门禁止运行时 import jhora；最先做 | 已验收（已部署 staging `795b8845`，BUG-1273） | `codex/special-lagnas-native-20261008`（未上线；部署以 `/api/health` 为准） |
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# TASK · 内测安全网修复单（issue #75，产品选 A）
|
||||
|
||||
- 日期：2026-10-15
|
||||
- 原任务书：`docs/tasks/TASK-beta-safety-net-20261014.md`
|
||||
- 被验 SHA：`53d8df257ddef8918034acef36df0b4b7daade2c`（`codex/beta-safety-net-20261014`）
|
||||
- 执行方：Grok，同一分支续做（先 merge 最新 `origin/staging`，不改写已推历史）
|
||||
- **决策记录**：产品负责人 2026-10-15 在 issue #75 选 **A**——**授权新增定时工作流 `.gitea/workflows/backup-production-daily.yml`**（每天北京时间 04:00 自动连生产做加密备份），但**每日备份不得改动「当前恢复点」记录**。恢复点仍只由产品负责人手动点「Create production recovery」创建。仍不改已有四个生产工作流的语义；仍不点任何生产按钮。
|
||||
|
||||
## 1. 问题
|
||||
|
||||
`deploy/run-production-recovery.sh` 无论由谁调用，结尾都会无条件写 `$DEPLOY_PATH/.state/recovery-baseline.env`（`RECOVERY_REFERENCE`、`RECOVERY_SHA256`、`RECOVERY_FORMAT=2` 等）。发布、迁移、回退都读这份文件当「当前恢复点」。每日备份复用这个脚本，于是每天 04:00 都会把它换成当天的自动备份——产品负责人上线前手动建的恢复点，隔一夜就被替换。
|
||||
|
||||
## 2. 修法要求
|
||||
|
||||
1. **每日备份不写 `recovery-baseline.env`**：`RECOVERY_BASENAME_PREFIX=production-daily` 时（或新增一个明确的模式参数，二选一，写进 PROGRESS），脚本仍完整执行「导出 → 加密 → 还原核对（含 #74 的表权限、列权限、属主三项比对）→ 写 `*-restore-verify.json`」，但**跳过写 `recovery-baseline.env`**，并在输出里打印 `baseline_updated=false`。手动恢复点路径行为不变，打印 `baseline_updated=true`。
|
||||
2. 每日备份的核对结果单独记录：在 `$DEPLOY_PATH/.state/` 下另写一个 `daily-backup-latest.env`（只含文件名、sha256、校验时间、格式版本，不含数据、不含密钥），供告警 / 运维查看；发布、迁移、回退工作流**不得**读取它。
|
||||
3. 合同测试（`tests/test_beta_safety_net.py` 或 `frontend/tests/staging-backend-workflows.test.ts`）：
|
||||
- 每日模式下 `recovery-baseline.env` 不被创建或修改（用临时目录 + 假 docker 跑脚本，断言文件 mtime / 内容不变）；
|
||||
- 手动模式下仍会写，且含 `RECOVERY_FORMAT=2`；
|
||||
- 三个读取恢复点的生产工作流 / 脚本不引用 `daily-backup-latest.env`。
|
||||
4. runbook 与 `deploy/README.md`：写清「每日备份只做存档与核对，不是发布用的恢复点；上线前仍要手动点 Create production recovery」。
|
||||
5. 清理脚本只删 `production-daily-*` 前缀的文件（原任务书已要求）——再确认不会删 `production-pre-migration-*` 手动恢复点。
|
||||
|
||||
## 3. 验收
|
||||
|
||||
- Claude 会对本单做完整验收（上一轮在发现定时工作流后就停下等拍板，其余部分尚未逐行验）：逐行看新工作流与各脚本、跑 `tests/test_beta_safety_net.py`、Python 全量、前端全量、`docker compose config`。
|
||||
- 交回前本机跑：`python -m pytest tests/test_beta_safety_net.py tests/test_repo_privacy_markers.py -q`、`npx tsx --test tests/runtime-sources-no-test-imports.test.ts tests/staging-backend-workflows.test.ts`。
|
||||
|
||||
## 4. 让步顺序
|
||||
|
||||
无。
|
||||
@@ -0,0 +1,45 @@
|
||||
# TASK · 内测安全网修复单 2（issue #75）
|
||||
|
||||
- 日期：2026-10-15
|
||||
- 前序：`TASK-beta-safety-net-20261014.md`、`TASK-beta-safety-net-fix-20261015.md`（产品选 A）
|
||||
- 被验 SHA：`5c0d895391b5d0806c98de669a2e9cdcd61e3335`（`codex/beta-safety-net-20261014`）
|
||||
- 执行方：Grok，同一分支续做（先 merge 最新 `origin/staging`，不改写已推历史）
|
||||
- 红线不变：仍不改已有四个生产工作流与 staging 工作流、`gated-paths.txt`、`is-docs-only-range.sh`；不点生产按钮
|
||||
|
||||
## 1. 验收结论：面向生产的部分全部过；一条阻断 + 三条要一起修
|
||||
|
||||
已确认成立：每日工作流只从 main 跑、与恢复点工作流同一把锁 / 同样的密钥处理、不回显密钥；每日模式**任何路径都不写** `recovery-baseline.env`（只有手动分支写），失败时什么状态都不写；`daily-backup-latest.env` 只有 4 个键；发布 / 迁移 / 回退不读它；手动恢复点（含 `RECOVERY_FORMAT=2` 与权限比对）不变；清理脚本只认 `^production-daily-\d{8}T\d{6}Z\.dump\.enc$` 的普通文件，碰不到 `production-pre-migration-*`；错误分类只外发校验过的 errorCode / 路由 / UUID，不发用户内容；compose 只加了 6 个服务一致的 `logging`。
|
||||
|
||||
### B1（阻断）· 测试在 Linux 上失败
|
||||
|
||||
`tests/test_beta_safety_net.py` 约 616–634 行 `test_feishu_payload_keeps_only_allowed_fields`：约 76–77 行用 `write_text` 写 `monitor.env`，默认权限 0644；`deploy/monitor/notify-feishu.sh` 约 18–23 行在 Linux 上拒绝非 0600 的环境文件 → 退出 1。macOS / Windows 上跳过了这项检查所以没发现。`release-quality-gate.yml` 在 Linux 上跑全树 pytest，提升 main 会被它挡住。
|
||||
- 修：测试里写完后 `os.chmod(env_file, 0o600)`。脚本里的检查是对的，不改。
|
||||
|
||||
### N1 · 新测试不进 staging 门禁
|
||||
|
||||
`scripts/run_quality_gate.py` 的 quick 清单（约 55–90 行）没有 `tests/test_beta_safety_net.py`，合同只在发布时才跑——B1 就是这样漏过去的。把它加进 quick 清单。
|
||||
|
||||
### N2 · 磁盘 ≥70% 后备份静默停摆
|
||||
|
||||
`run-production-recovery.sh` 约 149–153 行磁盘 ≥70% 拒绝；`backup-production-daily.yml` 约 145–146 行只在备份成功后才清理旧文件 → 一旦到 70%，旧文件永不清理、每天都失败；磁盘告警 80% 才响，也没人检查 `daily-backup-latest.env` 有多旧。
|
||||
- 修：清理放在备份**之前**（或 `if: always()`）；`healthcheck-alert.sh` 增加「最近一次每日备份超过 36 小时」告警（读 `daily-backup-latest.env` 的 `verified_at`）。加测试。
|
||||
|
||||
### N3 · 两个监控脚本在生产主机上大概率跑不起来
|
||||
|
||||
`deploy/monitor/error-digest.sh` 约 54–59 行、`deploy/find-request-log.sh` 约 300–307 行直接调 `docker`，其它生产脚本与恢复演练都用 `sudo -n docker`；也没导出 `DATABASE_ENV_FILE`，`docker-compose.postgres.yml` 会回落到不存在的 `../.env.staging.database`，Compose 拒绝加载。错误汇总由 cron 每 5 分钟跑，会每次失败、什么都不发。
|
||||
- 修：统一 `sudo -n docker`，显式传入生产的 compose 文件与 env 文件路径（与 `run-production-deploy.sh` 同一写法）；测试断言命令行形状。
|
||||
|
||||
## 2. 顺手修（不阻断，一并做）
|
||||
|
||||
- `notify-feishu.sh` 约 96–101 行：`curl` 加 `--fail`，并检查飞书返回体里的 `code == 0`；失败时不记成「已发送」。
|
||||
- `error-digest.sh`：同一类告警 30 分钟内不重复发。
|
||||
- `find-request-log.sh`：真实路径按时间戳排序（去掉 compose 前缀后再排），与 README 说法一致。
|
||||
|
||||
## 3. 验收标准
|
||||
|
||||
- Linux：`python3 -m pytest tests/test_beta_safety_net.py -q` 全绿；Claude 全量 pytest、前端全量无新增；`bash -n` 全过；quick 门禁包含新测试。
|
||||
- 交回前本机跑上面那条 pytest（Windows 上请用 WSL 或在测试里显式断言 0600 已生效）。
|
||||
|
||||
## 4. 让步顺序
|
||||
|
||||
B1、N1、N3 不可让；N2 的「过旧告警」可写理由后让；第 2 节可让。
|
||||
@@ -0,0 +1,18 @@
|
||||
# 生产恢复演练结果（2026-10-14）
|
||||
|
||||
内测开放前由产品做一次。步骤见 `docs/operations/production-restore-drill-20261014.md`。不要填写姓名、出生资料、邮箱、会话或密钥。
|
||||
|
||||
| 项 | 结果 |
|
||||
| --- | --- |
|
||||
| 日期 | |
|
||||
| 用的备份文件名（只要 `production-daily-` 开头的文件名） | |
|
||||
| 还原目标 | 一次性库 / 整库（圈一个） |
|
||||
| 一次性库是否已删除 | |
|
||||
| identity.users 行数 | |
|
||||
| public.profiles 行数 | |
|
||||
| public.credit_transactions 行数 | |
|
||||
| public 表张数 | |
|
||||
| 网站是否停过 | 一次性库应为否 |
|
||||
| 停写大约多久 | 一次性库填「未停」 |
|
||||
| 是否通过 | |
|
||||
| 备注（不要写用户内容） | |
|
||||
@@ -121,6 +121,9 @@ CORE_PYTEST_TARGETS = [
|
||||
"tests/test_upstream_import_plan.py",
|
||||
# Tracked-file privacy must run in staging's explicit quick test list.
|
||||
"tests/test_repo_privacy_markers.py",
|
||||
# Closed-beta safety net. The Linux 0600 monitor-env check only fails in
|
||||
# release-quality-gate unless this file is on the staging quick list.
|
||||
"tests/test_beta_safety_net.py",
|
||||
# Summary lines by default; failure keeps the last 200 lines and a full log (BUG-1230).
|
||||
"tests/test_run_quality_gate_output.py",
|
||||
"tests/test_upstream_git_import_b9a0ef8f.py",
|
||||
|
||||
@@ -0,0 +1,999 @@
|
||||
"""Contracts for the closed-beta safety net. Fixtures are fictional."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "backup-production-daily.yml"
|
||||
RECOVERY_WORKFLOW = ROOT / ".gitea" / "workflows" / "create-production-recovery.yml"
|
||||
RECOVERY_SCRIPT = ROOT / "deploy" / "run-production-recovery.sh"
|
||||
PRUNE_SCRIPT = ROOT / "deploy" / "prune-production-daily-backups.sh"
|
||||
HEALTH_SCRIPT = ROOT / "deploy" / "monitor" / "healthcheck-alert.sh"
|
||||
DIGEST_SCRIPT = ROOT / "deploy" / "monitor" / "error-digest.sh"
|
||||
FIND_SCRIPT = ROOT / "deploy" / "find-request-log.sh"
|
||||
NOTIFY_SCRIPT = ROOT / "deploy" / "monitor" / "notify-feishu.sh"
|
||||
SERVER_COMPOSE = ROOT / "deploy" / "docker-compose.server.yml"
|
||||
POSTGRES_COMPOSE = ROOT / "deploy" / "docker-compose.postgres.yml"
|
||||
|
||||
|
||||
def resolve_bash() -> str:
|
||||
found = shutil.which("bash")
|
||||
git_bash = Path(r"D:\Program Files\Git\bin\bash.exe")
|
||||
if os.name == "nt" and git_bash.exists():
|
||||
lowered = (found or "").lower()
|
||||
if not found or "system32" in lowered or "windowsapps" in lowered:
|
||||
return str(git_bash)
|
||||
if found:
|
||||
return found
|
||||
if git_bash.exists():
|
||||
return str(git_bash)
|
||||
return "bash"
|
||||
|
||||
|
||||
BASH = resolve_bash()
|
||||
|
||||
|
||||
def bash_path(path: Path) -> str:
|
||||
if os.name != "nt":
|
||||
return str(path)
|
||||
completed = subprocess.run(
|
||||
[BASH, "-lc", 'cygpath -u "$1"', "cygpath", str(path)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def run_bash(script: Path, extra: dict[str, str], args: list[str] | None = None) -> subprocess.CompletedProcess[str]:
|
||||
env = os.environ.copy()
|
||||
env.pop("FEISHU_WEBHOOK_URL", None)
|
||||
env.pop("FEISHU_SIGN_SECRET", None)
|
||||
env.update(extra)
|
||||
env["JYOTISHA_PYTHON"] = bash_path(Path(sys.executable))
|
||||
return subprocess.run(
|
||||
[BASH, bash_path(script), *(args or [])],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def write_backup_record(directory: Path, verified_at: str) -> Path:
|
||||
record = directory / "daily-backup-latest.env"
|
||||
write_text(
|
||||
record,
|
||||
"BACKUP_BASENAME=production-daily-20261015T010000Z.dump.enc\n"
|
||||
f"BACKUP_SHA256={'a' * 64}\n"
|
||||
f"VERIFIED_AT={verified_at}\n"
|
||||
"BACKUP_FORMAT=2\n",
|
||||
)
|
||||
return record
|
||||
|
||||
|
||||
def run_disk_mode(bin_dir: Path, script_env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
|
||||
cwd=ROOT,
|
||||
env={**os.environ, **script_env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
|
||||
|
||||
class HealthHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
if self.path != "/health":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
kind = getattr(self.server, "kind", "ok")
|
||||
if kind == "timeout":
|
||||
time.sleep(3)
|
||||
commit = getattr(self.server, "commit", "a" * 40)
|
||||
if kind == "500":
|
||||
status, body = 500, b'{"status":"ok"}'
|
||||
elif kind == "missing":
|
||||
status, body = 200, b"{}"
|
||||
elif kind == "degraded":
|
||||
status, body = 200, json.dumps({"status": "degraded", "deployment": {"gitCommit": commit}}).encode()
|
||||
else:
|
||||
status, body = 200, json.dumps({"status": "ok", "deployment": {"gitCommit": commit}}).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
class HookHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
body = self.rfile.read(length)
|
||||
self.server.bodies.append(body)
|
||||
self.close_connection = True
|
||||
code = int(getattr(self.server, "feishu_code", 0))
|
||||
reply = json.dumps({"code": code}).encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(reply)))
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.wfile.write(reply)
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
def serve(handler: type[BaseHTTPRequestHandler]) -> tuple[ThreadingHTTPServer, threading.Thread]:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
return server, thread
|
||||
|
||||
|
||||
def alert_lines(state: Path) -> list[str]:
|
||||
log = state / "alerts.log"
|
||||
if not log.exists():
|
||||
return []
|
||||
return [line for line in log.read_text(encoding="utf-8").splitlines() if line]
|
||||
|
||||
|
||||
def test_daily_workflow_is_scheduled_from_main_and_reuses_recovery() -> None:
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
assert "\tif:" not in text
|
||||
assert "if: gitea.ref == 'refs/heads/main'" in text
|
||||
assert 'cron: "0 20 * * *"' in text
|
||||
assert "workflow_dispatch:" in text
|
||||
assert 'git show "$GITEA_SHA:deploy/run-production-recovery.sh"' in text
|
||||
assert "RECOVERY_BASENAME_PREFIX='production-daily'" in text
|
||||
assert "prune-production-daily-backups.sh" in text
|
||||
prune_call = text.index("bash '$incoming/prune-production-daily-backups.sh'")
|
||||
recovery_call = text.index("bash '$incoming/run-production-recovery.sh'")
|
||||
assert prune_call < recovery_call
|
||||
assert "production-daily-" in text
|
||||
assert 'artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"' in text
|
||||
assert 'local_retention="${LOCAL_RETENTION_DAYS:-7}"' in text
|
||||
assert 'process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS' in text
|
||||
assert "set +x" in text
|
||||
assert "pg_dump" not in text
|
||||
assert "CHECK_RELEASE_GATE" not in text
|
||||
assert "observed_staging_sha" not in text
|
||||
assert "STAGING_URL" not in text
|
||||
assert "echo \"$SSH_PRIVATE_KEY" not in text
|
||||
assert "PRODUCTION_BACKUP_ENCRYPTION_KEY" not in text
|
||||
recovery = RECOVERY_WORKFLOW.read_text(encoding="utf-8")
|
||||
assert "observed_staging_sha" in recovery
|
||||
assert 'process.env["INPUT_RETENTION-DAYS"]="30"' in recovery
|
||||
|
||||
|
||||
def test_recovery_script_keeps_manual_names_unless_daily_prefix() -> None:
|
||||
text = RECOVERY_SCRIPT.read_text(encoding="utf-8")
|
||||
assert 'backup_prefix="production-pre-migration"' in text
|
||||
assert 'production-daily) backup_prefix="production-daily"' in text
|
||||
assert "openssl enc -aes-256-cbc -salt -pbkdf2" in text
|
||||
assert "usage_percent < 70" in text
|
||||
assert "flock -n 9" in text
|
||||
assert '[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ]' in text
|
||||
assert 'echo "baseline_updated=false" >&2' in text
|
||||
assert 'echo "baseline_updated=true" >&2' in text
|
||||
assert "daily-backup-latest.env" in text
|
||||
|
||||
|
||||
def install_fake_recovery_bin(bin_dir: Path) -> None:
|
||||
bin_dir.mkdir()
|
||||
write_text(
|
||||
bin_dir / "sudo",
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nif [[ \"${1:-}\" == \"-n\" ]]; then shift; fi\nexec \"$@\"\n",
|
||||
)
|
||||
write_text(
|
||||
bin_dir / "df",
|
||||
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' 'tmpfs 100 10 90 10% /'\n",
|
||||
)
|
||||
# Windows cannot chmod pytest's temp directories. These stand-ins only
|
||||
# exist on the test PATH. The production script still calls the real commands.
|
||||
write_text(
|
||||
bin_dir / "chmod",
|
||||
"#!/usr/bin/env bash\nexit 0\n",
|
||||
)
|
||||
# Git Bash on this Windows machine has no flock. The production script still calls it.
|
||||
write_text(
|
||||
bin_dir / "flock",
|
||||
"#!/usr/bin/env bash\nexit 0\n",
|
||||
)
|
||||
write_text(
|
||||
bin_dir / "install",
|
||||
"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
dirs=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-d) shift ;;
|
||||
-m) shift; shift ;;
|
||||
--) shift; break ;;
|
||||
-*) shift ;;
|
||||
*) dirs+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
mkdir -p "${dirs[@]}"
|
||||
""",
|
||||
)
|
||||
write_text(
|
||||
bin_dir / "docker",
|
||||
"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cmd="${1:-}"
|
||||
shift || true
|
||||
case "$cmd" in
|
||||
ps)
|
||||
printf '%s\\n' fixturecontainer
|
||||
;;
|
||||
inspect)
|
||||
printf '%s\\n' sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||
;;
|
||||
run)
|
||||
exit 0
|
||||
;;
|
||||
exec)
|
||||
tool=""
|
||||
sql=""
|
||||
prev=""
|
||||
for arg in "$@"; do
|
||||
if [[ "$prev" == "-c" ]]; then
|
||||
sql="$arg"
|
||||
fi
|
||||
case "$arg" in
|
||||
pg_dump|pg_restore|psql) tool="$arg" ;;
|
||||
esac
|
||||
prev="$arg"
|
||||
done
|
||||
case "$tool" in
|
||||
pg_dump)
|
||||
printf '%s\\n' fixture-dump
|
||||
;;
|
||||
pg_restore)
|
||||
cat >/dev/null
|
||||
;;
|
||||
psql)
|
||||
case "$sql" in
|
||||
*role_table_grants*) printf '%s\\n' 'reader|SELECT|profiles' ;;
|
||||
*column_privileges*) printf '%s\\n' 'reader|SELECT|profiles|id' ;;
|
||||
*pg_get_userbyid*) printf '%s\\n' 'profiles|app' ;;
|
||||
*identity.users*) printf '%s\\n' '1|2|3|4' ;;
|
||||
*"CREATE DATABASE"*|*"DROP DATABASE"*) ;;
|
||||
*)
|
||||
printf 'unexpected psql\\n' >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected docker exec\\n' >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected docker %s\\n' "$cmd" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
subprocess.run(
|
||||
[BASH, "-lc", 'chmod +x "$1/sudo" "$1/df" "$1/docker" "$1/chmod" "$1/install" "$1/flock"', "chmod", bash_path(bin_dir)],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def write_recovery_env(deploy: Path) -> None:
|
||||
write_text(
|
||||
deploy / ".env.production.database",
|
||||
"\n".join(
|
||||
[
|
||||
"POSTGRES_DB=fixturedatabase",
|
||||
"POSTGRES_USER=fixtureuser",
|
||||
"POSTGRES_PASSWORD=fixturepassword",
|
||||
"PRODUCTION_BACKUP_ENCRYPTION_KEY=fixtureencryptionkey",
|
||||
"",
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def run_recovery(deploy: Path, bin_dir: Path, prefix: str | None) -> subprocess.CompletedProcess[str]:
|
||||
env = os.environ.copy()
|
||||
env.pop("FEISHU_WEBHOOK_URL", None)
|
||||
env.pop("FEISHU_SIGN_SECRET", None)
|
||||
env.pop("RECOVERY_BASENAME_PREFIX", None)
|
||||
env["DEPLOY_PATH"] = bash_path(deploy)
|
||||
env["RECOVERY_RUN_ID"] = "4242"
|
||||
env["JYOTISHA_RECOVERY_FIXTURE"] = "1"
|
||||
if prefix:
|
||||
env["RECOVERY_BASENAME_PREFIX"] = prefix
|
||||
return subprocess.run(
|
||||
[
|
||||
BASH,
|
||||
"-lc",
|
||||
'export PATH="$1:$PATH"; exec bash "$2"',
|
||||
"bash",
|
||||
bash_path(bin_dir),
|
||||
bash_path(RECOVERY_SCRIPT),
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
|
||||
|
||||
def parse_env(text: str) -> dict[str, str]:
|
||||
parsed: dict[str, str] = {}
|
||||
for line in text.splitlines():
|
||||
key, value = line.split("=", 1)
|
||||
parsed[key] = value
|
||||
return parsed
|
||||
|
||||
|
||||
def test_daily_backup_does_not_replace_the_recovery_point(tmp_path: Path) -> None:
|
||||
deploy = tmp_path / "deploy"
|
||||
state = deploy / ".state"
|
||||
state.mkdir(parents=True)
|
||||
write_recovery_env(deploy)
|
||||
baseline = state / "recovery-baseline.env"
|
||||
original = (
|
||||
"RECOVERY_REFERENCE=gitea-actions-run-1/production-pre-migration-20261014T120000Z.dump.enc\n"
|
||||
"RECOVERY_CREATED_AT=2026-10-14T12:00:00Z\n"
|
||||
"RECOVERY_SHA256=" + ("ab" * 32) + "\n"
|
||||
"RESTORE_VERIFIED=true\n"
|
||||
"RECOVERY_FORMAT=2\n"
|
||||
"VERIFY_FILE=production-pre-migration-20261014T120000Z-restore-verify.json\n"
|
||||
)
|
||||
write_text(baseline, original)
|
||||
subprocess.run(
|
||||
[BASH, "-lc", 'touch -d "@$1" "$2"', "touch", "1700000000", bash_path(baseline)],
|
||||
check=True,
|
||||
)
|
||||
before_ns = baseline.stat().st_mtime_ns
|
||||
before_bytes = baseline.read_bytes()
|
||||
bin_dir = tmp_path / "bin"
|
||||
install_fake_recovery_bin(bin_dir)
|
||||
completed = run_recovery(deploy, bin_dir, "production-daily")
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert baseline.read_bytes() == before_bytes
|
||||
assert baseline.stat().st_mtime_ns == before_ns
|
||||
assert "baseline_updated" not in completed.stdout
|
||||
assert "baseline_updated=false" in completed.stderr.splitlines()
|
||||
assert "baseline_updated=true" not in completed.stderr
|
||||
assert len(completed.stdout.splitlines()) == 7
|
||||
stdout_env = parse_env(completed.stdout)
|
||||
assert stdout_env["BACKUP_BASENAME"].startswith("production-daily-")
|
||||
daily = state / "daily-backup-latest.env"
|
||||
daily_env = parse_env(daily.read_text(encoding="utf-8"))
|
||||
assert set(daily_env) == {"BACKUP_BASENAME", "BACKUP_SHA256", "VERIFIED_AT", "BACKUP_FORMAT"}
|
||||
assert daily_env["BACKUP_FORMAT"] == "2"
|
||||
assert daily_env["BACKUP_BASENAME"] == stdout_env["BACKUP_BASENAME"]
|
||||
assert daily_env["BACKUP_SHA256"] == stdout_env["RECOVERY_SHA256"]
|
||||
daily_text = daily.read_text(encoding="utf-8")
|
||||
assert "fixturepassword" not in daily_text
|
||||
assert "fixtureencryptionkey" not in daily_text
|
||||
assert "fixturedatabase" not in daily_text
|
||||
verify = deploy / "backups" / stdout_env["VERIFY_BASENAME"]
|
||||
payload = json.loads(verify.read_text(encoding="utf-8"))
|
||||
assert payload["checks"]["table_grants_match"] is True
|
||||
assert payload["checks"]["column_privileges_match"] is True
|
||||
assert payload["checks"]["table_owners_match"] is True
|
||||
assert "fixturepassword" not in verify.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_manual_recovery_still_writes_format_two(tmp_path: Path) -> None:
|
||||
deploy = tmp_path / "deploy"
|
||||
state = deploy / ".state"
|
||||
state.mkdir(parents=True)
|
||||
write_recovery_env(deploy)
|
||||
sentinel = state / "daily-backup-latest.env"
|
||||
write_text(sentinel, "BACKUP_BASENAME=keep\nBACKUP_FORMAT=2\n")
|
||||
before = sentinel.read_bytes()
|
||||
bin_dir = tmp_path / "bin"
|
||||
install_fake_recovery_bin(bin_dir)
|
||||
completed = run_recovery(deploy, bin_dir, None)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert "baseline_updated=true" in completed.stderr.splitlines()
|
||||
assert "baseline_updated=false" not in completed.stderr
|
||||
assert len(completed.stdout.splitlines()) == 7
|
||||
baseline_text = (state / "recovery-baseline.env").read_text(encoding="utf-8")
|
||||
baseline = parse_env(baseline_text)
|
||||
assert baseline["RECOVERY_FORMAT"] == "2"
|
||||
assert baseline["VERIFY_FILE"].startswith("production-pre-migration-")
|
||||
assert baseline["RECOVERY_REFERENCE"].startswith("gitea-actions-run-4242/production-pre-migration-")
|
||||
assert "fixturepassword" not in baseline_text
|
||||
assert "fixtureencryptionkey" not in baseline_text
|
||||
assert parse_env(completed.stdout)["BACKUP_BASENAME"].startswith("production-pre-migration-")
|
||||
assert sentinel.read_bytes() == before
|
||||
|
||||
|
||||
def test_release_migration_and_rollback_ignore_daily_backup_record() -> None:
|
||||
readers = [
|
||||
ROOT / ".gitea" / "workflows" / "deploy-production.yml",
|
||||
ROOT / ".gitea" / "workflows" / "migrate-production-database.yml",
|
||||
ROOT / "deploy" / "run-production-deploy.sh",
|
||||
ROOT / "deploy" / "run-production-migration.sh",
|
||||
]
|
||||
for path in readers:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert "daily-backup-latest.env" not in text, path.name
|
||||
assert "recovery-baseline.env" in text
|
||||
for path in (WORKFLOW, RECOVERY_WORKFLOW):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert "JYOTISHA_RECOVERY_FIXTURE" not in text
|
||||
assert '[[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d \' \')" == 7 ]]' in text
|
||||
|
||||
|
||||
def test_prune_deletes_only_expired_daily_files(tmp_path: Path) -> None:
|
||||
backup = tmp_path / "backups"
|
||||
nested = backup / "nested"
|
||||
nested.mkdir(parents=True)
|
||||
now = 1_700_000_000
|
||||
limit = 7 * 86400
|
||||
names = {
|
||||
"production-daily-20261001T000000Z.dump.enc": now - limit - 60,
|
||||
"production-daily-20261001T000000Z-restore-verify.json": now - limit - 60,
|
||||
"production-daily-20261007T000000Z.dump.enc": now - limit,
|
||||
"production-daily-20261007T000000Z-restore-verify.json": now - limit,
|
||||
"production-daily-20261014T000000Z.dump.enc": now - 3600,
|
||||
"production-daily-20261014T000000Z-restore-verify.json": now - 3600,
|
||||
"production-pre-migration-20261001T000000Z.dump.enc": now - limit - 3600,
|
||||
"production-pre-migration-20261001T000000Z-restore-verify.json": now - limit - 3600,
|
||||
"production-daily-notes.txt": now - limit - 3600,
|
||||
"production-daily-20261001T000000Z.dump.enc.bak": now - limit - 3600,
|
||||
"unrelated.txt": now - limit - 3600,
|
||||
}
|
||||
for name, epoch in names.items():
|
||||
path = backup / name
|
||||
write_text(path, "fictional\n")
|
||||
subprocess.run(
|
||||
[BASH, "-lc", 'touch -d "@$1" "$2"', "touch", str(epoch), bash_path(path)],
|
||||
check=True,
|
||||
)
|
||||
write_text(nested / "production-daily-20261001T000000Z.dump.enc", "nested\n")
|
||||
completed = run_bash(PRUNE_SCRIPT, {}, [bash_path(backup), "7", str(now)])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
deleted = completed.stdout.splitlines()
|
||||
assert deleted == ["production-daily-20261001T000000Z.dump.enc"]
|
||||
assert not (backup / "production-daily-20261001T000000Z.dump.enc").exists()
|
||||
assert not (backup / "production-daily-20261001T000000Z-restore-verify.json").exists()
|
||||
for name in names:
|
||||
if name.startswith("production-daily-20261001T000000Z.dump") or name.startswith(
|
||||
"production-daily-20261001T000000Z-restore"
|
||||
):
|
||||
continue
|
||||
assert (backup / name).exists(), name
|
||||
assert (nested / "production-daily-20261001T000000Z.dump.enc").exists()
|
||||
|
||||
|
||||
def test_health_alerts_after_three_failures_and_recovery_once(tmp_path: Path) -> None:
|
||||
server, _thread = serve(HealthHandler)
|
||||
server.kind = "500"
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
for _ in range(2):
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert alert_lines(state) == []
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "errorCode=health_down" in lines[0]
|
||||
assert "count=3" in lines[0]
|
||||
assert "route=/api/health" in lines[0]
|
||||
server.kind = "ok"
|
||||
server.commit = "b" * 40
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 2
|
||||
assert "errorCode=recovered" in lines[1]
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert len(alert_lines(state)) == 2
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_health_timeout_and_missing_fields_count_as_failures(tmp_path: Path) -> None:
|
||||
server, _thread = serve(HealthHandler)
|
||||
server.kind = "timeout"
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "1",
|
||||
}
|
||||
try:
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert alert_lines(state) == []
|
||||
server.kind = "missing"
|
||||
run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert any("errorCode=health_down" in line for line in alert_lines(state))
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_commit_change_alerts_once_outside_release_window(tmp_path: Path) -> None:
|
||||
server, _thread = serve(HealthHandler)
|
||||
server.kind = "ok"
|
||||
server.commit = "a" * 40
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert alert_lines(state) == []
|
||||
server.commit = "b" * 40
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "errorCode=commit_changed" in lines[0]
|
||||
assert "b" * 40 not in lines[0]
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert len(alert_lines(state)) == 1
|
||||
marker = state / "release-window"
|
||||
state.mkdir(parents=True, exist_ok=True)
|
||||
write_text(marker, "open\n")
|
||||
server.commit = "c" * 40
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert len(alert_lines(state)) == 1
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_disk_alerts_once_and_no_webhook_stays_local(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
write_text(
|
||||
bin_dir / "df",
|
||||
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 90 10 ${FAKE_DISK_PERCENT:?}% /\"\n",
|
||||
)
|
||||
subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True)
|
||||
state = tmp_path / "state"
|
||||
record = write_backup_record(tmp_path, "2026-10-15T01:00:00Z")
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_DAILY_BACKUP_RECORD": bash_path(record),
|
||||
"JYOTISHA_NOW": "2026-10-15T01:00:00Z",
|
||||
"FAKE_DISK_PERCENT": "81",
|
||||
}
|
||||
env.pop("FEISHU_WEBHOOK_URL", None)
|
||||
env.pop("FEISHU_SIGN_SECRET", None)
|
||||
completed = run_disk_mode(bin_dir, env)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert "errorCode=disk_high" in alert_lines(state)[0]
|
||||
assert "count=81" in alert_lines(state)[0]
|
||||
env["FAKE_DISK_PERCENT"] = "81"
|
||||
again = run_disk_mode(bin_dir, env)
|
||||
assert again.returncode == 0, again.stderr
|
||||
assert len(alert_lines(state)) == 1
|
||||
env["FAKE_DISK_PERCENT"] = "10"
|
||||
recovered = run_disk_mode(bin_dir, env)
|
||||
assert recovered.returncode == 0, recovered.stderr
|
||||
assert len(alert_lines(state)) == 2
|
||||
assert "errorCode=recovered" in alert_lines(state)[1]
|
||||
|
||||
|
||||
def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None:
|
||||
hook, _thread = serve(HookHandler)
|
||||
hook.bodies = []
|
||||
health, _thread = serve(HealthHandler)
|
||||
health.kind = "500"
|
||||
state = tmp_path / "state"
|
||||
secret = "fixture-sign-secret"
|
||||
env_file = tmp_path / "monitor.env"
|
||||
write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\nFEISHU_SIGN_SECRET={secret}\n")
|
||||
os.chmod(env_file, 0o600)
|
||||
if os.name != "nt":
|
||||
assert (env_file.stat().st_mode & 0o777) == 0o600
|
||||
notify = NOTIFY_SCRIPT.read_text(encoding="utf-8")
|
||||
assert '[ "$mode" = "600" ]' in notify
|
||||
assert "curl --silent --show-error --fail" in notify
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(env_file),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
for _ in range(3):
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert len(hook.bodies) == 1
|
||||
payload = json.loads(hook.bodies[0].decode("utf-8"))
|
||||
assert set(payload) == {"timestamp", "sign", "msg_type", "content"}
|
||||
assert payload["msg_type"] == "text"
|
||||
text = payload["content"]["text"]
|
||||
assert secret not in text
|
||||
assert secret not in hook.bodies[0].decode("utf-8")
|
||||
assert "errorCode=health_down" in text
|
||||
assert "@" not in text
|
||||
expected = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import base64,hashlib,hmac,os,sys; ts=sys.argv[1]; secret=sys.argv[2]; key=f'{ts}\\n{secret}'.encode(); print(base64.b64encode(hmac.new(key, digestmod=hashlib.sha256).digest()).decode())",
|
||||
payload["timestamp"],
|
||||
secret,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
assert payload["sign"] == expected
|
||||
finally:
|
||||
hook.shutdown()
|
||||
health.shutdown()
|
||||
|
||||
|
||||
def test_feishu_rejection_is_not_recorded_as_sent(tmp_path: Path) -> None:
|
||||
hook, _thread = serve(HookHandler)
|
||||
hook.bodies = []
|
||||
hook.feishu_code = 19001
|
||||
health, _thread = serve(HealthHandler)
|
||||
health.kind = "500"
|
||||
state = tmp_path / "state"
|
||||
env_file = tmp_path / "monitor.env"
|
||||
write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\n")
|
||||
os.chmod(env_file, 0o600)
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(env_file),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
rejected = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert rejected.returncode != 0
|
||||
assert len(hook.bodies) == 1
|
||||
assert alert_lines(state) == []
|
||||
assert "health_down" not in rejected.stdout
|
||||
finally:
|
||||
hook.shutdown()
|
||||
health.shutdown()
|
||||
|
||||
|
||||
def uuid(index: int) -> str:
|
||||
return f"00000000-0000-4000-8000-{index:012d}"
|
||||
|
||||
|
||||
def test_error_digest_drops_personal_text_and_respects_thresholds(tmp_path: Path) -> None:
|
||||
safe = [
|
||||
f"2026-10-15T00:00:{index:02d}Z web [agent-observability] {json.dumps({'requestId': uuid(index), 'errorCode': 'settlement_failed'})}"
|
||||
for index in range(1, 11)
|
||||
]
|
||||
unsafe = [
|
||||
"2026-10-15T00:01:00Z web person@example.com errorCode=should_drop",
|
||||
"2026-10-15T00:01:01Z web note 1900-01-01 birth_date",
|
||||
"2026-10-15T00:01:02Z web 对话内容 今天工作不顺",
|
||||
]
|
||||
fixture = tmp_path / "logs.txt"
|
||||
write_text(fixture, "\n".join([*safe, *unsafe]) + "\n")
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_LOG_FIXTURE": bash_path(fixture),
|
||||
"JYOTISHA_NOW": "2026-10-15T01:00:00Z",
|
||||
}
|
||||
below = tmp_path / "below.txt"
|
||||
write_text(below, "\n".join(safe[:9]) + "\n")
|
||||
quiet = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_LOG_FIXTURE": bash_path(below)}, ["--window"])
|
||||
assert quiet.returncode == 0, quiet.stderr
|
||||
assert alert_lines(state) == []
|
||||
completed = run_bash(DIGEST_SCRIPT, env, ["--window"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "errorCode=settlement_failed" in lines[0]
|
||||
assert "count=10" in lines[0]
|
||||
assert lines[0].count(",") == 4
|
||||
assert uuid(6) not in lines[0]
|
||||
assert "person@example.com" not in lines[0]
|
||||
assert "1900-01-01" not in lines[0]
|
||||
assert "今天工作不顺" not in lines[0]
|
||||
assert "should_drop" not in lines[0]
|
||||
soon = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_NOW": "2026-10-15T01:29:59Z"}, ["--window"])
|
||||
assert soon.returncode == 0, soon.stderr
|
||||
assert len(alert_lines(state)) == 1
|
||||
later = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_NOW": "2026-10-15T01:30:00Z"}, ["--window"])
|
||||
assert later.returncode == 0, later.stderr
|
||||
assert len(alert_lines(state)) == 2
|
||||
assert "errorCode=settlement_failed" in alert_lines(state)[1]
|
||||
|
||||
fivexx = tmp_path / "fivexx.txt"
|
||||
write_text(fivexx, "\n".join(f"2026-10-15T00:02:{index:02d}Z api status=502" for index in range(20)) + "\n")
|
||||
state_5xx = tmp_path / "state5"
|
||||
completed = run_bash(
|
||||
DIGEST_SCRIPT,
|
||||
{**env, "JYOTISHA_MONITOR_STATE_DIR": bash_path(state_5xx), "JYOTISHA_LOG_FIXTURE": bash_path(fivexx)},
|
||||
["--window"],
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
joined = "\n".join(alert_lines(state_5xx))
|
||||
assert "errorCode=http_5xx" in joined
|
||||
assert "count=20" in joined
|
||||
|
||||
|
||||
def test_daily_digest_summarizes_previous_shanghai_day(tmp_path: Path) -> None:
|
||||
fixture = tmp_path / "daily.txt"
|
||||
write_text(
|
||||
fixture,
|
||||
"\n".join(
|
||||
[
|
||||
'2026-10-14T02:00:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000001","errorCode":"settlement_failed"}',
|
||||
'2026-10-15T01:30:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000002","errorCode":"settlement_failed"}',
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
state = tmp_path / "state"
|
||||
completed = run_bash(
|
||||
DIGEST_SCRIPT,
|
||||
{
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_LOG_FIXTURE": bash_path(fixture),
|
||||
"JYOTISHA_NOW": "2026-10-15T01:00:00Z",
|
||||
},
|
||||
["--daily"],
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "count=1" in lines[0]
|
||||
assert "00000000-0000-4000-8000-000000000001" in lines[0]
|
||||
assert "00000000-0000-4000-8000-000000000002" not in lines[0]
|
||||
|
||||
|
||||
def test_find_request_log_sorts_and_rejects_injection(tmp_path: Path) -> None:
|
||||
log_dir = tmp_path / "logs"
|
||||
log_dir.mkdir()
|
||||
request_id = "11111111-2222-4333-8444-555555555555"
|
||||
write_text(log_dir / "caddy.log", f"2026-10-15T03:00:00Z caddy {request_id}\n")
|
||||
write_text(log_dir / "web.log", f"2026-10-15T01:00:00Z web {request_id}\nignore\n")
|
||||
write_text(log_dir / "api.log", f"2026-10-15T02:00:00Z api {request_id}\n")
|
||||
canary = tmp_path / "canary.txt"
|
||||
write_text(canary, "keep\n")
|
||||
found = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h"])
|
||||
assert found.returncode == 0, found.stderr
|
||||
assert found.stdout.splitlines() == [
|
||||
f"2026-10-15T01:00:00Z web {request_id}",
|
||||
f"2026-10-15T02:00:00Z api {request_id}",
|
||||
f"2026-10-15T03:00:00Z caddy {request_id}",
|
||||
]
|
||||
rejected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["; rm -rf /", "--since", "24h"])
|
||||
assert rejected.returncode == 2
|
||||
assert canary.read_text(encoding="utf-8") == "keep\n"
|
||||
injected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["$(rm canary.txt)", "--since", "1h"])
|
||||
assert injected.returncode == 2
|
||||
assert canary.exists()
|
||||
bad_since = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h;rm"])
|
||||
assert bad_since.returncode == 2
|
||||
compose_log = tmp_path / "compose.log"
|
||||
write_text(
|
||||
compose_log,
|
||||
"\n".join(
|
||||
[
|
||||
f"web-1 | 2026-10-15T01:00:00Z web {request_id}",
|
||||
f"api-1 | 2026-10-15T03:00:00Z api {request_id}",
|
||||
f"caddy-1 | 2026-10-15T02:00:00Z caddy {request_id}",
|
||||
f"api-1 | ignore {request_id}",
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
composed = run_bash(
|
||||
FIND_SCRIPT,
|
||||
{"JYOTISHA_COMPOSE_LOG_FIXTURE": bash_path(compose_log)},
|
||||
[request_id, "--since", "24h"],
|
||||
)
|
||||
assert composed.returncode == 0, composed.stderr
|
||||
assert composed.stdout.splitlines() == [
|
||||
f"2026-10-15T01:00:00Z web {request_id}",
|
||||
f"2026-10-15T02:00:00Z caddy {request_id}",
|
||||
f"2026-10-15T03:00:00Z api {request_id}",
|
||||
]
|
||||
|
||||
|
||||
def service_logging_blocks(text: str) -> dict[str, str]:
|
||||
blocks: dict[str, str] = {}
|
||||
service: str | None = None
|
||||
in_services = False
|
||||
capturing = False
|
||||
buf: list[str] = []
|
||||
for line in text.splitlines():
|
||||
if line.startswith("services:"):
|
||||
in_services = True
|
||||
continue
|
||||
if in_services and line and not line.startswith(" "):
|
||||
in_services = False
|
||||
if not in_services:
|
||||
continue
|
||||
if line.startswith(" ") and not line.startswith(" ") and line.endswith(":"):
|
||||
if capturing and service is not None:
|
||||
blocks[service] = "\n".join(buf)
|
||||
service = line.strip()[:-1]
|
||||
capturing = False
|
||||
buf = []
|
||||
continue
|
||||
if service and line == " logging:":
|
||||
capturing = True
|
||||
buf = [line]
|
||||
continue
|
||||
if capturing:
|
||||
if line.startswith(" ") and not line.startswith(" "):
|
||||
blocks[service] = "\n".join(buf)
|
||||
capturing = False
|
||||
buf = []
|
||||
else:
|
||||
buf.append(line)
|
||||
if capturing and service is not None:
|
||||
blocks[service] = "\n".join(buf)
|
||||
return blocks
|
||||
|
||||
|
||||
def test_compose_logging_is_present_and_identical() -> None:
|
||||
expected = "\n".join(
|
||||
[
|
||||
" logging:",
|
||||
" driver: json-file",
|
||||
" options:",
|
||||
' max-size: "20m"',
|
||||
' max-file: "10"',
|
||||
]
|
||||
)
|
||||
expected_names = {
|
||||
SERVER_COMPOSE: {"api", "web", "caddy"},
|
||||
POSTGRES_COMPOSE: {"postgres", "migrator", "migration-checker"},
|
||||
}
|
||||
for path, names in expected_names.items():
|
||||
blocks = service_logging_blocks(path.read_text(encoding="utf-8"))
|
||||
assert set(blocks) == names
|
||||
for name in names:
|
||||
assert blocks[name] == expected, name
|
||||
|
||||
|
||||
def test_monitor_commands_use_production_docker() -> None:
|
||||
for script in (DIGEST_SCRIPT, FIND_SCRIPT):
|
||||
text = script.read_text(encoding="utf-8")
|
||||
assert "sudo -n docker compose -p jyotisha-production" in text
|
||||
assert "export DATABASE_ENV_FILE='../.env.production.database'" in text
|
||||
assert "export APP_ENV_FILE='../.env.production'" in text
|
||||
assert "docker-compose.server.yml" in text
|
||||
assert "docker-compose.postgres.yml" in text
|
||||
assert "docker-compose.production.yml" in text
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("docker ") or stripped.startswith("docker compose"):
|
||||
raise AssertionError(stripped)
|
||||
finder = FIND_SCRIPT.read_text(encoding="utf-8")
|
||||
strip_at = finder.index("sed -E 's/^[^|]*\\| //'")
|
||||
sort_at = finder.index("sort", strip_at)
|
||||
assert strip_at < sort_at
|
||||
|
||||
|
||||
def test_safety_net_is_on_the_quick_gate() -> None:
|
||||
text = (ROOT / "scripts" / "run_quality_gate.py").read_text(encoding="utf-8")
|
||||
assert '"tests/test_beta_safety_net.py"' in text
|
||||
|
||||
|
||||
def install_fake_df(bin_dir: Path) -> None:
|
||||
bin_dir.mkdir()
|
||||
write_text(
|
||||
bin_dir / "df",
|
||||
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 10 90 ${FAKE_DISK_PERCENT:?}% /\"\n",
|
||||
)
|
||||
subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True)
|
||||
|
||||
|
||||
def test_stale_daily_backup_alerts_once_until_a_fresh_record(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
install_fake_df(bin_dir)
|
||||
state = tmp_path / "state"
|
||||
now = "2026-10-15T12:00:00Z"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_NOW": now,
|
||||
"JYOTISHA_DAILY_BACKUP_RECORD": bash_path(tmp_path / "missing-record.env"),
|
||||
"FAKE_DISK_PERCENT": "10",
|
||||
}
|
||||
missing = run_disk_mode(bin_dir, env)
|
||||
assert missing.returncode == 0, missing.stderr
|
||||
assert alert_lines(state) == [
|
||||
line for line in alert_lines(state) if "errorCode=backup_stale" in line and "count=36" in line
|
||||
]
|
||||
assert len(alert_lines(state)) == 1
|
||||
assert "production-daily" not in alert_lines(state)[0]
|
||||
assert ".env" not in alert_lines(state)[0]
|
||||
again = run_disk_mode(bin_dir, env)
|
||||
assert again.returncode == 0, again.stderr
|
||||
assert len(alert_lines(state)) == 1
|
||||
|
||||
exact = tmp_path / "exact"
|
||||
exact.mkdir()
|
||||
env["JYOTISHA_MONITOR_STATE_DIR"] = bash_path(exact)
|
||||
env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(exact, "2026-10-14T00:00:00Z"))
|
||||
boundary = run_disk_mode(bin_dir, env)
|
||||
assert boundary.returncode == 0, boundary.stderr
|
||||
assert alert_lines(exact) == []
|
||||
|
||||
stale_state = tmp_path / "stale-state"
|
||||
old_dir = tmp_path / "old"
|
||||
old_dir.mkdir()
|
||||
env["JYOTISHA_MONITOR_STATE_DIR"] = bash_path(stale_state)
|
||||
env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(old_dir, "2026-10-13T23:00:00Z"))
|
||||
old = run_disk_mode(bin_dir, env)
|
||||
assert old.returncode == 0, old.stderr
|
||||
assert len(alert_lines(stale_state)) == 1
|
||||
assert "errorCode=backup_stale" in alert_lines(stale_state)[0]
|
||||
assert "count=37" in alert_lines(stale_state)[0]
|
||||
fresh_dir = tmp_path / "fresh"
|
||||
fresh_dir.mkdir()
|
||||
env["JYOTISHA_DAILY_BACKUP_RECORD"] = bash_path(write_backup_record(fresh_dir, now))
|
||||
recovered = run_disk_mode(bin_dir, env)
|
||||
assert recovered.returncode == 0, recovered.stderr
|
||||
assert len(alert_lines(stale_state)) == 2
|
||||
assert "errorCode=recovered" in alert_lines(stale_state)[1]
|
||||
|
||||
|
||||
def test_shell_scripts_pass_bash_n() -> None:
|
||||
for script in (PRUNE_SCRIPT, HEALTH_SCRIPT, DIGEST_SCRIPT, FIND_SCRIPT, NOTIFY_SCRIPT, RECOVERY_SCRIPT):
|
||||
completed = subprocess.run(
|
||||
[BASH, "-n", bash_path(script)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert completed.returncode == 0, script.name + completed.stderr
|
||||
Reference in New Issue
Block a user