给内测补上每日加密备份、宕机告警和日志上限。
主机安装和一次恢复演练留给产品。页面没有变化。 Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
co-authored by
Grok (agent loop)
Grok (agent loop)
parent
71f7fac3a1
commit
53d8df257d
@@ -0,0 +1,183 @@
|
||||
name: Backup Production Daily
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 04:00 Asia/Shanghai (UTC+8). Gitea cron is UTC.
|
||||
- cron: "0 20 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
artifact_retention_days:
|
||||
description: Days to keep the encrypted artifact on Gitea
|
||||
required: false
|
||||
default: "14"
|
||||
type: string
|
||||
local_retention_days:
|
||||
description: Days to keep this workflow's encrypted files on the host
|
||||
required: false
|
||||
default: "7"
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
concurrency:
|
||||
group: production-mutation
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
backup:
|
||||
if: gitea.ref == 'refs/heads/main'
|
||||
runs-on: xiaoxin
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
GITEA_SHA: ${{ gitea.sha }}
|
||||
GITEA_REF: ${{ gitea.ref }}
|
||||
GITEA_API_URL: ${{ gitea.api_url }}
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }}
|
||||
DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }}
|
||||
DEPLOY_USER: ${{ vars.PRODUCTION_USER }}
|
||||
DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }}
|
||||
PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }}
|
||||
ARTIFACT_RETENTION_DAYS: ${{ inputs.artifact_retention_days }}
|
||||
LOCAL_RETENTION_DAYS: ${{ inputs.local_retention_days }}
|
||||
steps:
|
||||
- name: Create, restore-verify, and retrieve the daily encrypted backup
|
||||
id: backup
|
||||
env:
|
||||
SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "$GITEA_REF" == "refs/heads/main" ]]
|
||||
[[ "$GITEA_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"
|
||||
[ -n "$artifact_retention" ] || artifact_retention=14
|
||||
local_retention="${LOCAL_RETENTION_DAYS:-7}"
|
||||
[ -n "$local_retention" ] || local_retention=7
|
||||
[[ "$artifact_retention" =~ ^[1-9][0-9]{0,3}$ ]]
|
||||
[[ "$local_retention" =~ ^[1-9][0-9]{0,3}$ ]]
|
||||
[[ "$DEPLOY_HOST" == "118.194.235.34" ]]
|
||||
[[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 ))
|
||||
[[ "$DEPLOY_USER" == "deploy" ]]
|
||||
[[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]]
|
||||
[[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]]
|
||||
test -n "$PRODUCTION_KNOWN_HOSTS"
|
||||
echo "artifact_retention_days=$artifact_retention" >>"$GITHUB_OUTPUT"
|
||||
echo "local_retention_days=$local_retention" >>"$GITHUB_OUTPUT"
|
||||
ssh_root="${RUNNER_TEMP}/production-daily-ssh"
|
||||
key_path="$ssh_root/id_ed25519"
|
||||
known_hosts_path="$ssh_root/known_hosts"
|
||||
artifact_directory="artifacts/production-daily"
|
||||
install -m 700 -d "$ssh_root" "$artifact_directory"
|
||||
test -n "$SSH_PRIVATE_KEY_BASE64"
|
||||
printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode >"$key_path"
|
||||
printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' >"$known_hosts_path"
|
||||
chmod 600 "$key_path" "$known_hosts_path"
|
||||
ssh-keygen -y -f "$key_path" >/dev/null
|
||||
ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
|
||||
scp_options=(-i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
|
||||
remote="$DEPLOY_USER@$DEPLOY_HOST"
|
||||
incoming="$(ssh "${ssh_options[@]}" "$remote" 'mktemp -d /tmp/jyotisha-production-daily.XXXXXXXXXX')"
|
||||
[[ "$incoming" == /tmp/jyotisha-production-daily.* ]]
|
||||
cleanup() {
|
||||
ssh "${ssh_options[@]}" "$remote" "rm -rf -- '$incoming'" >/dev/null 2>&1 || true
|
||||
rm -rf -- "$ssh_root"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
git init .
|
||||
git remote remove origin 2>/dev/null || true
|
||||
git remote add origin https://git.copse.top/root/Jyotisha.git
|
||||
git fetch --no-tags origin "$GITEA_SHA"
|
||||
git checkout --detach --force "$GITEA_SHA"
|
||||
[[ "$(git rev-parse HEAD)" == "$GITEA_SHA" ]]
|
||||
git show "$GITEA_SHA:deploy/run-production-recovery.sh" >"$ssh_root/run-production-recovery.sh"
|
||||
git show "$GITEA_SHA:deploy/prune-production-daily-backups.sh" >"$ssh_root/prune-production-daily-backups.sh"
|
||||
chmod 700 "$ssh_root/run-production-recovery.sh" "$ssh_root/prune-production-daily-backups.sh"
|
||||
scp "${scp_options[@]}" \
|
||||
"$ssh_root/run-production-recovery.sh" \
|
||||
"$ssh_root/prune-production-daily-backups.sh" \
|
||||
"$remote:$incoming/"
|
||||
ssh "${ssh_options[@]}" "$remote" \
|
||||
"DEPLOY_PATH='$DEPLOY_PATH' RECOVERY_RUN_ID='$GITHUB_RUN_ID' RECOVERY_BASENAME_PREFIX='production-daily' bash '$incoming/run-production-recovery.sh'" \
|
||||
>"$ssh_root/recovery-output.env"
|
||||
[[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d ' ')" == 7 ]]
|
||||
recovery_reference="$(awk -F= '$1 == "RECOVERY_REFERENCE" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
recovery_created_at="$(awk -F= '$1 == "RECOVERY_CREATED_AT" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
recovery_verified_at="$(awk -F= '$1 == "RECOVERY_VERIFIED_AT" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
recovery_sha256="$(awk -F= '$1 == "RECOVERY_SHA256" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
restore_verified="$(awk -F= '$1 == "RESTORE_VERIFIED" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
backup_basename="$(awk -F= '$1 == "BACKUP_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
verify_basename="$(awk -F= '$1 == "VERIFY_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
|
||||
[[ "$backup_basename" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]]
|
||||
[[ "$verify_basename" == "${backup_basename%.dump.enc}-restore-verify.json" ]]
|
||||
[[ "$recovery_reference" == "gitea-actions-run-${GITHUB_RUN_ID}/${backup_basename}" ]]
|
||||
[[ "$recovery_created_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
[[ "$recovery_verified_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
[[ "$recovery_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
[[ "$restore_verified" == "true" ]]
|
||||
scp "${scp_options[@]}" \
|
||||
"$remote:$DEPLOY_PATH/backups/$backup_basename" \
|
||||
"$remote:$DEPLOY_PATH/backups/$verify_basename" \
|
||||
"$artifact_directory/"
|
||||
test -s "$artifact_directory/$backup_basename"
|
||||
test -s "$artifact_directory/$verify_basename"
|
||||
printf '%s %s\n' "$recovery_sha256" "$artifact_directory/$backup_basename" | sha256sum --check --status
|
||||
jq -e \
|
||||
--arg backup "$backup_basename" \
|
||||
--arg sha "$recovery_sha256" \
|
||||
--arg created "$recovery_created_at" \
|
||||
--arg verified "$recovery_verified_at" '
|
||||
.mode == "restore_verify" and .ok == true and
|
||||
.backup == $backup and .sha256 == $sha and
|
||||
.created_at == $created and .verified_at == $verified and
|
||||
.restore_database_removed == true and
|
||||
([.checks.identity_users, .checks.profiles, .checks.credit_transactions, .checks.public_tables] |
|
||||
all(type == "number" and . >= 0 and floor == .))
|
||||
' "$artifact_directory/$verify_basename" >/dev/null
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
|
||||
"$recovery_reference" "$recovery_created_at" "$recovery_verified_at" "$recovery_sha256" \
|
||||
"$backup_basename" "$verify_basename" >"$artifact_directory/attestation.env"
|
||||
chmod 600 "$artifact_directory"/*
|
||||
ssh "${ssh_options[@]}" "$remote" \
|
||||
"bash '$incoming/prune-production-daily-backups.sh' '$DEPLOY_PATH/backups' '$local_retention'"
|
||||
{
|
||||
echo "recovery_reference=$recovery_reference"
|
||||
echo "recovery_created_at=$recovery_created_at"
|
||||
echo "recovery_verified_at=$recovery_verified_at"
|
||||
echo "recovery_sha256=$recovery_sha256"
|
||||
} >>"$GITHUB_OUTPUT"
|
||||
echo "Daily backup restore-verified: reference=$recovery_reference created_at=$recovery_created_at verified_at=$recovery_verified_at"
|
||||
|
||||
- name: Upload encrypted off-site daily backup
|
||||
env:
|
||||
ARTIFACT_RETENTION_DAYS: ${{ steps.backup.outputs.artifact_retention_days }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "${ARTIFACT_RETENTION_DAYS:-}" =~ ^[1-9][0-9]{0,3}$ ]]
|
||||
test -n "${ACTIONS_RUNTIME_TOKEN:-}"
|
||||
test -n "${ACTIONS_RESULTS_URL:-}"
|
||||
test -n "${GITHUB_RUN_ID:-}"
|
||||
test -n "${GITHUB_REPOSITORY:-}"
|
||||
workdir="$(pwd -P)"
|
||||
docker run --rm \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
--volume "$workdir:$workdir" \
|
||||
--workdir "$workdir" \
|
||||
--env HOME=/tmp \
|
||||
--env "INPUT_NAME=production-daily-$GITHUB_RUN_ID" \
|
||||
--env INPUT_PATH=artifacts/production-daily/ \
|
||||
--env INPUT_OVERWRITE=false \
|
||||
--env ACTIONS_RUNTIME_TOKEN \
|
||||
--env ACTIONS_RESULTS_URL \
|
||||
--env GITHUB_RUN_ID \
|
||||
--env GITHUB_REPOSITORY \
|
||||
--env "GITHUB_SHA=$GITEA_SHA" \
|
||||
--env "GITHUB_WORKSPACE=$workdir" \
|
||||
--env ARTIFACT_RETENTION_DAYS \
|
||||
node:22-bookworm-slim \
|
||||
node -e 'process.env["INPUT_IF-NO-FILES-FOUND"]="error"; process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS; process.env["INPUT_COMPRESSION-LEVEL"]="0"; require("./.gitea/actions/upload-artifact/dist/index.js")'
|
||||
@@ -1,5 +1,13 @@
|
||||
# BLOCKED
|
||||
|
||||
## TASK-beta-safety-net-20261014：主机安装与一次恢复演练还没做（2026-10-15）
|
||||
|
||||
- **每日备份**：工作流要等这支分支进入 `main` 之后才会在北京时间 04:00 跑。本机没有生产主机，没有手动触发生产工作流。
|
||||
- **探活**：应装在 Gitea runner 上，不要装在生产主机。磁盘检查和错误汇总装在生产主机。飞书 Webhook 由产品写进模式 `0600` 的环境文件，不进仓库。没配之前只写本机日志。
|
||||
- **日志轮转**：编排已写上单文件 20 MB、最多 10 个文件。要等生产容器按新编排重建后才生效。本机没有对生产主机执行。
|
||||
- **恢复演练**：内测开放前按 `docs/operations/production-restore-drill-20261014.md` 做一次一次性库还原，结果填 `docs/testing/production-restore-drill-20261014.md`。模板现在是空的。
|
||||
- 解除：上面四件事做完，并把演练模板里的「是否通过」填上。不要把 Webhook、密钥或用户内容写进仓库。
|
||||
|
||||
## TASK-consult-prep-parallel-20261013：审核真实延迟、Postgres 中位数、多领域镜像（2026-10-14）
|
||||
|
||||
- **阿里云审核**：本机没有审核密钥。1500 毫秒挡首帧的结论来自测试里的假等待，不是真实供应商。开关 `CONSULT_PREP_PARALLEL` 保持关闭。
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
# 印度占星 Skill 更新日志
|
||||
|
||||
## 2026-10-15 — 内测前补上每日备份、宕机告警和日志上限（待合入 staging）
|
||||
|
||||
- 生产可以每天自动做一次加密备份。站点连续三次打不开、磁盘过满，或短时间错误变多时，可以通知到飞书群。容器日志有大小上限。可以按一轮请求的编号在服务器日志里查找。这些要等产品在主机上装好之后才生效。页面没有变化。Skill 版本不变（BUG-1354、BUG-1355、BUG-1356）。
|
||||
|
||||
## 2026-10-15 — 星盘页上升行补上星宿，表头对齐（待合入 staging）
|
||||
|
||||
- 本命盘和月亮盘的上升行现在有星宿、宿主和 pada。状态写「不适用」，不再是空横杠。分盘等表里居中的那几列，表头也居中。当前大运不再画左侧那条线，浅色底还在。已经打开过的星盘会重新计算。Skill 版本不变（BUG-1352、BUG-1353）。
|
||||
|
||||
@@ -394,6 +394,48 @@ smoke in `docs/operations/personal-report-staging.md` (section “Post-deploy
|
||||
report list / detail / create smoke”): `GET /api/reports` 200, owned
|
||||
`GET /api/reports/<id>` 200, and one new `POST /api/reports` through ready.
|
||||
|
||||
## 运维与告警
|
||||
|
||||
这些步骤要等本分支进入 `main` 之后，由产品在主机上安装。仓库里没有 Webhook 地址。告警正文只允许服务名、路由、错误码、次数、requestId 和时间。
|
||||
|
||||
### 备份
|
||||
|
||||
- 工作流：`.gitea/workflows/backup-production-daily.yml`。北京时间每天 04:00，也可手动触发。只从 `main` 运行，调用 `deploy/run-production-recovery.sh`，不另写一套导出。
|
||||
- 每日文件前缀是 `production-daily-`。手动恢复点仍是 `production-pre-migration-`。清理脚本 `deploy/prune-production-daily-backups.sh` 只删超过 7 天的每日文件和对应的校验文件。
|
||||
- Gitea 产物默认保留 14 天。手动恢复点工作流仍是 30 天，没有改。
|
||||
- 还原步骤：`docs/operations/production-restore-drill-20261014.md`。
|
||||
|
||||
### 告警装在哪
|
||||
|
||||
- 探活不要装在生产主机上。装在 Gitea runner 上，每分钟跑 `deploy/monitor/healthcheck-alert.sh --mode health`。它访问公网 `https://jyotisha.chat/api/health`，超时 10 秒。连续 3 次失败才告警，恢复只发一次。
|
||||
- 发布窗口：在 runner 上执行 `touch /var/lib/jyotisha-monitor/release-window`。45 分钟内，版本号变化不告警。
|
||||
- 磁盘检查装在生产主机上，每分钟跑同一个脚本的 `--mode disk`。使用率达到 80% 告警一次，降下来再发一次已恢复。
|
||||
- 错误汇总装在生产主机上。每 5 分钟 `deploy/monitor/error-digest.sh --window`。每天北京时间 09:00（UTC 01:00）跑 `--daily`，汇总前一天。同类错误 5 分钟内至少 10 次，或 5xx 至少 20 次，才发窗口告警。消息里最多 5 个 requestId。含邮箱、第二个日期或非 ASCII 文本的日志行整行丢掉。
|
||||
- Webhook 写在模式 `0600` 的环境文件里，建议路径 `/opt/jyotisha-production/.monitor.env`（runner 上用 `JYOTISHA_MONITOR_ENV` 指向自己的文件）。只放这两行，不要提交：
|
||||
|
||||
```text
|
||||
FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/<token>
|
||||
FEISHU_SIGN_SECRET=<只有开了签名校验才填>
|
||||
```
|
||||
|
||||
没配 Webhook 时只写本机 `alerts.log`，不外发。飞书消息是 `msg_type: text`。配了签名就带 `timestamp` 和 `sign`。
|
||||
|
||||
### 日志轮转
|
||||
|
||||
`deploy/docker-compose.server.yml` 和 `deploy/docker-compose.postgres.yml` 的每个服务都是 json-file，单文件 20 MB，最多 10 个文件。镜像、环境变量、端口和健康检查没有改。要等生产容器按新编排重建后才生效。
|
||||
|
||||
看守护进程默认值：`docker info --format '{{.LoggingDriver}}'`，以及主机上的 `/etc/docker/daemon.json`。编排里的 `logging` 会盖过守护进程对这几个服务的默认值。
|
||||
|
||||
### 按 requestId 查日志
|
||||
|
||||
在生产主机上，只读：
|
||||
|
||||
```bash
|
||||
deploy/find-request-log.sh <requestId> --since 24h
|
||||
```
|
||||
|
||||
`<requestId>` 必须是 UUID。其它写法会拒绝，避免被当成命令。`--since` 只接受类似 `24h`、`30m` 的形式。输出按时间排序，来自 web、api、caddy。
|
||||
|
||||
## Common operations
|
||||
|
||||
```bash
|
||||
|
||||
@@ -2,6 +2,11 @@ services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
shm_size: 128mb
|
||||
env_file:
|
||||
- ${DATABASE_ENV_FILE:-../.env.staging.database}
|
||||
@@ -30,6 +35,11 @@ services:
|
||||
image: ${WEB_IMAGE:-jyotisha-web:local}
|
||||
profiles: ["migration"]
|
||||
restart: "no"
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${DATABASE_ENV_FILE:-../.env.staging.database}
|
||||
working_dir: /app/frontend
|
||||
@@ -43,6 +53,11 @@ services:
|
||||
image: ${WEB_IMAGE:-jyotisha-web:local}
|
||||
profiles: ["migration-check"]
|
||||
restart: "no"
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${DATABASE_ENV_FILE:-../.env.staging.database}
|
||||
working_dir: /app/frontend
|
||||
|
||||
@@ -5,6 +5,11 @@ services:
|
||||
context: ..
|
||||
dockerfile: deploy/railway-api.Dockerfile
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${APP_ENV_FILE:-../.env.production}
|
||||
environment:
|
||||
@@ -35,6 +40,11 @@ services:
|
||||
NEXT_PUBLIC_SUPABASE_URL: ${NEXT_PUBLIC_SUPABASE_URL:-}
|
||||
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${NEXT_PUBLIC_SUPABASE_ANON_KEY:-}
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
env_file:
|
||||
- ${APP_ENV_FILE:-../.env.production}
|
||||
environment:
|
||||
@@ -57,6 +67,11 @@ services:
|
||||
caddy:
|
||||
image: caddy:2-alpine
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "20m"
|
||||
max-file: "10"
|
||||
environment:
|
||||
SITE_ADDRESS: ${SITE_ADDRESS:-https://jyotisha.chat}
|
||||
ports:
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
# Read-only search for one request id in web, api, and caddy logs.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
|
||||
if [ $# -lt 1 ] || [ $# -gt 3 ]; then
|
||||
echo "usage: find-request-log.sh <requestId> [--since 24h]" >&2
|
||||
exit 2
|
||||
fi
|
||||
request_id="$1"
|
||||
shift
|
||||
since="24h"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--since)
|
||||
since="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ "$request_id" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]] || {
|
||||
echo "requestId must be a UUID" >&2
|
||||
exit 2
|
||||
}
|
||||
[[ "$since" =~ ^[1-9][0-9]{0,3}[smhd]$ ]] || {
|
||||
echo "since must look like 24h" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
log_dir="${JYOTISHA_LOG_DIR:-}"
|
||||
if [ -n "$log_dir" ]; then
|
||||
[ -d "$log_dir" ] && [ ! -L "$log_dir" ] || {
|
||||
echo "log directory is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
tmp="$(mktemp)"
|
||||
for service in web api caddy; do
|
||||
file="$log_dir/$service.log"
|
||||
[ -f "$file" ] && [ ! -L "$file" ] || continue
|
||||
grep -F -- "$request_id" "$file" >>"$tmp" || true
|
||||
done
|
||||
if [ -s "$tmp" ]; then
|
||||
grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}T' "$tmp" | sort
|
||||
fi
|
||||
rm -f -- "$tmp"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
|
||||
docker compose -p jyotisha-production \
|
||||
--env-file "$compose_dir/.env.production" \
|
||||
-f "$compose_dir/deploy/docker-compose.server.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.production.yml" \
|
||||
logs --since "$since" --no-color --timestamps web api caddy |
|
||||
grep -F -- "$request_id" |
|
||||
sort
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Classify fictional or container log lines into alert rows.
|
||||
|
||||
Outbound fields are service, route, errorCode, count, and request ids.
|
||||
Lines with non-ASCII text, an email address, or a second calendar date are
|
||||
dropped so conversation text and birth dates cannot leave the host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
|
||||
UUID = re.compile(
|
||||
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
|
||||
)
|
||||
ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$")
|
||||
ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$")
|
||||
STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b")
|
||||
FIXTURE = re.compile(
|
||||
r"^(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?P<svc>web|api) (?P<rest>.*)$"
|
||||
)
|
||||
DOCKER = re.compile(
|
||||
r"^(?P<svc>web|api)-\d+\s+\|\s+"
|
||||
r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P<rest>.*)$"
|
||||
)
|
||||
CST = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def unsafe(line: str) -> bool:
|
||||
if any(ord(char) > 126 for char in line):
|
||||
return True
|
||||
lowered = line.lower()
|
||||
if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered:
|
||||
return True
|
||||
return len(DATE.findall(line)) >= 2
|
||||
|
||||
|
||||
def parse_line(line: str) -> tuple[str, str, str] | None:
|
||||
match = FIXTURE.match(line) or DOCKER.match(line)
|
||||
if not match:
|
||||
return None
|
||||
return match.group("ts"), match.group("svc"), match.group("rest")
|
||||
|
||||
|
||||
def timestamp_cst_date(timestamp: str):
|
||||
stamp = timestamp
|
||||
if "." in stamp:
|
||||
stamp = stamp.split(".", 1)[0] + "Z"
|
||||
moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
return moment.astimezone(CST).date()
|
||||
|
||||
|
||||
def extract_fields(rest: str) -> tuple[str, str, str, int | None]:
|
||||
route = "-"
|
||||
error_code = ""
|
||||
request_id = ""
|
||||
status: int | None = None
|
||||
marker = "[agent-observability]"
|
||||
if marker in rest:
|
||||
raw = rest.split(marker, 1)[1].strip()
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, dict):
|
||||
code = payload.get("errorCode")
|
||||
if isinstance(code, str) and ERROR_CODE.match(code):
|
||||
error_code = code
|
||||
candidate = payload.get("requestId")
|
||||
if isinstance(candidate, str) and UUID.match(candidate):
|
||||
request_id = candidate.lower()
|
||||
route_value = payload.get("route")
|
||||
if isinstance(route_value, str) and ROUTE.match(route_value):
|
||||
route = route_value
|
||||
status_match = STATUS.search(rest)
|
||||
if status_match:
|
||||
status = int(status_match.group(1))
|
||||
return route, error_code, request_id, status
|
||||
|
||||
|
||||
def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]:
|
||||
now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
yesterday = now.astimezone(CST).date() - timedelta(days=1)
|
||||
counts: OrderedDict[tuple[str, str, str], int] = OrderedDict()
|
||||
request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict()
|
||||
fivexx_counts: OrderedDict[str, int] = OrderedDict()
|
||||
fivexx_ids: OrderedDict[str, list[str]] = OrderedDict()
|
||||
|
||||
def add(key: tuple[str, str, str], request_id: str) -> None:
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
bucket = request_ids.setdefault(key, [])
|
||||
if request_id and request_id not in bucket and len(bucket) < 5:
|
||||
bucket.append(request_id)
|
||||
|
||||
for raw in lines:
|
||||
line = raw.rstrip("\r\n")
|
||||
if not line or unsafe(line):
|
||||
continue
|
||||
parsed = parse_line(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
timestamp, service, rest = parsed
|
||||
if mode == "daily" and timestamp_cst_date(timestamp) != yesterday:
|
||||
continue
|
||||
route, error_code, request_id, status = extract_fields(rest)
|
||||
if error_code:
|
||||
add((service, route, error_code), request_id)
|
||||
elif status is not None:
|
||||
add((service, route, f"http_{status}"), request_id)
|
||||
if status is not None and 500 <= status <= 599:
|
||||
fivexx_counts[service] = fivexx_counts.get(service, 0) + 1
|
||||
bucket = fivexx_ids.setdefault(service, [])
|
||||
if request_id and request_id not in bucket and len(bucket) < 5:
|
||||
bucket.append(request_id)
|
||||
|
||||
rows: list[str] = []
|
||||
|
||||
def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None:
|
||||
shown = ",".join(ids) if ids else "-"
|
||||
rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}")
|
||||
|
||||
for key, count in counts.items():
|
||||
if mode == "window" and count < error_threshold:
|
||||
continue
|
||||
if count <= 0:
|
||||
continue
|
||||
emit(*key, count, request_ids.get(key, []))
|
||||
for service, count in fivexx_counts.items():
|
||||
if mode == "window" and count < fivexx_threshold:
|
||||
continue
|
||||
if count <= 0:
|
||||
continue
|
||||
emit(service, "-", "http_5xx", count, fivexx_ids.get(service, []))
|
||||
return rows
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 5:
|
||||
print("usage: classify_error_log.py <fixture> <window|daily> <now> <error-threshold> <5xx-threshold>", file=sys.stderr)
|
||||
return 2
|
||||
fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv
|
||||
if mode not in {"window", "daily"}:
|
||||
print("unknown mode", file=sys.stderr)
|
||||
return 2
|
||||
error_threshold = int(error_threshold_text)
|
||||
fivexx_threshold = int(fivexx_threshold_text)
|
||||
with open(fixture, encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold):
|
||||
print(row)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# Count recent web/api errors and alert without sending log text.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=notify-feishu.sh
|
||||
source "$script_dir/notify-feishu.sh"
|
||||
|
||||
mode="window"
|
||||
case "${1:-}" in
|
||||
""|--window) mode="window" ;;
|
||||
--daily) mode="daily" ;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
||||
load_monitor_env "$env_file"
|
||||
|
||||
error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}"
|
||||
fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}"
|
||||
[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
||||
[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
||||
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2
|
||||
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
|
||||
fixture="${JYOTISHA_LOG_FIXTURE:-}"
|
||||
cleanup() {
|
||||
if [ -n "${generated_fixture:-}" ]; then
|
||||
rm -f -- "$generated_fixture"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
generated_fixture=""
|
||||
if [ -z "$fixture" ]; then
|
||||
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
|
||||
since="5m"
|
||||
[ "$mode" = "daily" ] && since="48h"
|
||||
generated_fixture="$(mktemp)"
|
||||
docker compose -p jyotisha-production \
|
||||
--env-file "$compose_dir/.env.production" \
|
||||
-f "$compose_dir/deploy/docker-compose.server.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.production.yml" \
|
||||
logs --since "$since" --no-color --timestamps web api >"$generated_fixture"
|
||||
fixture="$generated_fixture"
|
||||
fi
|
||||
[ -f "$fixture" ] && [ ! -L "$fixture" ] || {
|
||||
echo "log fixture is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then
|
||||
echo "could not classify logs" >&2
|
||||
exit 1
|
||||
fi
|
||||
stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
text=""
|
||||
if [ -n "$rows" ]; then
|
||||
while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do
|
||||
[ -n "${service:-}" ] || continue
|
||||
line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}"
|
||||
text="${text}${line}"$'\n'
|
||||
done <<<"$rows" || true
|
||||
elif [ "$mode" = "daily" ]; then
|
||||
text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n'
|
||||
fi
|
||||
|
||||
if [ -n "$text" ]; then
|
||||
post_monitor_alert "${text%$'\n'}"
|
||||
fi
|
||||
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe the public health URL, or the host disk. Install the health mode on
|
||||
# the Gitea runner, not on the production host. Install disk mode on the host.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=notify-feishu.sh
|
||||
source "$script_dir/notify-feishu.sh"
|
||||
|
||||
mode="health"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
mode="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--mode=*)
|
||||
mode="${1#--mode=}"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
case "$mode" in
|
||||
health|disk|all) ;;
|
||||
*)
|
||||
echo "unknown mode" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
||||
load_monitor_env "$env_file"
|
||||
|
||||
health_url="${JYOTISHA_HEALTH_URL:-https://jyotisha.chat/api/health}"
|
||||
timeout="${JYOTISHA_HEALTH_TIMEOUT:-10}"
|
||||
[[ "$timeout" =~ ^[1-9][0-9]?$ ]] && (( timeout <= 30 )) || {
|
||||
echo "health timeout must be 1 to 30 seconds" >&2
|
||||
exit 2
|
||||
}
|
||||
if [ "$mode" != "disk" ]; then
|
||||
[[ "$health_url" == "https://jyotisha.chat/api/health" \
|
||||
|| "$health_url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/health$ \
|
||||
|| "$health_url" =~ ^http://localhost:[0-9]{1,5}/health$ ]] || {
|
||||
echo "unexpected health url" >&2
|
||||
exit 2
|
||||
}
|
||||
fi
|
||||
|
||||
state_file="$state_dir/monitor.state"
|
||||
consecutive_failures=0
|
||||
alert_open=0
|
||||
last_commit=""
|
||||
commit_alerted=""
|
||||
disk_alert_open=0
|
||||
|
||||
read_state() {
|
||||
local line value
|
||||
[ -f "$state_file" ] && [ ! -L "$state_file" ] || return 0
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
consecutive_failures=[0-9]*)
|
||||
consecutive_failures="${line#consecutive_failures=}"
|
||||
;;
|
||||
alert_open=[01])
|
||||
alert_open="${line#alert_open=}"
|
||||
;;
|
||||
last_commit=)
|
||||
last_commit=""
|
||||
;;
|
||||
last_commit=*)
|
||||
value="${line#last_commit=}"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && last_commit="$value"
|
||||
;;
|
||||
commit_alerted=)
|
||||
commit_alerted=""
|
||||
;;
|
||||
commit_alerted=*)
|
||||
value="${line#commit_alerted=}"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && commit_alerted="$value"
|
||||
;;
|
||||
disk_alert_open=[01])
|
||||
disk_alert_open="${line#disk_alert_open=}"
|
||||
;;
|
||||
esac
|
||||
done <"$state_file"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
local tmp
|
||||
tmp="$state_file.tmp.$$"
|
||||
printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\n' \
|
||||
"$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" >"$tmp"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$tmp"
|
||||
fi
|
||||
mv "$tmp" "$state_file"
|
||||
}
|
||||
|
||||
stamp() {
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ
|
||||
}
|
||||
|
||||
send_alert() {
|
||||
local service="$1"
|
||||
local route="$2"
|
||||
local error_code="$3"
|
||||
local count="$4"
|
||||
local request_id="${5:--}"
|
||||
post_monitor_alert "service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_id} time=$(stamp)"
|
||||
}
|
||||
|
||||
release_window_open() {
|
||||
local marker="$state_dir/release-window"
|
||||
local mtime now age
|
||||
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
|
||||
mtime="$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker")"
|
||||
now="$(date +%s)"
|
||||
age=$((now - mtime))
|
||||
[ "$age" -ge 0 ] && [ "$age" -le 2700 ]
|
||||
}
|
||||
|
||||
check_health() {
|
||||
local body_file http_code curl_status status_line commit_line status commit healthy py
|
||||
body_file="$(mktemp)"
|
||||
http_code="000"
|
||||
curl_status=0
|
||||
http_code="$(curl --silent --show-error --connect-timeout "$timeout" --max-time "$timeout" \
|
||||
-o "$body_file" -w '%{http_code}' "$health_url")" || curl_status=$?
|
||||
status=""
|
||||
commit=""
|
||||
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ]; then
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
status_line="$("$py" - "$body_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
try:
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
except Exception:
|
||||
print("")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
if not isinstance(payload, dict):
|
||||
print("")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
status = payload.get("status")
|
||||
deployment = payload.get("deployment")
|
||||
commit = ""
|
||||
if isinstance(deployment, dict):
|
||||
raw = deployment.get("gitCommit")
|
||||
if isinstance(raw, str):
|
||||
commit = raw
|
||||
print(status if isinstance(status, str) else "")
|
||||
print(commit)
|
||||
PY
|
||||
)"
|
||||
status="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==1 {print}')"
|
||||
commit="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==2 {print}')"
|
||||
fi
|
||||
rm -f -- "$body_file"
|
||||
|
||||
healthy=0
|
||||
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ] && [ "$status" = "ok" ]; then
|
||||
healthy=1
|
||||
fi
|
||||
|
||||
if [ "$healthy" -eq 1 ]; then
|
||||
if [ "$alert_open" -eq 1 ]; then
|
||||
send_alert site /api/health recovered 0 -
|
||||
alert_open=0
|
||||
fi
|
||||
consecutive_failures=0
|
||||
if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
if [ -n "$last_commit" ] && [ "$commit" != "$last_commit" ] && [ "$commit_alerted" != "$commit" ]; then
|
||||
if ! release_window_open; then
|
||||
send_alert site /api/health commit_changed 1 -
|
||||
commit_alerted="$commit"
|
||||
fi
|
||||
fi
|
||||
last_commit="$commit"
|
||||
fi
|
||||
else
|
||||
consecutive_failures=$((consecutive_failures + 1))
|
||||
if [ "$consecutive_failures" -ge 3 ] && [ "$alert_open" -eq 0 ]; then
|
||||
send_alert site /api/health health_down "$consecutive_failures" -
|
||||
alert_open=1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
disk_percent() {
|
||||
df -Pk "${JYOTISHA_DISK_PATH:-/}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}'
|
||||
}
|
||||
|
||||
check_disk() {
|
||||
local percent
|
||||
percent="$(disk_percent)"
|
||||
[[ "$percent" =~ ^[0-9]+$ ]] && [ "$percent" -le 100 ] || {
|
||||
echo "disk usage is unreadable" >&2
|
||||
exit 1
|
||||
}
|
||||
if [ "$percent" -ge 80 ]; then
|
||||
if [ "$disk_alert_open" -eq 0 ]; then
|
||||
send_alert host - disk_high "$percent" -
|
||||
disk_alert_open=1
|
||||
fi
|
||||
elif [ "$disk_alert_open" -eq 1 ]; then
|
||||
send_alert host - recovered 0 -
|
||||
disk_alert_open=0
|
||||
fi
|
||||
}
|
||||
|
||||
read_state
|
||||
case "$mode" in
|
||||
health) check_health ;;
|
||||
disk) check_disk ;;
|
||||
all)
|
||||
check_health
|
||||
check_disk
|
||||
;;
|
||||
esac
|
||||
write_state
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared alert sender. Source this file; do not execute it.
|
||||
# Outbound text is limited to service, route, errorCode, count, requestId, time.
|
||||
|
||||
ensure_private_dir() {
|
||||
local dir="$1"
|
||||
mkdir -p -- "$dir"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 700 "$dir"
|
||||
fi
|
||||
}
|
||||
|
||||
load_monitor_env() {
|
||||
local file="$1"
|
||||
local line key value mode
|
||||
[ -n "$file" ] || return 0
|
||||
[ -f "$file" ] && [ ! -L "$file" ] || return 0
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
mode="$(stat -c %a "$file")"
|
||||
[ "$mode" = "600" ] || {
|
||||
echo "monitor env must be mode 0600" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
""|\#*) continue ;;
|
||||
FEISHU_WEBHOOK_URL=*|FEISHU_SIGN_SECRET=*)
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
case "$value" in
|
||||
*[$'\n\r`$']*)
|
||||
echo "monitor env value is unsafe" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf -v "$key" '%s' "$value"
|
||||
export "$key"
|
||||
;;
|
||||
esac
|
||||
done <"$file"
|
||||
}
|
||||
|
||||
feishu_sign() {
|
||||
local timestamp="$1"
|
||||
local secret="$2"
|
||||
local string_to_sign
|
||||
string_to_sign="$(printf '%s\n%s' "$timestamp" "$secret")"
|
||||
printf '' | openssl dgst -sha256 -hmac "$string_to_sign" -binary | openssl base64 -A
|
||||
}
|
||||
|
||||
webhook_allowed() {
|
||||
local url="$1"
|
||||
[[ "$url" =~ ^https://open\.(feishu\.cn|larksuite\.com|larkoffice\.com)/open-apis/bot/v2/hook/[A-Za-z0-9-]+$ ]] && return 0
|
||||
[[ "$url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/hook$ ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
alert_line_allowed() {
|
||||
local line="$1"
|
||||
[[ "$line" =~ ^service=[a-z0-9_-]+[[:space:]]route=(/|/[A-Za-z0-9/_.-]+|-)[[:space:]]errorCode=[a-z][a-z0-9._-]*[[:space:]]count=[0-9]+[[:space:]]requestId=(-|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}){0,4})[[:space:]]time=[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
}
|
||||
|
||||
post_monitor_alert() {
|
||||
local text="$1"
|
||||
local state_dir line timestamp sign payload_file
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
[ -n "$line" ] || continue
|
||||
alert_line_allowed "$line" || {
|
||||
echo "alert text has a field that is not allowed" >&2
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "$line" >>"$state_dir/alerts.log"
|
||||
done <<<"$text"
|
||||
|
||||
[ -n "${FEISHU_WEBHOOK_URL:-}" ] || return 0
|
||||
webhook_allowed "$FEISHU_WEBHOOK_URL" || {
|
||||
printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log"
|
||||
return 0
|
||||
}
|
||||
|
||||
payload_file="$(mktemp)"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$payload_file"
|
||||
fi
|
||||
if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then
|
||||
timestamp="$(date +%s)"
|
||||
sign="$(feishu_sign "$timestamp" "$FEISHU_SIGN_SECRET")"
|
||||
FEISHU_ALERT_TEXT="$text" FEISHU_ALERT_TIMESTAMP="$timestamp" FEISHU_ALERT_SIGN="$sign" \
|
||||
python_write_payload "$payload_file" signed
|
||||
else
|
||||
FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain
|
||||
fi
|
||||
curl --silent --show-error --connect-timeout 10 --max-time 10 \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Connection: close' \
|
||||
--data-binary @"$payload_file" \
|
||||
"$FEISHU_WEBHOOK_URL" >/dev/null
|
||||
rm -f -- "$payload_file"
|
||||
}
|
||||
|
||||
python_write_payload() {
|
||||
local destination="$1"
|
||||
local mode="$2"
|
||||
local py
|
||||
export FEISHU_ALERT_TEXT FEISHU_ALERT_TIMESTAMP FEISHU_ALERT_SIGN
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
"$py" - "$destination" "$mode" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
destination, mode = sys.argv[1], sys.argv[2]
|
||||
text = os.environ["FEISHU_ALERT_TEXT"]
|
||||
payload = {"msg_type": "text", "content": {"text": text}}
|
||||
if mode == "signed":
|
||||
payload["timestamp"] = os.environ["FEISHU_ALERT_TIMESTAMP"]
|
||||
payload["sign"] = os.environ["FEISHU_ALERT_SIGN"]
|
||||
with open(destination, "w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, separators=(",", ":"))
|
||||
PY
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
# Delete only expired production-daily backup files. The prefix is fixed so a
|
||||
# caller cannot aim this at manual recovery points or other files.
|
||||
set -euo pipefail
|
||||
|
||||
prune_production_daily_backups() {
|
||||
local directory="$1"
|
||||
local retention_days="$2"
|
||||
local now_epoch="${3:-}"
|
||||
local dump name mtime age limit base verify verify_name
|
||||
local -a doomed=()
|
||||
|
||||
[ -n "$directory" ] && [ -d "$directory" ] && [ ! -L "$directory" ] || {
|
||||
echo "backup directory is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "$retention_days" =~ ^[1-9][0-9]{0,3}$ ]] || {
|
||||
echo "retention days must be a positive integer" >&2
|
||||
exit 1
|
||||
}
|
||||
if [ -z "$now_epoch" ]; then
|
||||
now_epoch="$(date +%s)"
|
||||
fi
|
||||
[[ "$now_epoch" =~ ^[0-9]+$ ]] || {
|
||||
echo "clock is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
limit=$((retention_days * 86400))
|
||||
|
||||
shopt -s nullglob
|
||||
for dump in "$directory"/production-daily-*.dump.enc; do
|
||||
name="$(basename -- "$dump")"
|
||||
[[ "$name" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]] || continue
|
||||
[ -f "$dump" ] && [ ! -L "$dump" ] || continue
|
||||
mtime="$(stat -c %Y "$dump" 2>/dev/null || stat -f %m "$dump")"
|
||||
[[ "$mtime" =~ ^[0-9]+$ ]] || continue
|
||||
age=$((now_epoch - mtime))
|
||||
if (( age > limit )); then
|
||||
doomed+=("$name")
|
||||
fi
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
if ((${#doomed[@]} == 0)); then
|
||||
return 0
|
||||
fi
|
||||
for name in "${doomed[@]}"; do
|
||||
base="${name%.dump.enc}"
|
||||
verify="$directory/${base}-restore-verify.json"
|
||||
rm -f -- "$directory/$name"
|
||||
if [ -e "$verify" ] && [ -f "$verify" ] && [ ! -L "$verify" ]; then
|
||||
verify_name="$(basename -- "$verify")"
|
||||
if [[ "$verify_name" =~ ^production-daily-[0-9]{8}T[0-9]{6}Z-restore-verify\.json$ ]]; then
|
||||
rm -f -- "$verify"
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' "$name"
|
||||
done
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
prune_production_daily_backups "${1:-}" "${2:-}" "${3:-}"
|
||||
fi
|
||||
@@ -138,7 +138,19 @@ usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); p
|
||||
|
||||
created_compact="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
backup_basename="production-pre-migration-${created_compact}.dump.enc"
|
||||
# Daily backups set RECOVERY_BASENAME_PREFIX=production-daily so retention can
|
||||
# tell them from manual recovery points. Unset keeps the original name.
|
||||
backup_prefix="production-pre-migration"
|
||||
if [ -n "${RECOVERY_BASENAME_PREFIX:-}" ]; then
|
||||
case "$RECOVERY_BASENAME_PREFIX" in
|
||||
production-daily) backup_prefix="production-daily" ;;
|
||||
*)
|
||||
echo "unexpected recovery basename prefix" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
backup_basename="${backup_prefix}-${created_compact}.dump.enc"
|
||||
backup_partial="$backup_directory/.${backup_basename}.$$.partial"
|
||||
backup_file="$backup_directory/$backup_basename"
|
||||
restore_database="restore_verify_${created_compact,,}"
|
||||
|
||||
@@ -36182,3 +36182,48 @@
|
||||
- 复发自：无
|
||||
- 修复版本：分支 `codex/chart-ascendant-row-20261015`。
|
||||
|
||||
## BUG-1354 | 生产没有每日自动备份
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-15
|
||||
- 来源：产品 10-14 内测体检。任务书 `docs/tasks/TASK-beta-safety-net-20261014.md`。
|
||||
- 现象：生产备份只在有人手动点恢复点时才做。仓库里没有按天跑的备份，也没有定时器文件。
|
||||
- 触发条件：内测开始写入真实用户数据之后，若两次手动备份之间出错或磁盘故障，中间的数据没有恢复点。
|
||||
- 根因：备份是按「发布前做一次」设计的，没有持续运行的每日任务。
|
||||
- 修复：新增只从 `main` 运行的每日备份工作流，复用现有加密与还原校验脚本。每日文件用单独前缀，过期清理不会碰到手动恢复点。工作流要等进入 `main` 后才会在北京时间 04:00 跑。本机没有生产主机，没有手动触发。
|
||||
- 验证：合同测试锁定有每日时刻、只在 `main` 上跑、调用同一个恢复脚本、清理只删过期的每日前缀。虚构目录里，手动恢复点和其他文件都还在。
|
||||
- 防复发：上述合同测试。清理函数不接受外部传入的前缀。
|
||||
- 相关记录：BUG-1350、BUG-1355、BUG-1356。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。开工时本文件最大号是 BUG-1353。
|
||||
|
||||
## BUG-1355 | 站点打不开或磁盘过满时没有告警
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-15
|
||||
- 来源：同 BUG-1354。
|
||||
- 现象：容器自己的健康检查只会尝试重启，不会通知人。磁盘变满也没有通知。
|
||||
- 触发条件：公网健康检查连续失败，或生产主机磁盘使用率达到 80%。
|
||||
- 根因：上线时没有持续探活，也没有把告警发到人用的渠道。
|
||||
- 修复：探活脚本连续 3 次失败才发一条告警，恢复只发一次。磁盘检查单独在生产主机上跑。Webhook 从环境文件读取，没有配置就只写本机日志。外发正文不含用户内容。脚本还没装到 runner 或生产主机上。
|
||||
- 验证：本地桩服务覆盖正常、500、超时和缺少字段。连续 3 次才告警，恢复只出现一次。签名测试里的密钥没有出现在发出的正文里。
|
||||
- 防复发：上述脚本测试。告警行若含不允许的字段，脚本拒绝发送。
|
||||
- 相关记录：BUG-1354、BUG-1356。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。编号接在 BUG-1354 之后。
|
||||
|
||||
## BUG-1356 | 容器日志不轮转，也不能按一轮请求查找
|
||||
|
||||
- 状态：代码已交，主机安装待产品
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-15
|
||||
- 来源：同 BUG-1354。BUG-1306 取证时日志只在容器控制台，重建后查不到。
|
||||
- 现象：编排没有日志大小上限。知道一轮请求的编号时，没有只读的查找命令。
|
||||
- 触发条件：日志持续写入，或需要按 requestId 回看 web、api、caddy。
|
||||
- 根因：日志沿用守护进程的未知默认值，也没有按编号检索的入口。
|
||||
- 修复：每个服务的日志改为单文件 20 MB、最多 10 个文件。新增只读查找脚本，只接受 UUID。错误汇总只计数和最多 5 个编号，丢掉含邮箱、额外日期或非 ASCII 文本的行。要等生产容器按新编排重建后，轮转才生效。
|
||||
- 验证：两个编排文件里每个服务的日志段一致。虚构日志能按时间排序找到编号；命令注入写法被拒绝，旁边的标记文件还在。含邮箱和汉字的样本没有进入告警文本。
|
||||
- 防复发：上述合同测试和查找脚本的格式检查。
|
||||
- 相关记录：BUG-1306、BUG-1354、BUG-1355。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/beta-safety-net-20261014`。编号接在 BUG-1355 之后。
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# 生产备份恢复演练（2026-10-14）
|
||||
|
||||
给产品负责人照着做。这一次演练还原到一次性库，不碰正在服务的主库，网站不用停。真正整库还原会停写，放在文末，只有确认要覆盖现网数据时才做。
|
||||
|
||||
不要把备份文件、密钥、数据库口令、出生资料、姓名、邮箱或会话内容写进仓库、聊天或这份记录。
|
||||
|
||||
## 备份在哪里
|
||||
|
||||
- 每天北京时间 04:00，从 `main` 自动跑「Backup Production Daily」。也可以在 Gitea 里手动点一次。它只在 `main` 上跑。
|
||||
- 加密文件在生产主机 `/opt/jyotisha-production/backups/`，文件名是 `production-daily-日期T时间Z.dump.enc`。旁边有同名的 `-restore-verify.json`，里面只有行数和权限是否一致，没有用户正文。
|
||||
- 手动恢复点仍叫 `production-pre-migration-...`。每日清理只删过期的 `production-daily-` 文件，不会删手动恢复点。
|
||||
- Gitea 上的加密产物默认留 14 天。主机上的每日文件默认留 7 天。
|
||||
- 加密方式与现有恢复点相同：AES-256-CBC、PBKDF2。密钥在主机的数据库环境文件里，不要打印。
|
||||
|
||||
## 一次性库演练（不停机）
|
||||
|
||||
在生产主机上用 `deploy` 用户做。下面的库名换成当天的日期，不要用正在服务的那个库名。
|
||||
|
||||
1. 选定一个 `production-daily-*.dump.enc`。确认旁边的校验文件写着 `"ok": true` 和 `"restore_database_removed": true`。
|
||||
2. 进入 `/opt/jyotisha-production`。数据库口令和加密密钥已经在 `.env.production.database` 里。只在这台机器上读取，不要回显。
|
||||
3. 确认生产 Postgres 容器只有一个。容器名用 `docker ps` 看，下面记成 `<postgres>`。
|
||||
4. 建一次性库，名字必须不是主库名，例如 `restore_drill_20261015`：
|
||||
|
||||
```bash
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres> \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c 'CREATE DATABASE "restore_drill_20261015";'
|
||||
```
|
||||
|
||||
5. 解密并还原进这个一次性库。不要把输出重定向到主库。
|
||||
|
||||
```bash
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 \
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY \
|
||||
-in /opt/jyotisha-production/backups/production-daily-日期T时间Z.dump.enc |
|
||||
sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres> \
|
||||
pg_restore -U "$POSTGRES_USER" -d restore_drill_20261015 --exit-on-error
|
||||
```
|
||||
|
||||
6. 只记四个数字：`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表张数。不要把表里的内容抄出来。
|
||||
7. 删掉一次性库，并确认它已经不在：
|
||||
|
||||
```bash
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres> \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c 'DROP DATABASE "restore_drill_20261015" WITH (FORCE);'
|
||||
```
|
||||
|
||||
8. 把四个数字、文件名（不要附文件）、一次性库是否已删除，填进 `docs/testing/production-restore-drill-20261014.md`。
|
||||
|
||||
通过：还原命令退出码是 0，四个数字都是非负整数，一次性库已删除，主库没有被这条命令点名。
|
||||
|
||||
失败：停在失败的那一步。不要为了演练去改主库。把退出码交给验收，不要附带环境文件。
|
||||
|
||||
## 整库还原（会停机，平时不要做）
|
||||
|
||||
只有在确认现网库要被这份备份覆盖时才做。预留 30 分钟停写。这不是实测时间，数据变大就再加。
|
||||
|
||||
1. 停掉 `web` 和 `api`，让没有新的写入。Postgres 先留着。
|
||||
2. 用上面同一条解密和 `pg_restore`，目标改成主库名。这一步会覆盖主库里的现有数据。
|
||||
3. 核对四个行数，并对照校验文件里的权限三项是否为真。格式版本低于 2 的旧备份不要拿来做这件事。
|
||||
4. 再启动 `web` 和 `api`。打开 `https://jyotisha.chat/api/health`，`status` 应为 `ok`。
|
||||
5. 把实际停了多久填进演练模板。不要写用户内容。
|
||||
@@ -0,0 +1,34 @@
|
||||
# 进度 · 内测安全网（2026-10-15）
|
||||
|
||||
- 分支：`codex/beta-safety-net-20261014`
|
||||
- 基线：开工时 `origin/staging` 为 `71f7fac3a13f817f91755ff01b25b8706380ae19`
|
||||
- 前置：`09906cdc`（发布通路）和 `4c3485bc`、`3893783e`（#58）都已是这个基线的祖先。`deploy/docker-compose.server.yml` 在加日志段之前没有 #58 留下的未合入冲突。
|
||||
- 编号：本文件最大号是 BUG-1353。本单用 BUG-1354（没有每日备份）、BUG-1355（没有探活和磁盘告警）、BUG-1356（日志不轮转、不能按请求编号查）。
|
||||
|
||||
## 做了什么
|
||||
|
||||
- 新增每日备份工作流。北京时间 04:00，只在 `main` 上跑，调用现有的 `deploy/run-production-recovery.sh`。产物默认留 14 天。主机上只清理超过 7 天、且文件名严格是 `production-daily-` 的加密备份和对应校验文件。
|
||||
- 恢复脚本增加可选前缀。不设置时文件名仍是 `production-pre-migration-`。每日任务把它设为 `production-daily`。加密、还原校验、锁和磁盘 70% 拒绝没有改。
|
||||
- 探活、磁盘、错误汇总、飞书文本消息（可选签名）。没有 Webhook 时只写本机日志。
|
||||
- 两个编排文件的每个服务加上相同的日志上限：json-file，20 MB，10 个文件。镜像、环境、端口、健康检查没改。
|
||||
- 按 UUID 查 web / api / caddy 日志的只读脚本。拒绝其它写法。
|
||||
- 恢复演练步骤和空白结果模板。安装步骤写在 `deploy/README.md` 和 `BLOCKED.md`。
|
||||
|
||||
## 跑过的检查
|
||||
|
||||
| 检查 | 命令 | 结果 |
|
||||
| --- | --- | --- |
|
||||
| 安全网测试 | `.venv\Scripts\python.exe -m pytest tests/test_beta_safety_net.py -q` | 13 通过，退出码 0 |
|
||||
| 隐私标记 | `.venv\Scripts\python.exe -m pytest tests/test_repo_privacy_markers.py -q` | 退出码 0 |
|
||||
| 编排能否解析 | `docker compose -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml config --quiet` | 退出码 0。本机没有真实环境文件，检查时用了临时空文件，没有提交 |
|
||||
| shellcheck | 无 | 本机没有这条命令 |
|
||||
| 前端 tsc / lint / 全量测试 | 未跑 | 没有改网页 |
|
||||
| Python 快速门 | 未跑 | 按本轮约定不跑 |
|
||||
| 数据库测试 | 未跑 | 没有改数据库 |
|
||||
|
||||
## 偏离
|
||||
|
||||
- 没有改现有的四个生产工作流，也没有把它们的 SSH 步骤抽成共用脚本。新工作流自己复制了同一套 SSH，并调用同一个恢复脚本。
|
||||
- 没有往网页或接口日志里加新字段。现有观测记录已经有 requestId 和 errorCode。错误汇总会丢掉含有汉字、邮箱或第二个日期的整行，避免把对话带出去。因此中文报错不会进入计数。
|
||||
- 探活外发不带提交号，只带错误码 `commit_changed`。提交号留在 runner 本机状态里。
|
||||
- 本机没有生产主机，没有登录 runner，没有手动触发生产工作流。告警和定时备份要等产品安装，见 `BLOCKED.md`。
|
||||
@@ -338,7 +338,7 @@
|
||||
| `TASK-release-path-production-20261014.md` | `PROGRESS-release-path-production-20261014.md` | **发布通路 + 生产首发**：产品 10-14 授权改生产工作流（发布 SHA = 最近门禁成功提交 + 其后纯文档）；63 个迁移清单与演练脚本、首发 runbook；生产拒绝测试验证码。Grok 不 dispatch 生产工作流 | 第 2 次打回后续修 | `codex/release-path-production-20261014`（BUG-1348、BUG-1349） |
|
||||
| `TASK-release-path-production-fix-20261014.md` | `PROGRESS-release-path-production-fix-20261014.md` | **#74 第 1 次打回**：发布规则成立；测试写死 Windows 路径（Linux 全红）、主机演练依赖主机 node 且会留下生产数据副本、docs-only 头时回滚被拒；另补 CHECK_RELEASE_GATE 断言与 OTP 守卫规范化 | 已修，第 2 次仍未过 | `codex/release-path-production-20261014` |
|
||||
| `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2（产品选 A）**：恢复点要带属主与权限，旧格式不得回退；演练按备份里的授权比对，并在真实 PostgreSQL 17 上跑通一次 | 待验收 | `codex/release-path-production-20261014` |
|
||||
| `TASK-beta-safety-net-20261014.md` | — | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。**等发布通路单合入、#58 合入后开工** | 待前置 | — |
|
||||
| `TASK-beta-safety-net-20261014.md` | `PROGRESS-beta-safety-net-20261014.md` | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。发布通路与 #58 已在开工时的 `origin/staging` 上。主机安装和一次演练留给产品 | 待验收 | `codex/beta-safety-net-20261014`（BUG-1354～1356） |
|
||||
| `TASK-beta-capacity-20261014.md` | — | **内测容量**：按用户限流、重计算短排队与对话重试、兑换码当邀请码、staging 10/20 并发压测。**等 #58 合入后开工，压测等 #57 合入** | 待前置 | — |
|
||||
| `TASK-special-lagnas-native-20261008.md` | `PROGRESS-special-lagnas-native-20261008.md` | **特殊上升点原生计算（线上缺陷）**：sync6 U6 运行时调 PyJHora，生产镜像没有 PyJHora → staging 上 Hora/Ghati/Sree/Indu/Pranapada 全 blocked；原生实现、PyJHora 只作冻结对照、快速门禁止运行时 import jhora；最先做 | 已验收（已部署 staging `795b8845`，BUG-1273） | `codex/special-lagnas-native-20261008`（未上线；部署以 `/api/health` 为准） |
|
||||
| `TASK-ephemeris-nutation-20261008.md` | `PROGRESS-ephemeris-nutation-20261008.md` | **恒星黄道少扣章动（BUG-1275）**：行星 = 含章动热带视位置 − 平岁差，全体偏 Δψ（≤17″），大运日期差 2–3 天；改标准算法、PyJHora 冻结对照、scoring-12、v5 重跑 | 已验收（已部署 staging `db0134f9`，BUG-1275） | `codex/ephemeris-nutation-20261008`（未推送，未部署） |
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# 生产恢复演练结果（2026-10-14）
|
||||
|
||||
内测开放前由产品做一次。步骤见 `docs/operations/production-restore-drill-20261014.md`。不要填写姓名、出生资料、邮箱、会话或密钥。
|
||||
|
||||
| 项 | 结果 |
|
||||
| --- | --- |
|
||||
| 日期 | |
|
||||
| 用的备份文件名（只要 `production-daily-` 开头的文件名） | |
|
||||
| 还原目标 | 一次性库 / 整库（圈一个） |
|
||||
| 一次性库是否已删除 | |
|
||||
| identity.users 行数 | |
|
||||
| public.profiles 行数 | |
|
||||
| public.credit_transactions 行数 | |
|
||||
| public 表张数 | |
|
||||
| 网站是否停过 | 一次性库应为否 |
|
||||
| 停写大约多久 | 一次性库填「未停」 |
|
||||
| 是否通过 | |
|
||||
| 备注（不要写用户内容） | |
|
||||
@@ -0,0 +1,584 @@
|
||||
"""Contracts for the closed-beta safety net. Fixtures are fictional."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "backup-production-daily.yml"
|
||||
RECOVERY_WORKFLOW = ROOT / ".gitea" / "workflows" / "create-production-recovery.yml"
|
||||
RECOVERY_SCRIPT = ROOT / "deploy" / "run-production-recovery.sh"
|
||||
PRUNE_SCRIPT = ROOT / "deploy" / "prune-production-daily-backups.sh"
|
||||
HEALTH_SCRIPT = ROOT / "deploy" / "monitor" / "healthcheck-alert.sh"
|
||||
DIGEST_SCRIPT = ROOT / "deploy" / "monitor" / "error-digest.sh"
|
||||
FIND_SCRIPT = ROOT / "deploy" / "find-request-log.sh"
|
||||
NOTIFY_SCRIPT = ROOT / "deploy" / "monitor" / "notify-feishu.sh"
|
||||
SERVER_COMPOSE = ROOT / "deploy" / "docker-compose.server.yml"
|
||||
POSTGRES_COMPOSE = ROOT / "deploy" / "docker-compose.postgres.yml"
|
||||
|
||||
|
||||
def resolve_bash() -> str:
|
||||
found = shutil.which("bash")
|
||||
git_bash = Path(r"D:\Program Files\Git\bin\bash.exe")
|
||||
if os.name == "nt" and git_bash.exists():
|
||||
lowered = (found or "").lower()
|
||||
if not found or "system32" in lowered or "windowsapps" in lowered:
|
||||
return str(git_bash)
|
||||
if found:
|
||||
return found
|
||||
if git_bash.exists():
|
||||
return str(git_bash)
|
||||
return "bash"
|
||||
|
||||
|
||||
BASH = resolve_bash()
|
||||
|
||||
|
||||
def bash_path(path: Path) -> str:
|
||||
if os.name != "nt":
|
||||
return str(path)
|
||||
completed = subprocess.run(
|
||||
[BASH, "-lc", 'cygpath -u "$1"', "cygpath", str(path)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def run_bash(script: Path, extra: dict[str, str], args: list[str] | None = None) -> subprocess.CompletedProcess[str]:
|
||||
env = os.environ.copy()
|
||||
env.pop("FEISHU_WEBHOOK_URL", None)
|
||||
env.pop("FEISHU_SIGN_SECRET", None)
|
||||
env.update(extra)
|
||||
env["JYOTISHA_PYTHON"] = bash_path(Path(sys.executable))
|
||||
return subprocess.run(
|
||||
[BASH, bash_path(script), *(args or [])],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
class HealthHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
if self.path != "/health":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
kind = getattr(self.server, "kind", "ok")
|
||||
if kind == "timeout":
|
||||
time.sleep(3)
|
||||
commit = getattr(self.server, "commit", "a" * 40)
|
||||
if kind == "500":
|
||||
status, body = 500, b'{"status":"ok"}'
|
||||
elif kind == "missing":
|
||||
status, body = 200, b"{}"
|
||||
elif kind == "degraded":
|
||||
status, body = 200, json.dumps({"status": "degraded", "deployment": {"gitCommit": commit}}).encode()
|
||||
else:
|
||||
status, body = 200, json.dumps({"status": "ok", "deployment": {"gitCommit": commit}}).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
class HookHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
body = self.rfile.read(length)
|
||||
self.server.bodies.append(body)
|
||||
self.close_connection = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", "2")
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"ok")
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
def serve(handler: type[BaseHTTPRequestHandler]) -> tuple[ThreadingHTTPServer, threading.Thread]:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
return server, thread
|
||||
|
||||
|
||||
def alert_lines(state: Path) -> list[str]:
|
||||
log = state / "alerts.log"
|
||||
if not log.exists():
|
||||
return []
|
||||
return [line for line in log.read_text(encoding="utf-8").splitlines() if line]
|
||||
|
||||
|
||||
def test_daily_workflow_is_scheduled_from_main_and_reuses_recovery() -> None:
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
assert "\tif:" not in text
|
||||
assert "if: gitea.ref == 'refs/heads/main'" in text
|
||||
assert 'cron: "0 20 * * *"' in text
|
||||
assert "workflow_dispatch:" in text
|
||||
assert 'git show "$GITEA_SHA:deploy/run-production-recovery.sh"' in text
|
||||
assert "RECOVERY_BASENAME_PREFIX='production-daily'" in text
|
||||
assert "prune-production-daily-backups.sh" in text
|
||||
assert "production-daily-" in text
|
||||
assert 'artifact_retention="${ARTIFACT_RETENTION_DAYS:-14}"' in text
|
||||
assert 'local_retention="${LOCAL_RETENTION_DAYS:-7}"' in text
|
||||
assert 'process.env["INPUT_RETENTION-DAYS"]=process.env.ARTIFACT_RETENTION_DAYS' in text
|
||||
assert "set +x" in text
|
||||
assert "pg_dump" not in text
|
||||
assert "CHECK_RELEASE_GATE" not in text
|
||||
assert "observed_staging_sha" not in text
|
||||
assert "STAGING_URL" not in text
|
||||
assert "echo \"$SSH_PRIVATE_KEY" not in text
|
||||
assert "PRODUCTION_BACKUP_ENCRYPTION_KEY" not in text
|
||||
recovery = RECOVERY_WORKFLOW.read_text(encoding="utf-8")
|
||||
assert "observed_staging_sha" in recovery
|
||||
assert 'process.env["INPUT_RETENTION-DAYS"]="30"' in recovery
|
||||
|
||||
|
||||
def test_recovery_script_keeps_manual_names_unless_daily_prefix() -> None:
|
||||
text = RECOVERY_SCRIPT.read_text(encoding="utf-8")
|
||||
assert 'backup_prefix="production-pre-migration"' in text
|
||||
assert 'production-daily) backup_prefix="production-daily"' in text
|
||||
assert "openssl enc -aes-256-cbc -salt -pbkdf2" in text
|
||||
assert "usage_percent < 70" in text
|
||||
assert "flock -n 9" in text
|
||||
|
||||
|
||||
def test_prune_deletes_only_expired_daily_files(tmp_path: Path) -> None:
|
||||
backup = tmp_path / "backups"
|
||||
nested = backup / "nested"
|
||||
nested.mkdir(parents=True)
|
||||
now = 1_700_000_000
|
||||
limit = 7 * 86400
|
||||
names = {
|
||||
"production-daily-20261001T000000Z.dump.enc": now - limit - 60,
|
||||
"production-daily-20261001T000000Z-restore-verify.json": now - limit - 60,
|
||||
"production-daily-20261007T000000Z.dump.enc": now - limit,
|
||||
"production-daily-20261007T000000Z-restore-verify.json": now - limit,
|
||||
"production-daily-20261014T000000Z.dump.enc": now - 3600,
|
||||
"production-daily-20261014T000000Z-restore-verify.json": now - 3600,
|
||||
"production-pre-migration-20261001T000000Z.dump.enc": now - limit - 3600,
|
||||
"production-pre-migration-20261001T000000Z-restore-verify.json": now - limit - 3600,
|
||||
"production-daily-notes.txt": now - limit - 3600,
|
||||
"production-daily-20261001T000000Z.dump.enc.bak": now - limit - 3600,
|
||||
"unrelated.txt": now - limit - 3600,
|
||||
}
|
||||
for name, epoch in names.items():
|
||||
path = backup / name
|
||||
write_text(path, "fictional\n")
|
||||
subprocess.run(
|
||||
[BASH, "-lc", 'touch -d "@$1" "$2"', "touch", str(epoch), bash_path(path)],
|
||||
check=True,
|
||||
)
|
||||
write_text(nested / "production-daily-20261001T000000Z.dump.enc", "nested\n")
|
||||
completed = run_bash(PRUNE_SCRIPT, {}, [bash_path(backup), "7", str(now)])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
deleted = completed.stdout.splitlines()
|
||||
assert deleted == ["production-daily-20261001T000000Z.dump.enc"]
|
||||
assert not (backup / "production-daily-20261001T000000Z.dump.enc").exists()
|
||||
assert not (backup / "production-daily-20261001T000000Z-restore-verify.json").exists()
|
||||
for name in names:
|
||||
if name.startswith("production-daily-20261001T000000Z.dump") or name.startswith(
|
||||
"production-daily-20261001T000000Z-restore"
|
||||
):
|
||||
continue
|
||||
assert (backup / name).exists(), name
|
||||
assert (nested / "production-daily-20261001T000000Z.dump.enc").exists()
|
||||
|
||||
|
||||
def test_health_alerts_after_three_failures_and_recovery_once(tmp_path: Path) -> None:
|
||||
server, _thread = serve(HealthHandler)
|
||||
server.kind = "500"
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
for _ in range(2):
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert alert_lines(state) == []
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "errorCode=health_down" in lines[0]
|
||||
assert "count=3" in lines[0]
|
||||
assert "route=/api/health" in lines[0]
|
||||
server.kind = "ok"
|
||||
server.commit = "b" * 40
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 2
|
||||
assert "errorCode=recovered" in lines[1]
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert len(alert_lines(state)) == 2
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_health_timeout_and_missing_fields_count_as_failures(tmp_path: Path) -> None:
|
||||
server, _thread = serve(HealthHandler)
|
||||
server.kind = "timeout"
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "1",
|
||||
}
|
||||
try:
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert alert_lines(state) == []
|
||||
server.kind = "missing"
|
||||
run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert any("errorCode=health_down" in line for line in alert_lines(state))
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_commit_change_alerts_once_outside_release_window(tmp_path: Path) -> None:
|
||||
server, _thread = serve(HealthHandler)
|
||||
server.kind = "ok"
|
||||
server.commit = "a" * 40
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{server.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert alert_lines(state) == []
|
||||
server.commit = "b" * 40
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "errorCode=commit_changed" in lines[0]
|
||||
assert "b" * 40 not in lines[0]
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert len(alert_lines(state)) == 1
|
||||
marker = state / "release-window"
|
||||
state.mkdir(parents=True, exist_ok=True)
|
||||
write_text(marker, "open\n")
|
||||
server.commit = "c" * 40
|
||||
assert run_bash(HEALTH_SCRIPT, env, ["--mode", "health"]).returncode == 0
|
||||
assert len(alert_lines(state)) == 1
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_disk_alerts_once_and_no_webhook_stays_local(tmp_path: Path) -> None:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
write_text(
|
||||
bin_dir / "df",
|
||||
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' \"overlay 100 90 10 ${FAKE_DISK_PERCENT:?}% /\"\n",
|
||||
)
|
||||
subprocess.run([BASH, "-lc", 'chmod +x "$1/df"', "chmod", bash_path(bin_dir)], check=True)
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"FAKE_DISK_PERCENT": "81",
|
||||
}
|
||||
env.pop("FEISHU_WEBHOOK_URL", None)
|
||||
env.pop("FEISHU_SIGN_SECRET", None)
|
||||
completed = subprocess.run(
|
||||
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
|
||||
cwd=ROOT,
|
||||
env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert "errorCode=disk_high" in alert_lines(state)[0]
|
||||
assert "count=81" in alert_lines(state)[0]
|
||||
env["FAKE_DISK_PERCENT"] = "81"
|
||||
again = subprocess.run(
|
||||
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
|
||||
cwd=ROOT,
|
||||
env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
assert again.returncode == 0, again.stderr
|
||||
assert len(alert_lines(state)) == 1
|
||||
env["FAKE_DISK_PERCENT"] = "10"
|
||||
recovered = subprocess.run(
|
||||
[BASH, "-lc", 'export PATH="$1:$PATH"; exec bash "$2" --mode disk', "bash", bash_path(bin_dir), bash_path(HEALTH_SCRIPT)],
|
||||
cwd=ROOT,
|
||||
env={**os.environ, **env, "JYOTISHA_PYTHON": bash_path(Path(sys.executable))},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
assert recovered.returncode == 0, recovered.stderr
|
||||
assert len(alert_lines(state)) == 2
|
||||
assert "errorCode=recovered" in alert_lines(state)[1]
|
||||
|
||||
|
||||
def test_feishu_payload_keeps_only_allowed_fields(tmp_path: Path) -> None:
|
||||
hook, _thread = serve(HookHandler)
|
||||
hook.bodies = []
|
||||
health, _thread = serve(HealthHandler)
|
||||
health.kind = "500"
|
||||
state = tmp_path / "state"
|
||||
secret = "fixture-sign-secret"
|
||||
env_file = tmp_path / "monitor.env"
|
||||
write_text(env_file, f"FEISHU_WEBHOOK_URL=http://127.0.0.1:{hook.server_address[1]}/hook\nFEISHU_SIGN_SECRET={secret}\n")
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(env_file),
|
||||
"JYOTISHA_HEALTH_URL": f"http://127.0.0.1:{health.server_address[1]}/health",
|
||||
"JYOTISHA_HEALTH_TIMEOUT": "2",
|
||||
}
|
||||
try:
|
||||
for _ in range(3):
|
||||
completed = run_bash(HEALTH_SCRIPT, env, ["--mode", "health"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert len(hook.bodies) == 1
|
||||
payload = json.loads(hook.bodies[0].decode("utf-8"))
|
||||
assert set(payload) == {"timestamp", "sign", "msg_type", "content"}
|
||||
assert payload["msg_type"] == "text"
|
||||
text = payload["content"]["text"]
|
||||
assert secret not in text
|
||||
assert secret not in hook.bodies[0].decode("utf-8")
|
||||
assert "errorCode=health_down" in text
|
||||
assert "@" not in text
|
||||
expected = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import base64,hashlib,hmac,os,sys; ts=sys.argv[1]; secret=sys.argv[2]; key=f'{ts}\\n{secret}'.encode(); print(base64.b64encode(hmac.new(key, digestmod=hashlib.sha256).digest()).decode())",
|
||||
payload["timestamp"],
|
||||
secret,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
assert payload["sign"] == expected
|
||||
finally:
|
||||
hook.shutdown()
|
||||
health.shutdown()
|
||||
|
||||
|
||||
def uuid(index: int) -> str:
|
||||
return f"00000000-0000-4000-8000-{index:012d}"
|
||||
|
||||
|
||||
def test_error_digest_drops_personal_text_and_respects_thresholds(tmp_path: Path) -> None:
|
||||
safe = [
|
||||
f"2026-10-15T00:00:{index:02d}Z web [agent-observability] {json.dumps({'requestId': uuid(index), 'errorCode': 'settlement_failed'})}"
|
||||
for index in range(1, 11)
|
||||
]
|
||||
unsafe = [
|
||||
"2026-10-15T00:01:00Z web person@example.com errorCode=should_drop",
|
||||
"2026-10-15T00:01:01Z web note 1900-01-01 birth_date",
|
||||
"2026-10-15T00:01:02Z web 对话内容 今天工作不顺",
|
||||
]
|
||||
fixture = tmp_path / "logs.txt"
|
||||
write_text(fixture, "\n".join([*safe, *unsafe]) + "\n")
|
||||
state = tmp_path / "state"
|
||||
env = {
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_LOG_FIXTURE": bash_path(fixture),
|
||||
"JYOTISHA_NOW": "2026-10-15T01:00:00Z",
|
||||
}
|
||||
below = tmp_path / "below.txt"
|
||||
write_text(below, "\n".join(safe[:9]) + "\n")
|
||||
quiet = run_bash(DIGEST_SCRIPT, {**env, "JYOTISHA_LOG_FIXTURE": bash_path(below)}, ["--window"])
|
||||
assert quiet.returncode == 0, quiet.stderr
|
||||
assert alert_lines(state) == []
|
||||
completed = run_bash(DIGEST_SCRIPT, env, ["--window"])
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "errorCode=settlement_failed" in lines[0]
|
||||
assert "count=10" in lines[0]
|
||||
assert lines[0].count(",") == 4
|
||||
assert uuid(6) not in lines[0]
|
||||
assert "person@example.com" not in lines[0]
|
||||
assert "1900-01-01" not in lines[0]
|
||||
assert "今天工作不顺" not in lines[0]
|
||||
assert "should_drop" not in lines[0]
|
||||
|
||||
fivexx = tmp_path / "fivexx.txt"
|
||||
write_text(fivexx, "\n".join(f"2026-10-15T00:02:{index:02d}Z api status=502" for index in range(20)) + "\n")
|
||||
state_5xx = tmp_path / "state5"
|
||||
completed = run_bash(
|
||||
DIGEST_SCRIPT,
|
||||
{**env, "JYOTISHA_MONITOR_STATE_DIR": bash_path(state_5xx), "JYOTISHA_LOG_FIXTURE": bash_path(fivexx)},
|
||||
["--window"],
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
joined = "\n".join(alert_lines(state_5xx))
|
||||
assert "errorCode=http_5xx" in joined
|
||||
assert "count=20" in joined
|
||||
|
||||
|
||||
def test_daily_digest_summarizes_previous_shanghai_day(tmp_path: Path) -> None:
|
||||
fixture = tmp_path / "daily.txt"
|
||||
write_text(
|
||||
fixture,
|
||||
"\n".join(
|
||||
[
|
||||
'2026-10-14T02:00:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000001","errorCode":"settlement_failed"}',
|
||||
'2026-10-15T01:30:00Z api [agent-observability] {"requestId":"00000000-0000-4000-8000-000000000002","errorCode":"settlement_failed"}',
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
state = tmp_path / "state"
|
||||
completed = run_bash(
|
||||
DIGEST_SCRIPT,
|
||||
{
|
||||
"JYOTISHA_MONITOR_STATE_DIR": bash_path(state),
|
||||
"JYOTISHA_MONITOR_ENV": bash_path(tmp_path / "missing.env"),
|
||||
"JYOTISHA_LOG_FIXTURE": bash_path(fixture),
|
||||
"JYOTISHA_NOW": "2026-10-15T01:00:00Z",
|
||||
},
|
||||
["--daily"],
|
||||
)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
lines = alert_lines(state)
|
||||
assert len(lines) == 1
|
||||
assert "count=1" in lines[0]
|
||||
assert "00000000-0000-4000-8000-000000000001" in lines[0]
|
||||
assert "00000000-0000-4000-8000-000000000002" not in lines[0]
|
||||
|
||||
|
||||
def test_find_request_log_sorts_and_rejects_injection(tmp_path: Path) -> None:
|
||||
log_dir = tmp_path / "logs"
|
||||
log_dir.mkdir()
|
||||
request_id = "11111111-2222-4333-8444-555555555555"
|
||||
write_text(log_dir / "caddy.log", f"2026-10-15T03:00:00Z caddy {request_id}\n")
|
||||
write_text(log_dir / "web.log", f"2026-10-15T01:00:00Z web {request_id}\nignore\n")
|
||||
write_text(log_dir / "api.log", f"2026-10-15T02:00:00Z api {request_id}\n")
|
||||
canary = tmp_path / "canary.txt"
|
||||
write_text(canary, "keep\n")
|
||||
found = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h"])
|
||||
assert found.returncode == 0, found.stderr
|
||||
assert found.stdout.splitlines() == [
|
||||
f"2026-10-15T01:00:00Z web {request_id}",
|
||||
f"2026-10-15T02:00:00Z api {request_id}",
|
||||
f"2026-10-15T03:00:00Z caddy {request_id}",
|
||||
]
|
||||
rejected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["; rm -rf /", "--since", "24h"])
|
||||
assert rejected.returncode == 2
|
||||
assert canary.read_text(encoding="utf-8") == "keep\n"
|
||||
injected = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, ["$(rm canary.txt)", "--since", "1h"])
|
||||
assert injected.returncode == 2
|
||||
assert canary.exists()
|
||||
bad_since = run_bash(FIND_SCRIPT, {"JYOTISHA_LOG_DIR": bash_path(log_dir)}, [request_id, "--since", "24h;rm"])
|
||||
assert bad_since.returncode == 2
|
||||
|
||||
|
||||
def service_logging_blocks(text: str) -> dict[str, str]:
|
||||
blocks: dict[str, str] = {}
|
||||
service: str | None = None
|
||||
in_services = False
|
||||
capturing = False
|
||||
buf: list[str] = []
|
||||
for line in text.splitlines():
|
||||
if line.startswith("services:"):
|
||||
in_services = True
|
||||
continue
|
||||
if in_services and line and not line.startswith(" "):
|
||||
in_services = False
|
||||
if not in_services:
|
||||
continue
|
||||
if line.startswith(" ") and not line.startswith(" ") and line.endswith(":"):
|
||||
if capturing and service is not None:
|
||||
blocks[service] = "\n".join(buf)
|
||||
service = line.strip()[:-1]
|
||||
capturing = False
|
||||
buf = []
|
||||
continue
|
||||
if service and line == " logging:":
|
||||
capturing = True
|
||||
buf = [line]
|
||||
continue
|
||||
if capturing:
|
||||
if line.startswith(" ") and not line.startswith(" "):
|
||||
blocks[service] = "\n".join(buf)
|
||||
capturing = False
|
||||
buf = []
|
||||
else:
|
||||
buf.append(line)
|
||||
if capturing and service is not None:
|
||||
blocks[service] = "\n".join(buf)
|
||||
return blocks
|
||||
|
||||
|
||||
def test_compose_logging_is_present_and_identical() -> None:
|
||||
expected = "\n".join(
|
||||
[
|
||||
" logging:",
|
||||
" driver: json-file",
|
||||
" options:",
|
||||
' max-size: "20m"',
|
||||
' max-file: "10"',
|
||||
]
|
||||
)
|
||||
expected_names = {
|
||||
SERVER_COMPOSE: {"api", "web", "caddy"},
|
||||
POSTGRES_COMPOSE: {"postgres", "migrator", "migration-checker"},
|
||||
}
|
||||
for path, names in expected_names.items():
|
||||
blocks = service_logging_blocks(path.read_text(encoding="utf-8"))
|
||||
assert set(blocks) == names
|
||||
for name in names:
|
||||
assert blocks[name] == expected, name
|
||||
|
||||
|
||||
def test_shell_scripts_pass_bash_n() -> None:
|
||||
for script in (PRUNE_SCRIPT, HEALTH_SCRIPT, DIGEST_SCRIPT, FIND_SCRIPT, NOTIFY_SCRIPT, RECOVERY_SCRIPT):
|
||||
completed = subprocess.run(
|
||||
[BASH, "-n", bash_path(script)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert completed.returncode == 0, script.name + completed.stderr
|
||||
Reference in New Issue
Block a user