diff --git a/deploy/README.md b/deploy/README.md index 17aaabbd..5e468154 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -254,7 +254,7 @@ The deploy and migration workflows share the `staging-mutation` Actions concurre Use Gitea Actions → **Create Production Recovery Point** from the exact current `main` release SHA before every production schema migration. The manual workflow requires `main`, `staging`, public staging health, and the successful release gate to identify the same SHA. It uses the pinned production SSH identity, shares the `production-mutation` lock, and runs `deploy/run-production-recovery.sh` with shell tracing disabled. -The host script writes an AES-256-CBC/PBKDF2 encrypted custom-format PostgreSQL dump under `/opt/jyotisha-production/backups`, restores it into a uniquely named disposable database, validates non-sensitive row/table counts, removes only that disposable database, and writes a mode-`0600` verification manifest. Before acquiring the shared host lock it validates that `.state` and `backups` are real directories, then uses the constrained deployment-ownership sudo boundary to restore only those directories and an existing regular `mutation.lock` to the current `deploy` UID/GID; it never removes or replaces a lock inode. The workflow retrieves only the encrypted dump and verification metadata, verifies the SHA-256 digest, and uploads them as a 30-day Gitea Actions artifact with compression disabled. The artifact-backed `recovery_reference`, `recovery_created_at`, and `restore_verified=true` output are the inputs for **Migrate Production Database**. Do not use the attestation if backup, restore, retrieval, digest validation, ownership normalization, or artifact upload fails. +The host script writes an AES-256-CBC/PBKDF2 encrypted custom-format PostgreSQL dump under `/opt/jyotisha-production/backups`, restores it into a uniquely named disposable database, validates non-sensitive row/table counts, removes only that disposable database, and writes a mode-`0600` verification manifest. Before acquiring the shared host lock it validates that `.state` and `backups` are real directories, then reuses the existing passwordless Docker boundary to run the already-loaded PostgreSQL image with no network, a read-only root filesystem, all capabilities dropped except `CHOWN`, and only those two directories bind-mounted. That helper restores only the directory mount points and an existing regular `mutation.lock` to the current `deploy` UID/GID; it never recursively changes backup files or removes or replaces a lock inode. The workflow retrieves only the encrypted dump and verification metadata, verifies the SHA-256 digest, and uploads them as a 30-day Gitea Actions artifact with compression disabled. The artifact-backed `recovery_reference`, `recovery_created_at`, and `restore_verified=true` output are the inputs for **Migrate Production Database**. Do not use the attestation if backup, restore, retrieval, digest validation, ownership normalization, or artifact upload fails. Never restore over `jyotisha`, delete the PostgreSQL volume, print `.env.production.database`, expose `PRODUCTION_BACKUP_ENCRYPTION_KEY`, or substitute a staging recovery artifact. The encrypted local archive is preserved for repair; the Actions artifact supplies the required off-host copy. diff --git a/deploy/run-production-recovery.sh b/deploy/run-production-recovery.sh index fc5ad208..ebfc39f2 100755 --- a/deploy/run-production-recovery.sh +++ b/deploy/run-production-recovery.sh @@ -35,14 +35,38 @@ done install -d -m 700 "$state_directory" "$backup_directory" deployment_uid="$(id -u)" deployment_gid="$(id -g)" -sudo -n chown "$deployment_uid:$deployment_gid" "$state_directory" "$backup_directory" -chmod 700 "$state_directory" "$backup_directory" +ownership_targets=("$state_directory" "$backup_directory") if [ -e "$lock_file" ]; then [ -f "$lock_file" ] && [ ! -L "$lock_file" ] || { echo "production mutation lock is unsafe" >&2 exit 1 } - sudo -n chown "$deployment_uid:$deployment_gid" "$lock_file" + ownership_targets+=("$lock_file") +fi + +mapfile -t postgres_containers < <( + sudo -n docker ps -q \ + --filter 'label=com.docker.compose.project=jyotisha-production' \ + --filter 'label=com.docker.compose.service=postgres' +) +[ "${#postgres_containers[@]}" -eq 1 ] || { + echo "expected exactly one running production PostgreSQL container" >&2 + exit 1 +} +postgres_container="${postgres_containers[0]}" +ownership_image="$(sudo -n docker inspect --format '{{.Image}}' "$postgres_container")" +[[ "$ownership_image" =~ ^sha256:[0-9a-f]{64}$ ]] || { + echo "production PostgreSQL image identity is unsafe" >&2 + exit 1 +} +sudo -n docker run --rm --pull never --network none --read-only --user 0:0 \ + --cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges \ + --volume "$state_directory:$state_directory" \ + --volume "$backup_directory:$backup_directory" \ + --entrypoint chown "$ownership_image" \ + "$deployment_uid:$deployment_gid" "${ownership_targets[@]}" +chmod 700 "$state_directory" "$backup_directory" +if [ -e "$lock_file" ]; then chmod 600 "$lock_file" fi @@ -64,17 +88,6 @@ usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); p exit 1 } -mapfile -t postgres_containers < <( - sudo -n docker ps -q \ - --filter 'label=com.docker.compose.project=jyotisha-production' \ - --filter 'label=com.docker.compose.service=postgres' -) -[ "${#postgres_containers[@]}" -eq 1 ] || { - echo "expected exactly one running production PostgreSQL container" >&2 - exit 1 -} -postgres_container="${postgres_containers[0]}" - created_compact="$(date -u +%Y%m%dT%H%M%SZ)" created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" backup_basename="production-pre-migration-${created_compact}.dump.enc" diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index a4543f0f..c83b977c 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -3415,10 +3415,10 @@ - 首次发现：2026-08-15 - 最近更新：2026-08-15 - 影响面：`Create Production Recovery Point`、生产 schema migration 前置恢复门禁，以及后续 production deploy。 -- 用户现象：`main` 与 `staging` 已同步且 release gate 成功，但生产恢复 workflow Run 1865 在创建备份前失败，日志为 `/opt/jyotisha-production/.state/mutation.lock: Permission denied`；迁移和部署因此保持阻断，生产运行版本未改变。 -- 根因：生产 bootstrap 遗留的 `.state` 路径或既有 lock 仍为非 `deploy` 所有。原恢复脚本只执行 `install -d -m 700`；对已经存在的目录该命令不会恢复所有权，随后由 `deploy` 打开共享锁即被内核拒绝。失败发生在 `pg_dump` 前，因此本轮没有生成可用恢复证明，也没有执行 schema migration。 -- 修复：恢复脚本先对 `.state`、`backups` 和既有 `mutation.lock` 做类型与非符号链接校验，再通过已审查的部署树 sudo 边界仅恢复这些路径到当前 `deploy` UID/GID 和私有权限；不递归改动备份文件、不删除或替换 lock inode，随后仍用同一个 `flock -n` fail-closed 获取共享 host lock。同步更新生产 runbook 和静态安全契约测试。 +- 用户现象：`main` 与 `staging` 已同步且 release gate 成功，但生产恢复 workflow Run 1865 在创建备份前失败，日志为 `/opt/jyotisha-production/.state/mutation.lock: Permission denied`。首次修复后的 Run 1869 仍在 `pg_dump` 前失败，准确日志为 `sudo: a password is required`；迁移和部署因此持续阻断，生产运行版本未改变。 +- 根因：生产 bootstrap 遗留的 `.state` 路径或既有 lock 仍为非 `deploy` 所有。原恢复脚本只执行 `install -d -m 700`；对已经存在的目录该命令不会恢复所有权，随后由 `deploy` 打开共享锁即被内核拒绝。首次修复又错误假设主机已为 `deploy` 配置独立的免密 `chown`，但真实生产 sudo 边界只允许已审查的 Docker 命令，因此 `sudo -n chown` 立即失败。两次失败都发生在 `pg_dump` 前，没有生成可用恢复证明，也没有执行 schema migration。 +- 修复：恢复脚本先对 `.state`、`backups` 和既有 `mutation.lock` 做类型与非符号链接校验，取得当前运行 PostgreSQL 容器的不可变本地 image ID，再复用既有 `sudo -n docker` 边界启动一次性所有权修复容器：`--pull never`、无网络、只读根文件系统、`no-new-privileges`、删除全部 capability 后只保留 `CHOWN`，并且仅绑定 `.state` 与 `backups`。helper 只恢复两个目录 mount point 和既有普通 lock 到当前 `deploy` UID/GID；不递归改动历史备份、不删除或替换 lock inode，随后仍用同一个 `flock -n` fail-closed 获取共享 host lock。同步更新生产 runbook 和静态安全契约测试，明确不得再扩大主机 sudoers。 - 验证：本地 `bash -n`、YAML parse、聚焦 workflow contract 与 `git diff --check` 必须通过；远端必须重新完成 staging quality/deploy、exact-SHA release gate、真实 production dump + disposable restore + off-site artifact，再允许 migration/deploy。 - 防复发：生产私有状态目录必须保持 `deploy:deploy 0700`，共享 lock 必须是普通非 symlink 文件且不可通过删除重建来“修复”；任何恢复流程失败都不得手填 `restore_verified=true` 或跳过恢复门禁。 -- 相关记录：生产迁移 runbook、Run 1865 +- 相关记录：生产迁移 runbook、Run 1865、Run 1869 - 修复版本：待提交（精确 SHA 以重新发布后的远端分支与 production health 为准） diff --git a/docs/operations/production-server-migration-2026-08.md b/docs/operations/production-server-migration-2026-08.md index 40e5c73e..9807b5f7 100644 --- a/docs/operations/production-server-migration-2026-08.md +++ b/docs/operations/production-server-migration-2026-08.md @@ -60,7 +60,7 @@ Do not put the Ubuntu password, database URLs, Resend key, payment key, model-pr Perform this interactively before any workflow dispatch: 1. Patch Ubuntu and install Docker Engine, Compose v2, `rsync`, `curl`, `jq`, `flock`, and UFW. -2. Create a non-root `deploy` user, install a dedicated Ed25519 public key, and grant only the reviewed passwordless commands needed for Docker and deployment-tree ownership. +2. Create a non-root `deploy` user, install a dedicated Ed25519 public key, and grant only the reviewed passwordless Docker command used by the deployment workflows. Ownership normalization must reuse the capability-constrained helper container described below rather than adding a separate passwordless host `chown` rule. 3. Verify a second key-only session and rotate the exposed bootstrap password. For this host, the production owner explicitly requires password authentication to remain enabled for other operators; do not change `PasswordAuthentication`. Workflows must still use the dedicated deploy key. 4. Permit only the confirmed SSH port plus `80/tcp`, `443/tcp`, and `443/udp`. Do not publish `3000`, `5200`, `5432`, or the Docker API. 5. Create a 2–4 GB swap file and enable Docker log rotation. Keep at least 15 GB free before the first image pull and database import. @@ -90,7 +90,7 @@ Use distinct production credentials for PostgreSQL roles, Better Auth, Resend, b ## Database migration engineering gate -Before importing data, dispatch Gitea Actions → `Create Production Recovery Point` for the exact accepted release SHA. It validates the private production `.state` and `backups` paths, restores only their documented `deploy` ownership and an existing regular shared lock when bootstrap ownership has drifted, creates an AES-256-CBC/PBKDF2 encrypted custom-format dump on the production host, restores it into a disposable database, verifies non-sensitive table counts, removes only the disposable database, and uploads the encrypted dump plus verification manifest as a 30-day Gitea Actions artifact. It must preserve the existing lock inode and fail closed on symlinks or non-regular lock paths. The workflow prints the non-sensitive `recovery_reference` and exact UTC `recovery_created_at`; use those values only after the artifact upload succeeds. +Before importing data, dispatch Gitea Actions → `Create Production Recovery Point` for the exact accepted release SHA. It validates the private production `.state` and `backups` paths, obtains the immutable image ID of the already-running PostgreSQL container, and reuses the existing passwordless Docker boundary to run that local image with `--pull never`, no network, a read-only root filesystem, `no-new-privileges`, and only `CHOWN` retained. With only `.state` and `backups` bind-mounted, it restores the two directory mount points and an existing regular shared lock to documented `deploy` ownership when bootstrap ownership has drifted. It then creates an AES-256-CBC/PBKDF2 encrypted custom-format dump on the production host, restores it into a disposable database, verifies non-sensitive table counts, removes only that disposable database, and uploads the encrypted dump plus verification manifest as a 30-day Gitea Actions artifact. It must preserve the existing lock inode, avoid recursively changing historical backups, and fail closed on symlinks, non-regular lock paths, or an unexpected image identity. The workflow prints the non-sensitive `recovery_reference` and exact UTC `recovery_created_at`; use those values only after the artifact upload succeeds. Then dispatch Gitea Actions → `Migrate Production Database` for the same exact accepted release SHA. The workflow requires `main == staging == deploy_sha`, the same successful staging backend gate, the same manual release gate, and the public staging `/api/health` identity for that SHA. It also requires the recovery workflow's non-sensitive reference, exact UTC creation time, and `restore_verified=true`; the recovery point must be no more than 24 hours old and must already have passed the restore verification. It verifies the current production revision, obtains the gate-attested immutable Web image, runs the schema checker, applies only pending application schema migrations, and requires the checker to converge afterward. diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index b99e4606..904437df 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -1086,9 +1086,13 @@ test("production recovery workflow creates a verified encrypted off-site artifac assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/); assert.match(runner, /for directory in "\$state_directory" "\$backup_directory"/); - assert.match(runner, /sudo -n chown "\$deployment_uid:\$deployment_gid" "\$state_directory" "\$backup_directory"/); + assert.doesNotMatch(runner, /sudo -n chown/); + assert.match(runner, /ownership_image=.*docker inspect --format '\{\{\.Image\}\}'/); + assert.match(runner, /sudo -n docker run --rm --pull never --network none --read-only --user 0:0/); + assert.match(runner, /--cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges/); + assert.match(runner, /--entrypoint chown "\$ownership_image"/); + assert.match(runner, /"\$deployment_uid:\$deployment_gid" "\$\{ownership_targets\[@\]\}"/); assert.match(runner, /production mutation lock is unsafe/); - assert.match(runner, /sudo -n chown "\$deployment_uid:\$deployment_gid" "\$lock_file"/); assert.doesNotMatch(runner, /rm -f[^\n]*mutation\.lock|chown -R/); assert.match(runner, /another production mutation holds the host lock/); assert.match(runner, /usage_percent < 70/);