ops: automate production recovery attestation

.gitea/workflows/backend-quality-gate.yml

@@ -7,6 +7,7 @@ on:
       - '.gitea/workflows/deploy-staging.yml'
       - '.gitea/workflows/migrate-staging-database.yml'
       - '.gitea/workflows/migrate-production-database.yml'
+      - '.gitea/workflows/create-production-recovery.yml'
       - 'deploy/**'
       - 'frontend/**'
       - 'jyotish_vedic/**'

.gitea/workflows/create-production-recovery.yml

@@ -0,0 +1,213 @@
+name: Create Production Recovery Point (manual only)
+
+on:
+  workflow_dispatch:
+    inputs:
+      deploy_sha:
+        description: Exact accepted production release SHA this recovery point protects
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  actions: write
+
+concurrency:
+  group: production-mutation
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  recover:
+    runs-on: manman-linux
+    timeout-minutes: 30
+    env:
+      GITEA_SHA: ${{ gitea.sha }}
+      GITEA_API_URL: ${{ gitea.api_url }}
+      GITEA_REPOSITORY: ${{ gitea.repository }}
+      GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+      DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }}
+      DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }}
+      DEPLOY_USER: ${{ vars.PRODUCTION_USER }}
+      DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }}
+      STAGING_URL: ${{ vars.STAGING_URL }}
+      PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }}
+    steps:
+      - name: Validate exact accepted release
+        id: revision
+        env:
+          DEPLOY_SHA: ${{ inputs.deploy_sha }}
+        run: |
+          set -euo pipefail
+          [[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; }
+          [[ "$GITEA_SHA" == "$DEPLOY_SHA" ]] || { echo "dispatch recovery from the exact main release SHA" >&2; exit 1; }
+          [[ "$STAGING_URL" == "https://staging.jyotisha.chat" ]] || { echo "unexpected staging acceptance URL" >&2; exit 1; }
+          read_ref_sha() {
+            local branch="$1"
+            curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
+              --header "Authorization: token $GITEA_TOKEN" \
+              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" |
+              jq -er --arg ref "refs/heads/$branch" '
+                select(type == "array" and length == 1) |
+                .[0] | select(.ref == $ref) | .object.sha |
+                select(test("^[0-9a-f]{40}$"))
+              '
+          }
+          staging_head="$(read_ref_sha staging)"
+          main_head="$(read_ref_sha main)"
+          [[ "$main_head" == "$DEPLOY_SHA" && "$staging_head" == "$DEPLOY_SHA" ]] || {
+            echo "production recovery requires main and staging to equal deploy_sha" >&2
+            exit 1
+          }
+          observed_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 30 --retry 3 --retry-all-errors \
+            "$STAGING_URL/api/health" | jq -er '.deployment.gitCommit | select(test("^[0-9a-f]{40}$"))')"
+          [[ "$observed_staging_sha" == "$DEPLOY_SHA" ]] || {
+            echo "public staging has not accepted the requested SHA" >&2
+            exit 1
+          }
+          release_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
+            --header "Authorization: token $GITEA_TOKEN" \
+            "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&event=workflow_dispatch&status=success&limit=100")"
+          jq -e --arg sha "$DEPLOY_SHA" '
+            any(.workflow_runs[]?;
+              (.path | split("@")[0] | endswith("release-quality-gate.yml")) and
+              .head_sha == $sha and .event == "workflow_dispatch" and .conclusion == "success"
+            )
+          ' <<<"$release_runs" >/dev/null || {
+            echo "no successful exact-SHA manual release quality gate found" >&2
+            exit 1
+          }
+          echo "sha=$DEPLOY_SHA" >>"$GITHUB_OUTPUT"
+
+      - name: Checkout exact recovery controller
+        env:
+          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
+        run: |
+          set -euo pipefail
+          git init .
+          git remote remove origin 2>/dev/null || true
+          git remote add origin https://git.copse.top/root/Jyotisha.git
+          git fetch --no-tags origin "$DEPLOY_SHA"
+          git checkout --detach --force "$DEPLOY_SHA"
+          [[ "$(git rev-parse HEAD)" == "$DEPLOY_SHA" ]]
+
+      - name: Create, restore-verify, and retrieve encrypted recovery point
+        id: recovery
+        env:
+          SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }}
+          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
+        run: |
+          set -euo pipefail
+          set +x
+          [[ "$DEPLOY_HOST" == "118.194.235.34" ]]
+          [[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 ))
+          [[ "$DEPLOY_USER" == "deploy" ]]
+          [[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]]
+          [[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]]
+          test -n "$PRODUCTION_KNOWN_HOSTS"
+          ssh_root="${RUNNER_TEMP}/production-recovery-ssh"
+          key_path="$ssh_root/id_ed25519"
+          known_hosts_path="$ssh_root/known_hosts"
+          artifact_directory="artifacts/production-recovery"
+          install -m 700 -d "$ssh_root" "$artifact_directory"
+          test -n "$SSH_PRIVATE_KEY_BASE64"
+          printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode >"$key_path"
+          printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' >"$known_hosts_path"
+          chmod 600 "$key_path" "$known_hosts_path"
+          ssh-keygen -y -f "$key_path" >/dev/null
+          ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
+          scp_options=(-i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
+          remote="$DEPLOY_USER@$DEPLOY_HOST"
+          incoming="$(ssh "${ssh_options[@]}" "$remote" 'mktemp -d /tmp/jyotisha-production-recovery.XXXXXXXXXX')"
+          [[ "$incoming" == /tmp/jyotisha-production-recovery.* ]]
+          cleanup() {
+            ssh "${ssh_options[@]}" "$remote" "rm -rf -- '$incoming'" >/dev/null 2>&1 || true
+            rm -rf -- "$ssh_root"
+          }
+          trap cleanup EXIT
+          git show "$DEPLOY_SHA:deploy/run-production-recovery.sh" >"$ssh_root/run-production-recovery.sh"
+          chmod 700 "$ssh_root/run-production-recovery.sh"
+          scp "${scp_options[@]}" "$ssh_root/run-production-recovery.sh" "$remote:$incoming/run-production-recovery.sh"
+          ssh "${ssh_options[@]}" "$remote" \
+            "DEPLOY_PATH='$DEPLOY_PATH' RECOVERY_RUN_ID='$GITHUB_RUN_ID' bash '$incoming/run-production-recovery.sh'" \
+            >"$ssh_root/recovery-output.env"
+          [[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d ' ')" == 7 ]]
+          recovery_reference="$(awk -F= '$1 == "RECOVERY_REFERENCE" {print $2}' "$ssh_root/recovery-output.env")"
+          recovery_created_at="$(awk -F= '$1 == "RECOVERY_CREATED_AT" {print $2}' "$ssh_root/recovery-output.env")"
+          recovery_verified_at="$(awk -F= '$1 == "RECOVERY_VERIFIED_AT" {print $2}' "$ssh_root/recovery-output.env")"
+          recovery_sha256="$(awk -F= '$1 == "RECOVERY_SHA256" {print $2}' "$ssh_root/recovery-output.env")"
+          restore_verified="$(awk -F= '$1 == "RESTORE_VERIFIED" {print $2}' "$ssh_root/recovery-output.env")"
+          backup_basename="$(awk -F= '$1 == "BACKUP_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
+          verify_basename="$(awk -F= '$1 == "VERIFY_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
+          [[ "$backup_basename" =~ ^production-pre-migration-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]]
+          [[ "$verify_basename" == "${backup_basename%.dump.enc}-restore-verify.json" ]]
+          [[ "$recovery_reference" == "gitea-actions-run-${GITHUB_RUN_ID}/${backup_basename}" ]]
+          [[ "$recovery_created_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
+          [[ "$recovery_verified_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
+          [[ "$recovery_sha256" =~ ^[0-9a-f]{64}$ ]]
+          [[ "$restore_verified" == "true" ]]
+          scp "${scp_options[@]}" \
+            "$remote:$DEPLOY_PATH/backups/$backup_basename" \
+            "$remote:$DEPLOY_PATH/backups/$verify_basename" \
+            "$artifact_directory/"
+          test -s "$artifact_directory/$backup_basename"
+          test -s "$artifact_directory/$verify_basename"
+          printf '%s  %s\n' "$recovery_sha256" "$artifact_directory/$backup_basename" | sha256sum --check --status
+          jq -e \
+            --arg backup "$backup_basename" \
+            --arg sha "$recovery_sha256" \
+            --arg created "$recovery_created_at" \
+            --arg verified "$recovery_verified_at" '
+              .mode == "restore_verify" and .ok == true and
+              .backup == $backup and .sha256 == $sha and
+              .created_at == $created and .verified_at == $verified and
+              .restore_database_removed == true and
+              ([.checks.identity_users, .checks.profiles, .checks.credit_transactions, .checks.public_tables] |
+                all(type == "number" and . >= 0 and floor == .))
+            ' "$artifact_directory/$verify_basename" >/dev/null
+          printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
+            "$recovery_reference" "$recovery_created_at" "$recovery_verified_at" "$recovery_sha256" \
+            "$backup_basename" "$verify_basename" >"$artifact_directory/attestation.env"
+          chmod 600 "$artifact_directory"/*
+          {
+            echo "recovery_reference=$recovery_reference"
+            echo "recovery_created_at=$recovery_created_at"
+            echo "recovery_verified_at=$recovery_verified_at"
+            echo "recovery_sha256=$recovery_sha256"
+          } >>"$GITHUB_OUTPUT"
+          echo "Recovery restore-verified: reference=$recovery_reference created_at=$recovery_created_at verified_at=$recovery_verified_at"
+
+      - name: Upload encrypted off-site recovery artifact
+        run: |
+          set -euo pipefail
+          test -n "${ACTIONS_RUNTIME_TOKEN:-}"
+          test -n "${ACTIONS_RESULTS_URL:-}"
+          test -n "${GITHUB_RUN_ID:-}"
+          test -n "${GITHUB_REPOSITORY:-}"
+          workdir="$(pwd -P)"
+          docker run --rm \
+            --user "$(id -u):$(id -g)" \
+            --volume "$workdir:$workdir" \
+            --workdir "$workdir" \
+            --env HOME=/tmp \
+            --env "INPUT_NAME=production-recovery-$GITHUB_RUN_ID" \
+            --env INPUT_PATH=artifacts/production-recovery/ \
+            --env INPUT_OVERWRITE=false \
+            --env ACTIONS_RUNTIME_TOKEN \
+            --env ACTIONS_RESULTS_URL \
+            --env GITHUB_RUN_ID \
+            --env GITHUB_REPOSITORY \
+            --env "GITHUB_SHA=$GITEA_SHA" \
+            --env "GITHUB_WORKSPACE=$workdir" \
+            node:22-bookworm-slim \
+            node -e 'process.env["INPUT_IF-NO-FILES-FOUND"]="error"; process.env["INPUT_RETENTION-DAYS"]="30"; process.env["INPUT_COMPRESSION-LEVEL"]="0"; require("./.gitea/actions/upload-artifact/dist/index.js")'
+
+      - name: Publish recovery attestation
+        env:
+          RECOVERY_REFERENCE: ${{ steps.recovery.outputs.recovery_reference }}
+          RECOVERY_CREATED_AT: ${{ steps.recovery.outputs.recovery_created_at }}
+          RECOVERY_VERIFIED_AT: ${{ steps.recovery.outputs.recovery_verified_at }}
+          RECOVERY_SHA256: ${{ steps.recovery.outputs.recovery_sha256 }}
+        run: |
+          set -euo pipefail
+          echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT verified_at=$RECOVERY_VERIFIED_AT restore_verified=true sha256=$RECOVERY_SHA256"

deploy/README.md

@@ -250,6 +250,14 @@ Use this order for every staging revision:
 
 The deploy and migration workflows share the `staging-mutation` Actions concurrency group, and their live-tree sync plus Compose work runs under `/opt/jyotisha-staging/.state/mutation.lock`. The synchronized tree explicitly preserves `/backups/`, `.env*`, `.state`, and `.incoming`. The read-only checker exits before app changes when a migration is pending. Its message includes the exact SHA and the `Migrate Staging Database` workflow name. A failed migration does not re-dispatch deployment. Application rollback restores the previously recorded digest references and SHA, falling back to validated local image IDs only when transitioning from the pre-foundation local-image deployment; it does not roll back database state.
 
+### Production recovery point before schema migration
+
+Use Gitea Actions → **Create Production Recovery Point** from the exact current `main` release SHA before every production schema migration. The manual workflow requires `main`, `staging`, public staging health, and the successful release gate to identify the same SHA. It uses the pinned production SSH identity, shares the `production-mutation` lock, and runs `deploy/run-production-recovery.sh` with shell tracing disabled.
+
+The host script writes an AES-256-CBC/PBKDF2 encrypted custom-format PostgreSQL dump under `/opt/jyotisha-production/backups`, restores it into a uniquely named disposable database, validates non-sensitive row/table counts, removes only that disposable database, and writes a mode-`0600` verification manifest. The workflow retrieves only the encrypted dump and verification metadata, verifies the SHA-256 digest, and uploads them as a 30-day Gitea Actions artifact with compression disabled. The artifact-backed `recovery_reference`, `recovery_created_at`, and `restore_verified=true` output are the inputs for **Migrate Production Database**. Do not use the attestation if backup, restore, retrieval, digest validation, or artifact upload fails.
+
+Never restore over `jyotisha`, delete the PostgreSQL volume, print `.env.production.database`, expose `PRODUCTION_BACKUP_ENCRYPTION_KEY`, or substitute a staging recovery artifact. The encrypted local archive is preserved for repair; the Actions artifact supplies the required off-host copy.
+
 ### Local encrypted staging backups (three-copy limit)
 
 After the health check, run the repository backup helper from the synchronized staging checkout:

deploy/run-production-recovery.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set +x
+umask 077
+
+: "${DEPLOY_PATH:?DEPLOY_PATH is required}"
+: "${RECOVERY_RUN_ID:?RECOVERY_RUN_ID is required}"
+
+[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
+  echo "unexpected production path" >&2
+  exit 1
+}
+[[ "$RECOVERY_RUN_ID" =~ ^[0-9]+$ ]] || {
+  echo "recovery run id must be numeric" >&2
+  exit 1
+}
+
+state_directory="$DEPLOY_PATH/.state"
+backup_directory="$DEPLOY_PATH/backups"
+environment_file="$DEPLOY_PATH/.env.production.database"
+
+[ -f "$environment_file" ] && [ ! -L "$environment_file" ] || {
+  echo "production database environment file is missing or unsafe" >&2
+  exit 1
+}
+install -d -m 700 "$state_directory" "$backup_directory"
+[ ! -L "$state_directory" ] && [ ! -L "$backup_directory" ] || {
+  echo "production state or backup directory is unsafe" >&2
+  exit 1
+}
+
+exec 9>"$state_directory/mutation.lock"
+flock -n 9 || {
+  echo "another production mutation holds the host lock" >&2
+  exit 75
+}
+
+set -a
+# shellcheck disable=SC1090
+. "$environment_file"
+set +a
+: "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}"
+
+usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')"
+[[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || {
+  echo "production backup disk usage must remain below 70 percent" >&2
+  exit 1
+}
+
+mapfile -t postgres_containers < <(
+  sudo -n docker ps -q \
+    --filter 'label=com.docker.compose.project=jyotisha-production' \
+    --filter 'label=com.docker.compose.service=postgres'
+)
+[ "${#postgres_containers[@]}" -eq 1 ] || {
+  echo "expected exactly one running production PostgreSQL container" >&2
+  exit 1
+}
+postgres_container="${postgres_containers[0]}"
+
+created_compact="$(date -u +%Y%m%dT%H%M%SZ)"
+created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+backup_basename="production-pre-migration-${created_compact}.dump.enc"
+backup_partial="$backup_directory/.${backup_basename}.$$.partial"
+backup_file="$backup_directory/$backup_basename"
+restore_database="restore_verify_${created_compact,,}"
+restore_database="${restore_database//[^a-z0-9_]/_}"
+
+cleanup() {
+  rm -f -- "$backup_partial"
+  sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
+    psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
+    -c "DROP DATABASE IF EXISTS \"$restore_database\" WITH (FORCE);" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT HUP INT TERM
+
+sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
+  pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-acl |
+  openssl enc -aes-256-cbc -salt -pbkdf2 \
+    -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY >"$backup_partial"
+[ -s "$backup_partial" ]
+chmod 600 "$backup_partial"
+mv "$backup_partial" "$backup_file"
+backup_sha256="$(sha256sum "$backup_file" | awk '{print $1}')"
+[[ "$backup_sha256" =~ ^[0-9a-f]{64}$ ]]
+
+sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
+  psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
+  -c "CREATE DATABASE \"$restore_database\";" >/dev/null
+openssl enc -d -aes-256-cbc -pbkdf2 \
+  -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$backup_file" |
+  sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
+    pg_restore -U "$POSTGRES_USER" -d "$restore_database" \
+      --no-owner --no-acl --exit-on-error
+
+counts="$({
+  sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
+    psql -U "$POSTGRES_USER" -d "$restore_database" -At -F '|' -v ON_ERROR_STOP=1 -c \
+    "SELECT
+       (SELECT count(*) FROM identity.users),
+       (SELECT count(*) FROM public.profiles),
+       (SELECT count(*) FROM public.credit_transactions),
+       (SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public');"
+} | tail -n 1)"
+IFS='|' read -r identity_users profiles credit_transactions public_tables <<<"$counts"
+for value in "$identity_users" "$profiles" "$credit_transactions" "$public_tables"; do
+  [[ "$value" =~ ^[0-9]+$ ]] || {
+    echo "restore verification returned an invalid count" >&2
+    exit 1
+  }
+done
+
+sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
+  psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
+  -c "DROP DATABASE \"$restore_database\" WITH (FORCE);" >/dev/null
+verified_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+trap - EXIT HUP INT TERM
+
+verify_basename="${backup_basename%.dump.enc}-restore-verify.json"
+verify_file="$backup_directory/$verify_basename"
+printf '{\n  "mode": "restore_verify",\n  "ok": true,\n  "backup": "%s",\n  "sha256": "%s",\n  "created_at": "%s",\n  "verified_at": "%s",\n  "restore_database_removed": true,\n  "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s}\n}\n' \
+  "$backup_basename" "$backup_sha256" "$created_at" "$verified_at" \
+  "$identity_users" "$profiles" "$credit_transactions" "$public_tables" >"$verify_file"
+chmod 600 "$verify_file"
+
+recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}"
+state_tmp="$state_directory/recovery-baseline.env.tmp.$$"
+printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nVERIFY_FILE=%s\n' \
+  "$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp"
+chmod 600 "$state_tmp"
+mv "$state_tmp" "$state_directory/recovery-baseline.env"
+
+printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
+  "$recovery_reference" "$created_at" "$verified_at" "$backup_sha256" \
+  "$backup_basename" "$verify_basename"

docs/operations/production-server-migration-2026-08.md

@@ -90,7 +90,9 @@ Use distinct production credentials for PostgreSQL roles, Better Auth, Resend, b
 
 ## Database migration engineering gate
 
-Before importing data, dispatch Gitea Actions → `Migrate Production Database` for the exact accepted release SHA. The workflow requires `main == staging == deploy_sha`, the same successful staging backend gate, the same manual release gate, and the public staging `/api/health` identity for that SHA. It also requires a non-sensitive recovery reference, its exact UTC creation time, and `restore_verified=true`; the recovery point must be no more than 24 hours old and must already have passed a restore verification. It verifies the current production revision, obtains the gate-attested immutable Web image, runs the schema checker, applies only pending application schema migrations, and requires the checker to converge afterward.
+Before importing data, dispatch Gitea Actions → `Create Production Recovery Point` for the exact accepted release SHA. It creates an AES-256-CBC/PBKDF2 encrypted custom-format dump on the production host, restores it into a disposable database, verifies non-sensitive table counts, removes only the disposable database, and uploads the encrypted dump plus verification manifest as a 30-day Gitea Actions artifact. The workflow prints the non-sensitive `recovery_reference` and exact UTC `recovery_created_at`; use those values only after the artifact upload succeeds.
+
+Then dispatch Gitea Actions → `Migrate Production Database` for the same exact accepted release SHA. The workflow requires `main == staging == deploy_sha`, the same successful staging backend gate, the same manual release gate, and the public staging `/api/health` identity for that SHA. It also requires the recovery workflow's non-sensitive reference, exact UTC creation time, and `restore_verified=true`; the recovery point must be no more than 24 hours old and must already have passed the restore verification. It verifies the current production revision, obtains the gate-attested immutable Web image, runs the schema checker, applies only pending application schema migrations, and requires the checker to converge afterward.
 
 Schema migration files are committed sequentially and are not one atomic transaction as a set. If a later file or post-check fails, earlier files may remain applied; stop, preserve evidence, and restore from the attested recovery point when repair-in-place is not explicitly reviewed. Do not assume a failed workflow means the database is unchanged.
 
@@ -172,7 +174,7 @@ Both `jyotisha.chat` and `admin.jyotisha.chat` are required. The application rej
 - Confirm the exact release SHA is deployed and accepted on staging.
 - Run the manual release quality gate for that SHA.
 - Confirm final backup capacity, restore rehearsal, SMTP/OTP delivery, and rollback contacts.
-- Create and restore-verify a production recovery point no more than 24 hours before the schema migration; record its non-sensitive reference and UTC creation time.
+- Run `Create Production Recovery Point` no more than 24 hours before the schema migration; retain its encrypted off-site artifact and record the printed non-sensitive reference and UTC creation time.
 - Dispatch `Migrate Production Database` for the accepted SHA with that recovery attestation and confirm its post-check reports no pending schema migrations.
 - Record pending payment orders and long-running jobs; choose an explicit disposition for each.
 - Dispatch `Deploy production` with `verification_mode=internal` only after target schema/data preparation. This verifies the new host without depending on public DNS.
@@ -228,9 +230,10 @@ Normal release:
 
 1. Merge the reviewed `staging` release into `main` so both heads are the same SHA.
 2. Confirm the staging push gate, public staging SHA, and manual release gate all succeeded for that SHA.
-3. Open Gitea Actions → `Migrate Production Database`; enter the exact 40-character SHA, the no-more-than-24-hour-old recovery reference and UTC creation time, and confirm `restore_verified=true`. Wait for the post-migration checker to converge. Do not use this workflow for Supabase ETL.
-4. Run the trusted-host ETL phases and retain the redacted reconciliation manifests.
-5. Open Gitea Actions → `Deploy production`.
-6. Enter the same exact SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase.
+3. Open Gitea Actions → `Create Production Recovery Point`; enter the exact 40-character SHA and wait for the encrypted artifact upload plus restore verification to succeed. Record the printed `recovery_reference` and `recovery_created_at`.
+4. Open Gitea Actions → `Migrate Production Database`; enter the same SHA and the no-more-than-24-hour-old recovery attestation, then confirm `restore_verified=true`. Wait for the post-migration checker to converge. Do not use this workflow for Supabase ETL.
+5. Run the trusted-host ETL phases and retain the redacted reconciliation manifests when a data cutover is actually required. Ordinary forward schema releases do not rerun the one-shot Supabase ETL.
+6. Open Gitea Actions → `Deploy production`.
+7. Enter the same exact SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase.
 
 Application rollback accepts only an explicitly authorized, previously gate-attested SHA in reviewed `main` history. Database migrations and imported data are not rolled back by the application workflow.

frontend/tests/staging-backend-workflows.test.ts

@@ -46,6 +46,10 @@ const giteaProductionMigrationWorkflow = new URL(
   "../../.gitea/workflows/migrate-production-database.yml",
   import.meta.url,
 );
+const giteaProductionRecoveryWorkflow = new URL(
+  "../../.gitea/workflows/create-production-recovery.yml",
+  import.meta.url,
+);
 const giteaReleaseQualityWorkflow = new URL(
   "../../.gitea/workflows/release-quality-gate.yml",
   import.meta.url,
@@ -74,6 +78,10 @@ const productionMigrationScript = new URL(
   "../../deploy/run-production-migration.sh",
   import.meta.url,
 );
+const productionRecoveryScript = new URL(
+  "../../deploy/run-production-recovery.sh",
+  import.meta.url,
+);
 const productionSyncScript = new URL(
   "../../deploy/sync-production-tree.sh",
   import.meta.url,
@@ -1053,6 +1061,42 @@ test("production runner validates state and migrations before switching exact im
 });
 
 
+
+test("production recovery workflow creates a verified encrypted off-site artifact", () => {
+  const workflow = read(giteaProductionRecoveryWorkflow);
+  const runner = read(productionRecoveryScript);
+
+  assert.match(workflow, /^on:\n\s+workflow_dispatch:/m);
+  assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/);
+  assert.match(workflow, /permissions:\n\s+contents: read\n\s+actions: write/);
+  assert.match(workflow, /group: production-mutation/);
+  assert.match(workflow, /main_head[\s\S]*DEPLOY_SHA[\s\S]*staging_head[\s\S]*DEPLOY_SHA/);
+  assert.match(workflow, /endswith\("release-quality-gate\.yml"\)/);
+  assert.match(workflow, /observed_staging_sha[\s\S]*DEPLOY_SHA/);
+  assert.match(workflow, /git checkout --detach --force "\$DEPLOY_SHA"/);
+  assert.match(workflow, /SSH_PRIVATE_KEY_BASE64: \$\{\{ secrets\.PRODUCTION_SSH_PRIVATE_KEY \}\}/);
+  assert.match(workflow, /ServerAliveInterval=15.*ServerAliveCountMax=4/);
+  assert.match(workflow, /git show "\$DEPLOY_SHA:deploy\/run-production-recovery\.sh"/);
+  assert.match(workflow, /production-recovery-\$GITHUB_RUN_ID/);
+  assert.match(workflow, /INPUT_RETENTION-DAYS.*30/);
+  assert.match(workflow, /INPUT_COMPRESSION-LEVEL.*0/);
+  assert.match(workflow, /sha256sum --check --status/);
+  assert.match(workflow, /restore_database_removed == true/);
+  assert.doesNotMatch(workflow, /STAGING_BACKUP_ENCRYPTION_KEY|\.env\.production\.database[^\n]*(?:cat|awk)/);
+
+  assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/);
+  assert.match(runner, /another production mutation holds the host lock/);
+  assert.match(runner, /usage_percent < 70/);
+  assert.match(runner, /pg_dump[\s\S]*--format=custom --no-owner --no-acl/);
+  assert.match(runner, /openssl enc -aes-256-cbc -salt -pbkdf2/);
+  assert.match(runner, /openssl enc -d -aes-256-cbc -pbkdf2/);
+  assert.match(runner, /pg_restore[\s\S]*--no-owner --no-acl --exit-on-error/);
+  assert.match(runner, /DROP DATABASE IF EXISTS[\s\S]*WITH \(FORCE\)/);
+  assert.match(runner, /restore_database_removed/);
+  assert.match(runner, /gitea-actions-run-\$\{RECOVERY_RUN_ID\}/);
+  assert.doesNotMatch(runner, /docker compose down|down -v|dropdb jyotisha|rm -rf[^\n]*backup_directory/);
+});
+
 test("Gitea production schema migration is exact-SHA gated and isolated from ETL and deploy", () => {
   const workflow = read(giteaProductionMigrationWorkflow);
   const runner = read(productionMigrationScript);