ops: automate production recovery attestation

frontend/tests/staging-backend-workflows.test.ts

@@ -46,6 +46,10 @@ const giteaProductionMigrationWorkflow = new URL(
   "../../.gitea/workflows/migrate-production-database.yml",
   import.meta.url,
 );
+const giteaProductionRecoveryWorkflow = new URL(
+  "../../.gitea/workflows/create-production-recovery.yml",
+  import.meta.url,
+);
 const giteaReleaseQualityWorkflow = new URL(
   "../../.gitea/workflows/release-quality-gate.yml",
   import.meta.url,
@@ -74,6 +78,10 @@ const productionMigrationScript = new URL(
   "../../deploy/run-production-migration.sh",
   import.meta.url,
 );
+const productionRecoveryScript = new URL(
+  "../../deploy/run-production-recovery.sh",
+  import.meta.url,
+);
 const productionSyncScript = new URL(
   "../../deploy/sync-production-tree.sh",
   import.meta.url,
@@ -1053,6 +1061,42 @@ test("production runner validates state and migrations before switching exact im
 });
 
 
+
+test("production recovery workflow creates a verified encrypted off-site artifact", () => {
+  const workflow = read(giteaProductionRecoveryWorkflow);
+  const runner = read(productionRecoveryScript);
+
+  assert.match(workflow, /^on:\n\s+workflow_dispatch:/m);
+  assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/);
+  assert.match(workflow, /permissions:\n\s+contents: read\n\s+actions: write/);
+  assert.match(workflow, /group: production-mutation/);
+  assert.match(workflow, /main_head[\s\S]*DEPLOY_SHA[\s\S]*staging_head[\s\S]*DEPLOY_SHA/);
+  assert.match(workflow, /endswith\("release-quality-gate\.yml"\)/);
+  assert.match(workflow, /observed_staging_sha[\s\S]*DEPLOY_SHA/);
+  assert.match(workflow, /git checkout --detach --force "\$DEPLOY_SHA"/);
+  assert.match(workflow, /SSH_PRIVATE_KEY_BASE64: \$\{\{ secrets\.PRODUCTION_SSH_PRIVATE_KEY \}\}/);
+  assert.match(workflow, /ServerAliveInterval=15.*ServerAliveCountMax=4/);
+  assert.match(workflow, /git show "\$DEPLOY_SHA:deploy\/run-production-recovery\.sh"/);
+  assert.match(workflow, /production-recovery-\$GITHUB_RUN_ID/);
+  assert.match(workflow, /INPUT_RETENTION-DAYS.*30/);
+  assert.match(workflow, /INPUT_COMPRESSION-LEVEL.*0/);
+  assert.match(workflow, /sha256sum --check --status/);
+  assert.match(workflow, /restore_database_removed == true/);
+  assert.doesNotMatch(workflow, /STAGING_BACKUP_ENCRYPTION_KEY|\.env\.production\.database[^\n]*(?:cat|awk)/);
+
+  assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/);
+  assert.match(runner, /another production mutation holds the host lock/);
+  assert.match(runner, /usage_percent < 70/);
+  assert.match(runner, /pg_dump[\s\S]*--format=custom --no-owner --no-acl/);
+  assert.match(runner, /openssl enc -aes-256-cbc -salt -pbkdf2/);
+  assert.match(runner, /openssl enc -d -aes-256-cbc -pbkdf2/);
+  assert.match(runner, /pg_restore[\s\S]*--no-owner --no-acl --exit-on-error/);
+  assert.match(runner, /DROP DATABASE IF EXISTS[\s\S]*WITH \(FORCE\)/);
+  assert.match(runner, /restore_database_removed/);
+  assert.match(runner, /gitea-actions-run-\$\{RECOVERY_RUN_ID\}/);
+  assert.doesNotMatch(runner, /docker compose down|down -v|dropdb jyotisha|rm -rf[^\n]*backup_directory/);
+});
+
 test("Gitea production schema migration is exact-SHA gated and isolated from ETL and deploy", () => {
   const workflow = read(giteaProductionMigrationWorkflow);
   const runner = read(productionMigrationScript);