From 3a9a332be05aa7f155005bb0adcd15cb7ddadb9f Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Tue, 21 Jul 2026 04:31:43 +0800 Subject: [PATCH] fix: preserve staging rollback transition --- deploy/README.md | 12 +-- deploy/run-staging-deploy.sh | 26 +++-- deploy/run-staging-migration.sh | 2 +- ...-07-20-postgres-quality-gate-foundation.md | 2 +- frontend/scripts/db-migrate.mjs | 28 +++--- frontend/tests/database-foundation.test.ts | 9 ++ .../tests/staging-backend-workflows.test.ts | 98 ++++++++++++++++++- 7 files changed, 148 insertions(+), 29 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index d5d515af..240f7eb2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -163,15 +163,15 @@ After source sync and before `up`, the workflow validates `.env.staging` mode/se ### First-deploy sequence -1. Complete the server and GitHub bootstrap: create both mode-`0600` env files, configure the staging Environment variables/secrets, and configure the repository staging build variables. -2. Merge the reviewed change, then push the reviewed SHA to `staging`; do not rely on a `main` workflow dispatch to publish images. +1. Complete the server and GitHub bootstrap: create both mode-`0600` env files, preload the reviewed `postgres:17-alpine` image, configure the staging Environment variables/secrets, and configure the repository staging build variables. Deployment and migration workflows use `--pull never` for PostgreSQL, so database image upgrades remain an explicit operator-controlled maintenance action rather than an application-deploy side effect. +2. Merge the reviewed change to `main`, then fast-forward/push that exact reviewed SHA to `staging`; do not create a staging-only target or rely on a `main` workflow dispatch to publish images. 3. The `Staging Backend Quality Gate` runs for that push and, when successful, publishes API/web images plus an artifact binding the exact SHA to both immutable image digests. -4. The automatic `Deploy staging` workflow downloads that gate-run artifact, syncs the exact revision under the shared staging host lock, and validates both `.env.staging` and `.env.staging.database` before any app change. +4. The automatic `Deploy staging` workflow downloads that gate-run artifact, syncs only the trusted `main` controller's allowlisted `deploy/` files under the shared staging host lock, and validates both `.env.staging` and `.env.staging.database` before any app change. The target application's code is carried only by the digest-pinned images. 5. If environment validation fails, fix the server-side env files without committing or copying secrets, then manually rerun `Deploy staging` from `main` with the same successful SHA in `deploy_sha`; the workflow rechecks a successful staging gate for that exact SHA. 6. If the read-only checker reports a pending migration, stop app deployment and run `Migrate Staging Database` manually with the same full SHA; a successful migration re-dispatches `Deploy staging` with that same SHA. 7. Confirm `https://staging.jyotisha.chat/api/health` reports the exact SHA and private API health. -Application rollback uses the same workflow: manually dispatch `Deploy staging` from the `main` controller with a previous known-good full SHA that has a successful `Staging Backend Quality Gate` run, and explicitly set `allow_rollback=true`. Normal and migration-triggered deployments reject stale, divergent, or backward revisions. Rollback still consumes the selected gate run's digest manifest; database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. +Application rollback uses the same workflow: manually dispatch `Deploy staging` from the `main` controller with a previous known-good full SHA that has a successful `Staging Backend Quality Gate` run, and explicitly set `allow_rollback=true`. Normal and migration-triggered deployments reject stale, divergent, or backward revisions. Rollback still consumes the selected gate run's digest manifest, which is retained for 30 days; if it has expired, rerun that exact SHA's gate run to regenerate the immutable manifest before dispatching rollback. Database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. Inspect staging without printing secrets: @@ -228,7 +228,7 @@ PostgreSQL is private: `deploy/docker-compose.postgres.yml` has no `ports` mappi Use this order for every staging revision: -1. Merge to `staging` after reviewing the change. +1. Merge the reviewed revision to `main`, then fast-forward/push that same exact SHA to `staging`. 2. Wait for `Staging Backend Quality Gate` to pass and publish that exact full SHA's API/web digest manifest. 3. The automatic `Deploy staging` workflow checks the exact SHA in read-only migration-check mode before changing API, web, or Caddy. If it reports pending or drifted migrations, stop; do not retry the application deployment as if it were a migration. 4. Open **Migrate Staging Database -> Run workflow**, select **Use workflow from: main**, and enter the reported full lowercase 40-character SHA in `deploy_sha`. The controller validates that exact SHA against a successful `staging` gate and reviewed `main` history, starts only PostgreSQL, and runs the digest-pinned migrator without executing scripts from the target revision. @@ -236,7 +236,7 @@ Use this order for every staging revision: 6. Confirm `https://staging.jyotisha.chat/api/health` and verify that its deployment SHA is the SHA from step 2. 7. After health verification, create the local encrypted backup described below. -The deploy and migration workflows share the `staging-mutation` Actions concurrency group, and their live-tree sync plus Compose work runs under `/opt/jyotisha-staging/.state/mutation.lock`. The synchronized tree explicitly preserves `/backups/`, `.env*`, `.state`, and `.incoming`. The read-only checker exits before app changes when a migration is pending. Its message includes the exact SHA and the `Migrate Staging Database` workflow name. A failed migration does not re-dispatch deployment. Application rollback restores previously recorded digest references and SHA only; it does not roll back database state. +The deploy and migration workflows share the `staging-mutation` Actions concurrency group, and their live-tree sync plus Compose work runs under `/opt/jyotisha-staging/.state/mutation.lock`. The synchronized tree explicitly preserves `/backups/`, `.env*`, `.state`, and `.incoming`. The read-only checker exits before app changes when a migration is pending. Its message includes the exact SHA and the `Migrate Staging Database` workflow name. A failed migration does not re-dispatch deployment. Application rollback restores the previously recorded digest references and SHA, falling back to validated local image IDs only when transitioning from the pre-foundation local-image deployment; it does not roll back database state. ### Local encrypted staging backups (three-copy limit) diff --git a/deploy/run-staging-deploy.sh b/deploy/run-staging-deploy.sh index 34c9f1ad..153d658a 100755 --- a/deploy/run-staging-deploy.sh +++ b/deploy/run-staging-deploy.sh @@ -15,6 +15,7 @@ done sha_pattern='^[0-9a-f]{40}$' digest_pattern='^ghcr\.io/jesse-ux/jyotisha-(api|web)@sha256:[0-9a-f]{64}$' +image_id_pattern='^sha256:[0-9a-f]{64}$' if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] || [[ ! "$API_IMAGE" =~ $digest_pattern ]] || [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then @@ -95,6 +96,19 @@ if [ -n "$(container_id web)" ]; then previous_web_id="$(docker inspect --format '{{.Image}}' "$(container_id web)")" fi +rollback_image() { + local digest_ref="$1" + local image_id="$2" + if [[ "$digest_ref" =~ $digest_pattern ]]; then + printf '%s' "$digest_ref" + elif [[ "$image_id" =~ $image_id_pattern ]]; then + printf '%s' "$image_id" + fi +} + +previous_api_target="$(rollback_image "$previous_api_image" "$previous_api_id")" +previous_web_target="$(rollback_image "$previous_web_image" "$previous_web_id")" + bash "$INCOMING_PATH/deploy/sync-staging-tree.sh" \ "$INCOMING_PATH" "$DEPLOY_PATH" @@ -114,8 +128,8 @@ export SITE_ADDRESS='https://staging.jyotisha.chat' export GITHUB_SHA="$DEPLOY_SHA" "${compose[@]}" config --quiet -"${compose[@]}" pull api web postgres -"${compose[@]}" up -d --no-build --wait postgres +"${compose[@]}" pull api web +"${compose[@]}" up -d --no-build --pull never --wait postgres set +e "${compose[@]}" --profile migration-check run --rm migration-checker @@ -134,11 +148,11 @@ switched=false rollback() { local status=$? if [ "$switched" = "true" ] && - [[ "$previous_api_image" =~ $digest_pattern ]] && - [[ "$previous_web_image" =~ $digest_pattern ]] && + [ -n "$previous_api_target" ] && + [ -n "$previous_web_target" ] && [[ "$current_sha" =~ $sha_pattern ]]; then - echo "staging verification failed; restoring prior image digests" >&2 - API_IMAGE="$previous_api_image" WEB_IMAGE="$previous_web_image" \ + echo "staging verification failed; restoring prior application images" >&2 + API_IMAGE="$previous_api_target" WEB_IMAGE="$previous_web_target" \ GITHUB_SHA="$current_sha" \ "${compose[@]}" up -d --no-build --remove-orphans api web caddy || true fi diff --git a/deploy/run-staging-migration.sh b/deploy/run-staging-migration.sh index 9154af1b..cc0b9ee8 100755 --- a/deploy/run-staging-migration.sh +++ b/deploy/run-staging-migration.sh @@ -73,7 +73,7 @@ bash deploy/validate-staging-database-env.sh .env.staging.database export DATABASE_ENV_FILE='../.env.staging.database' compose=(docker compose -p jyotisha-staging -f deploy/docker-compose.postgres.yml) docker pull "$WEB_IMAGE" -"${compose[@]}" up -d --wait postgres +"${compose[@]}" up -d --no-build --pull never --wait postgres "${compose[@]}" --profile migration run --rm migrator "${compose[@]}" exec -T postgres psql -U postgres -d jyotisha -Atc \ 'select filename from migration.schema_migrations order by filename' diff --git a/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md b/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md index 9d19f725..ded1bba7 100644 --- a/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md +++ b/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md @@ -959,7 +959,7 @@ Each secret uses independently generated 32 random bytes. URL password is percen Document order: -1. Merge to `staging`. +1. Merge the reviewed revision to `main`, then fast-forward/push that exact SHA to `staging`. 2. Wait for backend quality gate and its exact-SHA image digest manifest. 3. If automatic deploy reports pending migrations, manually run `Migrate Staging Database` with the reported full SHA. 4. The successful migration workflow re-dispatches exact-SHA staging deploy automatically. diff --git a/frontend/scripts/db-migrate.mjs b/frontend/scripts/db-migrate.mjs index 737b9a73..8433dbc9 100644 --- a/frontend/scripts/db-migrate.mjs +++ b/frontend/scripts/db-migrate.mjs @@ -59,6 +59,20 @@ async function readLedger(client) { return new Map(result.rows.map((row) => [row.filename, row.checksum])); } +function assertLedgerFilesPresent(ledger, files) { + const reviewedFilenames = new Set(files.map((file) => file.filename)); + for (const filename of ledger.keys()) { + if (!reviewedFilenames.has(filename)) { + if (!migrationFilenamePattern.test(filename)) { + throw new SafeMigrationError( + "migration ledger contains an invalid filename", + ); + } + throw new SafeMigrationError(`migration file missing: ${filename}`); + } + } +} + export async function runMigrations({ connectionString, migrationsDirectory, @@ -78,19 +92,8 @@ export async function runMigrations({ if (check) { const ledger = await readLedger(client); - const reviewedFilenames = new Set(files.map((file) => file.filename)); const pending = []; - - for (const filename of ledger.keys()) { - if (!reviewedFilenames.has(filename)) { - if (!migrationFilenamePattern.test(filename)) { - throw new SafeMigrationError( - "migration ledger contains an invalid filename", - ); - } - throw new SafeMigrationError(`migration file missing: ${filename}`); - } - } + assertLedgerFilesPresent(ledger, files); for (const file of files) { const recordedChecksum = ledger.get(file.filename); @@ -123,6 +126,7 @@ export async function runMigrations({ ); const ledger = await readLedger(client); + assertLedgerFilesPresent(ledger, files); for (const file of files) { const recordedChecksum = ledger.get(file.filename); if (recordedChecksum !== undefined) { diff --git a/frontend/tests/database-foundation.test.ts b/frontend/tests/database-foundation.test.ts index 82f7a43e..d90a6d62 100644 --- a/frontend/tests/database-foundation.test.ts +++ b/frontend/tests/database-foundation.test.ts @@ -205,6 +205,15 @@ test("migration runner is serialized, atomic, drift-safe, and read-only in check missingFileCheck.stderr, new RegExp(`migration file missing: ${migrationFilename}`), ); + const missingFileApply = runMigration(schemaUrl, { + migrationsDirectory: emptyDirectory, + }); + results.push(missingFileApply); + assert.equal(missingFileApply.status, 1); + assert.match( + missingFileApply.stderr, + new RegExp(`migration file missing: ${migrationFilename}`), + ); const pendingPath = join(temporaryDirectory, pendingFilename); mkdirSync(dirname(pendingPath), { recursive: true }); diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 677391a4..6afc5283 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -208,7 +208,7 @@ test("remote deployment verifies running image IDs, RepoDigests, and application assert.match(runner, /grep -Fqx "\$expected_ref"/); assert.match(runner, /publicBody\.deployment\?\.gitCommit !== process\.env\.EXPECTED_SHA/); assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/); - assert.match(runner, /restoring prior image digests/); + assert.match(runner, /restoring prior application images/); assert.match( runner, /switched=true\n"\$\{compose\[@\]\}" up -d --no-build --remove-orphans\n/, @@ -216,11 +216,101 @@ test("remote deployment verifies running image IDs, RepoDigests, and application assert.doesNotMatch(runner, /jyotisha-(?:api|web):\$DEPLOY_SHA/); }); +test("first immutable deployment rolls back to validated local image IDs", () => { + const root = mkdtempSync(join(tmpdir(), "jyotisha-local-image-rollback-")); + const deploymentPath = join(root, "live"); + const incomingPath = join(deploymentPath, ".incoming", "run-1"); + const incomingDeploy = join(incomingPath, "deploy"); + const liveDeploy = join(deploymentPath, "deploy"); + const mockBin = join(root, "bin"); + const rollbackLog = join(root, "rollback.log"); + const previousSha = "1".repeat(40); + const previousApiId = `sha256:${"a".repeat(64)}`; + const previousWebId = `sha256:${"b".repeat(64)}`; + const nextSha = "2".repeat(40); + + mkdirSync(incomingDeploy, { recursive: true }); + mkdirSync(liveDeploy, { recursive: true }); + mkdirSync(join(deploymentPath, ".state"), { recursive: true }); + mkdirSync(mockBin); + writeFileSync(join(deploymentPath, ".state", "deployed-revision"), `${previousSha}\n`); + for (const script of [ + join(incomingDeploy, "sync-staging-tree.sh"), + join(liveDeploy, "validate-staging-env.sh"), + join(liveDeploy, "validate-staging-database-env.sh"), + ]) { + writeFileSync(script, "#!/usr/bin/env bash\nexit 0\n"); + chmodSync(script, 0o755); + } + writeFileSync( + join(mockBin, "docker"), + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "if [ \"$1\" = ps ]; then", + " case \"$*\" in", + " *service=api*) echo container-api ;;", + " *service=web*) echo container-web ;;", + " esac", + " exit 0", + "fi", + "if [ \"$1\" = inspect ]; then", + ` if [ \"\${!#}\" = container-api ]; then echo '${previousApiId}'; else echo '${previousWebId}'; fi`, + " exit 0", + "fi", + "if [ \"$1\" = image ]; then exit 0; fi", + "if [ \"$1\" = compose ]; then", + " if [[ \" $* \" == *\" up -d --no-build --remove-orphans \"* ]]; then", + ` printf '%s|%s|%s|%s\\n' \"\${API_IMAGE:-}\" \"\${WEB_IMAGE:-}\" \"\${GITHUB_SHA:-}\" \"$*\" >>'${rollbackLog}'`, + " [[ \"$*\" == *\" api web caddy\" ]] && exit 0", + " exit 42", + " fi", + " exit 0", + "fi", + "exit 1", + "", + ].join("\n"), + ); + chmodSync(join(mockBin, "docker"), 0o755); + writeFileSync(join(mockBin, "flock"), "#!/usr/bin/env bash\nexit 0\n"); + chmodSync(join(mockBin, "flock"), 0o755); + + try { + const result = spawnSync("bash", [fileURLToPath(deployScript)], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${mockBin}:${process.env.PATH ?? ""}`, + INCOMING_PATH: incomingPath, + DEPLOY_PATH: deploymentPath, + API_IMAGE: `ghcr.io/jesse-ux/jyotisha-api@sha256:${"c".repeat(64)}`, + WEB_IMAGE: `ghcr.io/jesse-ux/jyotisha-web@sha256:${"d".repeat(64)}`, + DEPLOY_SHA: nextSha, + EXPECTED_PREVIOUS_SHA: previousSha, + ALLOW_ROLLBACK: "false", + FORWARD_REVISION_VERIFIED: "true", + DOCKER_CONFIG: join(incomingPath, ".docker"), + STAGING_URL: "https://staging.jyotisha.chat", + }, + }); + assert.equal(result.status, 42, result.stderr); + const attempts = readFileSync(rollbackLog, "utf8").trim().split("\n"); + assert.equal(attempts.length, 2); + assert.match( + attempts[1], + new RegExp(`^${previousApiId}\\|${previousWebId}\\|${previousSha}\\|`), + ); + assert.match(attempts[1], /api web caddy$/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("normal deployment checks migrations but never applies them", () => { const runner = read(deployScript); assertOrder(runner, [ - "pull api web postgres", - "up -d --no-build --wait postgres", + "pull api web", + "up -d --no-build --pull never --wait postgres", "--profile migration-check run --rm migration-checker", "up -d --no-build --remove-orphans", ]); @@ -228,6 +318,7 @@ test("normal deployment checks migrations but never applies them", () => { assert.match(runner, /exit 3/); assert.doesNotMatch(runner, /--profile migration run --rm migrator/); assert.doesNotMatch(runner, /npm\s+run\s+db:migrate(?!:check)/); + assert.doesNotMatch(runner, /pull api web postgres/); }); test("manual migration uses only PostgreSQL and the digest-pinned migrator", () => { @@ -237,6 +328,7 @@ test("manual migration uses only PostgreSQL and the digest-pinned migrator", () assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); assert.match(runner, /docker pull "\$WEB_IMAGE"/); + assert.match(runner, /up -d --no-build --pull never --wait postgres/); assert.match(runner, /-f deploy\/docker-compose\.postgres\.yml/); assert.match(runner, /--profile migration run --rm migrator/); assert.match(runner, /select filename from migration\.schema_migrations order by filename/);