diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 9f0a2163..25842b41 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -24,6 +24,7 @@ permissions: concurrency: group: staging-mutation cancel-in-progress: false + queue: max jobs: deploy: @@ -78,7 +79,7 @@ jobs: [.workflow_runs[] | select( .head_sha == $sha and .head_branch == "staging" and .event == "push" and .conclusion == "success" - )] | sort_by(.run_attempt) | reverse | first + )] | sort_by(.id) | reverse | first ' <<<"$runs")" gate_run_id="$(jq -er '.id' <<<"$selected_run")" gate_run_attempt="$(jq -er '.run_attempt' <<<"$selected_run")" @@ -110,10 +111,11 @@ jobs: echo "allow_rollback=$allow_rollback" } >>"$GITHUB_OUTPUT" - - name: Checkout tested revision + - name: Checkout trusted main controller uses: actions/checkout@v4 with: - ref: ${{ steps.revision.outputs.sha }} + ref: main + fetch-depth: 0 persist-credentials: false - name: Download gate-produced image manifest @@ -133,12 +135,16 @@ jobs: node frontend/scripts/staging-image-manifest.mjs \ artifacts/staging-image/manifest.env "$DEPLOY_SHA" >>"$GITHUB_OUTPUT" - - name: Verify revision and staging target + - name: Verify reviewed revision and staging target env: DEPLOY_SHA: ${{ steps.revision.outputs.sha }} run: | set -euo pipefail - test "$(git rev-parse HEAD)" = "$DEPLOY_SHA" + git cat-file -e "$DEPLOY_SHA^{commit}" + git merge-base --is-ancestor "$DEPLOY_SHA" HEAD || { + echo "staging revision is not in the reviewed main history" >&2 + exit 1 + } test "$DEPLOY_HOST" = "118.26.111.127" test "$DEPLOY_PORT" = "22" test "$DEPLOY_USER" = "deploy" @@ -195,7 +201,7 @@ jobs: echo "forward_verified=$forward_verified" } >>"$GITHUB_OUTPUT" - - name: Stage tested sources in an isolated incoming directory + - name: Stage trusted controller files in an isolated incoming directory id: incoming run: | set -euo pipefail @@ -203,11 +209,10 @@ jobs: RSYNC_SSH="ssh $SSH_OPTIONS" incoming="$DEPLOY_PATH/.incoming/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$incoming'" - rsync -az --delete \ - --exclude='/.git/' --exclude='/.env*' --exclude='/backups/' \ - --exclude='/frontend/node_modules/' --exclude='/frontend/.next/' \ - -e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/" echo "path=$incoming" >>"$GITHUB_OUTPUT" + rsync -az --delete --prune-empty-dirs \ + --include='/deploy/' --include='/deploy/***' --exclude='*' \ + -e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/" - name: Log in to GHCR with run-local Docker state env: diff --git a/.github/workflows/migrate-staging-database.yml b/.github/workflows/migrate-staging-database.yml index 1a512802..0072cbff 100644 --- a/.github/workflows/migrate-staging-database.yml +++ b/.github/workflows/migrate-staging-database.yml @@ -11,6 +11,7 @@ on: concurrency: group: staging-mutation cancel-in-progress: false + queue: max permissions: contents: read @@ -51,7 +52,7 @@ jobs: [.workflow_runs[] | select( .head_sha == $sha and .head_branch == "staging" and .event == "push" and .conclusion == "success" - )] | sort_by(.run_attempt) | reverse | first + )] | sort_by(.id) | reverse | first ' <<<"$runs")" gate_run_id="$(jq -er '.id' <<<"$selected_run")" gate_run_attempt="$(jq -er '.run_attempt' <<<"$selected_run")" @@ -73,10 +74,11 @@ jobs: echo "gate_run_attempt=$gate_run_attempt" } >>"$GITHUB_OUTPUT" - - name: Checkout tested revision + - name: Checkout trusted main controller uses: actions/checkout@v4 with: - ref: ${{ steps.revision.outputs.sha }} + ref: main + fetch-depth: 0 persist-credentials: false - name: Download gate-produced image manifest @@ -96,12 +98,16 @@ jobs: node frontend/scripts/staging-image-manifest.mjs \ artifacts/staging-image/manifest.env "$DEPLOY_SHA" >>"$GITHUB_OUTPUT" - - name: Verify revision and staging target + - name: Verify reviewed revision and staging target env: DEPLOY_SHA: ${{ steps.revision.outputs.sha }} run: | set -euo pipefail - test "$(git rev-parse HEAD)" = "$DEPLOY_SHA" + git cat-file -e "$DEPLOY_SHA^{commit}" + git merge-base --is-ancestor "$DEPLOY_SHA" HEAD || { + echo "staging revision is not in the reviewed main history" >&2 + exit 1 + } test "$DEPLOY_HOST" = "118.26.111.127" test "$DEPLOY_PORT" = "22" test "$DEPLOY_USER" = "deploy" @@ -152,7 +158,7 @@ jobs: echo "forward_verified=true" } >>"$GITHUB_OUTPUT" - - name: Stage tested sources in an isolated incoming directory + - name: Stage trusted controller files in an isolated incoming directory id: incoming run: | set -euo pipefail @@ -160,11 +166,10 @@ jobs: RSYNC_SSH="ssh $SSH_OPTIONS" incoming="$DEPLOY_PATH/.incoming/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$incoming'" - rsync -az --delete \ - --exclude='/.git/' --exclude='/.env*' --exclude='/backups/' \ - --exclude='/frontend/node_modules/' --exclude='/frontend/.next/' \ - -e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/" echo "path=$incoming" >>"$GITHUB_OUTPUT" + rsync -az --delete --prune-empty-dirs \ + --include='/deploy/' --include='/deploy/***' --exclude='*' \ + -e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/" - name: Log in to GHCR with run-local Docker state env: diff --git a/deploy/README.md b/deploy/README.md index 5b72968a..d5d515af 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -145,7 +145,7 @@ Staging is isolated from production: | Supabase | separate `Jyotisha Staging` project | | GitHub Environment | `staging` | -The GitHub `staging` Environment contains the secret `STAGING_SSH_PRIVATE_KEY` and the variables `STAGING_HOST`, `STAGING_PORT`, `STAGING_USER`, `STAGING_PATH`, `STAGING_URL`, and `STAGING_KNOWN_HOSTS`. Its deployment branch policy allows the `main` controller branch: GitHub's `workflow_run` event executes from the default branch while the workflow separately requires the successfully tested upstream branch to be `staging`. The staging key, database, Supabase keys, and model-provider keys must not be shared with production. +The GitHub `staging` Environment contains the secret `STAGING_SSH_PRIVATE_KEY` and the variables `STAGING_HOST`, `STAGING_PORT`, `STAGING_USER`, `STAGING_PATH`, `STAGING_URL`, and `STAGING_KNOWN_HOSTS`. Its deployment branch policy allows the `main` controller branch: GitHub's `workflow_run` event executes from the default branch while the workflow separately requires the successfully tested upstream branch to be `staging`. The controller checks out only `main` with full history, requires the requested staging SHA to be an ancestor of that reviewed history, and uploads only the allowlisted `deploy/` control files. It never executes deployment validators or remote orchestration scripts from the target/rollback revision. The staging key, database, Supabase keys, and model-provider keys must not be shared with production. The repository-level public build inputs are configured at GitHub **Settings -> Secrets and variables -> Actions -> Variables** (the UI is also shown as **Settings → Secrets and variables → Actions → Variables**): `STAGING_SUPABASE_URL` and `STAGING_SUPABASE_ANON_KEY`. They are public build inputs, required for publish, and exposed to the browser; keep them staging-only and never print their values in workflow output, summaries, or support messages. The workflow passes them only as the `NEXT_PUBLIC_*` build arguments after non-empty/HTTPS validation. @@ -231,7 +231,7 @@ Use this order for every staging revision: 1. Merge to `staging` after reviewing the change. 2. Wait for `Staging Backend Quality Gate` to pass and publish that exact full SHA's API/web digest manifest. 3. The automatic `Deploy staging` workflow checks the exact SHA in read-only migration-check mode before changing API, web, or Caddy. If it reports pending or drifted migrations, stop; do not retry the application deployment as if it were a migration. -4. Open **Migrate Staging Database -> Run workflow** and enter the reported full lowercase 40-character SHA in `deploy_sha`. The workflow validates that exact SHA against a successful `staging` gate, checks it out, starts only PostgreSQL, and runs the reviewed migrator. +4. Open **Migrate Staging Database -> Run workflow**, select **Use workflow from: main**, and enter the reported full lowercase 40-character SHA in `deploy_sha`. The controller validates that exact SHA against a successful `staging` gate and reviewed `main` history, starts only PostgreSQL, and runs the digest-pinned migrator without executing scripts from the target revision. 5. A successful migration rechecks that `staging` still points at the same exact SHA, prints the ordered migration ledger, and dispatches the `main` controller for digest-pinned deployment with `allow_rollback=false`. If `staging` advanced during migration, it refuses the stale dispatch. Do not substitute a branch name, a short SHA, or a newer commit. 6. Confirm `https://staging.jyotisha.chat/api/health` and verify that its deployment SHA is the SHA from step 2. 7. After health verification, create the local encrypted backup described below. diff --git a/deploy/run-staging-deploy.sh b/deploy/run-staging-deploy.sh index a9c074a0..34c9f1ad 100755 --- a/deploy/run-staging-deploy.sh +++ b/deploy/run-staging-deploy.sh @@ -146,8 +146,8 @@ rollback() { } trap rollback ERR -"${compose[@]}" up -d --no-build --remove-orphans switched=true +"${compose[@]}" up -d --no-build --remove-orphans verify_container_image() { local service="$1" diff --git a/deploy/sync-staging-tree.sh b/deploy/sync-staging-tree.sh index bd5f899f..005766fa 100755 --- a/deploy/sync-staging-tree.sh +++ b/deploy/sync-staging-tree.sh @@ -9,6 +9,7 @@ fi rsync -az --delete \ --exclude='/.git/' \ --exclude='/.env*' \ + --exclude='/.docker/' \ --exclude='/backups/' \ --exclude='/.state/' \ --exclude='/.incoming/' \ diff --git a/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md b/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md index 2d8b89ab..9d19f725 100644 --- a/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md +++ b/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md @@ -23,6 +23,10 @@ - Staging publication uses full Git SHA tags for discovery, but deployment is authorized and pinned by the build outputs' `sha256` manifest digests. Never deploy a mutable tag such as `latest`, or treat a tag alone as image identity. +- The `main` workflow revision is the trusted deployment controller. Target and + rollback SHAs must already be ancestors of reviewed `main`; their code is + represented by the digest-pinned images, but their validators and remote + orchestration scripts are never executed with staging Environment privileges. - Finish each task with the focused commit shown. ## Planned Files diff --git a/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md b/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md index 2adbd688..1484b232 100644 --- a/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md +++ b/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md @@ -182,7 +182,7 @@ touching the running application. After the operator runs the manual migration workflow successfully, that workflow dispatches staging deployment again for the same full SHA. The check may read the migration ledger but may never apply SQL. -Deployment and migration share one Actions concurrency group and one host-side lock covering live-tree synchronization through their final database/application verification. Deployment records the previous application SHA, image digests, and image IDs. It rejects stale or backward automatic revisions, verifies running container image IDs/RepoDigests plus the application-reported SHA, and requires public and private health checks before updating deployed-revision state. An older application revision requires an explicit manual rollback authorization; application rollback does not claim to roll back database state. +Deployment and migration share one Actions concurrency group and one host-side lock covering live-tree synchronization through their final database/application verification. The `main` controller owns manifest validation and remote orchestration: it requires a target SHA already present in reviewed `main` history, uploads only allowlisted controller files, and never executes deployment scripts from the target or rollback revision. Deployment records the previous application SHA, image digests, and image IDs. It rejects stale or backward automatic revisions, verifies running container image IDs/RepoDigests plus the application-reported SHA, and requires public and private health checks before updating deployed-revision state. An older application revision requires an explicit manual rollback authorization; application rollback does not claim to roll back database state. ## Automatic backend quality gate diff --git a/frontend/db/migrations/20260720000100_backend_foundation.sql b/frontend/db/migrations/20260720000100_backend_foundation.sql index ca47fee9..b20e845e 100644 --- a/frontend/db/migrations/20260720000100_backend_foundation.sql +++ b/frontend/db/migrations/20260720000100_backend_foundation.sql @@ -7,6 +7,8 @@ grant usage on schema identity to identity_runtime, admin_runtime; grant usage on schema public to app_runtime, admin_runtime; grant usage on schema audit to admin_runtime; +alter default privileges for role schema_owner + revoke execute on functions from public; alter default privileges for role schema_owner in schema identity revoke all on tables from public; alter default privileges for role schema_owner in schema public diff --git a/frontend/tests/database-foundation.test.ts b/frontend/tests/database-foundation.test.ts index dda1715d..82f7a43e 100644 --- a/frontend/tests/database-foundation.test.ts +++ b/frontend/tests/database-foundation.test.ts @@ -373,13 +373,16 @@ test("foundation grants no direct runtime table DML and exposes only reviewed fu create table audit.admin_event_probe ( value text not null ); + create function identity.unreviewed_identity_probe() + returns text + language sql + as 'select ''not callable''::text'; create function audit.record_admin_event_probe(event_value text) returns void language sql security definer set search_path = pg_catalog, audit as 'insert into audit.admin_event_probe(value) values (event_value)'; - revoke all on function audit.record_admin_event_probe(text) from public; grant execute on function audit.record_admin_event_probe(text) to admin_runtime; `, ); @@ -412,6 +415,19 @@ test("foundation grants no direct runtime table DML and exposes only reviewed fu "update public.runtime_boundary_probe set value = 'denied'", ), ); + assert.equal( + fixture.psql( + "select has_function_privilege('identity_runtime', 'identity.unreviewed_identity_probe()', 'execute')", + ), + "f", + ); + assert.throws(() => + fixture.psqlAs( + "identity_runtime", + "identity-runtime-test-password", + "select identity.unreviewed_identity_probe()", + ), + ); assert.equal( fixture.psqlAs( diff --git a/frontend/tests/health-deployment.test.ts b/frontend/tests/health-deployment.test.ts index 41010ce5..5eb6ac80 100644 --- a/frontend/tests/health-deployment.test.ts +++ b/frontend/tests/health-deployment.test.ts @@ -132,6 +132,10 @@ test("staging deploy consumes only the isolated staging environment and tested r new URL("../../.github/workflows/deploy-staging.yml", import.meta.url), "utf8", ); + const syncController = readFileSync( + new URL("../../deploy/sync-staging-tree.sh", import.meta.url), + "utf8", + ); assert.match(qualityGate, /push:\s*\n\s*branches: \[staging\]/); assert.match(workflow, /workflows: \["Staging Backend Quality Gate"\]/); @@ -154,13 +158,18 @@ test("staging deploy consumes only the isolated staging environment and tested r assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/); assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/); assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/); - assert.match(workflow, /--exclude='\/\.env\*'/); - assert.match(workflow, /--exclude='\/backups\/'/); + assert.match( + workflow, + /--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/, + ); assert.match(workflow, /run-staging-deploy\.sh/); assert.match(workflow, /steps\.images\.outputs\.api_image/); assert.match(workflow, /steps\.images\.outputs\.web_image/); assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/); assert.doesNotMatch(workflow, /103\.117\.123\.53/); + assert.match(syncController, /--exclude='\/\.env\*'/); + assert.match(syncController, /--exclude='\/\.docker\/'/); + assert.match(syncController, /--exclude='\/backups\/'/); }); test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => { diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 9794f0ca..677391a4 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -92,6 +92,8 @@ test("live staging sync preserves env, state, incoming files, and encrypted back mkdirSync(source); mkdirSync(destination); writeFileSync(join(source, "revision.txt"), "new\n"); + mkdirSync(join(source, ".docker")); + writeFileSync(join(source, ".docker", "config.json"), "temporary-token\n"); writeFileSync(join(destination, "stale.txt"), "old\n"); for (const relative of [ ".env.staging", @@ -114,6 +116,7 @@ test("live staging sync preserves env, state, incoming files, and encrypted back assert.equal(result.status, 0, result.stderr); assert.equal(existsSync(join(destination, "stale.txt")), false); assert.equal(readFileSync(join(destination, "revision.txt"), "utf8"), "new\n"); + assert.equal(existsSync(join(destination, ".docker")), false); for (const relative of [ ".env.staging", ".env.staging.database", @@ -155,6 +158,7 @@ test("deploy and migration consume the exact successful gate artifact", () => { assert.match(workflow, /\.head_branch == "staging"/); assert.match(workflow, /\.event == "push"/); assert.match(workflow, /\.conclusion == "success"/); + assert.match(workflow, /sort_by\(\.id\) \| reverse \| first/); assert.match(workflow, /uses: actions\/download-artifact@v4/); assert.match(workflow, /run-id: \$\{\{ steps\.revision\.outputs\.gate_run_id \}\}/); assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/); @@ -162,6 +166,25 @@ test("deploy and migration consume the exact successful gate artifact", () => { } }); +test("main owns the deployment control plane and target revisions are data only", () => { + for (const workflow of [read(deployWorkflow), read(migrationWorkflow)]) { + assert.match(workflow, /name: Checkout trusted main controller[\s\S]*ref: main/); + assert.match(workflow, /fetch-depth: 0/); + assert.match(workflow, /git merge-base --is-ancestor "\$DEPLOY_SHA" HEAD/); + assert.match(workflow, /--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/); + assert.doesNotMatch(workflow, /ref: \$\{\{ steps\.revision\.outputs\.sha \}\}/); + } +}); + +test("staging mutations retain every pending deployment and migration", () => { + for (const workflow of [read(deployWorkflow), read(migrationWorkflow)]) { + assert.match( + workflow, + /concurrency:\n group: staging-mutation\n cancel-in-progress: false\n queue: max/, + ); + } +}); + test("automatic staging paths reject stale and divergent revisions", () => { const deployment = read(deployWorkflow); const migration = read(migrationWorkflow); @@ -186,6 +209,10 @@ test("remote deployment verifies running image IDs, RepoDigests, and application assert.match(runner, /publicBody\.deployment\?\.gitCommit !== process\.env\.EXPECTED_SHA/); assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/); assert.match(runner, /restoring prior image digests/); + assert.match( + runner, + /switched=true\n"\$\{compose\[@\]\}" up -d --no-build --remove-orphans\n/, + ); assert.doesNotMatch(runner, /jyotisha-(?:api|web):\$DEPLOY_SHA/); }); @@ -223,6 +250,10 @@ test("run-local registry state and incoming trees are always cleaned up", () => assert.match(workflow, /if: always\(\) && steps\.incoming\.outputs\.path != ''/); assert.match(workflow, /docker logout ghcr\.io/); assert.match(workflow, /rm -rf -- '\$INCOMING_PATH'/); + assert.match( + workflow, + /install -d -m 700 [^\n]*\$incoming[^\n]*\n\s+echo "path=\$incoming" >>"\$GITHUB_OUTPUT"\n\s+rsync/, + ); assert.doesNotMatch(workflow, /--password(?:\s|=)/); } });