fix: keep staging control plane on main

This commit is contained in:
Jesse_Chen
2026-07-21 04:19:26 +08:00
parent bd7c9830bb
commit 9e119df977
11 changed files with 100 additions and 27 deletions
+15 -10
View File
@@ -24,6 +24,7 @@ permissions:
concurrency:
group: staging-mutation
cancel-in-progress: false
queue: max
jobs:
deploy:
@@ -78,7 +79,7 @@ jobs:
[.workflow_runs[] | select(
.head_sha == $sha and .head_branch == "staging" and
.event == "push" and .conclusion == "success"
)] | sort_by(.run_attempt) | reverse | first
)] | sort_by(.id) | reverse | first
' <<<"$runs")"
gate_run_id="$(jq -er '.id' <<<"$selected_run")"
gate_run_attempt="$(jq -er '.run_attempt' <<<"$selected_run")"
@@ -110,10 +111,11 @@ jobs:
echo "allow_rollback=$allow_rollback"
} >>"$GITHUB_OUTPUT"
- name: Checkout tested revision
- name: Checkout trusted main controller
uses: actions/checkout@v4
with:
ref: ${{ steps.revision.outputs.sha }}
ref: main
fetch-depth: 0
persist-credentials: false
- name: Download gate-produced image manifest
@@ -133,12 +135,16 @@ jobs:
node frontend/scripts/staging-image-manifest.mjs \
artifacts/staging-image/manifest.env "$DEPLOY_SHA" >>"$GITHUB_OUTPUT"
- name: Verify revision and staging target
- name: Verify reviewed revision and staging target
env:
DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$DEPLOY_SHA"
git cat-file -e "$DEPLOY_SHA^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" HEAD || {
echo "staging revision is not in the reviewed main history" >&2
exit 1
}
test "$DEPLOY_HOST" = "118.26.111.127"
test "$DEPLOY_PORT" = "22"
test "$DEPLOY_USER" = "deploy"
@@ -195,7 +201,7 @@ jobs:
echo "forward_verified=$forward_verified"
} >>"$GITHUB_OUTPUT"
- name: Stage tested sources in an isolated incoming directory
- name: Stage trusted controller files in an isolated incoming directory
id: incoming
run: |
set -euo pipefail
@@ -203,11 +209,10 @@ jobs:
RSYNC_SSH="ssh $SSH_OPTIONS"
incoming="$DEPLOY_PATH/.incoming/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$incoming'"
rsync -az --delete \
--exclude='/.git/' --exclude='/.env*' --exclude='/backups/' \
--exclude='/frontend/node_modules/' --exclude='/frontend/.next/' \
-e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/"
echo "path=$incoming" >>"$GITHUB_OUTPUT"
rsync -az --delete --prune-empty-dirs \
--include='/deploy/' --include='/deploy/***' --exclude='*' \
-e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/"
- name: Log in to GHCR with run-local Docker state
env:
+15 -10
View File
@@ -11,6 +11,7 @@ on:
concurrency:
group: staging-mutation
cancel-in-progress: false
queue: max
permissions:
contents: read
@@ -51,7 +52,7 @@ jobs:
[.workflow_runs[] | select(
.head_sha == $sha and .head_branch == "staging" and
.event == "push" and .conclusion == "success"
)] | sort_by(.run_attempt) | reverse | first
)] | sort_by(.id) | reverse | first
' <<<"$runs")"
gate_run_id="$(jq -er '.id' <<<"$selected_run")"
gate_run_attempt="$(jq -er '.run_attempt' <<<"$selected_run")"
@@ -73,10 +74,11 @@ jobs:
echo "gate_run_attempt=$gate_run_attempt"
} >>"$GITHUB_OUTPUT"
- name: Checkout tested revision
- name: Checkout trusted main controller
uses: actions/checkout@v4
with:
ref: ${{ steps.revision.outputs.sha }}
ref: main
fetch-depth: 0
persist-credentials: false
- name: Download gate-produced image manifest
@@ -96,12 +98,16 @@ jobs:
node frontend/scripts/staging-image-manifest.mjs \
artifacts/staging-image/manifest.env "$DEPLOY_SHA" >>"$GITHUB_OUTPUT"
- name: Verify revision and staging target
- name: Verify reviewed revision and staging target
env:
DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$DEPLOY_SHA"
git cat-file -e "$DEPLOY_SHA^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" HEAD || {
echo "staging revision is not in the reviewed main history" >&2
exit 1
}
test "$DEPLOY_HOST" = "118.26.111.127"
test "$DEPLOY_PORT" = "22"
test "$DEPLOY_USER" = "deploy"
@@ -152,7 +158,7 @@ jobs:
echo "forward_verified=true"
} >>"$GITHUB_OUTPUT"
- name: Stage tested sources in an isolated incoming directory
- name: Stage trusted controller files in an isolated incoming directory
id: incoming
run: |
set -euo pipefail
@@ -160,11 +166,10 @@ jobs:
RSYNC_SSH="ssh $SSH_OPTIONS"
incoming="$DEPLOY_PATH/.incoming/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$incoming'"
rsync -az --delete \
--exclude='/.git/' --exclude='/.env*' --exclude='/backups/' \
--exclude='/frontend/node_modules/' --exclude='/frontend/.next/' \
-e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/"
echo "path=$incoming" >>"$GITHUB_OUTPUT"
rsync -az --delete --prune-empty-dirs \
--include='/deploy/' --include='/deploy/***' --exclude='*' \
-e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/"
- name: Log in to GHCR with run-local Docker state
env:
+2 -2
View File
@@ -145,7 +145,7 @@ Staging is isolated from production:
| Supabase | separate `Jyotisha Staging` project |
| GitHub Environment | `staging` |
The GitHub `staging` Environment contains the secret `STAGING_SSH_PRIVATE_KEY` and the variables `STAGING_HOST`, `STAGING_PORT`, `STAGING_USER`, `STAGING_PATH`, `STAGING_URL`, and `STAGING_KNOWN_HOSTS`. Its deployment branch policy allows the `main` controller branch: GitHub's `workflow_run` event executes from the default branch while the workflow separately requires the successfully tested upstream branch to be `staging`. The staging key, database, Supabase keys, and model-provider keys must not be shared with production.
The GitHub `staging` Environment contains the secret `STAGING_SSH_PRIVATE_KEY` and the variables `STAGING_HOST`, `STAGING_PORT`, `STAGING_USER`, `STAGING_PATH`, `STAGING_URL`, and `STAGING_KNOWN_HOSTS`. Its deployment branch policy allows the `main` controller branch: GitHub's `workflow_run` event executes from the default branch while the workflow separately requires the successfully tested upstream branch to be `staging`. The controller checks out only `main` with full history, requires the requested staging SHA to be an ancestor of that reviewed history, and uploads only the allowlisted `deploy/` control files. It never executes deployment validators or remote orchestration scripts from the target/rollback revision. The staging key, database, Supabase keys, and model-provider keys must not be shared with production.
The repository-level public build inputs are configured at GitHub **Settings -> Secrets and variables -> Actions -> Variables** (the UI is also shown as **Settings → Secrets and variables → Actions → Variables**): `STAGING_SUPABASE_URL` and `STAGING_SUPABASE_ANON_KEY`. They are public build inputs, required for publish, and exposed to the browser; keep them staging-only and never print their values in workflow output, summaries, or support messages. The workflow passes them only as the `NEXT_PUBLIC_*` build arguments after non-empty/HTTPS validation.
@@ -231,7 +231,7 @@ Use this order for every staging revision:
1. Merge to `staging` after reviewing the change.
2. Wait for `Staging Backend Quality Gate` to pass and publish that exact full SHA's API/web digest manifest.
3. The automatic `Deploy staging` workflow checks the exact SHA in read-only migration-check mode before changing API, web, or Caddy. If it reports pending or drifted migrations, stop; do not retry the application deployment as if it were a migration.
4. Open **Migrate Staging Database -> Run workflow** and enter the reported full lowercase 40-character SHA in `deploy_sha`. The workflow validates that exact SHA against a successful `staging` gate, checks it out, starts only PostgreSQL, and runs the reviewed migrator.
4. Open **Migrate Staging Database -> Run workflow**, select **Use workflow from: main**, and enter the reported full lowercase 40-character SHA in `deploy_sha`. The controller validates that exact SHA against a successful `staging` gate and reviewed `main` history, starts only PostgreSQL, and runs the digest-pinned migrator without executing scripts from the target revision.
5. A successful migration rechecks that `staging` still points at the same exact SHA, prints the ordered migration ledger, and dispatches the `main` controller for digest-pinned deployment with `allow_rollback=false`. If `staging` advanced during migration, it refuses the stale dispatch. Do not substitute a branch name, a short SHA, or a newer commit.
6. Confirm `https://staging.jyotisha.chat/api/health` and verify that its deployment SHA is the SHA from step 2.
7. After health verification, create the local encrypted backup described below.
+1 -1
View File
@@ -146,8 +146,8 @@ rollback() {
}
trap rollback ERR
"${compose[@]}" up -d --no-build --remove-orphans
switched=true
"${compose[@]}" up -d --no-build --remove-orphans
verify_container_image() {
local service="$1"
+1
View File
@@ -9,6 +9,7 @@ fi
rsync -az --delete \
--exclude='/.git/' \
--exclude='/.env*' \
--exclude='/.docker/' \
--exclude='/backups/' \
--exclude='/.state/' \
--exclude='/.incoming/' \
@@ -23,6 +23,10 @@
- Staging publication uses full Git SHA tags for discovery, but deployment is
authorized and pinned by the build outputs' `sha256` manifest digests. Never
deploy a mutable tag such as `latest`, or treat a tag alone as image identity.
- The `main` workflow revision is the trusted deployment controller. Target and
rollback SHAs must already be ancestors of reviewed `main`; their code is
represented by the digest-pinned images, but their validators and remote
orchestration scripts are never executed with staging Environment privileges.
- Finish each task with the focused commit shown.
## Planned Files
@@ -182,7 +182,7 @@ touching the running application. After the operator runs the manual migration
workflow successfully, that workflow dispatches staging deployment again for the
same full SHA. The check may read the migration ledger but may never apply SQL.
Deployment and migration share one Actions concurrency group and one host-side lock covering live-tree synchronization through their final database/application verification. Deployment records the previous application SHA, image digests, and image IDs. It rejects stale or backward automatic revisions, verifies running container image IDs/RepoDigests plus the application-reported SHA, and requires public and private health checks before updating deployed-revision state. An older application revision requires an explicit manual rollback authorization; application rollback does not claim to roll back database state.
Deployment and migration share one Actions concurrency group and one host-side lock covering live-tree synchronization through their final database/application verification. The `main` controller owns manifest validation and remote orchestration: it requires a target SHA already present in reviewed `main` history, uploads only allowlisted controller files, and never executes deployment scripts from the target or rollback revision. Deployment records the previous application SHA, image digests, and image IDs. It rejects stale or backward automatic revisions, verifies running container image IDs/RepoDigests plus the application-reported SHA, and requires public and private health checks before updating deployed-revision state. An older application revision requires an explicit manual rollback authorization; application rollback does not claim to roll back database state.
## Automatic backend quality gate
@@ -7,6 +7,8 @@ grant usage on schema identity to identity_runtime, admin_runtime;
grant usage on schema public to app_runtime, admin_runtime;
grant usage on schema audit to admin_runtime;
alter default privileges for role schema_owner
revoke execute on functions from public;
alter default privileges for role schema_owner in schema identity
revoke all on tables from public;
alter default privileges for role schema_owner in schema public
+17 -1
View File
@@ -373,13 +373,16 @@ test("foundation grants no direct runtime table DML and exposes only reviewed fu
create table audit.admin_event_probe (
value text not null
);
create function identity.unreviewed_identity_probe()
returns text
language sql
as 'select ''not callable''::text';
create function audit.record_admin_event_probe(event_value text)
returns void
language sql
security definer
set search_path = pg_catalog, audit
as 'insert into audit.admin_event_probe(value) values (event_value)';
revoke all on function audit.record_admin_event_probe(text) from public;
grant execute on function audit.record_admin_event_probe(text) to admin_runtime;
`,
);
@@ -412,6 +415,19 @@ test("foundation grants no direct runtime table DML and exposes only reviewed fu
"update public.runtime_boundary_probe set value = 'denied'",
),
);
assert.equal(
fixture.psql(
"select has_function_privilege('identity_runtime', 'identity.unreviewed_identity_probe()', 'execute')",
),
"f",
);
assert.throws(() =>
fixture.psqlAs(
"identity_runtime",
"identity-runtime-test-password",
"select identity.unreviewed_identity_probe()",
),
);
assert.equal(
fixture.psqlAs(
+11 -2
View File
@@ -132,6 +132,10 @@ test("staging deploy consumes only the isolated staging environment and tested r
new URL("../../.github/workflows/deploy-staging.yml", import.meta.url),
"utf8",
);
const syncController = readFileSync(
new URL("../../deploy/sync-staging-tree.sh", import.meta.url),
"utf8",
);
assert.match(qualityGate, /push:\s*\n\s*branches: \[staging\]/);
assert.match(workflow, /workflows: \["Staging Backend Quality Gate"\]/);
@@ -154,13 +158,18 @@ test("staging deploy consumes only the isolated staging environment and tested r
assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/);
assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/);
assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/);
assert.match(workflow, /--exclude='\/\.env\*'/);
assert.match(workflow, /--exclude='\/backups\/'/);
assert.match(
workflow,
/--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/,
);
assert.match(workflow, /run-staging-deploy\.sh/);
assert.match(workflow, /steps\.images\.outputs\.api_image/);
assert.match(workflow, /steps\.images\.outputs\.web_image/);
assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/);
assert.doesNotMatch(workflow, /103\.117\.123\.53/);
assert.match(syncController, /--exclude='\/\.env\*'/);
assert.match(syncController, /--exclude='\/\.docker\/'/);
assert.match(syncController, /--exclude='\/backups\/'/);
});
test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => {
@@ -92,6 +92,8 @@ test("live staging sync preserves env, state, incoming files, and encrypted back
mkdirSync(source);
mkdirSync(destination);
writeFileSync(join(source, "revision.txt"), "new\n");
mkdirSync(join(source, ".docker"));
writeFileSync(join(source, ".docker", "config.json"), "temporary-token\n");
writeFileSync(join(destination, "stale.txt"), "old\n");
for (const relative of [
".env.staging",
@@ -114,6 +116,7 @@ test("live staging sync preserves env, state, incoming files, and encrypted back
assert.equal(result.status, 0, result.stderr);
assert.equal(existsSync(join(destination, "stale.txt")), false);
assert.equal(readFileSync(join(destination, "revision.txt"), "utf8"), "new\n");
assert.equal(existsSync(join(destination, ".docker")), false);
for (const relative of [
".env.staging",
".env.staging.database",
@@ -155,6 +158,7 @@ test("deploy and migration consume the exact successful gate artifact", () => {
assert.match(workflow, /\.head_branch == "staging"/);
assert.match(workflow, /\.event == "push"/);
assert.match(workflow, /\.conclusion == "success"/);
assert.match(workflow, /sort_by\(\.id\) \| reverse \| first/);
assert.match(workflow, /uses: actions\/download-artifact@v4/);
assert.match(workflow, /run-id: \$\{\{ steps\.revision\.outputs\.gate_run_id \}\}/);
assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/);
@@ -162,6 +166,25 @@ test("deploy and migration consume the exact successful gate artifact", () => {
}
});
test("main owns the deployment control plane and target revisions are data only", () => {
for (const workflow of [read(deployWorkflow), read(migrationWorkflow)]) {
assert.match(workflow, /name: Checkout trusted main controller[\s\S]*ref: main/);
assert.match(workflow, /fetch-depth: 0/);
assert.match(workflow, /git merge-base --is-ancestor "\$DEPLOY_SHA" HEAD/);
assert.match(workflow, /--include='\/deploy\/' --include='\/deploy\/\*\*\*' --exclude='\*'/);
assert.doesNotMatch(workflow, /ref: \$\{\{ steps\.revision\.outputs\.sha \}\}/);
}
});
test("staging mutations retain every pending deployment and migration", () => {
for (const workflow of [read(deployWorkflow), read(migrationWorkflow)]) {
assert.match(
workflow,
/concurrency:\n group: staging-mutation\n cancel-in-progress: false\n queue: max/,
);
}
});
test("automatic staging paths reject stale and divergent revisions", () => {
const deployment = read(deployWorkflow);
const migration = read(migrationWorkflow);
@@ -186,6 +209,10 @@ test("remote deployment verifies running image IDs, RepoDigests, and application
assert.match(runner, /publicBody\.deployment\?\.gitCommit !== process\.env\.EXPECTED_SHA/);
assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/);
assert.match(runner, /restoring prior image digests/);
assert.match(
runner,
/switched=true\n"\$\{compose\[@\]\}" up -d --no-build --remove-orphans\n/,
);
assert.doesNotMatch(runner, /jyotisha-(?:api|web):\$DEPLOY_SHA/);
});
@@ -223,6 +250,10 @@ test("run-local registry state and incoming trees are always cleaned up", () =>
assert.match(workflow, /if: always\(\) && steps\.incoming\.outputs\.path != ''/);
assert.match(workflow, /docker logout ghcr\.io/);
assert.match(workflow, /rm -rf -- '\$INCOMING_PATH'/);
assert.match(
workflow,
/install -d -m 700 [^\n]*\$incoming[^\n]*\n\s+echo "path=\$incoming" >>"\$GITHUB_OUTPUT"\n\s+rsync/,
);
assert.doesNotMatch(workflow, /--password(?:\s|=)/);
}
});