From 092efa0ed25a38c2346863cc7b37fba4e6c42055 Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Tue, 28 Jul 2026 13:48:29 +0800 Subject: [PATCH] ops: manage staging rectification rollout --- ...onfigure-staging-rectification-rollout.yml | 92 ++++++++++ deploy/README.md | 2 + ...configure-staging-rectification-rollout.sh | 167 ++++++++++++++++++ .../tests/staging-backend-workflows.test.ts | 113 +++++++++++- 4 files changed, 369 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/configure-staging-rectification-rollout.yml create mode 100755 deploy/configure-staging-rectification-rollout.sh diff --git a/.github/workflows/configure-staging-rectification-rollout.yml b/.github/workflows/configure-staging-rectification-rollout.yml new file mode 100644 index 00000000..a18fb072 --- /dev/null +++ b/.github/workflows/configure-staging-rectification-rollout.yml @@ -0,0 +1,92 @@ +name: Configure Staging Rectification Rollout + +on: + workflow_dispatch: + inputs: + expected_deploy_sha: + description: Exact 40-character SHA currently deployed to staging + required: true + type: string + audience: + description: New-case creation audience + required: true + default: paused + type: choice + options: + - paused + - smoke_only + - public + synthetic_smoke_user_ids: + description: Comma-separated canonical UUIDs; required only for smoke_only + required: false + type: string + +permissions: + contents: read + +concurrency: + group: staging-mutation + cancel-in-progress: false + +jobs: + configure: + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: staging + url: ${{ vars.STAGING_URL }} + env: + DEPLOY_HOST: ${{ vars.STAGING_HOST }} + DEPLOY_PORT: ${{ vars.STAGING_PORT }} + DEPLOY_USER: ${{ vars.STAGING_USER }} + DEPLOY_PATH: ${{ vars.STAGING_PATH }} + STAGING_URL: ${{ vars.STAGING_URL }} + STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }} + EXPECTED_DEPLOY_SHA: ${{ inputs.expected_deploy_sha }} + ROLLOUT_AUDIENCE: ${{ inputs.audience }} + SYNTHETIC_SMOKE_USER_IDS: ${{ inputs.synthetic_smoke_user_ids }} + + steps: + - name: Checkout trusted controller + uses: actions/checkout@v4 + with: + ref: main + persist-credentials: false + + - name: Validate rollout request and staging target + run: | + set -euo pipefail + [[ "$EXPECTED_DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] + case "$ROLLOUT_AUDIENCE" in paused|smoke_only|public) ;; *) exit 1 ;; esac + if [ "$ROLLOUT_AUDIENCE" = smoke_only ]; then + [[ "$SYNTHETIC_SMOKE_USER_IDS" =~ ^[0-9a-f-]{36}(,[0-9a-f-]{36})*$ ]] + else + test -z "$SYNTHETIC_SMOKE_USER_IDS" + fi + test "$DEPLOY_HOST" = "118.26.111.127" + test "$DEPLOY_PORT" = "22" + test "$DEPLOY_USER" = "deploy" + test "$DEPLOY_PATH" = "/opt/jyotisha-staging" + test "$STAGING_URL" = "https://staging.jyotisha.chat" + test -n "$STAGING_KNOWN_HOSTS" + bash -n deploy/configure-staging-rectification-rollout.sh + + - name: Configure pinned staging SSH + env: + SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} + run: | + set -euo pipefail + test -n "$SSH_PRIVATE_KEY" + install -d -m 700 ~/.ssh + printf '%s\n' "$SSH_PRIVATE_KEY" >~/.ssh/jyotisha-staging + chmod 600 ~/.ssh/jyotisha-staging + printf '%s\n' "$STAGING_KNOWN_HOSTS" >~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + + - name: Apply rollout under staging mutation lock + run: | + set -euo pipefail + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=10" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "DEPLOY_PATH='$DEPLOY_PATH' EXPECTED_DEPLOY_SHA='$EXPECTED_DEPLOY_SHA' ROLLOUT_AUDIENCE='$ROLLOUT_AUDIENCE' SYNTHETIC_SMOKE_USER_IDS='$SYNTHETIC_SMOKE_USER_IDS' STAGING_URL='$STAGING_URL' bash -s" \ + < deploy/configure-staging-rectification-rollout.sh diff --git a/deploy/README.md b/deploy/README.md index c14ce6c3..fc9142b9 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -193,6 +193,8 @@ After source sync and before `up`, the workflow validates `.env.staging` mode/se 6. If the read-only checker reports a pending migration, stop app deployment and run `Migrate Staging Database` manually with the same full SHA; a successful migration re-dispatches `Deploy staging` with that same SHA. 7. Confirm `https://staging.jyotisha.chat/api/health` reports the exact SHA and private API health. +After the exact-SHA deployment and migrations are verified, use the manual `Configure Staging Rectification Rollout` workflow to change new-case creation. Supply the SHA currently reported by `/api/health`; choose `public` to open all staging accounts, `smoke_only` with canonical test-account UUIDs for a canary, or `paused` to close creation. The workflow updates only the four `RECTIFICATION_V3_*` rollout variables under the shared host lock, recreates `web` and `rectification-v4-worker` with the already deployed image, and rolls back the env file if health does not match the requested audience. Do not edit or print `.env.staging` through CI logs. + Application rollback uses the same workflow: manually dispatch `Deploy staging` from the `main` controller with a previous known-good full SHA that has a successful `Staging Backend Quality Gate` run, and explicitly set `allow_rollback=true`. Normal and migration-triggered deployments reject stale, divergent, or backward revisions. Rollback still consumes the selected gate run's digest manifest and is supported only during that artifact's 30-day retention window; after expiry, stop and prepare a separately reviewed republish/recovery change rather than substituting a mutable tag or assuming the old run can still be rerun. Database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. Inspect staging without printing secrets: diff --git a/deploy/configure-staging-rectification-rollout.sh b/deploy/configure-staging-rectification-rollout.sh new file mode 100755 index 00000000..4edfbf9b --- /dev/null +++ b/deploy/configure-staging-rectification-rollout.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +required=(DEPLOY_PATH EXPECTED_DEPLOY_SHA ROLLOUT_AUDIENCE STAGING_URL) +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "required staging rollout input is missing: $key" >&2 + exit 1 + fi +done + +sha_pattern='^[0-9a-f]{40}$' +uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' +[[ "$EXPECTED_DEPLOY_SHA" =~ $sha_pattern ]] || { + echo "invalid expected deployment SHA" >&2 + exit 1 +} +case "$ROLLOUT_AUDIENCE" in + paused|smoke_only|public) ;; + *) echo "invalid rollout audience" >&2; exit 1 ;; +esac + +smoke_user_ids="${SYNTHETIC_SMOKE_USER_IDS:-}" +if [ "$ROLLOUT_AUDIENCE" = "smoke_only" ]; then + [ -n "$smoke_user_ids" ] || { + echo "smoke_only requires at least one synthetic user UUID" >&2 + exit 1 + } + IFS=',' read -ra smoke_users <<<"$smoke_user_ids" + for user_id in "${smoke_users[@]}"; do + [[ "$user_id" =~ $uuid_pattern ]] || { + echo "invalid synthetic smoke user UUID" >&2 + exit 1 + } + done +else + [ -z "$smoke_user_ids" ] || { + echo "synthetic smoke users are only valid for smoke_only" >&2 + exit 1 + } +fi + +state_directory="$DEPLOY_PATH/.state" +env_file="$DEPLOY_PATH/.env.staging" +install -d -m 700 "$state_directory" +exec 9>"$state_directory/mutation.lock" +flock -n 9 || { + echo "another staging mutation holds the host lock" >&2 + exit 75 +} + +compose_files=( + -f deploy/docker-compose.server.yml + -f deploy/docker-compose.postgres.yml + -f deploy/docker-compose.staging.yml +) + +[ -f "$env_file" ] || { + echo "staging environment file is missing" >&2 + exit 1 +} +current_sha="$(<"$state_directory/deployed-revision")" +[ "$current_sha" = "$EXPECTED_DEPLOY_SHA" ] || { + echo "deployed staging revision does not match the approved rollout SHA" >&2 + exit 1 +} + +case "$ROLLOUT_AUDIENCE" in + public) + creation_enabled=true + smoke_sha="$EXPECTED_DEPLOY_SHA" + smoke_user_ids="" + ;; + smoke_only) + creation_enabled=true + smoke_sha="" + ;; + paused) + creation_enabled=false + smoke_sha="" + smoke_user_ids="" + ;; +esac + +backup="$(mktemp "$state_directory/rectification-rollout-backup.XXXXXX")" +temporary="$(mktemp "$DEPLOY_PATH/.env.staging.rollout.XXXXXX")" +declare -a compose=() +cleanup() { rm -f -- "$backup" "$temporary"; } +rollback() { + local status=$? + cp -p -- "$backup" "$env_file" + if [ "${#compose[@]}" -gt 0 ]; then + "${compose[@]}" up -d --no-build --pull never --force-recreate --no-deps web rectification-v4-worker >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT +cp -p -- "$env_file" "$backup" + +awk \ + -v create="$creation_enabled" \ + -v migrations="true" \ + -v smoke_sha="$smoke_sha" \ + -v smoke_users="$smoke_user_ids" ' +BEGIN { + values["RECTIFICATION_V3_CREATE_ENABLED"] = create + values["RECTIFICATION_V3_MIGRATIONS_READY"] = migrations + values["RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA"] = smoke_sha + values["RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS"] = smoke_users +} +{ + split($0, parts, "=") + key = parts[1] + if (key in values) { + if (!(key in written)) print key "=" values[key] + written[key] = 1 + next + } + print +} +END { + for (key in values) if (!(key in written)) print key "=" values[key] +} +' "$env_file" >"$temporary" +chmod 600 "$temporary" + +cd "$DEPLOY_PATH" +bash deploy/validate-staging-env.sh "$temporary" staging.jyotisha.chat deploy/Caddyfile.staging +mv -f -- "$temporary" "$env_file" +trap rollback ERR + +web_container="$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1)" +[ -n "$web_container" ] || { + echo "staging web container is missing" >&2 + false +} +export WEB_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$web_container")" +export APP_ENV_FILE='../.env.staging' +export DATABASE_ENV_FILE='../.env.staging.database' +export CADDYFILE_PATH='./Caddyfile.staging' +export SITE_ADDRESS='https://staging.jyotisha.chat' +export ADMIN_SITE_ADDRESS='https://admin.staging.jyotisha.chat' +export GITHUB_SHA="$EXPECTED_DEPLOY_SHA" +compose=(docker compose -p jyotisha-staging --env-file .env.staging "${compose_files[@]}") + +"${compose[@]}" config --quiet +"${compose[@]}" up -d --no-build --pull never --force-recreate --no-deps web rectification-v4-worker + +health="" +for _ in $(seq 1 30); do + health="$(curl --fail --silent --show-error "$STAGING_URL/api/health" 2>/dev/null || true)" + expected_ready=false + [ "$ROLLOUT_AUDIENCE" = public ] && expected_ready=true + if grep -Fq "\"gitCommit\":\"$EXPECTED_DEPLOY_SHA\"" <<<"$health" && + grep -Fq "\"creationAudience\":\"$ROLLOUT_AUDIENCE\"" <<<"$health" && + grep -Fq "\"readyForNewCases\":$expected_ready" <<<"$health"; then + trap - ERR + printf 'rectification rollout audience=%s deployed_sha=%s ready_for_new_cases=%s\n' \ + "$ROLLOUT_AUDIENCE" "$EXPECTED_DEPLOY_SHA" "$([ "$ROLLOUT_AUDIENCE" = public ] && echo true || echo false)" + exit 0 + fi + sleep 2 +done + +echo "staging rollout health verification failed" >&2 +false diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 2c849ddc..9986d41c 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -18,6 +18,10 @@ const migrationWorkflow = new URL( "../../.github/workflows/migrate-staging-database.yml", import.meta.url, ); +const rolloutWorkflow = new URL( + "../../.github/workflows/configure-staging-rectification-rollout.yml", + import.meta.url, +); const deployScript = new URL( "../../deploy/run-staging-deploy.sh", import.meta.url, @@ -26,6 +30,10 @@ const migrationScript = new URL( "../../deploy/run-staging-migration.sh", import.meta.url, ); +const rolloutScript = new URL( + "../../deploy/configure-staging-rectification-rollout.sh", + import.meta.url, +); const syncScript = new URL( "../../deploy/sync-staging-tree.sh", import.meta.url, @@ -49,7 +57,7 @@ function assertOrder(text: string, labels: string[]): void { } test("changed staging workflows are syntactically valid YAML", () => { - for (const workflow of [qualityWorkflow, deployWorkflow, migrationWorkflow]) { + for (const workflow of [qualityWorkflow, deployWorkflow, migrationWorkflow, rolloutWorkflow]) { const result = spawnSync( "ruby", ["-e", "require 'yaml'; YAML.parse_file(ARGV.fetch(0))", fileURLToPath(workflow)], @@ -151,16 +159,21 @@ test("all staging mutations share Actions serialization and one host lock", () = const deployRunner = read(deployScript); const migrationRunner = read(migrationScript); - for (const workflow of [deployment, migration]) { + const rollout = read(rolloutWorkflow); + const rolloutRunner = read(rolloutScript); + + for (const workflow of [deployment, migration, rollout]) { assert.match(workflow, /concurrency:\n\s+group: staging-mutation\n\s+cancel-in-progress: false/); } - for (const runner of [deployRunner, migrationRunner]) { + for (const runner of [deployRunner, migrationRunner, rolloutRunner]) { assert.match(runner, /state_directory="\$DEPLOY_PATH\/\.state"/); assert.match(runner, /state_directory\/mutation\.lock/); assert.match(runner, /flock -n 9/); - assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("sync-staging-tree.sh")); assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("docker")); } + for (const runner of [deployRunner, migrationRunner]) { + assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("sync-staging-tree.sh")); + } }); test("deploy and migration consume the exact successful gate artifact", () => { @@ -391,8 +404,98 @@ test("production remains manual-only and separate from staging database automati assert.doesNotMatch(production, /docker-compose\.postgres\.yml|db:migrate/); }); + +test("public rectification rollout rewrites only rollout gates and recreates web runtimes", () => { + const root = mkdtempSync(join(tmpdir(), "jyotisha-rollout-")); + const deploymentPath = join(root, "app"); + const statePath = join(deploymentPath, ".state"); + const deployPath = join(deploymentPath, "deploy"); + const mockBin = join(root, "bin"); + const sha = "8".repeat(40); + mkdirSync(statePath, { recursive: true }); + mkdirSync(deployPath, { recursive: true }); + mkdirSync(mockBin, { recursive: true }); + writeFileSync(join(statePath, "deployed-revision"), sha); + writeFileSync( + join(deploymentPath, ".env.staging"), + [ + "APP_ENV_FILE=../.env.staging", + "CADDYFILE_PATH=./Caddyfile.staging", + "SITE_ADDRESS=https://staging.jyotisha.chat", + "ADMIN_SITE_ADDRESS=https://admin.staging.jyotisha.chat", + "AUTH_PROVIDER=self-hosted", + "SELF_HOSTED_IDENTITY_ENABLED=true", + "AUTH_USER_ORIGIN=https://staging.jyotisha.chat", + "AUTH_ADMIN_ORIGIN=https://admin.staging.jyotisha.chat", + `IDENTITY_DATABASE_URL=postgresql://identity_runtime:${"i".repeat(40)}@postgres:5432/jyotisha`, + `APP_DATABASE_URL=postgresql://app_runtime:${"a".repeat(40)}@postgres:5432/jyotisha`, + `ADMIN_DATABASE_URL=postgresql://admin_runtime:${"d".repeat(40)}@postgres:5432/jyotisha`, + `BETTER_AUTH_USER_SECRET=${"u".repeat(32)}`, + `BETTER_AUTH_ADMIN_SECRET=${"v".repeat(32)}`, + "RESEND_API_KEY=re_test_key", + "RESEND_FROM_EMAIL=test@example.com", + "ADMIN_EMAILS=admin@example.com", + `JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=${"t".repeat(32)}`, + "KEEP_ME=unchanged", + "RECTIFICATION_V3_CREATE_ENABLED=false", + "RECTIFICATION_V3_MIGRATIONS_READY=false", + "RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA=old", + "RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS=00000000-0000-4000-8000-000000009001", + "", + ].join("\n"), + { mode: 0o600 }, + ); + writeFileSync( + join(deployPath, "validate-staging-env.sh"), + readFileSync(new URL("../../deploy/validate-staging-env.sh", import.meta.url), "utf8"), + ); + writeFileSync(join(mockBin, "flock"), "#!/usr/bin/env bash\nexit 0\n"); + writeFileSync( + join(mockBin, "docker"), + [ + "#!/usr/bin/env bash", + 'if [ "$1" = ps ]; then echo web-container; exit 0; fi', + `if [ "$1" = inspect ]; then echo ghcr.io/jesse-ux/jyotisha-web@sha256:${"b".repeat(64)}; exit 0; fi`, + `printf '%s\n' "$*" >>${join(root, "docker.log")}`, + ].join("\n"), + ); + writeFileSync( + join(mockBin, "curl"), + `#!/usr/bin/env bash\nprintf '%s' '{"deployment":{"gitCommit":"${sha}"},"rollout":{"conversationalRectificationV3":{"creationAudience":"public","readyForNewCases":true}}}'\n`, + ); + for (const command of ["flock", "docker", "curl"]) { + chmodSync(join(mockBin, command), 0o755); + } + + try { + const result = spawnSync("bash", [fileURLToPath(rolloutScript)], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${mockBin}:${process.env.PATH ?? ""}`, + DEPLOY_PATH: deploymentPath, + EXPECTED_DEPLOY_SHA: sha, + ROLLOUT_AUDIENCE: "public", + SYNTHETIC_SMOKE_USER_IDS: "", + STAGING_URL: "https://staging.jyotisha.chat", + }, + }); + assert.equal(result.status, 0, result.stderr); + const env = readFileSync(join(deploymentPath, ".env.staging"), "utf8"); + assert.match(env, /^KEEP_ME=unchanged$/m); + assert.match(env, /^RECTIFICATION_V3_CREATE_ENABLED=true$/m); + assert.match(env, /^RECTIFICATION_V3_MIGRATIONS_READY=true$/m); + assert.match(env, new RegExp(`^RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA=${sha}$`, "m")); + assert.match(env, /^RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS=$/m); + assert.equal((env.match(/^RECTIFICATION_V3_CREATE_ENABLED=/gm) ?? []).length, 1); + assert.match(readFileSync(join(root, "docker.log"), "utf8"), /force-recreate --no-deps web rectification-v4-worker/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("staging scripts pass shell syntax validation", () => { - for (const script of [deployScript, migrationScript, syncScript]) { + for (const script of [deployScript, migrationScript, rolloutScript, syncScript]) { const path = fileURLToPath(script); chmodSync(path, 0o755); const result = spawnSync("bash", ["-n", path], { encoding: "utf8" });