ops: manage staging rectification rollout

This commit is contained in:
Jesse_Chen
2026-07-28 13:48:29 +08:00
parent 8ade6ed5c8
commit 092efa0ed2
4 changed files with 369 additions and 5 deletions
@@ -0,0 +1,92 @@
name: Configure Staging Rectification Rollout
on:
workflow_dispatch:
inputs:
expected_deploy_sha:
description: Exact 40-character SHA currently deployed to staging
required: true
type: string
audience:
description: New-case creation audience
required: true
default: paused
type: choice
options:
- paused
- smoke_only
- public
synthetic_smoke_user_ids:
description: Comma-separated canonical UUIDs; required only for smoke_only
required: false
type: string
permissions:
contents: read
concurrency:
group: staging-mutation
cancel-in-progress: false
jobs:
configure:
runs-on: ubuntu-latest
timeout-minutes: 10
environment:
name: staging
url: ${{ vars.STAGING_URL }}
env:
DEPLOY_HOST: ${{ vars.STAGING_HOST }}
DEPLOY_PORT: ${{ vars.STAGING_PORT }}
DEPLOY_USER: ${{ vars.STAGING_USER }}
DEPLOY_PATH: ${{ vars.STAGING_PATH }}
STAGING_URL: ${{ vars.STAGING_URL }}
STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }}
EXPECTED_DEPLOY_SHA: ${{ inputs.expected_deploy_sha }}
ROLLOUT_AUDIENCE: ${{ inputs.audience }}
SYNTHETIC_SMOKE_USER_IDS: ${{ inputs.synthetic_smoke_user_ids }}
steps:
- name: Checkout trusted controller
uses: actions/checkout@v4
with:
ref: main
persist-credentials: false
- name: Validate rollout request and staging target
run: |
set -euo pipefail
[[ "$EXPECTED_DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
case "$ROLLOUT_AUDIENCE" in paused|smoke_only|public) ;; *) exit 1 ;; esac
if [ "$ROLLOUT_AUDIENCE" = smoke_only ]; then
[[ "$SYNTHETIC_SMOKE_USER_IDS" =~ ^[0-9a-f-]{36}(,[0-9a-f-]{36})*$ ]]
else
test -z "$SYNTHETIC_SMOKE_USER_IDS"
fi
test "$DEPLOY_HOST" = "118.26.111.127"
test "$DEPLOY_PORT" = "22"
test "$DEPLOY_USER" = "deploy"
test "$DEPLOY_PATH" = "/opt/jyotisha-staging"
test "$STAGING_URL" = "https://staging.jyotisha.chat"
test -n "$STAGING_KNOWN_HOSTS"
bash -n deploy/configure-staging-rectification-rollout.sh
- name: Configure pinned staging SSH
env:
SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }}
run: |
set -euo pipefail
test -n "$SSH_PRIVATE_KEY"
install -d -m 700 ~/.ssh
printf '%s\n' "$SSH_PRIVATE_KEY" >~/.ssh/jyotisha-staging
chmod 600 ~/.ssh/jyotisha-staging
printf '%s\n' "$STAGING_KNOWN_HOSTS" >~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
- name: Apply rollout under staging mutation lock
run: |
set -euo pipefail
SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=10"
ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \
"DEPLOY_PATH='$DEPLOY_PATH' EXPECTED_DEPLOY_SHA='$EXPECTED_DEPLOY_SHA' ROLLOUT_AUDIENCE='$ROLLOUT_AUDIENCE' SYNTHETIC_SMOKE_USER_IDS='$SYNTHETIC_SMOKE_USER_IDS' STAGING_URL='$STAGING_URL' bash -s" \
< deploy/configure-staging-rectification-rollout.sh
+2
View File
@@ -193,6 +193,8 @@ After source sync and before `up`, the workflow validates `.env.staging` mode/se
6. If the read-only checker reports a pending migration, stop app deployment and run `Migrate Staging Database` manually with the same full SHA; a successful migration re-dispatches `Deploy staging` with that same SHA.
7. Confirm `https://staging.jyotisha.chat/api/health` reports the exact SHA and private API health.
After the exact-SHA deployment and migrations are verified, use the manual `Configure Staging Rectification Rollout` workflow to change new-case creation. Supply the SHA currently reported by `/api/health`; choose `public` to open all staging accounts, `smoke_only` with canonical test-account UUIDs for a canary, or `paused` to close creation. The workflow updates only the four `RECTIFICATION_V3_*` rollout variables under the shared host lock, recreates `web` and `rectification-v4-worker` with the already deployed image, and rolls back the env file if health does not match the requested audience. Do not edit or print `.env.staging` through CI logs.
Application rollback uses the same workflow: manually dispatch `Deploy staging` from the `main` controller with a previous known-good full SHA that has a successful `Staging Backend Quality Gate` run, and explicitly set `allow_rollback=true`. Normal and migration-triggered deployments reject stale, divergent, or backward revisions. Rollback still consumes the selected gate run's digest manifest and is supported only during that artifact's 30-day retention window; after expiry, stop and prepare a separately reviewed republish/recovery change rather than substituting a mutable tag or assuming the old run can still be rerun. Database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal.
Inspect staging without printing secrets:
+167
View File
@@ -0,0 +1,167 @@
#!/usr/bin/env bash
set -euo pipefail
set +x
required=(DEPLOY_PATH EXPECTED_DEPLOY_SHA ROLLOUT_AUDIENCE STAGING_URL)
for key in "${required[@]}"; do
if [ -z "${!key:-}" ]; then
echo "required staging rollout input is missing: $key" >&2
exit 1
fi
done
sha_pattern='^[0-9a-f]{40}$'
uuid_pattern='^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
[[ "$EXPECTED_DEPLOY_SHA" =~ $sha_pattern ]] || {
echo "invalid expected deployment SHA" >&2
exit 1
}
case "$ROLLOUT_AUDIENCE" in
paused|smoke_only|public) ;;
*) echo "invalid rollout audience" >&2; exit 1 ;;
esac
smoke_user_ids="${SYNTHETIC_SMOKE_USER_IDS:-}"
if [ "$ROLLOUT_AUDIENCE" = "smoke_only" ]; then
[ -n "$smoke_user_ids" ] || {
echo "smoke_only requires at least one synthetic user UUID" >&2
exit 1
}
IFS=',' read -ra smoke_users <<<"$smoke_user_ids"
for user_id in "${smoke_users[@]}"; do
[[ "$user_id" =~ $uuid_pattern ]] || {
echo "invalid synthetic smoke user UUID" >&2
exit 1
}
done
else
[ -z "$smoke_user_ids" ] || {
echo "synthetic smoke users are only valid for smoke_only" >&2
exit 1
}
fi
state_directory="$DEPLOY_PATH/.state"
env_file="$DEPLOY_PATH/.env.staging"
install -d -m 700 "$state_directory"
exec 9>"$state_directory/mutation.lock"
flock -n 9 || {
echo "another staging mutation holds the host lock" >&2
exit 75
}
compose_files=(
-f deploy/docker-compose.server.yml
-f deploy/docker-compose.postgres.yml
-f deploy/docker-compose.staging.yml
)
[ -f "$env_file" ] || {
echo "staging environment file is missing" >&2
exit 1
}
current_sha="$(<"$state_directory/deployed-revision")"
[ "$current_sha" = "$EXPECTED_DEPLOY_SHA" ] || {
echo "deployed staging revision does not match the approved rollout SHA" >&2
exit 1
}
case "$ROLLOUT_AUDIENCE" in
public)
creation_enabled=true
smoke_sha="$EXPECTED_DEPLOY_SHA"
smoke_user_ids=""
;;
smoke_only)
creation_enabled=true
smoke_sha=""
;;
paused)
creation_enabled=false
smoke_sha=""
smoke_user_ids=""
;;
esac
backup="$(mktemp "$state_directory/rectification-rollout-backup.XXXXXX")"
temporary="$(mktemp "$DEPLOY_PATH/.env.staging.rollout.XXXXXX")"
declare -a compose=()
cleanup() { rm -f -- "$backup" "$temporary"; }
rollback() {
local status=$?
cp -p -- "$backup" "$env_file"
if [ "${#compose[@]}" -gt 0 ]; then
"${compose[@]}" up -d --no-build --pull never --force-recreate --no-deps web rectification-v4-worker >/dev/null 2>&1 || true
fi
exit "$status"
}
trap cleanup EXIT
cp -p -- "$env_file" "$backup"
awk \
-v create="$creation_enabled" \
-v migrations="true" \
-v smoke_sha="$smoke_sha" \
-v smoke_users="$smoke_user_ids" '
BEGIN {
values["RECTIFICATION_V3_CREATE_ENABLED"] = create
values["RECTIFICATION_V3_MIGRATIONS_READY"] = migrations
values["RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA"] = smoke_sha
values["RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS"] = smoke_users
}
{
split($0, parts, "=")
key = parts[1]
if (key in values) {
if (!(key in written)) print key "=" values[key]
written[key] = 1
next
}
print
}
END {
for (key in values) if (!(key in written)) print key "=" values[key]
}
' "$env_file" >"$temporary"
chmod 600 "$temporary"
cd "$DEPLOY_PATH"
bash deploy/validate-staging-env.sh "$temporary" staging.jyotisha.chat deploy/Caddyfile.staging
mv -f -- "$temporary" "$env_file"
trap rollback ERR
web_container="$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1)"
[ -n "$web_container" ] || {
echo "staging web container is missing" >&2
false
}
export WEB_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$web_container")"
export APP_ENV_FILE='../.env.staging'
export DATABASE_ENV_FILE='../.env.staging.database'
export CADDYFILE_PATH='./Caddyfile.staging'
export SITE_ADDRESS='https://staging.jyotisha.chat'
export ADMIN_SITE_ADDRESS='https://admin.staging.jyotisha.chat'
export GITHUB_SHA="$EXPECTED_DEPLOY_SHA"
compose=(docker compose -p jyotisha-staging --env-file .env.staging "${compose_files[@]}")
"${compose[@]}" config --quiet
"${compose[@]}" up -d --no-build --pull never --force-recreate --no-deps web rectification-v4-worker
health=""
for _ in $(seq 1 30); do
health="$(curl --fail --silent --show-error "$STAGING_URL/api/health" 2>/dev/null || true)"
expected_ready=false
[ "$ROLLOUT_AUDIENCE" = public ] && expected_ready=true
if grep -Fq "\"gitCommit\":\"$EXPECTED_DEPLOY_SHA\"" <<<"$health" &&
grep -Fq "\"creationAudience\":\"$ROLLOUT_AUDIENCE\"" <<<"$health" &&
grep -Fq "\"readyForNewCases\":$expected_ready" <<<"$health"; then
trap - ERR
printf 'rectification rollout audience=%s deployed_sha=%s ready_for_new_cases=%s\n' \
"$ROLLOUT_AUDIENCE" "$EXPECTED_DEPLOY_SHA" "$([ "$ROLLOUT_AUDIENCE" = public ] && echo true || echo false)"
exit 0
fi
sleep 2
done
echo "staging rollout health verification failed" >&2
false
@@ -18,6 +18,10 @@ const migrationWorkflow = new URL(
"../../.github/workflows/migrate-staging-database.yml",
import.meta.url,
);
const rolloutWorkflow = new URL(
"../../.github/workflows/configure-staging-rectification-rollout.yml",
import.meta.url,
);
const deployScript = new URL(
"../../deploy/run-staging-deploy.sh",
import.meta.url,
@@ -26,6 +30,10 @@ const migrationScript = new URL(
"../../deploy/run-staging-migration.sh",
import.meta.url,
);
const rolloutScript = new URL(
"../../deploy/configure-staging-rectification-rollout.sh",
import.meta.url,
);
const syncScript = new URL(
"../../deploy/sync-staging-tree.sh",
import.meta.url,
@@ -49,7 +57,7 @@ function assertOrder(text: string, labels: string[]): void {
}
test("changed staging workflows are syntactically valid YAML", () => {
for (const workflow of [qualityWorkflow, deployWorkflow, migrationWorkflow]) {
for (const workflow of [qualityWorkflow, deployWorkflow, migrationWorkflow, rolloutWorkflow]) {
const result = spawnSync(
"ruby",
["-e", "require 'yaml'; YAML.parse_file(ARGV.fetch(0))", fileURLToPath(workflow)],
@@ -151,16 +159,21 @@ test("all staging mutations share Actions serialization and one host lock", () =
const deployRunner = read(deployScript);
const migrationRunner = read(migrationScript);
for (const workflow of [deployment, migration]) {
const rollout = read(rolloutWorkflow);
const rolloutRunner = read(rolloutScript);
for (const workflow of [deployment, migration, rollout]) {
assert.match(workflow, /concurrency:\n\s+group: staging-mutation\n\s+cancel-in-progress: false/);
}
for (const runner of [deployRunner, migrationRunner]) {
for (const runner of [deployRunner, migrationRunner, rolloutRunner]) {
assert.match(runner, /state_directory="\$DEPLOY_PATH\/\.state"/);
assert.match(runner, /state_directory\/mutation\.lock/);
assert.match(runner, /flock -n 9/);
assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("sync-staging-tree.sh"));
assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("docker"));
}
for (const runner of [deployRunner, migrationRunner]) {
assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("sync-staging-tree.sh"));
}
});
test("deploy and migration consume the exact successful gate artifact", () => {
@@ -391,8 +404,98 @@ test("production remains manual-only and separate from staging database automati
assert.doesNotMatch(production, /docker-compose\.postgres\.yml|db:migrate/);
});
test("public rectification rollout rewrites only rollout gates and recreates web runtimes", () => {
const root = mkdtempSync(join(tmpdir(), "jyotisha-rollout-"));
const deploymentPath = join(root, "app");
const statePath = join(deploymentPath, ".state");
const deployPath = join(deploymentPath, "deploy");
const mockBin = join(root, "bin");
const sha = "8".repeat(40);
mkdirSync(statePath, { recursive: true });
mkdirSync(deployPath, { recursive: true });
mkdirSync(mockBin, { recursive: true });
writeFileSync(join(statePath, "deployed-revision"), sha);
writeFileSync(
join(deploymentPath, ".env.staging"),
[
"APP_ENV_FILE=../.env.staging",
"CADDYFILE_PATH=./Caddyfile.staging",
"SITE_ADDRESS=https://staging.jyotisha.chat",
"ADMIN_SITE_ADDRESS=https://admin.staging.jyotisha.chat",
"AUTH_PROVIDER=self-hosted",
"SELF_HOSTED_IDENTITY_ENABLED=true",
"AUTH_USER_ORIGIN=https://staging.jyotisha.chat",
"AUTH_ADMIN_ORIGIN=https://admin.staging.jyotisha.chat",
`IDENTITY_DATABASE_URL=postgresql://identity_runtime:${"i".repeat(40)}@postgres:5432/jyotisha`,
`APP_DATABASE_URL=postgresql://app_runtime:${"a".repeat(40)}@postgres:5432/jyotisha`,
`ADMIN_DATABASE_URL=postgresql://admin_runtime:${"d".repeat(40)}@postgres:5432/jyotisha`,
`BETTER_AUTH_USER_SECRET=${"u".repeat(32)}`,
`BETTER_AUTH_ADMIN_SECRET=${"v".repeat(32)}`,
"RESEND_API_KEY=re_test_key",
"RESEND_FROM_EMAIL=test@example.com",
"ADMIN_EMAILS=admin@example.com",
`JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=${"t".repeat(32)}`,
"KEEP_ME=unchanged",
"RECTIFICATION_V3_CREATE_ENABLED=false",
"RECTIFICATION_V3_MIGRATIONS_READY=false",
"RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA=old",
"RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS=00000000-0000-4000-8000-000000009001",
"",
].join("\n"),
{ mode: 0o600 },
);
writeFileSync(
join(deployPath, "validate-staging-env.sh"),
readFileSync(new URL("../../deploy/validate-staging-env.sh", import.meta.url), "utf8"),
);
writeFileSync(join(mockBin, "flock"), "#!/usr/bin/env bash\nexit 0\n");
writeFileSync(
join(mockBin, "docker"),
[
"#!/usr/bin/env bash",
'if [ "$1" = ps ]; then echo web-container; exit 0; fi',
`if [ "$1" = inspect ]; then echo ghcr.io/jesse-ux/jyotisha-web@sha256:${"b".repeat(64)}; exit 0; fi`,
`printf '%s\n' "$*" >>${join(root, "docker.log")}`,
].join("\n"),
);
writeFileSync(
join(mockBin, "curl"),
`#!/usr/bin/env bash\nprintf '%s' '{"deployment":{"gitCommit":"${sha}"},"rollout":{"conversationalRectificationV3":{"creationAudience":"public","readyForNewCases":true}}}'\n`,
);
for (const command of ["flock", "docker", "curl"]) {
chmodSync(join(mockBin, command), 0o755);
}
try {
const result = spawnSync("bash", [fileURLToPath(rolloutScript)], {
encoding: "utf8",
env: {
...process.env,
PATH: `${mockBin}:${process.env.PATH ?? ""}`,
DEPLOY_PATH: deploymentPath,
EXPECTED_DEPLOY_SHA: sha,
ROLLOUT_AUDIENCE: "public",
SYNTHETIC_SMOKE_USER_IDS: "",
STAGING_URL: "https://staging.jyotisha.chat",
},
});
assert.equal(result.status, 0, result.stderr);
const env = readFileSync(join(deploymentPath, ".env.staging"), "utf8");
assert.match(env, /^KEEP_ME=unchanged$/m);
assert.match(env, /^RECTIFICATION_V3_CREATE_ENABLED=true$/m);
assert.match(env, /^RECTIFICATION_V3_MIGRATIONS_READY=true$/m);
assert.match(env, new RegExp(`^RECTIFICATION_V3_SYNTHETIC_SMOKE_SHA=${sha}$`, "m"));
assert.match(env, /^RECTIFICATION_V3_SYNTHETIC_SMOKE_USER_IDS=$/m);
assert.equal((env.match(/^RECTIFICATION_V3_CREATE_ENABLED=/gm) ?? []).length, 1);
assert.match(readFileSync(join(root, "docker.log"), "utf8"), /force-recreate --no-deps web rectification-v4-worker/);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test("staging scripts pass shell syntax validation", () => {
for (const script of [deployScript, migrationScript, syncScript]) {
for (const script of [deployScript, migrationScript, rolloutScript, syncScript]) {
const path = fileURLToPath(script);
chmodSync(path, 0o755);
const result = spawnSync("bash", ["-n", path], { encoding: "utf8" });