diff --git a/.github/workflows/backend-quality-gate.yml b/.github/workflows/backend-quality-gate.yml index 9c36f2ff..ab9c71b2 100644 --- a/.github/workflows/backend-quality-gate.yml +++ b/.github/workflows/backend-quality-gate.yml @@ -2,6 +2,18 @@ name: Staging Backend Quality Gate on: pull_request: + paths: + - '.github/workflows/backend-quality-gate.yml' + - '.github/workflows/deploy-staging.yml' + - '.github/workflows/migrate-staging-database.yml' + - 'deploy/**' + - 'frontend/**' + - 'jyotish_vedic/**' + - 'scripts/**' + - 'tests/**' + - 'mcp_server.py' + - 'pyproject.toml' + - 'requirements*.txt' push: branches: [staging] workflow_dispatch: @@ -60,10 +72,7 @@ jobs: name: quick-quality-gate-diagnostics path: artifacts/quick-quality-gate.log - - name: Run database tests - run: npm run test:db --prefix frontend - - - name: Validate frontend + - name: Validate frontend and database contracts env: NEXT_PUBLIC_SUPABASE_URL: https://placeholder.supabase.co NEXT_PUBLIC_SUPABASE_ANON_KEY: placeholder @@ -109,6 +118,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and publish API image + id: api_build uses: docker/build-push-action@v6 with: context: . @@ -117,6 +127,7 @@ jobs: tags: ghcr.io/jesse-ux/jyotisha-api:${{ github.sha }} - name: Build and publish web image + id: web_build uses: docker/build-push-action@v6 with: context: . @@ -126,3 +137,28 @@ jobs: build-args: | NEXT_PUBLIC_SUPABASE_URL=${{ vars.STAGING_SUPABASE_URL }} NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ vars.STAGING_SUPABASE_ANON_KEY }} + + - name: Record immutable staging image manifest + env: + API_DIGEST: ${{ steps.api_build.outputs.digest }} + WEB_DIGEST: ${{ steps.web_build.outputs.digest }} + run: | + set -euo pipefail + [[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$API_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "$WEB_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + install -d -m 700 artifacts/staging-images + umask 077 + printf 'git_sha=%s\napi_digest=%s\nweb_digest=%s\n' \ + "$GITHUB_SHA" "$API_DIGEST" "$WEB_DIGEST" \ + > artifacts/staging-images/manifest.env + node frontend/scripts/staging-image-manifest.mjs \ + artifacts/staging-images/manifest.env "$GITHUB_SHA" >/dev/null + + - name: Upload immutable staging image manifest + uses: actions/upload-artifact@v4 + with: + name: staging-image-manifest-${{ github.sha }}-${{ github.run_attempt }} + path: artifacts/staging-images/manifest.env + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index bc884a5d..9f0a2163 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -7,9 +7,14 @@ on: workflow_dispatch: inputs: deploy_sha: - description: Exact 40-character commit SHA from a successful backend quality gate + description: Exact tested 40-character staging commit SHA required: true type: string + allow_rollback: + description: Explicitly permit a manual rollback to an older tested SHA + required: true + default: false + type: boolean permissions: contents: read @@ -17,7 +22,7 @@ permissions: packages: read concurrency: - group: staging + group: staging-mutation cancel-in-progress: false jobs: @@ -37,44 +42,103 @@ jobs: STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }} steps: - - name: Validate tested revision + - name: Validate tested revision and gate run id: revision env: REQUESTED_SHA: ${{ github.event.workflow_run.head_sha || inputs.deploy_sha }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + WORKFLOW_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + REQUESTED_ROLLBACK: ${{ inputs.allow_rollback || 'false' }} GH_TOKEN: ${{ github.token }} + shell: bash run: | - test "${#REQUESTED_SHA}" -eq 40 - case "$REQUESTED_SHA" in - *[!0-9a-fA-F]*) echo "deploy_sha must be a full hexadecimal commit SHA" >&2; exit 1 ;; - esac - DEPLOY_GIT_SHA="$(printf '%s' "$REQUESTED_SHA" | tr '[:upper:]' '[:lower:]')" + set -euo pipefail + [[ "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "deploy_sha must be a lowercase full commit SHA" >&2 + exit 1 + } + allow_rollback=false + if [ "$REQUESTED_ROLLBACK" = "true" ]; then + [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] || { + echo "rollback authorization is manual-only" >&2 + exit 1 + } + allow_rollback=true + fi + + gate_run_id="$WORKFLOW_RUN_ID" + gate_run_attempt="$WORKFLOW_RUN_ATTEMPT" if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - TESTED_RUNS="$(curl --fail --silent --show-error \ + runs="$(curl --fail --silent --show-error \ --header "Authorization: Bearer $GH_TOKEN" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ - "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/backend-quality-gate.yml/runs?head_sha=$DEPLOY_GIT_SHA&branch=staging&event=push&status=success&per_page=100")" - MATCHING_RUNS="$(printf '%s' "$TESTED_RUNS" | jq --arg sha "$DEPLOY_GIT_SHA" \ - '[.workflow_runs[] | select(.head_sha == $sha and .head_branch == "staging" and .event == "push" and .conclusion == "success")] | length')" - test "$MATCHING_RUNS" -ge 1 || { - echo "No successful Staging Backend Quality Gate run found for exact SHA $DEPLOY_GIT_SHA" >&2 - exit 1 - } + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/backend-quality-gate.yml/runs?head_sha=$REQUESTED_SHA&branch=staging&event=push&status=success&per_page=100")" + selected_run="$(jq -cer --arg sha "$REQUESTED_SHA" ' + [.workflow_runs[] | select( + .head_sha == $sha and .head_branch == "staging" and + .event == "push" and .conclusion == "success" + )] | sort_by(.run_attempt) | reverse | first + ' <<<"$runs")" + gate_run_id="$(jq -er '.id' <<<"$selected_run")" + gate_run_attempt="$(jq -er '.run_attempt' <<<"$selected_run")" fi - echo "sha=$DEPLOY_GIT_SHA" >> "$GITHUB_OUTPUT" + [[ "$gate_run_id" =~ ^[0-9]+$ ]] || { + echo "no successful exact-SHA staging quality gate run found" >&2 + exit 1 + } + [[ "$gate_run_attempt" =~ ^[1-9][0-9]*$ ]] || { + echo "invalid staging quality gate run attempt" >&2 + exit 1 + } + + staging_head="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/git/ref/heads/staging" | + jq -er '.object.sha')" + if [ "$allow_rollback" = "false" ] && [ "$REQUESTED_SHA" != "$staging_head" ]; then + echo "stale staging revision refused; use explicit manual rollback only when intended" >&2 + exit 1 + fi + + { + echo "sha=$REQUESTED_SHA" + echo "gate_run_id=$gate_run_id" + echo "gate_run_attempt=$gate_run_attempt" + echo "allow_rollback=$allow_rollback" + } >>"$GITHUB_OUTPUT" - name: Checkout tested revision uses: actions/checkout@v4 with: ref: ${{ steps.revision.outputs.sha }} + persist-credentials: false - - name: Verify checked-out revision + - name: Download gate-produced image manifest + uses: actions/download-artifact@v4 + with: + name: staging-image-manifest-${{ steps.revision.outputs.sha }}-${{ steps.revision.outputs.gate_run_attempt }} + path: artifacts/staging-image + github-token: ${{ github.token }} + run-id: ${{ steps.revision.outputs.gate_run_id }} + + - name: Validate immutable image manifest + id: images env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: test "$(git rev-parse HEAD)" = "$DEPLOY_GIT_SHA" - - - name: Validate staging target configuration + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} run: | + set -euo pipefail + node frontend/scripts/staging-image-manifest.mjs \ + artifacts/staging-image/manifest.env "$DEPLOY_SHA" >>"$GITHUB_OUTPUT" + + - name: Verify revision and staging target + env: + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$DEPLOY_SHA" test "$DEPLOY_HOST" = "118.26.111.127" test "$DEPLOY_PORT" = "22" test "$DEPLOY_USER" = "deploy" @@ -86,156 +150,99 @@ jobs: env: SSH_PRIVATE_KEY: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} run: | + set -euo pipefail test -n "$SSH_PRIVATE_KEY" install -m 700 -d ~/.ssh - printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/jyotisha-staging + printf '%s\n' "$SSH_PRIVATE_KEY" >~/.ssh/jyotisha-staging chmod 600 ~/.ssh/jyotisha-staging - printf '%s\n' "$STAGING_KNOWN_HOSTS" > ~/.ssh/known_hosts + printf '%s\n' "$STAGING_KNOWN_HOSTS" >~/.ssh/known_hosts chmod 600 ~/.ssh/known_hosts - - name: Record previous staging images + - name: Verify forward-only deployed revision id: previous env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + ALLOW_ROLLBACK: ${{ steps.revision.outputs.allow_rollback }} + GH_TOKEN: ${{ github.token }} + shell: bash run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - PREVIOUS_HEALTH_SHA="$(curl --fail --silent --show-error --max-time 10 "$STAGING_URL/api/health" 2>/dev/null | jq -r '.deployment.gitCommit // empty' || true)" - PREVIOUS_API_IMAGE="$(ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "container_id=\$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=api' | head -n 1); if [ -n \"\$container_id\" ]; then docker inspect --format '{{.Config.Image}}' \"\$container_id\"; fi")" - PREVIOUS_WEB_IMAGE="$(ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "container_id=\$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$container_id\" ]; then docker inspect --format '{{.Config.Image}}' \"\$container_id\"; fi")" - PREVIOUS_SHA="$(ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "container_id=\$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$container_id\" ]; then docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$container_id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; fi")" - if [ -z "$PREVIOUS_SHA" ] && [[ "$PREVIOUS_HEALTH_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then - PREVIOUS_SHA="$PREVIOUS_HEALTH_SHA" + set -euo pipefail + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" + previous_sha="$(ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then value=\$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1); printf '%s' \"\${value:-not-deployed}\"; else printf not-deployed; fi; fi")" + if [ "$previous_sha" != "not-deployed" ] && [[ ! "$previous_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid deployed staging revision state" >&2 + exit 1 fi - if [ -z "$PREVIOUS_SHA" ] && [[ "$PREVIOUS_WEB_IMAGE" =~ ^ghcr\.io/jesse-ux/jyotisha-web:([0-9a-f]{40})$ ]]; then - PREVIOUS_SHA="${BASH_REMATCH[1]}" + forward_verified=true + if [ "$ALLOW_ROLLBACK" = "false" ] && + [ "$previous_sha" != "not-deployed" ] && + [ "$previous_sha" != "$DEPLOY_SHA" ]; then + comparison="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")" + jq -e --arg base "$previous_sha" ' + .status == "ahead" and .merge_base_commit.sha == $base + ' <<<"$comparison" >/dev/null || { + echo "automatic staging rollback or divergent deploy refused" >&2 + exit 1 + } fi - if [ -n "$PREVIOUS_SHA" ]; then - test "${#PREVIOUS_SHA}" -eq 40 - case "$PREVIOUS_SHA" in - *[!0-9a-fA-F]*) echo "Previous staging SHA is unsafe" >&2; exit 1 ;; - esac - PREVIOUS_SHA="$(printf '%s' "$PREVIOUS_SHA" | tr '[:upper:]' '[:lower:]')" - fi - for image in "$PREVIOUS_API_IMAGE" "$PREVIOUS_WEB_IMAGE"; do - case "$image" in - "") ;; - *[!A-Za-z0-9._/@:-]*) echo "Previous staging image reference is unsafe" >&2; exit 1 ;; - esac - done { - echo "api_image=$PREVIOUS_API_IMAGE" - echo "web_image=$PREVIOUS_WEB_IMAGE" - echo "previous_sha=$PREVIOUS_SHA" - } >> "$GITHUB_OUTPUT" - { - echo "### Staging deployment state" - echo "- Previous verified SHA: \`${PREVIOUS_SHA:-not-deployed}\`" - echo "- Target SHA: \`$DEPLOY_GIT_SHA\`" - echo "- Previous API image: \`${PREVIOUS_API_IMAGE:-not-deployed}\`" - echo "- Previous web image: \`${PREVIOUS_WEB_IMAGE:-not-deployed}\`" - } >> "$GITHUB_STEP_SUMMARY" + echo "sha=$previous_sha" + echo "forward_verified=$forward_verified" + } >>"$GITHUB_OUTPUT" - - name: Sync tested staging sources + - name: Stage tested sources in an isolated incoming directory + id: incoming run: | + set -euo pipefail SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" RSYNC_SSH="ssh $SSH_OPTIONS" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 755 '$DEPLOY_PATH'" + incoming="$DEPLOY_PATH/.incoming/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$incoming'" rsync -az --delete \ - --exclude='.git/' \ - --exclude='.env*' \ - --exclude='frontend/node_modules/' \ - --exclude='frontend/.next/' \ - -e "$RSYNC_SSH" \ - ./ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + --exclude='/.git/' --exclude='/.env*' --exclude='/backups/' \ + --exclude='/frontend/node_modules/' --exclude='/frontend/.next/' \ + -e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/" + echo "path=$incoming" >>"$GITHUB_OUTPUT" - - name: Validate staging configuration - env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && bash deploy/validate-staging-env.sh .env.staging && bash deploy/validate-staging-database-env.sh .env.staging.database && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='ghcr.io/jesse-ux/jyotisha-api:$DEPLOY_GIT_SHA' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_GIT_SHA' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml config --quiet" - - - name: Log in to GHCR + - name: Log in to GHCR with run-local Docker state env: GHCR_TOKEN: ${{ github.token }} + INCOMING_PATH: ${{ steps.incoming.outputs.path }} run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - printf '%s' "$GHCR_TOKEN" | ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "docker login ghcr.io --username '$GITHUB_ACTOR' --password-stdin" - - - name: Pull exact staging images - env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='ghcr.io/jesse-ux/jyotisha-api:$DEPLOY_GIT_SHA' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_GIT_SHA' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml pull api web postgres" - - - name: Start and wait for staging PostgreSQL - env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='ghcr.io/jesse-ux/jyotisha-api:$DEPLOY_GIT_SHA' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_GIT_SHA' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml up -d --no-build --wait postgres" - - - name: Check staging migrations - id: migration_check - env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - set +e - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='ghcr.io/jesse-ux/jyotisha-api:$DEPLOY_GIT_SHA' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_GIT_SHA' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml --profile migration-check run --rm migration-checker" - CHECK_STATUS=$? - set -e - if [ "$CHECK_STATUS" -eq 3 ]; then - echo "Run the Migrate Staging Database workflow manually with exact SHA $DEPLOY_GIT_SHA; no API, web, or Caddy container was changed." >&2 - exit 3 - fi - if [ "$CHECK_STATUS" -ne 0 ]; then - echo "Staging migration check failed safely for exact SHA $DEPLOY_GIT_SHA" >&2 - exit "$CHECK_STATUS" - fi - - - name: Deploy exact staging images - env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='ghcr.io/jesse-ux/jyotisha-api:$DEPLOY_GIT_SHA' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_GIT_SHA' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml up -d --no-build --remove-orphans" - - - name: Verify staging - env: - DEPLOY_GIT_SHA: ${{ steps.revision.outputs.sha }} - run: | - curl --fail --silent --show-error --retry 12 --retry-delay 5 "$STAGING_URL/login" >/dev/null - test "$(curl --silent --output /dev/null --write-out '%{http_code}' "$STAGING_URL/api/account")" = "401" - test "$(curl --fail --silent --show-error "$STAGING_URL/api/health" | jq -r '.deployment.gitCommit')" = "$DEPLOY_GIT_SHA" + set -euo pipefail SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='ghcr.io/jesse-ux/jyotisha-api:$DEPLOY_GIT_SHA' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_GIT_SHA' GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml exec -T web node -e 'fetch(\"http://api:5200/api/health\").then(async r => { const body = await r.json(); if (!r.ok || body.status !== \"ok\" || body.swisseph_available !== true) process.exit(1); console.log(JSON.stringify(body)); })'" - echo "- Verified deployed SHA: \`$DEPLOY_GIT_SHA\`" >> "$GITHUB_STEP_SUMMARY" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$INCOMING_PATH/.docker'" + printf '%s' "$GHCR_TOKEN" | ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "DOCKER_CONFIG='$INCOMING_PATH/.docker' docker login ghcr.io --username '$GITHUB_ACTOR' --password-stdin" - - name: Roll back staging images - if: failure() && steps.migration_check.outcome == 'success' && steps.previous.outputs.api_image != '' && steps.previous.outputs.web_image != '' && steps.previous.outputs.previous_sha != '' + - name: Deploy and verify exact image digests under host lock env: - PREVIOUS_API_IMAGE: ${{ steps.previous.outputs.api_image }} - PREVIOUS_WEB_IMAGE: ${{ steps.previous.outputs.web_image }} - PREVIOUS_SHA: ${{ steps.previous.outputs.previous_sha }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + API_IMAGE: ${{ steps.images.outputs.api_image }} + WEB_IMAGE: ${{ steps.images.outputs.web_image }} + ALLOW_ROLLBACK: ${{ steps.revision.outputs.allow_rollback }} + EXPECTED_PREVIOUS_SHA: ${{ steps.previous.outputs.sha }} + FORWARD_REVISION_VERIFIED: ${{ steps.previous.outputs.forward_verified }} + INCOMING_PATH: ${{ steps.incoming.outputs.path }} run: | + set -euo pipefail SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && APP_ENV_FILE='../.env.staging' DATABASE_ENV_FILE='../.env.staging.database' CADDYFILE_PATH='./Caddyfile.staging' SITE_ADDRESS='https://staging.jyotisha.chat' API_IMAGE='$PREVIOUS_API_IMAGE' WEB_IMAGE='$PREVIOUS_WEB_IMAGE' GITHUB_SHA='$PREVIOUS_SHA' docker compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml up -d --no-build --remove-orphans api web caddy" + "INCOMING_PATH='$INCOMING_PATH' DEPLOY_PATH='$DEPLOY_PATH' API_IMAGE='$API_IMAGE' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$EXPECTED_PREVIOUS_SHA' ALLOW_ROLLBACK='$ALLOW_ROLLBACK' FORWARD_REVISION_VERIFIED='$FORWARD_REVISION_VERIFIED' DOCKER_CONFIG='$INCOMING_PATH/.docker' STAGING_URL='$STAGING_URL' bash '$INCOMING_PATH/deploy/run-staging-deploy.sh'" | + tee staging-deploy-result.txt + sed 's/^/- /' staging-deploy-result.txt >>"$GITHUB_STEP_SUMMARY" - - name: Log out of GHCR - if: always() + - name: Remove run-local staging files + if: always() && steps.incoming.outputs.path != '' continue-on-error: true + env: + INCOMING_PATH: ${{ steps.incoming.outputs.path }} run: | SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "docker logout ghcr.io >/dev/null 2>&1 || true" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "DOCKER_CONFIG='$INCOMING_PATH/.docker' docker logout ghcr.io >/dev/null 2>&1 || true; rm -rf -- '$INCOMING_PATH'" diff --git a/.github/workflows/migrate-staging-database.yml b/.github/workflows/migrate-staging-database.yml index 33b8dec6..1a512802 100644 --- a/.github/workflows/migrate-staging-database.yml +++ b/.github/workflows/migrate-staging-database.yml @@ -4,12 +4,12 @@ on: workflow_dispatch: inputs: deploy_sha: - description: Full tested commit SHA to migrate + description: Full tested staging commit SHA to migrate required: true type: string concurrency: - group: staging-database-migration + group: staging-mutation cancel-in-progress: false permissions: @@ -30,7 +30,7 @@ jobs: STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }} steps: - - name: Validate tested revision + - name: Validate current tested staging revision id: revision env: REQUESTED_SHA: ${{ inputs.deploy_sha }} @@ -38,45 +38,70 @@ jobs: shell: bash run: | set -euo pipefail - if [[ ! "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "deploy_sha must be a lowercase full 40-character commit SHA" >&2 + [[ "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "deploy_sha must be a lowercase full commit SHA" >&2 exit 1 - fi - TESTED_RUNS="$(curl --fail --silent --show-error \ + } + runs="$(curl --fail --silent --show-error \ --header "Authorization: Bearer $GH_TOKEN" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/backend-quality-gate.yml/runs?head_sha=$REQUESTED_SHA&branch=staging&event=push&status=success&per_page=100")" - if ! jq -e --arg sha "$REQUESTED_SHA" ' - (.workflow_runs | type == "array") and - any(.workflow_runs[]; - .head_sha == $sha and - .head_branch == "staging" and - .event == "push" and - .conclusion == "success" - ) - ' <<< "$TESTED_RUNS" >/dev/null; then - echo "No successful Staging Backend Quality Gate run found for exact SHA $REQUESTED_SHA on staging" >&2 + selected_run="$(jq -cer --arg sha "$REQUESTED_SHA" ' + [.workflow_runs[] | select( + .head_sha == $sha and .head_branch == "staging" and + .event == "push" and .conclusion == "success" + )] | sort_by(.run_attempt) | reverse | first + ' <<<"$runs")" + gate_run_id="$(jq -er '.id' <<<"$selected_run")" + gate_run_attempt="$(jq -er '.run_attempt' <<<"$selected_run")" + [[ "$gate_run_id" =~ ^[0-9]+$ ]] + [[ "$gate_run_attempt" =~ ^[1-9][0-9]*$ ]] + staging_head="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/git/ref/heads/staging" | + jq -er '.object.sha')" + [ "$REQUESTED_SHA" = "$staging_head" ] || { + echo "stale staging migration refused; migrate the current staging head" >&2 exit 1 - fi - printf 'deploy_sha=%s\n' "$REQUESTED_SHA" >> "$GITHUB_OUTPUT" + } + { + echo "sha=$REQUESTED_SHA" + echo "gate_run_id=$gate_run_id" + echo "gate_run_attempt=$gate_run_attempt" + } >>"$GITHUB_OUTPUT" - name: Checkout tested revision uses: actions/checkout@v4 with: - ref: ${{ steps.revision.outputs.deploy_sha }} + ref: ${{ steps.revision.outputs.sha }} persist-credentials: false - - name: Verify checked-out revision + - name: Download gate-produced image manifest + uses: actions/download-artifact@v4 + with: + name: staging-image-manifest-${{ steps.revision.outputs.sha }}-${{ steps.revision.outputs.gate_run_attempt }} + path: artifacts/staging-image + github-token: ${{ github.token }} + run-id: ${{ steps.revision.outputs.gate_run_id }} + + - name: Validate immutable migration image + id: images env: - DEPLOY_SHA: ${{ steps.revision.outputs.deploy_sha }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + run: | + set -euo pipefail + node frontend/scripts/staging-image-manifest.mjs \ + artifacts/staging-image/manifest.env "$DEPLOY_SHA" >>"$GITHUB_OUTPUT" + + - name: Verify revision and staging target + env: + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$DEPLOY_SHA" - - - name: Validate staging target configuration - run: | - set -euo pipefail test "$DEPLOY_HOST" = "118.26.111.127" test "$DEPLOY_PORT" = "22" test "$DEPLOY_USER" = "deploy" @@ -90,100 +115,114 @@ jobs: set -euo pipefail test -n "$SSH_PRIVATE_KEY" install -m 700 -d ~/.ssh - printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/jyotisha-staging + printf '%s\n' "$SSH_PRIVATE_KEY" >~/.ssh/jyotisha-staging chmod 600 ~/.ssh/jyotisha-staging - printf '%s\n' "$STAGING_KNOWN_HOSTS" > ~/.ssh/known_hosts + printf '%s\n' "$STAGING_KNOWN_HOSTS" >~/.ssh/known_hosts chmod 600 ~/.ssh/known_hosts - - name: Sync tested staging sources + - name: Verify forward-only migration revision + id: previous + env: + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" + previous_sha="$(ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then value=\$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1); printf '%s' \"\${value:-not-deployed}\"; else printf not-deployed; fi; fi")" + if [ "$previous_sha" != "not-deployed" ] && [[ ! "$previous_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid deployed staging revision state" >&2 + exit 1 + fi + if [ "$previous_sha" != "not-deployed" ] && [ "$previous_sha" != "$DEPLOY_SHA" ]; then + comparison="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")" + jq -e --arg base "$previous_sha" ' + .status == "ahead" and .merge_base_commit.sha == $base + ' <<<"$comparison" >/dev/null || { + echo "stale or divergent staging migration refused" >&2 + exit 1 + } + fi + { + echo "sha=$previous_sha" + echo "forward_verified=true" + } >>"$GITHUB_OUTPUT" + + - name: Stage tested sources in an isolated incoming directory + id: incoming run: | set -euo pipefail SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" RSYNC_SSH="ssh $SSH_OPTIONS" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 755 '$DEPLOY_PATH'" + incoming="$DEPLOY_PATH/.incoming/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$incoming'" rsync -az --delete \ - --exclude='.git/' \ - --exclude='.env*' \ - --exclude='frontend/node_modules/' \ - --exclude='frontend/.next/' \ - -e "$RSYNC_SSH" \ - ./ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" + --exclude='/.git/' --exclude='/.env*' --exclude='/backups/' \ + --exclude='/frontend/node_modules/' --exclude='/frontend/.next/' \ + -e "$RSYNC_SSH" ./ "$DEPLOY_USER@$DEPLOY_HOST:$incoming/" + echo "path=$incoming" >>"$GITHUB_OUTPUT" - - name: Validate staging environment files - run: | - set -euo pipefail - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && bash deploy/validate-staging-env.sh .env.staging staging.jyotisha.chat deploy/Caddyfile.staging && bash deploy/validate-staging-database-env.sh .env.staging.database" - - - name: Log in to GHCR + - name: Log in to GHCR with run-local Docker state env: GHCR_TOKEN: ${{ github.token }} + INCOMING_PATH: ${{ steps.incoming.outputs.path }} run: | set -euo pipefail - test -n "$GHCR_TOKEN" - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - printf '%s' "$GHCR_TOKEN" | ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "docker login ghcr.io --username '$GITHUB_ACTOR' --password-stdin" + SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "install -d -m 700 '$INCOMING_PATH/.docker'" + printf '%s' "$GHCR_TOKEN" | ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "DOCKER_CONFIG='$INCOMING_PATH/.docker' docker login ghcr.io --username '$GITHUB_ACTOR' --password-stdin" - - name: Pull exact migration image + - name: Apply exact-image migrations under host lock env: - DEPLOY_SHA: ${{ steps.revision.outputs.deploy_sha }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + WEB_IMAGE: ${{ steps.images.outputs.web_image }} + EXPECTED_PREVIOUS_SHA: ${{ steps.previous.outputs.sha }} + FORWARD_REVISION_VERIFIED: ${{ steps.previous.outputs.forward_verified }} + INCOMING_PATH: ${{ steps.incoming.outputs.path }} run: | set -euo pipefail SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "docker pull 'ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_SHA'" + "INCOMING_PATH='$INCOMING_PATH' DEPLOY_PATH='$DEPLOY_PATH' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$EXPECTED_PREVIOUS_SHA' FORWARD_REVISION_VERIFIED='$FORWARD_REVISION_VERIFIED' DOCKER_CONFIG='$INCOMING_PATH/.docker' bash '$INCOMING_PATH/deploy/run-staging-migration.sh'" - - name: Start and wait for staging PostgreSQL - run: | - set -euo pipefail - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && DATABASE_ENV_FILE='../.env.staging.database' docker compose -p jyotisha-staging -f deploy/docker-compose.postgres.yml up -d --wait postgres" - - - name: Apply reviewed staging migrations + - name: Dispatch current exact-SHA staging deployment env: - DEPLOY_SHA: ${{ steps.revision.outputs.deploy_sha }} - run: | - set -euo pipefail - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && DATABASE_ENV_FILE='../.env.staging.database' WEB_IMAGE='ghcr.io/jesse-ux/jyotisha-web:$DEPLOY_SHA' docker compose -p jyotisha-staging -f deploy/docker-compose.postgres.yml --profile migration run --rm migrator" - - - name: Print ordered migration ledger - run: | - set -euo pipefail - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && DATABASE_ENV_FILE='../.env.staging.database' docker compose -p jyotisha-staging -f deploy/docker-compose.postgres.yml exec -T postgres psql -U postgres -d jyotisha -Atc 'select filename from migration.schema_migrations order by filename'" - - - name: Dispatch exact-SHA staging deployment - env: - DEPLOY_SHA: ${{ steps.revision.outputs.deploy_sha }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} GH_TOKEN: ${{ github.token }} shell: bash run: | set -euo pipefail - if [[ ! "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "validated deploy SHA is unsafe" >&2 + staging_head="$(curl --fail --silent --show-error \ + --header "Authorization: Bearer $GH_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/git/ref/heads/staging" | + jq -er '.object.sha')" + [ "$DEPLOY_SHA" = "$staging_head" ] || { + echo "staging advanced during migration; refusing stale deployment dispatch" >&2 exit 1 - fi - PAYLOAD_FILE="$(mktemp)" - trap 'rm -f "$PAYLOAD_FILE"' EXIT - jq -n --arg deploy_sha "$DEPLOY_SHA" \ - '{ref: "staging", inputs: {deploy_sha: $deploy_sha}}' > "$PAYLOAD_FILE" - curl --fail --silent --show-error \ - --request POST \ + } + payload="$(jq -cn --arg deploy_sha "$DEPLOY_SHA" \ + '{ref:"main",inputs:{deploy_sha:$deploy_sha,allow_rollback:"false"}}')" + curl --fail --silent --show-error --request POST \ --header "Authorization: Bearer $GH_TOKEN" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ --header "Content-Type: application/json" \ - --data-binary "@$PAYLOAD_FILE" \ + --data "$payload" \ "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/workflows/deploy-staging.yml/dispatches" - - name: Log out of GHCR - if: always() + - name: Remove run-local staging files + if: always() && steps.incoming.outputs.path != '' continue-on-error: true + env: + INCOMING_PATH: ${{ steps.incoming.outputs.path }} run: | SSH_OPTIONS="-i $HOME/.ssh/jyotisha-staging -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "docker logout ghcr.io >/dev/null 2>&1 || true" + ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ + "DOCKER_CONFIG='$INCOMING_PATH/.docker' docker logout ghcr.io >/dev/null 2>&1 || true; rm -rf -- '$INCOMING_PATH'" diff --git a/deploy/README.md b/deploy/README.md index e837b168..5b72968a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -149,7 +149,7 @@ The GitHub `staging` Environment contains the secret `STAGING_SSH_PRIVATE_KEY` a The repository-level public build inputs are configured at GitHub **Settings -> Secrets and variables -> Actions -> Variables** (the UI is also shown as **Settings → Secrets and variables → Actions → Variables**): `STAGING_SUPABASE_URL` and `STAGING_SUPABASE_ANON_KEY`. They are public build inputs, required for publish, and exposed to the browser; keep them staging-only and never print their values in workflow output, summaries, or support messages. The workflow passes them only as the `NEXT_PUBLIC_*` build arguments after non-empty/HTTPS validation. -`Staging Backend Quality Gate` runs for `pull_request`, pushes to `staging`, and `workflow_dispatch`. It validates the Python/database/frontend contract; only a successful push to `staging` can publish immutable full-SHA GHCR images. `.github/workflows/deploy-staging.yml` consumes the successful gate's exact SHA, and its manual `deploy_sha` input must identify a full 40-character commit with a successful `staging` gate run. +`Staging Backend Quality Gate` runs for relevant `pull_request` paths, pushes to `staging`, and `workflow_dispatch`. It validates the Python/database/frontend contract; only a successful push to `staging` publishes the API/web images and a run-bound manifest containing their `sha256` digests. `.github/workflows/deploy-staging.yml` consumes that exact successful run, validates its manifest against the full 40-character commit, and deploys digest references rather than trusting the discoverability tags. The staging env file must include these non-secret selectors so Compose cannot fall back to production paths: @@ -165,13 +165,13 @@ After source sync and before `up`, the workflow validates `.env.staging` mode/se 1. Complete the server and GitHub bootstrap: create both mode-`0600` env files, configure the staging Environment variables/secrets, and configure the repository staging build variables. 2. Merge the reviewed change, then push the reviewed SHA to `staging`; do not rely on a `main` workflow dispatch to publish images. -3. The `Staging Backend Quality Gate` runs for that push and, when successful, publishes the SHA-tagged API/web images for that exact 40-character commit SHA. -4. The automatic `Deploy staging` workflow starts from that successful gate, syncs the exact SHA, and validates both `.env.staging` and `.env.staging.database` before any app change. +3. The `Staging Backend Quality Gate` runs for that push and, when successful, publishes API/web images plus an artifact binding the exact SHA to both immutable image digests. +4. The automatic `Deploy staging` workflow downloads that gate-run artifact, syncs the exact revision under the shared staging host lock, and validates both `.env.staging` and `.env.staging.database` before any app change. 5. If environment validation fails, fix the server-side env files without committing or copying secrets, then manually rerun `Deploy staging` from `main` with the same successful SHA in `deploy_sha`; the workflow rechecks a successful staging gate for that exact SHA. 6. If the read-only checker reports a pending migration, stop app deployment and run `Migrate Staging Database` manually with the same full SHA; a successful migration re-dispatches `Deploy staging` with that same SHA. 7. Confirm `https://staging.jyotisha.chat/api/health` reports the exact SHA and private API health. -Application rollback uses the same workflow: manually dispatch `Deploy staging` from `main` with a previous known-good full SHA that has a successful `Staging Backend Quality Gate` run. Database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. +Application rollback uses the same workflow: manually dispatch `Deploy staging` from the `main` controller with a previous known-good full SHA that has a successful `Staging Backend Quality Gate` run, and explicitly set `allow_rollback=true`. Normal and migration-triggered deployments reject stale, divergent, or backward revisions. Rollback still consumes the selected gate run's digest manifest; database migrations are separate and are not rolled back by an application deployment. Restore a staging database backup before running any destructive migration rehearsal. Inspect staging without printing secrets: @@ -229,14 +229,14 @@ PostgreSQL is private: `deploy/docker-compose.postgres.yml` has no `ports` mappi Use this order for every staging revision: 1. Merge to `staging` after reviewing the change. -2. Wait for `Staging Backend Quality Gate` to pass and for that exact full SHA's API/web images to be published. +2. Wait for `Staging Backend Quality Gate` to pass and publish that exact full SHA's API/web digest manifest. 3. The automatic `Deploy staging` workflow checks the exact SHA in read-only migration-check mode before changing API, web, or Caddy. If it reports pending or drifted migrations, stop; do not retry the application deployment as if it were a migration. 4. Open **Migrate Staging Database -> Run workflow** and enter the reported full lowercase 40-character SHA in `deploy_sha`. The workflow validates that exact SHA against a successful `staging` gate, checks it out, starts only PostgreSQL, and runs the reviewed migrator. -5. A successful migration prints the ordered migration ledger and re-dispatches `Deploy staging` automatically with the same exact SHA. Do not substitute a branch name, a short SHA, or a newer commit. +5. A successful migration rechecks that `staging` still points at the same exact SHA, prints the ordered migration ledger, and dispatches the `main` controller for digest-pinned deployment with `allow_rollback=false`. If `staging` advanced during migration, it refuses the stale dispatch. Do not substitute a branch name, a short SHA, or a newer commit. 6. Confirm `https://staging.jyotisha.chat/api/health` and verify that its deployment SHA is the SHA from step 2. 7. After health verification, create the local encrypted backup described below. -The read-only checker exits before app changes when a migration is pending. Its message includes the exact SHA and the `Migrate Staging Database` workflow name. A failed migration does not re-dispatch deployment. Application rollback restores a previously verified image/SHA only; it does not roll back database state. +The deploy and migration workflows share the `staging-mutation` Actions concurrency group, and their live-tree sync plus Compose work runs under `/opt/jyotisha-staging/.state/mutation.lock`. The synchronized tree explicitly preserves `/backups/`, `.env*`, `.state`, and `.incoming`. The read-only checker exits before app changes when a migration is pending. Its message includes the exact SHA and the `Migrate Staging Database` workflow name. A failed migration does not re-dispatch deployment. Application rollback restores previously recorded digest references and SHA only; it does not roll back database state. ### Local encrypted staging backups (three-copy limit) diff --git a/deploy/run-staging-deploy.sh b/deploy/run-staging-deploy.sh new file mode 100755 index 00000000..a9c074a0 --- /dev/null +++ b/deploy/run-staging-deploy.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +required=( + INCOMING_PATH DEPLOY_PATH API_IMAGE WEB_IMAGE DEPLOY_SHA + EXPECTED_PREVIOUS_SHA ALLOW_ROLLBACK DOCKER_CONFIG STAGING_URL +) +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "required staging deployment input is missing: $key" >&2 + exit 1 + fi +done + +sha_pattern='^[0-9a-f]{40}$' +digest_pattern='^ghcr\.io/jesse-ux/jyotisha-(api|web)@sha256:[0-9a-f]{64}$' +if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] || + [[ ! "$API_IMAGE" =~ $digest_pattern ]] || + [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then + echo "unsafe staging image identity" >&2 + exit 1 +fi +if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then + echo "invalid rollback authorization" >&2 + exit 1 +fi +case "$INCOMING_PATH" in + "$DEPLOY_PATH"/.incoming/*) ;; + *) echo "unsafe incoming staging path" >&2; exit 1 ;; +esac + +state_directory="$DEPLOY_PATH/.state" +install -d -m 700 "$state_directory" +exec 9>"$state_directory/mutation.lock" +flock -n 9 || { + echo "another staging mutation holds the host lock" >&2 + exit 75 +} + +current_sha="not-deployed" +if [ -f "$state_directory/deployed-revision" ]; then + current_sha="$(<"$state_directory/deployed-revision")" +else + existing_web="$(docker ps -aq \ + --filter 'label=com.docker.compose.project=jyotisha-staging' \ + --filter 'label=com.docker.compose.service=web' | head -n 1)" + if [ -n "$existing_web" ]; then + discovered_sha="$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ + "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)" + if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi + fi +fi +if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ $sha_pattern ]]; then + echo "invalid deployed staging revision state" >&2 + exit 1 +fi +if [ "$current_sha" != "$EXPECTED_PREVIOUS_SHA" ]; then + echo "staging revision changed while this deployment was waiting" >&2 + exit 1 +fi +if [ "$ALLOW_ROLLBACK" = "false" ] && + [ "$current_sha" != "not-deployed" ] && + [ "$current_sha" != "$DEPLOY_SHA" ] && + [ "${FORWARD_REVISION_VERIFIED:-false}" != "true" ]; then + echo "forward staging revision was not verified" >&2 + exit 1 +fi + +container_id() { + docker ps -aq \ + --filter 'label=com.docker.compose.project=jyotisha-staging' \ + --filter "label=com.docker.compose.service=$1" | head -n 1 +} + +repo_digest_for_container() { + local service="$1" + local repository="$2" + local id image_id + id="$(container_id "$service")" + [ -n "$id" ] || return 0 + image_id="$(docker inspect --format '{{.Image}}' "$id")" + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id" | + awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }' +} + +previous_api_image="$(repo_digest_for_container api ghcr.io/jesse-ux/jyotisha-api)" +previous_web_image="$(repo_digest_for_container web ghcr.io/jesse-ux/jyotisha-web)" +previous_api_id="" +previous_web_id="" +if [ -n "$(container_id api)" ]; then + previous_api_id="$(docker inspect --format '{{.Image}}' "$(container_id api)")" +fi +if [ -n "$(container_id web)" ]; then + previous_web_id="$(docker inspect --format '{{.Image}}' "$(container_id web)")" +fi + +bash "$INCOMING_PATH/deploy/sync-staging-tree.sh" \ + "$INCOMING_PATH" "$DEPLOY_PATH" + +cd "$DEPLOY_PATH" +bash deploy/validate-staging-env.sh \ + .env.staging staging.jyotisha.chat deploy/Caddyfile.staging +bash deploy/validate-staging-database-env.sh .env.staging.database + +compose=( + docker compose -p jyotisha-staging --env-file .env.staging + -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml +) +export APP_ENV_FILE='../.env.staging' +export DATABASE_ENV_FILE='../.env.staging.database' +export CADDYFILE_PATH='./Caddyfile.staging' +export SITE_ADDRESS='https://staging.jyotisha.chat' +export GITHUB_SHA="$DEPLOY_SHA" + +"${compose[@]}" config --quiet +"${compose[@]}" pull api web postgres +"${compose[@]}" up -d --no-build --wait postgres + +set +e +"${compose[@]}" --profile migration-check run --rm migration-checker +check_status=$? +set -e +if [ "$check_status" -eq 3 ]; then + echo "pending migrations: run Migrate Staging Database for $DEPLOY_SHA" >&2 + exit 3 +fi +if [ "$check_status" -ne 0 ]; then + echo "staging migration check failed safely" >&2 + exit "$check_status" +fi + +switched=false +rollback() { + local status=$? + if [ "$switched" = "true" ] && + [[ "$previous_api_image" =~ $digest_pattern ]] && + [[ "$previous_web_image" =~ $digest_pattern ]] && + [[ "$current_sha" =~ $sha_pattern ]]; then + echo "staging verification failed; restoring prior image digests" >&2 + API_IMAGE="$previous_api_image" WEB_IMAGE="$previous_web_image" \ + GITHUB_SHA="$current_sha" \ + "${compose[@]}" up -d --no-build --remove-orphans api web caddy || true + fi + exit "$status" +} +trap rollback ERR + +"${compose[@]}" up -d --no-build --remove-orphans +switched=true + +verify_container_image() { + local service="$1" + local expected_ref="$2" + local id expected_id running_id repo_digests + id="$(container_id "$service")" + [ -n "$id" ] + expected_id="$(docker image inspect --format '{{.Id}}' "$expected_ref")" + running_id="$(docker inspect --format '{{.Image}}' "$id")" + [ "$running_id" = "$expected_id" ] + repo_digests="$(docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$expected_id")" + grep -Fqx "$expected_ref" <<<"$repo_digests" +} +verify_container_image api "$API_IMAGE" +verify_container_image web "$WEB_IMAGE" + +"${compose[@]}" exec -T \ + -e EXPECTED_SHA="$DEPLOY_SHA" -e STAGING_URL="$STAGING_URL" \ + web node --input-type=module <<'NODE' +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +let login; +for (let attempt = 0; attempt < 12; attempt += 1) { + try { + login = await fetch(`${process.env.STAGING_URL}/login`); + if (login.ok) break; + } catch {} + await delay(5_000); +} +if (!login?.ok) process.exit(1); +const account = await fetch(`${process.env.STAGING_URL}/api/account`); +if (account.status !== 401) process.exit(1); +const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`); +const publicBody = await publicHealth.json(); +if (!publicHealth.ok || publicBody.deployment?.gitCommit !== process.env.EXPECTED_SHA) { + process.exit(1); +} +const privateHealth = await fetch("http://api:5200/api/health"); +const privateBody = await privateHealth.json(); +if (!privateHealth.ok || privateBody.status !== "ok" || privateBody.swisseph_available !== true) { + process.exit(1); +} +NODE + +revision_file="$state_directory/deployed-revision.tmp.$$" +printf '%s\n' "$DEPLOY_SHA" >"$revision_file" +chmod 600 "$revision_file" +mv -f "$revision_file" "$state_directory/deployed-revision" +trap - ERR + +printf 'previous_sha=%s\nprevious_api_image=%s\nprevious_api_id=%s\n' \ + "$current_sha" "${previous_api_image:-not-deployed}" "${previous_api_id:-not-deployed}" +printf 'previous_web_image=%s\nprevious_web_id=%s\nverified_sha=%s\n' \ + "${previous_web_image:-not-deployed}" "${previous_web_id:-not-deployed}" "$DEPLOY_SHA" diff --git a/deploy/run-staging-migration.sh b/deploy/run-staging-migration.sh new file mode 100755 index 00000000..9154af1b --- /dev/null +++ b/deploy/run-staging-migration.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +required=( + INCOMING_PATH DEPLOY_PATH WEB_IMAGE DEPLOY_SHA EXPECTED_PREVIOUS_SHA + DOCKER_CONFIG +) +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "required staging migration input is missing: $key" >&2 + exit 1 + fi +done + +[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "unsafe staging migration revision" >&2 + exit 1 +} +[[ "$WEB_IMAGE" =~ ^ghcr\.io/jesse-ux/jyotisha-web@sha256:[0-9a-f]{64}$ ]] || { + echo "unsafe staging migration image" >&2 + exit 1 +} +case "$INCOMING_PATH" in + "$DEPLOY_PATH"/.incoming/*) ;; + *) echo "unsafe incoming staging path" >&2; exit 1 ;; +esac + +state_directory="$DEPLOY_PATH/.state" +install -d -m 700 "$state_directory" +exec 9>"$state_directory/mutation.lock" +flock -n 9 || { + echo "another staging mutation holds the host lock" >&2 + exit 75 +} + +current_sha="not-deployed" +if [ -f "$state_directory/deployed-revision" ]; then + current_sha="$(<"$state_directory/deployed-revision")" +else + existing_web="$(docker ps -aq \ + --filter 'label=com.docker.compose.project=jyotisha-staging' \ + --filter 'label=com.docker.compose.service=web' | head -n 1)" + if [ -n "$existing_web" ]; then + discovered_sha="$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ + "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)" + if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi + fi +fi +if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid deployed staging revision state" >&2 + exit 1 +fi +[ "$current_sha" = "$EXPECTED_PREVIOUS_SHA" ] || { + echo "staging revision changed while this migration was waiting" >&2 + exit 1 +} +[ "$current_sha" = "not-deployed" ] || + [ "$current_sha" = "$DEPLOY_SHA" ] || + [ "${FORWARD_REVISION_VERIFIED:-false}" = "true" ] || { + echo "forward staging revision was not verified" >&2 + exit 1 + } + +bash "$INCOMING_PATH/deploy/sync-staging-tree.sh" \ + "$INCOMING_PATH" "$DEPLOY_PATH" + +cd "$DEPLOY_PATH" +bash deploy/validate-staging-env.sh \ + .env.staging staging.jyotisha.chat deploy/Caddyfile.staging +bash deploy/validate-staging-database-env.sh .env.staging.database + +export DATABASE_ENV_FILE='../.env.staging.database' +compose=(docker compose -p jyotisha-staging -f deploy/docker-compose.postgres.yml) +docker pull "$WEB_IMAGE" +"${compose[@]}" up -d --wait postgres +"${compose[@]}" --profile migration run --rm migrator +"${compose[@]}" exec -T postgres psql -U postgres -d jyotisha -Atc \ + 'select filename from migration.schema_migrations order by filename' diff --git a/deploy/sync-staging-tree.sh b/deploy/sync-staging-tree.sh new file mode 100755 index 00000000..bd5f899f --- /dev/null +++ b/deploy/sync-staging-tree.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ] || [ ! -d "$1" ] || [ ! -d "$2" ]; then + echo "usage: sync-staging-tree.sh SOURCE_DIRECTORY DESTINATION_DIRECTORY" >&2 + exit 1 +fi + +rsync -az --delete \ + --exclude='/.git/' \ + --exclude='/.env*' \ + --exclude='/backups/' \ + --exclude='/.state/' \ + --exclude='/.incoming/' \ + --exclude='/frontend/node_modules/' \ + --exclude='/frontend/.next/' \ + "$1/" "$2/" diff --git a/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md b/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md index 9da03f40..2d8b89ab 100644 --- a/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md +++ b/docs/superpowers/plans/2026-07-20-postgres-quality-gate-foundation.md @@ -4,7 +4,7 @@ **Goal:** Add the self-hosted PostgreSQL staging foundation, least-privilege roles, reviewed SQL migration runner, automatic backend quality gate, immutable GHCR images, and exact-SHA staging deploy path without moving authentication or business traffic off Supabase. -**Architecture:** PostgreSQL 17 runs on the private Compose network of the Hong Kong staging VPS. A separate deployment-user-owned database env file supplies bootstrap and schema credentials only to PostgreSQL and an opt-in migrator; normal web/API containers never receive them. Reviewed plain SQL is the schema source of truth, while `pg` and Drizzle provide the future runtime seam. PRs and `staging` pushes run database/backend/frontend/configuration tests; successful `staging` pushes publish SHA-tagged web/API images, and staging pulls that exact SHA. Migrations remain a separate manual workflow. +**Architecture:** PostgreSQL 17 runs on the private Compose network of the Hong Kong staging VPS. A separate deployment-user-owned database env file supplies bootstrap and schema credentials only to PostgreSQL and an opt-in migrator; normal web/API containers never receive them. Reviewed plain SQL is the schema source of truth, while `pg` and Drizzle provide the future runtime seam. PRs and `staging` pushes run database/backend/frontend/configuration tests; successful `staging` pushes publish web/API images and a run-bound SHA-to-digest manifest, and staging deploys only those exact digests. Migrations remain a separate manual workflow. **Tech Stack:** PostgreSQL 17 Alpine, Docker Compose, Node.js 22, Next.js 16, TypeScript, `pg`, Drizzle ORM, Python 3.12, GitHub Actions, GHCR, Bash, OpenSSL. @@ -20,7 +20,9 @@ - App deployment never runs schema migration. Migration is manual and separately serialized. - Before changing app containers, staging deploy runs the exact SHA image in read-only migration-check mode. No pending migration means automatic continuation. Pending or checksum-drifted migration stops before app changes; a successful manual migration dispatches staging deploy again for the same full SHA. - Production defaults remain manual-only and unchanged. -- Staging image tags are immutable full Git SHAs; never deploy `latest`. +- Staging publication uses full Git SHA tags for discovery, but deployment is + authorized and pinned by the build outputs' `sha256` manifest digests. Never + deploy a mutable tag such as `latest`, or treat a tag alone as image identity. - Finish each task with the focused commit shown. ## Planned Files @@ -742,7 +744,7 @@ Assert: - It never invokes the applying `db:migrate` command, `migrator` service, or `--profile migration`. - It runs the exact web image through `migration-checker`/`db:migrate:check` before changing any app container. - Pending migrations stop before `api`, `web`, or `caddy` changes and print the manual workflow name plus exact SHA. -- Rollback uses recorded prior image names. +- Rollback uses recorded prior digest references, image IDs, and SHA. - [ ] **Step 2: Confirm red** @@ -776,15 +778,17 @@ APP_ENV_FILE=../.env.staging DATABASE_ENV_FILE=../.env.staging.database CADDYFILE_PATH=./Caddyfile.staging SITE_ADDRESS=staging.jyotisha.chat -API_IMAGE=ghcr.io/jesse-ux/jyotisha-api: -WEB_IMAGE=ghcr.io/jesse-ux/jyotisha-web: +API_IMAGE=ghcr.io/jesse-ux/jyotisha-api@sha256: +WEB_IMAGE=ghcr.io/jesse-ux/jyotisha-web@sha256: ``` - Validate both env files and Compose config. - Send GHCR token through `docker login --password-stdin`; never save it in either env file. - Pull `api web postgres`, start/wait for PostgreSQL, then run the exact web image through `--profile migration-check run --rm migration-checker`. - Continue to `up -d --no-build --remove-orphans` only after check exit `0`; treat exit `3` as a safe stop with no application changes. -- Record prior container image names before switching. Roll back with those exact image names and `--no-build`. +- Download and validate the successful gate run's SHA-to-digest manifest. Record + prior container digest references, image IDs, and SHA before switching. Roll + back with those exact digest references and `--no-build`. - Log out in an always-running cleanup step. - Never run migrations. @@ -845,7 +849,7 @@ on: required: true type: string concurrency: - group: staging-database-migration + group: staging-mutation cancel-in-progress: false permissions: contents: read @@ -883,7 +887,13 @@ docker compose -p jyotisha-staging \ 'select filename from migration.schema_migrations order by filename' ``` -Authenticate GHCR through stdin and log out in cleanup. Do not start/restart app services. After migration and ledger reporting succeed, call the GitHub workflow-dispatch API for `deploy-staging.yml` with `ref: staging` and `inputs.deploy_sha` equal to the validated full SHA. +Authenticate GHCR through stdin using run-local Docker state and remove it in +cleanup. Do not start/restart app services. Deployment and migration share the +`staging-mutation` concurrency group and the host mutation lock. After migration +and ledger reporting succeed, recheck that `staging` still points at the validated +SHA, then call the GitHub workflow-dispatch API for `deploy-staging.yml` with the +`main` controller ref, `inputs.deploy_sha` equal to that full SHA, and +`inputs.allow_rollback` set to `false`. - [ ] **Step 4: Verify and commit** @@ -946,7 +956,7 @@ Each secret uses independently generated 32 random bytes. URL password is percen Document order: 1. Merge to `staging`. -2. Wait for backend quality gate and SHA images. +2. Wait for backend quality gate and its exact-SHA image digest manifest. 3. If automatic deploy reports pending migrations, manually run `Migrate Staging Database` with the reported full SHA. 4. The successful migration workflow re-dispatches exact-SHA staging deploy automatically. 5. Check `https://staging.jyotisha.chat/api/health`. diff --git a/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md b/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md index e6637407..2adbd688 100644 --- a/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md +++ b/docs/superpowers/specs/2026-07-20-supabase-exit-backend-design.md @@ -166,7 +166,7 @@ Sustained swap use, database saturation, disk above 70%, or unacceptable request ## Build and deployment -The VPS must not compile large application images while PostgreSQL is serving tests. GitHub Actions builds immutable web/API images, pushes SHA-addressed images to GitHub Container Registry, and the VPS only pulls and restarts them. +The VPS must not compile large application images while PostgreSQL is serving tests. GitHub Actions builds web/API images, publishes SHA tags only for discovery, and records the build outputs' immutable manifest digests in an artifact bound to the successful quality-gate run. The VPS validates that artifact and pulls digest references only. Application deployment and database migration remain different operations: @@ -182,7 +182,7 @@ touching the running application. After the operator runs the manual migration workflow successfully, that workflow dispatches staging deployment again for the same full SHA. The check may read the migration ledger but may never apply SQL. -Deployment records the previous application SHA and image digests. Public and private health checks must pass before a deployment is marked successful. Application rollback does not claim to roll back database state. +Deployment and migration share one Actions concurrency group and one host-side lock covering live-tree synchronization through their final database/application verification. Deployment records the previous application SHA, image digests, and image IDs. It rejects stale or backward automatic revisions, verifies running container image IDs/RepoDigests plus the application-reported SHA, and requires public and private health checks before updating deployed-revision state. An older application revision requires an explicit manual rollback authorization; application rollback does not claim to roll back database state. ## Automatic backend quality gate diff --git a/frontend/package.json b/frontend/package.json index 98400d7a..fd4f53ed 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,7 +9,7 @@ "start": "next start", "test": "tsx --test tests/*.test.ts", "test:db": "tsx --test --test-concurrency=1 tests/database-*.test.ts", - "test:deployment": "tsx --test tests/health-deployment.test.ts tests/staging-backend-workflows.test.ts", + "test:deployment": "tsx --test tests/health-deployment.test.ts tests/staging-backend-workflows.test.ts tests/staging-image-manifest.test.ts", "db:migrate": "node scripts/db-migrate.mjs", "db:migrate:check": "node scripts/db-migrate.mjs --check", "lint": "eslint", diff --git a/frontend/scripts/staging-image-manifest.mjs b/frontend/scripts/staging-image-manifest.mjs new file mode 100644 index 00000000..bb27f13d --- /dev/null +++ b/frontend/scripts/staging-image-manifest.mjs @@ -0,0 +1,74 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const shaPattern = /^[0-9a-f]{40}$/; +const digestPattern = /^sha256:[0-9a-f]{64}$/; +const expectedKeys = ["git_sha", "api_digest", "web_digest"]; + +export function parseStagingImageManifest(text, expectedSha) { + if (!shaPattern.test(expectedSha)) { + throw new Error("invalid expected staging revision"); + } + + const lines = text.endsWith("\n") ? text.slice(0, -1).split("\n") : text.split("\n"); + if (lines.length !== expectedKeys.length) { + throw new Error("invalid staging image manifest"); + } + + const values = new Map(); + for (const line of lines) { + const separator = line.indexOf("="); + if (separator <= 0) throw new Error("invalid staging image manifest"); + const key = line.slice(0, separator); + const value = line.slice(separator + 1); + if (!expectedKeys.includes(key) || values.has(key)) { + throw new Error("invalid staging image manifest"); + } + values.set(key, value); + } + + if (values.get("git_sha") !== expectedSha) { + throw new Error("staging image manifest revision mismatch"); + } + for (const key of ["api_digest", "web_digest"]) { + if (!digestPattern.test(values.get(key) ?? "")) { + throw new Error("invalid staging image digest"); + } + } + + return { + gitSha: expectedSha, + apiDigest: values.get("api_digest"), + webDigest: values.get("web_digest"), + apiImage: `ghcr.io/jesse-ux/jyotisha-api@${values.get("api_digest")}`, + webImage: `ghcr.io/jesse-ux/jyotisha-web@${values.get("web_digest")}`, + }; +} + +const invokedPath = process.argv[1] + ? pathToFileURL(resolve(process.argv[1])).href + : undefined; + +if (invokedPath === import.meta.url) { + try { + const [manifestPath, expectedSha] = process.argv.slice(2); + if (!manifestPath || !expectedSha) { + throw new Error("manifest path and expected revision are required"); + } + const manifest = parseStagingImageManifest( + await readFile(manifestPath, "utf8"), + expectedSha, + ); + process.stdout.write( + [ + `git_sha=${manifest.gitSha}`, + `api_image=${manifest.apiImage}`, + `web_image=${manifest.webImage}`, + ].join("\n") + "\n", + ); + } catch { + console.error("invalid staging image manifest"); + process.exitCode = 1; + } +} diff --git a/frontend/tests/health-deployment.test.ts b/frontend/tests/health-deployment.test.ts index 7f99772e..41010ce5 100644 --- a/frontend/tests/health-deployment.test.ts +++ b/frontend/tests/health-deployment.test.ts @@ -1,8 +1,6 @@ import assert from "node:assert/strict"; import { chmodSync, - existsSync, - mkdirSync, mkdtempSync, readFileSync, rmSync, @@ -145,8 +143,7 @@ test("staging deploy consumes only the isolated staging environment and tested r assert.match(workflow, /packages: read/); assert.match(workflow, /environment:\s*\n\s*name: staging/); assert.match(workflow, /deploy_sha:/); - assert.doesNotMatch(workflow, /default: staging/); - assert.match(workflow, /test "\$\{#REQUESTED_SHA\}" -eq 40/); + assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/); assert.match( workflow, /actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=/, @@ -157,80 +154,15 @@ test("staging deploy consumes only the isolated staging environment and tested r assert.match(workflow, /test "\$DEPLOY_HOST" = "118\.26\.111\.127"/); assert.match(workflow, /test "\$DEPLOY_USER" = "deploy"/); assert.match(workflow, /test "\$DEPLOY_PATH" = "\/opt\/jyotisha-staging"/); - assert.match(workflow, /--exclude='\.env\*'/); - assert.match( - workflow, - /docker compose -p jyotisha-staging --env-file \.env\.staging/, - ); - assert.match( - workflow, - /bash deploy\/validate-staging-env\.sh \.env\.staging/, - ); - assert.match( - workflow, - /bash deploy\/validate-staging-database-env\.sh \.env\.staging\.database/, - ); - assert.match(workflow, /-f deploy\/docker-compose\.server\.yml/); - assert.match(workflow, /-f deploy\/docker-compose\.postgres\.yml/); - assert.match(workflow, /deployment\.gitCommit/); + assert.match(workflow, /--exclude='\/\.env\*'/); + assert.match(workflow, /--exclude='\/backups\/'/); + assert.match(workflow, /run-staging-deploy\.sh/); + assert.match(workflow, /steps\.images\.outputs\.api_image/); + assert.match(workflow, /steps\.images\.outputs\.web_image/); assert.doesNotMatch(workflow, /PRODUCTION_SSH_PRIVATE_KEY/); assert.doesNotMatch(workflow, /103\.117\.123\.53/); }); -test("staging rsync preserves every destination env variant during delete", () => { - const workflow = readFileSync( - new URL("../../.github/workflows/deploy-staging.yml", import.meta.url), - "utf8", - ); - const envExclusion = workflow.match(/--exclude='([^']*\.env[^']*)'/)?.[1]; - - assert.equal(envExclusion, ".env*"); - - const root = mkdtempSync(join(tmpdir(), "jyotisha-staging-rsync-")); - const source = join(root, "source"); - const destination = join(root, "destination"); - mkdirSync(source); - mkdirSync(destination); - writeFileSync(join(source, "app.txt"), "new revision\n"); - for (const name of [ - ".env", - ".env.local", - ".env.staging", - ".env.staging.backup", - ]) { - writeFileSync(join(destination, name), "preserve\n"); - } - - try { - const result = spawnSync( - "rsync", - [ - "-a", - "--delete", - `--exclude=${envExclusion}`, - `${source}/`, - `${destination}/`, - ], - { encoding: "utf8" }, - ); - assert.equal(result.status, 0, result.stderr); - for (const name of [ - ".env", - ".env.local", - ".env.staging", - ".env.staging.backup", - ]) { - assert.equal( - existsSync(join(destination, name)), - true, - `${name} was deleted`, - ); - } - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - test("staging env validator rejects selector drift, duplicates, and unsafe permissions", () => { const validator = fileURLToPath( new URL("../../deploy/validate-staging-env.sh", import.meta.url), diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 0bf6ee6e..9794f0ca 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -1,1384 +1,247 @@ import assert from "node:assert/strict"; -import { - chmodSync, - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; -import test from "node:test"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; -type YamlNode = { - key: string; - value: string; - indent: number; - start: number; - end: number; -}; - -type WorkflowDocument = { - lines: string[]; - root: YamlNode[]; -}; - -type WorkflowStep = { - node: YamlNode; - name: string; -}; - -const backendWorkflowUrl = new URL( +const qualityWorkflow = new URL( "../../.github/workflows/backend-quality-gate.yml", import.meta.url, ); -const deploymentWorkflowUrl = new URL( +const deployWorkflow = new URL( "../../.github/workflows/deploy-staging.yml", import.meta.url, ); -const migrationWorkflowUrl = new URL( +const migrationWorkflow = new URL( "../../.github/workflows/migrate-staging-database.yml", import.meta.url, ); -const deploymentReadmeUrl = new URL( - "../../deploy/README.md", +const deployScript = new URL( + "../../deploy/run-staging-deploy.sh", + import.meta.url, +); +const migrationScript = new URL( + "../../deploy/run-staging-migration.sh", + import.meta.url, +); +const syncScript = new URL( + "../../deploy/sync-staging-tree.sh", import.meta.url, ); -function markdownSection(readme: string, heading: string): string { - const lines = readme.split("\n"); - const start = lines.indexOf(heading); - assert.notEqual(start, -1, `missing README heading: ${heading}`); - const level = heading.match(/^#+/)?.[0].length ?? 0; - let inFence = false; - let end = -1; - for (let index = start + 1; index < lines.length; index += 1) { - if (lines[index].trim().startsWith("```")) { - inFence = !inFence; - continue; - } - if ( - !inFence && - /^#+\s/.test(lines[index]) && - (lines[index].match(/^#+/)?.[0].length ?? 0) <= level - ) { - end = index; - break; - } +function read(url: URL): string { + return readFileSync(url, "utf8"); +} + +function assertOrder(text: string, labels: string[]): void { + let previous = -1; + for (const label of labels) { + const index = text.indexOf(label); + assert.ok(index > previous, `${label} is missing or out of order`); + previous = index; } - return lines.slice(start, end === -1 ? lines.length : end).join("\n"); } -function fencedCodeBlocks(section: string, language: string): string[] { - const lines = section.split("\n"); - const blocks: string[] = []; - for (let index = 0; index < lines.length; index += 1) { - if (lines[index] !== `\`\`\`${language}`) continue; - const end = lines.indexOf("```", index + 1); - assert.notEqual(end, -1, `unterminated ${language} code block`); - blocks.push(lines.slice(index + 1, end).join("\n")); - index = end; - } - return blocks; -} - -function indentation(line: string): number { - return line.match(/^ */)?.[0].length ?? 0; -} - -function mappingNodes( - lines: string[], - indent: number, - start = 0, - end = lines.length, -): YamlNode[] { - const nodes: YamlNode[] = []; - for (let index = start; index < end; index += 1) { - const line = lines[index]; - if (!line.trim() || indentation(line) !== indent) continue; - const match = line.slice(indent).match(/^([^:#][^:]*):(?:\s+(.*))?$/); - if (!match) continue; - let nodeEnd = end; - for (let cursor = index + 1; cursor < end; cursor += 1) { - if (!lines[cursor].trim()) continue; - if (indentation(lines[cursor]) <= indent) { - nodeEnd = cursor; - break; - } - } - nodes.push({ - key: match[1], - value: match[2] ?? "", - indent, - start: index, - end: nodeEnd, - }); - } - return nodes; -} - -function parseWorkflow(url = backendWorkflowUrl): WorkflowDocument { - const lines = readFileSync(url, "utf8").split("\n"); - return { lines, root: mappingNodes(lines, 0) }; -} - -function requiredNode(nodes: YamlNode[], key: string): YamlNode { - const matches = nodes.filter((node) => node.key === key); - assert.equal(matches.length, 1, `expected exactly one YAML key: ${key}`); - return matches[0]; -} - -function children(document: WorkflowDocument, parent: YamlNode): YamlNode[] { - return mappingNodes( - document.lines, - parent.indent + 2, - parent.start + 1, - parent.end, - ); -} - -function child( - document: WorkflowDocument, - parent: YamlNode, - key: string, -): YamlNode { - return requiredNode(children(document, parent), key); -} - -function blockScalar(document: WorkflowDocument, node: YamlNode): string { - assert.equal(node.value, "|", `${node.key} must be a literal block scalar`); - const contentIndent = node.indent + 2; - return document.lines - .slice(node.start + 1, node.end) - .filter((line) => line.trim()) - .map((line) => { - assert.ok( - indentation(line) >= contentIndent, - `${node.key} contains an outdented block-scalar line`, - ); - return line.slice(contentIndent); - }) - .join("\n"); -} - -function job(document: WorkflowDocument, name: string): YamlNode { - return child(document, requiredNode(document.root, "jobs"), name); -} - -function steps(document: WorkflowDocument, jobNode: YamlNode): WorkflowStep[] { - const stepsNode = child(document, jobNode, "steps"); - const stepIndent = stepsNode.indent + 2; - const result: WorkflowStep[] = []; - for ( - let index = stepsNode.start + 1; - index < stepsNode.end; - index += 1 - ) { - const line = document.lines[index]; - if (indentation(line) !== stepIndent) continue; - const match = line.slice(stepIndent).match(/^- name:\s+(.+)$/); - if (!match) continue; - let stepEnd = stepsNode.end; - for (let cursor = index + 1; cursor < stepsNode.end; cursor += 1) { - if ( - indentation(document.lines[cursor]) === stepIndent && - document.lines[cursor].slice(stepIndent).startsWith("- ") - ) { - stepEnd = cursor; - break; - } - } - result.push({ - name: match[1], - node: { - key: match[1], - value: "", - indent: stepIndent, - start: index, - end: stepEnd, - }, - }); - } - return result; -} - -function requiredStep( - document: WorkflowDocument, - jobNode: YamlNode, - name: string, -): WorkflowStep { - const matches = steps(document, jobNode).filter((step) => step.name === name); - assert.equal(matches.length, 1, `expected exactly one workflow step: ${name}`); - return matches[0]; -} - -function stepField( - document: WorkflowDocument, - step: WorkflowStep, - key: string, -): YamlNode { - return requiredNode( - mappingNodes( - document.lines, - step.node.indent + 2, - step.node.start + 1, - step.node.end, - ), - key, - ); -} - -function stepRun(document: WorkflowDocument, step: WorkflowStep): string { - const run = stepField(document, step, "run"); - return run.value === "|" ? blockScalar(document, run) : run.value; -} - -function logicalShellLines(script: string): string[] { - return script - .replace(/\\\n\s*/g, " ") - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); -} - -test("production manual-trigger wording is scoped and staging automation stays explicit", () => { - const readme = readFileSync(deploymentReadmeUrl, "utf8"); - const production = markdownSection( - readme, - "## Manual deployment with GitHub Actions", - ); - const staging = markdownSection(readme, "## Staging deployment"); - - assert.doesNotMatch( - readme, - /^Pushes and pull requests do not start GitHub Actions automatically\./m, - ); - assert.match( - production, - /^Production pushes and pull requests do not start GitHub Actions automatically\./m, - ); - assert.match( - staging, - /Staging Backend Quality Gate[\s\S]*only a successful push to `staging` can publish/i, - ); - assert.match(staging, /automatic `Deploy staging`/i); -}); - -test("first staging deployment is executable and cannot route publish through main", () => { - const readme = readFileSync(deploymentReadmeUrl, "utf8"); - const staging = markdownSection(readme, "## Staging deployment"); - const firstDeploy = markdownSection(staging, "### First-deploy sequence").split( - "\n\nApplication rollback uses", - )[0]; - - assert.doesNotMatch(firstDeploy, /first deployment should be manual/i); - assert.doesNotMatch( - firstDeploy, - /Staging Backend Quality Gate[^\n]*from `main`/i, - ); - for (const phrase of [ - "complete the server and GitHub bootstrap", - "push the reviewed SHA to `staging`", - "publishes the SHA-tagged API/web images", - "automatic `Deploy staging`", - "validates both `.env.staging` and `.env.staging.database`", - "same successful SHA", - "pending migration", - "Migrate Staging Database", - "https://staging.jyotisha.chat/api/health", - ]) { - assert.match(firstDeploy, new RegExp(phrase.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i")); - } - - const orderedPhrases = [ - "complete the server and GitHub bootstrap", - "push the reviewed SHA to `staging`", - "publishes the SHA-tagged API/web images", - "validates both `.env.staging` and `.env.staging.database`", - "same successful SHA", - "pending migration", - "https://staging.jyotisha.chat/api/health", - ]; - const indexes = orderedPhrases.map((phrase) => firstDeploy.indexOf(phrase)); - for (let index = 1; index < indexes.length; index += 1) { - assert.ok( - indexes[index] > indexes[index - 1], - `first-deploy step is out of order: ${orderedPhrases[index - 1]} -> ${orderedPhrases[index]}`, +test("changed staging workflows are syntactically valid YAML", () => { + for (const workflow of [qualityWorkflow, deployWorkflow, migrationWorkflow]) { + const result = spawnSync( + "ruby", + ["-e", "require 'yaml'; YAML.parse_file(ARGV.fetch(0))", fileURLToPath(workflow)], + { encoding: "utf8" }, ); + assert.equal(result.status, 0, result.stderr); } }); -test("restore drill command block is fail-fast, archive-safe, and narrowly destructive", () => { - const readme = readFileSync(deploymentReadmeUrl, "utf8"); - const operations = markdownSection(readme, "## Staging PostgreSQL operations"); - const restore = markdownSection( - operations, - "### Restore drill into a disposable database", - ); - const blocks = fencedCodeBlocks(restore, "bash"); - assert.equal(blocks.length, 1, "restore drill must have one executable bash block"); - const script = blocks[0]; +test("quality gate validates relevant changes once and publishes a digest manifest", () => { + const workflow = read(qualityWorkflow); - assert.match(script, /^set -euo pipefail\n/); - assert.doesNotMatch(script, //); - assert.match(script, /find "\$BACKUP_DIR"[^\n]*-type f[^\n]*-name/); - assert.match(script, /BACKUP_FILE=.*\$\(.*find/); - assert.match(script, /test -n "\$BACKUP_FILE"/); - assert.match(script, /test -f "\$BACKUP_FILE"/); - assert.match(script, /test ! -L "\$BACKUP_FILE"/); - assert.match(script, /test -s "\$BACKUP_FILE"/); - - const traps = script.match(/^trap .*$/gm) ?? []; - assert.equal(traps.length, 1, "restore drill must have one cleanup trap"); - assert.match(traps[0], /RESTORE_DUMP/); - assert.doesNotMatch(traps[0], /BACKUP_FILE|BACKUP_DIR|jyotisha_restore_check/); - - const decrypt = script.indexOf("openssl enc -d"); - const create = script.indexOf("createdb -U postgres jyotisha_restore_check"); - const restoreDb = script.indexOf("pg_restore"); - const drop = script.indexOf("dropdb -U postgres --if-exists jyotisha_restore_check"); - assert.ok(decrypt >= 0 && decrypt < create, "decrypt must precede database creation"); - assert.ok(create < restoreDb, "database creation must precede pg_restore"); - assert.ok(restoreDb < drop, "dropdb must follow a successful pg_restore"); - assert.match(script, /-pass env:STAGING_BACKUP_ENCRYPTION_KEY/); - assert.match(script, /rm -f -- "\$RESTORE_DUMP"/); - assert.doesNotMatch(script, /dropdb[^\n]*\|\|/); -}); - -test("operations runbook documents the staging database boundary and deployment order", () => { - const readme = readFileSync(deploymentReadmeUrl, "utf8"); - const staging = markdownSection(readme, "## Staging deployment"); - const operations = markdownSection( - readme, - "## Staging PostgreSQL operations", - ); - - assert.match(operations, /Staging PostgreSQL operations/); - assert.match(staging, /\/opt\/jyotisha-staging\/\.env\.staging` \(`?0600`?\)/); - assert.match(staging, /\/opt\/jyotisha-staging\/\.env\.staging\.database` \(`?0600`?\)/); - assert.match(operations, /umask 077[\s\S]*touch \.env\.staging\.database[\s\S]*chmod 600 \.env\.staging\.database/); - - for (const key of [ - "POSTGRES_DB=jyotisha", - "POSTGRES_USER=postgres", - "POSTGRES_PASSWORD=", - "SCHEMA_OWNER_PASSWORD=", - "IDENTITY_RUNTIME_PASSWORD=", - "APP_RUNTIME_PASSWORD=", - "ADMIN_RUNTIME_PASSWORD=", - "MIGRATION_RUNNER_PASSWORD=", - "BACKUP_READER_PASSWORD=", - "STAGING_BACKUP_ENCRYPTION_KEY=", - "SCHEMA_DATABASE_URL=postgresql://schema_owner:@postgres:5432/jyotisha", - ]) { - assert.match(operations, new RegExp(key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))); + assert.match(workflow, /pull_request:\n\s+paths:/); + for (const path of ["frontend/**", "deploy/**", "scripts/**", "tests/**"]) { + assert.match(workflow, new RegExp(`'${path.replaceAll("*", "\\*")}'`)); } - assert.match(operations, /independently generated 32 random bytes/); - assert.match(operations, /openssl rand -base64 32/); - assert.match(operations, /percent-encod(?:e|ed)[^\n]*URL password/i); - assert.match(operations, /SCHEMA_DATABASE_URL[^\n]*must not[^\n]*\.env\.staging/i); - assert.match( - operations, - /there is no `?SCHEMA_DATABASE_URL`? in `?\.env\.staging`?/i, - ); - - assert.match(operations, /private[^\n]*PostgreSQL|PostgreSQL[^\n]*private/i); - assert.match(staging, /no published host port|no host port/i); - assert.match(operations, /127\.0\.0\.1:\$\{POSTGRES_HOST_PORT:-55432\}:5432/); - - for (const workflow of [ - "Staging Backend Quality Gate", - "Migrate Staging Database", - "Deploy staging", - ]) { - assert.match(staging + operations, new RegExp(workflow)); - } - assert.match(staging, /pull_request/); - assert.match(staging, /push[^\n]*staging|staging[^\n]*push/i); - assert.match(staging, /workflow_dispatch/); - assert.match(staging, /deploy_sha/); - assert.match(staging, /exact[^\n]*40-character[^\n]*SHA/i); - assert.match(operations, /re-dispatch|redispatch/i); - assert.match(operations, /same[^\n]*SHA/i); - - for (const variable of [ - "STAGING_SUPABASE_URL", - "STAGING_SUPABASE_ANON_KEY", - "STAGING_HOST", - "STAGING_PORT", - "STAGING_USER", - "STAGING_PATH", - "STAGING_URL", - "STAGING_KNOWN_HOSTS", - ]) { - assert.match(staging, new RegExp(variable)); - } - assert.match(staging, /Settings[ ]*[→>-][ ]*Secrets and variables[ ]*[→>-][ ]*Actions[ ]*[→>-][ ]*Variables/); - assert.match(staging, /public build inputs[^\n]*required for publish/i); - assert.match(staging, /never print[^\n]*(?:values|keys)/i); - assert.match(staging, /STAGING_SSH_PRIVATE_KEY/); - - const mergeIndex = operations.indexOf("Merge to `staging`"); - const gateIndex = operations.indexOf("Staging Backend Quality Gate", mergeIndex); - const migrationIndex = operations.indexOf("Migrate Staging Database", gateIndex); - const redispatchIndex = operations.search(/re-dispatch|redispatch/i); - const healthIndex = operations.indexOf( - "https://staging.jyotisha.chat/api/health", - redispatchIndex, - ); - const backupIndex = operations.indexOf("backup-staging-postgres.sh", healthIndex); - assert.ok(mergeIndex >= 0, "runbook must state the staging merge step"); - assert.ok(gateIndex > mergeIndex, "quality gate must follow the staging merge"); - assert.ok(migrationIndex > gateIndex, "manual migration must follow the gate"); - assert.ok(redispatchIndex > migrationIndex, "migration must redispatch the same SHA"); - assert.ok(healthIndex > redispatchIndex, "health check must follow redispatch"); - assert.ok(backupIndex > healthIndex, "backup must follow health verification"); + assert.match(workflow, /push:\n\s+branches: \[staging\]/); + assert.match(workflow, /workflow_dispatch:/); + assert.equal((workflow.match(/npm test --prefix frontend/g) ?? []).length, 1); + assert.doesNotMatch(workflow, /npm run test:db --prefix frontend/); + assert.match(workflow, /id: api_build[\s\S]*steps\.api_build\.outputs\.digest/); + assert.match(workflow, /id: web_build[\s\S]*steps\.web_build\.outputs\.digest/); + assert.match(workflow, /\^sha256:\[0-9a-f\]\{64\}\$/); + assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/); + assert.match(workflow, /name: staging-image-manifest-\$\{\{ github\.sha \}\}/); + assert.match(workflow, /uses: actions\/upload-artifact@v4/); + assert.doesNotMatch(workflow, /(?:^|:)latest$/m); }); -test("operations runbook documents three encrypted local backups and a safe restore drill", () => { - const readme = readFileSync(deploymentReadmeUrl, "utf8"); - const operations = markdownSection( - readme, - "## Staging PostgreSQL operations", - ); - - assert.match(operations, /newest three|three[^\n]*encrypted local backups/i); - assert.match(operations, /AES-256-CBC/i); - assert.match(operations, /PBKDF2|pbkdf2/); - assert.match(operations, /custom[^\n]*format|format:[ ]*custom/i); - assert.match(operations, /no off[- ]site[^\n]*staging[^\n]*(?:recovery|backup)/i); - assert.match( - operations, - /\.\/deploy\/backup-staging-postgres\.sh[\s\S]*\.env\.staging\.database[\s\S]*\/opt\/jyotisha-staging\/backups\/staging-db/, - ); - assert.match(operations, /-pass env:STAGING_BACKUP_ENCRYPTION_KEY/); - assert.match(operations, /passphrase[^\n]*(?:argv|command line|output|history)/i); - assert.match(operations, /jyotisha_restore_check/); - assert.match(operations, /pg_restore/); - assert.match(operations, /dropdb[^\n]*jyotisha_restore_check|DROP DATABASE[^\n]*jyotisha_restore_check/i); - assert.match(operations, /temporary decrypted dump/); - assert.match(operations, /delete[^\n]*(?:only|just)[^\n]*(?:disposable database|jyotisha_restore_check)[^\n]*(?:and|,)[^\n]*temporary decrypted dump/i); - assert.match(operations, /do not[^\n]*(?:docker compose[^\n]*down|down -v)[^\n]*(?:restore|drill|staging)/i); - assert.match(operations, /does not authorize[^\n]*(?:production|cutover)|no production[^\n]*cutover[^\n]*authoriz/i); -}); - -test("backend quality gate has structured staging triggers and concurrency", () => { - const document = parseWorkflow(); - - assert.equal(requiredNode(document.root, "name").value, "Staging Backend Quality Gate"); - const triggers = requiredNode(document.root, "on"); - assert.equal(child(document, triggers, "pull_request").value, ""); - const push = child(document, triggers, "push"); - assert.equal(child(document, push, "branches").value, "[staging]"); - assert.equal(child(document, triggers, "workflow_dispatch").value, ""); - - const concurrency = requiredNode(document.root, "concurrency"); - assert.equal( - child(document, concurrency, "group").value, - "backend-quality-${{ github.workflow }}-${{ github.ref }}", - ); - assert.equal(child(document, concurrency, "cancel-in-progress").value, "true"); - const permissions = requiredNode(document.root, "permissions"); - assert.deepEqual( - children(document, permissions).map(({ key, value }) => [key, value]), - [["contents", "read"]], - ); - assert.deepEqual( - children(document, requiredNode(document.root, "jobs")).map( - ({ key }) => key, - ), - ["validate", "publish"], - ); -}); - -test("validation locates every command in its intended job and step", () => { - const document = parseWorkflow(); - const validate = job(document, "validate"); - - assert.equal(child(document, validate, "runs-on").value, "ubuntu-latest"); - assert.equal(child(document, validate, "timeout-minutes").value, "30"); - assert.equal( - stepField( - document, - requiredStep(document, validate, "Set up Python"), - "uses", - ).value, - "actions/setup-python@v5", - ); - assert.equal( - stepField( - document, - requiredStep(document, validate, "Set up Node"), - "uses", - ).value, - "actions/setup-node@v4", - ); - assert.equal( - child( - document, - stepField( - document, - requiredStep(document, validate, "Set up Python"), - "with", - ), - "python-version", - ).value, - "'3.12'", - ); - assert.equal( - child( - document, - stepField( - document, - requiredStep(document, validate, "Set up Node"), - "with", - ), - "node-version", - ).value, - "'22'", - ); - - const install = blockScalar( - document, - stepField( - document, - requiredStep(document, validate, "Install dependencies"), - "run", - ), - ); - assert.match( - install, - /python -m pip install -r requirements\.txt -r requirements-dev\.txt/, - ); - assert.match(install, /npm ci --prefix frontend/); - - const pythonStep = requiredStep( - document, - validate, - "Run Python quick quality gate", - ); - assert.equal(stepField(document, pythonStep, "shell").value, "bash"); - const pythonGate = blockScalar( - document, - stepField(document, pythonStep, "run"), - ); - assert.match(pythonGate, /^set -o pipefail\n/); - assert.match( - pythonGate, - /ruff check scripts\/run_quality_gate\.py tests\/test_varga_bphs\.py \\\n\s+tests\/test_ashtakavarga_invariants\.py tests\/test_cli_smoke\.py \\\n\s+tests\/test_yoga_rules_integrity\.py/, - ); - assert.match( - pythonGate, - /python -m py_compile scripts\/\*\.py jyotish_vedic\/\*\.py mcp_server\.py/, - ); - assert.match( - pythonGate, - /python scripts\/run_quality_gate\.py \\\n\s+--profile quick --skip-yoga-logic --skip-frontend-runtime \\\n\s+2>&1 \| tee artifacts\/quick-quality-gate\.log/, - ); - assert.match(pythonGate, /python -m build --no-isolation/); - - assert.equal( - stepField( - document, - requiredStep(document, validate, "Run database tests"), - "run", - ).value, - "npm run test:db --prefix frontend", - ); - const frontend = requiredStep(document, validate, "Validate frontend"); - const frontendEnv = stepField(document, frontend, "env"); - assert.equal( - child(document, frontendEnv, "NEXT_PUBLIC_SUPABASE_URL").value, - "https://placeholder.supabase.co", - ); - assert.equal( - child(document, frontendEnv, "NEXT_PUBLIC_SUPABASE_ANON_KEY").value, - "placeholder", - ); - assert.equal( - blockScalar(document, stepField(document, frontend, "run")), - [ - "npm test --prefix frontend", - "npm run lint --prefix frontend", - "npm run build --prefix frontend", - ].join("\n"), - ); - assert.equal( - children(document, validate).some((node) => node.key === "permissions"), - false, - ); -}); - -test("diagnostic artifact upload is always executed in validation", () => { - const document = parseWorkflow(); - const upload = requiredStep( - document, - job(document, "validate"), - "Upload quick quality gate diagnostics", - ); - - assert.equal(stepField(document, upload, "if").value, "always()"); - assert.equal( - stepField(document, upload, "uses").value, - "actions/upload-artifact@v4", - ); - assert.equal( - child(document, stepField(document, upload, "with"), "path").value, - "artifacts/quick-quality-gate.log", - ); -}); - -test("publishing has job-local permissions and immutable staging SHA images", () => { - const document = parseWorkflow(); - const publish = job(document, "publish"); - const publishSteps = steps(document, publish); - const publishStepNames = publishSteps.map(({ name }) => name); - - assert.equal( - child(document, publish, "if").value, - "github.event_name == 'push' && github.ref == 'refs/heads/staging'", - ); - assert.equal(child(document, publish, "needs").value, "validate"); - assert.deepEqual( - children(document, child(document, publish, "permissions")).map( - ({ key, value }) => [key, value], - ), - [ - ["contents", "read"], - ["packages", "write"], - ], - ); - - assert.ok( - publishStepNames.indexOf("Validate staging web build variables") < - publishStepNames.indexOf("Build and publish API image"), - ); - assert.ok( - publishStepNames.indexOf("Validate staging web build variables") < - publishStepNames.indexOf("Build and publish web image"), - ); - - const stagingVariables = requiredStep( - document, - publish, - "Validate staging web build variables", - ); - const stagingVariableEnv = stepField(document, stagingVariables, "env"); - assert.equal( - child(document, stagingVariableEnv, "STAGING_SUPABASE_URL").value, - "${{ vars.STAGING_SUPABASE_URL }}", - ); - assert.equal( - child(document, stagingVariableEnv, "STAGING_SUPABASE_ANON_KEY").value, - "${{ vars.STAGING_SUPABASE_ANON_KEY }}", - ); - - const login = requiredStep(document, publish, "Log in to GHCR"); - assert.equal(stepField(document, login, "uses").value, "docker/login-action@v3"); - assert.equal( - child(document, stepField(document, login, "with"), "password").value, - "${{ secrets.GITHUB_TOKEN }}", - ); - - for (const [name, dockerfile, tag] of [ - [ - "Build and publish API image", - "deploy/railway-api.Dockerfile", - "ghcr.io/jesse-ux/jyotisha-api:${{ github.sha }}", - ], - [ - "Build and publish web image", - "deploy/railway-web.Dockerfile", - "ghcr.io/jesse-ux/jyotisha-web:${{ github.sha }}", - ], - ]) { - const build = requiredStep(document, publish, name); - assert.equal( - stepField(document, build, "uses").value, - "docker/build-push-action@v6", - ); - const options = stepField(document, build, "with"); - assert.equal(child(document, options, "context").value, "."); - assert.equal(child(document, options, "file").value, dockerfile); - assert.equal(child(document, options, "push").value, "true"); - assert.equal(child(document, options, "tags").value, tag); - assert.doesNotMatch(tag, /(?:^|:)latest$/); - } - - const webOptions = stepField( - document, - requiredStep(document, publish, "Build and publish web image"), - "with", - ); - assert.equal( - blockScalar(document, child(document, webOptions, "build-args")), - [ - "NEXT_PUBLIC_SUPABASE_URL=${{ vars.STAGING_SUPABASE_URL }}", - "NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ vars.STAGING_SUPABASE_ANON_KEY }}", - ].join("\n"), - ); - - const validateFrontend = requiredStep( - document, - job(document, "validate"), - "Validate frontend", - ); - const outsideValidationBuild = [ - ...document.lines.slice(0, validateFrontend.node.start), - ...document.lines.slice(validateFrontend.node.end), - ].join("\n"); - assert.doesNotMatch(outsideValidationBuild, /placeholder/); -}); - -test("publishing fails closed for missing or invalid staging web variables without exposing values", () => { - const document = parseWorkflow(); - const validation = requiredStep( - document, - job(document, "publish"), - "Validate staging web build variables", - ); - const script = stepRun(document, validation); - const secretFixture = "anon-fixture-must-not-appear"; - const validUrl = "https://project-ref.supabase.co"; - const invalidUrl = "http://project-ref.supabase.co"; - const run = (url: string, anonKey: string) => - spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { - ...process.env, - STAGING_SUPABASE_URL: url, - STAGING_SUPABASE_ANON_KEY: anonKey, - }, - }); - - assert.match(script, /test -n "\$STAGING_SUPABASE_URL"/); - assert.match(script, /test -n "\$STAGING_SUPABASE_ANON_KEY"/); - assert.doesNotMatch(script, /echo[^\n]*\$STAGING_SUPABASE_(?:URL|ANON_KEY)/); - - for (const failed of [ - run("", secretFixture), - run(validUrl, ""), - run(invalidUrl, secretFixture), - ]) { - assert.notEqual(failed.status, 0); - assert.doesNotMatch(`${failed.stdout}\n${failed.stderr}`, new RegExp(secretFixture)); - assert.doesNotMatch(`${failed.stdout}\n${failed.stderr}`, new RegExp(invalidUrl)); - } - assert.equal(run(validUrl, secretFixture).status, 0); -}); - -test("deployment test script covers health and backend workflow contracts", () => { +test("deployment test command includes manifest behavior coverage", () => { const packageJson = JSON.parse( readFileSync(new URL("../package.json", import.meta.url), "utf8"), ) as { scripts: Record }; - assert.equal( - packageJson.scripts["test:deployment"], - "tsx --test tests/health-deployment.test.ts tests/staging-backend-workflows.test.ts", - ); + const command = packageJson.scripts["test:deployment"]; + assert.match(command, /health-deployment\.test\.ts/); + assert.match(command, /staging-backend-workflows\.test\.ts/); + assert.match(command, /staging-image-manifest\.test\.ts/); }); -test("staging deploy structurally follows the backend gate and validates an exact manual SHA", () => { - const document = parseWorkflow(deploymentWorkflowUrl); - - assert.equal(requiredNode(document.root, "name").value, "Deploy staging"); - const triggers = requiredNode(document.root, "on"); - const workflowRun = child(document, triggers, "workflow_run"); - assert.equal( - child(document, workflowRun, "workflows").value, - '["Staging Backend Quality Gate"]', - ); - assert.equal(child(document, workflowRun, "types").value, "[completed]"); - const dispatch = child(document, triggers, "workflow_dispatch"); - const deploySha = child(document, child(document, dispatch, "inputs"), "deploy_sha"); - assert.equal(child(document, deploySha, "required").value, "true"); - assert.equal(child(document, deploySha, "type").value, "string"); - - assert.deepEqual( - children(document, requiredNode(document.root, "permissions")).map( - ({ key, value }) => [key, value], - ), - [ - ["contents", "read"], - ["actions", "read"], - ["packages", "read"], - ], - ); - const deploy = job(document, "deploy"); - assert.match(child(document, deploy, "if").value, /conclusion == 'success'/); - const revision = requiredStep(document, deploy, "Validate tested revision"); - assert.equal( - child(document, stepField(document, revision, "env"), "REQUESTED_SHA").value, - "${{ github.event.workflow_run.head_sha || inputs.deploy_sha }}", - ); - const validation = stepRun(document, revision); - assert.match(validation, /test "\$\{#REQUESTED_SHA\}" -eq 40/); - assert.match( - validation, - /actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=\$DEPLOY_GIT_SHA/, - ); - assert.match(validation, /head_branch == "staging"/); - assert.match(validation, /conclusion == "success"/); -}); - -test("staging deploy pins every Compose call and gates app changes on the read-only checker", () => { - const document = parseWorkflow(deploymentWorkflowUrl); - const deploy = job(document, "deploy"); - const deploySteps = steps(document, deploy); - const names = deploySteps.map(({ name }) => name); - const index = (name: string) => { - const found = names.indexOf(name); - assert.notEqual(found, -1, `missing deployment step: ${name}`); - return found; - }; - - assert.ok(index("Pull exact staging images") < index("Start and wait for staging PostgreSQL")); - assert.ok(index("Start and wait for staging PostgreSQL") < index("Check staging migrations")); - assert.ok(index("Check staging migrations") < index("Deploy exact staging images")); - assert.ok(index("Deploy exact staging images") < index("Verify staging")); - assert.ok(index("Verify staging") < index("Roll back staging images")); - assert.ok(index("Roll back staging images") < index("Log out of GHCR")); - - const scripts = deploySteps - .map((step) => { - const fields = mappingNodes( - document.lines, - step.node.indent + 2, - step.node.start + 1, - step.node.end, - ); - const run = fields.find(({ key }) => key === "run"); - return run ? (run.value === "|" ? blockScalar(document, run) : run.value) : ""; - }) - .filter(Boolean); - const composeCommands = scripts - .flatMap(logicalShellLines) - .filter((line) => line.includes("docker compose")); - assert.equal(composeCommands.length, 7); - const rollbackComposeCommands = composeCommands.filter((command) => - command.includes("API_IMAGE='$PREVIOUS_API_IMAGE'"), - ); - assert.equal(rollbackComposeCommands.length, 1); - const targetComposeCommands = composeCommands.filter( - (command) => !command.includes("API_IMAGE='$PREVIOUS_API_IMAGE'"), - ); - assert.equal(targetComposeCommands.length, 6); - for (const command of targetComposeCommands) { - assert.match( - command, - /API_IMAGE='ghcr\.io\/jesse-ux\/jyotisha-api:\$DEPLOY_GIT_SHA'/, - ); - assert.match( - command, - /WEB_IMAGE='ghcr\.io\/jesse-ux\/jyotisha-web:\$DEPLOY_GIT_SHA'/, - ); +test("live staging sync preserves env, state, incoming files, and encrypted backups", () => { + const root = mkdtempSync(join(tmpdir(), "jyotisha-live-sync-")); + const source = join(root, "source"); + const destination = join(root, "destination"); + mkdirSync(source); + mkdirSync(destination); + writeFileSync(join(source, "revision.txt"), "new\n"); + writeFileSync(join(destination, "stale.txt"), "old\n"); + for (const relative of [ + ".env.staging", + ".env.staging.database", + ".state/deployed-revision", + ".incoming/other-run/payload", + "backups/staging-db/20260720.dump.gz.gpg", + ]) { + const target = join(destination, relative); + mkdirSync(join(target, ".."), { recursive: true }); + writeFileSync(target, "preserve\n"); } - assert.match(rollbackComposeCommands[0], /WEB_IMAGE='\$PREVIOUS_WEB_IMAGE'/); - assert.match(rollbackComposeCommands[0], /GITHUB_SHA='\$PREVIOUS_SHA'/); - for (const command of composeCommands) { - assert.match(command, /ssh /); - assert.match(command, /APP_ENV_FILE='\.\.\/\.env\.staging'/); - assert.match(command, /DATABASE_ENV_FILE='\.\.\/\.env\.staging\.database'/); - assert.match(command, /CADDYFILE_PATH='\.\/Caddyfile\.staging'/); - assert.match(command, /SITE_ADDRESS='https:\/\/staging\.jyotisha\.chat'/); - assert.match(command, /--env-file \.env\.staging/); - assert.match(command, /-f deploy\/docker-compose\.server\.yml/); - assert.match(command, /-f deploy\/docker-compose\.postgres\.yml/); - } - - const login = stepRun(document, requiredStep(document, deploy, "Log in to GHCR")); - assert.match(login, /printf '%s' "\$GHCR_TOKEN" \| ssh /); - assert.match(login, /docker login ghcr\.io .*--password-stdin/); - assert.doesNotMatch(login, /--password(?:\s|=)/); - - assert.match( - stepRun(document, requiredStep(document, deploy, "Pull exact staging images")), - /pull api web postgres/, - ); - assert.match( - stepRun( - document, - requiredStep(document, deploy, "Start and wait for staging PostgreSQL"), - ), - /up -d --no-build --wait postgres/, - ); - - const check = requiredStep(document, deploy, "Check staging migrations"); - assert.equal(stepField(document, check, "id").value, "migration_check"); - const checkScript = stepRun(document, check); - assert.match(checkScript, /--profile migration-check run --rm migration-checker/); - assert.match(checkScript, /"\$CHECK_STATUS" -eq 3/); - assert.match(checkScript, /Migrate Staging Database/); - assert.match(checkScript, /\$DEPLOY_GIT_SHA/); - assert.match(checkScript, /exit 3/); - assert.match( - readFileSync( - new URL("../../deploy/docker-compose.postgres.yml", import.meta.url), - "utf8", - ), - /command: \["npm", "run", "db:migrate:check"\]/, - ); - - const workflow = readFileSync(deploymentWorkflowUrl, "utf8"); - assert.doesNotMatch(workflow, /npm\s+run\s+db:migrate(?!:check)/); - assert.doesNotMatch(workflow, /run\s+--rm\s+migrator/); - assert.doesNotMatch(workflow, /--profile\s+migration(?:\s|["'])/); - assert.doesNotMatch(workflow, /(?:up -d[^\n]*--build|docker compose build)/); - - const applicationDeploy = stepRun( - document, - requiredStep(document, deploy, "Deploy exact staging images"), - ); - assert.match(applicationDeploy, /up -d --no-build --remove-orphans/); - const previous = requiredStep(document, deploy, "Record previous staging images"); - assert.equal(stepField(document, previous, "id").value, "previous"); - const previousScript = stepRun(document, previous); - assert.match(previousScript, /docker inspect --format '\{\{\.Config\.Image\}\}'/); - assert.match(previousScript, /api_image=\$PREVIOUS_API_IMAGE/); - assert.match(previousScript, /web_image=\$PREVIOUS_WEB_IMAGE/); - assert.match(previousScript, /previous_sha=\$PREVIOUS_SHA/); - assert.doesNotMatch( - previousScript, - /(?:api_image|web_image|previous_sha)=not-deployed/, - ); - assert.match(previousScript, /\$\{PREVIOUS_SHA:-not-deployed\}/); - - const rollback = requiredStep(document, deploy, "Roll back staging images"); - assert.equal( - stepField(document, rollback, "if").value, - "failure() && steps.migration_check.outcome == 'success' && steps.previous.outputs.api_image != '' && steps.previous.outputs.web_image != '' && steps.previous.outputs.previous_sha != ''", - ); - const rollbackEnv = stepField(document, rollback, "env"); - assert.equal( - child(document, rollbackEnv, "PREVIOUS_API_IMAGE").value, - "${{ steps.previous.outputs.api_image }}", - ); - assert.equal( - child(document, rollbackEnv, "PREVIOUS_WEB_IMAGE").value, - "${{ steps.previous.outputs.web_image }}", - ); - assert.equal( - child(document, rollbackEnv, "PREVIOUS_SHA").value, - "${{ steps.previous.outputs.previous_sha }}", - ); - assert.match(stepRun(document, rollback), /API_IMAGE='\$PREVIOUS_API_IMAGE'/); - assert.match(stepRun(document, rollback), /WEB_IMAGE='\$PREVIOUS_WEB_IMAGE'/); - assert.match(stepRun(document, rollback), /GITHUB_SHA='\$PREVIOUS_SHA'/); - assert.match(stepRun(document, rollback), /up -d --no-build/); - assert.doesNotMatch(stepRun(document, rollback), /not-deployed/); - - const logout = requiredStep(document, deploy, "Log out of GHCR"); - assert.equal(stepField(document, logout, "if").value, "always()"); - assert.equal(stepField(document, logout, "continue-on-error").value, "true"); - assert.match(stepRun(document, logout), /docker logout ghcr\.io/); -}); - -test("pending migration status exits 3 with the manual workflow and exact SHA", () => { - const document = parseWorkflow(deploymentWorkflowUrl); - const check = requiredStep( - document, - job(document, "deploy"), - "Check staging migrations", - ); - const script = stepRun(document, check); - const root = mkdtempSync(join(tmpdir(), "jyotisha-migration-check-")); - const fakeBin = join(root, "bin"); - const fakeSsh = join(fakeBin, "ssh"); - const exactSha = "0123456789abcdef0123456789abcdef01234567"; - mkdirSync(fakeBin); - writeFileSync(fakeSsh, '#!/usr/bin/env bash\nexit "${FAKE_SSH_STATUS:?}"\n'); - chmodSync(fakeSsh, 0o700); - - const run = (status: number) => - spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { - ...process.env, - DEPLOY_GIT_SHA: exactSha, - DEPLOY_HOST: "staging.example.invalid", - DEPLOY_PORT: "22", - DEPLOY_USER: "deploy", - DEPLOY_PATH: "/opt/jyotisha-staging", - FAKE_SSH_STATUS: String(status), - HOME: root, - PATH: `${fakeBin}:${process.env.PATH}`, - }, - }); try { - const pending = run(3); - assert.equal(pending.status, 3, pending.stderr); - assert.match(`${pending.stdout}\n${pending.stderr}`, /Migrate Staging Database/); - assert.match(`${pending.stdout}\n${pending.stderr}`, new RegExp(exactSha)); - assert.equal(run(0).status, 0); - } finally { - rmSync(root, { recursive: true, force: true }); - } -}); - -test("staging migration is manual-only, serialized, and least-privileged", () => { - const document = parseWorkflow(migrationWorkflowUrl); - - assert.equal(requiredNode(document.root, "name").value, "Migrate Staging Database"); - const triggers = requiredNode(document.root, "on"); - assert.deepEqual(children(document, triggers).map(({ key }) => key), [ - "workflow_dispatch", - ]); - const dispatch = child(document, triggers, "workflow_dispatch"); - const inputs = child(document, dispatch, "inputs"); - assert.deepEqual(children(document, inputs).map(({ key }) => key), [ - "deploy_sha", - ]); - const deploySha = child(document, inputs, "deploy_sha"); - assert.equal(child(document, deploySha, "required").value, "true"); - assert.equal(child(document, deploySha, "type").value, "string"); - - const concurrency = requiredNode(document.root, "concurrency"); - assert.equal( - child(document, concurrency, "group").value, - "staging-database-migration", - ); - assert.equal(child(document, concurrency, "cancel-in-progress").value, "false"); - assert.deepEqual( - children(document, requiredNode(document.root, "permissions")).map( - ({ key, value }) => [key, value], - ), - [ - ["contents", "read"], - ["actions", "write"], - ["packages", "read"], - ], - ); - assert.deepEqual( - children(document, requiredNode(document.root, "jobs")).map(({ key }) => key), - ["migrate"], - ); - - const migrate = job(document, "migrate"); - assert.equal(child(document, migrate, "environment").value, "staging"); - assert.equal(child(document, migrate, "runs-on").value, "ubuntu-latest"); - assert.equal(child(document, migrate, "timeout-minutes").value, "20"); -}); - -test("staging migration accepts only an exact lowercase SHA with a successful staging gate", () => { - const document = parseWorkflow(migrationWorkflowUrl); - const migrate = job(document, "migrate"); - const revision = requiredStep(document, migrate, "Validate tested revision"); - const revisionEnv = stepField(document, revision, "env"); - assert.equal( - child(document, revisionEnv, "REQUESTED_SHA").value, - "${{ inputs.deploy_sha }}", - ); - assert.equal( - child(document, revisionEnv, "GH_TOKEN").value, - "${{ github.token }}", - ); - const validation = stepRun(document, revision); - assert.match(validation, /\^\[0-9a-f\]\{40\}\$/); - assert.match( - validation, - /actions\/workflows\/backend-quality-gate\.yml\/runs\?head_sha=\$REQUESTED_SHA&branch=staging&event=push&status=success/, - ); - assert.match(validation, /\.head_sha == \$sha/); - assert.match(validation, /\.head_branch == "staging"/); - assert.match(validation, /\.event == "push"/); - assert.match(validation, /\.conclusion == "success"/); - assert.equal( - validation.match(/>> "\$GITHUB_OUTPUT"/g)?.length, - 1, - "only the validated single-line SHA may reach GITHUB_OUTPUT", - ); - - const checkout = requiredStep(document, migrate, "Checkout tested revision"); - assert.equal(stepField(document, checkout, "uses").value, "actions/checkout@v4"); - assert.equal( - child(document, stepField(document, checkout, "with"), "ref").value, - "${{ steps.revision.outputs.deploy_sha }}", - ); - const verify = requiredStep(document, migrate, "Verify checked-out revision"); - assert.match(stepRun(document, verify), /git rev-parse HEAD/); - assert.match(stepRun(document, verify), /"\$DEPLOY_SHA"/); -}); - -test("tested-revision validation fails closed before checkout or migration", () => { - const document = parseWorkflow(migrationWorkflowUrl); - const script = stepRun( - document, - requiredStep(document, job(document, "migrate"), "Validate tested revision"), - ); - const root = mkdtempSync(join(tmpdir(), "jyotisha-migration-gate-")); - const fakeBin = join(root, "bin"); - const fakeCurl = join(fakeBin, "curl"); - const curlLog = join(root, "curl.log"); - const githubOutput = join(root, "github-output"); - const exactSha = "0123456789abcdef0123456789abcdef01234567"; - mkdirSync(fakeBin); - writeFileSync( - fakeCurl, - [ - "#!/usr/bin/env bash", - 'printf \'%s\\n\' "$@" >> "$FAKE_CURL_LOG"', - 'printf \'%s\' "$FAKE_GATE_JSON"', - ].join("\n"), - ); - chmodSync(fakeCurl, 0o700); - - const run = (sha: string, gateJson: object) => { - writeFileSync(curlLog, ""); - writeFileSync(githubOutput, ""); - const result = spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { - ...process.env, - FAKE_CURL_LOG: curlLog, - FAKE_GATE_JSON: JSON.stringify(gateJson), - GH_TOKEN: "gate-token-fixture", - GITHUB_API_URL: "https://api.github.test", - GITHUB_OUTPUT: githubOutput, - GITHUB_REPOSITORY: "jesse-ux/Jyotisha", - PATH: `${fakeBin}:${process.env.PATH}`, - REQUESTED_SHA: sha, - }, - }); - return { - calls: readFileSync(curlLog, "utf8"), - output: readFileSync(githubOutput, "utf8"), - result, - }; - }; - - try { - for (const invalidSha of [exactSha.toUpperCase(), exactSha.slice(1), `${exactSha}\n`]) { - const invalid = run(invalidSha, { workflow_runs: [] }); - assert.notEqual(invalid.result.status, 0); - assert.equal(invalid.calls, ""); - assert.equal(invalid.output, ""); + const result = spawnSync( + "bash", + [fileURLToPath(syncScript), source, destination], + { encoding: "utf8" }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(existsSync(join(destination, "stale.txt")), false); + assert.equal(readFileSync(join(destination, "revision.txt"), "utf8"), "new\n"); + for (const relative of [ + ".env.staging", + ".env.staging.database", + ".state/deployed-revision", + ".incoming/other-run/payload", + "backups/staging-db/20260720.dump.gz.gpg", + ]) { + assert.equal(existsSync(join(destination, relative)), true, relative); } - - const wrongBranch = run(exactSha, { - workflow_runs: [ - { - conclusion: "success", - event: "push", - head_branch: "main", - head_sha: exactSha, - }, - ], - }); - assert.notEqual(wrongBranch.result.status, 0); - assert.equal(wrongBranch.output, ""); - - const accepted = run(exactSha, { - workflow_runs: [ - { - conclusion: "success", - event: "push", - head_branch: "staging", - head_sha: exactSha, - }, - ], - }); - assert.equal(accepted.result.status, 0, accepted.result.stderr); - assert.equal(accepted.output, `deploy_sha=${exactSha}\n`); - assert.match(accepted.calls, new RegExp(`head_sha=${exactSha}`)); - assert.doesNotMatch( - `${accepted.result.stdout}\n${accepted.result.stderr}`, - /gate-token-fixture/, - ); } finally { rmSync(root, { recursive: true, force: true }); } }); -test("staging migration preserves env files and runs only PostgreSQL plus migrator", () => { - const document = parseWorkflow(migrationWorkflowUrl); - const migrate = job(document, "migrate"); - const names = steps(document, migrate).map(({ name }) => name); - const stepIndex = (name: string) => { - const index = names.indexOf(name); - assert.notEqual(index, -1, `missing migration step: ${name}`); - return index; - }; - assert.ok( - stepIndex("Validate staging environment files") < - stepIndex("Start and wait for staging PostgreSQL"), - ); - assert.ok( - stepIndex("Start and wait for staging PostgreSQL") < - stepIndex("Apply reviewed staging migrations"), - ); - assert.ok( - stepIndex("Apply reviewed staging migrations") < - stepIndex("Print ordered migration ledger"), - ); - assert.ok( - stepIndex("Print ordered migration ledger") < - stepIndex("Dispatch exact-SHA staging deployment"), - ); +test("all staging mutations share Actions serialization and one host lock", () => { + const deployment = read(deployWorkflow); + const migration = read(migrationWorkflow); + const deployRunner = read(deployScript); + const migrationRunner = read(migrationScript); - const target = stepRun( - document, - requiredStep(document, migrate, "Validate staging target configuration"), - ); - assert.match(target, /DEPLOY_HOST" = "118\.26\.111\.127/); - assert.match(target, /DEPLOY_USER" = "deploy/); - assert.match(target, /DEPLOY_PATH" = "\/opt\/jyotisha-staging/); - assert.match(target, /test -n "\$STAGING_KNOWN_HOSTS"/); - const ssh = stepRun( - document, - requiredStep(document, migrate, "Configure pinned staging SSH"), - ); - assert.match(ssh, /STAGING_KNOWN_HOSTS/); - assert.doesNotMatch(ssh, /ssh-keyscan/); - - const sync = stepRun( - document, - requiredStep(document, migrate, "Sync tested staging sources"), - ); - assert.match(sync, /rsync -az --delete/); - assert.match(sync, /--exclude='\.env\*'/); - const validators = stepRun( - document, - requiredStep(document, migrate, "Validate staging environment files"), - ); - assert.match(validators, /validate-staging-env\.sh \.env\.staging/); - assert.match( - validators, - /validate-staging-database-env\.sh \.env\.staging\.database/, - ); - - const migrationSteps = steps(document, migrate); - const scripts = migrationSteps.map((step) => { - const run = mappingNodes( - document.lines, - step.node.indent + 2, - step.node.start + 1, - step.node.end, - ).find(({ key }) => key === "run"); - return run ? (run.value === "|" ? blockScalar(document, run) : run.value) : ""; - }); - const composeCommands = scripts - .flatMap(logicalShellLines) - .filter((line) => line.includes("docker compose")); - assert.equal(composeCommands.length, 3); - for (const command of composeCommands) { - assert.match(command, /-f deploy\/docker-compose\.postgres\.yml/); - assert.doesNotMatch(command, /docker-compose\.server\.yml/); - assert.doesNotMatch(command, /\brestart\b/); + for (const workflow of [deployment, migration]) { + assert.match(workflow, /concurrency:\n\s+group: staging-mutation\n\s+cancel-in-progress: false/); } - const startCommands = composeCommands.filter((command) => /\sup\s/.test(command)); - assert.equal(startCommands.length, 1); - assert.match(startCommands[0], /up -d --wait postgres/); - assert.doesNotMatch(startCommands[0], /\b(?:api|web|caddy)\b/); - - const apply = stepRun( - document, - requiredStep(document, migrate, "Apply reviewed staging migrations"), - ); - assert.match( - apply, - /WEB_IMAGE='ghcr\.io\/jesse-ux\/jyotisha-web:\$DEPLOY_SHA'/, - ); - assert.match(apply, /--profile migration run --rm migrator/); - const ledger = stepRun( - document, - requiredStep(document, migrate, "Print ordered migration ledger"), - ); - assert.match(ledger, /psql -U postgres -d jyotisha -Atc/); - assert.match( - ledger, - /select filename from migration\.schema_migrations order by filename/, - ); - assert.doesNotMatch( - readFileSync(migrationWorkflowUrl, "utf8"), - /docker-compose\.server\.yml/, - ); -}); - -test("successful migration dispatches exact-SHA staging deploy and always logs out", () => { - const document = parseWorkflow(migrationWorkflowUrl); - const migrate = job(document, "migrate"); - const dispatch = requiredStep( - document, - migrate, - "Dispatch exact-SHA staging deployment", - ); - const dispatchFields = mappingNodes( - document.lines, - dispatch.node.indent + 2, - dispatch.node.start + 1, - dispatch.node.end, - ); - assert.equal(dispatchFields.some(({ key }) => key === "if"), false); - assert.equal(dispatchFields.some(({ key }) => key === "continue-on-error"), false); - const script = stepRun(document, dispatch); - assert.match( - script, - /actions\/workflows\/deploy-staging\.yml\/dispatches/, - ); - assert.match(script, /--data-binary "@\$PAYLOAD_FILE"/); - - const logout = requiredStep(document, migrate, "Log out of GHCR"); - assert.equal(stepField(document, logout, "if").value, "always()"); - assert.equal(stepField(document, logout, "continue-on-error").value, "true"); - assert.match(stepRun(document, logout), /docker logout ghcr\.io/); - - const root = mkdtempSync(join(tmpdir(), "jyotisha-migration-dispatch-")); - const fakeBin = join(root, "bin"); - const fakeCurl = join(fakeBin, "curl"); - const body = join(root, "body.json"); - const curlLog = join(root, "curl.log"); - const exactSha = "0123456789abcdef0123456789abcdef01234567"; - mkdirSync(fakeBin); - writeFileSync( - fakeCurl, - [ - "#!/usr/bin/env bash", - 'printf \'%s\\n\' "$@" > "$FAKE_CURL_LOG"', - 'for argument in "$@"; do', - ' case "$argument" in', - ' @*) cp -- "${argument#@}" "$FAKE_REQUEST_BODY" ;;', - " esac", - "done", - 'exit "${FAKE_CURL_STATUS:-0}"', - ].join("\n"), - ); - chmodSync(fakeCurl, 0o700); - - const run = (status: number) => - spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { - ...process.env, - DEPLOY_SHA: exactSha, - FAKE_CURL_LOG: curlLog, - FAKE_CURL_STATUS: String(status), - FAKE_REQUEST_BODY: body, - GH_TOKEN: "dispatch-token-fixture", - GITHUB_API_URL: "https://api.github.test", - GITHUB_REPOSITORY: "jesse-ux/Jyotisha", - PATH: `${fakeBin}:${process.env.PATH}`, - }, - }); - - try { - const accepted = run(0); - assert.equal(accepted.status, 0, accepted.stderr); - assert.deepEqual(JSON.parse(readFileSync(body, "utf8")), { - inputs: { deploy_sha: exactSha }, - ref: "staging", - }); - assert.match( - readFileSync(curlLog, "utf8"), - /actions\/workflows\/deploy-staging\.yml\/dispatches/, - ); - assert.doesNotMatch( - `${accepted.stdout}\n${accepted.stderr}`, - /dispatch-token-fixture/, - ); - assert.notEqual(run(22).status, 0, "workflow dispatch API failure must fail"); - } finally { - rmSync(root, { recursive: true, force: true }); + for (const runner of [deployRunner, migrationRunner]) { + assert.match(runner, /state_directory="\$DEPLOY_PATH\/\.state"/); + assert.match(runner, /state_directory\/mutation\.lock/); + assert.match(runner, /flock -n 9/); + assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("sync-staging-tree.sh")); + assert.ok(runner.indexOf("flock -n 9") < runner.indexOf("docker")); + } +}); + +test("deploy and migration consume the exact successful gate artifact", () => { + const deployment = read(deployWorkflow); + const migration = read(migrationWorkflow); + + for (const workflow of [deployment, migration]) { + assert.match(workflow, /backend-quality-gate\.yml\/runs\?head_sha=/); + assert.match(workflow, /\.head_branch == "staging"/); + assert.match(workflow, /\.event == "push"/); + assert.match(workflow, /\.conclusion == "success"/); + assert.match(workflow, /uses: actions\/download-artifact@v4/); + assert.match(workflow, /run-id: \$\{\{ steps\.revision\.outputs\.gate_run_id \}\}/); + assert.match(workflow, /node frontend\/scripts\/staging-image-manifest\.mjs/); + assert.doesNotMatch(workflow, /jyotisha-(?:api|web):\$[A-Z_]*SHA/); + } +}); + +test("automatic staging paths reject stale and divergent revisions", () => { + const deployment = read(deployWorkflow); + const migration = read(migrationWorkflow); + + assert.match(deployment, /allow_rollback:/); + assert.match(deployment, /rollback authorization is manual-only/); + assert.match(deployment, /stale staging revision refused/); + assert.match(deployment, /compare\/\$previous_sha\.\.\.\$DEPLOY_SHA/); + assert.match(deployment, /\.status == "ahead" and \.merge_base_commit\.sha == \$base/); + assert.match(migration, /stale staging migration refused/); + assert.match(migration, /staging advanced during migration; refusing stale deployment dispatch/); + assert.match(migration, /\{ref:"main",inputs:\{deploy_sha:\$deploy_sha,allow_rollback:"false"\}\}/); +}); + +test("remote deployment verifies running image IDs, RepoDigests, and application SHA", () => { + const runner = read(deployScript); + + assert.match(runner, /docker inspect --format '\{\{\.Image\}\}'/); + assert.match(runner, /docker image inspect --format '\{\{\.Id\}\}' "\$expected_ref"/); + assert.match(runner, /RepoDigests/); + assert.match(runner, /grep -Fqx "\$expected_ref"/); + assert.match(runner, /publicBody\.deployment\?\.gitCommit !== process\.env\.EXPECTED_SHA/); + assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/); + assert.match(runner, /restoring prior image digests/); + assert.doesNotMatch(runner, /jyotisha-(?:api|web):\$DEPLOY_SHA/); +}); + +test("normal deployment checks migrations but never applies them", () => { + const runner = read(deployScript); + assertOrder(runner, [ + "pull api web postgres", + "up -d --no-build --wait postgres", + "--profile migration-check run --rm migration-checker", + "up -d --no-build --remove-orphans", + ]); + assert.match(runner, /pending migrations: run Migrate Staging Database/); + assert.match(runner, /exit 3/); + assert.doesNotMatch(runner, /--profile migration run --rm migrator/); + assert.doesNotMatch(runner, /npm\s+run\s+db:migrate(?!:check)/); +}); + +test("manual migration uses only PostgreSQL and the digest-pinned migrator", () => { + const workflow = read(migrationWorkflow); + const runner = read(migrationScript); + + assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); + assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); + assert.match(runner, /docker pull "\$WEB_IMAGE"/); + assert.match(runner, /-f deploy\/docker-compose\.postgres\.yml/); + assert.match(runner, /--profile migration run --rm migrator/); + assert.match(runner, /select filename from migration\.schema_migrations order by filename/); + assert.doesNotMatch(runner, /docker-compose\.server\.yml/); + assert.doesNotMatch(runner, /\bup\b[^\n]*(?:api|web|caddy)/); +}); + +test("run-local registry state and incoming trees are always cleaned up", () => { + for (const workflow of [read(deployWorkflow), read(migrationWorkflow)]) { + assert.match(workflow, /DOCKER_CONFIG='\$INCOMING_PATH\/\.docker'/); + assert.match(workflow, /if: always\(\) && steps\.incoming\.outputs\.path != ''/); + assert.match(workflow, /docker logout ghcr\.io/); + assert.match(workflow, /rm -rf -- '\$INCOMING_PATH'/); + assert.doesNotMatch(workflow, /--password(?:\s|=)/); + } +}); + +test("production remains manual-only and separate from staging database automation", () => { + const production = readFileSync( + new URL("../../.github/workflows/deploy-production.yml", import.meta.url), + "utf8", + ); + assert.match(production, /^on:\n\s+workflow_dispatch:/m); + assert.doesNotMatch(production, /workflow_run:|\n\s+push:/); + assert.doesNotMatch(production, /docker-compose\.postgres\.yml|db:migrate/); +}); + +test("staging scripts pass shell syntax validation", () => { + for (const script of [deployScript, migrationScript, syncScript]) { + const path = fileURLToPath(script); + chmodSync(path, 0o755); + const result = spawnSync("bash", ["-n", path], { encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); } }); diff --git a/frontend/tests/staging-image-manifest.test.ts b/frontend/tests/staging-image-manifest.test.ts new file mode 100644 index 00000000..2fc9e656 --- /dev/null +++ b/frontend/tests/staging-image-manifest.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { parseStagingImageManifest } from "../scripts/staging-image-manifest.mjs"; + +const gitSha = "0123456789abcdef0123456789abcdef01234567"; +const apiDigest = `sha256:${"a".repeat(64)}`; +const webDigest = `sha256:${"b".repeat(64)}`; + +function validManifest(): string { + return [ + `git_sha=${gitSha}`, + `api_digest=${apiDigest}`, + `web_digest=${webDigest}`, + "", + ].join("\n"); +} + +test("manifest produces immutable GHCR digest references", () => { + assert.deepEqual(parseStagingImageManifest(validManifest(), gitSha), { + gitSha, + apiDigest, + webDigest, + apiImage: `ghcr.io/jesse-ux/jyotisha-api@${apiDigest}`, + webImage: `ghcr.io/jesse-ux/jyotisha-web@${webDigest}`, + }); +}); + +test("manifest rejects revision drift, mutable tags, duplicates, extras, and malformed digests", () => { + const invalid = [ + validManifest().replace(gitSha, "f".repeat(40)), + validManifest().replace(apiDigest, `${gitSha}`), + validManifest().replace(apiDigest, `sha256:${"A".repeat(64)}`), + validManifest().replace( + `web_digest=${webDigest}`, + `api_digest=${apiDigest}`, + ), + `${validManifest()}extra=value\n`, + validManifest().replace("api_digest=", "api_image=ghcr.io/example:"), + ]; + + for (const contents of invalid) { + assert.throws(() => parseStagingImageManifest(contents, gitSha)); + } +});