From 3b81fa9689ebc483f964636608ce92b543998be2 Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Mon, 20 Jul 2026 15:23:24 +0800 Subject: [PATCH] fix: keep deployment traffic on ready web --- .github/workflows/deploy-production.yml | 8 ++++++-- deploy/Caddyfile | 5 ++++- deploy/docker-compose.server.yml | 10 +++++++++- frontend/tests/health-deployment.test.ts | 24 ++++++++++++++++++++++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index c5f65174..9bb6d544 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout tested revision uses: actions/checkout@v4 with: - ref: ${{ github.sha }} + ref: ${{ github.event.workflow_run.head_sha || github.sha }} - name: Configure SSH env: @@ -38,7 +38,7 @@ jobs: - name: Sync and rebuild env: - DEPLOY_GIT_SHA: ${{ github.sha }} + DEPLOY_GIT_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} run: | SSH_OPTIONS="-i $HOME/.ssh/jyotisha-production -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" RSYNC_SSH="ssh $SSH_OPTIONS" @@ -54,9 +54,13 @@ jobs: "cd '$DEPLOY_PATH' && GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build --remove-orphans" - name: Verify production + env: + DEPLOY_GIT_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} run: | curl --fail --silent --show-error --retry 12 --retry-delay 5 https://jyotisha.chat/login >/dev/null test "$(curl --silent --output /dev/null --write-out '%{http_code}' https://jyotisha.chat/api/account)" = "401" + curl --fail --silent --show-error --retry 12 --retry-delay 5 https://jyotisha.chat/api/health | \ + node -e 'const body = JSON.parse(require("node:fs").readFileSync(0, "utf8")); if (body.deployment?.gitCommit !== process.env.DEPLOY_GIT_SHA) process.exit(1); console.log(JSON.stringify(body));' ssh -i ~/.ssh/jyotisha-production -p "$DEPLOY_PORT" \ -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20 \ "$DEPLOY_USER@$DEPLOY_HOST" \ diff --git a/deploy/Caddyfile b/deploy/Caddyfile index a686b6fe..baca242f 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -1,6 +1,9 @@ {$SITE_ADDRESS:https://jyotisha.chat} { encode zstd gzip - reverse_proxy web:3000 + reverse_proxy web:3000 { + lb_try_duration 10s + lb_try_interval 250ms + } } www.jyotisha.chat { diff --git a/deploy/docker-compose.server.yml b/deploy/docker-compose.server.yml index caa615f6..d14cf5ec 100644 --- a/deploy/docker-compose.server.yml +++ b/deploy/docker-compose.server.yml @@ -31,6 +31,13 @@ services: JYOTISH_API_BASE: http://api:5200 expose: - "3000" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/api/health').then(r=>{if(!r.ok)process.exit(1)})"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 30s + start_interval: 1s depends_on: api: condition: service_healthy @@ -49,7 +56,8 @@ services: - caddy_data:/data - caddy_config:/config depends_on: - - web + web: + condition: service_healthy volumes: caddy_data: diff --git a/frontend/tests/health-deployment.test.ts b/frontend/tests/health-deployment.test.ts index 734f1d51..887b0110 100644 --- a/frontend/tests/health-deployment.test.ts +++ b/frontend/tests/health-deployment.test.ts @@ -2,6 +2,12 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; +function serviceBlock(compose: string, service: string) { + const match = compose.match(new RegExp(`^ ${service}:\\n([\\s\\S]*?)(?=^ [a-z][a-z0-9_-]*:|^volumes:)`, "m")); + assert.ok(match, `expected ${service} service in compose file`); + return match[1]; +} + test("health endpoint exposes deployment identity for production verification", () => { const source = readFileSync(new URL("../src/app/api/health/route.ts", import.meta.url), "utf8"); @@ -11,11 +17,25 @@ test("health endpoint exposes deployment identity for production verification", assert.match(source, /gitCommit/); }); -test("production deployment passes the tested revision into the web runtime", () => { +test("production traffic waits for a healthy web container and retries short replacement gaps", () => { const compose = readFileSync(new URL("../../deploy/docker-compose.server.yml", import.meta.url), "utf8"); + const caddyfile = readFileSync(new URL("../../deploy/Caddyfile", import.meta.url), "utf8"); + const web = serviceBlock(compose, "web"); + const caddy = serviceBlock(compose, "caddy"); + + assert.match(web, /GITHUB_SHA: \$\{GITHUB_SHA\}/); + assert.match(web, /healthcheck:\n\s+test: \["CMD", "node", "-e", "fetch\('http:\/\/127\.0\.0\.1:3000\/api\/health'\)\.then\(r=>\{if\(!r\.ok\)process\.exit\(1\)\}\)"\]/); + assert.match(web, /start_period: 30s/); + assert.match(web, /start_interval: 1s/); + assert.match(caddy, /web:\n\s+condition: service_healthy/); + assert.match(caddyfile, /reverse_proxy web:3000 \{\n\s+lb_try_duration 10s\n\s+lb_try_interval 250ms\n\s+\}/); +}); + +test("production verification accepts only the SHA exposed by the deployed health endpoint", () => { const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8"); - assert.match(compose, /GITHUB_SHA: \$\{GITHUB_SHA\}/); assert.match(workflow, /DEPLOY_GIT_SHA: \$\{\{ github\.event\.workflow_run\.head_sha \|\| github\.sha \}\}/); assert.match(workflow, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/); + assert.match(workflow, /curl --fail --silent --show-error --retry 12 --retry-delay 5 https:\/\/jyotisha\.chat\/api\/health/); + assert.match(workflow, /body\.deployment\?\.gitCommit !== process\.env\.DEPLOY_GIT_SHA/); });