fix(staging): refresh dual-host edge verification

Merge reviewed bounded staging public identity convergence fix after successful PR gate 1519.

deploy/run-staging-deploy.sh

@@ -177,6 +177,7 @@ trap rollback ERR
 
 switched=true
 "${compose[@]}" up -d --no-build --remove-orphans
+"${compose[@]}" up -d --no-build --force-recreate --no-deps caddy
 
 verify_container_image() {
   local service="$1"
@@ -198,31 +199,60 @@ verify_container_image rectification-v4-worker "$WEB_IMAGE"
   -e EXPECTED_SHA="$DEPLOY_SHA" -e STAGING_URL="$STAGING_URL" \
   web node --input-type=module <<'NODE'
 const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-let login;
-for (let attempt = 0; attempt < 12; attempt += 1) {
+let observed = {};
+for (let attempt = 1; attempt <= 12; attempt += 1) {
   try {
-    login = await fetch(`${process.env.STAGING_URL}/login`);
-    if (login.ok) break;
-  } catch {}
-  await delay(5_000);
-}
-if (!login?.ok) process.exit(1);
-const adminPage = await fetch(`${process.env.STAGING_URL}/admin`, { redirect: "manual" });
-if (adminPage.status !== 307 || adminPage.headers.get("location") !== "/login") process.exit(1);
-const adminApi = await fetch(`${process.env.STAGING_URL}/api/admin/session`);
-if (adminApi.status !== 401) process.exit(1);
-const account = await fetch(`${process.env.STAGING_URL}/api/account`);
-if (account.status !== 401) process.exit(1);
-const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`);
-const publicBody = await publicHealth.json();
-if (!publicHealth.ok || publicBody.deployment?.gitCommit !== process.env.EXPECTED_SHA) {
-  process.exit(1);
-}
-const privateHealth = await fetch("http://api:5200/api/health");
-const privateBody = await privateHealth.json();
-if (!privateHealth.ok || privateBody.status !== "ok" || privateBody.swisseph_available !== true) {
-  process.exit(1);
+    const login = await fetch(`${process.env.STAGING_URL}/login`);
+    const userAdminPage = await fetch(`${process.env.STAGING_URL}/admin`, { redirect: "manual" });
+    const userAdminApi = await fetch(`${process.env.STAGING_URL}/api/admin/session`);
+    const adminPage = await fetch(`${process.env.ADMIN_USER_ORIGIN}/admin`, { redirect: "manual" });
+    const adminApi = await fetch(`${process.env.ADMIN_USER_ORIGIN}/api/admin/session`);
+    const account = await fetch(`${process.env.STAGING_URL}/api/account`);
+    const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`);
+    const publicBody = await publicHealth.json();
+    const privateHealth = await fetch("http://api:5200/api/health");
+    const privateBody = await privateHealth.json();
+    observed = {
+      attempt,
+      login: login.status,
+      userAdminPage: userAdminPage.status,
+      userAdminApi: userAdminApi.status,
+      adminPage: adminPage.status,
+      adminLocation: adminPage.headers.get("location"),
+      adminApi: adminApi.status,
+      account: account.status,
+      publicHealth: publicHealth.status,
+      publicSha: publicBody.deployment?.gitCommit ?? "missing",
+      privateHealth: privateHealth.status,
+      privateStatus: privateBody.status ?? "missing",
+      swissephAvailable: privateBody.swisseph_available === true,
+    };
+    if (
+      login.ok
+      && userAdminPage.status === 404
+      && userAdminApi.status === 404
+      && adminPage.status === 307
+      && adminPage.headers.get("location") === "/login"
+      && adminApi.status === 401
+      && account.status === 401
+      && publicHealth.ok
+      && publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA
+      && privateHealth.ok
+      && privateBody.status === "ok"
+      && privateBody.swisseph_available === true
+    ) {
+      process.exit(0);
+    }
+  } catch (error) {
+    observed = {
+      attempt,
+      error: error instanceof Error ? error.name : "verification_error",
+    };
+  }
+  if (attempt < 12) await delay(5_000);
 }
+console.error("staging verification predicates did not converge", JSON.stringify(observed));
+process.exit(1);
 NODE
 
 revision_file="$state_directory/deployed-revision.tmp.$$"

docs/BUG_HISTORY.md

@@ -2352,3 +2352,35 @@
 - 相关记录：BUG-129、BUG-131、ERR-096、ERR-099
 - 复发自：无
 - 修复版本：待 frontend build bounded timeout 与 staging exact-SHA 验收
+
+## BUG-137 | staging deploy 在公网 upstream 尚未收敛时用旧 SHA 立即判失败
+
+- 状态：investigating
+- 首次发现：2026-08-06
+- 最近更新：2026-08-06
+- 影响面：Gitea `Deploy staging` 最终公网验证、自动回滚；数据库迁移已成功，production 未受影响。
+- 用户现象：exact-SHA gate `1514` 与 migration `1516` 成功后，deploy `1517`、`1518` 均启动目标 web/API image 并达到容器 healthy，却在约 3 秒后的公网 verification 返回非零，随后成功恢复旧 web/worker；公网和 `.state/deployed-revision` 均保持旧 SHA `e59f15d352787f3d05425ba8c459d092e9801a20`。
+- 触发条件：Compose 切换到目标容器后，旧 Caddy upstream 在短暂收敛窗口仍可让 `/login` 返回 200；脚本只轮询 `/login`，然后对公网 health SHA 和其余 predicate 仅检查一次，读取旧 SHA 时立即触发回滚。
+- 根因：发布验证把“login 可达”和“公网已路由到 exact SHA”拆成了不对称检查；容器健康与代理 upstream 收敛不是同一时刻，单次 SHA 检查形成确定性 race。目标 image 隔离 probe 已确认注入的 `GITHUB_SHA` 为目标 SHA。
+- 修复：在原 60 秒总预算内，每 5 秒原子重查 login、admin 未登录重定向、admin API/account 401、公网 health exact SHA、私有 API health 和 Swiss Ephemeris；仅当所有 predicate 同轮满足才成功。预算耗尽仍 fail closed 并只输出状态码、observed SHA、health 状态等脱敏摘要，不输出正文、env 或凭据。
+- 验证：待合同测试、PR gate、新 reviewed exact-SHA push gate、immutable artifact、migration check 与真实 staging deploy 验收；完成前不得标记 resolved。
+- 防复发：发布验证必须等待最终外部路由 identity，而不能把单个 readiness endpoint 当作代理收敛证明；所有重试必须有总上限，失败记录仅含非敏感 predicate 状态并保持自动回滚。
+- 相关记录：BUG-129、BUG-136、ERR-099、ERR-100
+- 复发自：无
+- 修复版本：待 staging verification convergence gate 与 exact-SHA deploy 验收
+
+## BUG-138 | staging Caddy 保留旧单文件 bind inode 且 admin 验证误走用户域名
+
+- 状态：investigating
+- 首次发现：2026-08-06
+- 最近更新：2026-08-06
+- 影响面：staging Caddy 双 host 路由、admin TLS、`Deploy staging` 未登录边界验证；production 未受影响。
+- 用户现象：修复公网 SHA 收敛后，deploy `1522` 明确观测目标 SHA、私有 API 和 Swiss Ephemeris 均正常，但在用户域名上得到 `/admin -> 307 /` 与 `/api/admin/session -> 403`；同时 `admin.staging.jyotisha.chat` TLS 握手失败。workflow 在 60 秒后自动恢复旧应用，未写入新 deployed-revision。
+- 触发条件：controller 通过原子目录同步替换 `deploy/Caddyfile.staging`，但长期运行的 Caddy 容器仍持有旧单文件 bind mount inode；随后 checker 又把 admin 页面/API 请求错误发送到 `STAGING_URL` 而不是 `ADMIN_USER_ORIGIN`。
+- 根因：host 文件与 Caddy 容器 mount inode 漂移。只读现场证据显示 host Caddyfile 含 admin host、运行容器内文件不含，inode/size/mtime 均不同；staging VPS 从两台权威 nameserver 查询 admin A 记录均为 `118.26.111.127`，排除 DNS 缺失。用户域名按新 Caddy 合同本应对 admin paths 404，因此旧 checker 的 307/401 期待也违反双 host 边界。
+- 修复：应用 Compose 切换后显式 `--force-recreate --no-deps caddy`，使其重新挂载 gate-attested Caddyfile；完整 convergence 同轮要求用户域名 admin page/API 均 404、admin origin page 307 到 `/login`、admin API 401，并继续要求 exact public SHA、account 401、私有 API/Swiss 健康。失败仍 bounded、fail closed 并自动恢复旧应用。
+- 验证：待 shell/workflow 合同、PR gate、新 exact-SHA push artifact、真实 Caddy admin 证书/路由和 deploy 验收；完成前不得标记 resolved。
+- 防复发：原子替换单文件 bind mount 后必须 recreate/reload 长期运行服务；部署 smoke 必须分别使用各自主机 origin，不能在 user host 上测试 admin host 合同。保留 authority DNS、mount inode 和 TLS 检查作为脱敏现场证据。
+- 相关记录：BUG-134、BUG-137、ERR-097、ERR-100、ERR-101
+- 复发自：无
+- 修复版本：待 Caddy mount refresh 与双 host exact-SHA staging 验收

docs/research/pre_work_error_ledger.md

@@ -175,6 +175,18 @@ Staging push gate `1507` completed all frontend tests (`1472/1472`), lint with z
 
 Prevention: wrap the Gitea frontend production build in a command-level 600-second timeout with an explicit nonzero failure; retain the 45-minute job timeout as a second boundary. Never skip the build, reuse an old artifact, or treat a manual validation-only run as a successful staging push gate. Only a successful exact-SHA push gate may publish images and trigger deployment.
 
+## ERR-100 | Staging public verification checked exact SHA only once before proxy convergence | investigating 2026-08-06
+
+Exact-SHA gate `1514` and migration `1516` succeeded, but deploy runs `1517` and `1518` each reached healthy target web/API containers and then failed public verification roughly three seconds later. Both runs restored the prior application image and retained the old deployed-revision state. The verifier retried only `/login`; once that endpoint returned 200 through the existing Caddy route, it performed exactly one check of public health SHA and the remaining authorization/private-health predicates. This permits an old upstream response to trigger immediate rollback during container/proxy convergence even though the new container itself carries the expected SHA. No application release completed and production was not involved.
+
+Prevention: within the existing bounded 60-second budget, retry the complete predicate set together: login, admin redirect, logged-out admin/account responses, public exact SHA, private API status, and Swiss Ephemeris availability. Success requires every predicate in the same attempt. On exhaustion, fail closed and emit only redacted status codes, observed SHA, and health booleans; never log response bodies, env, credentials, or error stacks.
+
+## ERR-101 | Atomic Caddyfile replacement left the running Caddy bind-mounted to the old inode | investigating 2026-08-06
+
+The bounded verifier in deploy `1522` proved that public health had converged to the target exact SHA and private API/Swiss checks passed, but admin checks on the user host returned the expected application fail-closed statuses rather than admin-host statuses. Read-only host inspection then found the reviewed host `deploy/Caddyfile.staging` contained the admin virtual host while `/etc/caddy/Caddyfile` inside the long-running Caddy container did not; their inode, size, and mtime differed even though Docker reported the expected bind source. The tree sync atomically replaced the source file, leaving the existing bind mount attached to the old inode. Both authoritative Spaceship nameservers returned the staging VPS address for the admin host when queried from the VPS, so this was not an absent DNS record. The failed deploy restored the prior application and did not advance deployed-revision.
+
+Prevention: after syncing a gate-attested single-file bind mount, force-recreate Caddy under the shared host lock before public verification. Verify user and admin origins separately: user-host admin paths must be 404; admin-host anonymous page/API must be 307-to-login and 401. Preserve bounded convergence, exact public SHA, private health, automatic rollback, and redacted diagnostics. Do not weaken identity host routing or modify DNS based on intercepted local resolver results.
+
 ## Fragment Sweep Command Set
 
 ## ERR-086 | Steve Jobs jyotishganit artifacts used non-San-Francisco coordinates | mitigated 2026-07-21

frontend/tests/staging-backend-workflows.test.ts

@@ -644,7 +644,14 @@ test("remote deployment verifies running image IDs, RepoDigests, and application
   assert.match(runner, /"\$\{docker_command\[@\]\}" image inspect --format '\{\{\.Id\}\}' "\$expected_ref"/);
   assert.match(runner, /RepoDigests/);
   assert.match(runner, /grep -Fqx "\$expected_ref"/);
-  assert.match(runner, /publicBody\.deployment\?\.gitCommit !== process\.env\.EXPECTED_SHA/);
+  assert.match(runner, /for \(let attempt = 1; attempt <= 12; attempt \+= 1\)/);
+  assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/);
+  assert.match(runner, /staging verification predicates did not converge/);
+  assert.match(runner, /publicSha: publicBody\.deployment\?\.gitCommit \?\? "missing"/);
+  assert.match(runner, /up -d --no-build --force-recreate --no-deps caddy/);
+  assert.match(runner, /userAdminPage\.status === 404/);
+  assert.match(runner, /userAdminApi\.status === 404/);
+  assert.match(runner, /fetch\(`\$\{process\.env\.ADMIN_USER_ORIGIN\}\/admin`/);
   assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/);
   assert.match(runner, /restoring prior application images/);
   assert.match(
@@ -771,9 +778,14 @@ test("normal deployment checks migrations but never applies them", () => {
   assert.doesNotMatch(runner, /npm\s+run\s+db:migrate(?!:check)/);
   assert.doesNotMatch(runner, /pull api web postgres/);
   assert.match(runner, /verify_container_image rectification-v4-worker \"\$WEB_IMAGE\"/);
-  assert.match(runner, /adminPage\.status !== 307/);
-  assert.match(runner, /adminPage\.headers\.get\("location"\) !== "\/login"/);
-  assert.match(runner, /adminApi\.status !== 401/);
+  assertOrder(runner, [
+    "up -d --no-build --remove-orphans",
+    "up -d --no-build --force-recreate --no-deps caddy",
+    "verify_container_image api",
+  ]);
+  assert.match(runner, /adminPage\.status === 307/);
+  assert.match(runner, /adminPage\.headers\.get\("location"\) === "\/login"/);
+  assert.match(runner, /adminApi\.status === 401/);
 });
 
 test("staging runs the rectification V4 worker from the immutable web image", () => {