Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 91a39f2528 | |||
| 6c1dcbe857 | |||
| e718b2c6d9 | |||
| 28f4207f29 | |||
| 3221b6a9a4 | |||
| 8dc61e3135 |
@@ -177,6 +177,7 @@ trap rollback ERR
|
||||
|
||||
switched=true
|
||||
"${compose[@]}" up -d --no-build --remove-orphans
|
||||
"${compose[@]}" up -d --no-build --force-recreate --no-deps caddy
|
||||
|
||||
verify_container_image() {
|
||||
local service="$1"
|
||||
@@ -198,31 +199,60 @@ verify_container_image rectification-v4-worker "$WEB_IMAGE"
|
||||
-e EXPECTED_SHA="$DEPLOY_SHA" -e STAGING_URL="$STAGING_URL" \
|
||||
web node --input-type=module <<'NODE'
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
let login;
|
||||
for (let attempt = 0; attempt < 12; attempt += 1) {
|
||||
let observed = {};
|
||||
for (let attempt = 1; attempt <= 12; attempt += 1) {
|
||||
try {
|
||||
login = await fetch(`${process.env.STAGING_URL}/login`);
|
||||
if (login.ok) break;
|
||||
} catch {}
|
||||
await delay(5_000);
|
||||
}
|
||||
if (!login?.ok) process.exit(1);
|
||||
const adminPage = await fetch(`${process.env.STAGING_URL}/admin`, { redirect: "manual" });
|
||||
if (adminPage.status !== 307 || adminPage.headers.get("location") !== "/login") process.exit(1);
|
||||
const adminApi = await fetch(`${process.env.STAGING_URL}/api/admin/session`);
|
||||
if (adminApi.status !== 401) process.exit(1);
|
||||
const account = await fetch(`${process.env.STAGING_URL}/api/account`);
|
||||
if (account.status !== 401) process.exit(1);
|
||||
const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`);
|
||||
const publicBody = await publicHealth.json();
|
||||
if (!publicHealth.ok || publicBody.deployment?.gitCommit !== process.env.EXPECTED_SHA) {
|
||||
process.exit(1);
|
||||
}
|
||||
const privateHealth = await fetch("http://api:5200/api/health");
|
||||
const privateBody = await privateHealth.json();
|
||||
if (!privateHealth.ok || privateBody.status !== "ok" || privateBody.swisseph_available !== true) {
|
||||
process.exit(1);
|
||||
const login = await fetch(`${process.env.STAGING_URL}/login`);
|
||||
const userAdminPage = await fetch(`${process.env.STAGING_URL}/admin`, { redirect: "manual" });
|
||||
const userAdminApi = await fetch(`${process.env.STAGING_URL}/api/admin/session`);
|
||||
const adminPage = await fetch(`${process.env.ADMIN_USER_ORIGIN}/admin`, { redirect: "manual" });
|
||||
const adminApi = await fetch(`${process.env.ADMIN_USER_ORIGIN}/api/admin/session`);
|
||||
const account = await fetch(`${process.env.STAGING_URL}/api/account`);
|
||||
const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`);
|
||||
const publicBody = await publicHealth.json();
|
||||
const privateHealth = await fetch("http://api:5200/api/health");
|
||||
const privateBody = await privateHealth.json();
|
||||
observed = {
|
||||
attempt,
|
||||
login: login.status,
|
||||
userAdminPage: userAdminPage.status,
|
||||
userAdminApi: userAdminApi.status,
|
||||
adminPage: adminPage.status,
|
||||
adminLocation: adminPage.headers.get("location"),
|
||||
adminApi: adminApi.status,
|
||||
account: account.status,
|
||||
publicHealth: publicHealth.status,
|
||||
publicSha: publicBody.deployment?.gitCommit ?? "missing",
|
||||
privateHealth: privateHealth.status,
|
||||
privateStatus: privateBody.status ?? "missing",
|
||||
swissephAvailable: privateBody.swisseph_available === true,
|
||||
};
|
||||
if (
|
||||
login.ok
|
||||
&& userAdminPage.status === 404
|
||||
&& userAdminApi.status === 404
|
||||
&& adminPage.status === 307
|
||||
&& adminPage.headers.get("location") === "/login"
|
||||
&& adminApi.status === 401
|
||||
&& account.status === 401
|
||||
&& publicHealth.ok
|
||||
&& publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA
|
||||
&& privateHealth.ok
|
||||
&& privateBody.status === "ok"
|
||||
&& privateBody.swisseph_available === true
|
||||
) {
|
||||
process.exit(0);
|
||||
}
|
||||
} catch (error) {
|
||||
observed = {
|
||||
attempt,
|
||||
error: error instanceof Error ? error.name : "verification_error",
|
||||
};
|
||||
}
|
||||
if (attempt < 12) await delay(5_000);
|
||||
}
|
||||
console.error("staging verification predicates did not converge", JSON.stringify(observed));
|
||||
process.exit(1);
|
||||
NODE
|
||||
|
||||
revision_file="$state_directory/deployed-revision.tmp.$$"
|
||||
|
||||
+50
-18
@@ -2259,7 +2259,7 @@
|
||||
|
||||
## BUG-131 | staging quality gate exact-SHA checkout 因过严低速阈值单次失败
|
||||
|
||||
- 状态:investigating
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:Gitea `Staging Backend Quality Gate` validate/publish 的 exact-SHA checkout;staging mutation controller、应用数据面与 production 未受影响。
|
||||
@@ -2267,15 +2267,15 @@
|
||||
- 触发条件:quality gate 的 exact-SHA `--depth=1` fetch 只有单次调用,并把低速失败设为连续 30 秒低于 1024 B/s;当前 Gitea 链路在约 20 KiB/s 波动后短时低于阈值。
|
||||
- 根因:`BUG-129` 消除了 mutation workflow 的 Git object fetch,但 quality gate 自身仍必须取得待测源码;其 checkout 没有 bounded retry,且低速阈值对当前受限链路过严。旧测试只断言 exact SHA/clean tree,没有覆盖 checkout retry 与低速边界。
|
||||
- 修复:validate/publish 两处 exact-SHA checkout 均改为最多 3 次、每次 hard timeout 300 秒;保留 connect timeout 15 秒,将低速失败收紧为连续 60 秒低于 1 B/s。每次仍只抓 `--depth=1 --no-tags origin "$GITEA_SHA"`,耗尽后明确 fail closed,不复用旧 artifact、不放宽 exact-SHA 或 clean-tree 校验。
|
||||
- 验证:过期基线 PR gate `1488` 成功;待在最新并发主线上完成本地 workflow/YAML/run-script/pre-work 回归、完整 PR gate、staging push gate 和自动 deploy,完成前不得标记 resolved。
|
||||
- 验证:过期基线 PR gate `1488`、最新 controller PR gates `1519`/`1523`、staging push gate `1525` 均成功;最终 gate 对 exact SHA `6c1dcbe857006ec6ae7463b57b2b7d5947da4851` 完成 validate/publish,artifact ID `12` 成功上传,deploy `1526` 成功。
|
||||
- 防复发:质量门禁和 mutation controller 的网络边界分别测试;quality gate checkout 必须覆盖 attempt 数、hard timeout、低速阈值、exact-SHA refspec、最终错误和 clean-tree identity。
|
||||
- 相关记录:BUG-129、ERR-095、ERR-096
|
||||
- 复发自:无;属于同一 Gitea 链路在 quality-gate 阶段的独立缺口
|
||||
- 修复版本:待 bounded exact-SHA gate checkout 与 staging 验收
|
||||
- 修复版本:`6c1dcbe857006ec6ae7463b57b2b7d5947da4851`(最终 staging 验收)
|
||||
|
||||
## BUG-132 | 新增后台页面未同步能力审计精确路由集合
|
||||
|
||||
- 状态:investigating
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:`tests/test_api_server_security.py::test_capability_audit_scans_registry_and_local_sources`、Gitea staging quality gate;新增后台页面实现本身未由本记录改动。
|
||||
@@ -2283,15 +2283,15 @@
|
||||
- 触发条件:新增 administrators、audit logs、consultations、credit transactions、customers、feature flags、model releases、models、orders、products、roles、security、subscriptions、usage 页面后执行 capability audit 精确集合回归。
|
||||
- 根因:并发后台功能更新了真实 App Router 页面,却未同步能力审计的完整预期集合;这是 `BUG-126` 同类防复发模式在后台模块复发,说明新增页面的同变更门禁仍未统一执行。
|
||||
- 修复:保留精确集合比较,将实际新增的 14 个后台页面按排序加入预期列表;不删除既有个人报告页,不改成子集或数量下限,不修改并发后台业务实现。
|
||||
- 验证:待本地目标测试、完整 Gitea PR gate、staging push gate;完成前不得标记 resolved。
|
||||
- 验证:本地 staging controller/contracts 35/35、完整 Gitea PR gates `1503`/`1505`/`1519`/`1523`、最终 staging push gate `1525` 与 deploy `1526` 均成功。
|
||||
- 防复发:任何 `frontend/src/app/**/page.tsx` 新增或删除必须在同一提交更新 capability audit 精确路由集合,且 quality gate 失败不得通过放宽断言绕过。
|
||||
- 相关记录:BUG-126、BUG-131
|
||||
- 复发自:BUG-126
|
||||
- 修复版本:待能力审计同步与 staging 验收
|
||||
- 修复版本:`6c1dcbe857006ec6ae7463b57b2b7d5947da4851`(最终 staging 验收)
|
||||
|
||||
## BUG-133 | admin users Route Handler 重导出 runtime 导致 production build 失败
|
||||
|
||||
- 状态:investigating
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:`frontend/src/app/api/admin/users/route.ts`、Next.js production build、Gitea staging quality gate;customer handler 权限和业务逻辑未受改动。
|
||||
@@ -2299,15 +2299,15 @@
|
||||
- 触发条件:legacy `/api/admin/users` Route Handler 通过 `export { ..., runtime } from "../customers/route"` 同时重导出 handlers 和 route segment config。
|
||||
- 根因:Next.js 要求 `runtime` 等 route segment config 在当前 route 文件中可被静态解析,不允许从另一 Route Handler 重导出;既有精确合同测试反而固化了非法 re-export,且并发功能本地 production build 未闭环。
|
||||
- 修复:在 users route 本文件静态声明 `export const runtime = "nodejs"`,只重导出 DELETE/GET/PATCH/POST/PUT handlers;不复制 handler、不修改权限或客户数据逻辑。合同测试改为强制本地 runtime 常量并拒绝 runtime re-export。
|
||||
- 验证:admin users 目标合同、`tsc --noEmit`、默认 Turbopack `next build`、`next build --webpack`、全 route segment config re-export 扫描与 `git diff --check` 已通过;Gitea PR gate `1499` 在 SHA `015f1e501e2132d2abc6bbbcb8e6af074c326b47` 上通过并完成 production build。待最新并发主线 staging gate 与部署验收,完成前不得标记 resolved。
|
||||
- 验证:admin users 目标合同、`tsc --noEmit`、默认 Turbopack `next build`、`next build --webpack`、全 route segment config re-export 扫描与 `git diff --check` 已通过;Gitea PR gate `1499` 及最终 PR/push gates `1523`/`1525` 均完成 production build,deploy `1526` 成功。
|
||||
- 防复发:Route Handler 的 `runtime`、`dynamic`、`revalidate` 等 segment config 必须本地静态声明;handler 可复用,但 segment config 不得 re-export。新增 alias route 必须经过 production build,而不只运行文本合同测试。
|
||||
- 相关记录:BUG-131、BUG-132
|
||||
- 复发自:无
|
||||
- 修复版本:待 Route Handler 静态 config 修复与 staging 验收
|
||||
- 修复版本:`6c1dcbe857006ec6ae7463b57b2b7d5947da4851`(最终 staging 验收)
|
||||
|
||||
## BUG-134 | staging admin origin selector 未配置导致 exact-SHA 自动部署失败
|
||||
|
||||
- 状态:investigating
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:staging `.env.staging` 的公开 identity selector、自动 deploy run `1502`;应用容器、业务数据库和 production 未被修改。
|
||||
@@ -2315,15 +2315,15 @@
|
||||
- 触发条件:包含双 host self-hosted identity validator 的 controller 部署到现有 staging host,而 `.env.staging` 尚未包含精确且唯一的 `ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat`。
|
||||
- 根因:并发 admin rollout 将 admin host origin 加入应用和 validator 合同,但 staging host-managed env 未在发布前同步新增的非密钥 selector;quality gate 验证仓库合同,不读取主机 secret/env,因此直到 mutation 前远端校验才暴露漂移。
|
||||
- 修复:已在共享 staging mutation lock 下,仅向原文件原子补入公开 `ADMIN_USER_ORIGIN` selector,保留全部既有内容、`deploy:deploy` owner 和 `0600` mode;未输出、复制或重写其他 secret 值。随后完整 validator 暴露独立的 service runtime 漂移,转由 `BUG-135` 处理;仍待 exact-SHA artifact 重新部署。
|
||||
- 验证:脱敏只读检查先确认 `.env.staging` 为 `deploy:deploy 0600`、`AUTH_USER_ORIGIN` 精确且唯一、`ADMIN_USER_ORIGIN` 计数为 0;原子修复后 `ADMIN_USER_ORIGIN` 精确且唯一,正式 staging env validator 已通过。待最新 staging gate、exact-SHA deploy、公网/host SHA、容器 restart、日志和未登录边界验收。
|
||||
- 验证:脱敏只读检查先确认 `.env.staging` 为 `deploy:deploy 0600`、`AUTH_USER_ORIGIN` 精确且唯一、`ADMIN_USER_ORIGIN` 计数为 0;原子修复后 `ADMIN_USER_ORIGIN` 精确且唯一,正式 validators 与 deploy `1526` 通过。最终 user host admin paths 为 404;admin host `/admin` 为 307 `/login`、session API 为 401;容器 restart count 均为 0。
|
||||
- 防复发:任何新增 staging host-managed selector 必须在同一 rollout runbook 中包含 deploy 前 presence/exact-value 检查;quality gate 成功不能替代 host env validation。env 修复必须共享 mutation lock、原子替换并保持 owner/mode,严禁打印 raw env。
|
||||
- 相关记录:BUG-128、BUG-133、ERR-093、ERR-097
|
||||
- 复发自:无
|
||||
- 修复版本:待 staging env selector 对齐与 exact-SHA 部署验收
|
||||
- 修复版本:`6c1dcbe857006ec6ae7463b57b2b7d5947da4851`(staging host selector 与双 host 验收)
|
||||
|
||||
## BUG-135 | staging service runtime 缺失且 admin runtime 错误继承 BYPASSRLS 角色
|
||||
|
||||
- 状态:investigating
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:staging 私有 PostgreSQL runtime roles、`.env.staging`、`.env.staging.database`、个人报告 service client 与后台最小权限;production 未受影响。
|
||||
@@ -2331,15 +2331,15 @@
|
||||
- 触发条件:在早期初始化的 staging 数据卷上部署依赖独立 service client 和最新 admin RBAC 的应用;bootstrap 脚本只在空数据卷初始化时执行,现有 host env/roles 未随 reviewed compatibility contract 对齐。
|
||||
- 根因:staging host bootstrap 漂移。数据库保留了 `service_role`、identity/app/admin runtime roles,但没有后来合同要求的 `service_runtime`;旧 admin runtime membership 又违反当前 bootstrap 和 RBAC 的明确 revoke 边界。quality gate 不读取 host env 或运行时 role catalog,因此直到远端部署前校验与现场权限审计才暴露。
|
||||
- 修复:在共享 mutation lock 下生成独立 staging-only 随机凭据,通过 PostgreSQL stdin 创建/设置 `service_runtime`,授予 `service_role` membership 和数据库 CONNECT;将 raw password 仅原子写入 `.env.staging.database`,percent-encoded URL 仅原子写入 `.env.staging`,两文件保持 `deploy:deploy 0600`。随后撤销 `admin_runtime` 的 `service_role` membership;未重启容器、未输出凭据、未改 production。已应用的 `20260806000000_personal_reports.sql` checksum 与仓库一致,保持历史迁移不可变。
|
||||
- 验证:两个正式 env validator 均通过;`service_runtime` 真实密码登录、`service_role` membership 和 CONNECT 均通过;`admin_runtime` membership=false,`service_runtime` membership=true;两份 service 配置各精确 1 条且非空,owner/mode 保持正确。PR gate `1503` 已通过;待最终合并 SHA 的 staging gate、migration check、exact-SHA deploy、service client smoke、RLS/容器/日志验收,完成前不得标记 resolved。
|
||||
- 验证:两个正式 env validator 均通过;`service_runtime` 真实密码登录、`service_role` membership 和 CONNECT 均通过;最终审计再次确认 `service_membership=true`、`service_connect=true`、`admin_service_membership=false`、`admin_bypassrls=false`、`service_can_login=true`。两份 env 均为 `deploy:deploy 0600`,migration `1516`、push gate `1525` 和 deploy `1526` 成功。
|
||||
- 防复发:非空数据卷不能依赖 `/docker-entrypoint-initdb.d` 自动重放;每次新增 runtime role 或 host-managed URL 都必须有兼容性 role repair、脱敏 pre-deploy presence 检查和真实登录/role-membership smoke。`admin_runtime` 永不得继承 `service_role`;service writes 只能使用独立 `SERVICE_DATABASE_URL`。已应用迁移不得为修正文案而改 checksum。
|
||||
- 相关记录:BUG-128、BUG-134、ERR-093、ERR-098
|
||||
- 复发自:无
|
||||
- 修复版本:待 staging service runtime 对齐与 exact-SHA 部署验收
|
||||
- 修复版本:`6c1dcbe857006ec6ae7463b57b2b7d5947da4851`(staging role/env 与 exact-SHA 验收)
|
||||
|
||||
## BUG-136 | staging quality gate frontend build 无界卡住并耗尽 45 分钟 job
|
||||
|
||||
- 状态:investigating
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:Gitea `Staging Backend Quality Gate` validate job、staging artifact publication;应用代码、staging host 和 production 未被本次失败修改。
|
||||
@@ -2347,8 +2347,40 @@
|
||||
- 触发条件:质量门禁执行 `npm run build --prefix frontend` 没有命令级 bounded timeout;Turbopack 在编译后静态生成/收尾阶段无输出卡住时只能等待 job-level timeout。
|
||||
- 根因:quality gate 只有 45 分钟 job 上限,缺少针对生产构建步骤的 fail-closed deadline;此前 PR gate `1503/1505` 同一代码完整 build 通过,说明本次是 runner/build hang,不是已观测的业务编译错误。
|
||||
- 修复:在 Gitea validate 中将 frontend production build 包在 `timeout 600` 内,超时输出明确事实并以非零状态失败;不跳过 build、不降低测试、不发布旧 artifact。新增 workflow contract 锁定该 bounded timeout。
|
||||
- 验证:待本地 workflow contract、完整 PR gate、同一 reviewed main/staging SHA 的 push gate、immutable manifest 和 deploy 验收;完成前不得标记 resolved。
|
||||
- 验证:本地 workflow contracts 35/35;PR gates `1511`/`1519`/`1523` 和 staging push gates `1514`/`1521`/`1525` 均在 600 秒 command deadline 内完成 production build;最终 gate `1525` publish 与 deploy `1526` 成功。
|
||||
- 防复发:所有可能长时间静默的编译、镜像构建和外部网络步骤都必须有命令级 deadline,且 deadline 失败必须 fail closed;保留 job-level timeout 作为第二层上限,不把 timeout 当成功。
|
||||
- 相关记录:BUG-129、BUG-131、ERR-096、ERR-099
|
||||
- 复发自:无
|
||||
- 修复版本:待 frontend build bounded timeout 与 staging exact-SHA 验收
|
||||
- 修复版本:`8dc61e3135d8afb96b6683714f41a1716761164e`;最终验收 `6c1dcbe857006ec6ae7463b57b2b7d5947da4851`
|
||||
|
||||
## BUG-137 | staging deploy 在公网 upstream 尚未收敛时用旧 SHA 立即判失败
|
||||
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:Gitea `Deploy staging` 最终公网验证、自动回滚;数据库迁移已成功,production 未受影响。
|
||||
- 用户现象:exact-SHA gate `1514` 与 migration `1516` 成功后,deploy `1517`、`1518` 均启动目标 web/API image 并达到容器 healthy,却在约 3 秒后的公网 verification 返回非零,随后成功恢复旧 web/worker;公网和 `.state/deployed-revision` 均保持旧 SHA `e59f15d352787f3d05425ba8c459d092e9801a20`。
|
||||
- 触发条件:Compose 切换到目标容器后,旧 Caddy upstream 在短暂收敛窗口仍可让 `/login` 返回 200;脚本只轮询 `/login`,然后对公网 health SHA 和其余 predicate 仅检查一次,读取旧 SHA 时立即触发回滚。
|
||||
- 根因:发布验证把“login 可达”和“公网已路由到 exact SHA”拆成了不对称检查;容器健康与代理 upstream 收敛不是同一时刻,单次 SHA 检查形成确定性 race。目标 image 隔离 probe 已确认注入的 `GITHUB_SHA` 为目标 SHA。
|
||||
- 修复:在原 60 秒总预算内,每 5 秒原子重查 login、admin 未登录重定向、admin API/account 401、公网 health exact SHA、私有 API health 和 Swiss Ephemeris;仅当所有 predicate 同轮满足才成功。预算耗尽仍 fail closed 并只输出状态码、observed SHA、health 状态等脱敏摘要,不输出正文、env 或凭据。
|
||||
- 验证:合同测试 35/35;deploy `1522` 的脱敏摘要证明 bounded verifier 等待到目标 public SHA 后才报告独立 admin-host 问题;后续 PR gate `1523`、push gate `1525`、artifact ID `12` 和 deploy `1526` 均成功,最终公网/host state/main/staging 完整 SHA 一致。
|
||||
- 防复发:发布验证必须等待最终外部路由 identity,而不能把单个 readiness endpoint 当作代理收敛证明;所有重试必须有总上限,失败记录仅含非敏感 predicate 状态并保持自动回滚。
|
||||
- 相关记录:BUG-129、BUG-136、ERR-099、ERR-100
|
||||
- 复发自:无
|
||||
- 修复版本:`28f4207f295276968427686a21971fd53abb42ec`;最终验收 `6c1dcbe857006ec6ae7463b57b2b7d5947da4851`
|
||||
|
||||
## BUG-138 | staging Caddy 保留旧单文件 bind inode 且 admin 验证误走用户域名
|
||||
|
||||
- 状态:resolved
|
||||
- 首次发现:2026-08-06
|
||||
- 最近更新:2026-08-06
|
||||
- 影响面:staging Caddy 双 host 路由、admin TLS、`Deploy staging` 未登录边界验证;production 未受影响。
|
||||
- 用户现象:修复公网 SHA 收敛后,deploy `1522` 明确观测目标 SHA、私有 API 和 Swiss Ephemeris 均正常,但在用户域名上得到 `/admin -> 307 /` 与 `/api/admin/session -> 403`;同时 `admin.staging.jyotisha.chat` TLS 握手失败。workflow 在 60 秒后自动恢复旧应用,未写入新 deployed-revision。
|
||||
- 触发条件:controller 通过原子目录同步替换 `deploy/Caddyfile.staging`,但长期运行的 Caddy 容器仍持有旧单文件 bind mount inode;随后 checker 又把 admin 页面/API 请求错误发送到 `STAGING_URL` 而不是 `ADMIN_USER_ORIGIN`。
|
||||
- 根因:host 文件与 Caddy 容器 mount inode 漂移。只读现场证据显示 host Caddyfile 含 admin host、运行容器内文件不含,inode/size/mtime 均不同;staging VPS 从两台权威 nameserver 查询 admin A 记录均为 `118.26.111.127`,排除 DNS 缺失。用户域名按新 Caddy 合同本应对 admin paths 404,因此旧 checker 的 307/401 期待也违反双 host 边界。
|
||||
- 修复:应用 Compose 切换后显式 `--force-recreate --no-deps caddy`,使其重新挂载 gate-attested Caddyfile;完整 convergence 同轮要求用户域名 admin page/API 均 404、admin origin page 307 到 `/login`、admin API 401,并继续要求 exact public SHA、account 401、私有 API/Swiss 健康。失败仍 bounded、fail closed 并自动恢复旧应用。
|
||||
- 验证:shell/workflow contracts 35/35、PR gate `1523`、push gate `1525`、artifact ID `12` 和 deploy `1526` 成功。Caddy 被 force-recreate,容器内配置包含 admin host;公网 user host `/admin` 与 `/api/admin/session` 均 404,admin host `/` 为 308 `/admin`、`/admin` 为 307 `/login`、session API 为 401;Caddy restart count 为 0。
|
||||
- 防复发:原子替换单文件 bind mount 后必须 recreate/reload 长期运行服务;部署 smoke 必须分别使用各自主机 origin,不能在 user host 上测试 admin host 合同。保留 authority DNS、mount inode 和 TLS 检查作为脱敏现场证据。
|
||||
- 相关记录:BUG-134、BUG-137、ERR-097、ERR-100、ERR-101
|
||||
- 复发自:无
|
||||
- 修复版本:`6c1dcbe857006ec6ae7463b57b2b7d5947da4851`
|
||||
|
||||
@@ -11,8 +11,10 @@ This record contains only release identities, aggregate operational evidence, sc
|
||||
- Security/control-plane merge: `f7a615a5bf11ed95b3a6c7e6d28dfe8150a825ef`
|
||||
- Staging migration and application-under-test SHA: `f7a615a5bf11ed95b3a6c7e6d28dfe8150a825ef`
|
||||
- Docs-only acceptance attestation deployment SHA: `02cc483b7c303e6cc0f26fb31462c50adb007f12`
|
||||
- Final gate-attested controller deployment SHA: `e59f15d352787f3d05425ba8c459d092e9801a20`
|
||||
- Application rollback target: `49da8f916960030d5760d8dedf4e77820732a527`, subject to retained successful gate artifacts. The additive database migration remains in place after application rollback.
|
||||
- Prior gate-attested controller deployment SHA: `e59f15d352787f3d05425ba8c459d092e9801a20`
|
||||
- Final combined report/admin/control-plane deployment SHA: `6c1dcbe857006ec6ae7463b57b2b7d5947da4851`
|
||||
- Final immutable staging artifact: gate run `1525`, artifact ID `12`; target and controller SHA both equal the final deployment SHA.
|
||||
- Application rollback target before the final release: `e59f15d352787f3d05425ba8c459d092e9801a20`, subject to retained successful gate artifacts. Additive database migrations remain in place after application rollback.
|
||||
|
||||
At deployment verification, Gitea `main`, Gitea `staging`, `/opt/jyotisha-staging/.state/deployed-revision`, and public `/api/health` all reported the same full SHA.
|
||||
|
||||
@@ -44,7 +46,13 @@ The complete frontend suite could not be made fully executable on the local macO
|
||||
- Docs-only attestation deploy `1474`: success. The trusted-main fetch paused for an extended period before recovering; no SSH/staging mutation occurred during the pause. Public health and host state then moved to the exact attestation SHA with zero container restarts.
|
||||
- First bounded-fetch staging gate `1479`: success; automatic deploy `1480` then exhausted all three 120-second full-history fetch attempts and failed closed before SSH. Public/state remained on the prior healthy SHA.
|
||||
- Gate-attested controller PR gate `1481` and staging gate `1483`: success. Gate `1483` published the first four-field manifest plus SHA-256-bound `controller.tar` bundle for `e59f15d352787f3d05425ba8c459d092e9801a20`.
|
||||
- Final automatic deploy `1484`: success in one minute. Mutation-time `git fetch` count was zero; controller hash validation ran; the staging SSH secret remained masked and no private-key material appeared. Public health, host state, Gitea `main`, and Gitea `staging` all matched the exact final SHA. See `BUG-129` / `ERR-095`.
|
||||
- Final automatic deploy `1484`: success in one minute. Mutation-time `git fetch` count was zero; controller hash validation ran; the staging SSH secret remained masked and no private-key material appeared. Public health, host state, Gitea `main`, and Gitea `staging` all matched that release SHA. See `BUG-129` / `ERR-095`.
|
||||
- Later combined rollout gate `1514` succeeded for `8dc61e3135d8afb96b6683714f41a1716761164e`; automatic deploy `1515` correctly stopped before application mutation because seven admin/control-plane migrations were pending.
|
||||
- Manual staging migration `1516`: success. It applied `20260806010000_admin_rbac.sql` through `20260806070000_admin_mfa.sql`; previously applied personal-report migrations remained unchanged.
|
||||
- Deploys `1517`/`1518` and `1522` failed closed and restored the prior application while exposing two controller-only defects: one-shot proxy identity verification and a stale Caddy single-file bind mount/admin-host mismatch. No failed run advanced deployed-revision.
|
||||
- Controller PR gates `1519` and `1523`: success, including frontend 1472/1472, lint with zero errors, production build, database fixtures, workflow contracts, and bounded build execution.
|
||||
- Final staging push gate `1525`: success for full SHA `6c1dcbe857006ec6ae7463b57b2b7d5947da4851`; artifact ID `12` bound immutable API/web digests and the SHA-256 controller bundle.
|
||||
- Final automatic deploy `1526`: success. It consumed target/controller gate run `1525`, reported `verified_sha=6c1dcbe857006ec6ae7463b57b2b7d5947da4851`, refreshed Caddy's reviewed bind mount, passed dual-host identity boundaries, and left all containers at restart count zero.
|
||||
|
||||
## Security incident and containment
|
||||
|
||||
@@ -72,7 +80,10 @@ Post-deploy schema inspection reported:
|
||||
- policies: owner SELECT and owner DELETE;
|
||||
- `authenticated`: SELECT, DELETE only;
|
||||
- `service_role`: SELECT, INSERT, UPDATE, DELETE;
|
||||
- personal-report migration ledger count: 1.
|
||||
- personal-report migration ledger count: 1;
|
||||
- personal-report migration checksum: `fc03b6f48a62a615d4ed1c601451f30ef2e72bbb30c0f7ff6e8a935ea72be970`, matching the immutable repository file;
|
||||
- seven combined admin/billing/operations migrations reported present after migration run `1516`;
|
||||
- final runtime-role audit: service role membership/login/CONNECT true; admin service membership and BYPASSRLS false.
|
||||
|
||||
A two-owner synthetic RLS test ran entirely inside one PostgreSQL transaction and then rolled back:
|
||||
|
||||
@@ -90,6 +101,8 @@ No report document or birth fact was needed or persisted for this check.
|
||||
- Public `/api/health`: `ok`; deployment SHA exact match.
|
||||
- `/login`: 200.
|
||||
- Logged-out `/api/account`: 401.
|
||||
- User host `/admin` and `/api/admin/session`: 404.
|
||||
- Admin host `/`: 308 to `/admin`; `/admin`: 307 to `/login`; logged-out `/api/admin/session`: 401.
|
||||
- Logged-out report GET and POST: 401.
|
||||
- Logged-out report reader route: reachable; its data API remains authenticated.
|
||||
- Internal Python `/api/health`: 200, `status=ok`, `swisseph_available=true`.
|
||||
@@ -120,7 +133,7 @@ This is an idle snapshot only. It does not satisfy the planned single-user/two-u
|
||||
- Post-deploy encrypted staging backup: success.
|
||||
- Retention after post-deploy backup: 3 archives.
|
||||
- Backup directory mode: 0700.
|
||||
- Latest archive mode: 0600; non-empty (717,008 bytes).
|
||||
- Latest archive mode: 0600; non-empty. Final post-release archive size: 915,952 bytes.
|
||||
- No secret was passed in argv or printed.
|
||||
|
||||
## Blocked / user handoff
|
||||
|
||||
@@ -151,29 +151,41 @@ After exact-SHA staging gate `1473` succeeded, automatic deploy `1474` stopped m
|
||||
|
||||
Prevention: Gitea mutation workflows must perform no Git object operations. A successful staging gate packages its already-verified exact-SHA `deploy/` controller plus manifest validator into `controller.tar`, binds its SHA-256 into the strict image manifest, and uploads both as one immutable artifact. Deploy/migration must verify artifact run/SHA, controller digest, archive paths/types/duplicates/size, current `main == staging` refs, and a complete Gitea compare commit-DAG path before mutation; any missing or inconsistent evidence fails closed. Manual rollback still uses the current reviewed controller, never the old target's controller. Preserve exact-SHA images, forward-only defaults, shared mutation lock, and bounded API/artifact requests. Verified by PR gate `1481`, staging gate `1483`, and one-minute exact-SHA deploy `1484`; mutation-time `git fetch` was zero and all post-deploy health/schema/permission checks passed.
|
||||
|
||||
## ERR-096 | Quality-gate exact-SHA checkout failed on a transient low-speed window | investigating 2026-08-06
|
||||
## ERR-096 | Quality-gate exact-SHA checkout failed on a transient low-speed window | resolved 2026-08-06
|
||||
|
||||
Staging gate `1485` failed before validation when its single exact-SHA shallow fetch hit the configured 30-second/1024-B/s low-speed abort, producing `curl 28`, `early EOF`, and no publish artifact. No deployment was triggered and the previous exact-SHA staging application remained healthy. This is separate from mutation-time Git removal: quality validation still must acquire the source under test.
|
||||
|
||||
Prevention: both validate and publish exact-SHA checkouts use three bounded 300-second attempts, a 15-second connect timeout, and a 60-second/1-B/s stalled-transfer threshold. Preserve `--depth=1 --no-tags origin "$GITEA_SHA"`, exact HEAD equality, clean-tree checks, artifact non-reuse, and fail-closed exhaustion. Never report a skipped publish job as successful artifact publication.
|
||||
Prevention: both validate and publish exact-SHA checkouts use three bounded 300-second attempts, a 15-second connect timeout, and a 60-second/1-B/s stalled-transfer threshold. Preserve `--depth=1 --no-tags origin "$GITEA_SHA"`, exact HEAD equality, clean-tree checks, artifact non-reuse, and fail-closed exhaustion. Never report a skipped publish job as successful artifact publication. Closure evidence: exact-SHA PR gate `1523`, staging push gate `1525`, immutable artifact ID `12`, and deploy `1526` all succeeded for the reviewed release chain.
|
||||
|
||||
## ERR-097 | Staging host env missed the reviewed admin-origin selector | investigating 2026-08-06
|
||||
## ERR-097 | Staging host env missed the reviewed admin-origin selector | resolved 2026-08-06
|
||||
|
||||
Staging gate `1500` successfully validated and published the exact SHA, but automatic deploy `1502` stopped before app mutation with `invalid staging selector: ADMIN_USER_ORIGIN`. A redacted read-only check confirmed `.env.staging` remained `deploy:deploy 0600`, had one exact user origin, and had zero admin-origin definitions. Public staging therefore remained on the prior healthy SHA; this was not a migration failure and production was not involved.
|
||||
|
||||
Prevention: when a reviewed identity rollout adds a host-managed non-secret selector, update the staging env under the shared mutation lock before deploying the dependent controller. Modify only the named public selector through a mode-`0600` atomic replacement that preserves deployment-tree UID/GID; never print or copy the raw env. Re-run the exact validator and require the same gate-attested SHA in `main`, `staging`, host state, and public health before closure.
|
||||
Prevention: when a reviewed identity rollout adds a host-managed non-secret selector, update the staging env under the shared mutation lock before deploying the dependent controller. Modify only the named public selector through a mode-`0600` atomic replacement that preserves deployment-tree UID/GID; never print or copy the raw env. Re-run the exact validator and require the same gate-attested SHA in `main`, `staging`, host state, and public health before closure. Closure evidence: both env files remained `deploy:deploy 0600`, deploy `1526` passed both validators, and dual-host anonymous boundaries matched the reviewed contract.
|
||||
|
||||
## ERR-098 | Non-empty staging volume missed service runtime bootstrap and retained an obsolete privileged membership | investigating 2026-08-06
|
||||
## ERR-098 | Non-empty staging volume missed service runtime bootstrap and retained an obsolete privileged membership | resolved 2026-08-06
|
||||
|
||||
After the admin-origin selector was repaired, the full staging validators exposed that both service runtime env entries were absent. Redacted role inspection then showed `service_role` existed but `service_runtime` did not, while `admin_runtime` retained membership in the BYPASSRLS role contrary to the reviewed bootstrap/RBAC contract. The running old web image had no recoverable service URL, so no password was guessed or copied. Under the shared mutation lock, a new staging-only credential was generated, the dedicated login role and CONNECT/membership were established through PostgreSQL stdin, the two host env files were atomically updated with their separate raw/URL representations, and the obsolete admin membership was revoked. Both validators and a real service login passed; production was not involved.
|
||||
|
||||
Prevention: `/docker-entrypoint-initdb.d` is not a compatibility mechanism for an existing PostgreSQL volume. Every newly required runtime role must have a reviewed non-destructive repair path plus pre-deploy role/presence probes. Keep `admin_runtime` outside `service_role`; only `service_runtime` may assume the BYPASSRLS role through the dedicated service URL. Never display role passwords, pass them in argv, or edit an already-ledgered migration checksum to retrofit host bootstrap behavior.
|
||||
Prevention: `/docker-entrypoint-initdb.d` is not a compatibility mechanism for an existing PostgreSQL volume. Every newly required runtime role must have a reviewed non-destructive repair path plus pre-deploy role/presence probes. Keep `admin_runtime` outside `service_role`; only `service_runtime` may assume the BYPASSRLS role through the dedicated service URL. Never display role passwords, pass them in argv, or edit an already-ledgered migration checksum to retrofit host bootstrap behavior. Closure evidence: final metadata audit reported service membership/login/CONNECT true, admin service membership and BYPASSRLS false, with migration `1516` and deploy `1526` successful.
|
||||
|
||||
## ERR-099 | Gitea frontend production build hung after successful compilation until job timeout | investigating 2026-08-06
|
||||
## ERR-099 | Gitea frontend production build hung after successful compilation until job timeout | resolved 2026-08-06
|
||||
|
||||
Staging push gate `1507` completed all frontend tests (`1472/1472`), lint with zero errors, and Turbopack compilation in 38.2 seconds, then emitted no further build output for roughly 44 minutes. The 45-minute validate job expired, publish was skipped, no artifact or deployment was produced, and public staging remained on the previous healthy SHA. The same code had completed production builds in PR gates `1503` and `1505`, so the observed failure is a runner/build-finalization hang rather than a reported compile error. A manually dispatched diagnostic run does not satisfy the push-only publication contract and must not be treated as an immutable release artifact.
|
||||
|
||||
Prevention: wrap the Gitea frontend production build in a command-level 600-second timeout with an explicit nonzero failure; retain the 45-minute job timeout as a second boundary. Never skip the build, reuse an old artifact, or treat a manual validation-only run as a successful staging push gate. Only a successful exact-SHA push gate may publish images and trigger deployment.
|
||||
Prevention: wrap the Gitea frontend production build in a command-level 600-second timeout with an explicit nonzero failure; retain the 45-minute job timeout as a second boundary. Never skip the build, reuse an old artifact, or treat a manual validation-only run as a successful staging push gate. Only a successful exact-SHA push gate may publish images and trigger deployment. Closure evidence: PR gates `1511`/`1519`/`1523` and push gates `1514`/`1521`/`1525` completed production builds within the command deadline; final publish/deploy succeeded.
|
||||
|
||||
## ERR-100 | Staging public verification checked exact SHA only once before proxy convergence | resolved 2026-08-06
|
||||
|
||||
Exact-SHA gate `1514` and migration `1516` succeeded, but deploy runs `1517` and `1518` each reached healthy target web/API containers and then failed public verification roughly three seconds later. Both runs restored the prior application image and retained the old deployed-revision state. The verifier retried only `/login`; once that endpoint returned 200 through the existing Caddy route, it performed exactly one check of public health SHA and the remaining authorization/private-health predicates. This permits an old upstream response to trigger immediate rollback during container/proxy convergence even though the new container itself carries the expected SHA. No application release completed and production was not involved.
|
||||
|
||||
Prevention: within the existing bounded 60-second budget, retry the complete predicate set together: login, admin redirect, logged-out admin/account responses, public exact SHA, private API status, and Swiss Ephemeris availability. Success requires every predicate in the same attempt. On exhaustion, fail closed and emit only redacted status codes, observed SHA, and health booleans; never log response bodies, env, credentials, or error stacks. Closure evidence: deploy `1522` produced the intended redacted convergence evidence, and deploy `1526` passed the complete same-attempt predicate set.
|
||||
|
||||
## ERR-101 | Atomic Caddyfile replacement left the running Caddy bind-mounted to the old inode | resolved 2026-08-06
|
||||
|
||||
The bounded verifier in deploy `1522` proved that public health had converged to the target exact SHA and private API/Swiss checks passed, but admin checks on the user host returned the expected application fail-closed statuses rather than admin-host statuses. Read-only host inspection then found the reviewed host `deploy/Caddyfile.staging` contained the admin virtual host while `/etc/caddy/Caddyfile` inside the long-running Caddy container did not; their inode, size, and mtime differed even though Docker reported the expected bind source. The tree sync atomically replaced the source file, leaving the existing bind mount attached to the old inode. Both authoritative Spaceship nameservers returned the staging VPS address for the admin host when queried from the VPS, so this was not an absent DNS record. The failed deploy restored the prior application and did not advance deployed-revision.
|
||||
|
||||
Prevention: after syncing a gate-attested single-file bind mount, force-recreate Caddy under the shared host lock before public verification. Verify user and admin origins separately: user-host admin paths must be 404; admin-host anonymous page/API must be 307-to-login and 401. Preserve bounded convergence, exact public SHA, private health, automatic rollback, and redacted diagnostics. Do not weaken identity host routing or modify DNS based on intercepted local resolver results. Closure evidence: deploy `1526` force-recreated Caddy, the container mounted the reviewed dual-host file, public TLS/routes passed, and all container restart counts remained zero.
|
||||
|
||||
## Fragment Sweep Command Set
|
||||
|
||||
|
||||
@@ -644,7 +644,14 @@ test("remote deployment verifies running image IDs, RepoDigests, and application
|
||||
assert.match(runner, /"\$\{docker_command\[@\]\}" image inspect --format '\{\{\.Id\}\}' "\$expected_ref"/);
|
||||
assert.match(runner, /RepoDigests/);
|
||||
assert.match(runner, /grep -Fqx "\$expected_ref"/);
|
||||
assert.match(runner, /publicBody\.deployment\?\.gitCommit !== process\.env\.EXPECTED_SHA/);
|
||||
assert.match(runner, /for \(let attempt = 1; attempt <= 12; attempt \+= 1\)/);
|
||||
assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/);
|
||||
assert.match(runner, /staging verification predicates did not converge/);
|
||||
assert.match(runner, /publicSha: publicBody\.deployment\?\.gitCommit \?\? "missing"/);
|
||||
assert.match(runner, /up -d --no-build --force-recreate --no-deps caddy/);
|
||||
assert.match(runner, /userAdminPage\.status === 404/);
|
||||
assert.match(runner, /userAdminApi\.status === 404/);
|
||||
assert.match(runner, /fetch\(`\$\{process\.env\.ADMIN_USER_ORIGIN\}\/admin`/);
|
||||
assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/);
|
||||
assert.match(runner, /restoring prior application images/);
|
||||
assert.match(
|
||||
@@ -771,9 +778,14 @@ test("normal deployment checks migrations but never applies them", () => {
|
||||
assert.doesNotMatch(runner, /npm\s+run\s+db:migrate(?!:check)/);
|
||||
assert.doesNotMatch(runner, /pull api web postgres/);
|
||||
assert.match(runner, /verify_container_image rectification-v4-worker \"\$WEB_IMAGE\"/);
|
||||
assert.match(runner, /adminPage\.status !== 307/);
|
||||
assert.match(runner, /adminPage\.headers\.get\("location"\) !== "\/login"/);
|
||||
assert.match(runner, /adminApi\.status !== 401/);
|
||||
assertOrder(runner, [
|
||||
"up -d --no-build --remove-orphans",
|
||||
"up -d --no-build --force-recreate --no-deps caddy",
|
||||
"verify_container_image api",
|
||||
]);
|
||||
assert.match(runner, /adminPage\.status === 307/);
|
||||
assert.match(runner, /adminPage\.headers\.get\("location"\) === "\/login"/);
|
||||
assert.match(runner, /adminApi\.status === 401/);
|
||||
});
|
||||
|
||||
test("staging runs the rectification V4 worker from the immutable web image", () => {
|
||||
|
||||
Reference in New Issue
Block a user