From 9c20efd16d4b4f9c9dc1f8a7b4c1f23a9f7ec002 Mon Sep 17 00:00:00 2001 From: Jesse Date: Thu, 6 Aug 2026 20:13:59 +0800 Subject: [PATCH] fix(staging): retry exact gate checkout --- .gitea/workflows/backend-quality-gate.yml | 32 ++++++++++++++++--- docs/BUG_HISTORY.md | 16 ++++++++++ docs/research/pre_work_error_ledger.md | 6 ++++ .../tests/staging-backend-workflows.test.ts | 6 ++++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/backend-quality-gate.yml b/.gitea/workflows/backend-quality-gate.yml index d2d2bbf4..d9f88445 100644 --- a/.gitea/workflows/backend-quality-gate.yml +++ b/.gitea/workflows/backend-quality-gate.yml @@ -40,8 +40,20 @@ jobs: git init . git remote remove origin 2>/dev/null || true git remote add origin https://git.copse.top/root/Jyotisha.git - git -c http.connectTimeout=15 -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=30 \ - fetch --depth=1 --no-tags origin "$GITEA_SHA" + fetch_succeeded=false + for attempt in 1 2 3; do + if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \ + fetch --depth=1 --no-tags origin "$GITEA_SHA"; then + fetch_succeeded=true + break + fi + if [ "$attempt" -eq 3 ]; then + echo "exact staging gate checkout failed after $attempt bounded attempts" >&2 + exit 1 + fi + sleep $((attempt * 10)) + done + [[ "$fetch_succeeded" == true ]] git checkout --detach --force "$GITEA_SHA" git clean -ffdx test "$(git rev-parse HEAD)" = "$GITEA_SHA" @@ -191,8 +203,20 @@ jobs: git init . git remote remove origin 2>/dev/null || true git remote add origin https://git.copse.top/root/Jyotisha.git - git -c http.connectTimeout=15 -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=30 \ - fetch --depth=1 --no-tags origin "$GITEA_SHA" + fetch_succeeded=false + for attempt in 1 2 3; do + if timeout 300 git -c http.connectTimeout=15 -c http.lowSpeedLimit=1 -c http.lowSpeedTime=60 \ + fetch --depth=1 --no-tags origin "$GITEA_SHA"; then + fetch_succeeded=true + break + fi + if [ "$attempt" -eq 3 ]; then + echo "exact staging gate checkout failed after $attempt bounded attempts" >&2 + exit 1 + fi + sleep $((attempt * 10)) + done + [[ "$fetch_succeeded" == true ]] git checkout --detach --force "$GITEA_SHA" git clean -ffdx test "$(git rev-parse HEAD)" = "$GITEA_SHA" diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index a8890c33..e919c4fd 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -2240,3 +2240,19 @@ - 相关记录：BUG-128、ERR-094、ERR-095 - 复发自：BUG-129 第一轮修复未覆盖对象范围 - 修复版本：`e59f15d352787f3d05425ba8c459d092e9801a20`；gate-attested controller bundle 已完成 staging exact-SHA 验收 + +## BUG-130 | staging quality gate exact-SHA checkout 因过严低速阈值单次失败 + +- 状态：investigating +- 首次发现：2026-08-06 +- 最近更新：2026-08-06 +- 影响面：Gitea `Staging Backend Quality Gate` validate/publish 的 exact-SHA checkout；staging mutation controller、应用数据面与 production 未受影响。 +- 用户现象：docs-attestation staging gate run `1485` 在 validate 的首步失败；Gitea 已枚举/压缩 3,246/2,895 个 shallow objects，但客户端传输降速后触发 `curl 28 Operation too slow`、`early EOF`。publish 被依赖关系跳过，自动 deploy 未触发；公网继续健康运行 `e59f15d352787f3d05425ba8c459d092e9801a20`。 +- 触发条件：quality gate 的 exact-SHA `--depth=1` fetch 只有单次调用，并把低速失败设为连续 30 秒低于 1024 B/s；当前 Gitea 链路在约 20 KiB/s 波动后短时低于阈值。 +- 根因：`BUG-129` 消除了 mutation workflow 的 Git object fetch，但 quality gate 自身仍必须取得待测源码；其 checkout 没有 bounded retry，且低速阈值对当前受限链路过严。旧测试只断言 exact SHA/clean tree，没有覆盖 checkout retry 与低速边界。 +- 修复：validate/publish 两处 exact-SHA checkout 均改为最多 3 次、每次 hard timeout 300 秒；保留 connect timeout 15 秒，将低速失败收紧为连续 60 秒低于 1 B/s。每次仍只抓 `--depth=1 --no-tags origin "$GITEA_SHA"`，耗尽后明确 fail closed，不复用旧 artifact、不放宽 exact-SHA 或 clean-tree 校验。 +- 验证：待本地 workflow/YAML/run-script/pre-work 回归、完整 PR gate、staging push gate 和自动 deploy；完成前不得标记 resolved。 +- 防复发：质量门禁和 mutation controller 的网络边界分别测试；quality gate checkout 必须覆盖 attempt 数、hard timeout、低速阈值、exact-SHA refspec、最终错误和 clean-tree identity。 +- 相关记录：BUG-129、ERR-095、ERR-096 +- 复发自：无；属于同一 Gitea 链路在 quality-gate 阶段的独立缺口 +- 修复版本：待 bounded exact-SHA gate checkout 与 staging 验收 diff --git a/docs/research/pre_work_error_ledger.md b/docs/research/pre_work_error_ledger.md index 8494a2dd..3c6c7415 100644 --- a/docs/research/pre_work_error_ledger.md +++ b/docs/research/pre_work_error_ledger.md @@ -151,6 +151,12 @@ After exact-SHA staging gate `1473` succeeded, automatic deploy `1474` stopped m Prevention: Gitea mutation workflows must perform no Git object operations. A successful staging gate packages its already-verified exact-SHA `deploy/` controller plus manifest validator into `controller.tar`, binds its SHA-256 into the strict image manifest, and uploads both as one immutable artifact. Deploy/migration must verify artifact run/SHA, controller digest, archive paths/types/duplicates/size, current `main == staging` refs, and a complete Gitea compare commit-DAG path before mutation; any missing or inconsistent evidence fails closed. Manual rollback still uses the current reviewed controller, never the old target's controller. Preserve exact-SHA images, forward-only defaults, shared mutation lock, and bounded API/artifact requests. Verified by PR gate `1481`, staging gate `1483`, and one-minute exact-SHA deploy `1484`; mutation-time `git fetch` was zero and all post-deploy health/schema/permission checks passed. +## ERR-096 | Quality-gate exact-SHA checkout failed on a transient low-speed window | investigating 2026-08-06 + +Staging gate `1485` failed before validation when its single exact-SHA shallow fetch hit the configured 30-second/1024-B/s low-speed abort, producing `curl 28`, `early EOF`, and no publish artifact. No deployment was triggered and the previous exact-SHA staging application remained healthy. This is separate from mutation-time Git removal: quality validation still must acquire the source under test. + +Prevention: both validate and publish exact-SHA checkouts use three bounded 300-second attempts, a 15-second connect timeout, and a 60-second/1-B/s stalled-transfer threshold. Preserve `--depth=1 --no-tags origin "$GITEA_SHA"`, exact HEAD equality, clean-tree checks, artifact non-reuse, and fail-closed exhaustion. Never report a skipped publish job as successful artifact publication. + ## Fragment Sweep Command Set ## ERR-086 | Steve Jobs jyotishganit artifacts used non-San-Francisco coordinates | mitigated 2026-07-21 diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 50dee9a9..139b9bda 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -141,6 +141,12 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest", assert.match(workflow, /GITEA_SHA: \$\{\{ gitea\.sha \}\}/); assert.equal((workflow.match(/git clean -ffdx/g) ?? []).length, 2); assert.equal((workflow.match(/git status --porcelain --untracked-files=all/g) ?? []).length, 2); + assert.equal((workflow.match(/for attempt in 1 2 3/g) ?? []).length, 5); + assert.equal((workflow.match(/timeout 300 git -c http\.connectTimeout=15 -c http\.lowSpeedLimit=1 -c http\.lowSpeedTime=60/g) ?? []).length, 2); + assert.equal((workflow.match(/fetch --depth=1 --no-tags origin "\$GITEA_SHA"/g) ?? []).length, 2); + assert.equal((workflow.match(/exact staging gate checkout failed after \$attempt bounded attempts/g) ?? []).length, 2); + assert.equal((workflow.match(/\[\[ "\$fetch_succeeded" == true \]\]/g) ?? []).length, 2); + assert.doesNotMatch(workflow, /http\.lowSpeedLimit=1024|http\.lowSpeedTime=30/); assert.match(workflow, /python3 -m venv \.venv/); assert.match(workflow, /PIP_INDEX_URL: https:\/\/mirrors\.aliyun\.com\/pypi\/simple\//); assert.match(workflow, /NPM_CONFIG_REGISTRY: https:\/\/registry\.npmmirror\.com/);