diff --git a/.gitea/workflows/backend-quality-gate.yml b/.gitea/workflows/backend-quality-gate.yml index 904e1382..bd3f18bd 100644 --- a/.gitea/workflows/backend-quality-gate.yml +++ b/.gitea/workflows/backend-quality-gate.yml @@ -251,8 +251,15 @@ jobs: "timezonefinder==8.2.5" \ -r requirements.txt -r requirements-dev.txt workdir="$(pwd -P)" + npm_diagnostics="$(mktemp -d "${TMPDIR:-/tmp}/jyotisha-npm-XXXXXXXX")" + cleanup_npm() { + if [ -s "$npm_diagnostics/container.cid" ]; then + docker rm -f "$(cat "$npm_diagnostics/container.cid")" >/dev/null 2>&1 || true + fi + } + trap cleanup_npm EXIT set +e - docker run --rm \ + docker run --cidfile "$npm_diagnostics/container.cid" \ --cpus=1.5 \ --memory=2g \ --memory-swap=2g \ @@ -261,9 +268,11 @@ jobs: --volume "$workdir:$workdir" \ --workdir "$workdir" \ --env HOME=/tmp \ + --env LC_ALL=C \ + --volume "$npm_diagnostics:/npm-diagnostics" \ --env "NPM_CONFIG_REGISTRY=$NPM_CONFIG_REGISTRY" \ "$NODE_TOOL_SOURCE_IMAGE" \ - timeout --signal=TERM --kill-after=30s 900s \ + timeout --verbose --signal=TERM --kill-after=30s 900s \ npm ci --prefix frontend \ --no-audit \ --no-fund \ @@ -272,10 +281,24 @@ jobs: --fetch-timeout=60000 \ --fetch-retries=2 \ --fetch-retry-mintimeout=1000 \ - --fetch-retry-maxtimeout=10000 - npm_ci_status=$? + --fetch-retry-maxtimeout=10000 \ + --foreground-scripts \ + --loglevel=http \ + --logs-dir=/npm-diagnostics/npm 2>&1 | tee "$npm_diagnostics/install.log" + npm_ci_status=${PIPESTATUS[0]} set -e - if [ "$npm_ci_status" -eq 124 ]; then + if [ "$npm_ci_status" -ne 0 ]; then + echo "npm diagnostics retained at $npm_diagnostics (runner-local; do not publish unredacted logs)" >&2 + npm_oom="$(docker inspect --format '{{.State.OOMKilled}}' "$(cat "$npm_diagnostics/container.cid" 2>/dev/null)" 2>/dev/null || echo unknown)" + echo "npm container exit=$npm_ci_status OOMKilled=$npm_oom" >&2 + if [ "$npm_oom" = true ]; then + exit "$npm_ci_status" + fi + fi + if [ "$npm_ci_status" -eq 124 ] || { + [ "$npm_ci_status" -eq 137 ] && + grep -Eq '^timeout: sending signal (TERM|KILL) to command' "$npm_diagnostics/install.log" + }; then echo "frontend npm ci exceeded bounded 900-second timeout; check npm mirror/network or dependency postinstall hang" >&2 exit 124 fi @@ -283,6 +306,9 @@ jobs: echo "frontend npm ci failed with status $npm_ci_status inside bounded Node container" >&2 exit "$npm_ci_status" fi + cleanup_npm + trap - EXIT + rm -rf -- "$npm_diagnostics" - name: Validate backend, package, frontend, and database contracts run: | diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 066b0a89..f8d44f9c 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -2509,9 +2509,9 @@ ## BUG-149 | staging quality gate npm ci 缺少安装级 deadline 导致 45 分钟黑洞 -- 状态：resolved（local candidate，远端 gate/deploy 待本提交） +- 状态：investigating（2026-09-26 安装卡住复发；既有 deadline 生效，底层卡点待 runner 证据） - 首次发现：2026-08-09 -- 最近更新：2026-08-09 +- 最近更新：2026-09-26 - 影响面：Gitea staging quality gate 的依赖安装阶段、`manman-linux` runner 与 Gitea/runner 控制面可用性；测试、lint、build、exact-SHA checkout/publish/deploy 合同未改变。 - 用户现象：Run 1618（SHA `ffbe505c`）在 Python pip 完成后进入 digest-pinned Node 容器执行 `npm ci`；步骤无后续 npm 输出，直到 45 分钟 job timeout，publish/deploy skipped。 - 触发条件：self-hosted runner 在受限 Node 容器中执行 frontend `npm ci`，但安装命令本身没有 fail-closed deadline，npm registry/fetch 也没有比 job-level 更短的诊断边界。 @@ -2523,6 +2523,26 @@ - 复发自：无 - 修复版本：待本次提交 / gate / deploy +### 2026-09-26 诊断补充（未修改 workflow / 未重跑部署） + +- 版本：远端 staging 与隔离诊断 worktree 均为 `40d7930ab4c8c44e504f3a6caaee351ed262f172`。Gitea run `2937`（显示序号 `1522`）、job `6464` 在 `xiaoxin` runner 的 `Install dependencies` 失败；Python pip 成功，测试步骤与 publish 跳过。 +- 原始错误：`frontend npm ci failed with status 137 inside bounded Node container`。npm 最后一条警告时间为 `2026-09-26T08:28:41.857Z`，失败时间为 `08:44:11.716Z`，约 930 秒，吻合现有 `timeout --signal=TERM --kill-after=30s 900s`。本地 GNU timeout 缩时验证：子进程忽略 TERM 后被 KILL，退出码确为 137。直接退出路径高度符合超时强杀，但仅凭该退出码不能证明或排除 runner OOM。 +- 对照：此前成功 run `2934` / job `6456` 在 `07:05:36Z` 输出 `added 859 packages in 40s`。两次的 workflow、`frontend/package.json` 与 lockfile 完全相同；两次都出现 `posthog-node@5.41.0` 要求 Node `^20.20.0 || >=22.22.0`、实际 `22.16.0` 的 EBADENGINE 警告，因此不能把该警告认定为本轮直接失败原因。 +- 历史防线：900 秒安装 deadline、资源上限与 fetch timeout 均仍存在，阻止了旧 45 分钟黑洞；workflow 仅对 124 输出超时专用文案，137 落入通用失败分支。既有 `staging-backend-workflows.test.ts` 只锁定命令文本与边界，未验证 TERM 不退出后 137 的诊断语义，也不能防止外部安装链路卡住。 +- 未证实部分：npm 下载/解压/postinstall 的具体卡点及内存/PID 状态。容器使用 `--rm`，npm 日志位于容器 HOME `/tmp` 且未持久化；现有 Actions 日志不足以区分镜像网络、安装脚本或资源问题，不虚构具体故障包。 +- 后续最小排查：由产品负责人授权后，为安装容器持久化 npm debug 日志并保留退出/OOM 证据，在相同 SHA 重跑；不要仅调大超时或禁用安装脚本。Node engine 版本告警需另行对齐，不冒充本轮根因修复。 +- 运行态：只读健康检查返回 `status=ok`，web/API SHA 均为此前成功版本 `f74825a27cca881af3373eb0c77f8f52135da378`，本轮最新代码未部署。未执行 push、migration 或 workflow dispatch。 + + +### 2026-09-26 本地诊断修补（未推送 / 未部署，原始卡死仍 investigating） + +- 授权与范围：产品负责人要求修复上述 workflow；仅修改 backend quality gate 的 npm 安装诊断及其回归，不升级 Node、不更换 registry、不增加重试、不放宽 900 秒 deadline 或 CPU/memory/PID 边界。 +- 已确认缺陷：GNU timeout 的 TERM 后强杀可能返回 137，旧分支只识别 124；`--rm` 和容器内临时 HOME 丢失 OOM 状态及 npm debug 日志。它们是诊断缺陷，不是已证实的 npm 卡死根因。 +- 本地修补：使用独占 cidfile 和 EXIT trap 清理本次容器；先读取 `State.OOMKilled`，只有 124 或「137 + timeout 实际发送信号的日志」才报告超时，OOM/未知失败仍 fail closed。用 `PIPESTATUS[0]` 保留 Docker 的失败码，不让 tee 掩盖失败。mode-0700 临时目录挂载 npm debug 日志并保存安装输出，成功删除、失败保留在 runner 本地；日志未经脱敏不发布。开启 foreground scripts 与 HTTP 进度用于定位下载或 postinstall 卡点。 +- 回归：直接提取 workflow 中的安装 shell，以 Docker stub 执行成功、124、强杀 137、OOM 137、未知 137、容器启动失败 125 六种场景，验证分类、退出码、日志保留与定向清理。修补前强杀场景复现通用 137 失败，修补后通过；使用主 checkout 的 Python venv 提供 PyYAML，`node --test frontend/tests/staging-backend-workflows.test.ts` 为 44/44 passed；`bash -n` 与 `git diff --check` 通过。系统 Python 缺 PyYAML 的首次检查失败属于本机工具环境，不是 workflow YAML 错误。 +- 真实容器验证：本机 Docker Linux/amd64 使用 CI 同一 pinned Node 镜像、同一 lockfile 和资源限制，修改后安装 shell 成功安装 859 包（约 2 分钟）。将验证副本的期限缩至 1 秒 + 1 秒、命令替换为忽略 TERM 的进程，真实 Docker 返回 137、`OOMKilled=false`，workflow 报告超时并返回 124；容器由 trap 删除。此人为强杀实验只验证退出诊断，不复现 npm 卡死。 +- 剩余边界：本机无法复现 xiaoxin 的原始卡死；未在真实 runner 验证修补版本，未执行 push、workflow dispatch、migration 或 deploy。BUG-149 保持 investigating，下一步是获准推送后以新 SHA 的 runner 日志判断下载、安装脚本或资源卡点，不能将本次诊断修补称为安装问题彻底解决。 + ## BUG-150 | staging publish 的 Webpack 镜像构建耗尽共享 Gitea 资源 - 状态：resolved（local candidate，远端 gate/deploy 待本提交） diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 52592dba..2d0c69d4 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -279,8 +279,10 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest", assert.match(installStep, /rm -rf -- \.venv/); assert.match(installStep, /python3 -m venv --clear \.venv/); assert.match(installStep, /workdir="\$\(pwd -P\)"/); - assert.match(installStep, /docker run --rm/); - assert.doesNotMatch(installStep, /timeout --signal=TERM --kill-after=30s 900s docker run/); + assert.match(installStep, /docker run --cidfile/); + assert.match(installStep, /--logs-dir=\/npm-diagnostics\/npm/); + assert.match(installStep, /trap cleanup_npm EXIT/); + assert.doesNotMatch(installStep, /timeout[^\n]*docker run/); assert.match(installStep, /--cpus=1\.5/); assert.match(installStep, /--memory=2g/); assert.match(installStep, /--memory-swap=2g/); @@ -290,8 +292,8 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest", assert.match(installStep, /--workdir "\$workdir"/); assert.match(installStep, /--env HOME=\/tmp/); assert.match(installStep, /--env "NPM_CONFIG_REGISTRY=\$NPM_CONFIG_REGISTRY"/); - assert.match(installStep, /"\$NODE_TOOL_SOURCE_IMAGE" \\\n\s+timeout --signal=TERM --kill-after=30s 900s \\\n\s+npm ci --prefix frontend \\\n\s+--no-audit \\\n\s+--no-fund \\\n\s+--progress=false \\\n\s+--maxsockets=4 \\\n\s+--fetch-timeout=60000 \\\n\s+--fetch-retries=2 \\\n\s+--fetch-retry-mintimeout=1000 \\\n\s+--fetch-retry-maxtimeout=10000/); - assert.match(installStep, /npm_ci_status=\$\?/); + assert.match(installStep, /"\$NODE_TOOL_SOURCE_IMAGE" \\\n\s+timeout --verbose --signal=TERM --kill-after=30s 900s \\\n\s+npm ci --prefix frontend \\\n\s+--no-audit \\\n\s+--no-fund \\\n\s+--progress=false \\\n\s+--maxsockets=4 \\\n\s+--fetch-timeout=60000 \\\n\s+--fetch-retries=2 \\\n\s+--fetch-retry-mintimeout=1000 \\\n\s+--fetch-retry-maxtimeout=10000/); + assert.match(installStep, /npm_ci_status=\$\{PIPESTATUS\[0\]\}/); assert.match(installStep, /frontend npm ci exceeded bounded 900-second timeout/); assert.match(installStep, /frontend npm ci failed with status \$npm_ci_status inside bounded Node container/); assert.doesNotMatch(installStep, /--ignore-scripts|--omit(?:=|\s+)optional|--force/); @@ -1627,3 +1629,70 @@ test("gate checkouts fetch the exact SHA from a host-persistent mirror and fall ]); } }); + +test("bounded npm install distinguishes forced timeout, OOM, and unknown failure and cleans its container", () => { + const workflow = readFileSync(giteaQualityWorkflow, "utf8"); + const start = workflow.indexOf(' workdir="$(pwd -P)"'); + assert.notEqual(start, -1); + const script = "set -euo pipefail\n" + workflow.slice(start) + .split("\n - name: Validate backend")[0].replace(/^ {10}/gm, ""); + const root = mkdtempSync(join(tmpdir(), "npm-workflow-test-")); + try { + const docker = join(root, "docker"); + writeFileSync(docker, `#!/bin/bash +case "$1" in + run) + while [ "$#" -gt 0 ]; do + if [ "$1" = --cidfile ] && [ "$FAKE_STATUS" != 125 ]; then printf test-container > "$2"; fi + shift + done + if [ "$FAKE_TIMEOUT" = true ]; then echo "timeout: sending signal TERM to command 'npm'" >&2; fi + exit "$FAKE_STATUS" ;; + inspect) + if [ "$FAKE_STATUS" = 125 ]; then exit 1; fi + echo "$FAKE_OOM" ;; + rm) echo "$*" >> "$CLEANUP_LOG" ;; +esac +`); + chmodSync(docker, 0o755); + for (const [status, oom, timedOut, expectedStatus, message] of [ + [137, false, true, 124, "exceeded bounded 900-second timeout"], + [0, false, false, 0, ""], + [124, false, true, 124, "exceeded bounded 900-second timeout"], + [137, true, true, 137, "OOMKilled=true"], + [137, false, false, 137, "failed with status 137"], + [125, false, false, 125, "failed with status 125"], + ] as const) { + const cleanupLog = join(root, "cleanup.log"); + writeFileSync(cleanupLog, ""); + const result = spawnSync("bash", ["-c", script], { + cwd: root, + env: { ...process.env, PATH: `${root}:${process.env.PATH}`, TMPDIR: root, + NODE_TOOL_SOURCE_IMAGE: "test-image", NPM_CONFIG_REGISTRY: "https://registry.npmmirror.com", + FAKE_STATUS: String(status), FAKE_OOM: String(oom), FAKE_TIMEOUT: String(timedOut), + CLEANUP_LOG: cleanupLog }, + encoding: "utf8", timeout: 10000, + }); + const output = result.stdout + result.stderr; + assert.equal(result.status, expectedStatus, output); + assert.ok(output.includes(message), output); + if (status === 125) { + assert.equal(readFileSync(cleanupLog, "utf8"), ""); + assert.match(output, /OOMKilled=unknown/); + } else { + assert.match(readFileSync(cleanupLog, "utf8"), /rm -f test-container/); + } + const diagnostics = readdirSync(root).filter(name => name.startsWith("jyotisha-npm-")); + if (status !== 0) { + assert.equal(diagnostics.length, 1, output); + assert.ok(existsSync(join(root, diagnostics[0], "install.log"))); + assert.match(output, /npm diagnostics retained at/); + rmSync(join(root, diagnostics[0]), { recursive: true, force: true }); + } else { + assert.equal(diagnostics.length, 0); + } + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +});