fix staging npm ci in-container timeout
This commit is contained in:
@@ -168,6 +168,7 @@ jobs:
|
||||
"timezonefinder==8.2.5" \
|
||||
-r requirements.txt -r requirements-dev.txt
|
||||
workdir="$(pwd -P)"
|
||||
set +e
|
||||
docker run --rm \
|
||||
--cpus=1.5 \
|
||||
--memory=2g \
|
||||
@@ -179,11 +180,26 @@ jobs:
|
||||
--env HOME=/tmp \
|
||||
--env "NPM_CONFIG_REGISTRY=$NPM_CONFIG_REGISTRY" \
|
||||
"$NODE_TOOL_SOURCE_IMAGE" \
|
||||
timeout --signal=TERM --kill-after=30s 900s \
|
||||
npm ci --prefix frontend \
|
||||
--no-audit \
|
||||
--no-fund \
|
||||
--progress=false \
|
||||
--maxsockets=4
|
||||
--maxsockets=4 \
|
||||
--fetch-timeout=60000 \
|
||||
--fetch-retries=2 \
|
||||
--fetch-retry-mintimeout=1000 \
|
||||
--fetch-retry-maxtimeout=10000
|
||||
npm_ci_status=$?
|
||||
set -e
|
||||
if [ "$npm_ci_status" -eq 124 ]; then
|
||||
echo "frontend npm ci exceeded bounded 900-second timeout; check npm mirror/network or dependency postinstall hang" >&2
|
||||
exit 124
|
||||
fi
|
||||
if [ "$npm_ci_status" -ne 0 ]; then
|
||||
echo "frontend npm ci failed with status $npm_ci_status inside bounded Node container" >&2
|
||||
exit "$npm_ci_status"
|
||||
fi
|
||||
|
||||
- name: Validate backend, package, frontend, and database contracts
|
||||
run: |
|
||||
|
||||
+7
-7
@@ -2537,18 +2537,18 @@
|
||||
- 复发自:无
|
||||
- 修复版本:2026-08-08 staging 变更
|
||||
|
||||
## BUG-149 | staging quality gate 宿主 npm ci 触发控制面持续 502 并被基础设施终止
|
||||
## BUG-149 | staging quality gate npm ci 缺少安装级 deadline 导致 45 分钟黑洞
|
||||
|
||||
- 状态:resolved(local candidate,远端 gate/deploy 待本提交)
|
||||
- 首次发现:2026-08-09
|
||||
- 最近更新:2026-08-09
|
||||
- 影响面:Gitea staging quality gate 的依赖安装阶段、`manman-linux` runner 与 Gitea/runner 控制面可用性;测试、lint、build、exact-SHA checkout/publish/deploy 合同未改变。
|
||||
- 用户现象:quality gate 在宿主机直接执行 `npm ci` 期间,Gitea 与 runner 控制面持续返回 502;步骤日志没有 npm 安装错误,任务最终由基础设施终止,无法进入后续远端 gate/deploy 收口。
|
||||
- 触发条件:self-hosted runner 在宿主环境执行无独立资源边界的 frontend `npm ci`,同时由共享控制面维持 job 调度、状态与日志链路。
|
||||
- 根因:依赖安装直接占用宿主资源且缺少 CPU、内存、swap、PID 与并发 socket 上限,故障证据落在 runner/控制面资源争用和基础设施终止边界,而不是 npm 包解析或安装错误。
|
||||
- 修复:仅将 `npm ci` 放入 digest-pinned Node 容器,并限制为 CPU 1.5、memory 2g、no swap、pids 256、`npm_config_maxsockets=4`;测试、lint、build 与 exact-SHA 行为保持原样,不扩大容器化范围,也不改变发布和部署信任边界。
|
||||
- 验证:本地 deployment contracts 45/45、workflow contracts 29/29;独立 review 无 P0/P1。远端 staging gate/deploy 待本提交后验证,本记录不提前声称远端收口。
|
||||
- 防复发:宿主依赖安装必须维持 digest-pinned Node runtime 和显式资源上限;合同测试需锁定只有 `npm ci` 在受限容器内执行,并持续确认测试、lint、build、exact-SHA checkout/publish/deploy 语义未漂移。
|
||||
- 用户现象:Run 1618(SHA `ffbe505c`)在 Python pip 完成后进入 digest-pinned Node 容器执行 `npm ci`;步骤无后续 npm 输出,直到 45 分钟 job timeout,publish/deploy skipped。
|
||||
- 触发条件:self-hosted runner 在受限 Node 容器中执行 frontend `npm ci`,但安装命令本身没有 fail-closed deadline,npm registry/fetch 也没有比 job-level 更短的诊断边界。
|
||||
- 根因:BUG-149 的宿主资源争用已由容器边界缓解,但剩余风险转移到容器内 `npm ci` 黑洞:安装命令缺少比 45 分钟 job-level timeout 更短的 hard deadline,且 npm fetch/retry 没有显式网络诊断边界;当前没有 OOM 或 PID 耗尽证据。
|
||||
- 修复:仍只将 `npm ci` 放入 digest-pinned Node 容器;保留 CPU 1.5、memory 2g、no swap、pids 256,并在容器内用 `timeout --signal=TERM --kill-after=30s 900s` 包住安装,避免外层 timeout 杀掉 docker CLI 后留下 orphan 容器。npm 增加 `--fetch-timeout=60000`、2 次重试和 1s/10s retry 上下限;超时或非零退出都输出明确诊断并 fail closed。测试、lint、build 与 exact-SHA 行为保持原样,不回退到宿主 npm。
|
||||
- 验证:本地 `node --test frontend/tests/staging-backend-workflows.test.ts` 32/32 passed;`git diff --check` 通过。远端 staging gate/deploy 待本提交后验证,本记录不提前声称远端收口。
|
||||
- 防复发:依赖安装必须维持 digest-pinned Node runtime、显式资源上限、命令级 hard timeout 与 npm 网络 timeout;合同测试需锁定只有 `npm ci` 在受限容器内执行,并持续确认测试、lint、build、exact-SHA checkout/publish/deploy 语义未漂移。
|
||||
- 相关记录:BUG-129、BUG-136、BUG-142
|
||||
- 复发自:无
|
||||
- 修复版本:待本次提交 / gate / deploy
|
||||
|
||||
@@ -208,6 +208,7 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest",
|
||||
)?.[0] ?? "";
|
||||
assert.match(installStep, /workdir="\$\(pwd -P\)"/);
|
||||
assert.match(installStep, /docker run --rm/);
|
||||
assert.doesNotMatch(installStep, /timeout --signal=TERM --kill-after=30s 900s docker run/);
|
||||
assert.match(installStep, /--cpus=1\.5/);
|
||||
assert.match(installStep, /--memory=2g/);
|
||||
assert.match(installStep, /--memory-swap=2g/);
|
||||
@@ -217,7 +218,10 @@ test("Gitea quality gate validates before publishing an immutable ACR manifest",
|
||||
assert.match(installStep, /--workdir "\$workdir"/);
|
||||
assert.match(installStep, /--env HOME=\/tmp/);
|
||||
assert.match(installStep, /--env "NPM_CONFIG_REGISTRY=\$NPM_CONFIG_REGISTRY"/);
|
||||
assert.match(installStep, /"\$NODE_TOOL_SOURCE_IMAGE" \\\n\s+npm ci --prefix frontend \\\n\s+--no-audit \\\n\s+--no-fund \\\n\s+--progress=false \\\n\s+--maxsockets=4/);
|
||||
assert.match(installStep, /"\$NODE_TOOL_SOURCE_IMAGE" \\\n\s+timeout --signal=TERM --kill-after=30s 900s \\\n\s+npm ci --prefix frontend \\\n\s+--no-audit \\\n\s+--no-fund \\\n\s+--progress=false \\\n\s+--maxsockets=4 \\\n\s+--fetch-timeout=60000 \\\n\s+--fetch-retries=2 \\\n\s+--fetch-retry-mintimeout=1000 \\\n\s+--fetch-retry-maxtimeout=10000/);
|
||||
assert.match(installStep, /npm_ci_status=\$\?/);
|
||||
assert.match(installStep, /frontend npm ci exceeded bounded 900-second timeout/);
|
||||
assert.match(installStep, /frontend npm ci failed with status \$npm_ci_status inside bounded Node container/);
|
||||
assert.doesNotMatch(installStep, /--ignore-scripts|--omit(?:=|\s+)optional|--force/);
|
||||
assert.match(workflow, /docker compose version --short/);
|
||||
assert.match(workflow, /docker compose --help \| grep -q -- '--project-name'/);
|
||||
|
||||
Reference in New Issue
Block a user