diff --git a/deploy/railway-api.Dockerfile b/deploy/railway-api.Dockerfile index b96fcc3e..49672296 100644 --- a/deploy/railway-api.Dockerfile +++ b/deploy/railway-api.Dockerfile @@ -2,15 +2,18 @@ FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/python:3.12-slim ENV PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 \ - PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ \ + PIP_INDEX_URL=https://mirrors.huaweicloud.com/repository/pypi/simple/ \ PIP_DEFAULT_TIMEOUT=60 WORKDIR /app COPY requirements.txt deploy/patch_vedastro_update_check.py ./ -RUN sed -i 's|http://deb.debian.org|https://mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources \ +# Apt packages are their own layer so a later pip timeout can reuse them. +# Huawei mirrors share the SWR path already used for the base image; Aliyun +# debian/pypi at ~22 kB/s cannot finish this RUN inside the 60-minute publish job. +RUN sed -i 's|http://deb.debian.org|https://mirrors.huaweicloud.com|g' /etc/apt/sources.list.d/debian.sources \ && apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update \ - && apt-get install -y --no-install-recommends build-essential nodejs \ - && python -m pip install -r requirements.txt \ + && apt-get install -y --no-install-recommends build-essential nodejs +RUN python -m pip install -r requirements.txt \ && python patch_vedastro_update_check.py \ && apt-get purge -y --auto-remove build-essential \ && rm -rf /var/lib/apt/lists/* diff --git a/deploy/reclaim-runner-disk.sh b/deploy/reclaim-runner-disk.sh index 3f6ea914..60d70dc0 100755 --- a/deploy/reclaim-runner-disk.sh +++ b/deploy/reclaim-runner-disk.sh @@ -66,20 +66,26 @@ if [ -n "$STALE_IMAGES" ]; then printf '%s\n' "$STALE_IMAGES" | xargs -r docker image rm || true fi -# BuildKit layer cache is what lets the publish job reuse the Dockerfile -# `npm ci` layer instead of rebuilding it from scratch behind the mirror (a -# 7-minute image build versus 47). `docker image prune --force` above removes -# only dangling images; BuildKit cache records live in the builder store, not -# in dangling images, so nothing before this point touches them. Reclaim them -# in tiers: drop entries nobody has used for 72 hours, re-measure, and escalate -# to `--all` only when the runner is still below the threshold. -docker builder prune --force --filter until=72h - -TIERED_GIB="$(free_gib)" -echo "docker root $DOCKER_ROOT has ${TIERED_GIB} GiB free after aged build-cache reclaim" -if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then - echo "still below ${MINIMUM_FREE_GIB} GiB; escalating to a full BuildKit cache prune" - docker builder prune --force --all +# BuildKit layer cache is what lets the publish job reuse apt/pip/`npm ci` +# layers instead of rebuilding behind a slow mirror (7-minute image builds +# versus 47–60). `docker image prune --force` above removes only dangling +# images; BuildKit cache lives in the builder store. Do not touch it while +# the runner still has at least MINIMUM_FREE_GIB. Aged and full prunes are +# reachable only after re-measuring and finding the disk actually short. +CACHE_GIB="$(free_gib)" +echo "docker root $DOCKER_ROOT has ${CACHE_GIB} GiB free before BuildKit reclaim" +if [ "$CACHE_GIB" -lt "$MINIMUM_FREE_GIB" ]; then + echo "below ${MINIMUM_FREE_GIB} GiB; pruning BuildKit cache unused for 72h" + docker builder prune --force --filter until=72h + TIERED_GIB="$(free_gib)" + echo "docker root $DOCKER_ROOT has ${TIERED_GIB} GiB free after aged build-cache reclaim" + if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then + echo "still below ${MINIMUM_FREE_GIB} GiB; escalating to a full BuildKit cache prune" + docker builder prune --force --all + fi +else + echo "keeping BuildKit layer cache; ${CACHE_GIB} GiB free" + TIERED_GIB="$CACHE_GIB" fi AFTER_GIB="$(free_gib)" diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 52c5945b..a879402b 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -13039,3 +13039,19 @@ - 相关记录：BUG-981、BUG-733、BUG-712；F3 同步确认 BUG-984 为 BUG-981 端到端阻塞项。 - 复发自：BUG-733（同进程可证的等价关系被跨机 golden 代替）。 - 修复版本：`25232ce4`，第三环境 review 通过；本次按产品授权合入 staging，详见 `docs/tasks/PROGRESS-rectification-cross-midnight-gate-fix-20260920.md`。 + +## BUG-986 | staging publish 冷构建在 22 kB/s 镜像源下撞 60 分钟超时 + +- 状态：investigating +- 首次发现：2026-09-21 +- 最近更新：2026-09-21 +- 影响面：Gitea `backend-quality-gate` publish job、`deploy/railway-api.Dockerfile`、`deploy/reclaim-runner-disk.sh` +- 用户现象：run 2828 的 validate 通过后，`Build and publish exact-SHA ACR images` 在 API 镜像 `pip install pandas` 期间被取消；结论 `failure`，`context deadline exceeded`。 +- 触发条件：向 staging 推送代码后 publish；runner 空闲 677 GiB，但距上次命中的 API 依赖层已超过 72 小时。 +- 根因：`reclaim-runner-disk.sh` 在磁盘充足时仍执行 `docker builder prune --filter until=72h`，丢掉 BuildKit 层缓存。随后 Aliyun debian/pypi 约 22 kB/s：`apt-get update` 9.4 MB 用 7 分、102 MB 软件包约 40 分，pyswisseph 6.9 MB 又 5 分，pandas 12.4 MB 下载未完成即达 publish 的 60 分钟上限。业务测试未失败。 +- 修复：磁盘不低于 `MINIMUM_FREE_GIB` 时保留 BuildKit 缓存；API 镜像 apt 与 pip 分两层，apt/pypi 改走华为云镜像（与 SWR 基础镜像同路径）。不改 workflow 超时、评分或确认门。 +- 验证：定向更新 `staging-backend-workflows.test.ts` 与 `test_railway_deployment.py`；远端以新 staging 门禁为准，本记录不提前标 resolved。 +- 防复发：磁盘充足时不得 prune BuildKit；API 依赖安装不得把 apt 与 pip 绑在同一层以致超时后整层作废。 +- 相关记录：BUG-142 +- 复发自：BUG-142（publish 超时预算；本次是缓存被过早丢掉后的冷构建） +- 修复版本：待本修复合入 staging 的门禁 run diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index f9cc2ef1..756eea4e 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -406,6 +406,7 @@ test("both gate jobs reclaim runner disk before they need it, and only unheld re assert.match(script, /docker image prune --force\n/); assert.match(script, /docker builder prune --force --filter until=72h/); assert.match(script, /docker builder prune --force --all/); + assert.match(script, /keeping BuildKit layer cache/); assert.match(script, /grep -v -F -e "api-\$KEEP_TAG_SHA" -e "web-\$KEEP_TAG_SHA"/); assert.match(script, /PostgreSQL fixtures and image builds need at least \$\{MINIMUM_FREE_GIB\} GiB/); // A gate that silently proceeds on a full disk fails 23 database tests instead @@ -1307,21 +1308,32 @@ test("runner disk reclaim keeps BuildKit cache unless the runner is actually sho // Unconditionally pruning the whole BuildKit store made every publish job // rebuild the Dockerfile `npm ci` layer from scratch (47-minute image builds). - // Aged cache goes first; `--all` is reachable only after re-measuring and - // finding the runner still below MINIMUM_FREE_GIB. + // Run 2828 then showed the remaining 72h prune still drops that cache when + // disk is plentiful, forcing a 22 kB/s apt/pip rebuild that hits the + // 60-minute publish timeout. Aged and `--all` prunes run only after the + // post-image-prune measurement is below MINIMUM_FREE_GIB. assert.doesNotMatch(script, /^docker builder prune --force --all$/m); + assert.doesNotMatch(script, /^docker builder prune --force --filter until=72h$/m); assert.equal((script.match(/docker builder prune --force --all/g) ?? []).length, 1); + assert.equal((script.match(/docker builder prune --force --filter until=72h/g) ?? []).length, 1); assertOrder(script, [ "docker image prune --force", + 'CACHE_GIB="$(free_gib)"', + 'if [ "$CACHE_GIB" -lt "$MINIMUM_FREE_GIB" ]; then', "docker builder prune --force --filter until=72h", 'TIERED_GIB="$(free_gib)"', 'if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then', "docker builder prune --force --all", + "keeping BuildKit layer cache", 'AFTER_GIB="$(free_gib)"', ]); assert.match( script, - /if \[ "\$TIERED_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+docker builder prune --force --all\nfi\n/, + /if \[ "\$CACHE_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+docker builder prune --force --filter until=72h/, + ); + assert.match( + script, + /if \[ "\$TIERED_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+docker builder prune --force --all\n\s+fi\n/, ); // The final threshold check still fails the job instead of silently proceeding. assert.match(script, /if \[ "\$AFTER_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+exit 1/); diff --git a/tests/test_railway_deployment.py b/tests/test_railway_deployment.py index 382ac665..99d1682e 100644 --- a/tests/test_railway_deployment.py +++ b/tests/test_railway_deployment.py @@ -15,6 +15,11 @@ def test_railway_services_use_the_product_frontend_and_dynamic_ports() -> None: assert "FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/python:3.12-slim" in api assert "daocloud" not in api.lower() + assert "PIP_INDEX_URL=https://mirrors.huaweicloud.com/repository/pypi/simple/" in api + assert "mirrors.aliyun.com" not in api + assert api.index("apt-get install -y --no-install-recommends build-essential nodejs") < api.index( + "python -m pip install -r requirements.txt" + ) assert "COPY SKILL.md mcp_server.py" in api assert "--host 0.0.0.0" in api and "${PORT:-5200}" in api assert "http.server" not in api