diff --git a/deploy/run-staging-deploy.sh b/deploy/run-staging-deploy.sh index 4bffa70a..c5ec0858 100755 --- a/deploy/run-staging-deploy.sh +++ b/deploy/run-staging-deploy.sh @@ -143,6 +143,16 @@ export SITE_ADDRESS='https://staging.jyotisha.chat' export GITHUB_SHA="$DEPLOY_SHA" "${compose[@]}" config --quiet +# Exact-SHA images accumulate after each deploy. Pulling a new digest then +# fails with containerd "no space left on device" (run 2841 / f9685f28). +# Running jyotisha-staging containers keep their layers; unused tags go. +# Do not prune named volumes: that would risk the Postgres data volume. +echo "staging host docker disk before image reclaim" +"${docker_command[@]}" system df || true +"${docker_command[@]}" image prune --force +"${docker_command[@]}" image prune --all --force +echo "staging host docker disk after image reclaim" +"${docker_command[@]}" system df || true "${compose[@]}" pull api web "${compose[@]}" up -d --no-build --pull never --wait postgres diff --git a/deploy/run-staging-migration.sh b/deploy/run-staging-migration.sh index 6cb35c54..9fba41fb 100755 --- a/deploy/run-staging-migration.sh +++ b/deploy/run-staging-migration.sh @@ -85,6 +85,16 @@ bash deploy/validate-staging-database-env.sh .env.staging.database export DATABASE_ENV_FILE='../.env.staging.database' compose=("${docker_command[@]}" compose -p jyotisha-staging -f deploy/docker-compose.postgres.yml) +# Exact-SHA images accumulate after each deploy. Pulling a new digest then +# fails with containerd "no space left on device" (run 2841 / f9685f28). +# Running jyotisha-staging containers keep their layers; unused tags go. +# Do not prune named volumes: that would risk the Postgres data volume. +echo "staging host docker disk before image reclaim" +"${docker_command[@]}" system df || true +"${docker_command[@]}" image prune --force +"${docker_command[@]}" image prune --all --force +echo "staging host docker disk after image reclaim" +"${docker_command[@]}" system df || true "${docker_command[@]}" pull "$WEB_IMAGE" "${compose[@]}" up -d --no-build --pull never --wait postgres "${compose[@]}" exec -T postgres psql -v ON_ERROR_STOP=1 -U postgres -d jyotisha \ diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index a77cba1a..3219f58c 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -13166,4 +13166,20 @@ - 防复发:会话标题和行内操作必须共享同一个行级状态面。不得给 `.session-menu-trigger` 单独的 canvas/选中底。 - 相关记录:BUG-024 - 复发自:BUG-024(旧测试没锁住 trigger 的 canvas 悬停底) -- 修复版本:待本修复合入 staging +- 修复版本:`f9685f28`(已进 staging;部署被 BUG-994 / migrate run 2841 磁盘写满挡住) + +## BUG-994 | staging 迁移拉 digest 镜像时 containerd 磁盘写满 + +- 状态:investigating +- 首次发现:2026-09-21 +- 最近更新:2026-09-21 +- 影响面:`deploy/run-staging-migration.sh`、`deploy/run-staging-deploy.sh`、staging 主机 Docker 镜像层、`migrate-staging-database` run 2841 +- 用户现象:无终端用户可见功能回归。`f9685f28` 门禁 validate 全绿、镜像已发布,但测试站没有切到该 SHA:迁移在主机 `docker pull` 新 web digest 时失败,后续 `deploy-staging` 未执行。当时线上仍是上一版 `d2cec179`。 +- 触发条件:向 staging 推送会触发门禁的改动后,publish 派发 `migrate-staging-database`,主机再拉一枚新的 exact-SHA web 镜像。 +- 根因:每次部署留下不可变 digest 镜像,脚本在 pull 前不回收未使用镜像。run 2841 在 `Apply digest-pinned migration under host lock` 拉 `copse/jyotisha@sha256:a00887de…` 时,containerd ingest 报 `write /var/lib/containerd/io.containerd.content.v1.content/ingest/631c40bfaf7318a3ce6ca6ead3b0f8ce0eb25e2a2e93851c76bfc9eb5060c25d/data: no space left on device`。门禁 runner 的 `reclaim-runner-disk.sh` 只管构建机,不管 staging 主机。 +- 修复:迁移与部署脚本在 pull 前打印 `docker system df`,执行 `image prune --force` 再 `image prune --all --force`。正在跑的 `jyotisha-staging` 容器会保住当前层。禁止 `volume prune`,避免误删 Postgres 数据卷。 +- 验证:源码合同 `staging-backend-workflows.test.ts` 锁定 prune 出现在 pull 之前且不得 `volume prune`。端到端证据待本轮门禁的 migrate / deploy 转绿;转绿前不得标 resolved。 +- 防复发:staging 主机拉 digest 前必须先回收未使用镜像。不得用 `volume prune` 换磁盘。 +- 相关记录:BUG-150、BUG-993 +- 复发自:无 +- 修复版本:待本轮含 prune-before-pull 的提交合入 staging 且 migrate 转绿 diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 756eea4e..52592dba 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -1008,7 +1008,13 @@ test("normal deployment checks migrations but never applies them", () => { for (const workflow of stagingWorkflows) { assert.doesNotMatch(workflow, /SERVICE_RUNTIME_PASSWORD|SERVICE_DATABASE_URL/); } + // 原值: pull api web 前不回收镜像 + // 新值: image prune --all --force 必须出现在 pull 之前,且不得 volume prune + // 原因: BUG-994 / migrate run 2841 在 staging 主机拉 digest 时 containerd 写满 + assert.match(runner, /image prune --all --force/); + assert.doesNotMatch(runner, /volume prune --/); assertOrder(runner, [ + "image prune --all --force", "pull api web", "up -d --no-build --pull never --wait postgres", "--profile migration-check run --rm migration-checker", @@ -1036,6 +1042,15 @@ test("manual migration uses only PostgreSQL and the digest-pinned migrator", () assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); assert.match(runner, /"\$\{docker_command\[@\]\}" pull "\$WEB_IMAGE"/); + // 原值: docker pull "$WEB_IMAGE" 前不回收镜像 + // 新值: image prune --all --force 必须出现在 pull 之前,且不得 volume prune + // 原因: BUG-994 / migrate run 2841 在 staging 主机拉 digest 时 containerd 写满 + assert.match(runner, /image prune --all --force/); + assert.doesNotMatch(runner, /volume prune --/); + assertOrder(runner, [ + "image prune --all --force", + '"${docker_command[@]}" pull "$WEB_IMAGE"', + ]); assert.match(runner, /up -d --no-build --pull never --wait postgres/); assert.match(runner, /-f deploy\/docker-compose\.postgres\.yml/); assertOrder(runner, [