diff --git a/deploy/reclaim-runner-disk.sh b/deploy/reclaim-runner-disk.sh index 38c31802..4f40f12e 100755 --- a/deploy/reclaim-runner-disk.sh +++ b/deploy/reclaim-runner-disk.sh @@ -29,6 +29,30 @@ docker container prune --force --filter until=6h docker volume ls --quiet --filter 'name=^jyotisha-postgres-' --filter dangling=true | xargs -r docker volume rm docker network prune --force --filter until=6h + +# Compose networks from crashed or finished fixtures keep occupying Docker's +# default address pools until they are removed. `network prune --filter until=6h` +# leaves same-day leftovers, and the next `docker compose up` then fails with +# "all predefined address pools have been fully subnetted". +# `docker network rm` refuses networks that still have endpoints, so a +# concurrent job's live fixture is left alone. +echo "removing unused jyotisha-postgres compose networks" +removed_networks=0 +kept_networks=0 +while IFS= read -r network; do + if [ -z "$network" ]; then + continue + fi + if docker network rm "$network" >/dev/null 2>&1; then + echo "removed unused network $network" + removed_networks=$((removed_networks + 1)) + else + echo "kept in-use network $network" + kept_networks=$((kept_networks + 1)) + fi +done < <(docker network ls --format '{{.Name}}' --filter 'name=jyotisha-postgres-') +echo "jyotisha-postgres networks: removed=${removed_networks} kept=${kept_networks}" + docker image prune --force # Exact-SHA release images are immutable in the registry once published, so the diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 37de57dd..3f72e8ea 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -4183,3 +4183,19 @@ - 相关记录：BUG-277（同一段结算逻辑；它删掉的缓冲正是兜底此前不触发的原因之一）、BUG-214（同为契约门与可见输出的耦合）、BUG-271（同一批「失败在回执里查不到」） - 复发自：无 - 修复版本：待提交 + +## BUG-281 | staging 质量门 1731/1735：xiaoxin 上 Docker 地址池耗尽，4 个 PostgreSQL fixture 起不来 + +- 状态：resolved（本地修复，待提交与发布） +- 首次发现：2026-08-18 +- 最近更新：2026-08-18 +- 影响面：Gitea `backend-quality-gate.yml` 的 `npm test --prefix frontend`、`frontend/tests/helpers/postgres-fixture.ts`、`deploy/reclaim-runner-disk.sh`、xiaoxin runner 上的 Docker user-defined networks +- 用户现象：`927bdd7a` 的 validate 在 1735 项前端测试里红 4 项，日志末尾是两条已通过的 truth-source 合同。失败原文都是 `docker compose up -d --wait postgres` 创建 `jyotisha-postgres-*-app` 网络时返回 `all predefined address pools have been fully subnetted`。 +- 触发条件：向 `staging` 推送后，xiaoxin（20 核）上 `tsx --test` 按 `os.availableParallelism()` 并行跑全部 `frontend/tests/*.test.ts`。约 20 个文件会各自 `startPostgresFixture()`，每个 Compose 项目占用一个 user-defined network。 +- 根因：两层。其一，质量门跑的是全量并行 `npm test`，而 `test:db` 才把数据库套件串行化；20 核 runner 上一轮实测创建了 26 个 `jyotisha-postgres-*` 网络，其中 4 个在分配子网时失败。其二，BUG-266 的回收脚本只做 `docker network prune --filter until=6h`，同一天内崩溃或未拆掉的 fixture 网络继续占着 Docker default address pools；该 prune 本来就不会删仍有 endpoint 的网络，6 小时门槛对地址池没有额外保护，只会把当天残留留到下次 `compose up` 爆掉。失败的 4 项分别在 `database-local-business`、`identity-auth-integration`、`model-configuration-security`、`rectification-pr4-database`，与 `927bdd7a` 的咨询副运改动无关。 +- 修复：fixture 在 `docker compose up` 前领取最多 2 个并发槽（可用 `JYOTISHA_POSTGRES_MAX_FIXTURES` 覆盖），持有期覆盖整个 Compose 项目寿命；槽位用目录锁，进程已死则回收，启动失败与 `stop()` 都会释放。回收脚本在既有 6 小时 prune 之后，对名字匹配 `jyotisha-postgres-` 的网络逐个 `docker network rm`：空闲的删掉，仍有 endpoint 的留下并发 job。 +- 验证：`postgres-fixture-contract.test.ts` 2/2、`staging-backend-workflows.test.ts` 38/38 通过（含回收脚本会 `network rm` 残留 `jyotisha-postgres-` 网络、仍拒绝 `docker rm --force` / `system prune`，以及部署脚本 shell 语法）。未做的验证：没有在 xiaoxin 上复跑完整质量门。 +- 防复发：Docker user-defined network 和磁盘是两类资源。磁盘回收不能代替地址池回收；`until=6h` 对「今天刚留下的空网络」无效。每个并行测试文件一个 Compose 项目，在多核 runner 上会按核数打满 default address pools。数据库 fixture 必须自己限制并发，不能依赖 `tsx` 默认并行度。 +- 相关记录：BUG-266（同一 runner 上的磁盘耗尽与 6 小时网络 prune）、BUG-264（本地全量 `npm test` 并发跑 `database-*` fixture 的既有抖动） +- 复发自：BUG-266（回收只覆盖磁盘与 6 小时以上空网络，未覆盖地址池） +- 修复版本：待提交 diff --git a/frontend/tests/helpers/postgres-fixture.ts b/frontend/tests/helpers/postgres-fixture.ts index b696e6bd..d8fc641f 100644 --- a/frontend/tests/helpers/postgres-fixture.ts +++ b/frontend/tests/helpers/postgres-fixture.ts @@ -3,6 +3,7 @@ import { chmodSync, mkdirSync, mkdtempSync, + readFileSync, rmSync, writeFileSync, } from "node:fs"; @@ -76,15 +77,101 @@ function releasePort(reservation: PortReservation): void { rmSync(reservation.directory, { force: true, recursive: true }); } +const DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES = 2; +const FIXTURE_SLOT_WAIT_MS = 5 * 60 * 1000; + +type FixtureSlot = { id: number; release(): void }; + +function maxConcurrentPostgresFixtures(): number { + const raw = process.env.JYOTISHA_POSTGRES_MAX_FIXTURES; + if (raw === undefined || raw === "") { + return DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES; + } + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error("JYOTISHA_POSTGRES_MAX_FIXTURES must be a positive integer"); + } + return parsed; +} + +function fixtureSlotsRoot(): string { + return join(tmpdir(), "jyotisha-postgres-slots"); +} + +function fixtureSlotDirectory(id: number): string { + return join(fixtureSlotsRoot(), `slot-${id}`); +} + +function pidIsAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function tryClaimSlot(id: number): boolean { + const directory = fixtureSlotDirectory(id); + mkdirSync(fixtureSlotsRoot(), { recursive: true }); + try { + mkdirSync(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + try { + const pid = Number(readFileSync(join(directory, "pid"), "utf8")); + if (pidIsAlive(pid)) return false; + rmSync(directory, { force: true, recursive: true }); + mkdirSync(directory); + } catch { + return false; + } + } + writeFileSync(join(directory, "pid"), String(process.pid)); + return true; +} + +function acquireFixtureSlot(): FixtureSlot { + const limit = maxConcurrentPostgresFixtures(); + const deadline = Date.now() + FIXTURE_SLOT_WAIT_MS; + while (Date.now() < deadline) { + for (let id = 0; id < limit; id += 1) { + if (tryClaimSlot(id)) { + return { + id, + release() { + rmSync(fixtureSlotDirectory(id), { force: true, recursive: true }); + }, + }; + } + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50); + } + throw new Error( + `timed out waiting for a PostgreSQL fixture slot (${limit} concurrent compose networks). Docker address pools cannot host one network per parallel test file.`, + ); +} + export function startPostgresFixture(): PostgresFixture { + const slot = acquireFixtureSlot(); const projectName = `jyotisha-postgres-${process.pid}-${Date.now()}`; - const temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-")); + let temporaryDirectory: string; + try { + temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-")); + } catch (error) { + slot.release(); + throw error; + } const databaseEnvFile = join(temporaryDirectory, "database.env"); let portReservation: PortReservation; try { portReservation = reserveAvailablePort(); } catch (error) { rmSync(temporaryDirectory, { force: true, recursive: true }); + slot.release(); throw error; } const hostPort = portReservation.port; @@ -137,6 +224,7 @@ export function startPostgresFixture(): PostgresFixture { } finally { releasePort(portReservation); rmSync(temporaryDirectory, { force: true, recursive: true }); + slot.release(); } throw error; } @@ -190,6 +278,7 @@ export function startPostgresFixture(): PostgresFixture { } finally { releasePort(portReservation); rmSync(temporaryDirectory, { force: true, recursive: true }); + slot.release(); } }, }; diff --git a/frontend/tests/postgres-fixture-contract.test.ts b/frontend/tests/postgres-fixture-contract.test.ts new file mode 100644 index 00000000..74acbf7c --- /dev/null +++ b/frontend/tests/postgres-fixture-contract.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const fixtureSource = readFileSync( + fileURLToPath(new URL("./helpers/postgres-fixture.ts", import.meta.url)), + "utf8", +); + +test("postgres fixtures cap concurrent compose networks so Docker address pools cannot be exhausted", () => { + assert.match(fixtureSource, /JYOTISHA_POSTGRES_MAX_FIXTURES/); + assert.match(fixtureSource, /DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES = 2/); + assert.match(fixtureSource, /function acquireFixtureSlot\(/); + assert.match(fixtureSource, /process\.kill\(\s*pid,\s*0\s*\)/); + assert.match( + fixtureSource, + /export function startPostgresFixture\(\)[\s\S]*acquireFixtureSlot\(\)/, + ); + assert.match( + fixtureSource, + /timed out waiting for a PostgreSQL fixture slot/, + ); +}); + +test("postgres fixture start failure and stop both release the compose-network slot", () => { + assert.match( + fixtureSource, + /catch \(error\) \{[\s\S]*releasePort\(portReservation\)[\s\S]*slot\.release\(\)/, + ); + assert.match( + fixtureSource, + /stop\(\) \{[\s\S]*releasePort\(portReservation\)[\s\S]*slot\.release\(\)/, + ); +}); diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index acd73fe3..0e83661a 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -407,13 +407,18 @@ test("both gate jobs reclaim runner disk before they need it, and only unheld re assert.match(script, /docker container prune --force --filter until=6h/); assert.match(script, /--filter 'name=\^jyotisha-postgres-' --filter dangling=true/); assert.match(script, /docker network prune --force --filter until=6h/); + assert.match(script, /docker network ls[^\n]*--filter 'name=jyotisha-postgres-'/); + assert.match(script, /docker network rm "\$network"/); + assert.match(script, /all predefined address pools have been fully subnetted/); assert.match(script, /docker image prune --force\n/); assert.match(script, /docker builder prune --force --all/); assert.match(script, /grep -v -F -e "api-\$KEEP_TAG_SHA" -e "web-\$KEEP_TAG_SHA"/); assert.match(script, /PostgreSQL fixtures and image builds need at least \$\{MINIMUM_FREE_GIB\} GiB/); // A gate that silently proceeds on a full disk fails 23 database tests instead // of naming the disk, and pruning what a container still holds would delete a - // concurrent job's fixture out from under it. + // concurrent job's fixture out from under it. Network rm is the same rule: + // unused jyotisha-postgres leftovers occupy address pools, but an in-use + // network still has endpoints and docker network rm will refuse it. assert.match(script, /if \[ "\$AFTER_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+exit 1/); assert.doesNotMatch(script, /docker system prune|--volumes|prune[^\n]*--all --force|docker (?:rm|volume rm|kill)[^\n]*--force/); });