fix(ci): cap postgres fixtures and reclaim leftover compose networks

xiaoxin's 20-core npm test opened one Docker network per database file and exhausted default address pools. Limit concurrent fixtures and remove unused jyotisha-postgres networks before the gate.

Co-authored-by: Cursor <cursoragent@cursor.com>

deploy/reclaim-runner-disk.sh

@@ -29,6 +29,30 @@ docker container prune --force --filter until=6h
 docker volume ls --quiet --filter 'name=^jyotisha-postgres-' --filter dangling=true |
   xargs -r docker volume rm
 docker network prune --force --filter until=6h
+
+# Compose networks from crashed or finished fixtures keep occupying Docker's
+# default address pools until they are removed. `network prune --filter until=6h`
+# leaves same-day leftovers, and the next `docker compose up` then fails with
+# "all predefined address pools have been fully subnetted".
+# `docker network rm` refuses networks that still have endpoints, so a
+# concurrent job's live fixture is left alone.
+echo "removing unused jyotisha-postgres compose networks"
+removed_networks=0
+kept_networks=0
+while IFS= read -r network; do
+  if [ -z "$network" ]; then
+    continue
+  fi
+  if docker network rm "$network" >/dev/null 2>&1; then
+    echo "removed unused network $network"
+    removed_networks=$((removed_networks + 1))
+  else
+    echo "kept in-use network $network"
+    kept_networks=$((kept_networks + 1))
+  fi
+done < <(docker network ls --format '{{.Name}}' --filter 'name=jyotisha-postgres-')
+echo "jyotisha-postgres networks: removed=${removed_networks} kept=${kept_networks}"
+
 docker image prune --force
 
 # Exact-SHA release images are immutable in the registry once published, so the

docs/BUG_HISTORY.md

@@ -4183,3 +4183,19 @@
 - 相关记录：BUG-277（同一段结算逻辑；它删掉的缓冲正是兜底此前不触发的原因之一）、BUG-214（同为契约门与可见输出的耦合）、BUG-271（同一批「失败在回执里查不到」）
 - 复发自：无
 - 修复版本：待提交
+
+## BUG-281 | staging 质量门 1731/1735：xiaoxin 上 Docker 地址池耗尽，4 个 PostgreSQL fixture 起不来
+
+- 状态：resolved（本地修复，待提交与发布）
+- 首次发现：2026-08-18
+- 最近更新：2026-08-18
+- 影响面：Gitea `backend-quality-gate.yml` 的 `npm test --prefix frontend`、`frontend/tests/helpers/postgres-fixture.ts`、`deploy/reclaim-runner-disk.sh`、xiaoxin runner 上的 Docker user-defined networks
+- 用户现象：`927bdd7a` 的 validate 在 1735 项前端测试里红 4 项，日志末尾是两条已通过的 truth-source 合同。失败原文都是 `docker compose up -d --wait postgres` 创建 `jyotisha-postgres-*-app` 网络时返回 `all predefined address pools have been fully subnetted`。
+- 触发条件：向 `staging` 推送后，xiaoxin（20 核）上 `tsx --test` 按 `os.availableParallelism()` 并行跑全部 `frontend/tests/*.test.ts`。约 20 个文件会各自 `startPostgresFixture()`，每个 Compose 项目占用一个 user-defined network。
+- 根因：两层。其一，质量门跑的是全量并行 `npm test`，而 `test:db` 才把数据库套件串行化；20 核 runner 上一轮实测创建了 26 个 `jyotisha-postgres-*` 网络，其中 4 个在分配子网时失败。其二，BUG-266 的回收脚本只做 `docker network prune --filter until=6h`，同一天内崩溃或未拆掉的 fixture 网络继续占着 Docker default address pools；该 prune 本来就不会删仍有 endpoint 的网络，6 小时门槛对地址池没有额外保护，只会把当天残留留到下次 `compose up` 爆掉。失败的 4 项分别在 `database-local-business`、`identity-auth-integration`、`model-configuration-security`、`rectification-pr4-database`，与 `927bdd7a` 的咨询副运改动无关。
+- 修复：fixture 在 `docker compose up` 前领取最多 2 个并发槽（可用 `JYOTISHA_POSTGRES_MAX_FIXTURES` 覆盖），持有期覆盖整个 Compose 项目寿命；槽位用目录锁，进程已死则回收，启动失败与 `stop()` 都会释放。回收脚本在既有 6 小时 prune 之后，对名字匹配 `jyotisha-postgres-` 的网络逐个 `docker network rm`：空闲的删掉，仍有 endpoint 的留下并发 job。
+- 验证：`postgres-fixture-contract.test.ts` 2/2、`staging-backend-workflows.test.ts` 38/38 通过（含回收脚本会 `network rm` 残留 `jyotisha-postgres-` 网络、仍拒绝 `docker rm --force` / `system prune`，以及部署脚本 shell 语法）。未做的验证：没有在 xiaoxin 上复跑完整质量门。
+- 防复发：Docker user-defined network 和磁盘是两类资源。磁盘回收不能代替地址池回收；`until=6h` 对「今天刚留下的空网络」无效。每个并行测试文件一个 Compose 项目，在多核 runner 上会按核数打满 default address pools。数据库 fixture 必须自己限制并发，不能依赖 `tsx` 默认并行度。
+- 相关记录：BUG-266（同一 runner 上的磁盘耗尽与 6 小时网络 prune）、BUG-264（本地全量 `npm test` 并发跑 `database-*` fixture 的既有抖动）
+- 复发自：BUG-266（回收只覆盖磁盘与 6 小时以上空网络，未覆盖地址池）
+- 修复版本：待提交

frontend/tests/helpers/postgres-fixture.ts

@@ -3,6 +3,7 @@ import {
   chmodSync,
   mkdirSync,
   mkdtempSync,
+  readFileSync,
   rmSync,
   writeFileSync,
 } from "node:fs";
@@ -76,15 +77,101 @@ function releasePort(reservation: PortReservation): void {
   rmSync(reservation.directory, { force: true, recursive: true });
 }
 
+const DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES = 2;
+const FIXTURE_SLOT_WAIT_MS = 5 * 60 * 1000;
+
+type FixtureSlot = { id: number; release(): void };
+
+function maxConcurrentPostgresFixtures(): number {
+  const raw = process.env.JYOTISHA_POSTGRES_MAX_FIXTURES;
+  if (raw === undefined || raw === "") {
+    return DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES;
+  }
+  const parsed = Number(raw);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error("JYOTISHA_POSTGRES_MAX_FIXTURES must be a positive integer");
+  }
+  return parsed;
+}
+
+function fixtureSlotsRoot(): string {
+  return join(tmpdir(), "jyotisha-postgres-slots");
+}
+
+function fixtureSlotDirectory(id: number): string {
+  return join(fixtureSlotsRoot(), `slot-${id}`);
+}
+
+function pidIsAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) {
+    return false;
+  }
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function tryClaimSlot(id: number): boolean {
+  const directory = fixtureSlotDirectory(id);
+  mkdirSync(fixtureSlotsRoot(), { recursive: true });
+  try {
+    mkdirSync(directory);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    try {
+      const pid = Number(readFileSync(join(directory, "pid"), "utf8"));
+      if (pidIsAlive(pid)) return false;
+      rmSync(directory, { force: true, recursive: true });
+      mkdirSync(directory);
+    } catch {
+      return false;
+    }
+  }
+  writeFileSync(join(directory, "pid"), String(process.pid));
+  return true;
+}
+
+function acquireFixtureSlot(): FixtureSlot {
+  const limit = maxConcurrentPostgresFixtures();
+  const deadline = Date.now() + FIXTURE_SLOT_WAIT_MS;
+  while (Date.now() < deadline) {
+    for (let id = 0; id < limit; id += 1) {
+      if (tryClaimSlot(id)) {
+        return {
+          id,
+          release() {
+            rmSync(fixtureSlotDirectory(id), { force: true, recursive: true });
+          },
+        };
+      }
+    }
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
+  }
+  throw new Error(
+    `timed out waiting for a PostgreSQL fixture slot (${limit} concurrent compose networks). Docker address pools cannot host one network per parallel test file.`,
+  );
+}
+
 export function startPostgresFixture(): PostgresFixture {
+  const slot = acquireFixtureSlot();
   const projectName = `jyotisha-postgres-${process.pid}-${Date.now()}`;
-  const temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-"));
+  let temporaryDirectory: string;
+  try {
+    temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-"));
+  } catch (error) {
+    slot.release();
+    throw error;
+  }
   const databaseEnvFile = join(temporaryDirectory, "database.env");
   let portReservation: PortReservation;
   try {
     portReservation = reserveAvailablePort();
   } catch (error) {
     rmSync(temporaryDirectory, { force: true, recursive: true });
+    slot.release();
     throw error;
   }
   const hostPort = portReservation.port;
@@ -137,6 +224,7 @@ export function startPostgresFixture(): PostgresFixture {
     } finally {
       releasePort(portReservation);
       rmSync(temporaryDirectory, { force: true, recursive: true });
+      slot.release();
     }
     throw error;
   }
@@ -190,6 +278,7 @@ export function startPostgresFixture(): PostgresFixture {
       } finally {
         releasePort(portReservation);
         rmSync(temporaryDirectory, { force: true, recursive: true });
+        slot.release();
       }
     },
   };

frontend/tests/postgres-fixture-contract.test.ts

@@ -0,0 +1,35 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const fixtureSource = readFileSync(
+  fileURLToPath(new URL("./helpers/postgres-fixture.ts", import.meta.url)),
+  "utf8",
+);
+
+test("postgres fixtures cap concurrent compose networks so Docker address pools cannot be exhausted", () => {
+  assert.match(fixtureSource, /JYOTISHA_POSTGRES_MAX_FIXTURES/);
+  assert.match(fixtureSource, /DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES = 2/);
+  assert.match(fixtureSource, /function acquireFixtureSlot\(/);
+  assert.match(fixtureSource, /process\.kill\(\s*pid,\s*0\s*\)/);
+  assert.match(
+    fixtureSource,
+    /export function startPostgresFixture\(\)[\s\S]*acquireFixtureSlot\(\)/,
+  );
+  assert.match(
+    fixtureSource,
+    /timed out waiting for a PostgreSQL fixture slot/,
+  );
+});
+
+test("postgres fixture start failure and stop both release the compose-network slot", () => {
+  assert.match(
+    fixtureSource,
+    /catch \(error\) \{[\s\S]*releasePort\(portReservation\)[\s\S]*slot\.release\(\)/,
+  );
+  assert.match(
+    fixtureSource,
+    /stop\(\) \{[\s\S]*releasePort\(portReservation\)[\s\S]*slot\.release\(\)/,
+  );
+});

frontend/tests/staging-backend-workflows.test.ts

@@ -407,13 +407,18 @@ test("both gate jobs reclaim runner disk before they need it, and only unheld re
   assert.match(script, /docker container prune --force --filter until=6h/);
   assert.match(script, /--filter 'name=\^jyotisha-postgres-' --filter dangling=true/);
   assert.match(script, /docker network prune --force --filter until=6h/);
+  assert.match(script, /docker network ls[^\n]*--filter 'name=jyotisha-postgres-'/);
+  assert.match(script, /docker network rm "\$network"/);
+  assert.match(script, /all predefined address pools have been fully subnetted/);
   assert.match(script, /docker image prune --force\n/);
   assert.match(script, /docker builder prune --force --all/);
   assert.match(script, /grep -v -F -e "api-\$KEEP_TAG_SHA" -e "web-\$KEEP_TAG_SHA"/);
   assert.match(script, /PostgreSQL fixtures and image builds need at least \$\{MINIMUM_FREE_GIB\} GiB/);
   // A gate that silently proceeds on a full disk fails 23 database tests instead
   // of naming the disk, and pruning what a container still holds would delete a
-  // concurrent job's fixture out from under it.
+  // concurrent job's fixture out from under it. Network rm is the same rule:
+  // unused jyotisha-postgres leftovers occupy address pools, but an in-use
+  // network still has endpoints and docker network rm will refuse it.
   assert.match(script, /if \[ "\$AFTER_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+exit 1/);
   assert.doesNotMatch(script, /docker system prune|--volumes|prune[^\n]*--all --force|docker (?:rm|volume rm|kill)[^\n]*--force/);
 });