fix(ci): cap postgres fixtures and reclaim leftover compose networks

xiaoxin's 20-core npm test opened one Docker network per database file and exhausted default address pools. Limit concurrent fixtures and remove unused jyotisha-postgres networks before the gate.

Co-authored-by: Cursor <cursoragent@cursor.com>

frontend/tests/helpers/postgres-fixture.ts

@@ -3,6 +3,7 @@ import {
   chmodSync,
   mkdirSync,
   mkdtempSync,
+  readFileSync,
   rmSync,
   writeFileSync,
 } from "node:fs";
@@ -76,15 +77,101 @@ function releasePort(reservation: PortReservation): void {
   rmSync(reservation.directory, { force: true, recursive: true });
 }
 
+const DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES = 2;
+const FIXTURE_SLOT_WAIT_MS = 5 * 60 * 1000;
+
+type FixtureSlot = { id: number; release(): void };
+
+function maxConcurrentPostgresFixtures(): number {
+  const raw = process.env.JYOTISHA_POSTGRES_MAX_FIXTURES;
+  if (raw === undefined || raw === "") {
+    return DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES;
+  }
+  const parsed = Number(raw);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error("JYOTISHA_POSTGRES_MAX_FIXTURES must be a positive integer");
+  }
+  return parsed;
+}
+
+function fixtureSlotsRoot(): string {
+  return join(tmpdir(), "jyotisha-postgres-slots");
+}
+
+function fixtureSlotDirectory(id: number): string {
+  return join(fixtureSlotsRoot(), `slot-${id}`);
+}
+
+function pidIsAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) {
+    return false;
+  }
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function tryClaimSlot(id: number): boolean {
+  const directory = fixtureSlotDirectory(id);
+  mkdirSync(fixtureSlotsRoot(), { recursive: true });
+  try {
+    mkdirSync(directory);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    try {
+      const pid = Number(readFileSync(join(directory, "pid"), "utf8"));
+      if (pidIsAlive(pid)) return false;
+      rmSync(directory, { force: true, recursive: true });
+      mkdirSync(directory);
+    } catch {
+      return false;
+    }
+  }
+  writeFileSync(join(directory, "pid"), String(process.pid));
+  return true;
+}
+
+function acquireFixtureSlot(): FixtureSlot {
+  const limit = maxConcurrentPostgresFixtures();
+  const deadline = Date.now() + FIXTURE_SLOT_WAIT_MS;
+  while (Date.now() < deadline) {
+    for (let id = 0; id < limit; id += 1) {
+      if (tryClaimSlot(id)) {
+        return {
+          id,
+          release() {
+            rmSync(fixtureSlotDirectory(id), { force: true, recursive: true });
+          },
+        };
+      }
+    }
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
+  }
+  throw new Error(
+    `timed out waiting for a PostgreSQL fixture slot (${limit} concurrent compose networks). Docker address pools cannot host one network per parallel test file.`,
+  );
+}
+
 export function startPostgresFixture(): PostgresFixture {
+  const slot = acquireFixtureSlot();
   const projectName = `jyotisha-postgres-${process.pid}-${Date.now()}`;
-  const temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-"));
+  let temporaryDirectory: string;
+  try {
+    temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-"));
+  } catch (error) {
+    slot.release();
+    throw error;
+  }
   const databaseEnvFile = join(temporaryDirectory, "database.env");
   let portReservation: PortReservation;
   try {
     portReservation = reserveAvailablePort();
   } catch (error) {
     rmSync(temporaryDirectory, { force: true, recursive: true });
+    slot.release();
     throw error;
   }
   const hostPort = portReservation.port;
@@ -137,6 +224,7 @@ export function startPostgresFixture(): PostgresFixture {
     } finally {
       releasePort(portReservation);
       rmSync(temporaryDirectory, { force: true, recursive: true });
+      slot.release();
     }
     throw error;
   }
@@ -190,6 +278,7 @@ export function startPostgresFixture(): PostgresFixture {
       } finally {
         releasePort(portReservation);
         rmSync(temporaryDirectory, { force: true, recursive: true });
+        slot.release();
       }
     },
   };

frontend/tests/postgres-fixture-contract.test.ts

@@ -0,0 +1,35 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const fixtureSource = readFileSync(
+  fileURLToPath(new URL("./helpers/postgres-fixture.ts", import.meta.url)),
+  "utf8",
+);
+
+test("postgres fixtures cap concurrent compose networks so Docker address pools cannot be exhausted", () => {
+  assert.match(fixtureSource, /JYOTISHA_POSTGRES_MAX_FIXTURES/);
+  assert.match(fixtureSource, /DEFAULT_MAX_CONCURRENT_POSTGRES_FIXTURES = 2/);
+  assert.match(fixtureSource, /function acquireFixtureSlot\(/);
+  assert.match(fixtureSource, /process\.kill\(\s*pid,\s*0\s*\)/);
+  assert.match(
+    fixtureSource,
+    /export function startPostgresFixture\(\)[\s\S]*acquireFixtureSlot\(\)/,
+  );
+  assert.match(
+    fixtureSource,
+    /timed out waiting for a PostgreSQL fixture slot/,
+  );
+});
+
+test("postgres fixture start failure and stop both release the compose-network slot", () => {
+  assert.match(
+    fixtureSource,
+    /catch \(error\) \{[\s\S]*releasePort\(portReservation\)[\s\S]*slot\.release\(\)/,
+  );
+  assert.match(
+    fixtureSource,
+    /stop\(\) \{[\s\S]*releasePort\(portReservation\)[\s\S]*slot\.release\(\)/,
+  );
+});

frontend/tests/staging-backend-workflows.test.ts

@@ -407,13 +407,18 @@ test("both gate jobs reclaim runner disk before they need it, and only unheld re
   assert.match(script, /docker container prune --force --filter until=6h/);
   assert.match(script, /--filter 'name=\^jyotisha-postgres-' --filter dangling=true/);
   assert.match(script, /docker network prune --force --filter until=6h/);
+  assert.match(script, /docker network ls[^\n]*--filter 'name=jyotisha-postgres-'/);
+  assert.match(script, /docker network rm "\$network"/);
+  assert.match(script, /all predefined address pools have been fully subnetted/);
   assert.match(script, /docker image prune --force\n/);
   assert.match(script, /docker builder prune --force --all/);
   assert.match(script, /grep -v -F -e "api-\$KEEP_TAG_SHA" -e "web-\$KEEP_TAG_SHA"/);
   assert.match(script, /PostgreSQL fixtures and image builds need at least \$\{MINIMUM_FREE_GIB\} GiB/);
   // A gate that silently proceeds on a full disk fails 23 database tests instead
   // of naming the disk, and pruning what a container still holds would delete a
-  // concurrent job's fixture out from under it.
+  // concurrent job's fixture out from under it. Network rm is the same rule:
+  // unused jyotisha-postgres leftovers occupy address pools, but an in-use
+  // network still has endpoints and docker network rm will refuse it.
   assert.match(script, /if \[ "\$AFTER_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+exit 1/);
   assert.doesNotMatch(script, /docker system prune|--volumes|prune[^\n]*--all --force|docker (?:rm|volume rm|kill)[^\n]*--force/);
 });