diff --git a/deploy/reclaim-runner-disk.sh b/deploy/reclaim-runner-disk.sh index 4f40f12e..3f6ea914 100755 --- a/deploy/reclaim-runner-disk.sh +++ b/deploy/reclaim-runner-disk.sh @@ -66,7 +66,21 @@ if [ -n "$STALE_IMAGES" ]; then printf '%s\n' "$STALE_IMAGES" | xargs -r docker image rm || true fi -docker builder prune --force --all +# BuildKit layer cache is what lets the publish job reuse the Dockerfile +# `npm ci` layer instead of rebuilding it from scratch behind the mirror (a +# 7-minute image build versus 47). `docker image prune --force` above removes +# only dangling images; BuildKit cache records live in the builder store, not +# in dangling images, so nothing before this point touches them. Reclaim them +# in tiers: drop entries nobody has used for 72 hours, re-measure, and escalate +# to `--all` only when the runner is still below the threshold. +docker builder prune --force --filter until=72h + +TIERED_GIB="$(free_gib)" +echo "docker root $DOCKER_ROOT has ${TIERED_GIB} GiB free after aged build-cache reclaim" +if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then + echo "still below ${MINIMUM_FREE_GIB} GiB; escalating to a full BuildKit cache prune" + docker builder prune --force --all +fi AFTER_GIB="$(free_gib)" echo "docker root $DOCKER_ROOT has ${AFTER_GIB} GiB free after reclaim" diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 47589d88..2cca69a0 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -398,6 +398,7 @@ test("both gate jobs reclaim runner disk before they need it, and only unheld re assert.match(script, /docker network rm "\$network"/); assert.match(script, /all predefined address pools have been fully subnetted/); assert.match(script, /docker image prune --force\n/); + assert.match(script, /docker builder prune --force --filter until=72h/); assert.match(script, /docker builder prune --force --all/); assert.match(script, /grep -v -F -e "api-\$KEEP_TAG_SHA" -e "web-\$KEEP_TAG_SHA"/); assert.match(script, /PostgreSQL fixtures and image builds need at least \$\{MINIMUM_FREE_GIB\} GiB/); @@ -1224,3 +1225,28 @@ test("Gitea production schema migration is exact-SHA gated and isolated from ETL assert.doesNotMatch(runner, /migrate-supabase-production|docker-compose\.server\.yml|docker-compose\.production\.yml/); assert.doesNotMatch(runner, /\bup\b[^\n]*(?:api|web|caddy)|Caddyfile|PRODUCTION_URL|PRODUCTION_ADMIN_URL|mv -f[^\n]*deployed-revision/); }); + +test("runner disk reclaim keeps BuildKit cache unless the runner is actually short on disk", () => { + const script = read(reclaimRunnerDiskScript); + + // Unconditionally pruning the whole BuildKit store made every publish job + // rebuild the Dockerfile `npm ci` layer from scratch (47-minute image builds). + // Aged cache goes first; `--all` is reachable only after re-measuring and + // finding the runner still below MINIMUM_FREE_GIB. + assert.doesNotMatch(script, /^docker builder prune --force --all$/m); + assert.equal((script.match(/docker builder prune --force --all/g) ?? []).length, 1); + assertOrder(script, [ + "docker image prune --force", + "docker builder prune --force --filter until=72h", + 'TIERED_GIB="$(free_gib)"', + 'if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then', + "docker builder prune --force --all", + 'AFTER_GIB="$(free_gib)"', + ]); + assert.match( + script, + /if \[ "\$TIERED_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+docker builder prune --force --all\nfi\n/, + ); + // The final threshold check still fails the job instead of silently proceeding. + assert.match(script, /if \[ "\$AFTER_GIB" -lt "\$MINIMUM_FREE_GIB" \]; then\n\s+echo[^\n]+\n\s+exit 1/); +});