Jesse_Chen
75e288b0c6
deploy/reclaim-runner-disk.sh ran `docker builder prune --force --all` at the start of every gate job, so each publish rebuilt the Dockerfile `npm ci` layer from scratch behind the registry mirror (7-minute image builds became 47). Reclaim build cache in tiers instead: prune entries unused for 72h, re-measure free space, and escalate to `--all` only while still below MINIMUM_FREE_GIB. The final threshold check and the "only unheld resources" rules are unchanged. Contract test asserts the aged prune runs first and that `--all` is only reachable inside the re-measured conditional. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VawU7Xfd5jS9wUEXz1XYmS
95 lines
3.7 KiB
Bash
Executable File
95 lines
3.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
MINIMUM_FREE_GIB="${MINIMUM_FREE_GIB:-10}"
|
|
IMAGE_REPOSITORY="${IMAGE_REPOSITORY:-*/copse/jyotisha}"
|
|
KEEP_IMAGE_SHA="${KEEP_IMAGE_SHA:-}"
|
|
|
|
if ! [[ "$MINIMUM_FREE_GIB" =~ ^[0-9]+$ ]]; then
|
|
echo "MINIMUM_FREE_GIB must be a whole number of GiB" >&2
|
|
exit 1
|
|
fi
|
|
|
|
DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')"
|
|
if [ -z "$DOCKER_ROOT" ] || [ ! -d "$DOCKER_ROOT" ]; then
|
|
echo "unable to resolve the Docker root directory to measure" >&2
|
|
exit 1
|
|
fi
|
|
|
|
free_gib() {
|
|
df -Pk "$DOCKER_ROOT" | awk 'NR == 2 { printf "%d", $4 / 1048576 }'
|
|
}
|
|
|
|
BEFORE_GIB="$(free_gib)"
|
|
echo "docker root $DOCKER_ROOT has ${BEFORE_GIB} GiB free before reclaim"
|
|
|
|
# Every reclaim below is restricted to resources no container currently holds,
|
|
# so a quality-gate job running concurrently on this runner keeps its fixtures.
|
|
docker container prune --force --filter until=6h
|
|
docker volume ls --quiet --filter 'name=^jyotisha-postgres-' --filter dangling=true |
|
|
xargs -r docker volume rm
|
|
docker network prune --force --filter until=6h
|
|
|
|
# Compose networks from crashed or finished fixtures keep occupying Docker's
|
|
# default address pools until they are removed. `network prune --filter until=6h`
|
|
# leaves same-day leftovers, and the next `docker compose up` then fails with
|
|
# "all predefined address pools have been fully subnetted".
|
|
# `docker network rm` refuses networks that still have endpoints, so a
|
|
# concurrent job's live fixture is left alone.
|
|
echo "removing unused jyotisha-postgres compose networks"
|
|
removed_networks=0
|
|
kept_networks=0
|
|
while IFS= read -r network; do
|
|
if [ -z "$network" ]; then
|
|
continue
|
|
fi
|
|
if docker network rm "$network" >/dev/null 2>&1; then
|
|
echo "removed unused network $network"
|
|
removed_networks=$((removed_networks + 1))
|
|
else
|
|
echo "kept in-use network $network"
|
|
kept_networks=$((kept_networks + 1))
|
|
fi
|
|
done < <(docker network ls --format '{{.Name}}' --filter 'name=jyotisha-postgres-')
|
|
echo "jyotisha-postgres networks: removed=${removed_networks} kept=${kept_networks}"
|
|
|
|
docker image prune --force
|
|
|
|
# Exact-SHA release images are immutable in the registry once published, so the
|
|
# runner's local copies are rebuildable cache rather than release state.
|
|
KEEP_TAG_SHA="${KEEP_IMAGE_SHA:-no-image-to-keep}"
|
|
STALE_IMAGES="$(docker image ls --format '{{.Repository}}:{{.Tag}}' \
|
|
--filter "reference=$IMAGE_REPOSITORY:api-*" \
|
|
--filter "reference=$IMAGE_REPOSITORY:web-*" |
|
|
grep -v -F -e "api-$KEEP_TAG_SHA" -e "web-$KEEP_TAG_SHA" || true)"
|
|
if [ -n "$STALE_IMAGES" ]; then
|
|
printf '%s\n' "$STALE_IMAGES" | xargs -r docker image rm || true
|
|
fi
|
|
|
|
# BuildKit layer cache is what lets the publish job reuse the Dockerfile
|
|
# `npm ci` layer instead of rebuilding it from scratch behind the mirror (a
|
|
# 7-minute image build versus 47). `docker image prune --force` above removes
|
|
# only dangling images; BuildKit cache records live in the builder store, not
|
|
# in dangling images, so nothing before this point touches them. Reclaim them
|
|
# in tiers: drop entries nobody has used for 72 hours, re-measure, and escalate
|
|
# to `--all` only when the runner is still below the threshold.
|
|
docker builder prune --force --filter until=72h
|
|
|
|
TIERED_GIB="$(free_gib)"
|
|
echo "docker root $DOCKER_ROOT has ${TIERED_GIB} GiB free after aged build-cache reclaim"
|
|
if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then
|
|
echo "still below ${MINIMUM_FREE_GIB} GiB; escalating to a full BuildKit cache prune"
|
|
docker builder prune --force --all
|
|
fi
|
|
|
|
AFTER_GIB="$(free_gib)"
|
|
echo "docker root $DOCKER_ROOT has ${AFTER_GIB} GiB free after reclaim"
|
|
docker system df
|
|
|
|
if [ "$AFTER_GIB" -lt "$MINIMUM_FREE_GIB" ]; then
|
|
echo "runner kept only ${AFTER_GIB} GiB free; PostgreSQL fixtures and image builds need at least ${MINIMUM_FREE_GIB} GiB" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "runner disk reclaimed"
|