jesse-ux
0c3c9d6b42
Gitea run 2828 validated, then publish hit the 60-minute deadline while Aliyun apt/pip crawled at about 22 kB/s after a 72h cache prune on a runner with 677 GiB free. Skip BuildKit prune when disk is above the minimum, split API apt/pip layers, and point apt/pypi at Huawei Cloud.
101 lines
4.0 KiB
Bash
Executable File
101 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
MINIMUM_FREE_GIB="${MINIMUM_FREE_GIB:-10}"
|
||
IMAGE_REPOSITORY="${IMAGE_REPOSITORY:-*/copse/jyotisha}"
|
||
KEEP_IMAGE_SHA="${KEEP_IMAGE_SHA:-}"
|
||
|
||
if ! [[ "$MINIMUM_FREE_GIB" =~ ^[0-9]+$ ]]; then
|
||
echo "MINIMUM_FREE_GIB must be a whole number of GiB" >&2
|
||
exit 1
|
||
fi
|
||
|
||
DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')"
|
||
if [ -z "$DOCKER_ROOT" ] || [ ! -d "$DOCKER_ROOT" ]; then
|
||
echo "unable to resolve the Docker root directory to measure" >&2
|
||
exit 1
|
||
fi
|
||
|
||
free_gib() {
|
||
df -Pk "$DOCKER_ROOT" | awk 'NR == 2 { printf "%d", $4 / 1048576 }'
|
||
}
|
||
|
||
BEFORE_GIB="$(free_gib)"
|
||
echo "docker root $DOCKER_ROOT has ${BEFORE_GIB} GiB free before reclaim"
|
||
|
||
# Every reclaim below is restricted to resources no container currently holds,
|
||
# so a quality-gate job running concurrently on this runner keeps its fixtures.
|
||
docker container prune --force --filter until=6h
|
||
docker volume ls --quiet --filter 'name=^jyotisha-postgres-' --filter dangling=true |
|
||
xargs -r docker volume rm
|
||
docker network prune --force --filter until=6h
|
||
|
||
# Compose networks from crashed or finished fixtures keep occupying Docker's
|
||
# default address pools until they are removed. `network prune --filter until=6h`
|
||
# leaves same-day leftovers, and the next `docker compose up` then fails with
|
||
# "all predefined address pools have been fully subnetted".
|
||
# `docker network rm` refuses networks that still have endpoints, so a
|
||
# concurrent job's live fixture is left alone.
|
||
echo "removing unused jyotisha-postgres compose networks"
|
||
removed_networks=0
|
||
kept_networks=0
|
||
while IFS= read -r network; do
|
||
if [ -z "$network" ]; then
|
||
continue
|
||
fi
|
||
if docker network rm "$network" >/dev/null 2>&1; then
|
||
echo "removed unused network $network"
|
||
removed_networks=$((removed_networks + 1))
|
||
else
|
||
echo "kept in-use network $network"
|
||
kept_networks=$((kept_networks + 1))
|
||
fi
|
||
done < <(docker network ls --format '{{.Name}}' --filter 'name=jyotisha-postgres-')
|
||
echo "jyotisha-postgres networks: removed=${removed_networks} kept=${kept_networks}"
|
||
|
||
docker image prune --force
|
||
|
||
# Exact-SHA release images are immutable in the registry once published, so the
|
||
# runner's local copies are rebuildable cache rather than release state.
|
||
KEEP_TAG_SHA="${KEEP_IMAGE_SHA:-no-image-to-keep}"
|
||
STALE_IMAGES="$(docker image ls --format '{{.Repository}}:{{.Tag}}' \
|
||
--filter "reference=$IMAGE_REPOSITORY:api-*" \
|
||
--filter "reference=$IMAGE_REPOSITORY:web-*" |
|
||
grep -v -F -e "api-$KEEP_TAG_SHA" -e "web-$KEEP_TAG_SHA" || true)"
|
||
if [ -n "$STALE_IMAGES" ]; then
|
||
printf '%s\n' "$STALE_IMAGES" | xargs -r docker image rm || true
|
||
fi
|
||
|
||
# BuildKit layer cache is what lets the publish job reuse apt/pip/`npm ci`
|
||
# layers instead of rebuilding behind a slow mirror (7-minute image builds
|
||
# versus 47–60). `docker image prune --force` above removes only dangling
|
||
# images; BuildKit cache lives in the builder store. Do not touch it while
|
||
# the runner still has at least MINIMUM_FREE_GIB. Aged and full prunes are
|
||
# reachable only after re-measuring and finding the disk actually short.
|
||
CACHE_GIB="$(free_gib)"
|
||
echo "docker root $DOCKER_ROOT has ${CACHE_GIB} GiB free before BuildKit reclaim"
|
||
if [ "$CACHE_GIB" -lt "$MINIMUM_FREE_GIB" ]; then
|
||
echo "below ${MINIMUM_FREE_GIB} GiB; pruning BuildKit cache unused for 72h"
|
||
docker builder prune --force --filter until=72h
|
||
TIERED_GIB="$(free_gib)"
|
||
echo "docker root $DOCKER_ROOT has ${TIERED_GIB} GiB free after aged build-cache reclaim"
|
||
if [ "$TIERED_GIB" -lt "$MINIMUM_FREE_GIB" ]; then
|
||
echo "still below ${MINIMUM_FREE_GIB} GiB; escalating to a full BuildKit cache prune"
|
||
docker builder prune --force --all
|
||
fi
|
||
else
|
||
echo "keeping BuildKit layer cache; ${CACHE_GIB} GiB free"
|
||
TIERED_GIB="$CACHE_GIB"
|
||
fi
|
||
|
||
AFTER_GIB="$(free_gib)"
|
||
echo "docker root $DOCKER_ROOT has ${AFTER_GIB} GiB free after reclaim"
|
||
docker system df
|
||
|
||
if [ "$AFTER_GIB" -lt "$MINIMUM_FREE_GIB" ]; then
|
||
echo "runner kept only ${AFTER_GIB} GiB free; PostgreSQL fixtures and image builds need at least ${MINIMUM_FREE_GIB} GiB" >&2
|
||
exit 1
|
||
fi
|
||
|
||
echo "runner disk reclaimed"
|