Jesse_Chen
37e62094d7
xiaoxin's 20-core npm test opened one Docker network per database file and exhausted default address pools. Limit concurrent fixtures and remove unused jyotisha-postgres networks before the gate. Co-authored-by: Cursor <cursoragent@cursor.com>
81 lines
2.9 KiB
Bash
Executable File
81 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
MINIMUM_FREE_GIB="${MINIMUM_FREE_GIB:-10}"
|
|
IMAGE_REPOSITORY="${IMAGE_REPOSITORY:-*/copse/jyotisha}"
|
|
KEEP_IMAGE_SHA="${KEEP_IMAGE_SHA:-}"
|
|
|
|
if ! [[ "$MINIMUM_FREE_GIB" =~ ^[0-9]+$ ]]; then
|
|
echo "MINIMUM_FREE_GIB must be a whole number of GiB" >&2
|
|
exit 1
|
|
fi
|
|
|
|
DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')"
|
|
if [ -z "$DOCKER_ROOT" ] || [ ! -d "$DOCKER_ROOT" ]; then
|
|
echo "unable to resolve the Docker root directory to measure" >&2
|
|
exit 1
|
|
fi
|
|
|
|
free_gib() {
|
|
df -Pk "$DOCKER_ROOT" | awk 'NR == 2 { printf "%d", $4 / 1048576 }'
|
|
}
|
|
|
|
BEFORE_GIB="$(free_gib)"
|
|
echo "docker root $DOCKER_ROOT has ${BEFORE_GIB} GiB free before reclaim"
|
|
|
|
# Every reclaim below is restricted to resources no container currently holds,
|
|
# so a quality-gate job running concurrently on this runner keeps its fixtures.
|
|
docker container prune --force --filter until=6h
|
|
docker volume ls --quiet --filter 'name=^jyotisha-postgres-' --filter dangling=true |
|
|
xargs -r docker volume rm
|
|
docker network prune --force --filter until=6h
|
|
|
|
# Compose networks from crashed or finished fixtures keep occupying Docker's
|
|
# default address pools until they are removed. `network prune --filter until=6h`
|
|
# leaves same-day leftovers, and the next `docker compose up` then fails with
|
|
# "all predefined address pools have been fully subnetted".
|
|
# `docker network rm` refuses networks that still have endpoints, so a
|
|
# concurrent job's live fixture is left alone.
|
|
echo "removing unused jyotisha-postgres compose networks"
|
|
removed_networks=0
|
|
kept_networks=0
|
|
while IFS= read -r network; do
|
|
if [ -z "$network" ]; then
|
|
continue
|
|
fi
|
|
if docker network rm "$network" >/dev/null 2>&1; then
|
|
echo "removed unused network $network"
|
|
removed_networks=$((removed_networks + 1))
|
|
else
|
|
echo "kept in-use network $network"
|
|
kept_networks=$((kept_networks + 1))
|
|
fi
|
|
done < <(docker network ls --format '{{.Name}}' --filter 'name=jyotisha-postgres-')
|
|
echo "jyotisha-postgres networks: removed=${removed_networks} kept=${kept_networks}"
|
|
|
|
docker image prune --force
|
|
|
|
# Exact-SHA release images are immutable in the registry once published, so the
|
|
# runner's local copies are rebuildable cache rather than release state.
|
|
KEEP_TAG_SHA="${KEEP_IMAGE_SHA:-no-image-to-keep}"
|
|
STALE_IMAGES="$(docker image ls --format '{{.Repository}}:{{.Tag}}' \
|
|
--filter "reference=$IMAGE_REPOSITORY:api-*" \
|
|
--filter "reference=$IMAGE_REPOSITORY:web-*" |
|
|
grep -v -F -e "api-$KEEP_TAG_SHA" -e "web-$KEEP_TAG_SHA" || true)"
|
|
if [ -n "$STALE_IMAGES" ]; then
|
|
printf '%s\n' "$STALE_IMAGES" | xargs -r docker image rm || true
|
|
fi
|
|
|
|
docker builder prune --force --all
|
|
|
|
AFTER_GIB="$(free_gib)"
|
|
echo "docker root $DOCKER_ROOT has ${AFTER_GIB} GiB free after reclaim"
|
|
docker system df
|
|
|
|
if [ "$AFTER_GIB" -lt "$MINIMUM_FREE_GIB" ]; then
|
|
echo "runner kept only ${AFTER_GIB} GiB free; PostgreSQL fixtures and image builds need at least ${MINIMUM_FREE_GIB} GiB" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "runner disk reclaimed"
|