#!/usr/bin/env bash set -euo pipefail set +x umask 077 : "${DEPLOY_PATH:?DEPLOY_PATH is required}" : "${RECOVERY_RUN_ID:?RECOVERY_RUN_ID is required}" [ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || { echo "unexpected production path" >&2 exit 1 } [[ "$RECOVERY_RUN_ID" =~ ^[0-9]+$ ]] || { echo "recovery run id must be numeric" >&2 exit 1 } state_directory="$DEPLOY_PATH/.state" backup_directory="$DEPLOY_PATH/backups" environment_file="$DEPLOY_PATH/.env.production.database" lock_file="$state_directory/mutation.lock" [ -f "$environment_file" ] && [ ! -L "$environment_file" ] || { echo "production database environment file is missing or unsafe" >&2 exit 1 } for directory in "$state_directory" "$backup_directory"; do if [ -e "$directory" ]; then [ -d "$directory" ] && [ ! -L "$directory" ] || { echo "production state or backup directory is unsafe" >&2 exit 1 } fi done install -d -m 700 "$state_directory" "$backup_directory" deployment_uid="$(id -u)" deployment_gid="$(id -g)" ownership_mounts=( --volume "$state_directory:$state_directory" --volume "$backup_directory:$backup_directory" ) ownership_targets=() if [ -e "$lock_file" ]; then [ -f "$lock_file" ] && [ ! -L "$lock_file" ] || { echo "production mutation lock is unsafe" >&2 exit 1 } ownership_lock_target="/mutation.lock" ownership_mounts+=(--mount "type=bind,src=$lock_file,dst=$ownership_lock_target") ownership_targets+=("$ownership_lock_target") fi ownership_targets+=("$state_directory" "$backup_directory") mapfile -t postgres_containers < <( sudo -n docker ps -q \ --filter 'label=com.docker.compose.project=jyotisha-production' \ --filter 'label=com.docker.compose.service=postgres' ) [ "${#postgres_containers[@]}" -eq 1 ] || { echo "expected exactly one running production PostgreSQL container" >&2 exit 1 } postgres_container="${postgres_containers[0]}" ownership_image="$(sudo -n docker inspect --format '{{.Image}}' "$postgres_container")" [[ "$ownership_image" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "production PostgreSQL image identity is unsafe" >&2 exit 1 } sudo -n docker run --rm --pull never --network none --read-only --user 0:0 \ --cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges \ "${ownership_mounts[@]}" \ --entrypoint chown "$ownership_image" \ "$deployment_uid:$deployment_gid" "${ownership_targets[@]}" chmod 700 "$state_directory" "$backup_directory" if [ -e "$lock_file" ]; then chmod 600 "$lock_file" fi exec 9>"$lock_file" flock -n 9 || { echo "another production mutation holds the host lock" >&2 exit 75 } set -a # shellcheck disable=SC1090 . "$environment_file" set +a : "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}" usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')" [[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || { echo "production backup disk usage must remain below 70 percent" >&2 exit 1 } created_compact="$(date -u +%Y%m%dT%H%M%SZ)" created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" backup_basename="production-pre-migration-${created_compact}.dump.enc" backup_partial="$backup_directory/.${backup_basename}.$$.partial" backup_file="$backup_directory/$backup_basename" restore_database="restore_verify_${created_compact,,}" restore_database="${restore_database//[^a-z0-9_]/_}" cleanup() { rm -f -- "$backup_partial" sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ -c "DROP DATABASE IF EXISTS \"$restore_database\" WITH (FORCE);" >/dev/null 2>&1 || true } trap cleanup EXIT HUP INT TERM sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-acl | openssl enc -aes-256-cbc -salt -pbkdf2 \ -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY >"$backup_partial" [ -s "$backup_partial" ] chmod 600 "$backup_partial" mv "$backup_partial" "$backup_file" backup_sha256="$(sha256sum "$backup_file" | awk '{print $1}')" [[ "$backup_sha256" =~ ^[0-9a-f]{64}$ ]] sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ -c "CREATE DATABASE \"$restore_database\";" >/dev/null openssl enc -d -aes-256-cbc -pbkdf2 \ -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$backup_file" | sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ pg_restore -U "$POSTGRES_USER" -d "$restore_database" \ --no-owner --no-acl --exit-on-error counts="$({ sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ psql -U "$POSTGRES_USER" -d "$restore_database" -At -F '|' -v ON_ERROR_STOP=1 -c \ "SELECT (SELECT count(*) FROM identity.users), (SELECT count(*) FROM public.profiles), (SELECT count(*) FROM public.credit_transactions), (SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public');" } | tail -n 1)" IFS='|' read -r identity_users profiles credit_transactions public_tables <<<"$counts" for value in "$identity_users" "$profiles" "$credit_transactions" "$public_tables"; do [[ "$value" =~ ^[0-9]+$ ]] || { echo "restore verification returned an invalid count" >&2 exit 1 } done sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ -c "DROP DATABASE \"$restore_database\" WITH (FORCE);" >/dev/null verified_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" trap - EXIT HUP INT TERM verify_basename="${backup_basename%.dump.enc}-restore-verify.json" verify_file="$backup_directory/$verify_basename" printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s}\n}\n' \ "$backup_basename" "$backup_sha256" "$created_at" "$verified_at" \ "$identity_users" "$profiles" "$credit_transactions" "$public_tables" >"$verify_file" chmod 600 "$verify_file" recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}" state_tmp="$state_directory/recovery-baseline.env.tmp.$$" printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nVERIFY_FILE=%s\n' \ "$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp" chmod 600 "$state_tmp" mv "$state_tmp" "$state_directory/recovery-baseline.env" printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \ "$recovery_reference" "$created_at" "$verified_at" "$backup_sha256" \ "$backup_basename" "$verify_basename"