jesse-uxandGrok (agent loop)
5c0d895391
Product chose to keep the scheduled backup, but the current recovery point must stay the one created by hand. Daily runs still encrypt and restore-verify, then record a separate latest-backup note. Co-Authored-By: Grok (agent loop)
266 lines
10 KiB
Bash
Executable File
266 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
set +x
|
|
umask 077
|
|
|
|
: "${DEPLOY_PATH:?DEPLOY_PATH is required}"
|
|
: "${RECOVERY_RUN_ID:?RECOVERY_RUN_ID is required}"
|
|
|
|
# Tests set JYOTISHA_RECOVERY_FIXTURE=1 and point DEPLOY_PATH at a temp directory.
|
|
# Production workflows never set it, so the live path check stays closed.
|
|
if [ "${JYOTISHA_RECOVERY_FIXTURE:-}" = "1" ]; then
|
|
[ -d "$DEPLOY_PATH" ] && [ ! -L "$DEPLOY_PATH" ] || {
|
|
echo "unexpected production path" >&2
|
|
exit 1
|
|
}
|
|
case "$DEPLOY_PATH" in
|
|
/*) ;;
|
|
*)
|
|
echo "unexpected production path" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
else
|
|
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
|
|
echo "unexpected production path" >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
[[ "$RECOVERY_RUN_ID" =~ ^[0-9]+$ ]] || {
|
|
echo "recovery run id must be numeric" >&2
|
|
exit 1
|
|
}
|
|
|
|
state_directory="$DEPLOY_PATH/.state"
|
|
backup_directory="$DEPLOY_PATH/backups"
|
|
environment_file="$DEPLOY_PATH/.env.production.database"
|
|
lock_file="$state_directory/mutation.lock"
|
|
|
|
[ -f "$environment_file" ] && [ ! -L "$environment_file" ] || {
|
|
echo "production database environment file is missing or unsafe" >&2
|
|
exit 1
|
|
}
|
|
for directory in "$state_directory" "$backup_directory"; do
|
|
if [ -e "$directory" ]; then
|
|
[ -d "$directory" ] && [ ! -L "$directory" ] || {
|
|
echo "production state or backup directory is unsafe" >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
done
|
|
install -d -m 700 "$state_directory" "$backup_directory"
|
|
deployment_uid="$(id -u)"
|
|
deployment_gid="$(id -g)"
|
|
ownership_mounts=(
|
|
--volume "$state_directory:$state_directory"
|
|
--volume "$backup_directory:$backup_directory"
|
|
)
|
|
ownership_targets=()
|
|
if [ -e "$lock_file" ]; then
|
|
[ -f "$lock_file" ] && [ ! -L "$lock_file" ] || {
|
|
echo "production mutation lock is unsafe" >&2
|
|
exit 1
|
|
}
|
|
ownership_lock_target="/mutation.lock"
|
|
ownership_mounts+=(--mount "type=bind,src=$lock_file,dst=$ownership_lock_target")
|
|
ownership_targets+=("$ownership_lock_target")
|
|
fi
|
|
ownership_targets+=("$state_directory" "$backup_directory")
|
|
|
|
mapfile -t postgres_containers < <(
|
|
sudo -n docker ps -q \
|
|
--filter 'label=com.docker.compose.project=jyotisha-production' \
|
|
--filter 'label=com.docker.compose.service=postgres'
|
|
)
|
|
[ "${#postgres_containers[@]}" -eq 1 ] || {
|
|
echo "expected exactly one running production PostgreSQL container" >&2
|
|
exit 1
|
|
}
|
|
postgres_container="${postgres_containers[0]}"
|
|
ownership_image="$(sudo -n docker inspect --format '{{.Image}}' "$postgres_container")"
|
|
[[ "$ownership_image" =~ ^sha256:[0-9a-f]{64}$ ]] || {
|
|
echo "production PostgreSQL image identity is unsafe" >&2
|
|
exit 1
|
|
}
|
|
sudo -n docker run --rm --pull never --network none --read-only --user 0:0 \
|
|
--cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges \
|
|
"${ownership_mounts[@]}" \
|
|
--entrypoint chown "$ownership_image" \
|
|
"$deployment_uid:$deployment_gid" "${ownership_targets[@]}"
|
|
chmod 700 "$state_directory" "$backup_directory"
|
|
if [ -e "$lock_file" ]; then
|
|
chmod 600 "$lock_file"
|
|
fi
|
|
|
|
exec 9>"$lock_file"
|
|
flock -n 9 || {
|
|
echo "another production mutation holds the host lock" >&2
|
|
exit 75
|
|
}
|
|
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
. "$environment_file"
|
|
set +a
|
|
: "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}"
|
|
|
|
privilege_sql() {
|
|
case "$1" in
|
|
table_grants)
|
|
printf '%s' "select grantee || '|' || privilege_type || '|' || table_name from information_schema.role_table_grants where table_schema = 'public' order by 1"
|
|
;;
|
|
column_privileges)
|
|
printf '%s' "select grantee || '|' || privilege_type || '|' || table_name || '|' || column_name from information_schema.column_privileges where table_schema = 'public' order by 1"
|
|
;;
|
|
table_owners)
|
|
printf '%s' "select c.relname || '|' || pg_get_userbyid(c.relowner) from pg_class c join pg_namespace n on n.oid = c.relnamespace where n.nspname = 'public' and c.relkind in ('r', 'p') order by 1"
|
|
;;
|
|
*)
|
|
echo "unknown privilege check" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
privilege_snapshot() {
|
|
local database="$1" kind="$2" destination="$3"
|
|
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
psql -U "$POSTGRES_USER" -d "$database" -At -v ON_ERROR_STOP=1 \
|
|
-c "$(privilege_sql "$kind")" | sort >"$destination"
|
|
}
|
|
|
|
privileges_match() {
|
|
local kind="$1" source_file restore_file status=0
|
|
source_file="$(mktemp)"
|
|
restore_file="$(mktemp)"
|
|
if ! privilege_snapshot "$POSTGRES_DB" "$kind" "$source_file"; then
|
|
rm -f -- "$source_file" "$restore_file"
|
|
return 1
|
|
fi
|
|
if ! privilege_snapshot "$restore_database" "$kind" "$restore_file"; then
|
|
rm -f -- "$source_file" "$restore_file"
|
|
return 1
|
|
fi
|
|
cmp -s "$source_file" "$restore_file" || status=$?
|
|
rm -f -- "$source_file" "$restore_file"
|
|
return "$status"
|
|
}
|
|
|
|
usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')"
|
|
[[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || {
|
|
echo "production backup disk usage must remain below 70 percent" >&2
|
|
exit 1
|
|
}
|
|
|
|
created_compact="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
# Daily backups set RECOVERY_BASENAME_PREFIX=production-daily so retention can
|
|
# tell them from manual recovery points. Unset keeps the original name.
|
|
# That same prefix is the switch that skips recovery-baseline.env. Daily runs
|
|
# still export, encrypt, and restore-verify, then record daily-backup-latest.env.
|
|
backup_prefix="production-pre-migration"
|
|
if [ -n "${RECOVERY_BASENAME_PREFIX:-}" ]; then
|
|
case "$RECOVERY_BASENAME_PREFIX" in
|
|
production-daily) backup_prefix="production-daily" ;;
|
|
*)
|
|
echo "unexpected recovery basename prefix" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
backup_basename="${backup_prefix}-${created_compact}.dump.enc"
|
|
backup_partial="$backup_directory/.${backup_basename}.$$.partial"
|
|
backup_file="$backup_directory/$backup_basename"
|
|
restore_database="restore_verify_${created_compact,,}"
|
|
restore_database="${restore_database//[^a-z0-9_]/_}"
|
|
|
|
cleanup() {
|
|
rm -f -- "$backup_partial"
|
|
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
|
-c "DROP DATABASE IF EXISTS \"$restore_database\" WITH (FORCE);" >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT HUP INT TERM
|
|
|
|
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom |
|
|
openssl enc -aes-256-cbc -salt -pbkdf2 \
|
|
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY >"$backup_partial"
|
|
[ -s "$backup_partial" ]
|
|
chmod 600 "$backup_partial"
|
|
mv "$backup_partial" "$backup_file"
|
|
backup_sha256="$(sha256sum "$backup_file" | awk '{print $1}')"
|
|
[[ "$backup_sha256" =~ ^[0-9a-f]{64}$ ]]
|
|
|
|
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
|
-c "CREATE DATABASE \"$restore_database\";" >/dev/null
|
|
openssl enc -d -aes-256-cbc -pbkdf2 \
|
|
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$backup_file" |
|
|
sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
pg_restore -U "$POSTGRES_USER" -d "$restore_database" \
|
|
--exit-on-error
|
|
|
|
counts="$({
|
|
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
psql -U "$POSTGRES_USER" -d "$restore_database" -At -F '|' -v ON_ERROR_STOP=1 -c \
|
|
"SELECT
|
|
(SELECT count(*) FROM identity.users),
|
|
(SELECT count(*) FROM public.profiles),
|
|
(SELECT count(*) FROM public.credit_transactions),
|
|
(SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public');"
|
|
} | tail -n 1)"
|
|
IFS='|' read -r identity_users profiles credit_transactions public_tables <<<"$counts"
|
|
for value in "$identity_users" "$profiles" "$credit_transactions" "$public_tables"; do
|
|
[[ "$value" =~ ^[0-9]+$ ]] || {
|
|
echo "restore verification returned an invalid count" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
table_grants_match=false
|
|
column_privileges_match=false
|
|
table_owners_match=false
|
|
if privileges_match table_grants; then table_grants_match=true; fi
|
|
if privileges_match column_privileges; then column_privileges_match=true; fi
|
|
if privileges_match table_owners; then table_owners_match=true; fi
|
|
if [ "$table_grants_match" != true ] || [ "$column_privileges_match" != true ] || [ "$table_owners_match" != true ]; then
|
|
echo "restore privilege verification failed" >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
|
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
|
-c "DROP DATABASE \"$restore_database\" WITH (FORCE);" >/dev/null
|
|
verified_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
trap - EXIT HUP INT TERM
|
|
|
|
verify_basename="${backup_basename%.dump.enc}-restore-verify.json"
|
|
verify_file="$backup_directory/$verify_basename"
|
|
printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s, "table_grants_match": %s, "column_privileges_match": %s, "table_owners_match": %s}\n}\n' \
|
|
"$backup_basename" "$backup_sha256" "$created_at" "$verified_at" \
|
|
"$identity_users" "$profiles" "$credit_transactions" "$public_tables" \
|
|
"$table_grants_match" "$column_privileges_match" "$table_owners_match" >"$verify_file"
|
|
chmod 600 "$verify_file"
|
|
|
|
recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}"
|
|
if [ "$backup_prefix" = "production-daily" ]; then
|
|
daily_tmp="$state_directory/daily-backup-latest.env.tmp.$$"
|
|
printf 'BACKUP_BASENAME=%s\nBACKUP_SHA256=%s\nVERIFIED_AT=%s\nBACKUP_FORMAT=2\n' \
|
|
"$backup_basename" "$backup_sha256" "$verified_at" >"$daily_tmp"
|
|
chmod 600 "$daily_tmp"
|
|
mv "$daily_tmp" "$state_directory/daily-backup-latest.env"
|
|
echo "baseline_updated=false" >&2
|
|
else
|
|
state_tmp="$state_directory/recovery-baseline.env.tmp.$$"
|
|
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nRECOVERY_FORMAT=2\nVERIFY_FILE=%s\n' \
|
|
"$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp"
|
|
chmod 600 "$state_tmp"
|
|
mv "$state_tmp" "$state_directory/recovery-baseline.env"
|
|
echo "baseline_updated=true" >&2
|
|
fi
|
|
|
|
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
|
|
"$recovery_reference" "$created_at" "$verified_at" "$backup_sha256" \
|
|
"$backup_basename" "$verify_basename"
|