ops: automate production recovery attestation
This commit is contained in:
Jesse_Chen
@@ -250,6 +250,14 @@ Use this order for every staging revision:
|
||||
|
||||
The deploy and migration workflows share the `staging-mutation` Actions concurrency group, and their live-tree sync plus Compose work runs under `/opt/jyotisha-staging/.state/mutation.lock`. The synchronized tree explicitly preserves `/backups/`, `.env*`, `.state`, and `.incoming`. The read-only checker exits before app changes when a migration is pending. Its message includes the exact SHA and the `Migrate Staging Database` workflow name. A failed migration does not re-dispatch deployment. Application rollback restores the previously recorded digest references and SHA, falling back to validated local image IDs only when transitioning from the pre-foundation local-image deployment; it does not roll back database state.
|
||||
|
||||
### Production recovery point before schema migration
|
||||
|
||||
Use Gitea Actions → **Create Production Recovery Point** from the exact current `main` release SHA before every production schema migration. The manual workflow requires `main`, `staging`, public staging health, and the successful release gate to identify the same SHA. It uses the pinned production SSH identity, shares the `production-mutation` lock, and runs `deploy/run-production-recovery.sh` with shell tracing disabled.
|
||||
|
||||
The host script writes an AES-256-CBC/PBKDF2 encrypted custom-format PostgreSQL dump under `/opt/jyotisha-production/backups`, restores it into a uniquely named disposable database, validates non-sensitive row/table counts, removes only that disposable database, and writes a mode-`0600` verification manifest. The workflow retrieves only the encrypted dump and verification metadata, verifies the SHA-256 digest, and uploads them as a 30-day Gitea Actions artifact with compression disabled. The artifact-backed `recovery_reference`, `recovery_created_at`, and `restore_verified=true` output are the inputs for **Migrate Production Database**. Do not use the attestation if backup, restore, retrieval, digest validation, or artifact upload fails.
|
||||
|
||||
Never restore over `jyotisha`, delete the PostgreSQL volume, print `.env.production.database`, expose `PRODUCTION_BACKUP_ENCRYPTION_KEY`, or substitute a staging recovery artifact. The encrypted local archive is preserved for repair; the Actions artifact supplies the required off-host copy.
|
||||
|
||||
### Local encrypted staging backups (three-copy limit)
|
||||
|
||||
After the health check, run the repository backup helper from the synchronized staging checkout:
|
||||
|
||||
Executable
+135
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
: "${DEPLOY_PATH:?DEPLOY_PATH is required}"
|
||||
: "${RECOVERY_RUN_ID:?RECOVERY_RUN_ID is required}"
|
||||
|
||||
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
|
||||
echo "unexpected production path" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "$RECOVERY_RUN_ID" =~ ^[0-9]+$ ]] || {
|
||||
echo "recovery run id must be numeric" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
state_directory="$DEPLOY_PATH/.state"
|
||||
backup_directory="$DEPLOY_PATH/backups"
|
||||
environment_file="$DEPLOY_PATH/.env.production.database"
|
||||
|
||||
[ -f "$environment_file" ] && [ ! -L "$environment_file" ] || {
|
||||
echo "production database environment file is missing or unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
install -d -m 700 "$state_directory" "$backup_directory"
|
||||
[ ! -L "$state_directory" ] && [ ! -L "$backup_directory" ] || {
|
||||
echo "production state or backup directory is unsafe" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
exec 9>"$state_directory/mutation.lock"
|
||||
flock -n 9 || {
|
||||
echo "another production mutation holds the host lock" >&2
|
||||
exit 75
|
||||
}
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$environment_file"
|
||||
set +a
|
||||
: "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}"
|
||||
|
||||
usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')"
|
||||
[[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || {
|
||||
echo "production backup disk usage must remain below 70 percent" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
mapfile -t postgres_containers < <(
|
||||
sudo -n docker ps -q \
|
||||
--filter 'label=com.docker.compose.project=jyotisha-production' \
|
||||
--filter 'label=com.docker.compose.service=postgres'
|
||||
)
|
||||
[ "${#postgres_containers[@]}" -eq 1 ] || {
|
||||
echo "expected exactly one running production PostgreSQL container" >&2
|
||||
exit 1
|
||||
}
|
||||
postgres_container="${postgres_containers[0]}"
|
||||
|
||||
created_compact="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
backup_basename="production-pre-migration-${created_compact}.dump.enc"
|
||||
backup_partial="$backup_directory/.${backup_basename}.$$.partial"
|
||||
backup_file="$backup_directory/$backup_basename"
|
||||
restore_database="restore_verify_${created_compact,,}"
|
||||
restore_database="${restore_database//[^a-z0-9_]/_}"
|
||||
|
||||
cleanup() {
|
||||
rm -f -- "$backup_partial"
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "DROP DATABASE IF EXISTS \"$restore_database\" WITH (FORCE);" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-acl |
|
||||
openssl enc -aes-256-cbc -salt -pbkdf2 \
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY >"$backup_partial"
|
||||
[ -s "$backup_partial" ]
|
||||
chmod 600 "$backup_partial"
|
||||
mv "$backup_partial" "$backup_file"
|
||||
backup_sha256="$(sha256sum "$backup_file" | awk '{print $1}')"
|
||||
[[ "$backup_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "CREATE DATABASE \"$restore_database\";" >/dev/null
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 \
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$backup_file" |
|
||||
sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
pg_restore -U "$POSTGRES_USER" -d "$restore_database" \
|
||||
--no-owner --no-acl --exit-on-error
|
||||
|
||||
counts="$({
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$restore_database" -At -F '|' -v ON_ERROR_STOP=1 -c \
|
||||
"SELECT
|
||||
(SELECT count(*) FROM identity.users),
|
||||
(SELECT count(*) FROM public.profiles),
|
||||
(SELECT count(*) FROM public.credit_transactions),
|
||||
(SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public');"
|
||||
} | tail -n 1)"
|
||||
IFS='|' read -r identity_users profiles credit_transactions public_tables <<<"$counts"
|
||||
for value in "$identity_users" "$profiles" "$credit_transactions" "$public_tables"; do
|
||||
[[ "$value" =~ ^[0-9]+$ ]] || {
|
||||
echo "restore verification returned an invalid count" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "DROP DATABASE \"$restore_database\" WITH (FORCE);" >/dev/null
|
||||
verified_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
trap - EXIT HUP INT TERM
|
||||
|
||||
verify_basename="${backup_basename%.dump.enc}-restore-verify.json"
|
||||
verify_file="$backup_directory/$verify_basename"
|
||||
printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s}\n}\n' \
|
||||
"$backup_basename" "$backup_sha256" "$created_at" "$verified_at" \
|
||||
"$identity_users" "$profiles" "$credit_transactions" "$public_tables" >"$verify_file"
|
||||
chmod 600 "$verify_file"
|
||||
|
||||
recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}"
|
||||
state_tmp="$state_directory/recovery-baseline.env.tmp.$$"
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nVERIFY_FILE=%s\n' \
|
||||
"$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp"
|
||||
chmod 600 "$state_tmp"
|
||||
mv "$state_tmp" "$state_directory/recovery-baseline.env"
|
||||
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
|
||||
"$recovery_reference" "$created_at" "$verified_at" "$backup_sha256" \
|
||||
"$backup_basename" "$verify_basename"
|
||||
Reference in New Issue
Block a user