fix(release): daily backups no longer replace the recovery point
Product chose to keep the scheduled backup, but the current recovery point must stay the one created by hand. Daily runs still encrypt and restore-verify, then record a separate latest-backup note. Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
co-authored by
Grok (agent loop)
Grok (agent loop)
parent
6ba63470a9
commit
5c0d895391
@@ -167,6 +167,262 @@ def test_recovery_script_keeps_manual_names_unless_daily_prefix() -> None:
|
||||
assert "openssl enc -aes-256-cbc -salt -pbkdf2" in text
|
||||
assert "usage_percent < 70" in text
|
||||
assert "flock -n 9" in text
|
||||
assert '[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ]' in text
|
||||
assert 'echo "baseline_updated=false" >&2' in text
|
||||
assert 'echo "baseline_updated=true" >&2' in text
|
||||
assert "daily-backup-latest.env" in text
|
||||
|
||||
|
||||
def install_fake_recovery_bin(bin_dir: Path) -> None:
|
||||
bin_dir.mkdir()
|
||||
write_text(
|
||||
bin_dir / "sudo",
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nif [[ \"${1:-}\" == \"-n\" ]]; then shift; fi\nexec \"$@\"\n",
|
||||
)
|
||||
write_text(
|
||||
bin_dir / "df",
|
||||
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted' 'tmpfs 100 10 90 10% /'\n",
|
||||
)
|
||||
# Windows cannot chmod pytest's temp directories. These stand-ins only
|
||||
# exist on the test PATH. The production script still calls the real commands.
|
||||
write_text(
|
||||
bin_dir / "chmod",
|
||||
"#!/usr/bin/env bash\nexit 0\n",
|
||||
)
|
||||
# Git Bash on this Windows machine has no flock. The production script still calls it.
|
||||
write_text(
|
||||
bin_dir / "flock",
|
||||
"#!/usr/bin/env bash\nexit 0\n",
|
||||
)
|
||||
write_text(
|
||||
bin_dir / "install",
|
||||
"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
dirs=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-d) shift ;;
|
||||
-m) shift; shift ;;
|
||||
--) shift; break ;;
|
||||
-*) shift ;;
|
||||
*) dirs+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
mkdir -p "${dirs[@]}"
|
||||
""",
|
||||
)
|
||||
write_text(
|
||||
bin_dir / "docker",
|
||||
"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cmd="${1:-}"
|
||||
shift || true
|
||||
case "$cmd" in
|
||||
ps)
|
||||
printf '%s\\n' fixturecontainer
|
||||
;;
|
||||
inspect)
|
||||
printf '%s\\n' sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
|
||||
;;
|
||||
run)
|
||||
exit 0
|
||||
;;
|
||||
exec)
|
||||
tool=""
|
||||
sql=""
|
||||
prev=""
|
||||
for arg in "$@"; do
|
||||
if [[ "$prev" == "-c" ]]; then
|
||||
sql="$arg"
|
||||
fi
|
||||
case "$arg" in
|
||||
pg_dump|pg_restore|psql) tool="$arg" ;;
|
||||
esac
|
||||
prev="$arg"
|
||||
done
|
||||
case "$tool" in
|
||||
pg_dump)
|
||||
printf '%s\\n' fixture-dump
|
||||
;;
|
||||
pg_restore)
|
||||
cat >/dev/null
|
||||
;;
|
||||
psql)
|
||||
case "$sql" in
|
||||
*role_table_grants*) printf '%s\\n' 'reader|SELECT|profiles' ;;
|
||||
*column_privileges*) printf '%s\\n' 'reader|SELECT|profiles|id' ;;
|
||||
*pg_get_userbyid*) printf '%s\\n' 'profiles|app' ;;
|
||||
*identity.users*) printf '%s\\n' '1|2|3|4' ;;
|
||||
*"CREATE DATABASE"*|*"DROP DATABASE"*) ;;
|
||||
*)
|
||||
printf 'unexpected psql\\n' >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected docker exec\\n' >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected docker %s\\n' "$cmd" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
subprocess.run(
|
||||
[BASH, "-lc", 'chmod +x "$1/sudo" "$1/df" "$1/docker" "$1/chmod" "$1/install" "$1/flock"', "chmod", bash_path(bin_dir)],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def write_recovery_env(deploy: Path) -> None:
|
||||
write_text(
|
||||
deploy / ".env.production.database",
|
||||
"\n".join(
|
||||
[
|
||||
"POSTGRES_DB=fixturedatabase",
|
||||
"POSTGRES_USER=fixtureuser",
|
||||
"POSTGRES_PASSWORD=fixturepassword",
|
||||
"PRODUCTION_BACKUP_ENCRYPTION_KEY=fixtureencryptionkey",
|
||||
"",
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def run_recovery(deploy: Path, bin_dir: Path, prefix: str | None) -> subprocess.CompletedProcess[str]:
|
||||
env = os.environ.copy()
|
||||
env.pop("FEISHU_WEBHOOK_URL", None)
|
||||
env.pop("FEISHU_SIGN_SECRET", None)
|
||||
env.pop("RECOVERY_BASENAME_PREFIX", None)
|
||||
env["DEPLOY_PATH"] = bash_path(deploy)
|
||||
env["RECOVERY_RUN_ID"] = "4242"
|
||||
env["JYOTISHA_RECOVERY_FIXTURE"] = "1"
|
||||
if prefix:
|
||||
env["RECOVERY_BASENAME_PREFIX"] = prefix
|
||||
return subprocess.run(
|
||||
[
|
||||
BASH,
|
||||
"-lc",
|
||||
'export PATH="$1:$PATH"; exec bash "$2"',
|
||||
"bash",
|
||||
bash_path(bin_dir),
|
||||
bash_path(RECOVERY_SCRIPT),
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
|
||||
|
||||
def parse_env(text: str) -> dict[str, str]:
|
||||
parsed: dict[str, str] = {}
|
||||
for line in text.splitlines():
|
||||
key, value = line.split("=", 1)
|
||||
parsed[key] = value
|
||||
return parsed
|
||||
|
||||
|
||||
def test_daily_backup_does_not_replace_the_recovery_point(tmp_path: Path) -> None:
|
||||
deploy = tmp_path / "deploy"
|
||||
state = deploy / ".state"
|
||||
state.mkdir(parents=True)
|
||||
write_recovery_env(deploy)
|
||||
baseline = state / "recovery-baseline.env"
|
||||
original = (
|
||||
"RECOVERY_REFERENCE=gitea-actions-run-1/production-pre-migration-20261014T120000Z.dump.enc\n"
|
||||
"RECOVERY_CREATED_AT=2026-10-14T12:00:00Z\n"
|
||||
"RECOVERY_SHA256=" + ("ab" * 32) + "\n"
|
||||
"RESTORE_VERIFIED=true\n"
|
||||
"RECOVERY_FORMAT=2\n"
|
||||
"VERIFY_FILE=production-pre-migration-20261014T120000Z-restore-verify.json\n"
|
||||
)
|
||||
write_text(baseline, original)
|
||||
subprocess.run(
|
||||
[BASH, "-lc", 'touch -d "@$1" "$2"', "touch", "1700000000", bash_path(baseline)],
|
||||
check=True,
|
||||
)
|
||||
before_ns = baseline.stat().st_mtime_ns
|
||||
before_bytes = baseline.read_bytes()
|
||||
bin_dir = tmp_path / "bin"
|
||||
install_fake_recovery_bin(bin_dir)
|
||||
completed = run_recovery(deploy, bin_dir, "production-daily")
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert baseline.read_bytes() == before_bytes
|
||||
assert baseline.stat().st_mtime_ns == before_ns
|
||||
assert "baseline_updated" not in completed.stdout
|
||||
assert "baseline_updated=false" in completed.stderr.splitlines()
|
||||
assert "baseline_updated=true" not in completed.stderr
|
||||
assert len(completed.stdout.splitlines()) == 7
|
||||
stdout_env = parse_env(completed.stdout)
|
||||
assert stdout_env["BACKUP_BASENAME"].startswith("production-daily-")
|
||||
daily = state / "daily-backup-latest.env"
|
||||
daily_env = parse_env(daily.read_text(encoding="utf-8"))
|
||||
assert set(daily_env) == {"BACKUP_BASENAME", "BACKUP_SHA256", "VERIFIED_AT", "BACKUP_FORMAT"}
|
||||
assert daily_env["BACKUP_FORMAT"] == "2"
|
||||
assert daily_env["BACKUP_BASENAME"] == stdout_env["BACKUP_BASENAME"]
|
||||
assert daily_env["BACKUP_SHA256"] == stdout_env["RECOVERY_SHA256"]
|
||||
daily_text = daily.read_text(encoding="utf-8")
|
||||
assert "fixturepassword" not in daily_text
|
||||
assert "fixtureencryptionkey" not in daily_text
|
||||
assert "fixturedatabase" not in daily_text
|
||||
verify = deploy / "backups" / stdout_env["VERIFY_BASENAME"]
|
||||
payload = json.loads(verify.read_text(encoding="utf-8"))
|
||||
assert payload["checks"]["table_grants_match"] is True
|
||||
assert payload["checks"]["column_privileges_match"] is True
|
||||
assert payload["checks"]["table_owners_match"] is True
|
||||
assert "fixturepassword" not in verify.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_manual_recovery_still_writes_format_two(tmp_path: Path) -> None:
|
||||
deploy = tmp_path / "deploy"
|
||||
state = deploy / ".state"
|
||||
state.mkdir(parents=True)
|
||||
write_recovery_env(deploy)
|
||||
sentinel = state / "daily-backup-latest.env"
|
||||
write_text(sentinel, "BACKUP_BASENAME=keep\nBACKUP_FORMAT=2\n")
|
||||
before = sentinel.read_bytes()
|
||||
bin_dir = tmp_path / "bin"
|
||||
install_fake_recovery_bin(bin_dir)
|
||||
completed = run_recovery(deploy, bin_dir, None)
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
assert "baseline_updated=true" in completed.stderr.splitlines()
|
||||
assert "baseline_updated=false" not in completed.stderr
|
||||
assert len(completed.stdout.splitlines()) == 7
|
||||
baseline_text = (state / "recovery-baseline.env").read_text(encoding="utf-8")
|
||||
baseline = parse_env(baseline_text)
|
||||
assert baseline["RECOVERY_FORMAT"] == "2"
|
||||
assert baseline["VERIFY_FILE"].startswith("production-pre-migration-")
|
||||
assert baseline["RECOVERY_REFERENCE"].startswith("gitea-actions-run-4242/production-pre-migration-")
|
||||
assert "fixturepassword" not in baseline_text
|
||||
assert "fixtureencryptionkey" not in baseline_text
|
||||
assert parse_env(completed.stdout)["BACKUP_BASENAME"].startswith("production-pre-migration-")
|
||||
assert sentinel.read_bytes() == before
|
||||
|
||||
|
||||
def test_release_migration_and_rollback_ignore_daily_backup_record() -> None:
|
||||
readers = [
|
||||
ROOT / ".gitea" / "workflows" / "deploy-production.yml",
|
||||
ROOT / ".gitea" / "workflows" / "migrate-production-database.yml",
|
||||
ROOT / "deploy" / "run-production-deploy.sh",
|
||||
ROOT / "deploy" / "run-production-migration.sh",
|
||||
]
|
||||
for path in readers:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert "daily-backup-latest.env" not in text, path.name
|
||||
assert "recovery-baseline.env" in text
|
||||
for path in (WORKFLOW, RECOVERY_WORKFLOW):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert "JYOTISHA_RECOVERY_FIXTURE" not in text
|
||||
assert '[[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d \' \')" == 7 ]]' in text
|
||||
|
||||
|
||||
def test_prune_deletes_only_expired_daily_files(tmp_path: Path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user