feat: add encrypted staging database backups
This commit is contained in:
Jesse_Chen
Executable
+105
@@ -0,0 +1,105 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
set +x
|
||||||
|
|
||||||
|
if [ "$#" -ne 2 ]; then
|
||||||
|
echo "usage: backup-staging-postgres.sh DATABASE_ENV_FILE BACKUP_DIRECTORY" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
SCRIPT_DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPOSITORY_ROOT="$(cd "$SCRIPT_DIRECTORY/.." && pwd)"
|
||||||
|
VALIDATOR="$SCRIPT_DIRECTORY/validate-staging-database-env.sh"
|
||||||
|
|
||||||
|
DATABASE_ENV_FILE="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
|
||||||
|
export DATABASE_ENV_FILE
|
||||||
|
BACKUP_DIRECTORY_INPUT="$2"
|
||||||
|
|
||||||
|
if [ "$BACKUP_DIRECTORY_INPUT" = "/" ]; then
|
||||||
|
echo "backup directory must not be the filesystem root" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
"$VALIDATOR" "$DATABASE_ENV_FILE" >/dev/null
|
||||||
|
|
||||||
|
read_environment_value() {
|
||||||
|
local key="$1"
|
||||||
|
local value
|
||||||
|
|
||||||
|
value="$(sed -n -E "s/^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=[[:space:]]*(.*)$/\\2/p" "$DATABASE_ENV_FILE")"
|
||||||
|
case "$value" in
|
||||||
|
\"*\") value="${value#\"}"; value="${value%\"}" ;;
|
||||||
|
\'*\') value="${value#\'}"; value="${value%\'}" ;;
|
||||||
|
esac
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
POSTGRES_DB="$(read_environment_value POSTGRES_DB)"
|
||||||
|
POSTGRES_USER="$(read_environment_value POSTGRES_USER)"
|
||||||
|
STAGING_BACKUP_ENCRYPTION_KEY="$(read_environment_value STAGING_BACKUP_ENCRYPTION_KEY)"
|
||||||
|
export STAGING_BACKUP_ENCRYPTION_KEY
|
||||||
|
|
||||||
|
mkdir -p "$BACKUP_DIRECTORY_INPUT"
|
||||||
|
chmod 0700 "$BACKUP_DIRECTORY_INPUT"
|
||||||
|
BACKUP_DIRECTORY="$(cd "$BACKUP_DIRECTORY_INPUT" && pwd -P)"
|
||||||
|
|
||||||
|
DISK_USAGE="$(df -Pk "$BACKUP_DIRECTORY" | awk 'NR == 2 { gsub(/%/, "", $5); print $5 }')"
|
||||||
|
if ! [[ "$DISK_USAGE" =~ ^[0-9]+$ ]] || [ "$DISK_USAGE" -ge 70 ]; then
|
||||||
|
echo "backup directory disk usage must be below 70 percent" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
BACKUP_TIMESTAMP="${BACKUP_TIMESTAMP:-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||||
|
if ! [[ "$BACKUP_TIMESTAMP" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
|
||||||
|
echo "backup timestamp must use YYYYMMDDTHHMMSSZ" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
FILE_NAME="jyotisha-staging-${BACKUP_TIMESTAMP}.dump.enc"
|
||||||
|
FINAL_FILE="$BACKUP_DIRECTORY/$FILE_NAME"
|
||||||
|
PARTIAL_FILE="$BACKUP_DIRECTORY/.${FILE_NAME}.$$.partial"
|
||||||
|
|
||||||
|
if [ -e "$FINAL_FILE" ] || [ -L "$FINAL_FILE" ]; then
|
||||||
|
echo "backup destination already exists" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cleanup_partial() {
|
||||||
|
local status="$?"
|
||||||
|
if [ -n "${PARTIAL_FILE:-}" ] && [ -e "$PARTIAL_FILE" ]; then
|
||||||
|
rm -f "$PARTIAL_FILE"
|
||||||
|
fi
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
trap cleanup_partial EXIT HUP INT TERM
|
||||||
|
|
||||||
|
umask 077
|
||||||
|
: > "$PARTIAL_FILE"
|
||||||
|
chmod 0600 "$PARTIAL_FILE"
|
||||||
|
|
||||||
|
cd "$REPOSITORY_ROOT"
|
||||||
|
docker compose -p "${COMPOSE_PROJECT_NAME:-jyotisha-staging}" \
|
||||||
|
-f deploy/docker-compose.postgres.yml exec -T postgres \
|
||||||
|
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner |
|
||||||
|
openssl enc -aes-256-cbc -salt -pbkdf2 \
|
||||||
|
-pass env:STAGING_BACKUP_ENCRYPTION_KEY > "$PARTIAL_FILE"
|
||||||
|
|
||||||
|
chmod 0600 "$PARTIAL_FILE"
|
||||||
|
mv "$PARTIAL_FILE" "$FINAL_FILE"
|
||||||
|
PARTIAL_FILE=""
|
||||||
|
|
||||||
|
completed=()
|
||||||
|
while IFS= read -r path; do
|
||||||
|
name="${path##*/}"
|
||||||
|
if [[ "$name" =~ ^jyotisha-staging-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]]; then
|
||||||
|
completed+=("$name")
|
||||||
|
fi
|
||||||
|
done < <(find "$BACKUP_DIRECTORY" -maxdepth 1 -type f -name 'jyotisha-staging-*.dump.enc' -print | LC_ALL=C sort)
|
||||||
|
|
||||||
|
if [ "${#completed[@]}" -gt 3 ]; then
|
||||||
|
for ((index = 0; index < ${#completed[@]} - 3; index += 1)); do
|
||||||
|
rm -f "$BACKUP_DIRECTORY/${completed[$index]}"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'path=%s count=%s\n' "$FINAL_FILE" "$(( ${#completed[@]} > 3 ? 3 : ${#completed[@]} ))"
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import {
|
||||||
|
chmodSync,
|
||||||
|
mkdtempSync,
|
||||||
|
readdirSync,
|
||||||
|
readFileSync,
|
||||||
|
rmSync,
|
||||||
|
statSync,
|
||||||
|
writeFileSync,
|
||||||
|
} from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { test } from "node:test";
|
||||||
|
import { startPostgresFixture } from "./helpers/postgres-fixture";
|
||||||
|
|
||||||
|
const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url));
|
||||||
|
const backupScript = join(repositoryRoot, "deploy/backup-staging-postgres.sh");
|
||||||
|
const fixtureSecrets = [
|
||||||
|
"postgres-test-password",
|
||||||
|
"schema-owner-test-password",
|
||||||
|
"identity-runtime-test-password",
|
||||||
|
"app-runtime-test-password",
|
||||||
|
"admin-runtime-test-password",
|
||||||
|
"migration-runner-test-password",
|
||||||
|
"backup-reader-test-password",
|
||||||
|
"staging-backup-test-password",
|
||||||
|
];
|
||||||
|
|
||||||
|
function listBackups(directory: string): string[] {
|
||||||
|
return readdirSync(directory)
|
||||||
|
.filter((name) => /^jyotisha-staging-\d{8}T\d{6}Z\.dump\.enc$/.test(name))
|
||||||
|
.sort();
|
||||||
|
}
|
||||||
|
|
||||||
|
function listDumpArchive(fixture: ReturnType<typeof startPostgresFixture>, dump: Buffer): void {
|
||||||
|
const hostRestore = spawnSync("pg_restore", ["--list"], {
|
||||||
|
input: dump,
|
||||||
|
encoding: "utf8",
|
||||||
|
});
|
||||||
|
if (hostRestore.status === 0) return;
|
||||||
|
|
||||||
|
const result = spawnSync(
|
||||||
|
"docker",
|
||||||
|
[
|
||||||
|
"compose",
|
||||||
|
"--project-name",
|
||||||
|
fixture.projectName,
|
||||||
|
"--env-file",
|
||||||
|
fixture.databaseEnvFile,
|
||||||
|
"-f",
|
||||||
|
"../deploy/docker-compose.postgres.yml",
|
||||||
|
"-f",
|
||||||
|
"../deploy/docker-compose.postgres-ci.yml",
|
||||||
|
"exec",
|
||||||
|
"-T",
|
||||||
|
"postgres",
|
||||||
|
"pg_restore",
|
||||||
|
"--list",
|
||||||
|
],
|
||||||
|
{ cwd: join(repositoryRoot, "frontend"), input: dump, encoding: "utf8" },
|
||||||
|
);
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("staging backups are encrypted, atomic, private, and retain the newest three", () => {
|
||||||
|
const fixture = startPostgresFixture();
|
||||||
|
const backupDirectory = mkdtempSync(join(tmpdir(), "jyotisha-staging-backup-"));
|
||||||
|
const commandDirectory = mkdtempSync(join(tmpdir(), "jyotisha-backup-command-"));
|
||||||
|
const diskUsageCommand = join(commandDirectory, "df");
|
||||||
|
const timestamps = [
|
||||||
|
"20260720T010101Z",
|
||||||
|
"20260720T010102Z",
|
||||||
|
"20260720T010103Z",
|
||||||
|
"20260720T010104Z",
|
||||||
|
];
|
||||||
|
|
||||||
|
try {
|
||||||
|
writeFileSync(
|
||||||
|
diskUsageCommand,
|
||||||
|
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'\nprintf '%s\\n' '/dev/test 1000 100 900 10% /tmp'\n",
|
||||||
|
{ mode: 0o700 },
|
||||||
|
);
|
||||||
|
chmodSync(diskUsageCommand, 0o700);
|
||||||
|
|
||||||
|
for (const timestamp of timestamps) {
|
||||||
|
const result = spawnSync(
|
||||||
|
"bash",
|
||||||
|
[backupScript, fixture.databaseEnvFile, backupDirectory],
|
||||||
|
{
|
||||||
|
cwd: repositoryRoot,
|
||||||
|
encoding: "utf8",
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
BACKUP_TIMESTAMP: timestamp,
|
||||||
|
COMPOSE_PROJECT_NAME: fixture.projectName,
|
||||||
|
PATH: `${commandDirectory}:${process.env.PATH}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(fixtureSecrets.join("|")));
|
||||||
|
}
|
||||||
|
|
||||||
|
const backups = listBackups(backupDirectory);
|
||||||
|
assert.deepEqual(backups, [
|
||||||
|
"jyotisha-staging-20260720T010102Z.dump.enc",
|
||||||
|
"jyotisha-staging-20260720T010103Z.dump.enc",
|
||||||
|
"jyotisha-staging-20260720T010104Z.dump.enc",
|
||||||
|
]);
|
||||||
|
assert.deepEqual(
|
||||||
|
readdirSync(backupDirectory).filter((name) => name.endsWith(".partial")),
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
assert.equal(statSync(backupDirectory).mode & 0o777, 0o700);
|
||||||
|
for (const backup of backups) {
|
||||||
|
assert.equal(statSync(join(backupDirectory, backup)).mode & 0o777, 0o600);
|
||||||
|
}
|
||||||
|
|
||||||
|
const encrypted = readFileSync(join(backupDirectory, backups[0]));
|
||||||
|
const decrypted = spawnSync(
|
||||||
|
"openssl",
|
||||||
|
["enc", "-d", "-aes-256-cbc", "-pbkdf2", "-pass", "env:STAGING_BACKUP_ENCRYPTION_KEY"],
|
||||||
|
{
|
||||||
|
input: encrypted,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
STAGING_BACKUP_ENCRYPTION_KEY: "staging-backup-test-password",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(decrypted.status, 0, decrypted.stderr.toString());
|
||||||
|
listDumpArchive(fixture, decrypted.stdout);
|
||||||
|
} finally {
|
||||||
|
fixture.stop();
|
||||||
|
rmSync(backupDirectory, { force: true, recursive: true });
|
||||||
|
rmSync(commandDirectory, { force: true, recursive: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user