feat: add encrypted staging database backups
This commit is contained in:
Jesse_Chen
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set +x
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: backup-staging-postgres.sh DATABASE_ENV_FILE BACKUP_DIRECTORY" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPOSITORY_ROOT="$(cd "$SCRIPT_DIRECTORY/.." && pwd)"
|
||||
VALIDATOR="$SCRIPT_DIRECTORY/validate-staging-database-env.sh"
|
||||
|
||||
DATABASE_ENV_FILE="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
|
||||
export DATABASE_ENV_FILE
|
||||
BACKUP_DIRECTORY_INPUT="$2"
|
||||
|
||||
if [ "$BACKUP_DIRECTORY_INPUT" = "/" ]; then
|
||||
echo "backup directory must not be the filesystem root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$VALIDATOR" "$DATABASE_ENV_FILE" >/dev/null
|
||||
|
||||
read_environment_value() {
|
||||
local key="$1"
|
||||
local value
|
||||
|
||||
value="$(sed -n -E "s/^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=[[:space:]]*(.*)$/\\2/p" "$DATABASE_ENV_FILE")"
|
||||
case "$value" in
|
||||
\"*\") value="${value#\"}"; value="${value%\"}" ;;
|
||||
\'*\') value="${value#\'}"; value="${value%\'}" ;;
|
||||
esac
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
POSTGRES_DB="$(read_environment_value POSTGRES_DB)"
|
||||
POSTGRES_USER="$(read_environment_value POSTGRES_USER)"
|
||||
STAGING_BACKUP_ENCRYPTION_KEY="$(read_environment_value STAGING_BACKUP_ENCRYPTION_KEY)"
|
||||
export STAGING_BACKUP_ENCRYPTION_KEY
|
||||
|
||||
mkdir -p "$BACKUP_DIRECTORY_INPUT"
|
||||
chmod 0700 "$BACKUP_DIRECTORY_INPUT"
|
||||
BACKUP_DIRECTORY="$(cd "$BACKUP_DIRECTORY_INPUT" && pwd -P)"
|
||||
|
||||
DISK_USAGE="$(df -Pk "$BACKUP_DIRECTORY" | awk 'NR == 2 { gsub(/%/, "", $5); print $5 }')"
|
||||
if ! [[ "$DISK_USAGE" =~ ^[0-9]+$ ]] || [ "$DISK_USAGE" -ge 70 ]; then
|
||||
echo "backup directory disk usage must be below 70 percent" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BACKUP_TIMESTAMP="${BACKUP_TIMESTAMP:-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
if ! [[ "$BACKUP_TIMESTAMP" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
|
||||
echo "backup timestamp must use YYYYMMDDTHHMMSSZ" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILE_NAME="jyotisha-staging-${BACKUP_TIMESTAMP}.dump.enc"
|
||||
FINAL_FILE="$BACKUP_DIRECTORY/$FILE_NAME"
|
||||
PARTIAL_FILE="$BACKUP_DIRECTORY/.${FILE_NAME}.$$.partial"
|
||||
|
||||
if [ -e "$FINAL_FILE" ] || [ -L "$FINAL_FILE" ]; then
|
||||
echo "backup destination already exists" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup_partial() {
|
||||
local status="$?"
|
||||
if [ -n "${PARTIAL_FILE:-}" ] && [ -e "$PARTIAL_FILE" ]; then
|
||||
rm -f "$PARTIAL_FILE"
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup_partial EXIT HUP INT TERM
|
||||
|
||||
umask 077
|
||||
: > "$PARTIAL_FILE"
|
||||
chmod 0600 "$PARTIAL_FILE"
|
||||
|
||||
cd "$REPOSITORY_ROOT"
|
||||
docker compose -p "${COMPOSE_PROJECT_NAME:-jyotisha-staging}" \
|
||||
-f deploy/docker-compose.postgres.yml exec -T postgres \
|
||||
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner |
|
||||
openssl enc -aes-256-cbc -salt -pbkdf2 \
|
||||
-pass env:STAGING_BACKUP_ENCRYPTION_KEY > "$PARTIAL_FILE"
|
||||
|
||||
chmod 0600 "$PARTIAL_FILE"
|
||||
mv "$PARTIAL_FILE" "$FINAL_FILE"
|
||||
PARTIAL_FILE=""
|
||||
|
||||
completed=()
|
||||
while IFS= read -r path; do
|
||||
name="${path##*/}"
|
||||
if [[ "$name" =~ ^jyotisha-staging-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]]; then
|
||||
completed+=("$name")
|
||||
fi
|
||||
done < <(find "$BACKUP_DIRECTORY" -maxdepth 1 -type f -name 'jyotisha-staging-*.dump.enc' -print | LC_ALL=C sort)
|
||||
|
||||
if [ "${#completed[@]}" -gt 3 ]; then
|
||||
for ((index = 0; index < ${#completed[@]} - 3; index += 1)); do
|
||||
rm -f "$BACKUP_DIRECTORY/${completed[$index]}"
|
||||
done
|
||||
fi
|
||||
|
||||
printf 'path=%s count=%s\n' "$FINAL_FILE" "$(( ${#completed[@]} > 3 ? 3 : ${#completed[@]} ))"
|
||||
@@ -0,0 +1,140 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { test } from "node:test";
|
||||
import { startPostgresFixture } from "./helpers/postgres-fixture";
|
||||
|
||||
const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url));
|
||||
const backupScript = join(repositoryRoot, "deploy/backup-staging-postgres.sh");
|
||||
const fixtureSecrets = [
|
||||
"postgres-test-password",
|
||||
"schema-owner-test-password",
|
||||
"identity-runtime-test-password",
|
||||
"app-runtime-test-password",
|
||||
"admin-runtime-test-password",
|
||||
"migration-runner-test-password",
|
||||
"backup-reader-test-password",
|
||||
"staging-backup-test-password",
|
||||
];
|
||||
|
||||
function listBackups(directory: string): string[] {
|
||||
return readdirSync(directory)
|
||||
.filter((name) => /^jyotisha-staging-\d{8}T\d{6}Z\.dump\.enc$/.test(name))
|
||||
.sort();
|
||||
}
|
||||
|
||||
function listDumpArchive(fixture: ReturnType<typeof startPostgresFixture>, dump: Buffer): void {
|
||||
const hostRestore = spawnSync("pg_restore", ["--list"], {
|
||||
input: dump,
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (hostRestore.status === 0) return;
|
||||
|
||||
const result = spawnSync(
|
||||
"docker",
|
||||
[
|
||||
"compose",
|
||||
"--project-name",
|
||||
fixture.projectName,
|
||||
"--env-file",
|
||||
fixture.databaseEnvFile,
|
||||
"-f",
|
||||
"../deploy/docker-compose.postgres.yml",
|
||||
"-f",
|
||||
"../deploy/docker-compose.postgres-ci.yml",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"pg_restore",
|
||||
"--list",
|
||||
],
|
||||
{ cwd: join(repositoryRoot, "frontend"), input: dump, encoding: "utf8" },
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
}
|
||||
|
||||
test("staging backups are encrypted, atomic, private, and retain the newest three", () => {
|
||||
const fixture = startPostgresFixture();
|
||||
const backupDirectory = mkdtempSync(join(tmpdir(), "jyotisha-staging-backup-"));
|
||||
const commandDirectory = mkdtempSync(join(tmpdir(), "jyotisha-backup-command-"));
|
||||
const diskUsageCommand = join(commandDirectory, "df");
|
||||
const timestamps = [
|
||||
"20260720T010101Z",
|
||||
"20260720T010102Z",
|
||||
"20260720T010103Z",
|
||||
"20260720T010104Z",
|
||||
];
|
||||
|
||||
try {
|
||||
writeFileSync(
|
||||
diskUsageCommand,
|
||||
"#!/usr/bin/env bash\nprintf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'\nprintf '%s\\n' '/dev/test 1000 100 900 10% /tmp'\n",
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
chmodSync(diskUsageCommand, 0o700);
|
||||
|
||||
for (const timestamp of timestamps) {
|
||||
const result = spawnSync(
|
||||
"bash",
|
||||
[backupScript, fixture.databaseEnvFile, backupDirectory],
|
||||
{
|
||||
cwd: repositoryRoot,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
BACKUP_TIMESTAMP: timestamp,
|
||||
COMPOSE_PROJECT_NAME: fixture.projectName,
|
||||
PATH: `${commandDirectory}:${process.env.PATH}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(fixtureSecrets.join("|")));
|
||||
}
|
||||
|
||||
const backups = listBackups(backupDirectory);
|
||||
assert.deepEqual(backups, [
|
||||
"jyotisha-staging-20260720T010102Z.dump.enc",
|
||||
"jyotisha-staging-20260720T010103Z.dump.enc",
|
||||
"jyotisha-staging-20260720T010104Z.dump.enc",
|
||||
]);
|
||||
assert.deepEqual(
|
||||
readdirSync(backupDirectory).filter((name) => name.endsWith(".partial")),
|
||||
[],
|
||||
);
|
||||
assert.equal(statSync(backupDirectory).mode & 0o777, 0o700);
|
||||
for (const backup of backups) {
|
||||
assert.equal(statSync(join(backupDirectory, backup)).mode & 0o777, 0o600);
|
||||
}
|
||||
|
||||
const encrypted = readFileSync(join(backupDirectory, backups[0]));
|
||||
const decrypted = spawnSync(
|
||||
"openssl",
|
||||
["enc", "-d", "-aes-256-cbc", "-pbkdf2", "-pass", "env:STAGING_BACKUP_ENCRYPTION_KEY"],
|
||||
{
|
||||
input: encrypted,
|
||||
env: {
|
||||
...process.env,
|
||||
STAGING_BACKUP_ENCRYPTION_KEY: "staging-backup-test-password",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(decrypted.status, 0, decrypted.stderr.toString());
|
||||
listDumpArchive(fixture, decrypted.stdout);
|
||||
} finally {
|
||||
fixture.stop();
|
||||
rmSync(backupDirectory, { force: true, recursive: true });
|
||||
rmSync(commandDirectory, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user