From 10bec0408bcbc49403fb16d71f912804c1c9706a Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Mon, 20 Jul 2026 18:02:19 +0800 Subject: [PATCH] feat: add private staging postgres topology --- deploy/docker-compose.postgres-ci.yml | 4 + deploy/docker-compose.postgres.yml | 59 ++++++++++ deploy/postgres/001-bootstrap-roles.sh | 72 ++++++++++++ deploy/validate-staging-database-env.sh | 91 +++++++++++++++ frontend/package.json | 1 + frontend/tests/database-topology.test.ts | 38 ++++++ frontend/tests/helpers/postgres-fixture.ts | 128 +++++++++++++++++++++ 7 files changed, 393 insertions(+) create mode 100644 deploy/docker-compose.postgres-ci.yml create mode 100644 deploy/docker-compose.postgres.yml create mode 100755 deploy/postgres/001-bootstrap-roles.sh create mode 100755 deploy/validate-staging-database-env.sh create mode 100644 frontend/tests/database-topology.test.ts create mode 100644 frontend/tests/helpers/postgres-fixture.ts diff --git a/deploy/docker-compose.postgres-ci.yml b/deploy/docker-compose.postgres-ci.yml new file mode 100644 index 00000000..574d1f02 --- /dev/null +++ b/deploy/docker-compose.postgres-ci.yml @@ -0,0 +1,4 @@ +services: + postgres: + ports: + - "127.0.0.1:${POSTGRES_HOST_PORT:-55432}:5432" diff --git a/deploy/docker-compose.postgres.yml b/deploy/docker-compose.postgres.yml new file mode 100644 index 00000000..7d26d900 --- /dev/null +++ b/deploy/docker-compose.postgres.yml @@ -0,0 +1,59 @@ +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + shm_size: 128mb + env_file: + - ${DATABASE_ENV_FILE:-../.env.staging.database} + command: + - postgres + - -c + - max_connections=30 + - -c + - shared_buffers=256MB + - -c + - effective_cache_size=1GB + - -c + - work_mem=4MB + volumes: + - postgres_data:/var/lib/postgresql/data + - ./postgres/001-bootstrap-roles.sh:/docker-entrypoint-initdb.d/001-bootstrap-roles.sh:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""] + interval: 5s + timeout: 5s + retries: 20 + start_period: 10s + networks: [app] + + migrator: + image: ${WEB_IMAGE:-jyotisha-web:local} + profiles: ["migration"] + restart: "no" + env_file: + - ${DATABASE_ENV_FILE:-../.env.staging.database} + working_dir: /app/frontend + command: ["npm", "run", "db:migrate"] + depends_on: + postgres: + condition: service_healthy + networks: [app] + + migration-checker: + image: ${WEB_IMAGE:-jyotisha-web:local} + profiles: ["migration-check"] + restart: "no" + env_file: + - ${DATABASE_ENV_FILE:-../.env.staging.database} + working_dir: /app/frontend + command: ["npm", "run", "db:migrate:check"] + depends_on: + postgres: + condition: service_healthy + networks: [app] + +volumes: + postgres_data: + +networks: + app: diff --git a/deploy/postgres/001-bootstrap-roles.sh b/deploy/postgres/001-bootstrap-roles.sh new file mode 100755 index 00000000..83d6c945 --- /dev/null +++ b/deploy/postgres/001-bootstrap-roles.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +required=( + POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD + SCHEMA_OWNER_PASSWORD IDENTITY_RUNTIME_PASSWORD APP_RUNTIME_PASSWORD + ADMIN_RUNTIME_PASSWORD MIGRATION_RUNNER_PASSWORD BACKUP_READER_PASSWORD +) +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "required database bootstrap variable is missing: $key" >&2 + exit 1 + fi +done + +psql --set ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" \ + --dbname "$POSTGRES_DB" \ + --set database_name="$POSTGRES_DB" \ + --set schema_owner_password="$SCHEMA_OWNER_PASSWORD" \ + --set identity_runtime_password="$IDENTITY_RUNTIME_PASSWORD" \ + --set app_runtime_password="$APP_RUNTIME_PASSWORD" \ + --set admin_runtime_password="$ADMIN_RUNTIME_PASSWORD" \ + --set migration_runner_password="$MIGRATION_RUNNER_PASSWORD" \ + --set backup_reader_password="$BACKUP_READER_PASSWORD" <<'SQL' +SELECT format( + 'CREATE ROLE schema_owner WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L', + :'schema_owner_password' +) WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'schema_owner' +) \gexec +SELECT format( + 'CREATE ROLE identity_runtime WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L', + :'identity_runtime_password' +) WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'identity_runtime' +) \gexec +SELECT format( + 'CREATE ROLE app_runtime WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L', + :'app_runtime_password' +) WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'app_runtime' +) \gexec +SELECT format( + 'CREATE ROLE admin_runtime WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L', + :'admin_runtime_password' +) WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'admin_runtime' +) \gexec +SELECT format( + 'CREATE ROLE migration_runner WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L', + :'migration_runner_password' +) WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'migration_runner' +) \gexec +SELECT format( + 'CREATE ROLE backup_reader WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L', + :'backup_reader_password' +) WHERE NOT EXISTS ( + SELECT 1 FROM pg_roles WHERE rolname = 'backup_reader' +) \gexec + +SELECT format( + 'GRANT CONNECT, CREATE ON DATABASE %I TO schema_owner', + :'database_name' +) \gexec +SELECT format( + 'GRANT CONNECT ON DATABASE %I TO identity_runtime, app_runtime, admin_runtime, migration_runner, backup_reader', + :'database_name' +) \gexec +SQL diff --git a/deploy/validate-staging-database-env.sh b/deploy/validate-staging-database-env.sh new file mode 100755 index 00000000..f317dd40 --- /dev/null +++ b/deploy/validate-staging-database-env.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +ENV_FILE="${1:-.env.staging.database}" + +if [ ! -e "$ENV_FILE" ]; then + echo "staging database environment file is missing" >&2 + exit 1 +fi + +if [ -L "$ENV_FILE" ]; then + echo "staging database environment file must not be a symlink" >&2 + exit 1 +fi + +if [ ! -f "$ENV_FILE" ]; then + echo "staging database environment path must be a regular file" >&2 + exit 1 +fi + +if MODE="$(stat -c '%a' "$ENV_FILE" 2>/dev/null)"; then + : +else + MODE="$(stat -f '%Lp' "$ENV_FILE")" +fi + +if [ "$MODE" != "600" ]; then + echo "staging database environment file must have mode 0600" >&2 + exit 1 +fi + +if OWNER="$(stat -c '%u' "$ENV_FILE" 2>/dev/null)"; then + : +else + OWNER="$(stat -f '%u' "$ENV_FILE")" +fi + +if [ "$OWNER" != "$(id -u)" ]; then + echo "staging database environment file must be owned by the current user" >&2 + exit 1 +fi + +definition_count() { + local key="$1" + grep -Ec "^(export[[:space:]]+)?${key}=" "$ENV_FILE" || true +} + +environment_value() { + local key="$1" + sed -n -E "s/^(export[[:space:]]+)?${key}=(.*)$/\\2/p" "$ENV_FILE" +} + +require_once_non_empty() { + local key="$1" + local count + local value + count="$(definition_count "$key")" + value="$(environment_value "$key")" + if [ "$count" -ne 1 ] || [ -z "$value" ]; then + echo "required staging database environment variable is missing or duplicated: $key" >&2 + exit 1 + fi +} + +required=( + POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD + SCHEMA_OWNER_PASSWORD IDENTITY_RUNTIME_PASSWORD APP_RUNTIME_PASSWORD + ADMIN_RUNTIME_PASSWORD MIGRATION_RUNNER_PASSWORD BACKUP_READER_PASSWORD + STAGING_BACKUP_ENCRYPTION_KEY SCHEMA_DATABASE_URL +) +for key in "${required[@]}"; do + require_once_non_empty "$key" +done + +if [ "$(environment_value POSTGRES_DB)" != "jyotisha" ]; then + echo "invalid staging database selector: POSTGRES_DB" >&2 + exit 1 +fi + +if [ "$(environment_value POSTGRES_USER)" != "postgres" ]; then + echo "invalid staging database selector: POSTGRES_USER" >&2 + exit 1 +fi + +if ! [[ "$(environment_value SCHEMA_DATABASE_URL)" =~ ^postgresql://schema_owner:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then + echo "invalid staging database selector: SCHEMA_DATABASE_URL" >&2 + exit 1 +fi + +echo "staging database environment validated" diff --git a/frontend/package.json b/frontend/package.json index 10d41ed1..e538b24a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -8,6 +8,7 @@ "build": "next build", "start": "next start", "test": "tsx --test tests/*.test.ts", + "test:db": "tsx --test --test-concurrency=1 tests/database-*.test.ts", "lint": "eslint", "data:china": "node scripts/pull-china-locations.mjs" }, diff --git a/frontend/tests/database-topology.test.ts b/frontend/tests/database-topology.test.ts new file mode 100644 index 00000000..f6309675 --- /dev/null +++ b/frontend/tests/database-topology.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { startPostgresFixture } from "./helpers/postgres-fixture"; + +test("staging postgres is private and CI binds loopback only", () => { + const staging = readFileSync("../deploy/docker-compose.postgres.yml", "utf8"); + const ci = readFileSync("../deploy/docker-compose.postgres-ci.yml", "utf8"); + assert.match(staging, /image:\s*postgres:17-alpine/); + assert.doesNotMatch(staging, /^\s+ports:/m); + assert.match(ci, /127\.0\.0\.1:\$\{POSTGRES_HOST_PORT:-55432\}:5432/); +}); + +test("database roles have no cluster privileges", () => { + const fixture = startPostgresFixture(); + try { + assert.equal( + fixture.psql(` + select rolname || ':' || rolsuper || ':' || rolcreatedb || ':' || + rolcreaterole || ':' || rolbypassrls + from pg_roles + where rolname in ('schema_owner','identity_runtime','app_runtime', + 'admin_runtime','migration_runner','backup_reader') + order by rolname + `), + [ + "admin_runtime:f:f:f:f", + "app_runtime:f:f:f:f", + "backup_reader:f:f:f:f", + "identity_runtime:f:f:f:f", + "migration_runner:f:f:f:f", + "schema_owner:f:f:f:f", + ].join("\n"), + ); + } finally { + fixture.stop(); + } +}); diff --git a/frontend/tests/helpers/postgres-fixture.ts b/frontend/tests/helpers/postgres-fixture.ts new file mode 100644 index 00000000..6e5cee46 --- /dev/null +++ b/frontend/tests/helpers/postgres-fixture.ts @@ -0,0 +1,128 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +export type PostgresFixture = { + projectName: string; + databaseEnvFile: string; + hostPort: number; + connectionUrl(role: string, password: string): string; + psql(sql: string): string; + stop(): void; +}; + +const databaseEnvironment = `POSTGRES_DB=jyotisha +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres-test-password +SCHEMA_OWNER_PASSWORD=schema-owner-test-password +IDENTITY_RUNTIME_PASSWORD=identity-runtime-test-password +APP_RUNTIME_PASSWORD=app-runtime-test-password +ADMIN_RUNTIME_PASSWORD=admin-runtime-test-password +MIGRATION_RUNNER_PASSWORD=migration-runner-test-password +BACKUP_READER_PASSWORD=backup-reader-test-password +STAGING_BACKUP_ENCRYPTION_KEY=staging-backup-test-password +SCHEMA_DATABASE_URL=postgresql://schema_owner:schema-owner-test-password@postgres:5432/jyotisha +`; + +function isPortAvailable(port: number): boolean { + return ( + spawnSync( + process.execPath, + [ + "-e", + `const server = require("node:net").createServer(); +server.once("error", () => process.exit(1)); +server.listen({ host: "127.0.0.1", port: Number(process.argv[1]) }, () => + server.close(() => process.exit(0)), +);`, + String(port), + ], + { stdio: "ignore" }, + ).status === 0 + ); +} + +function findAvailablePort(): number { + for (let port = 55432; port <= 55531; port += 1) { + if (isPortAvailable(port)) { + return port; + } + } + + throw new Error("no available PostgreSQL test port in 55432..55531"); +} + +export function startPostgresFixture(): PostgresFixture { + const projectName = `jyotisha-postgres-${process.pid}-${Date.now()}`; + const temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-")); + const databaseEnvFile = join(temporaryDirectory, "database.env"); + const hostPort = findAvailablePort(); + const composeArguments = [ + "compose", + "--project-name", + projectName, + "--env-file", + databaseEnvFile, + "-f", + "../deploy/docker-compose.postgres.yml", + "-f", + "../deploy/docker-compose.postgres-ci.yml", + ]; + const environment = { + ...process.env, + DATABASE_ENV_FILE: databaseEnvFile, + POSTGRES_HOST_PORT: String(hostPort), + }; + + writeFileSync(databaseEnvFile, databaseEnvironment, { mode: 0o600 }); + chmodSync(databaseEnvFile, 0o600); + + try { + execFileSync( + "docker", + [...composeArguments, "up", "-d", "--wait", "postgres"], + { env: environment, stdio: "inherit" }, + ); + } catch (error) { + try { + execFileSync( + "docker", + [...composeArguments, "down", "-v", "--remove-orphans"], + { env: environment, stdio: "inherit" }, + ); + } finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); + } + throw error; + } + + return { + projectName, + databaseEnvFile, + hostPort, + connectionUrl(role, password) { + return `postgresql://${role}:${password}@127.0.0.1:${hostPort}/jyotisha`; + }, + psql(sql) { + return execFileSync( + "docker", + [...composeArguments, "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "jyotisha", "-Atc", sql], + { encoding: "utf8", env: environment }, + ) + .trim() + .replace(/(^|:)false(?=:|$)/gm, "$1f"); + }, + stop() { + try { + execFileSync( + "docker", + [...composeArguments, "down", "-v", "--remove-orphans"], + { env: environment, stdio: "inherit" }, + ); + } finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); + } + }, + }; +}