feat: add private staging postgres topology

2026-07-20 18:02:19 +08:00
parent d913291cee
commit 10bec0408b
7 changed files with 393 additions and 0 deletions
@@ -0,0 +1,4 @@
+services:
+  postgres:
+    ports:
+      - "127.0.0.1:${POSTGRES_HOST_PORT:-55432}:5432"
@@ -0,0 +1,59 @@
+services:
+  postgres:
+    image: postgres:17-alpine
+    restart: unless-stopped
+    shm_size: 128mb
+    env_file:
+      - ${DATABASE_ENV_FILE:-../.env.staging.database}
+    command:
+      - postgres
+      - -c
+      - max_connections=30
+      - -c
+      - shared_buffers=256MB
+      - -c
+      - effective_cache_size=1GB
+      - -c
+      - work_mem=4MB
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./postgres/001-bootstrap-roles.sh:/docker-entrypoint-initdb.d/001-bootstrap-roles.sh:ro
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+      start_period: 10s
+    networks: [app]
+
+  migrator:
+    image: ${WEB_IMAGE:-jyotisha-web:local}
+    profiles: ["migration"]
+    restart: "no"
+    env_file:
+      - ${DATABASE_ENV_FILE:-../.env.staging.database}
+    working_dir: /app/frontend
+    command: ["npm", "run", "db:migrate"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks: [app]
+
+  migration-checker:
+    image: ${WEB_IMAGE:-jyotisha-web:local}
+    profiles: ["migration-check"]
+    restart: "no"
+    env_file:
+      - ${DATABASE_ENV_FILE:-../.env.staging.database}
+    working_dir: /app/frontend
+    command: ["npm", "run", "db:migrate:check"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks: [app]
+
+volumes:
+  postgres_data:
+
+networks:
+  app:
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set +x
+
+required=(
+  POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD
+  SCHEMA_OWNER_PASSWORD IDENTITY_RUNTIME_PASSWORD APP_RUNTIME_PASSWORD
+  ADMIN_RUNTIME_PASSWORD MIGRATION_RUNNER_PASSWORD BACKUP_READER_PASSWORD
+)
+for key in "${required[@]}"; do
+  if [ -z "${!key:-}" ]; then
+    echo "required database bootstrap variable is missing: $key" >&2
+    exit 1
+  fi
+done
+
+psql --set ON_ERROR_STOP=1 \
+  --username "$POSTGRES_USER" \
+  --dbname "$POSTGRES_DB" \
+  --set database_name="$POSTGRES_DB" \
+  --set schema_owner_password="$SCHEMA_OWNER_PASSWORD" \
+  --set identity_runtime_password="$IDENTITY_RUNTIME_PASSWORD" \
+  --set app_runtime_password="$APP_RUNTIME_PASSWORD" \
+  --set admin_runtime_password="$ADMIN_RUNTIME_PASSWORD" \
+  --set migration_runner_password="$MIGRATION_RUNNER_PASSWORD" \
+  --set backup_reader_password="$BACKUP_READER_PASSWORD" <<'SQL'
+SELECT format(
+  'CREATE ROLE schema_owner WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L',
+  :'schema_owner_password'
+) WHERE NOT EXISTS (
+  SELECT 1 FROM pg_roles WHERE rolname = 'schema_owner'
+) \gexec
+SELECT format(
+  'CREATE ROLE identity_runtime WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L',
+  :'identity_runtime_password'
+) WHERE NOT EXISTS (
+  SELECT 1 FROM pg_roles WHERE rolname = 'identity_runtime'
+) \gexec
+SELECT format(
+  'CREATE ROLE app_runtime WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L',
+  :'app_runtime_password'
+) WHERE NOT EXISTS (
+  SELECT 1 FROM pg_roles WHERE rolname = 'app_runtime'
+) \gexec
+SELECT format(
+  'CREATE ROLE admin_runtime WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L',
+  :'admin_runtime_password'
+) WHERE NOT EXISTS (
+  SELECT 1 FROM pg_roles WHERE rolname = 'admin_runtime'
+) \gexec
+SELECT format(
+  'CREATE ROLE migration_runner WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L',
+  :'migration_runner_password'
+) WHERE NOT EXISTS (
+  SELECT 1 FROM pg_roles WHERE rolname = 'migration_runner'
+) \gexec
+SELECT format(
+  'CREATE ROLE backup_reader WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT PASSWORD %L',
+  :'backup_reader_password'
+) WHERE NOT EXISTS (
+  SELECT 1 FROM pg_roles WHERE rolname = 'backup_reader'
+) \gexec
+
+SELECT format(
+  'GRANT CONNECT, CREATE ON DATABASE %I TO schema_owner',
+  :'database_name'
+) \gexec
+SELECT format(
+  'GRANT CONNECT ON DATABASE %I TO identity_runtime, app_runtime, admin_runtime, migration_runner, backup_reader',
+  :'database_name'
+) \gexec
+SQL
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set +x
+
+ENV_FILE="${1:-.env.staging.database}"
+
+if [ ! -e "$ENV_FILE" ]; then
+  echo "staging database environment file is missing" >&2
+  exit 1
+fi
+
+if [ -L "$ENV_FILE" ]; then
+  echo "staging database environment file must not be a symlink" >&2
+  exit 1
+fi
+
+if [ ! -f "$ENV_FILE" ]; then
+  echo "staging database environment path must be a regular file" >&2
+  exit 1
+fi
+
+if MODE="$(stat -c '%a' "$ENV_FILE" 2>/dev/null)"; then
+  :
+else
+  MODE="$(stat -f '%Lp' "$ENV_FILE")"
+fi
+
+if [ "$MODE" != "600" ]; then
+  echo "staging database environment file must have mode 0600" >&2
+  exit 1
+fi
+
+if OWNER="$(stat -c '%u' "$ENV_FILE" 2>/dev/null)"; then
+  :
+else
+  OWNER="$(stat -f '%u' "$ENV_FILE")"
+fi
+
+if [ "$OWNER" != "$(id -u)" ]; then
+  echo "staging database environment file must be owned by the current user" >&2
+  exit 1
+fi
+
+definition_count() {
+  local key="$1"
+  grep -Ec "^(export[[:space:]]+)?${key}=" "$ENV_FILE" || true
+}
+
+environment_value() {
+  local key="$1"
+  sed -n -E "s/^(export[[:space:]]+)?${key}=(.*)$/\\2/p" "$ENV_FILE"
+}
+
+require_once_non_empty() {
+  local key="$1"
+  local count
+  local value
+  count="$(definition_count "$key")"
+  value="$(environment_value "$key")"
+  if [ "$count" -ne 1 ] || [ -z "$value" ]; then
+    echo "required staging database environment variable is missing or duplicated: $key" >&2
+    exit 1
+  fi
+}
+
+required=(
+  POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD
+  SCHEMA_OWNER_PASSWORD IDENTITY_RUNTIME_PASSWORD APP_RUNTIME_PASSWORD
+  ADMIN_RUNTIME_PASSWORD MIGRATION_RUNNER_PASSWORD BACKUP_READER_PASSWORD
+  STAGING_BACKUP_ENCRYPTION_KEY SCHEMA_DATABASE_URL
+)
+for key in "${required[@]}"; do
+  require_once_non_empty "$key"
+done
+
+if [ "$(environment_value POSTGRES_DB)" != "jyotisha" ]; then
+  echo "invalid staging database selector: POSTGRES_DB" >&2
+  exit 1
+fi
+
+if [ "$(environment_value POSTGRES_USER)" != "postgres" ]; then
+  echo "invalid staging database selector: POSTGRES_USER" >&2
+  exit 1
+fi
+
+if ! [[ "$(environment_value SCHEMA_DATABASE_URL)" =~ ^postgresql://schema_owner:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then
+  echo "invalid staging database selector: SCHEMA_DATABASE_URL" >&2
+  exit 1
+fi
+
+echo "staging database environment validated"
@@ -8,6 +8,7 @@
    "build": "next build",
    "start": "next start",
    "test": "tsx --test tests/*.test.ts",
+    "test:db": "tsx --test --test-concurrency=1 tests/database-*.test.ts",
    "lint": "eslint",
    "data:china": "node scripts/pull-china-locations.mjs"
  },
@@ -0,0 +1,38 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { test } from "node:test";
+import { startPostgresFixture } from "./helpers/postgres-fixture";
+
+test("staging postgres is private and CI binds loopback only", () => {
+  const staging = readFileSync("../deploy/docker-compose.postgres.yml", "utf8");
+  const ci = readFileSync("../deploy/docker-compose.postgres-ci.yml", "utf8");
+  assert.match(staging, /image:\s*postgres:17-alpine/);
+  assert.doesNotMatch(staging, /^\s+ports:/m);
+  assert.match(ci, /127\.0\.0\.1:\$\{POSTGRES_HOST_PORT:-55432\}:5432/);
+});
+
+test("database roles have no cluster privileges", () => {
+  const fixture = startPostgresFixture();
+  try {
+    assert.equal(
+      fixture.psql(`
+        select rolname || ':' || rolsuper || ':' || rolcreatedb || ':' ||
+               rolcreaterole || ':' || rolbypassrls
+        from pg_roles
+        where rolname in ('schema_owner','identity_runtime','app_runtime',
+          'admin_runtime','migration_runner','backup_reader')
+        order by rolname
+      `),
+      [
+        "admin_runtime:f:f:f:f",
+        "app_runtime:f:f:f:f",
+        "backup_reader:f:f:f:f",
+        "identity_runtime:f:f:f:f",
+        "migration_runner:f:f:f:f",
+        "schema_owner:f:f:f:f",
+      ].join("\n"),
+    );
+  } finally {
+    fixture.stop();
+  }
+});
@@ -0,0 +1,128 @@
+import { execFileSync, spawnSync } from "node:child_process";
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+export type PostgresFixture = {
+  projectName: string;
+  databaseEnvFile: string;
+  hostPort: number;
+  connectionUrl(role: string, password: string): string;
+  psql(sql: string): string;
+  stop(): void;
+};
+
+const databaseEnvironment = `POSTGRES_DB=jyotisha
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres-test-password
+SCHEMA_OWNER_PASSWORD=schema-owner-test-password
+IDENTITY_RUNTIME_PASSWORD=identity-runtime-test-password
+APP_RUNTIME_PASSWORD=app-runtime-test-password
+ADMIN_RUNTIME_PASSWORD=admin-runtime-test-password
+MIGRATION_RUNNER_PASSWORD=migration-runner-test-password
+BACKUP_READER_PASSWORD=backup-reader-test-password
+STAGING_BACKUP_ENCRYPTION_KEY=staging-backup-test-password
+SCHEMA_DATABASE_URL=postgresql://schema_owner:schema-owner-test-password@postgres:5432/jyotisha
+`;
+
+function isPortAvailable(port: number): boolean {
+  return (
+    spawnSync(
+      process.execPath,
+      [
+        "-e",
+        `const server = require("node:net").createServer();
+server.once("error", () => process.exit(1));
+server.listen({ host: "127.0.0.1", port: Number(process.argv[1]) }, () =>
+  server.close(() => process.exit(0)),
+);`,
+        String(port),
+      ],
+      { stdio: "ignore" },
+    ).status === 0
+  );
+}
+
+function findAvailablePort(): number {
+  for (let port = 55432; port <= 55531; port += 1) {
+    if (isPortAvailable(port)) {
+      return port;
+    }
+  }
+
+  throw new Error("no available PostgreSQL test port in 55432..55531");
+}
+
+export function startPostgresFixture(): PostgresFixture {
+  const projectName = `jyotisha-postgres-${process.pid}-${Date.now()}`;
+  const temporaryDirectory = mkdtempSync(join(tmpdir(), "jyotisha-postgres-"));
+  const databaseEnvFile = join(temporaryDirectory, "database.env");
+  const hostPort = findAvailablePort();
+  const composeArguments = [
+    "compose",
+    "--project-name",
+    projectName,
+    "--env-file",
+    databaseEnvFile,
+    "-f",
+    "../deploy/docker-compose.postgres.yml",
+    "-f",
+    "../deploy/docker-compose.postgres-ci.yml",
+  ];
+  const environment = {
+    ...process.env,
+    DATABASE_ENV_FILE: databaseEnvFile,
+    POSTGRES_HOST_PORT: String(hostPort),
+  };
+
+  writeFileSync(databaseEnvFile, databaseEnvironment, { mode: 0o600 });
+  chmodSync(databaseEnvFile, 0o600);
+
+  try {
+    execFileSync(
+      "docker",
+      [...composeArguments, "up", "-d", "--wait", "postgres"],
+      { env: environment, stdio: "inherit" },
+    );
+  } catch (error) {
+    try {
+      execFileSync(
+        "docker",
+        [...composeArguments, "down", "-v", "--remove-orphans"],
+        { env: environment, stdio: "inherit" },
+      );
+    } finally {
+      rmSync(temporaryDirectory, { force: true, recursive: true });
+    }
+    throw error;
+  }
+
+  return {
+    projectName,
+    databaseEnvFile,
+    hostPort,
+    connectionUrl(role, password) {
+      return `postgresql://${role}:${password}@127.0.0.1:${hostPort}/jyotisha`;
+    },
+    psql(sql) {
+      return execFileSync(
+        "docker",
+        [...composeArguments, "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "jyotisha", "-Atc", sql],
+        { encoding: "utf8", env: environment },
+      )
+        .trim()
+        .replace(/(^|:)false(?=:|$)/gm, "$1f");
+    },
+    stop() {
+      try {
+        execFileSync(
+          "docker",
+          [...composeArguments, "down", "-v", "--remove-orphans"],
+          { env: environment, stdio: "inherit" },
+        );
+      } finally {
+        rmSync(temporaryDirectory, { force: true, recursive: true });
+      }
+    },
+  };
+}