diff --git a/.gitea/workflows/backend-quality-gate.yml b/.gitea/workflows/backend-quality-gate.yml index b263f793..1452c832 100644 --- a/.gitea/workflows/backend-quality-gate.yml +++ b/.gitea/workflows/backend-quality-gate.yml @@ -6,6 +6,7 @@ on: - '.gitea/workflows/backend-quality-gate.yml' - '.gitea/workflows/deploy-staging.yml' - '.gitea/workflows/migrate-staging-database.yml' + - '.gitea/workflows/migrate-production-database.yml' - 'deploy/**' - 'frontend/**' - 'jyotish_vedic/**' diff --git a/.gitea/workflows/migrate-production-database.yml b/.gitea/workflows/migrate-production-database.yml new file mode 100644 index 00000000..c9a2be91 --- /dev/null +++ b/.gitea/workflows/migrate-production-database.yml @@ -0,0 +1,363 @@ +name: Migrate Production Database (manual only) + +on: + workflow_dispatch: + inputs: + deploy_sha: + description: Full current production release SHA to migrate + required: true + type: string + recovery_reference: + description: Backup or PITR recovery reference; multiple migration files are not atomic as a set + required: true + type: string + recovery_created_at: + description: Recovery point creation time in UTC, exactly YYYY-MM-DDTHH:MM:SSZ and no more than 24 hours old + required: true + type: string + restore_verified: + description: Confirm that this recovery point has passed a restore verification + required: true + default: false + type: boolean + +permissions: + contents: read + actions: read + +concurrency: + group: production-mutation + cancel-in-progress: false + queue: max + +jobs: + migrate: + runs-on: manman-linux + timeout-minutes: 20 + env: + GITEA_SHA: ${{ gitea.sha }} + GITEA_API_URL: ${{ gitea.api_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + REGISTRY_HOST: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com + IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha + DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }} + DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }} + DEPLOY_USER: ${{ vars.PRODUCTION_USER }} + DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }} + STAGING_URL: ${{ vars.STAGING_URL }} + PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }} + steps: + - name: Validate current production revision and successful gates + id: revision + env: + DEPLOY_SHA: ${{ inputs.deploy_sha }} + RECOVERY_REFERENCE: ${{ inputs.recovery_reference }} + RECOVERY_CREATED_AT: ${{ inputs.recovery_created_at }} + RESTORE_VERIFIED: ${{ inputs.restore_verified }} + run: | + set -euo pipefail + [[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; } + [[ "$STAGING_URL" == "https://staging.jyotisha.chat" ]] || { echo "unexpected staging acceptance URL" >&2; exit 1; } + [[ "$RECOVERY_REFERENCE" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}$ ]] || { + echo "recovery_reference must be 1-200 safe reference characters" >&2 + exit 1 + } + [[ "$RECOVERY_CREATED_AT" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || { + echo "recovery_created_at must be UTC in YYYY-MM-DDTHH:MM:SSZ format" >&2 + exit 1 + } + [[ "$RESTORE_VERIFIED" == "true" ]] || { + echo "restore_verified=true is required for a production schema migration" >&2 + exit 1 + } + python3 - "$RECOVERY_CREATED_AT" <<'PY' + from datetime import datetime, timedelta, timezone + import sys + + try: + created_at = datetime.strptime(sys.argv[1], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + except ValueError as error: + raise SystemExit(f"invalid recovery_created_at: {error}") + age = datetime.now(timezone.utc) - created_at + if age < timedelta(0) or age > timedelta(hours=24): + raise SystemExit("recovery_created_at must be no more than 24 hours old and not in the future") + PY + echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT restore_verified=true" + echo "WARNING: migration files run sequentially and are not atomic as a whole; recovery may be required after a partial migration." >&2 + read_ref_sha() { + local branch="$1" + curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" | + jq -er --arg ref "refs/heads/$branch" ' + select(type == "array" and length == 1) | + .[0] | select(.ref == $ref) | .object.sha | + select(test("^[0-9a-f]{40}$")) + ' + } + staging_head="$(read_ref_sha staging)" + main_head="$(read_ref_sha main)" + [[ "$main_head" == "$DEPLOY_SHA" && "$staging_head" == "$DEPLOY_SHA" ]] || { + echo "production migration requires main and staging to equal deploy_sha" >&2 + exit 1 + } + [[ "$GITEA_SHA" == "$DEPLOY_SHA" ]] || { + echo "dispatch the production migration workflow from the exact main release SHA" >&2 + exit 1 + } + runs="$(curl --fail --silent --show-error \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&branch=staging&event=push&status=success&limit=100")" + selected_run="$(jq -cer --arg sha "$DEPLOY_SHA" ' + [.workflow_runs[] | select( + (.path | split("@")[0] | endswith("backend-quality-gate.yml")) and + .head_sha == $sha and .head_branch == "staging" and + .event == "push" and .conclusion == "success" + )] | sort_by(.id) | reverse | first + ' <<<"$runs")" + gate_run_id="$(jq -er '.id' <<<"$selected_run")" + [[ "$gate_run_id" =~ ^[0-9]+$ ]] || { + echo "no successful exact-SHA staging backend quality gate found" >&2 + exit 1 + } + release_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&event=workflow_dispatch&status=success&limit=100")" + jq -e --arg sha "$DEPLOY_SHA" ' + any(.workflow_runs[]?; + (.path | split("@")[0] | endswith("release-quality-gate.yml")) and + .head_sha == $sha and .event == "workflow_dispatch" and .conclusion == "success" + ) + ' <<<"$release_runs" >/dev/null || { + echo "no successful exact-SHA manual release quality gate found" >&2 + exit 1 + } + observed_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 30 --retry 3 --retry-all-errors \ + "$STAGING_URL/api/health" | jq -er '.deployment.gitCommit | select(test("^[0-9a-f]{40}$"))')" + [[ "$observed_staging_sha" == "$DEPLOY_SHA" ]] || { + echo "public staging has not accepted the requested SHA" >&2 + exit 1 + } + { + echo "sha=$DEPLOY_SHA" + echo "gate_run_id=$gate_run_id" + echo "recovery_reference=$RECOVERY_REFERENCE" + echo "recovery_created_at=$RECOVERY_CREATED_AT" + echo "restore_verified=true" + } >>"$GITHUB_OUTPUT" + + - name: Prepare pinned Node tooling + env: + NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c + NODE_TOOL_IMAGE: node:22-bookworm-slim + run: | + set -euo pipefail + if ! docker image inspect "$NODE_TOOL_SOURCE_IMAGE" >/dev/null 2>&1; then + for attempt in 1 2 3; do + if timeout 180 docker pull "$NODE_TOOL_SOURCE_IMAGE"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "Failed to preload $NODE_TOOL_IMAGE after $attempt attempts" >&2 + exit 1 + fi + sleep $((attempt * 15)) + done + fi + docker tag "$NODE_TOOL_SOURCE_IMAGE" "$NODE_TOOL_IMAGE" + docker image inspect "$NODE_TOOL_IMAGE" >/dev/null + tool_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-node-tools.XXXXXX")" + cat > "$tool_dir/node" <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + workdir="$(pwd -P)" + exec docker run --rm \ + --user "$(id -u):$(id -g)" \ + --volume "$workdir:$workdir" \ + --workdir "$workdir" \ + --env HOME=/tmp \ + node:22-bookworm-slim "${0##*/}" "$@" + EOF + chmod 0755 "$tool_dir/node" + ln -s node "$tool_dir/npm" + test -n "${GITHUB_PATH:-}" + printf '%s\n' "$tool_dir" >> "$GITHUB_PATH" + export PATH="$tool_dir:$PATH" + node --version + npm --version + + - name: Download gate-produced migration manifest + env: + GATE_RUN_ID: ${{ steps.revision.outputs.gate_run_id }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + run: | + set -euo pipefail + artifact_prefix="staging-image-manifest-$DEPLOY_SHA-" + artifacts="$(curl --fail --silent --show-error \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs/$GATE_RUN_ID/artifacts?limit=100")" + selected_artifact="$(jq -cer --arg prefix "$artifact_prefix" ' + [(.artifacts // [])[] + | select(.expired == false and (.name | startswith($prefix))) + | . + {attempt: ((.name | ltrimstr($prefix)) | tonumber?)} + | select(.attempt != null and .attempt >= 1) + ] | sort_by(.attempt, .id) | reverse | first + ' <<<"$artifacts")" + artifact_name="$(jq -er '.name' <<<"$selected_artifact")" + artifact_id="$(jq -er '.id' <<<"$selected_artifact")" + artifact_attempt="${artifact_name#"$artifact_prefix"}" + [[ "$artifact_name" == "$artifact_prefix"* ]] + [[ "$artifact_attempt" =~ ^[1-9][0-9]*$ ]] + [[ "$artifact_id" =~ ^[0-9]+$ ]] + install -d -m 700 artifacts/staging-image + curl --fail --silent --show-error --location \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$artifact_id/zip" \ + --output "${RUNNER_TEMP}/staging-image-manifest.zip" + python3 - "${RUNNER_TEMP}/staging-image-manifest.zip" artifacts/staging-image <<'PY' + import pathlib, stat, sys, zipfile + archive = pathlib.Path(sys.argv[1]) + destination = pathlib.Path(sys.argv[2]) + allowed = {"manifest.env", "controller.tar"} + with zipfile.ZipFile(archive) as bundle: + entries = bundle.infolist() + names = [entry.filename for entry in entries] + if len(names) != len(set(names)) or set(names) != allowed: + raise SystemExit("invalid production migration artifact bundle") + if sum(entry.file_size for entry in entries) > 3 * 1024 * 1024: + raise SystemExit("production migration artifact bundle is too large") + for entry in entries: + path = pathlib.PurePosixPath(entry.filename) + mode = entry.external_attr >> 16 + if path.is_absolute() or ".." in path.parts or path.name != entry.filename: + raise SystemExit("unsafe production migration artifact path") + if mode and not stat.S_ISREG(mode): + raise SystemExit("unsafe production migration artifact type") + target = destination / entry.filename + with bundle.open(entry) as source, target.open("xb") as output: + output.write(source.read()) + PY + [[ -f artifacts/staging-image/manifest.env ]] + [[ -f artifacts/staging-image/controller.tar ]] + + - name: Validate gate-attested controller and digest-pinned migration image + id: image + env: + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + run: | + set -euo pipefail + manifest=artifacts/staging-image/manifest.env + controller_tar=artifacts/staging-image/controller.tar + [[ "$(wc -l < "$manifest" | tr -d ' ')" == 4 ]] + manifest_sha="$(awk -F= '$1 == "git_sha" {print $2}' "$manifest")" + expected_controller_digest="$(awk -F= '$1 == "controller_sha256" {print $2}' "$manifest")" + [[ "$manifest_sha" == "$DEPLOY_SHA" && "$expected_controller_digest" =~ ^[0-9a-f]{64}$ ]] + printf '%s %s\n' "$expected_controller_digest" "$controller_tar" | sha256sum --check --status + python3 - "$controller_tar" <<'PY' + import pathlib, sys, tarfile + archive = pathlib.Path(sys.argv[1]) + required = {"deploy/run-production-migration.sh", "frontend/scripts/staging-image-manifest.mjs"} + with tarfile.open(archive, "r:") as bundle: + members = bundle.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)) or not required.issubset(names): + raise SystemExit("invalid production migration controller bundle") + if sum(member.size for member in members) > 2 * 1024 * 1024: + raise SystemExit("production migration controller bundle is too large") + for member in members: + path = pathlib.PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()): + raise SystemExit("unsafe production migration controller bundle") + PY + install -d -m 700 artifacts/staging-image/extracted + tar -xf "$controller_tar" -C artifacts/staging-image/extracted + node artifacts/staging-image/extracted/frontend/scripts/staging-image-manifest.mjs \ + "$manifest" "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT" + + - name: Apply production schema migration under pinned SSH identity + env: + SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + WEB_IMAGE: ${{ steps.image.outputs.web_image }} + RECOVERY_REFERENCE: ${{ steps.revision.outputs.recovery_reference }} + RECOVERY_CREATED_AT: ${{ steps.revision.outputs.recovery_created_at }} + RESTORE_VERIFIED: ${{ steps.revision.outputs.restore_verified }} + run: | + set -euo pipefail + [[ "$DEPLOY_HOST" == "118.194.235.34" ]] + [[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 )) + [[ "$DEPLOY_USER" == "deploy" ]] + [[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]] + test -n "$PRODUCTION_KNOWN_HOSTS" + ssh_root="${RUNNER_TEMP}/production-migration-ssh" + key_path="$ssh_root/id_ed25519" + known_hosts_path="$ssh_root/known_hosts" + incoming="" + install -m 700 -d "$ssh_root" + test -n "$SSH_PRIVATE_KEY_BASE64" + printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode > "$key_path" + printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' > "$known_hosts_path" + chmod 600 "$key_path" "$known_hosts_path" + ssh-keygen -y -f "$key_path" >/dev/null + ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path") + remote="$DEPLOY_USER@$DEPLOY_HOST" + require_current_release_heads() { + current_staging="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" | + jq -er 'select(type == "array" and length == 1) | .[0] | + select(.ref == "refs/heads/staging") | .object.sha | + select(test("^[0-9a-f]{40}$"))')" + current_main="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/main" | + jq -er 'select(type == "array" and length == 1) | .[0] | + select(.ref == "refs/heads/main") | .object.sha | + select(test("^[0-9a-f]{40}$"))')" + [[ "$current_main" == "$DEPLOY_SHA" && "$current_staging" == "$DEPLOY_SHA" ]] || { + echo "main or staging advanced during production migration; refusing stale mutation" >&2 + exit 1 + } + } + cleanup() { + if [[ -n "$incoming" ]]; then + ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; sudo -n rm -rf -- '$incoming'" >/dev/null 2>&1 || true + fi + rm -rf -- "$ssh_root" + } + trap cleanup EXIT + incoming="$(ssh "${ssh_options[@]}" "$remote" "mktemp -d /tmp/jyotisha-production-migration.XXXXXXXXXX")" + [[ "$incoming" == /tmp/jyotisha-production-migration.* ]] + ssh "${ssh_options[@]}" "$remote" "install -d -m 700 '$incoming/.docker'" + scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" artifacts/staging-image/controller.tar "$remote:$incoming/controller.tar" + ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/controller.tar' -C '$incoming' && rm -f -- '$incoming/controller.tar'" + previous_sha="$(ssh "${ssh_options[@]}" "$remote" "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(sudo -n docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-production' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then sudo -n docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; else printf not-deployed; fi; fi")" + [[ "$previous_sha" == not-deployed || "$previous_sha" =~ ^[0-9a-f]{40}$ ]] || exit 1 + forward_verified=false + if [[ "$previous_sha" != not-deployed && "$previous_sha" != "$DEPLOY_SHA" ]]; then + comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")" + jq -e --arg base "$previous_sha" --arg head "$DEPLOY_SHA" ' + (.commits // []) as $commits | + def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha]; + def reaches($sha; $seen): + if $sha == $base then true + elif ($seen | index($sha)) != null then false + else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end; + (.total_commits | type) == "number" and + .total_commits == ($commits | length) and ($commits | length) > 0 and + ([$commits[].sha] | length == (unique | length)) and reaches($head; []) + ' <<<"$comparison" >/dev/null || { echo "production migration rollback or divergence refused" >&2; exit 1; } + forward_verified=true + fi + require_current_release_heads + printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin" + ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' FORWARD_REVISION_VERIFIED='$forward_verified' RECOVERY_REFERENCE='$RECOVERY_REFERENCE' RECOVERY_CREATED_AT='$RECOVERY_CREATED_AT' RESTORE_VERIFIED='$RESTORE_VERIFIED' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' bash '$incoming/deploy/run-production-migration.sh'" + require_current_release_heads + + - name: Operator action + run: echo 'Schema migration complete. Production ETL and application deployment remain separate manual operations.' diff --git a/deploy/postgres/001-bootstrap-roles.sh b/deploy/postgres/001-bootstrap-roles.sh index e55cb784..e4e82344 100755 --- a/deploy/postgres/001-bootstrap-roles.sh +++ b/deploy/postgres/001-bootstrap-roles.sh @@ -81,6 +81,8 @@ SELECT format( SELECT 1 FROM pg_roles WHERE rolname = 'backup_reader' ) \gexec +GRANT schema_owner TO migration_runner; + SELECT format( 'GRANT CONNECT, CREATE ON DATABASE %I TO schema_owner', :'database_name' diff --git a/deploy/run-production-migration.sh b/deploy/run-production-migration.sh new file mode 100755 index 00000000..6d1a66e1 --- /dev/null +++ b/deploy/run-production-migration.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +required=( + INCOMING_PATH DEPLOY_PATH WEB_IMAGE DEPLOY_SHA EXPECTED_PREVIOUS_SHA + RECOVERY_REFERENCE RECOVERY_CREATED_AT RESTORE_VERIFIED DOCKER_CONFIG +) +case "${DOCKER_BIN:-docker}" in + docker) docker_command=(docker) ;; + "sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;; + *) echo "unsafe production Docker command" >&2; exit 1 ;; +esac +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "required production migration input is missing: $key" >&2 + exit 1 + fi +done + +[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "unsafe production migration revision" >&2 + exit 1 +} +image_pattern='^crpi-d1feco6itet73spp\.cn-hongkong\.personal\.cr\.aliyuncs\.com/copse/jyotisha@sha256:[0-9a-f]{64}$' +[[ "$WEB_IMAGE" =~ $image_pattern ]] || { + echo "unsafe production migration image" >&2 + exit 1 +} +case "$INCOMING_PATH" in + /tmp/jyotisha-production-migration.*) ;; + *) echo "unsafe incoming production migration path" >&2; exit 1 ;; +esac +[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || { + echo "unsafe production deployment path" >&2 + exit 1 +} +[ "$DOCKER_CONFIG" = "$INCOMING_PATH/.docker" ] || { + echo "unsafe production Docker configuration path" >&2 + exit 1 +} +[[ "$RECOVERY_REFERENCE" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}$ ]] || { + echo "unsafe production recovery reference" >&2 + exit 1 +} +[[ "$RECOVERY_CREATED_AT" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || { + echo "unsafe production recovery creation time" >&2 + exit 1 +} +[ "$RESTORE_VERIFIED" = "true" ] || { + echo "production recovery point must have restore_verified=true" >&2 + exit 1 +} +recovery_created_epoch="$(date -u -d "$RECOVERY_CREATED_AT" +%s 2>/dev/null)" || { + echo "invalid production recovery creation time" >&2 + exit 1 +} +recovery_now_epoch="$(date -u +%s)" +recovery_age_seconds=$((recovery_now_epoch - recovery_created_epoch)) +(( recovery_age_seconds >= 0 && recovery_age_seconds <= 24 * 60 * 60 )) || { + echo "production recovery point must be no more than 24 hours old and not in the future" >&2 + exit 1 +} +echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT restore_verified=true" +echo "WARNING: migration files run sequentially and are not atomic as a whole; recovery may be required after a partial migration." >&2 + +state_directory="$DEPLOY_PATH/.state" +install -d -m 700 "$state_directory" +exec 9>"$state_directory/mutation.lock" +flock -n 9 || { + echo "another production mutation holds the host lock" >&2 + exit 75 +} + +current_sha="not-deployed" +if [ -f "$state_directory/deployed-revision" ]; then + current_sha="$(<"$state_directory/deployed-revision")" +else + existing_web="$("${docker_command[@]}" ps -aq \ + --filter 'label=com.docker.compose.project=jyotisha-production' \ + --filter 'label=com.docker.compose.service=web' | head -n 1)" + if [ -n "$existing_web" ]; then + discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ + "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)" + if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi + fi +fi +if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid deployed production revision state" >&2 + exit 1 +fi +[ "$current_sha" = "$EXPECTED_PREVIOUS_SHA" ] || { + echo "production revision changed while this migration was waiting" >&2 + exit 1 +} +[ "$current_sha" = "not-deployed" ] || + [ "$current_sha" = "$DEPLOY_SHA" ] || + [ "${FORWARD_REVISION_VERIFIED:-false}" = "true" ] || { + echo "forward production revision was not verified" >&2 + exit 1 + } + +bash "$INCOMING_PATH/deploy/sync-production-tree.sh" \ + "$INCOMING_PATH" "$DEPLOY_PATH" + +cd "$DEPLOY_PATH" +EXPECTED_PRODUCTION_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")" +[[ "$EXPECTED_PRODUCTION_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || { + echo "production deployment owner is invalid" >&2 + exit 1 +} +export EXPECTED_PRODUCTION_ENV_OWNER_UID +bash deploy/validate-production-env.sh .env.production +bash deploy/validate-production-database-env.sh .env.production.database + +export DATABASE_ENV_FILE='../.env.production.database' +compose=("${docker_command[@]}" compose -p jyotisha-production -f deploy/docker-compose.postgres.yml) +"${docker_command[@]}" pull "$WEB_IMAGE" +"${compose[@]}" config --quiet +"${compose[@]}" up -d --no-build --pull never --wait postgres + +membership="$("${compose[@]}" exec -T postgres psql -v ON_ERROR_STOP=1 -U postgres -d jyotisha -Atc \ + "select pg_has_role('migration_runner', 'schema_owner', 'member')")" +[ "$membership" = "t" ] || { + echo "migration_runner must be allowed to SET ROLE schema_owner before production migration" >&2 + exit 1 +} + +environment_value() { + local key="$1" + local value + value="$(sed -n -E "s/^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=[[:space:]]*(.*)$/\\2/p" .env.production.database)" + case "$value" in + \"*\") value="${value:1:${#value}-2}" ;; + \'*\') value="${value:1:${#value}-2}" ;; + esac + printf '%s' "$value" +} + +percent_encode() { + local value="$1" + local encoded="" + local character hex index + LC_ALL=C + for ((index = 0; index < ${#value}; index += 1)); do + character="${value:index:1}" + case "$character" in + [a-zA-Z0-9.~_-]) encoded+="$character" ;; + *) + printf -v hex '%%%02X' "'$character" + encoded+="$hex" + ;; + esac + done + printf '%s' "$encoded" +} + +migration_runner_password="$(environment_value MIGRATION_RUNNER_PASSWORD)" +[ -n "$migration_runner_password" ] || { + echo "production migration runner password is missing" >&2 + exit 1 +} +encoded_migration_runner_password="$(percent_encode "$migration_runner_password")" +unset migration_runner_password +migration_runner_database_url="postgresql://migration_runner:${encoded_migration_runner_password}@postgres:5432/jyotisha?options=-c%20role%3Dschema_owner" +unset encoded_migration_runner_password + +migration_environment="$(mktemp "$state_directory/production-migration-env.XXXXXXXXXX")" +cleanup_migration_environment() { + rm -f -- "$migration_environment" +} +trap cleanup_migration_environment EXIT +chmod 600 "$migration_environment" +printf 'SCHEMA_DATABASE_URL=%s\n' "$migration_runner_database_url" >"$migration_environment" +unset migration_runner_database_url + +set +e +DATABASE_ENV_FILE="$migration_environment" \ + "${compose[@]}" --profile migration-check run --rm migration-checker +precheck_status=$? +set -e +if [ "$precheck_status" -ne 0 ] && [ "$precheck_status" -ne 3 ]; then + echo "production migration precheck failed safely" >&2 + exit "$precheck_status" +fi + +DATABASE_ENV_FILE="$migration_environment" \ + "${compose[@]}" --profile migration run --rm migrator + +set +e +DATABASE_ENV_FILE="$migration_environment" \ + "${compose[@]}" --profile migration-check run --rm migration-checker +postcheck_status=$? +set -e +if [ "$postcheck_status" -ne 0 ]; then + echo "production migration postcheck did not converge" >&2 + exit "$postcheck_status" +fi + +echo "production schema migration verified for $DEPLOY_SHA" diff --git a/docs/operations/production-server-migration-2026-08.md b/docs/operations/production-server-migration-2026-08.md index 64a78cc4..1a67912d 100644 --- a/docs/operations/production-server-migration-2026-08.md +++ b/docs/operations/production-server-migration-2026-08.md @@ -13,7 +13,7 @@ This is not a volume copy. A full Supabase dump must not be restored over the ta ## Release invariants -The production workflow is manual-only and accepts a full lowercase 40-character `deploy_sha`. A normal deployment proceeds only when all of the following identify that exact SHA: +The production deployment and schema-migration workflows are manual-only and accept a full lowercase 40-character `deploy_sha`. A normal production mutation proceeds only when all of the following identify that exact SHA: 1. current `main`; 2. current `staging`; @@ -21,7 +21,7 @@ The production workflow is manual-only and accepts a full lowercase 40-character 4. a successful manually triggered `Jyotish Release Quality Gate`; 5. the public staging `/api/health` deployment identity. -The workflow consumes the exact API and Web image digests recorded by the staging gate. It does not build on the 2-core/4-GB production host, import user data, run schema migrations, or change DNS. +The deployment workflow consumes the exact API and Web image digests recorded by the staging gate. It does not build on the 2-core/4-GB production host, import user data, run schema migrations, or change DNS. The separate `Migrate Production Database` workflow uses the same gate-attested Web image only to run the schema checker/migrator/checker sequence; it does not run ETL, deploy the application, or change DNS. Both workflows share the `production-mutation` lock. ## Required Gitea configuration @@ -80,7 +80,13 @@ Use distinct production credentials for PostgreSQL roles, Better Auth, Resend, b ## Database migration engineering gate -Before production cutover, implement and review `frontend/scripts/migrate-supabase-production.mjs` with `--preflight`, `--apply`, and `--verify` modes. Until that tool and its fixtures pass, production data cutover is blocked. +Before importing data, dispatch Gitea Actions → `Migrate Production Database` for the exact accepted release SHA. The workflow requires `main == staging == deploy_sha`, the same successful staging backend gate, the same manual release gate, and the public staging `/api/health` identity for that SHA. It also requires a non-sensitive recovery reference, its exact UTC creation time, and `restore_verified=true`; the recovery point must be no more than 24 hours old and must already have passed a restore verification. It verifies the current production revision, obtains the gate-attested immutable Web image, runs the schema checker, applies only pending application schema migrations, and requires the checker to converge afterward. + +Schema migration files are committed sequentially and are not one atomic transaction as a set. If a later file or post-check fails, earlier files may remain applied; stop, preserve evidence, and restore from the attested recovery point when repair-in-place is not explicitly reviewed. Do not assume a failed workflow means the database is unchanged. + +The production database must already grant `migration_runner` membership in `schema_owner`; `deploy/postgres/001-bootstrap-roles.sh` grants only that migration role the ability to `SET ROLE schema_owner`. Identity, app, service, admin, and backup runtime roles must not receive this membership. The workflow checks the membership and refuses to add it itself. + +The reviewed data-transfer entry point is `frontend/scripts/migrate-supabase-production.mjs`. It has separate `--preflight`, `--apply`, and `--verify` modes; the application deployment workflow never runs it automatically. Production cutover remains blocked until the exact production snapshot has completed an isolated rehearsal and final verification. The tool must: @@ -96,6 +102,24 @@ The tool must: - emit only counts, state aggregates, and normalized SHA-256 manifests—not email addresses, birth data, tokens, or connection strings; - run all target writes in a transaction and roll back on failure. +The operator supplies these values only on the trusted migration host; do not store the database URLs or encryption keys in Gitea: + +- `SUPABASE_SOURCE_DATABASE_URL`: the consistent read-only Supabase snapshot/source URL; +- `PRODUCTION_TARGET_DATABASE_URL`: the PostgreSQL 17 target URL using the migration role; +- `PRODUCTION_OWNER_USER_ID`: the UUID of the designated active source administrator; +- `PRODUCTION_CIPHERTEXT_MODE=preserve|exclude`; preserve additionally requires `PRODUCTION_CIPHERTEXT_KEYS_CONFIRMED=true`. + +Run each phase separately and retain its redacted JSON manifest: + +```bash +cd frontend +node scripts/migrate-supabase-production.mjs --preflight +node scripts/migrate-supabase-production.mjs --apply +node scripts/migrate-supabase-production.mjs --verify +``` + +`--apply` is intentionally one-shot: it refuses a populated target. If apply fails, discard or restore the isolated target, correct the cause, and rerun from an empty migrated schema rather than improvising a partial resume. + Do not import platform schemas, source roles/grants, Supabase migration ledgers, sessions, refresh tokens, or provider tokens. Do not use a full-database `pg_restore` against the target. Because migrated users have no portable password/session, all sessions are invalidated and users sign in again through email OTP. Administrators re-enrol MFA. @@ -104,7 +128,7 @@ Because migrated users have no portable password/session, all sessions are inval Complete at least one isolated full-data rehearsal before scheduling the final window: -1. Apply all target schema migrations to an empty rehearsal database. +1. Apply all target schema migrations to an empty rehearsal database using the same schema migrator path as `Migrate Production Database`. 2. Run migration preflight, apply, post-import reconciliation, and verify. 3. Verify source/target row counts, primary-key set hashes, normalized row hashes, credit totals, payment state totals, subscriptions, reports, consultations, and rectification records. 4. Verify one Owner exists, every active admin has a target role, and database roles remain isolated. @@ -135,6 +159,8 @@ Both `jyotisha.chat` and `admin.jyotisha.chat` are required. The application rej - Confirm the exact release SHA is deployed and accepted on staging. - Run the manual release quality gate for that SHA. - Confirm final backup capacity, restore rehearsal, SMTP/OTP delivery, and rollback contacts. +- Create and restore-verify a production recovery point no more than 24 hours before the schema migration; record its non-sensitive reference and UTC creation time. +- Dispatch `Migrate Production Database` for the accepted SHA with that recovery attestation and confirm its post-check reports no pending schema migrations. - Record pending payment orders and long-running jobs; choose an explicit disposition for each. - Dispatch `Deploy production` with `verification_mode=internal` only after target schema/data preparation. This verifies the new host without depending on public DNS. @@ -189,7 +215,9 @@ Normal release: 1. Merge the reviewed `staging` release into `main` so both heads are the same SHA. 2. Confirm the staging push gate, public staging SHA, and manual release gate all succeeded for that SHA. -3. Open Gitea Actions → `Deploy production`. -4. Enter the exact 40-character SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase. +3. Open Gitea Actions → `Migrate Production Database`; enter the exact 40-character SHA, the no-more-than-24-hour-old recovery reference and UTC creation time, and confirm `restore_verified=true`. Wait for the post-migration checker to converge. Do not use this workflow for Supabase ETL. +4. Run the trusted-host ETL phases and retain the redacted reconciliation manifests. +5. Open Gitea Actions → `Deploy production`. +6. Enter the same exact SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase. Application rollback accepts only an explicitly authorized, previously gate-attested SHA in reviewed `main` history. Database migrations and imported data are not rolled back by the application workflow. diff --git a/docs/research/pre_work_error_ledger.md b/docs/research/pre_work_error_ledger.md index 20106b5d..6a0fa122 100644 --- a/docs/research/pre_work_error_ledger.md +++ b/docs/research/pre_work_error_ledger.md @@ -251,3 +251,9 @@ Prevention: keep evidence-request, life-event, private-candidate, public-recap, After accumulated historical evidence produced a very narrow winning segment, that segment could contain fewer than two linked samples or discriminating divisional themes. Packet construction treated this valid “not enough distinction yet” state as a dependency failure, so a later answer returned 503 even though scoring and the astrology service were healthy. Prevention: classify insufficient candidate-range discrimination explicitly; when a newly narrowed segment cannot support the technical evidence contract, retain the prior candidate range, preserve scored evidence, clear the unconfirmed result, and continue conversational collection. + +## ERR-102 | Gitea returned HTTP 502 during final remote synchronization check | active 2026-08-09 + +Two final `git fetch origin --prune` attempts against the configured primary Gitea remote failed before ref exchange with HTTP `502`. The last locally verified refs remain available, but this run cannot prove that they are still current and must not claim a completed remote synchronization or push. + +Prevention: retry fetch and `git ls-remote` before any push or release action, compare the full `main`, `staging`, and migration-branch SHAs, and stop if Gitea remains unavailable. Do not substitute cached refs, the GitHub mirror, or a successful local commit for current Gitea synchronization evidence. diff --git a/frontend/scripts/migrate-supabase-production.mjs b/frontend/scripts/migrate-supabase-production.mjs new file mode 100644 index 00000000..01f6ce73 --- /dev/null +++ b/frontend/scripts/migrate-supabase-production.mjs @@ -0,0 +1,1029 @@ +import { createHash } from "node:crypto"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Pool } from "pg"; + +import { runMigrations } from "./db-migrate.mjs"; +import { normalizeSupabaseUsers } from "./import-supabase-auth-users.mjs"; + +export class SafeProductionMigrationError extends Error {} + +const ALLOWED_TARGET_ROWS = new Set([ + "public.admin_permissions", + "public.admin_role_permissions", + "public.admin_roles", + "public.billing_products", + "public.feature_flags", + "public.notification_templates", + "public.product_entitlements", +]); + +const SEED_TABLES = new Map([ + ["admin_permissions", ["permission_key"]], + ["admin_roles", ["code"]], + ["billing_products", ["code", "version"]], + ["feature_flags", ["flag_key", "version"]], + ["notification_templates", ["template_key", "channel", "version"]], +]); + +const SEED_RELATIONS = new Map([ + ["admin_role_permissions", ["role_id", "permission_id"]], + ["product_entitlements", ["product_id", "feature_key"]], +]); + +const CIPHERTEXT_COLUMNS = new Map([ + ["epay_settings", new Set(["encrypted_key"])], + ["model_providers", new Set(["encrypted_api_key"])], +]); + +const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +function quoted(identifier) { + return `"${String(identifier).replaceAll('"', '""')}"`; +} + +function qualified(schema, table) { + return `${quoted(schema)}.${quoted(table)}`; +} + +function requiredUrl(env, name) { + const value = env[name]?.trim(); + if (!value) throw new SafeProductionMigrationError(`${name} is required`); + if (!/^postgres(?:ql)?:\/\//.test(value)) { + throw new SafeProductionMigrationError(`${name} must be a PostgreSQL URL`); + } + return value; +} + +export function readConfiguration(env) { + const sourceUrl = requiredUrl(env, "SUPABASE_SOURCE_DATABASE_URL"); + const targetUrl = requiredUrl(env, "PRODUCTION_TARGET_DATABASE_URL"); + if (sourceUrl === targetUrl) { + throw new SafeProductionMigrationError("source and target databases must differ"); + } + const ownerUserId = env.PRODUCTION_OWNER_USER_ID?.trim().toLowerCase(); + if (!ownerUserId || !uuidPattern.test(ownerUserId)) { + throw new SafeProductionMigrationError("PRODUCTION_OWNER_USER_ID must be a UUID"); + } + const ciphertextMode = env.PRODUCTION_CIPHERTEXT_MODE?.trim(); + if (!new Set(["preserve", "exclude"]).has(ciphertextMode)) { + throw new SafeProductionMigrationError( + "PRODUCTION_CIPHERTEXT_MODE must be preserve or exclude", + ); + } + if ( + ciphertextMode === "preserve" && + env.PRODUCTION_CIPHERTEXT_KEYS_CONFIRMED !== "true" + ) { + throw new SafeProductionMigrationError( + "preserving ciphertext requires confirmed production encryption keys", + ); + } + return { sourceUrl, targetUrl, ownerUserId, ciphertextMode }; +} + +export function parseMode(arguments_) { + const modes = arguments_.filter((argument) => + ["--preflight", "--apply", "--verify"].includes(argument), + ); + if (modes.length !== 1 || modes.length !== arguments_.length) { + throw new SafeProductionMigrationError( + "choose exactly one of --preflight, --apply, or --verify", + ); + } + return modes[0].slice(2); +} + +function asArray(value) { + if (Array.isArray(value)) return value; + if (typeof value !== "string") return []; + return value.replace(/^\{/, "").replace(/\}$/, "").split(",").filter(Boolean); +} + +export async function readSchema(client, schema) { + const columnsResult = await client.query( + ` + select c.table_name, c.column_name, c.is_nullable = 'YES' as nullable, + c.column_default, c.is_generated <> 'NEVER' as generated, + c.is_identity = 'YES' as identity, c.identity_generation, + c.ordinal_position + from information_schema.columns c + join information_schema.tables t + on t.table_schema = c.table_schema and t.table_name = c.table_name + where c.table_schema = $1 and t.table_type = 'BASE TABLE' + order by c.table_name, c.ordinal_position + `, + [schema], + ); + const primaryKeysResult = await client.query( + ` + select kcu.table_name, + array_agg(kcu.column_name order by kcu.ordinal_position) as columns + from information_schema.table_constraints tc + join information_schema.key_column_usage kcu + on kcu.constraint_schema = tc.constraint_schema + and kcu.constraint_name = tc.constraint_name + and kcu.table_name = tc.table_name + where tc.table_schema = $1 and tc.constraint_type = 'PRIMARY KEY' + group by kcu.table_name + `, + [schema], + ); + const foreignKeysResult = await client.query( + ` + select n.nspname as schema_name, r.relname as table_name, + rn.nspname as ref_schema, rr.relname as ref_table, + array( + select a.attname + from unnest(c.conkey) with ordinality as key(attnum, ord) + join pg_attribute a on a.attrelid = c.conrelid and a.attnum = key.attnum + order by key.ord + ) as columns, + array( + select a.attname + from unnest(c.confkey) with ordinality as key(attnum, ord) + join pg_attribute a on a.attrelid = c.confrelid and a.attnum = key.attnum + order by key.ord + ) as ref_columns + from pg_constraint c + join pg_class r on r.oid = c.conrelid + join pg_namespace n on n.oid = r.relnamespace + join pg_class rr on rr.oid = c.confrelid + join pg_namespace rn on rn.oid = rr.relnamespace + where c.contype = 'f' and n.nspname = $1 + `, + [schema], + ); + + const tables = new Map(); + for (const row of columnsResult.rows) { + if (!tables.has(row.table_name)) { + tables.set(row.table_name, { columns: [], primaryKey: [], foreignKeys: [] }); + } + tables.get(row.table_name).columns.push({ + name: row.column_name, + nullable: row.nullable, + defaultValue: row.column_default, + generated: row.generated, + identity: row.identity, + identityGeneration: row.identity_generation, + }); + } + for (const row of primaryKeysResult.rows) { + if (tables.has(row.table_name)) tables.get(row.table_name).primaryKey = asArray(row.columns); + } + for (const row of foreignKeysResult.rows) { + if (!tables.has(row.table_name)) continue; + tables.get(row.table_name).foreignKeys.push({ + columns: asArray(row.columns), + refSchema: row.ref_schema, + refTable: row.ref_table, + refColumns: asArray(row.ref_columns), + }); + } + return tables; +} + +function columnMap(table) { + return new Map(table.columns.map((column) => [column.name, column])); +} + +function commonColumns(sourceTable, targetTable) { + const sourceColumns = new Set(sourceTable.columns.map((column) => column.name)); + return targetTable.columns + .filter((column) => !column.generated && sourceColumns.has(column.name)) + .map((column) => column.name); +} + +function assertCompatibleTable(tableName, sourceTable, targetTable) { + if (targetTable.primaryKey.length === 0) { + throw new SafeProductionMigrationError(`target table has no primary key: ${tableName}`); + } + const sourceColumns = new Set(sourceTable.columns.map((column) => column.name)); + const targetColumns = new Set(targetTable.columns.map((column) => column.name)); + if (sourceTable.columns.some((column) => !targetColumns.has(column.name))) { + throw new SafeProductionMigrationError(`target schema is missing a source column: ${tableName}`); + } + for (const column of targetTable.columns) { + if ( + !column.generated && + !column.nullable && + column.defaultValue === null && + !sourceColumns.has(column.name) + ) { + throw new SafeProductionMigrationError(`source schema is missing a required target column: ${tableName}`); + } + } + for (const primaryKeyColumn of targetTable.primaryKey) { + if (!sourceColumns.has(primaryKeyColumn)) { + throw new SafeProductionMigrationError(`source schema is missing a target primary key: ${tableName}`); + } + } +} + +function strictDependencies(tableName, table, selectedTables) { + const columns = columnMap(table); + return table.foreignKeys + .filter( + (foreignKey) => + foreignKey.refSchema === "public" && + foreignKey.refTable !== tableName && + selectedTables.has(foreignKey.refTable) && + foreignKey.columns.every((name) => columns.get(name)?.nullable === false), + ) + .map((foreignKey) => foreignKey.refTable); +} + +export function transferPlan(sourceTables, targetTables) { + const selected = new Set( + [...sourceTables.keys()].filter((table) => targetTables.has(table)), + ); + for (const table of selected) { + const targetTable = targetTables.get(table); + assertCompatibleTable(table, sourceTables.get(table), targetTable); + const columns = columnMap(targetTable); + if (targetTable.foreignKeys.some( + (foreignKey) => foreignKey.refSchema === "public" && + foreignKey.refTable === table && + foreignKey.columns.every((name) => columns.get(name)?.nullable === false), + )) { + throw new SafeProductionMigrationError("non-nullable self reference blocks migration"); + } + } + + const general = new Set( + [...selected].filter( + (table) => !SEED_TABLES.has(table) && !SEED_RELATIONS.has(table), + ), + ); + const remaining = new Set(general); + const ordered = []; + while (remaining.size > 0) { + const ready = [...remaining] + .filter((table) => + strictDependencies(table, targetTables.get(table), general).every( + (dependency) => !remaining.has(dependency), + ), + ) + .sort(); + if (ready.length === 0) { + throw new SafeProductionMigrationError("non-nullable foreign-key cycle blocks migration"); + } + for (const table of ready) { + remaining.delete(table); + ordered.push(table); + } + } + return { + selected, + ordered, + seedTables: [...SEED_TABLES.keys()].filter((table) => selected.has(table)), + seedRelations: [...SEED_RELATIONS.keys()].filter((table) => selected.has(table)), + }; +} + +function normalizeValue(value) { + if (value instanceof Date) return value.toISOString(); + if (Buffer.isBuffer(value)) return value.toString("base64"); + if (Array.isArray(value)) return value.map(normalizeValue); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value).sort().map((key) => [key, normalizeValue(value[key])]), + ); + } + return value; +} + +function canonicalRow(row, columns) { + return Object.fromEntries(columns.map((column) => [column, normalizeValue(row[column])])); +} + +export function rowsSha256(rows, columns) { + const hash = createHash("sha256"); + const values = rows.map((row) => JSON.stringify(canonicalRow(row, columns))).sort(); + for (const value of values) hash.update(value).update("\n"); + return hash.digest("hex"); +} + +async function readRows(client, schema, table, columns, orderColumns = []) { + if (columns.length === 0) return []; + const order = orderColumns.length + ? ` order by ${orderColumns.map(quoted).join(", ")}` + : ""; + return ( + await client.query( + `select ${columns.map(quoted).join(", ")} from ${qualified(schema, table)}${order}`, + ) + ).rows; +} + +async function countRows(client, schema, table) { + const result = await client.query(`select count(*)::bigint as count from ${qualified(schema, table)}`); + return Number(result.rows[0].count); +} + +function bannedState(value, now = new Date()) { + if (value === null || value === undefined || value === "") { + return { banned: false, banExpires: null }; + } + if (String(value).toLowerCase() === "infinity") { + return { banned: true, banExpires: null }; + } + const date = value instanceof Date ? value : new Date(value); + if (!Number.isFinite(date.getTime())) { + throw new SafeProductionMigrationError("source contains an invalid banned_until value"); + } + return date > now + ? { banned: true, banExpires: date } + : { banned: false, banExpires: null }; +} + +export function normalizeAuthUsers(rows, now = new Date(), activeAdminUserIds = new Set()) { + let portableUsers; + try { + portableUsers = normalizeSupabaseUsers(rows); + } catch { + throw new SafeProductionMigrationError("source contains invalid or duplicate auth identities"); + } + return portableUsers.map((user, index) => { + const { banned, banExpires } = bannedState(rows[index].banned_until, now); + return { + id: user.id, + name: user.name, + email: user.email, + email_verified: user.emailVerified, + email_verified_at: user.emailVerifiedAt, + image: user.image, + role: activeAdminUserIds.has(user.id) ? "admin" : "user", + banned, + ban_reason: banned ? "migrated blocked-user state" : null, + ban_expires: banExpires, + created_at: user.createdAt, + updated_at: user.updatedAt, + two_factor_enabled: false, + }; + }); +} + +export function normalizeAuthUser(row, now = new Date()) { + return normalizeAuthUsers([row], now)[0]; +} + +async function readSourceUsers(source, sourceAuthSchema, activeAdminUserIds) { + const table = sourceAuthSchema.get("users"); + if (!table) throw new SafeProductionMigrationError("source auth.users is missing"); + const available = new Set(table.columns.map((column) => column.name)); + const required = [ + "id", + "email", + "raw_user_meta_data", + "email_confirmed_at", + "created_at", + "updated_at", + ]; + if (required.some((column) => !available.has(column))) { + throw new SafeProductionMigrationError("source auth.users is missing portable identity columns"); + } + const columns = [...required, ...(available.has("banned_until") ? ["banned_until"] : [])]; + return normalizeAuthUsers( + await readRows(source, "auth", "users", columns, ["id"]), + new Date(), + activeAdminUserIds, + ); +} + +async function readActiveAdminUserIds(source, sourceTables) { + if (!sourceTables.has("admin_users")) { + throw new SafeProductionMigrationError("source public.admin_users is missing"); + } + const result = await source.query( + "select user_id from public.admin_users where revoked_at is null order by user_id", + ); + return new Set(result.rows.map((row) => String(row.user_id).toLowerCase())); +} + +export function assertActiveAdminUsers(users, activeAdminUserIds, ownerUserId) { + if (!activeAdminUserIds.has(ownerUserId)) { + throw new SafeProductionMigrationError("the designated Owner is not an active source administrator"); + } + const usersById = new Map(users.map((user) => [user.id, user])); + const owner = usersById.get(ownerUserId); + if (!owner) throw new SafeProductionMigrationError("the designated Owner is absent from source auth users"); + if (owner.banned) throw new SafeProductionMigrationError("the designated Owner is blocked"); + for (const userId of activeAdminUserIds) { + const user = usersById.get(userId); + if (!user) throw new SafeProductionMigrationError("an active source administrator is absent from auth users"); + if (user.banned) throw new SafeProductionMigrationError("an active source administrator is blocked"); + } +} + +export async function assertTargetEmpty(target, targetTables) { + if (await countRows(target, "identity", "users")) { + throw new SafeProductionMigrationError("target identity database is not empty"); + } + if (await countRows(target, "auth", "users")) { + throw new SafeProductionMigrationError("target auth compatibility table is not empty"); + } + for (const table of targetTables.keys()) { + if (ALLOWED_TARGET_ROWS.has(`public.${table}`)) continue; + if (await countRows(target, "public", table)) { + throw new SafeProductionMigrationError("target business database is not empty"); + } + } +} + +async function assertNoUnmappedSourceTables(source, sourceTables, targetTables) { + for (const table of sourceTables.keys()) { + if (targetTables.has(table)) continue; + if (await countRows(source, "public", table)) { + throw new SafeProductionMigrationError("source contains an unsupported non-empty public table"); + } + } +} + +async function assertActiveAdminRoles(source, sourceTables, ownerUserId) { + if (!sourceTables.has("admin_user_roles") || !sourceTables.has("admin_roles")) { + const result = await source.query( + `select count(*)::bigint as count from public.admin_users where revoked_at is null and user_id <> $1`, + [ownerUserId], + ); + if (Number(result.rows[0].count) > 0) { + throw new SafeProductionMigrationError("an active source administrator has no canonical target role"); + } + return; + } + const result = await source.query( + ` + select count(*)::bigint as count + from public.admin_users au + where au.revoked_at is null and au.user_id <> $1 + and not exists ( + select 1 + from public.admin_user_roles aur + join public.admin_roles ar on ar.id = aur.role_id + where aur.admin_user_id = au.user_id + and ar.code in ('owner','model_admin','billing_admin','operations','support','auditor') + ) + `, + [ownerUserId], + ); + if (Number(result.rows[0].count) > 0) { + throw new SafeProductionMigrationError("an active source administrator has no canonical target role"); + } +} + +function remapForeignKeys(row, table, maps) { + const result = { ...row }; + for (const foreignKey of table.foreignKeys) { + const map = maps.get(`${foreignKey.refSchema}.${foreignKey.refTable}`); + if (!map || foreignKey.columns.length !== 1 || foreignKey.refColumns[0] !== "id") continue; + const column = foreignKey.columns[0]; + if (result[column] === null || result[column] === undefined) continue; + const mapped = map.get(String(result[column])); + if (!mapped) throw new SafeProductionMigrationError("a configuration foreign key could not be mapped"); + result[column] = mapped; + } + return result; +} + +function applyCiphertextPolicy(row, table, ciphertextMode) { + if (ciphertextMode !== "exclude") return row; + const columns = CIPHERTEXT_COLUMNS.get(table); + if (!columns) return row; + return Object.fromEntries( + Object.entries(row).map(([column, value]) => [column, columns.has(column) ? null : value]), + ); +} + +function deferredForeignKeys(tableName, table, selectedTables) { + const columns = columnMap(table); + return table.foreignKeys.filter( + (foreignKey) => + foreignKey.refSchema === "public" && + selectedTables.has(foreignKey.refTable) && + (foreignKey.refTable === tableName || + foreignKey.columns.some((name) => columns.get(name)?.nullable === true)), + ); +} + +function parameterList(length) { + return Array.from({ length }, (_, index) => `$${index + 1}`).join(", "); +} + +async function insertIdentityUsers(target, users) { + const columns = [ + "id", "name", "email", "email_verified", "email_verified_at", "image", "role", + "banned", "ban_reason", "ban_expires", "created_at", "updated_at", "two_factor_enabled", + ]; + const sql = `insert into identity.users (${columns.map(quoted).join(", ")}) values (${parameterList(columns.length)})`; + for (const user of users) { + await target.query(sql, columns.map((column) => user[column])); + } +} + +async function mergeSeedTable(source, target, tableName, sourceTable, targetTable, maps) { + const naturalKey = SEED_TABLES.get(tableName); + const columns = commonColumns(sourceTable, targetTable).filter((column) => column !== "id"); + if (naturalKey.some((column) => !columns.includes(column))) { + throw new SafeProductionMigrationError("a seed table is missing its natural key"); + } + const rows = await readRows(source, "public", tableName, ["id", ...columns], sourceTable.primaryKey); + const map = new Map(); + const updateColumns = columns.filter((column) => !naturalKey.includes(column)); + const assignments = updateColumns.length + ? updateColumns.map((column) => `${quoted(column)} = excluded.${quoted(column)}`).join(", ") + : `${quoted(naturalKey[0])} = excluded.${quoted(naturalKey[0])}`; + const sql = ` + insert into ${qualified("public", tableName)} (${columns.map(quoted).join(", ")}) + values (${parameterList(columns.length)}) + on conflict (${naturalKey.map(quoted).join(", ")}) do update set ${assignments} + returning id + `; + for (const sourceRow of rows) { + const row = applyCiphertextPolicy( + remapForeignKeys(sourceRow, targetTable, maps), + tableName, + "preserve", + ); + const result = await target.query(sql, columns.map((column) => row[column])); + map.set(String(sourceRow.id), result.rows[0].id); + } + maps.set(`public.${tableName}`, map); + return rows.length; +} + +async function mergeSeedRelation(source, target, tableName, sourceTable, targetTable, maps, ciphertextMode) { + const naturalKey = SEED_RELATIONS.get(tableName); + const columns = commonColumns(sourceTable, targetTable).filter((column) => column !== "id"); + const rows = await readRows(source, "public", tableName, commonColumns(sourceTable, targetTable), sourceTable.primaryKey); + const updateColumns = columns.filter((column) => !naturalKey.includes(column)); + const conflict = updateColumns.length + ? `do update set ${updateColumns.map((column) => `${quoted(column)} = excluded.${quoted(column)}`).join(", ")}` + : "do nothing"; + const sql = ` + insert into ${qualified("public", tableName)} (${columns.map(quoted).join(", ")}) + values (${parameterList(columns.length)}) + on conflict (${naturalKey.map(quoted).join(", ")}) ${conflict} + `; + let map; + if (commonColumns(sourceTable, targetTable).includes("id")) map = new Map(); + for (const sourceRow of rows) { + const row = applyCiphertextPolicy( + remapForeignKeys(sourceRow, targetTable, maps), + tableName, + ciphertextMode, + ); + await target.query(sql, columns.map((column) => row[column])); + if (map) { + const where = naturalKey.map((column, index) => `${quoted(column)} = $${index + 1}`).join(" and "); + const result = await target.query( + `select id from ${qualified("public", tableName)} where ${where}`, + naturalKey.map((column) => row[column]), + ); + map.set(String(sourceRow.id), result.rows[0].id); + } + } + if (map) maps.set(`public.${tableName}`, map); + return rows.length; +} + +async function copyTable(source, target, tableName, sourceTable, targetTable, maps, selectedTables, ciphertextMode) { + const columns = commonColumns(sourceTable, targetTable); + const rows = await readRows(source, "public", tableName, columns, sourceTable.primaryKey); + const deferred = deferredForeignKeys(tableName, targetTable, selectedTables); + const deferredColumns = new Set( + deferred.flatMap((foreignKey) => { + const columnsByName = columnMap(targetTable); + return foreignKey.columns.filter((column) => columnsByName.get(column)?.nullable); + }), + ); + const updateColumns = columns.filter((column) => !targetTable.primaryKey.includes(column)); + const conflict = updateColumns.length + ? `do update set ${updateColumns.map((column) => `${quoted(column)} = excluded.${quoted(column)}`).join(", ")}` + : "do nothing"; + const overriding = columns.some((column) => columnMap(targetTable).get(column)?.identity) + ? " overriding system value" + : ""; + const sql = ` + insert into ${qualified("public", tableName)} (${columns.map(quoted).join(", ")})${overriding} + values (${parameterList(columns.length)}) + on conflict (${targetTable.primaryKey.map(quoted).join(", ")}) ${conflict} + `; + for (const sourceRow of rows) { + let row = applyCiphertextPolicy( + remapForeignKeys(sourceRow, targetTable, maps), + tableName, + ciphertextMode, + ); + row = { ...row }; + for (const column of deferredColumns) row[column] = null; + await target.query(sql, columns.map((column) => row[column])); + } + for (const column of targetTable.columns.filter( + (column) => column.identity && columns.includes(column.name), + )) { + await target.query( + `select setval(pg_get_serial_sequence($1, $2), coalesce(max(${quoted(column.name)}), 1), max(${quoted(column.name)}) is not null) from ${qualified("public", tableName)}`, + [`public.${tableName}`, column.name], + ); + } + return { count: rows.length, deferred }; +} + +async function restoreDeferredForeignKeys(source, target, tableName, sourceTable, targetTable, foreignKeys, maps) { + if (foreignKeys.length === 0) return; + const updateColumns = [...new Set(foreignKeys.flatMap((foreignKey) => foreignKey.columns))]; + const columns = [...new Set([...sourceTable.primaryKey, ...updateColumns])]; + const rows = await readRows(source, "public", tableName, columns, sourceTable.primaryKey); + for (const sourceRow of rows) { + const row = remapForeignKeys(sourceRow, targetTable, maps); + const assignments = updateColumns.map((column, index) => `${quoted(column)} = $${index + 1}`); + const where = sourceTable.primaryKey.map( + (column, index) => `${quoted(column)} = $${updateColumns.length + index + 1}`, + ); + await target.query( + `update ${qualified("public", tableName)} set ${assignments.join(", ")} where ${where.join(" and ")}`, + [...updateColumns.map((column) => row[column]), ...sourceTable.primaryKey.map((column) => row[column])], + ); + } +} + +async function forceOwner(target, ownerUserId) { + const role = await target.query("select id from public.admin_roles where code = 'owner'"); + if (role.rows.length !== 1) throw new SafeProductionMigrationError("target Owner role is missing"); + await target.query( + ` + insert into public.admin_users (user_id, created_by, revoked_at, revoked_by) + values ($1, $1, null, null) + on conflict (user_id) do update set + revoked_at = null, + revoked_by = null, + updated_at = case + when public.admin_users.revoked_at is not null or public.admin_users.revoked_by is not null + then now() + else public.admin_users.updated_at + end + `, + [ownerUserId], + ); + await target.query( + ` + insert into public.admin_user_roles (admin_user_id, role_id, assigned_by) + values ($1, $2, $1) + on conflict (admin_user_id, role_id) do nothing + `, + [ownerUserId, role.rows[0].id], + ); +} + +async function assertTargetAdminState(target, ownerUserId) { + const result = await target.query( + ` + select + count(*) filter (where au.user_id = $1 and au.revoked_at is null and ar.code = 'owner')::int as owner_count, + ( + select count(*)::int from public.admin_users active + where active.revoked_at is null and not exists ( + select 1 from public.admin_user_roles roles where roles.admin_user_id = active.user_id + ) + ) as admins_without_roles, + ( + select count(*)::int from public.admin_users active + where active.revoked_at is null and not exists ( + select 1 from identity.users users + where users.id = active.user_id and users.role = 'admin' and users.banned = false + ) + ) as unusable_identity_admins + from public.admin_users au + join public.admin_user_roles aur on aur.admin_user_id = au.user_id + join public.admin_roles ar on ar.id = aur.role_id + `, + [ownerUserId], + ); + if ( + result.rows[0].owner_count !== 1 || + result.rows[0].admins_without_roles !== 0 || + result.rows[0].unusable_identity_admins !== 0 + ) { + throw new SafeProductionMigrationError("target administrator reconciliation failed"); + } +} + +async function preflightContext(source, target, config, { requireEmpty = true } = {}) { + const [sourcePublic, sourceAuth, targetPublic] = await Promise.all([ + readSchema(source, "public"), + readSchema(source, "auth"), + readSchema(target, "public"), + ]); + const activeAdminUserIds = await readActiveAdminUserIds(source, sourcePublic); + const users = await readSourceUsers(source, sourceAuth, activeAdminUserIds); + assertActiveAdminUsers(users, activeAdminUserIds, config.ownerUserId); + if (requireEmpty) await assertTargetEmpty(target, targetPublic); + await assertNoUnmappedSourceTables(source, sourcePublic, targetPublic); + await assertActiveAdminRoles(source, sourcePublic, config.ownerUserId); + const plan = transferPlan(sourcePublic, targetPublic); + return { sourcePublic, targetPublic, users, plan }; +} + +async function applyMigration(source, target, config, context) { + const { sourcePublic, targetPublic, users, plan } = context; + const counts = { identity_users: users.length }; + const maps = new Map(); + await insertIdentityUsers(target, users); + + for (const tableName of plan.seedTables) { + counts[`public.${tableName}`] = await mergeSeedTable( + source, target, tableName, sourcePublic.get(tableName), targetPublic.get(tableName), maps, + ); + } + for (const tableName of plan.seedRelations) { + counts[`public.${tableName}`] = await mergeSeedRelation( + source, target, tableName, sourcePublic.get(tableName), targetPublic.get(tableName), maps, + config.ciphertextMode, + ); + } + + const deferredByTable = new Map(); + for (const tableName of plan.ordered) { + const result = await copyTable( + source, target, tableName, sourcePublic.get(tableName), targetPublic.get(tableName), maps, + plan.selected, config.ciphertextMode, + ); + counts[`public.${tableName}`] = result.count; + deferredByTable.set(tableName, result.deferred); + } + for (const tableName of plan.ordered) { + await restoreDeferredForeignKeys( + source, target, tableName, sourcePublic.get(tableName), targetPublic.get(tableName), + deferredByTable.get(tableName), maps, + ); + } + await forceOwner(target, config.ownerUserId); + await assertTargetAdminState(target, config.ownerUserId); + return counts; +} + +async function migrationFilesAreCurrent(targetUrl) { + const scriptDirectory = dirname(fileURLToPath(import.meta.url)); + const output = []; + const status = await runMigrations({ + connectionString: targetUrl, + migrationsDirectories: [ + resolve(scriptDirectory, "../db/migrations"), + resolve(scriptDirectory, "../supabase/migrations"), + ], + logger: { log: (value) => output.push(String(value)) }, + check: true, + }); + if (status !== 0) throw new SafeProductionMigrationError("target has pending schema migrations"); +} + +async function buildMapsForVerification(source, target, sourcePublic, plan) { + const maps = new Map(); + for (const [tableName, naturalKey] of SEED_TABLES) { + if (!plan.selected.has(tableName)) continue; + const sourceTable = sourcePublic.get(tableName); + const sourceRows = await readRows( + source, "public", tableName, + ["id", ...naturalKey], sourceTable.primaryKey, + ); + const map = new Map(); + for (const row of sourceRows) { + const where = naturalKey.map((column, index) => `${quoted(column)} = $${index + 1}`).join(" and "); + const match = await target.query( + `select id from ${qualified("public", tableName)} where ${where}`, + naturalKey.map((column) => row[column]), + ); + if (match.rows.length !== 1) throw new SafeProductionMigrationError("seed reconciliation failed"); + map.set(String(row.id), match.rows[0].id); + } + maps.set(`public.${tableName}`, map); + } + return maps; +} + +async function tableManifest(source, target, tableName, sourceTable, targetTable, maps, ciphertextMode, seedSubset) { + let columns = commonColumns(sourceTable, targetTable); + if (seedSubset) columns = columns.filter((column) => column !== "id"); + const sourceRows = await readRows(source, "public", tableName, columns, sourceTable.primaryKey); + const expected = sourceRows.map((row) => + applyCiphertextPolicy(remapForeignKeys(row, targetTable, maps), tableName, ciphertextMode), + ); + let targetRows; + if (seedSubset) { + const naturalKey = SEED_TABLES.get(tableName) ?? SEED_RELATIONS.get(tableName); + targetRows = []; + for (const row of expected) { + const where = naturalKey.map((column, index) => `${quoted(column)} = $${index + 1}`).join(" and "); + const match = await target.query( + `select ${columns.map(quoted).join(", ")} from ${qualified("public", tableName)} where ${where}`, + naturalKey.map((column) => row[column]), + ); + targetRows.push(...match.rows); + } + } else { + targetRows = await readRows(target, "public", tableName, columns, targetTable.primaryKey); + } + const sourceHash = rowsSha256(expected, columns); + const targetHash = rowsSha256(targetRows, columns); + const keyColumns = seedSubset + ? (SEED_TABLES.get(tableName) ?? SEED_RELATIONS.get(tableName)) + : sourceTable.primaryKey; + const sourceKeyHash = rowsSha256(expected, keyColumns); + const targetKeyHash = rowsSha256(targetRows, keyColumns); + return { + source_count: expected.length, + target_count: targetRows.length, + primary_key_sha256: sourceKeyHash, + target_primary_key_sha256: targetKeyHash, + normalized_sha256: sourceHash, + target_normalized_sha256: targetHash, + ok: + expected.length === targetRows.length && + sourceKeyHash === targetKeyHash && + sourceHash === targetHash, + }; +} + +async function queryAggregate(client, text) { + return (await client.query(text)).rows.map((row) => normalizeValue(row)); +} + +async function reconciliationAggregates(client, tables) { + const result = {}; + if (tables.has("credit_transactions")) { + result.credits = await queryAggregate( + client, + `select transaction_type as state, count(*)::bigint as count, coalesce(sum(amount),0)::text as amount from public.credit_transactions group by transaction_type order by transaction_type`, + ); + } + if (tables.has("payment_orders")) { + result.orders = await queryAggregate( + client, + `select status, count(*)::bigint as count, coalesce(sum(money_cents),0)::text as money_cents, coalesce(sum(refund_amount_cents),0)::text as refund_cents from public.payment_orders group by status order by status`, + ); + } + if (tables.has("user_subscriptions")) { + result.subscriptions = await queryAggregate( + client, + `select status, count(*)::bigint as count from public.user_subscriptions group by status order by status`, + ); + } + if (tables.has("personal_reports")) { + result.personal_reports = await queryAggregate( + client, + `select status, count(*)::bigint as count from public.personal_reports group by status order by status`, + ); + } + if (tables.has("consultation_requests")) { + result.consultations = await queryAggregate( + client, + `select status, count(*)::bigint as count from public.consultation_requests group by status order by status`, + ); + } + const rectificationTables = [...tables.keys()].filter((table) => table.includes("rectification")); + result.rectification = []; + for (const table of rectificationTables.sort()) { + result.rectification.push({ table, count: await countRows(client, "public", table) }); + } + return result; +} + +async function verifyMigration(source, target, config, context) { + const { sourcePublic, targetPublic, users, plan } = context; + const maps = await buildMapsForVerification(source, target, sourcePublic, plan); + const identityColumns = Object.keys(users[0] ?? normalizeAuthUser({ + id: "00000000-0000-4000-8000-000000000000", + email: "empty@example.invalid", + raw_user_meta_data: {}, email_confirmed_at: null, created_at: null, updated_at: null, + })); + const targetUsers = await readRows(target, "identity", "users", identityColumns, ["id"]); + const identityHash = rowsSha256(users, identityColumns); + const targetIdentityHash = rowsSha256(targetUsers, identityColumns); + const tables = {}; + for (const tableName of [...plan.selected].sort()) { + tables[`public.${tableName}`] = await tableManifest( + source, target, tableName, sourcePublic.get(tableName), targetPublic.get(tableName), maps, + config.ciphertextMode, SEED_TABLES.has(tableName) || SEED_RELATIONS.has(tableName), + ); + } + await assertTargetAdminState(target, config.ownerUserId); + const sourceAggregates = await reconciliationAggregates(source, sourcePublic); + const targetAggregates = await reconciliationAggregates(target, targetPublic); + const aggregatesOk = JSON.stringify(sourceAggregates) === JSON.stringify(targetAggregates); + const ok = + users.length === targetUsers.length && + identityHash === targetIdentityHash && + Object.values(tables).every((table) => table.ok) && + aggregatesOk; + return { + mode: "verify", + ok, + identity: { + source_count: users.length, + target_count: targetUsers.length, + normalized_sha256: identityHash, + target_normalized_sha256: targetIdentityHash, + }, + tables, + aggregates: { source: sourceAggregates, target: targetAggregates, ok: aggregatesOk }, + ciphertext_mode: config.ciphertextMode, + }; +} + +export async function run(mode, env, dependencies = {}) { + const config = readConfiguration(env); + const PoolClass = dependencies.Pool ?? Pool; + await (dependencies.checkMigrations ?? migrationFilesAreCurrent)(config.targetUrl); + const sourcePool = new PoolClass({ + connectionString: config.sourceUrl, + application_name: "jyotisha-production-migration-source", + max: 1, + }); + const targetPool = new PoolClass({ + connectionString: config.targetUrl, + application_name: "jyotisha-production-migration-target", + max: 1, + }); + let source; + let target; + let sourceTransaction = false; + let targetTransaction = false; + try { + source = await sourcePool.connect(); + target = await targetPool.connect(); + await source.query("begin isolation level repeatable read read only"); + sourceTransaction = true; + if (mode === "verify") { + await target.query("begin isolation level repeatable read read only"); + } else { + await target.query("begin"); + if (mode === "apply") { + await target.query("select pg_advisory_xact_lock(hashtext('jyotisha_production_data_migration'))"); + } + } + targetTransaction = true; + const context = await preflightContext(source, target, config, { + requireEmpty: mode !== "verify", + }); + + if (mode === "preflight") { + await target.query("rollback"); + targetTransaction = false; + await source.query("commit"); + sourceTransaction = false; + return { + mode, + ok: true, + source_users: context.users.length, + source_public_tables: context.plan.selected.size, + target_business_empty: true, + owner_ready: true, + ciphertext_mode: config.ciphertextMode, + }; + } + if (mode === "apply") { + const counts = await applyMigration(source, target, config, context); + await target.query("commit"); + targetTransaction = false; + await source.query("commit"); + sourceTransaction = false; + return { mode, ok: true, imported: counts, ciphertext_mode: config.ciphertextMode }; + } + const manifest = await verifyMigration(source, target, config, context); + await target.query("commit"); + targetTransaction = false; + await source.query("commit"); + sourceTransaction = false; + return manifest; + } catch (error) { + if (targetTransaction && target) await target.query("rollback").catch(() => {}); + if (sourceTransaction && source) await source.query("rollback").catch(() => {}); + throw error; + } finally { + source?.release(); + target?.release(); + await sourcePool.end().catch(() => {}); + await targetPool.end().catch(() => {}); + } +} + +function safeMessage(error) { + return error instanceof SafeProductionMigrationError + ? error.message + : "production data migration failed"; +} + +const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : ""; +if (import.meta.url === invokedPath) { + let mode; + try { + mode = parseMode(process.argv.slice(2)); + const result = await run(mode, process.env); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.ok) process.exitCode = 2; + } catch (error) { + process.stderr.write(`${safeMessage(error)}\n`); + process.exitCode = 1; + } +} diff --git a/frontend/tests/database-topology.test.ts b/frontend/tests/database-topology.test.ts index 1b2f6b88..5830c1e2 100644 --- a/frontend/tests/database-topology.test.ts +++ b/frontend/tests/database-topology.test.ts @@ -56,6 +56,24 @@ test("database roles have no cluster privileges", () => { `), "t", ); + assert.equal( + fixture.psql(` + select role_name || ':' || case when pg_has_role(role_name, 'schema_owner', 'MEMBER') then 'true' else 'false' end + from unnest(array[ + 'migration_runner', 'identity_runtime', 'app_runtime', + 'service_runtime', 'admin_runtime', 'backup_reader' + ]) role_name + order by role_name + `), + [ + "admin_runtime:f", + "app_runtime:f", + "backup_reader:f", + "identity_runtime:f", + "migration_runner:true", + "service_runtime:f", + ].join("\n"), + ); assert.equal( fixture.psql(` select coalesce(string_agg(privilege_type, ',' order by privilege_type), '') diff --git a/frontend/tests/health-deployment.test.ts b/frontend/tests/health-deployment.test.ts index 26b953fc..30886dfe 100644 --- a/frontend/tests/health-deployment.test.ts +++ b/frontend/tests/health-deployment.test.ts @@ -437,3 +437,92 @@ test("production API probes health rapidly while a replacement container starts" assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/); }); + +test("self-hosted production Caddy isolates user and admin hosts", () => { + const caddy = readFileSync( + new URL("../../deploy/Caddyfile.production.selfhosted", import.meta.url), + "utf8", + ); + + assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/jyotisha\.chat\}/); + assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/); + assert.match(caddy, /respond @adminPaths "Not found" 404/); + assert.match(caddy, /^https:\/\/admin\.jyotisha\.chat \{$/m); + assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2); + assert.match(caddy, /@root path \/\n\s+redir @root \/admin 308/); + assert.doesNotMatch(caddy, /staging\.jyotisha\.chat|:443 \{/); +}); + +test("production env validators accept only self-hosted production selectors and role URLs", () => { + const appValidator = fileURLToPath( + new URL("../../deploy/validate-production-env.sh", import.meta.url), + ); + const databaseValidator = fileURLToPath( + new URL("../../deploy/validate-production-database-env.sh", import.meta.url), + ); + const root = mkdtempSync(join(tmpdir(), "jyotisha-production-env-")); + const appEnv = join(root, ".env.production"); + const databaseEnv = join(root, ".env.production.database"); + const appLines = [ + "APP_ENV_FILE=../.env.production", + "CADDYFILE_PATH=./Caddyfile.production.selfhosted", + "SITE_ADDRESS=https://jyotisha.chat", + "AUTH_PROVIDER=self-hosted", + "SELF_HOSTED_IDENTITY_ENABLED=true", + "AUTH_USER_ORIGIN=https://jyotisha.chat", + "ADMIN_USER_ORIGIN=https://admin.jyotisha.chat", + "IDENTITY_DATABASE_URL=postgresql://identity_runtime:identity-runtime-test-password@postgres:5432/jyotisha", + "APP_DATABASE_URL=postgresql://app_runtime:app-runtime-test-password@postgres:5432/jyotisha", + "SERVICE_DATABASE_URL=postgresql://service_runtime:service-runtime-test-password@postgres:5432/jyotisha", + "ADMIN_DATABASE_URL=postgresql://admin_runtime:admin-runtime-test-password@postgres:5432/jyotisha", + "BETTER_AUTH_USER_SECRET=user-secret-that-is-at-least-32-bytes-long", + "RESEND_API_KEY=re_test_key_that_must_not_be_printed", + "RESEND_FROM_EMAIL=Jyotisha Production ", + "ADMIN_EMAILS=admin@example.com", + "EPAY_CONFIG_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", + "EPAY_CHAT_ENABLED=false", + "JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=dynamic-token-that-is-at-least-32-bytes", + "PERSONAL_REPORT_ENABLED=true", + "PERSONAL_REPORT_DAILY_LIMIT=5", + ]; + const databaseLines = [ + "POSTGRES_DB=jyotisha", + "POSTGRES_USER=postgres", + "POSTGRES_PASSWORD=postgres-password", + "SCHEMA_OWNER_PASSWORD=schema-owner-password", + "IDENTITY_RUNTIME_PASSWORD=identity-runtime-password", + "APP_RUNTIME_PASSWORD=app-runtime-password", + "SERVICE_RUNTIME_PASSWORD=service-runtime-password", + "ADMIN_RUNTIME_PASSWORD=admin-runtime-password", + "MIGRATION_RUNNER_PASSWORD=migration-runner-password", + "BACKUP_READER_PASSWORD=backup-reader-password", + "PRODUCTION_BACKUP_ENCRYPTION_KEY=backup-encryption-key", + "SCHEMA_DATABASE_URL=postgresql://schema_owner:schema-owner-password@postgres:5432/jyotisha", + ]; + const writeEnv = (path: string, lines: string[]) => { + writeFileSync(path, `${lines.join("\n")}\n`); + chmodSync(path, 0o600); + }; + + try { + writeEnv(appEnv, appLines); + writeEnv(databaseEnv, databaseLines); + assert.equal(spawnSync("bash", [appValidator, appEnv], { encoding: "utf8" }).status, 0); + assert.equal(spawnSync("bash", [databaseValidator, databaseEnv], { encoding: "utf8" }).status, 0); + + writeEnv(appEnv, appLines.map((line) => + line.startsWith("ADMIN_USER_ORIGIN=") + ? "ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat" + : line, + )); + assert.notEqual(spawnSync("bash", [appValidator, appEnv], { encoding: "utf8" }).status, 0); + + writeEnv(databaseEnv, databaseLines.filter((line) => + !line.startsWith("PRODUCTION_BACKUP_ENCRYPTION_KEY="), + )); + assert.notEqual(spawnSync("bash", [databaseValidator, databaseEnv], { encoding: "utf8" }).status, 0); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/frontend/tests/production-data-migration.test.ts b/frontend/tests/production-data-migration.test.ts new file mode 100644 index 00000000..4c43392c --- /dev/null +++ b/frontend/tests/production-data-migration.test.ts @@ -0,0 +1,277 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { + SafeProductionMigrationError, + assertActiveAdminUsers, + assertTargetEmpty, + normalizeAuthUser, + normalizeAuthUsers, + parseMode, + readConfiguration, + readSchema, + rowsSha256, + transferPlan, +} from "../scripts/migrate-supabase-production.mjs"; + +const scriptPath = fileURLToPath( + new URL("../scripts/migrate-supabase-production.mjs", import.meta.url), +); + +function table( + columns: Array<{ + name: string; + nullable?: boolean; + defaultValue?: string | null; + generated?: boolean; + identity?: boolean; + identityGeneration?: string | null; + }>, + primaryKey = ["id"], + foreignKeys: Array<{ + columns: string[]; + refSchema: string; + refTable: string; + refColumns: string[]; + }> = [], +) { + return { + columns: columns.map((column) => ({ + nullable: false, + defaultValue: null, + generated: false, + identity: false, + identityGeneration: null, + ...column, + })), + primaryKey, + foreignKeys, + }; +} + +test("CLI requires one explicit migration mode", () => { + assert.equal(parseMode(["--preflight"]), "preflight"); + assert.equal(parseMode(["--apply"]), "apply"); + assert.equal(parseMode(["--verify"]), "verify"); + assert.throws(() => parseMode([]), SafeProductionMigrationError); + assert.throws(() => parseMode(["--apply", "--verify"]), SafeProductionMigrationError); +}); + +test("configuration requires an explicit Owner and ciphertext decision", () => { + const base = { + SUPABASE_SOURCE_DATABASE_URL: "postgresql://source.invalid/jyotisha", + PRODUCTION_TARGET_DATABASE_URL: "postgresql://target.invalid/jyotisha", + PRODUCTION_OWNER_USER_ID: "018f4e6d-7a11-7000-8000-000000000001", + }; + + assert.equal( + readConfiguration({ ...base, PRODUCTION_CIPHERTEXT_MODE: "exclude" }).ciphertextMode, + "exclude", + ); + assert.throws( + () => readConfiguration({ ...base, PRODUCTION_CIPHERTEXT_MODE: "preserve" }), + /confirmed production encryption keys/, + ); + assert.equal( + readConfiguration({ + ...base, + PRODUCTION_CIPHERTEXT_MODE: "preserve", + PRODUCTION_CIPHERTEXT_KEYS_CONFIRMED: "true", + }).ciphertextMode, + "preserve", + ); +}); + +test("Supabase identity transform preserves UUID and ban state without credentials", () => { + const source = { + id: "018F4E6D-7A11-7000-8000-000000000001", + email: " Person@Example.com ", + raw_user_meta_data: { full_name: "Person One", avatar_url: "https://example.invalid/a.png" }, + email_confirmed_at: new Date("2026-07-01T00:00:00Z"), + banned_until: new Date("2027-01-01T00:00:00Z"), + created_at: new Date("2026-06-01T00:00:00Z"), + updated_at: new Date("2026-07-02T00:00:00Z"), + encrypted_password: "must-not-migrate", + refresh_token: "must-not-migrate", + mfa_secret: "must-not-migrate", + }; + const user = normalizeAuthUser(source, new Date("2026-08-09T00:00:00Z")); + + assert.equal(user.id, source.id.toLowerCase()); + assert.equal(user.email, "person@example.com"); + assert.equal(user.banned, true); + assert.deepEqual(user.ban_expires, source.banned_until); + assert.equal(user.two_factor_enabled, false); + assert.doesNotMatch(JSON.stringify(user), /must-not-migrate|password|refresh_token|mfa_secret/); +}); + +test("production identity preflight rejects duplicate canonical emails", () => { + const base = { + id: "018f4e6d-7a11-7000-8000-000000000001", + email: "person@example.com", + raw_user_meta_data: {}, + email_confirmed_at: null, + created_at: new Date("2026-06-01T00:00:00Z"), + updated_at: new Date("2026-06-01T00:00:00Z"), + }; + + assert.throws( + () => normalizeAuthUsers([ + base, + { ...base, id: "018f4e6d-7a11-7000-8000-000000000002", email: " PERSON@example.com " }, + ]), + /invalid or duplicate auth identities/, + ); +}); + +test("active administrators become usable identity admins", () => { + const ownerId = "018f4e6d-7a11-7000-8000-000000000001"; + const adminId = "018f4e6d-7a11-7000-8000-000000000002"; + const sourceUsers = [ownerId, adminId].map((id) => ({ + id, + email: `${id}@example.com`, + raw_user_meta_data: {}, + email_confirmed_at: null, + banned_until: null, + created_at: new Date("2026-06-01T00:00:00Z"), + updated_at: new Date("2026-06-01T00:00:00Z"), + })); + const activeAdminUserIds = new Set([ownerId, adminId]); + const users = normalizeAuthUsers( + sourceUsers, + new Date("2026-08-09T00:00:00Z"), + activeAdminUserIds, + ); + + assert.deepEqual(users.map((user) => user.role), ["admin", "admin"]); + assert.doesNotThrow(() => assertActiveAdminUsers(users, activeAdminUserIds, ownerId)); + assert.throws( + () => assertActiveAdminUsers(users, new Set([adminId]), ownerId), + /Owner is not an active source administrator/, + ); + assert.throws( + () => assertActiveAdminUsers([{ ...users[0], banned: true }, users[1]], activeAdminUserIds, ownerId), + /Owner is blocked/, + ); +}); + +test("schema reader preserves PostgreSQL identity metadata", async () => { + const client = { + async query(text: string) { + if (text.includes("information_schema.columns")) { + return { + rows: [{ + table_name: "redemption_attempts", + column_name: "id", + nullable: false, + column_default: null, + generated: false, + identity: true, + identity_generation: "ALWAYS", + }], + }; + } + if (text.includes("PRIMARY KEY")) { + return { rows: [{ table_name: "redemption_attempts", columns: ["id"] }] }; + } + return { rows: [] }; + }, + }; + + const schema = await readSchema(client, "public"); + assert.deepEqual(schema.get("redemption_attempts")?.columns[0], { + name: "id", + nullable: false, + defaultValue: null, + generated: false, + identity: true, + identityGeneration: "ALWAYS", + }); +}); + +test("transfer plan rejects source-only public columns", () => { + const source = new Map([ + ["profiles", table([{ name: "id" }, { name: "legacy_value" }])], + ]); + const target = new Map([ + ["profiles", table([{ name: "id" }])], + ]); + + assert.throws(() => transferPlan(source, target), /target schema is missing a source column/); +}); + +test("transfer plan uses non-nullable dependencies and rejects unsafe cycles", () => { + const source = new Map([ + ["parent", table([{ name: "id" }])], + ["child", table([{ name: "id" }, { name: "parent_id" }])], + ]); + const target = new Map([ + ["parent", table([{ name: "id" }])], + ["child", table( + [{ name: "id" }, { name: "parent_id" }], + ["id"], + [{ columns: ["parent_id"], refSchema: "public", refTable: "parent", refColumns: ["id"] }], + )], + ]); + assert.deepEqual(transferPlan(source, target).ordered, ["parent", "child"]); + + target.get("parent")!.columns.push({ + name: "child_id", nullable: false, defaultValue: null, generated: false, + }); + source.get("parent")!.columns.push({ + name: "child_id", nullable: false, defaultValue: null, generated: false, + }); + target.get("parent")!.foreignKeys.push({ + columns: ["child_id"], refSchema: "public", refTable: "child", refColumns: ["id"], + }); + assert.throws(() => transferPlan(source, target), /foreign-key cycle/); +}); + +test("target preflight rejects existing business rows but permits migration seeds", async () => { + const counts = new Map([ + ["identity.users", 0], + ["auth.users", 0], + ["public.admin_roles", 6], + ["public.profiles", 1], + ]); + const client = { + async query(text: string) { + const match = text.match(/from\s+"(identity|auth|public)"\."([a-z_]+)"/i); + assert.ok(match, text); + return { rows: [{ count: String(counts.get(`${match[1]}.${match[2]}`) ?? 0) }] }; + }, + }; + const targetTables = new Map([ + ["admin_roles", table([{ name: "id" }])], + ["profiles", table([{ name: "id" }])], + ]); + + await assert.rejects(() => assertTargetEmpty(client, targetTables), /not empty/); + counts.set("public.profiles", 0); + await assert.doesNotReject(() => assertTargetEmpty(client, targetTables)); +}); + +test("reconciliation hashes are stable and the script has no wildcard data reads", () => { + const rows = [ + { id: "b", payload: { z: 2, a: 1 } }, + { id: "a", payload: { a: 1, z: 2 } }, + ]; + assert.equal(rowsSha256(rows, ["id", "payload"]), rowsSha256([...rows].reverse(), ["id", "payload"])); + + const source = readFileSync(scriptPath, "utf8"); + assert.doesNotMatch(source, /select\s+\*/i); + assert.match(source, /begin isolation level repeatable read read only/i); + assert.match(source, /pg_advisory_xact_lock/); + assert.match(source, /target business database is not empty/); + assert.match(source, /rollback/); + assert.match(source, /--preflight/); + assert.match(source, /--apply/); + assert.match(source, /--verify/); + assert.match(source, /overriding system value/i); + assert.match(source, /setval\(pg_get_serial_sequence/i); + assert.match(source, /target_primary_key_sha256/); + assert.match(source, /sourceKeyHash === targetKeyHash/); + assert.match(source, /users\.role = 'admin' and users\.banned = false/); +}); diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 26227ca1..2432a13a 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -42,6 +42,10 @@ const giteaProductionWorkflow = new URL( "../../.gitea/workflows/deploy-production.yml", import.meta.url, ); +const giteaProductionMigrationWorkflow = new URL( + "../../.gitea/workflows/migrate-production-database.yml", + import.meta.url, +); const resetStagingAccountWorkflow = new URL( "../../.github/workflows/reset-staging-account.yml", import.meta.url, @@ -62,10 +66,22 @@ const productionDeployScript = new URL( "../../deploy/run-production-deploy.sh", import.meta.url, ); +const productionMigrationScript = new URL( + "../../deploy/run-production-migration.sh", + import.meta.url, +); const productionSyncScript = new URL( "../../deploy/sync-production-tree.sh", import.meta.url, ); +const productionEnvValidator = new URL( + "../../deploy/validate-production-env.sh", + import.meta.url, +); +const productionDatabaseEnvValidator = new URL( + "../../deploy/validate-production-database-env.sh", + import.meta.url, +); function read(url: URL): string { return readFileSync(url, "utf8"); @@ -89,6 +105,7 @@ test("changed staging workflows are syntactically valid YAML", () => { giteaDeployWorkflow, giteaMigrationWorkflow, giteaProductionWorkflow, + giteaProductionMigrationWorkflow, ]) { const result = spawnSync( "python", @@ -844,13 +861,16 @@ test("production deploy is manual-only and consumes the accepted staging artifac }); -test("staging scripts pass shell syntax validation", () => { +test("deployment scripts pass shell syntax validation", () => { for (const script of [ deployScript, migrationScript, syncScript, productionDeployScript, + productionMigrationScript, productionSyncScript, + productionEnvValidator, + productionDatabaseEnvValidator, ]) { const path = fileURLToPath(script); chmodSync(path, 0o755); @@ -858,3 +878,128 @@ test("staging scripts pass shell syntax validation", () => { assert.equal(result.status, 0, result.stderr); } }); + +test("Gitea production deploy consumes only gate-attested digests under manual control", () => { + const workflow = read(giteaProductionWorkflow); + + assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); + assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); + assert.match(workflow, /deploy_sha:[\s\S]*allow_rollback:[\s\S]*verification_mode:/); + assert.match(workflow, /runs-on: manman-linux/); + assert.match(workflow, /group: production-mutation/); + assert.match(workflow, /branch=staging&event=push&status=success/); + assert.match(workflow, /endswith\("backend-quality-gate\.yml"\)/); + assert.match(workflow, /endswith\("release-quality-gate\.yml"\)/); + assert.match(workflow, /public staging has not accepted the requested SHA/); + assert.match(workflow, /main and staging must identify the same reviewed release/); + assert.match(workflow, /controller_sha256/); + assert.match(workflow, /sha256sum --check --status/); + assert.match(workflow, /steps\.images\.outputs\.api_image/); + assert.match(workflow, /steps\.images\.outputs\.web_image/); + assert.match(workflow, /SSH_PRIVATE_KEY_BASE64: \$\{\{ secrets\.PRODUCTION_SSH_PRIVATE_KEY \}\}/); + assert.match(workflow, /\[\[ "\$DEPLOY_HOST" == "118\.194\.235\.34" \]\]/); + assert.match(workflow, /\[\[ "\$DEPLOY_USER" == "deploy" \]\]/); + assert.match(workflow, /\[\[ "\$DEPLOY_PATH" == "\/opt\/jyotisha-production" \]\]/); + assert.match(workflow, /bash '\$incoming\/deploy\/run-production-deploy\.sh'/); + assert.doesNotMatch(workflow, /docker compose[^\n]*build|db:migrate/); +}); + +test("production runner validates state and migrations before switching exact images", () => { + const runner = read(productionDeployScript); + const sync = read(productionSyncScript); + + assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/); + assert.match(runner, /another production mutation holds the host lock/); + assert.match(runner, /sync-production-tree\.sh/); + assertOrder(runner, [ + "validate-production-env.sh", + "validate-production-database-env.sh", + "compose=(", + '"${compose[@]}" config --quiet', + "pull api web", + "up -d --no-build --pull never --wait postgres", + "--profile migration-check run --rm migration-checker", + ]); + assert.match(runner, /-f deploy\/docker-compose\.production\.yml/); + assert.match(runner, /pending migrations: run Migrate Production Database/); + assert.doesNotMatch(runner, /--profile migration run --rm migrator/); + assert.match(runner, /if \[ "\$VERIFICATION_MODE" = "public" \]/); + assert.match(runner, /up -d --no-build api web/); + assert.match(runner, /up -d --no-build --force-recreate --no-deps caddy/); + assert.match(runner, /\["identity", "IDENTITY_DATABASE_URL"\]/); + assert.match(runner, /\["service", "SERVICE_DATABASE_URL"\]/); + assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/); + assert.match(runner, /production verification predicates did not converge/); + assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/); + assert.match(sync, /--exclude='\/\.env\*'/); + assert.match(sync, /--exclude='\/backups\/'/); + assert.match(sync, /--exclude='\/\.state\/'/); +}); + + +test("Gitea production schema migration is exact-SHA gated and isolated from ETL and deploy", () => { + const workflow = read(giteaProductionMigrationWorkflow); + const runner = read(productionMigrationScript); + + assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); + assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); + assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/); + assert.match(workflow, /main_head.*DEPLOY_SHA.*staging_head.*DEPLOY_SHA/s); + assert.match(workflow, /GITEA_SHA.*DEPLOY_SHA/); + assert.match(workflow, /STAGING_URL: \$\{\{ vars\.STAGING_URL \}\}/); + assert.match(workflow, /\[\[ "\$STAGING_URL" == "https:\/\/staging\.jyotisha\.chat" \]\]/); + assert.match(workflow, /"\$STAGING_URL\/api\/health"/); + assert.match(workflow, /observed_staging_sha.*DEPLOY_SHA/s); + assert.match(workflow, /branch=staging&event=push&status=success/); + assert.match(workflow, /endswith\("backend-quality-gate\.yml"\)/); + assert.match(workflow, /endswith\("release-quality-gate\.yml"\)/); + assert.match(workflow, /group: production-mutation/); + assert.match(workflow, /SSH_PRIVATE_KEY_BASE64: \$\{\{ secrets\.PRODUCTION_SSH_PRIVATE_KEY \}\}/); + assert.match(workflow, /PRODUCTION_KNOWN_HOSTS/); + assert.match(workflow, /\[\[ "\$DEPLOY_HOST" == "118\.194\.235\.34" \]\]/); + assert.match(workflow, /\[\[ "\$DEPLOY_USER" == "deploy" \]\]/); + assert.match(workflow, /\[\[ "\$DEPLOY_PATH" == "\/opt\/jyotisha-production" \]\]/); + assert.match(workflow, /sha256sum --check --status/); + assert.match(workflow, /recovery_reference:[\s\S]*required: true[\s\S]*type: string/); + assert.match(workflow, /recovery_created_at:[\s\S]*required: true[\s\S]*type: string/); + assert.match(workflow, /restore_verified:[\s\S]*required: true[\s\S]*default: false[\s\S]*type: boolean/); + assert.match(workflow, /\[\[ "\$RESTORE_VERIFIED" == "true" \]\]/); + assert.match(workflow, /%Y-%m-%dT%H:%M:%SZ/); + assert.match(workflow, /age > timedelta\(hours=24\)/); + assert.match(workflow, /migration files run sequentially and are not atomic as a whole/); + assert.match(workflow, /RECOVERY_REFERENCE='\$RECOVERY_REFERENCE'/); + assert.match(workflow, /RECOVERY_CREATED_AT='\$RECOVERY_CREATED_AT'/); + assert.match(workflow, /RESTORE_VERIFIED='\$RESTORE_VERIFIED'/); + assert.match(workflow, /run-production-migration\.sh/); + assert.doesNotMatch(workflow, /migrate-supabase-production|run-production-deploy|verification_mode|PRODUCTION_URL|CADDY/); + + assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/); + assert.match(runner, /another production mutation holds the host lock/); + assert.match(runner, /\[ "\$DEPLOY_PATH" = "\/opt\/jyotisha-production" \]/); + assert.match(runner, /sync-production-tree\.sh/); + assert.match(runner, /validate-production-env\.sh/); + assert.match(runner, /validate-production-database-env\.sh/); + assert.match(runner, /RECOVERY_REFERENCE RECOVERY_CREATED_AT RESTORE_VERIFIED/); + assert.match(runner, /date -u -d "\$RECOVERY_CREATED_AT" \+%s/); + assert.match(runner, /recovery_age_seconds <= 24 \* 60 \* 60/); + assert.match(runner, /production recovery point must have restore_verified=true/); + assert.match(runner, /migration files run sequentially and are not atomic as a whole/); + assert.match(runner, /migration_runner_database_url="postgresql:\/\/migration_runner:/); + assert.match(runner, /role%3Dschema_owner/); + assert.match(runner, /pg_has_role\('migration_runner', 'schema_owner', 'member'\)/); + assert.match( + runner, + /printf 'SCHEMA_DATABASE_URL=%s\\n' "\$migration_runner_database_url" >"\$migration_environment"/, + ); + assert.doesNotMatch(runner, /(?:awk|cat)[^\n]*\.env\.production\.database[^\n]*migration_environment/); + assert.doesNotMatch(runner, />>"\$migration_environment"/); + assert.equal((runner.match(/--profile migration-check run --rm migration-checker/g) ?? []).length, 2); + assertOrder(runner, [ + "--profile migration-check run --rm migration-checker", + "--profile migration run --rm migrator", + "postcheck_status", + ]); + assert.match(runner, /production migration postcheck did not converge/); + assert.doesNotMatch(runner, /migrate-supabase-production|docker-compose\.server\.yml|docker-compose\.production\.yml/); + assert.doesNotMatch(runner, /\bup\b[^\n]*(?:api|web|caddy)|Caddyfile|PRODUCTION_URL|PRODUCTION_ADMIN_URL|mv -f[^\n]*deployed-revision/); +});