ops: add production PostgreSQL migration tooling
This commit is contained in:
Jesse_Chen
@@ -6,6 +6,7 @@ on:
|
|||||||
- '.gitea/workflows/backend-quality-gate.yml'
|
- '.gitea/workflows/backend-quality-gate.yml'
|
||||||
- '.gitea/workflows/deploy-staging.yml'
|
- '.gitea/workflows/deploy-staging.yml'
|
||||||
- '.gitea/workflows/migrate-staging-database.yml'
|
- '.gitea/workflows/migrate-staging-database.yml'
|
||||||
|
- '.gitea/workflows/migrate-production-database.yml'
|
||||||
- 'deploy/**'
|
- 'deploy/**'
|
||||||
- 'frontend/**'
|
- 'frontend/**'
|
||||||
- 'jyotish_vedic/**'
|
- 'jyotish_vedic/**'
|
||||||
|
|||||||
@@ -0,0 +1,363 @@
|
|||||||
|
name: Migrate Production Database (manual only)
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
deploy_sha:
|
||||||
|
description: Full current production release SHA to migrate
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
recovery_reference:
|
||||||
|
description: Backup or PITR recovery reference; multiple migration files are not atomic as a set
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
recovery_created_at:
|
||||||
|
description: Recovery point creation time in UTC, exactly YYYY-MM-DDTHH:MM:SSZ and no more than 24 hours old
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
restore_verified:
|
||||||
|
description: Confirm that this recovery point has passed a restore verification
|
||||||
|
required: true
|
||||||
|
default: false
|
||||||
|
type: boolean
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
actions: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: production-mutation
|
||||||
|
cancel-in-progress: false
|
||||||
|
queue: max
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
migrate:
|
||||||
|
runs-on: manman-linux
|
||||||
|
timeout-minutes: 20
|
||||||
|
env:
|
||||||
|
GITEA_SHA: ${{ gitea.sha }}
|
||||||
|
GITEA_API_URL: ${{ gitea.api_url }}
|
||||||
|
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
REGISTRY_HOST: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com
|
||||||
|
IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha
|
||||||
|
DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }}
|
||||||
|
DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }}
|
||||||
|
DEPLOY_USER: ${{ vars.PRODUCTION_USER }}
|
||||||
|
DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }}
|
||||||
|
STAGING_URL: ${{ vars.STAGING_URL }}
|
||||||
|
PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }}
|
||||||
|
steps:
|
||||||
|
- name: Validate current production revision and successful gates
|
||||||
|
id: revision
|
||||||
|
env:
|
||||||
|
DEPLOY_SHA: ${{ inputs.deploy_sha }}
|
||||||
|
RECOVERY_REFERENCE: ${{ inputs.recovery_reference }}
|
||||||
|
RECOVERY_CREATED_AT: ${{ inputs.recovery_created_at }}
|
||||||
|
RESTORE_VERIFIED: ${{ inputs.restore_verified }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; }
|
||||||
|
[[ "$STAGING_URL" == "https://staging.jyotisha.chat" ]] || { echo "unexpected staging acceptance URL" >&2; exit 1; }
|
||||||
|
[[ "$RECOVERY_REFERENCE" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}$ ]] || {
|
||||||
|
echo "recovery_reference must be 1-200 safe reference characters" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ "$RECOVERY_CREATED_AT" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || {
|
||||||
|
echo "recovery_created_at must be UTC in YYYY-MM-DDTHH:MM:SSZ format" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ "$RESTORE_VERIFIED" == "true" ]] || {
|
||||||
|
echo "restore_verified=true is required for a production schema migration" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
python3 - "$RECOVERY_CREATED_AT" <<'PY'
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import sys
|
||||||
|
|
||||||
|
try:
|
||||||
|
created_at = datetime.strptime(sys.argv[1], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||||
|
except ValueError as error:
|
||||||
|
raise SystemExit(f"invalid recovery_created_at: {error}")
|
||||||
|
age = datetime.now(timezone.utc) - created_at
|
||||||
|
if age < timedelta(0) or age > timedelta(hours=24):
|
||||||
|
raise SystemExit("recovery_created_at must be no more than 24 hours old and not in the future")
|
||||||
|
PY
|
||||||
|
echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT restore_verified=true"
|
||||||
|
echo "WARNING: migration files run sequentially and are not atomic as a whole; recovery may be required after a partial migration." >&2
|
||||||
|
read_ref_sha() {
|
||||||
|
local branch="$1"
|
||||||
|
curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" |
|
||||||
|
jq -er --arg ref "refs/heads/$branch" '
|
||||||
|
select(type == "array" and length == 1) |
|
||||||
|
.[0] | select(.ref == $ref) | .object.sha |
|
||||||
|
select(test("^[0-9a-f]{40}$"))
|
||||||
|
'
|
||||||
|
}
|
||||||
|
staging_head="$(read_ref_sha staging)"
|
||||||
|
main_head="$(read_ref_sha main)"
|
||||||
|
[[ "$main_head" == "$DEPLOY_SHA" && "$staging_head" == "$DEPLOY_SHA" ]] || {
|
||||||
|
echo "production migration requires main and staging to equal deploy_sha" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ "$GITEA_SHA" == "$DEPLOY_SHA" ]] || {
|
||||||
|
echo "dispatch the production migration workflow from the exact main release SHA" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
runs="$(curl --fail --silent --show-error \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&branch=staging&event=push&status=success&limit=100")"
|
||||||
|
selected_run="$(jq -cer --arg sha "$DEPLOY_SHA" '
|
||||||
|
[.workflow_runs[] | select(
|
||||||
|
(.path | split("@")[0] | endswith("backend-quality-gate.yml")) and
|
||||||
|
.head_sha == $sha and .head_branch == "staging" and
|
||||||
|
.event == "push" and .conclusion == "success"
|
||||||
|
)] | sort_by(.id) | reverse | first
|
||||||
|
' <<<"$runs")"
|
||||||
|
gate_run_id="$(jq -er '.id' <<<"$selected_run")"
|
||||||
|
[[ "$gate_run_id" =~ ^[0-9]+$ ]] || {
|
||||||
|
echo "no successful exact-SHA staging backend quality gate found" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
release_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&event=workflow_dispatch&status=success&limit=100")"
|
||||||
|
jq -e --arg sha "$DEPLOY_SHA" '
|
||||||
|
any(.workflow_runs[]?;
|
||||||
|
(.path | split("@")[0] | endswith("release-quality-gate.yml")) and
|
||||||
|
.head_sha == $sha and .event == "workflow_dispatch" and .conclusion == "success"
|
||||||
|
)
|
||||||
|
' <<<"$release_runs" >/dev/null || {
|
||||||
|
echo "no successful exact-SHA manual release quality gate found" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
observed_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 30 --retry 3 --retry-all-errors \
|
||||||
|
"$STAGING_URL/api/health" | jq -er '.deployment.gitCommit | select(test("^[0-9a-f]{40}$"))')"
|
||||||
|
[[ "$observed_staging_sha" == "$DEPLOY_SHA" ]] || {
|
||||||
|
echo "public staging has not accepted the requested SHA" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
{
|
||||||
|
echo "sha=$DEPLOY_SHA"
|
||||||
|
echo "gate_run_id=$gate_run_id"
|
||||||
|
echo "recovery_reference=$RECOVERY_REFERENCE"
|
||||||
|
echo "recovery_created_at=$RECOVERY_CREATED_AT"
|
||||||
|
echo "restore_verified=true"
|
||||||
|
} >>"$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Prepare pinned Node tooling
|
||||||
|
env:
|
||||||
|
NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c
|
||||||
|
NODE_TOOL_IMAGE: node:22-bookworm-slim
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if ! docker image inspect "$NODE_TOOL_SOURCE_IMAGE" >/dev/null 2>&1; then
|
||||||
|
for attempt in 1 2 3; do
|
||||||
|
if timeout 180 docker pull "$NODE_TOOL_SOURCE_IMAGE"; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if [ "$attempt" -eq 3 ]; then
|
||||||
|
echo "Failed to preload $NODE_TOOL_IMAGE after $attempt attempts" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep $((attempt * 15))
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
docker tag "$NODE_TOOL_SOURCE_IMAGE" "$NODE_TOOL_IMAGE"
|
||||||
|
docker image inspect "$NODE_TOOL_IMAGE" >/dev/null
|
||||||
|
tool_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-node-tools.XXXXXX")"
|
||||||
|
cat > "$tool_dir/node" <<'EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
workdir="$(pwd -P)"
|
||||||
|
exec docker run --rm \
|
||||||
|
--user "$(id -u):$(id -g)" \
|
||||||
|
--volume "$workdir:$workdir" \
|
||||||
|
--workdir "$workdir" \
|
||||||
|
--env HOME=/tmp \
|
||||||
|
node:22-bookworm-slim "${0##*/}" "$@"
|
||||||
|
EOF
|
||||||
|
chmod 0755 "$tool_dir/node"
|
||||||
|
ln -s node "$tool_dir/npm"
|
||||||
|
test -n "${GITHUB_PATH:-}"
|
||||||
|
printf '%s\n' "$tool_dir" >> "$GITHUB_PATH"
|
||||||
|
export PATH="$tool_dir:$PATH"
|
||||||
|
node --version
|
||||||
|
npm --version
|
||||||
|
|
||||||
|
- name: Download gate-produced migration manifest
|
||||||
|
env:
|
||||||
|
GATE_RUN_ID: ${{ steps.revision.outputs.gate_run_id }}
|
||||||
|
DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
artifact_prefix="staging-image-manifest-$DEPLOY_SHA-"
|
||||||
|
artifacts="$(curl --fail --silent --show-error \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs/$GATE_RUN_ID/artifacts?limit=100")"
|
||||||
|
selected_artifact="$(jq -cer --arg prefix "$artifact_prefix" '
|
||||||
|
[(.artifacts // [])[]
|
||||||
|
| select(.expired == false and (.name | startswith($prefix)))
|
||||||
|
| . + {attempt: ((.name | ltrimstr($prefix)) | tonumber?)}
|
||||||
|
| select(.attempt != null and .attempt >= 1)
|
||||||
|
] | sort_by(.attempt, .id) | reverse | first
|
||||||
|
' <<<"$artifacts")"
|
||||||
|
artifact_name="$(jq -er '.name' <<<"$selected_artifact")"
|
||||||
|
artifact_id="$(jq -er '.id' <<<"$selected_artifact")"
|
||||||
|
artifact_attempt="${artifact_name#"$artifact_prefix"}"
|
||||||
|
[[ "$artifact_name" == "$artifact_prefix"* ]]
|
||||||
|
[[ "$artifact_attempt" =~ ^[1-9][0-9]*$ ]]
|
||||||
|
[[ "$artifact_id" =~ ^[0-9]+$ ]]
|
||||||
|
install -d -m 700 artifacts/staging-image
|
||||||
|
curl --fail --silent --show-error --location \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$artifact_id/zip" \
|
||||||
|
--output "${RUNNER_TEMP}/staging-image-manifest.zip"
|
||||||
|
python3 - "${RUNNER_TEMP}/staging-image-manifest.zip" artifacts/staging-image <<'PY'
|
||||||
|
import pathlib, stat, sys, zipfile
|
||||||
|
archive = pathlib.Path(sys.argv[1])
|
||||||
|
destination = pathlib.Path(sys.argv[2])
|
||||||
|
allowed = {"manifest.env", "controller.tar"}
|
||||||
|
with zipfile.ZipFile(archive) as bundle:
|
||||||
|
entries = bundle.infolist()
|
||||||
|
names = [entry.filename for entry in entries]
|
||||||
|
if len(names) != len(set(names)) or set(names) != allowed:
|
||||||
|
raise SystemExit("invalid production migration artifact bundle")
|
||||||
|
if sum(entry.file_size for entry in entries) > 3 * 1024 * 1024:
|
||||||
|
raise SystemExit("production migration artifact bundle is too large")
|
||||||
|
for entry in entries:
|
||||||
|
path = pathlib.PurePosixPath(entry.filename)
|
||||||
|
mode = entry.external_attr >> 16
|
||||||
|
if path.is_absolute() or ".." in path.parts or path.name != entry.filename:
|
||||||
|
raise SystemExit("unsafe production migration artifact path")
|
||||||
|
if mode and not stat.S_ISREG(mode):
|
||||||
|
raise SystemExit("unsafe production migration artifact type")
|
||||||
|
target = destination / entry.filename
|
||||||
|
with bundle.open(entry) as source, target.open("xb") as output:
|
||||||
|
output.write(source.read())
|
||||||
|
PY
|
||||||
|
[[ -f artifacts/staging-image/manifest.env ]]
|
||||||
|
[[ -f artifacts/staging-image/controller.tar ]]
|
||||||
|
|
||||||
|
- name: Validate gate-attested controller and digest-pinned migration image
|
||||||
|
id: image
|
||||||
|
env:
|
||||||
|
DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
manifest=artifacts/staging-image/manifest.env
|
||||||
|
controller_tar=artifacts/staging-image/controller.tar
|
||||||
|
[[ "$(wc -l < "$manifest" | tr -d ' ')" == 4 ]]
|
||||||
|
manifest_sha="$(awk -F= '$1 == "git_sha" {print $2}' "$manifest")"
|
||||||
|
expected_controller_digest="$(awk -F= '$1 == "controller_sha256" {print $2}' "$manifest")"
|
||||||
|
[[ "$manifest_sha" == "$DEPLOY_SHA" && "$expected_controller_digest" =~ ^[0-9a-f]{64}$ ]]
|
||||||
|
printf '%s %s\n' "$expected_controller_digest" "$controller_tar" | sha256sum --check --status
|
||||||
|
python3 - "$controller_tar" <<'PY'
|
||||||
|
import pathlib, sys, tarfile
|
||||||
|
archive = pathlib.Path(sys.argv[1])
|
||||||
|
required = {"deploy/run-production-migration.sh", "frontend/scripts/staging-image-manifest.mjs"}
|
||||||
|
with tarfile.open(archive, "r:") as bundle:
|
||||||
|
members = bundle.getmembers()
|
||||||
|
names = [member.name for member in members]
|
||||||
|
if len(names) != len(set(names)) or not required.issubset(names):
|
||||||
|
raise SystemExit("invalid production migration controller bundle")
|
||||||
|
if sum(member.size for member in members) > 2 * 1024 * 1024:
|
||||||
|
raise SystemExit("production migration controller bundle is too large")
|
||||||
|
for member in members:
|
||||||
|
path = pathlib.PurePosixPath(member.name)
|
||||||
|
if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()):
|
||||||
|
raise SystemExit("unsafe production migration controller bundle")
|
||||||
|
PY
|
||||||
|
install -d -m 700 artifacts/staging-image/extracted
|
||||||
|
tar -xf "$controller_tar" -C artifacts/staging-image/extracted
|
||||||
|
node artifacts/staging-image/extracted/frontend/scripts/staging-image-manifest.mjs \
|
||||||
|
"$manifest" "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Apply production schema migration under pinned SSH identity
|
||||||
|
env:
|
||||||
|
SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }}
|
||||||
|
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||||
|
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
|
DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
|
||||||
|
WEB_IMAGE: ${{ steps.image.outputs.web_image }}
|
||||||
|
RECOVERY_REFERENCE: ${{ steps.revision.outputs.recovery_reference }}
|
||||||
|
RECOVERY_CREATED_AT: ${{ steps.revision.outputs.recovery_created_at }}
|
||||||
|
RESTORE_VERIFIED: ${{ steps.revision.outputs.restore_verified }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
[[ "$DEPLOY_HOST" == "118.194.235.34" ]]
|
||||||
|
[[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 ))
|
||||||
|
[[ "$DEPLOY_USER" == "deploy" ]]
|
||||||
|
[[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]]
|
||||||
|
test -n "$PRODUCTION_KNOWN_HOSTS"
|
||||||
|
ssh_root="${RUNNER_TEMP}/production-migration-ssh"
|
||||||
|
key_path="$ssh_root/id_ed25519"
|
||||||
|
known_hosts_path="$ssh_root/known_hosts"
|
||||||
|
incoming=""
|
||||||
|
install -m 700 -d "$ssh_root"
|
||||||
|
test -n "$SSH_PRIVATE_KEY_BASE64"
|
||||||
|
printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode > "$key_path"
|
||||||
|
printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' > "$known_hosts_path"
|
||||||
|
chmod 600 "$key_path" "$known_hosts_path"
|
||||||
|
ssh-keygen -y -f "$key_path" >/dev/null
|
||||||
|
ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
|
||||||
|
remote="$DEPLOY_USER@$DEPLOY_HOST"
|
||||||
|
require_current_release_heads() {
|
||||||
|
current_staging="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" |
|
||||||
|
jq -er 'select(type == "array" and length == 1) | .[0] |
|
||||||
|
select(.ref == "refs/heads/staging") | .object.sha |
|
||||||
|
select(test("^[0-9a-f]{40}$"))')"
|
||||||
|
current_main="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/main" |
|
||||||
|
jq -er 'select(type == "array" and length == 1) | .[0] |
|
||||||
|
select(.ref == "refs/heads/main") | .object.sha |
|
||||||
|
select(test("^[0-9a-f]{40}$"))')"
|
||||||
|
[[ "$current_main" == "$DEPLOY_SHA" && "$current_staging" == "$DEPLOY_SHA" ]] || {
|
||||||
|
echo "main or staging advanced during production migration; refusing stale mutation" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cleanup() {
|
||||||
|
if [[ -n "$incoming" ]]; then
|
||||||
|
ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; sudo -n rm -rf -- '$incoming'" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
rm -rf -- "$ssh_root"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
incoming="$(ssh "${ssh_options[@]}" "$remote" "mktemp -d /tmp/jyotisha-production-migration.XXXXXXXXXX")"
|
||||||
|
[[ "$incoming" == /tmp/jyotisha-production-migration.* ]]
|
||||||
|
ssh "${ssh_options[@]}" "$remote" "install -d -m 700 '$incoming/.docker'"
|
||||||
|
scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" artifacts/staging-image/controller.tar "$remote:$incoming/controller.tar"
|
||||||
|
ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/controller.tar' -C '$incoming' && rm -f -- '$incoming/controller.tar'"
|
||||||
|
previous_sha="$(ssh "${ssh_options[@]}" "$remote" "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(sudo -n docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-production' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then sudo -n docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; else printf not-deployed; fi; fi")"
|
||||||
|
[[ "$previous_sha" == not-deployed || "$previous_sha" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
||||||
|
forward_verified=false
|
||||||
|
if [[ "$previous_sha" != not-deployed && "$previous_sha" != "$DEPLOY_SHA" ]]; then
|
||||||
|
comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
|
||||||
|
--header "Authorization: token $GITEA_TOKEN" \
|
||||||
|
"$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")"
|
||||||
|
jq -e --arg base "$previous_sha" --arg head "$DEPLOY_SHA" '
|
||||||
|
(.commits // []) as $commits |
|
||||||
|
def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha];
|
||||||
|
def reaches($sha; $seen):
|
||||||
|
if $sha == $base then true
|
||||||
|
elif ($seen | index($sha)) != null then false
|
||||||
|
else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end;
|
||||||
|
(.total_commits | type) == "number" and
|
||||||
|
.total_commits == ($commits | length) and ($commits | length) > 0 and
|
||||||
|
([$commits[].sha] | length == (unique | length)) and reaches($head; [])
|
||||||
|
' <<<"$comparison" >/dev/null || { echo "production migration rollback or divergence refused" >&2; exit 1; }
|
||||||
|
forward_verified=true
|
||||||
|
fi
|
||||||
|
require_current_release_heads
|
||||||
|
printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin"
|
||||||
|
ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' FORWARD_REVISION_VERIFIED='$forward_verified' RECOVERY_REFERENCE='$RECOVERY_REFERENCE' RECOVERY_CREATED_AT='$RECOVERY_CREATED_AT' RESTORE_VERIFIED='$RESTORE_VERIFIED' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' bash '$incoming/deploy/run-production-migration.sh'"
|
||||||
|
require_current_release_heads
|
||||||
|
|
||||||
|
- name: Operator action
|
||||||
|
run: echo 'Schema migration complete. Production ETL and application deployment remain separate manual operations.'
|
||||||
@@ -81,6 +81,8 @@ SELECT format(
|
|||||||
SELECT 1 FROM pg_roles WHERE rolname = 'backup_reader'
|
SELECT 1 FROM pg_roles WHERE rolname = 'backup_reader'
|
||||||
) \gexec
|
) \gexec
|
||||||
|
|
||||||
|
GRANT schema_owner TO migration_runner;
|
||||||
|
|
||||||
SELECT format(
|
SELECT format(
|
||||||
'GRANT CONNECT, CREATE ON DATABASE %I TO schema_owner',
|
'GRANT CONNECT, CREATE ON DATABASE %I TO schema_owner',
|
||||||
:'database_name'
|
:'database_name'
|
||||||
|
|||||||
Executable
+200
@@ -0,0 +1,200 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
set +x
|
||||||
|
|
||||||
|
required=(
|
||||||
|
INCOMING_PATH DEPLOY_PATH WEB_IMAGE DEPLOY_SHA EXPECTED_PREVIOUS_SHA
|
||||||
|
RECOVERY_REFERENCE RECOVERY_CREATED_AT RESTORE_VERIFIED DOCKER_CONFIG
|
||||||
|
)
|
||||||
|
case "${DOCKER_BIN:-docker}" in
|
||||||
|
docker) docker_command=(docker) ;;
|
||||||
|
"sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;;
|
||||||
|
*) echo "unsafe production Docker command" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
for key in "${required[@]}"; do
|
||||||
|
if [ -z "${!key:-}" ]; then
|
||||||
|
echo "required production migration input is missing: $key" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || {
|
||||||
|
echo "unsafe production migration revision" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
image_pattern='^crpi-d1feco6itet73spp\.cn-hongkong\.personal\.cr\.aliyuncs\.com/copse/jyotisha@sha256:[0-9a-f]{64}$'
|
||||||
|
[[ "$WEB_IMAGE" =~ $image_pattern ]] || {
|
||||||
|
echo "unsafe production migration image" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
case "$INCOMING_PATH" in
|
||||||
|
/tmp/jyotisha-production-migration.*) ;;
|
||||||
|
*) echo "unsafe incoming production migration path" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
|
||||||
|
echo "unsafe production deployment path" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[ "$DOCKER_CONFIG" = "$INCOMING_PATH/.docker" ] || {
|
||||||
|
echo "unsafe production Docker configuration path" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ "$RECOVERY_REFERENCE" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}$ ]] || {
|
||||||
|
echo "unsafe production recovery reference" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ "$RECOVERY_CREATED_AT" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || {
|
||||||
|
echo "unsafe production recovery creation time" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[ "$RESTORE_VERIFIED" = "true" ] || {
|
||||||
|
echo "production recovery point must have restore_verified=true" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
recovery_created_epoch="$(date -u -d "$RECOVERY_CREATED_AT" +%s 2>/dev/null)" || {
|
||||||
|
echo "invalid production recovery creation time" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
recovery_now_epoch="$(date -u +%s)"
|
||||||
|
recovery_age_seconds=$((recovery_now_epoch - recovery_created_epoch))
|
||||||
|
(( recovery_age_seconds >= 0 && recovery_age_seconds <= 24 * 60 * 60 )) || {
|
||||||
|
echo "production recovery point must be no more than 24 hours old and not in the future" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT restore_verified=true"
|
||||||
|
echo "WARNING: migration files run sequentially and are not atomic as a whole; recovery may be required after a partial migration." >&2
|
||||||
|
|
||||||
|
state_directory="$DEPLOY_PATH/.state"
|
||||||
|
install -d -m 700 "$state_directory"
|
||||||
|
exec 9>"$state_directory/mutation.lock"
|
||||||
|
flock -n 9 || {
|
||||||
|
echo "another production mutation holds the host lock" >&2
|
||||||
|
exit 75
|
||||||
|
}
|
||||||
|
|
||||||
|
current_sha="not-deployed"
|
||||||
|
if [ -f "$state_directory/deployed-revision" ]; then
|
||||||
|
current_sha="$(<"$state_directory/deployed-revision")"
|
||||||
|
else
|
||||||
|
existing_web="$("${docker_command[@]}" ps -aq \
|
||||||
|
--filter 'label=com.docker.compose.project=jyotisha-production' \
|
||||||
|
--filter 'label=com.docker.compose.service=web' | head -n 1)"
|
||||||
|
if [ -n "$existing_web" ]; then
|
||||||
|
discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
|
||||||
|
"$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)"
|
||||||
|
if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
||||||
|
echo "invalid deployed production revision state" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
[ "$current_sha" = "$EXPECTED_PREVIOUS_SHA" ] || {
|
||||||
|
echo "production revision changed while this migration was waiting" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[ "$current_sha" = "not-deployed" ] ||
|
||||||
|
[ "$current_sha" = "$DEPLOY_SHA" ] ||
|
||||||
|
[ "${FORWARD_REVISION_VERIFIED:-false}" = "true" ] || {
|
||||||
|
echo "forward production revision was not verified" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
bash "$INCOMING_PATH/deploy/sync-production-tree.sh" \
|
||||||
|
"$INCOMING_PATH" "$DEPLOY_PATH"
|
||||||
|
|
||||||
|
cd "$DEPLOY_PATH"
|
||||||
|
EXPECTED_PRODUCTION_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")"
|
||||||
|
[[ "$EXPECTED_PRODUCTION_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || {
|
||||||
|
echo "production deployment owner is invalid" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
export EXPECTED_PRODUCTION_ENV_OWNER_UID
|
||||||
|
bash deploy/validate-production-env.sh .env.production
|
||||||
|
bash deploy/validate-production-database-env.sh .env.production.database
|
||||||
|
|
||||||
|
export DATABASE_ENV_FILE='../.env.production.database'
|
||||||
|
compose=("${docker_command[@]}" compose -p jyotisha-production -f deploy/docker-compose.postgres.yml)
|
||||||
|
"${docker_command[@]}" pull "$WEB_IMAGE"
|
||||||
|
"${compose[@]}" config --quiet
|
||||||
|
"${compose[@]}" up -d --no-build --pull never --wait postgres
|
||||||
|
|
||||||
|
membership="$("${compose[@]}" exec -T postgres psql -v ON_ERROR_STOP=1 -U postgres -d jyotisha -Atc \
|
||||||
|
"select pg_has_role('migration_runner', 'schema_owner', 'member')")"
|
||||||
|
[ "$membership" = "t" ] || {
|
||||||
|
echo "migration_runner must be allowed to SET ROLE schema_owner before production migration" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
environment_value() {
|
||||||
|
local key="$1"
|
||||||
|
local value
|
||||||
|
value="$(sed -n -E "s/^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=[[:space:]]*(.*)$/\\2/p" .env.production.database)"
|
||||||
|
case "$value" in
|
||||||
|
\"*\") value="${value:1:${#value}-2}" ;;
|
||||||
|
\'*\') value="${value:1:${#value}-2}" ;;
|
||||||
|
esac
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
percent_encode() {
|
||||||
|
local value="$1"
|
||||||
|
local encoded=""
|
||||||
|
local character hex index
|
||||||
|
LC_ALL=C
|
||||||
|
for ((index = 0; index < ${#value}; index += 1)); do
|
||||||
|
character="${value:index:1}"
|
||||||
|
case "$character" in
|
||||||
|
[a-zA-Z0-9.~_-]) encoded+="$character" ;;
|
||||||
|
*)
|
||||||
|
printf -v hex '%%%02X' "'$character"
|
||||||
|
encoded+="$hex"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
printf '%s' "$encoded"
|
||||||
|
}
|
||||||
|
|
||||||
|
migration_runner_password="$(environment_value MIGRATION_RUNNER_PASSWORD)"
|
||||||
|
[ -n "$migration_runner_password" ] || {
|
||||||
|
echo "production migration runner password is missing" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
encoded_migration_runner_password="$(percent_encode "$migration_runner_password")"
|
||||||
|
unset migration_runner_password
|
||||||
|
migration_runner_database_url="postgresql://migration_runner:${encoded_migration_runner_password}@postgres:5432/jyotisha?options=-c%20role%3Dschema_owner"
|
||||||
|
unset encoded_migration_runner_password
|
||||||
|
|
||||||
|
migration_environment="$(mktemp "$state_directory/production-migration-env.XXXXXXXXXX")"
|
||||||
|
cleanup_migration_environment() {
|
||||||
|
rm -f -- "$migration_environment"
|
||||||
|
}
|
||||||
|
trap cleanup_migration_environment EXIT
|
||||||
|
chmod 600 "$migration_environment"
|
||||||
|
printf 'SCHEMA_DATABASE_URL=%s\n' "$migration_runner_database_url" >"$migration_environment"
|
||||||
|
unset migration_runner_database_url
|
||||||
|
|
||||||
|
set +e
|
||||||
|
DATABASE_ENV_FILE="$migration_environment" \
|
||||||
|
"${compose[@]}" --profile migration-check run --rm migration-checker
|
||||||
|
precheck_status=$?
|
||||||
|
set -e
|
||||||
|
if [ "$precheck_status" -ne 0 ] && [ "$precheck_status" -ne 3 ]; then
|
||||||
|
echo "production migration precheck failed safely" >&2
|
||||||
|
exit "$precheck_status"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DATABASE_ENV_FILE="$migration_environment" \
|
||||||
|
"${compose[@]}" --profile migration run --rm migrator
|
||||||
|
|
||||||
|
set +e
|
||||||
|
DATABASE_ENV_FILE="$migration_environment" \
|
||||||
|
"${compose[@]}" --profile migration-check run --rm migration-checker
|
||||||
|
postcheck_status=$?
|
||||||
|
set -e
|
||||||
|
if [ "$postcheck_status" -ne 0 ]; then
|
||||||
|
echo "production migration postcheck did not converge" >&2
|
||||||
|
exit "$postcheck_status"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "production schema migration verified for $DEPLOY_SHA"
|
||||||
@@ -13,7 +13,7 @@ This is not a volume copy. A full Supabase dump must not be restored over the ta
|
|||||||
|
|
||||||
## Release invariants
|
## Release invariants
|
||||||
|
|
||||||
The production workflow is manual-only and accepts a full lowercase 40-character `deploy_sha`. A normal deployment proceeds only when all of the following identify that exact SHA:
|
The production deployment and schema-migration workflows are manual-only and accept a full lowercase 40-character `deploy_sha`. A normal production mutation proceeds only when all of the following identify that exact SHA:
|
||||||
|
|
||||||
1. current `main`;
|
1. current `main`;
|
||||||
2. current `staging`;
|
2. current `staging`;
|
||||||
@@ -21,7 +21,7 @@ The production workflow is manual-only and accepts a full lowercase 40-character
|
|||||||
4. a successful manually triggered `Jyotish Release Quality Gate`;
|
4. a successful manually triggered `Jyotish Release Quality Gate`;
|
||||||
5. the public staging `/api/health` deployment identity.
|
5. the public staging `/api/health` deployment identity.
|
||||||
|
|
||||||
The workflow consumes the exact API and Web image digests recorded by the staging gate. It does not build on the 2-core/4-GB production host, import user data, run schema migrations, or change DNS.
|
The deployment workflow consumes the exact API and Web image digests recorded by the staging gate. It does not build on the 2-core/4-GB production host, import user data, run schema migrations, or change DNS. The separate `Migrate Production Database` workflow uses the same gate-attested Web image only to run the schema checker/migrator/checker sequence; it does not run ETL, deploy the application, or change DNS. Both workflows share the `production-mutation` lock.
|
||||||
|
|
||||||
## Required Gitea configuration
|
## Required Gitea configuration
|
||||||
|
|
||||||
@@ -80,7 +80,13 @@ Use distinct production credentials for PostgreSQL roles, Better Auth, Resend, b
|
|||||||
|
|
||||||
## Database migration engineering gate
|
## Database migration engineering gate
|
||||||
|
|
||||||
Before production cutover, implement and review `frontend/scripts/migrate-supabase-production.mjs` with `--preflight`, `--apply`, and `--verify` modes. Until that tool and its fixtures pass, production data cutover is blocked.
|
Before importing data, dispatch Gitea Actions → `Migrate Production Database` for the exact accepted release SHA. The workflow requires `main == staging == deploy_sha`, the same successful staging backend gate, the same manual release gate, and the public staging `/api/health` identity for that SHA. It also requires a non-sensitive recovery reference, its exact UTC creation time, and `restore_verified=true`; the recovery point must be no more than 24 hours old and must already have passed a restore verification. It verifies the current production revision, obtains the gate-attested immutable Web image, runs the schema checker, applies only pending application schema migrations, and requires the checker to converge afterward.
|
||||||
|
|
||||||
|
Schema migration files are committed sequentially and are not one atomic transaction as a set. If a later file or post-check fails, earlier files may remain applied; stop, preserve evidence, and restore from the attested recovery point when repair-in-place is not explicitly reviewed. Do not assume a failed workflow means the database is unchanged.
|
||||||
|
|
||||||
|
The production database must already grant `migration_runner` membership in `schema_owner`; `deploy/postgres/001-bootstrap-roles.sh` grants only that migration role the ability to `SET ROLE schema_owner`. Identity, app, service, admin, and backup runtime roles must not receive this membership. The workflow checks the membership and refuses to add it itself.
|
||||||
|
|
||||||
|
The reviewed data-transfer entry point is `frontend/scripts/migrate-supabase-production.mjs`. It has separate `--preflight`, `--apply`, and `--verify` modes; the application deployment workflow never runs it automatically. Production cutover remains blocked until the exact production snapshot has completed an isolated rehearsal and final verification.
|
||||||
|
|
||||||
The tool must:
|
The tool must:
|
||||||
|
|
||||||
@@ -96,6 +102,24 @@ The tool must:
|
|||||||
- emit only counts, state aggregates, and normalized SHA-256 manifests—not email addresses, birth data, tokens, or connection strings;
|
- emit only counts, state aggregates, and normalized SHA-256 manifests—not email addresses, birth data, tokens, or connection strings;
|
||||||
- run all target writes in a transaction and roll back on failure.
|
- run all target writes in a transaction and roll back on failure.
|
||||||
|
|
||||||
|
The operator supplies these values only on the trusted migration host; do not store the database URLs or encryption keys in Gitea:
|
||||||
|
|
||||||
|
- `SUPABASE_SOURCE_DATABASE_URL`: the consistent read-only Supabase snapshot/source URL;
|
||||||
|
- `PRODUCTION_TARGET_DATABASE_URL`: the PostgreSQL 17 target URL using the migration role;
|
||||||
|
- `PRODUCTION_OWNER_USER_ID`: the UUID of the designated active source administrator;
|
||||||
|
- `PRODUCTION_CIPHERTEXT_MODE=preserve|exclude`; preserve additionally requires `PRODUCTION_CIPHERTEXT_KEYS_CONFIRMED=true`.
|
||||||
|
|
||||||
|
Run each phase separately and retain its redacted JSON manifest:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd frontend
|
||||||
|
node scripts/migrate-supabase-production.mjs --preflight
|
||||||
|
node scripts/migrate-supabase-production.mjs --apply
|
||||||
|
node scripts/migrate-supabase-production.mjs --verify
|
||||||
|
```
|
||||||
|
|
||||||
|
`--apply` is intentionally one-shot: it refuses a populated target. If apply fails, discard or restore the isolated target, correct the cause, and rerun from an empty migrated schema rather than improvising a partial resume.
|
||||||
|
|
||||||
Do not import platform schemas, source roles/grants, Supabase migration ledgers, sessions, refresh tokens, or provider tokens. Do not use a full-database `pg_restore` against the target.
|
Do not import platform schemas, source roles/grants, Supabase migration ledgers, sessions, refresh tokens, or provider tokens. Do not use a full-database `pg_restore` against the target.
|
||||||
|
|
||||||
Because migrated users have no portable password/session, all sessions are invalidated and users sign in again through email OTP. Administrators re-enrol MFA.
|
Because migrated users have no portable password/session, all sessions are invalidated and users sign in again through email OTP. Administrators re-enrol MFA.
|
||||||
@@ -104,7 +128,7 @@ Because migrated users have no portable password/session, all sessions are inval
|
|||||||
|
|
||||||
Complete at least one isolated full-data rehearsal before scheduling the final window:
|
Complete at least one isolated full-data rehearsal before scheduling the final window:
|
||||||
|
|
||||||
1. Apply all target schema migrations to an empty rehearsal database.
|
1. Apply all target schema migrations to an empty rehearsal database using the same schema migrator path as `Migrate Production Database`.
|
||||||
2. Run migration preflight, apply, post-import reconciliation, and verify.
|
2. Run migration preflight, apply, post-import reconciliation, and verify.
|
||||||
3. Verify source/target row counts, primary-key set hashes, normalized row hashes, credit totals, payment state totals, subscriptions, reports, consultations, and rectification records.
|
3. Verify source/target row counts, primary-key set hashes, normalized row hashes, credit totals, payment state totals, subscriptions, reports, consultations, and rectification records.
|
||||||
4. Verify one Owner exists, every active admin has a target role, and database roles remain isolated.
|
4. Verify one Owner exists, every active admin has a target role, and database roles remain isolated.
|
||||||
@@ -135,6 +159,8 @@ Both `jyotisha.chat` and `admin.jyotisha.chat` are required. The application rej
|
|||||||
- Confirm the exact release SHA is deployed and accepted on staging.
|
- Confirm the exact release SHA is deployed and accepted on staging.
|
||||||
- Run the manual release quality gate for that SHA.
|
- Run the manual release quality gate for that SHA.
|
||||||
- Confirm final backup capacity, restore rehearsal, SMTP/OTP delivery, and rollback contacts.
|
- Confirm final backup capacity, restore rehearsal, SMTP/OTP delivery, and rollback contacts.
|
||||||
|
- Create and restore-verify a production recovery point no more than 24 hours before the schema migration; record its non-sensitive reference and UTC creation time.
|
||||||
|
- Dispatch `Migrate Production Database` for the accepted SHA with that recovery attestation and confirm its post-check reports no pending schema migrations.
|
||||||
- Record pending payment orders and long-running jobs; choose an explicit disposition for each.
|
- Record pending payment orders and long-running jobs; choose an explicit disposition for each.
|
||||||
- Dispatch `Deploy production` with `verification_mode=internal` only after target schema/data preparation. This verifies the new host without depending on public DNS.
|
- Dispatch `Deploy production` with `verification_mode=internal` only after target schema/data preparation. This verifies the new host without depending on public DNS.
|
||||||
|
|
||||||
@@ -189,7 +215,9 @@ Normal release:
|
|||||||
|
|
||||||
1. Merge the reviewed `staging` release into `main` so both heads are the same SHA.
|
1. Merge the reviewed `staging` release into `main` so both heads are the same SHA.
|
||||||
2. Confirm the staging push gate, public staging SHA, and manual release gate all succeeded for that SHA.
|
2. Confirm the staging push gate, public staging SHA, and manual release gate all succeeded for that SHA.
|
||||||
3. Open Gitea Actions → `Deploy production`.
|
3. Open Gitea Actions → `Migrate Production Database`; enter the exact 40-character SHA, the no-more-than-24-hour-old recovery reference and UTC creation time, and confirm `restore_verified=true`. Wait for the post-migration checker to converge. Do not use this workflow for Supabase ETL.
|
||||||
4. Enter the exact 40-character SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase.
|
4. Run the trusted-host ETL phases and retain the redacted reconciliation manifests.
|
||||||
|
5. Open Gitea Actions → `Deploy production`.
|
||||||
|
6. Enter the same exact SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase.
|
||||||
|
|
||||||
Application rollback accepts only an explicitly authorized, previously gate-attested SHA in reviewed `main` history. Database migrations and imported data are not rolled back by the application workflow.
|
Application rollback accepts only an explicitly authorized, previously gate-attested SHA in reviewed `main` history. Database migrations and imported data are not rolled back by the application workflow.
|
||||||
|
|||||||
@@ -251,3 +251,9 @@ Prevention: keep evidence-request, life-event, private-candidate, public-recap,
|
|||||||
After accumulated historical evidence produced a very narrow winning segment, that segment could contain fewer than two linked samples or discriminating divisional themes. Packet construction treated this valid “not enough distinction yet” state as a dependency failure, so a later answer returned 503 even though scoring and the astrology service were healthy.
|
After accumulated historical evidence produced a very narrow winning segment, that segment could contain fewer than two linked samples or discriminating divisional themes. Packet construction treated this valid “not enough distinction yet” state as a dependency failure, so a later answer returned 503 even though scoring and the astrology service were healthy.
|
||||||
|
|
||||||
Prevention: classify insufficient candidate-range discrimination explicitly; when a newly narrowed segment cannot support the technical evidence contract, retain the prior candidate range, preserve scored evidence, clear the unconfirmed result, and continue conversational collection.
|
Prevention: classify insufficient candidate-range discrimination explicitly; when a newly narrowed segment cannot support the technical evidence contract, retain the prior candidate range, preserve scored evidence, clear the unconfirmed result, and continue conversational collection.
|
||||||
|
|
||||||
|
## ERR-102 | Gitea returned HTTP 502 during final remote synchronization check | active 2026-08-09
|
||||||
|
|
||||||
|
Two final `git fetch origin --prune` attempts against the configured primary Gitea remote failed before ref exchange with HTTP `502`. The last locally verified refs remain available, but this run cannot prove that they are still current and must not claim a completed remote synchronization or push.
|
||||||
|
|
||||||
|
Prevention: retry fetch and `git ls-remote` before any push or release action, compare the full `main`, `staging`, and migration-branch SHAs, and stop if Gitea remains unavailable. Do not substitute cached refs, the GitHub mirror, or a successful local commit for current Gitea synchronization evidence.
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -56,6 +56,24 @@ test("database roles have no cluster privileges", () => {
|
|||||||
`),
|
`),
|
||||||
"t",
|
"t",
|
||||||
);
|
);
|
||||||
|
assert.equal(
|
||||||
|
fixture.psql(`
|
||||||
|
select role_name || ':' || case when pg_has_role(role_name, 'schema_owner', 'MEMBER') then 'true' else 'false' end
|
||||||
|
from unnest(array[
|
||||||
|
'migration_runner', 'identity_runtime', 'app_runtime',
|
||||||
|
'service_runtime', 'admin_runtime', 'backup_reader'
|
||||||
|
]) role_name
|
||||||
|
order by role_name
|
||||||
|
`),
|
||||||
|
[
|
||||||
|
"admin_runtime:f",
|
||||||
|
"app_runtime:f",
|
||||||
|
"backup_reader:f",
|
||||||
|
"identity_runtime:f",
|
||||||
|
"migration_runner:true",
|
||||||
|
"service_runtime:f",
|
||||||
|
].join("\n"),
|
||||||
|
);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
fixture.psql(`
|
fixture.psql(`
|
||||||
select coalesce(string_agg(privilege_type, ',' order by privilege_type), '')
|
select coalesce(string_agg(privilege_type, ',' order by privilege_type), '')
|
||||||
|
|||||||
@@ -437,3 +437,92 @@ test("production API probes health rapidly while a replacement container starts"
|
|||||||
|
|
||||||
assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/);
|
assert.match(compose, /healthcheck:[\s\S]*start_period:\s*30s[\s\S]*start_interval:\s*1s/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("self-hosted production Caddy isolates user and admin hosts", () => {
|
||||||
|
const caddy = readFileSync(
|
||||||
|
new URL("../../deploy/Caddyfile.production.selfhosted", import.meta.url),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/jyotisha\.chat\}/);
|
||||||
|
assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/);
|
||||||
|
assert.match(caddy, /respond @adminPaths "Not found" 404/);
|
||||||
|
assert.match(caddy, /^https:\/\/admin\.jyotisha\.chat \{$/m);
|
||||||
|
assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2);
|
||||||
|
assert.match(caddy, /@root path \/\n\s+redir @root \/admin 308/);
|
||||||
|
assert.doesNotMatch(caddy, /staging\.jyotisha\.chat|:443 \{/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("production env validators accept only self-hosted production selectors and role URLs", () => {
|
||||||
|
const appValidator = fileURLToPath(
|
||||||
|
new URL("../../deploy/validate-production-env.sh", import.meta.url),
|
||||||
|
);
|
||||||
|
const databaseValidator = fileURLToPath(
|
||||||
|
new URL("../../deploy/validate-production-database-env.sh", import.meta.url),
|
||||||
|
);
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "jyotisha-production-env-"));
|
||||||
|
const appEnv = join(root, ".env.production");
|
||||||
|
const databaseEnv = join(root, ".env.production.database");
|
||||||
|
const appLines = [
|
||||||
|
"APP_ENV_FILE=../.env.production",
|
||||||
|
"CADDYFILE_PATH=./Caddyfile.production.selfhosted",
|
||||||
|
"SITE_ADDRESS=https://jyotisha.chat",
|
||||||
|
"AUTH_PROVIDER=self-hosted",
|
||||||
|
"SELF_HOSTED_IDENTITY_ENABLED=true",
|
||||||
|
"AUTH_USER_ORIGIN=https://jyotisha.chat",
|
||||||
|
"ADMIN_USER_ORIGIN=https://admin.jyotisha.chat",
|
||||||
|
"IDENTITY_DATABASE_URL=postgresql://identity_runtime:identity-runtime-test-password@postgres:5432/jyotisha",
|
||||||
|
"APP_DATABASE_URL=postgresql://app_runtime:app-runtime-test-password@postgres:5432/jyotisha",
|
||||||
|
"SERVICE_DATABASE_URL=postgresql://service_runtime:service-runtime-test-password@postgres:5432/jyotisha",
|
||||||
|
"ADMIN_DATABASE_URL=postgresql://admin_runtime:admin-runtime-test-password@postgres:5432/jyotisha",
|
||||||
|
"BETTER_AUTH_USER_SECRET=user-secret-that-is-at-least-32-bytes-long",
|
||||||
|
"RESEND_API_KEY=re_test_key_that_must_not_be_printed",
|
||||||
|
"RESEND_FROM_EMAIL=Jyotisha Production <login@jyotisha.chat>",
|
||||||
|
"ADMIN_EMAILS=admin@example.com",
|
||||||
|
"EPAY_CONFIG_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
||||||
|
"MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY=BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=",
|
||||||
|
"EPAY_CHAT_ENABLED=false",
|
||||||
|
"JYOTISH_DYNAMIC_RECTIFICATION_TOKEN=dynamic-token-that-is-at-least-32-bytes",
|
||||||
|
"PERSONAL_REPORT_ENABLED=true",
|
||||||
|
"PERSONAL_REPORT_DAILY_LIMIT=5",
|
||||||
|
];
|
||||||
|
const databaseLines = [
|
||||||
|
"POSTGRES_DB=jyotisha",
|
||||||
|
"POSTGRES_USER=postgres",
|
||||||
|
"POSTGRES_PASSWORD=postgres-password",
|
||||||
|
"SCHEMA_OWNER_PASSWORD=schema-owner-password",
|
||||||
|
"IDENTITY_RUNTIME_PASSWORD=identity-runtime-password",
|
||||||
|
"APP_RUNTIME_PASSWORD=app-runtime-password",
|
||||||
|
"SERVICE_RUNTIME_PASSWORD=service-runtime-password",
|
||||||
|
"ADMIN_RUNTIME_PASSWORD=admin-runtime-password",
|
||||||
|
"MIGRATION_RUNNER_PASSWORD=migration-runner-password",
|
||||||
|
"BACKUP_READER_PASSWORD=backup-reader-password",
|
||||||
|
"PRODUCTION_BACKUP_ENCRYPTION_KEY=backup-encryption-key",
|
||||||
|
"SCHEMA_DATABASE_URL=postgresql://schema_owner:schema-owner-password@postgres:5432/jyotisha",
|
||||||
|
];
|
||||||
|
const writeEnv = (path: string, lines: string[]) => {
|
||||||
|
writeFileSync(path, `${lines.join("\n")}\n`);
|
||||||
|
chmodSync(path, 0o600);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
writeEnv(appEnv, appLines);
|
||||||
|
writeEnv(databaseEnv, databaseLines);
|
||||||
|
assert.equal(spawnSync("bash", [appValidator, appEnv], { encoding: "utf8" }).status, 0);
|
||||||
|
assert.equal(spawnSync("bash", [databaseValidator, databaseEnv], { encoding: "utf8" }).status, 0);
|
||||||
|
|
||||||
|
writeEnv(appEnv, appLines.map((line) =>
|
||||||
|
line.startsWith("ADMIN_USER_ORIGIN=")
|
||||||
|
? "ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat"
|
||||||
|
: line,
|
||||||
|
));
|
||||||
|
assert.notEqual(spawnSync("bash", [appValidator, appEnv], { encoding: "utf8" }).status, 0);
|
||||||
|
|
||||||
|
writeEnv(databaseEnv, databaseLines.filter((line) =>
|
||||||
|
!line.startsWith("PRODUCTION_BACKUP_ENCRYPTION_KEY="),
|
||||||
|
));
|
||||||
|
assert.notEqual(spawnSync("bash", [databaseValidator, databaseEnv], { encoding: "utf8" }).status, 0);
|
||||||
|
} finally {
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,277 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import test from "node:test";
|
||||||
|
|
||||||
|
import {
|
||||||
|
SafeProductionMigrationError,
|
||||||
|
assertActiveAdminUsers,
|
||||||
|
assertTargetEmpty,
|
||||||
|
normalizeAuthUser,
|
||||||
|
normalizeAuthUsers,
|
||||||
|
parseMode,
|
||||||
|
readConfiguration,
|
||||||
|
readSchema,
|
||||||
|
rowsSha256,
|
||||||
|
transferPlan,
|
||||||
|
} from "../scripts/migrate-supabase-production.mjs";
|
||||||
|
|
||||||
|
const scriptPath = fileURLToPath(
|
||||||
|
new URL("../scripts/migrate-supabase-production.mjs", import.meta.url),
|
||||||
|
);
|
||||||
|
|
||||||
|
function table(
|
||||||
|
columns: Array<{
|
||||||
|
name: string;
|
||||||
|
nullable?: boolean;
|
||||||
|
defaultValue?: string | null;
|
||||||
|
generated?: boolean;
|
||||||
|
identity?: boolean;
|
||||||
|
identityGeneration?: string | null;
|
||||||
|
}>,
|
||||||
|
primaryKey = ["id"],
|
||||||
|
foreignKeys: Array<{
|
||||||
|
columns: string[];
|
||||||
|
refSchema: string;
|
||||||
|
refTable: string;
|
||||||
|
refColumns: string[];
|
||||||
|
}> = [],
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
columns: columns.map((column) => ({
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: null,
|
||||||
|
generated: false,
|
||||||
|
identity: false,
|
||||||
|
identityGeneration: null,
|
||||||
|
...column,
|
||||||
|
})),
|
||||||
|
primaryKey,
|
||||||
|
foreignKeys,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test("CLI requires one explicit migration mode", () => {
|
||||||
|
assert.equal(parseMode(["--preflight"]), "preflight");
|
||||||
|
assert.equal(parseMode(["--apply"]), "apply");
|
||||||
|
assert.equal(parseMode(["--verify"]), "verify");
|
||||||
|
assert.throws(() => parseMode([]), SafeProductionMigrationError);
|
||||||
|
assert.throws(() => parseMode(["--apply", "--verify"]), SafeProductionMigrationError);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("configuration requires an explicit Owner and ciphertext decision", () => {
|
||||||
|
const base = {
|
||||||
|
SUPABASE_SOURCE_DATABASE_URL: "postgresql://source.invalid/jyotisha",
|
||||||
|
PRODUCTION_TARGET_DATABASE_URL: "postgresql://target.invalid/jyotisha",
|
||||||
|
PRODUCTION_OWNER_USER_ID: "018f4e6d-7a11-7000-8000-000000000001",
|
||||||
|
};
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
readConfiguration({ ...base, PRODUCTION_CIPHERTEXT_MODE: "exclude" }).ciphertextMode,
|
||||||
|
"exclude",
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => readConfiguration({ ...base, PRODUCTION_CIPHERTEXT_MODE: "preserve" }),
|
||||||
|
/confirmed production encryption keys/,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
readConfiguration({
|
||||||
|
...base,
|
||||||
|
PRODUCTION_CIPHERTEXT_MODE: "preserve",
|
||||||
|
PRODUCTION_CIPHERTEXT_KEYS_CONFIRMED: "true",
|
||||||
|
}).ciphertextMode,
|
||||||
|
"preserve",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Supabase identity transform preserves UUID and ban state without credentials", () => {
|
||||||
|
const source = {
|
||||||
|
id: "018F4E6D-7A11-7000-8000-000000000001",
|
||||||
|
email: " Person@Example.com ",
|
||||||
|
raw_user_meta_data: { full_name: "Person One", avatar_url: "https://example.invalid/a.png" },
|
||||||
|
email_confirmed_at: new Date("2026-07-01T00:00:00Z"),
|
||||||
|
banned_until: new Date("2027-01-01T00:00:00Z"),
|
||||||
|
created_at: new Date("2026-06-01T00:00:00Z"),
|
||||||
|
updated_at: new Date("2026-07-02T00:00:00Z"),
|
||||||
|
encrypted_password: "must-not-migrate",
|
||||||
|
refresh_token: "must-not-migrate",
|
||||||
|
mfa_secret: "must-not-migrate",
|
||||||
|
};
|
||||||
|
const user = normalizeAuthUser(source, new Date("2026-08-09T00:00:00Z"));
|
||||||
|
|
||||||
|
assert.equal(user.id, source.id.toLowerCase());
|
||||||
|
assert.equal(user.email, "person@example.com");
|
||||||
|
assert.equal(user.banned, true);
|
||||||
|
assert.deepEqual(user.ban_expires, source.banned_until);
|
||||||
|
assert.equal(user.two_factor_enabled, false);
|
||||||
|
assert.doesNotMatch(JSON.stringify(user), /must-not-migrate|password|refresh_token|mfa_secret/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("production identity preflight rejects duplicate canonical emails", () => {
|
||||||
|
const base = {
|
||||||
|
id: "018f4e6d-7a11-7000-8000-000000000001",
|
||||||
|
email: "person@example.com",
|
||||||
|
raw_user_meta_data: {},
|
||||||
|
email_confirmed_at: null,
|
||||||
|
created_at: new Date("2026-06-01T00:00:00Z"),
|
||||||
|
updated_at: new Date("2026-06-01T00:00:00Z"),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert.throws(
|
||||||
|
() => normalizeAuthUsers([
|
||||||
|
base,
|
||||||
|
{ ...base, id: "018f4e6d-7a11-7000-8000-000000000002", email: " PERSON@example.com " },
|
||||||
|
]),
|
||||||
|
/invalid or duplicate auth identities/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("active administrators become usable identity admins", () => {
|
||||||
|
const ownerId = "018f4e6d-7a11-7000-8000-000000000001";
|
||||||
|
const adminId = "018f4e6d-7a11-7000-8000-000000000002";
|
||||||
|
const sourceUsers = [ownerId, adminId].map((id) => ({
|
||||||
|
id,
|
||||||
|
email: `${id}@example.com`,
|
||||||
|
raw_user_meta_data: {},
|
||||||
|
email_confirmed_at: null,
|
||||||
|
banned_until: null,
|
||||||
|
created_at: new Date("2026-06-01T00:00:00Z"),
|
||||||
|
updated_at: new Date("2026-06-01T00:00:00Z"),
|
||||||
|
}));
|
||||||
|
const activeAdminUserIds = new Set([ownerId, adminId]);
|
||||||
|
const users = normalizeAuthUsers(
|
||||||
|
sourceUsers,
|
||||||
|
new Date("2026-08-09T00:00:00Z"),
|
||||||
|
activeAdminUserIds,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(users.map((user) => user.role), ["admin", "admin"]);
|
||||||
|
assert.doesNotThrow(() => assertActiveAdminUsers(users, activeAdminUserIds, ownerId));
|
||||||
|
assert.throws(
|
||||||
|
() => assertActiveAdminUsers(users, new Set([adminId]), ownerId),
|
||||||
|
/Owner is not an active source administrator/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => assertActiveAdminUsers([{ ...users[0], banned: true }, users[1]], activeAdminUserIds, ownerId),
|
||||||
|
/Owner is blocked/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("schema reader preserves PostgreSQL identity metadata", async () => {
|
||||||
|
const client = {
|
||||||
|
async query(text: string) {
|
||||||
|
if (text.includes("information_schema.columns")) {
|
||||||
|
return {
|
||||||
|
rows: [{
|
||||||
|
table_name: "redemption_attempts",
|
||||||
|
column_name: "id",
|
||||||
|
nullable: false,
|
||||||
|
column_default: null,
|
||||||
|
generated: false,
|
||||||
|
identity: true,
|
||||||
|
identity_generation: "ALWAYS",
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (text.includes("PRIMARY KEY")) {
|
||||||
|
return { rows: [{ table_name: "redemption_attempts", columns: ["id"] }] };
|
||||||
|
}
|
||||||
|
return { rows: [] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const schema = await readSchema(client, "public");
|
||||||
|
assert.deepEqual(schema.get("redemption_attempts")?.columns[0], {
|
||||||
|
name: "id",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: null,
|
||||||
|
generated: false,
|
||||||
|
identity: true,
|
||||||
|
identityGeneration: "ALWAYS",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("transfer plan rejects source-only public columns", () => {
|
||||||
|
const source = new Map([
|
||||||
|
["profiles", table([{ name: "id" }, { name: "legacy_value" }])],
|
||||||
|
]);
|
||||||
|
const target = new Map([
|
||||||
|
["profiles", table([{ name: "id" }])],
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.throws(() => transferPlan(source, target), /target schema is missing a source column/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("transfer plan uses non-nullable dependencies and rejects unsafe cycles", () => {
|
||||||
|
const source = new Map([
|
||||||
|
["parent", table([{ name: "id" }])],
|
||||||
|
["child", table([{ name: "id" }, { name: "parent_id" }])],
|
||||||
|
]);
|
||||||
|
const target = new Map([
|
||||||
|
["parent", table([{ name: "id" }])],
|
||||||
|
["child", table(
|
||||||
|
[{ name: "id" }, { name: "parent_id" }],
|
||||||
|
["id"],
|
||||||
|
[{ columns: ["parent_id"], refSchema: "public", refTable: "parent", refColumns: ["id"] }],
|
||||||
|
)],
|
||||||
|
]);
|
||||||
|
assert.deepEqual(transferPlan(source, target).ordered, ["parent", "child"]);
|
||||||
|
|
||||||
|
target.get("parent")!.columns.push({
|
||||||
|
name: "child_id", nullable: false, defaultValue: null, generated: false,
|
||||||
|
});
|
||||||
|
source.get("parent")!.columns.push({
|
||||||
|
name: "child_id", nullable: false, defaultValue: null, generated: false,
|
||||||
|
});
|
||||||
|
target.get("parent")!.foreignKeys.push({
|
||||||
|
columns: ["child_id"], refSchema: "public", refTable: "child", refColumns: ["id"],
|
||||||
|
});
|
||||||
|
assert.throws(() => transferPlan(source, target), /foreign-key cycle/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("target preflight rejects existing business rows but permits migration seeds", async () => {
|
||||||
|
const counts = new Map([
|
||||||
|
["identity.users", 0],
|
||||||
|
["auth.users", 0],
|
||||||
|
["public.admin_roles", 6],
|
||||||
|
["public.profiles", 1],
|
||||||
|
]);
|
||||||
|
const client = {
|
||||||
|
async query(text: string) {
|
||||||
|
const match = text.match(/from\s+"(identity|auth|public)"\."([a-z_]+)"/i);
|
||||||
|
assert.ok(match, text);
|
||||||
|
return { rows: [{ count: String(counts.get(`${match[1]}.${match[2]}`) ?? 0) }] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const targetTables = new Map([
|
||||||
|
["admin_roles", table([{ name: "id" }])],
|
||||||
|
["profiles", table([{ name: "id" }])],
|
||||||
|
]);
|
||||||
|
|
||||||
|
await assert.rejects(() => assertTargetEmpty(client, targetTables), /not empty/);
|
||||||
|
counts.set("public.profiles", 0);
|
||||||
|
await assert.doesNotReject(() => assertTargetEmpty(client, targetTables));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("reconciliation hashes are stable and the script has no wildcard data reads", () => {
|
||||||
|
const rows = [
|
||||||
|
{ id: "b", payload: { z: 2, a: 1 } },
|
||||||
|
{ id: "a", payload: { a: 1, z: 2 } },
|
||||||
|
];
|
||||||
|
assert.equal(rowsSha256(rows, ["id", "payload"]), rowsSha256([...rows].reverse(), ["id", "payload"]));
|
||||||
|
|
||||||
|
const source = readFileSync(scriptPath, "utf8");
|
||||||
|
assert.doesNotMatch(source, /select\s+\*/i);
|
||||||
|
assert.match(source, /begin isolation level repeatable read read only/i);
|
||||||
|
assert.match(source, /pg_advisory_xact_lock/);
|
||||||
|
assert.match(source, /target business database is not empty/);
|
||||||
|
assert.match(source, /rollback/);
|
||||||
|
assert.match(source, /--preflight/);
|
||||||
|
assert.match(source, /--apply/);
|
||||||
|
assert.match(source, /--verify/);
|
||||||
|
assert.match(source, /overriding system value/i);
|
||||||
|
assert.match(source, /setval\(pg_get_serial_sequence/i);
|
||||||
|
assert.match(source, /target_primary_key_sha256/);
|
||||||
|
assert.match(source, /sourceKeyHash === targetKeyHash/);
|
||||||
|
assert.match(source, /users\.role = 'admin' and users\.banned = false/);
|
||||||
|
});
|
||||||
@@ -42,6 +42,10 @@ const giteaProductionWorkflow = new URL(
|
|||||||
"../../.gitea/workflows/deploy-production.yml",
|
"../../.gitea/workflows/deploy-production.yml",
|
||||||
import.meta.url,
|
import.meta.url,
|
||||||
);
|
);
|
||||||
|
const giteaProductionMigrationWorkflow = new URL(
|
||||||
|
"../../.gitea/workflows/migrate-production-database.yml",
|
||||||
|
import.meta.url,
|
||||||
|
);
|
||||||
const resetStagingAccountWorkflow = new URL(
|
const resetStagingAccountWorkflow = new URL(
|
||||||
"../../.github/workflows/reset-staging-account.yml",
|
"../../.github/workflows/reset-staging-account.yml",
|
||||||
import.meta.url,
|
import.meta.url,
|
||||||
@@ -62,10 +66,22 @@ const productionDeployScript = new URL(
|
|||||||
"../../deploy/run-production-deploy.sh",
|
"../../deploy/run-production-deploy.sh",
|
||||||
import.meta.url,
|
import.meta.url,
|
||||||
);
|
);
|
||||||
|
const productionMigrationScript = new URL(
|
||||||
|
"../../deploy/run-production-migration.sh",
|
||||||
|
import.meta.url,
|
||||||
|
);
|
||||||
const productionSyncScript = new URL(
|
const productionSyncScript = new URL(
|
||||||
"../../deploy/sync-production-tree.sh",
|
"../../deploy/sync-production-tree.sh",
|
||||||
import.meta.url,
|
import.meta.url,
|
||||||
);
|
);
|
||||||
|
const productionEnvValidator = new URL(
|
||||||
|
"../../deploy/validate-production-env.sh",
|
||||||
|
import.meta.url,
|
||||||
|
);
|
||||||
|
const productionDatabaseEnvValidator = new URL(
|
||||||
|
"../../deploy/validate-production-database-env.sh",
|
||||||
|
import.meta.url,
|
||||||
|
);
|
||||||
|
|
||||||
function read(url: URL): string {
|
function read(url: URL): string {
|
||||||
return readFileSync(url, "utf8");
|
return readFileSync(url, "utf8");
|
||||||
@@ -89,6 +105,7 @@ test("changed staging workflows are syntactically valid YAML", () => {
|
|||||||
giteaDeployWorkflow,
|
giteaDeployWorkflow,
|
||||||
giteaMigrationWorkflow,
|
giteaMigrationWorkflow,
|
||||||
giteaProductionWorkflow,
|
giteaProductionWorkflow,
|
||||||
|
giteaProductionMigrationWorkflow,
|
||||||
]) {
|
]) {
|
||||||
const result = spawnSync(
|
const result = spawnSync(
|
||||||
"python",
|
"python",
|
||||||
@@ -844,13 +861,16 @@ test("production deploy is manual-only and consumes the accepted staging artifac
|
|||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
test("staging scripts pass shell syntax validation", () => {
|
test("deployment scripts pass shell syntax validation", () => {
|
||||||
for (const script of [
|
for (const script of [
|
||||||
deployScript,
|
deployScript,
|
||||||
migrationScript,
|
migrationScript,
|
||||||
syncScript,
|
syncScript,
|
||||||
productionDeployScript,
|
productionDeployScript,
|
||||||
|
productionMigrationScript,
|
||||||
productionSyncScript,
|
productionSyncScript,
|
||||||
|
productionEnvValidator,
|
||||||
|
productionDatabaseEnvValidator,
|
||||||
]) {
|
]) {
|
||||||
const path = fileURLToPath(script);
|
const path = fileURLToPath(script);
|
||||||
chmodSync(path, 0o755);
|
chmodSync(path, 0o755);
|
||||||
@@ -858,3 +878,128 @@ test("staging scripts pass shell syntax validation", () => {
|
|||||||
assert.equal(result.status, 0, result.stderr);
|
assert.equal(result.status, 0, result.stderr);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("Gitea production deploy consumes only gate-attested digests under manual control", () => {
|
||||||
|
const workflow = read(giteaProductionWorkflow);
|
||||||
|
|
||||||
|
assert.match(workflow, /^on:\n\s+workflow_dispatch:/m);
|
||||||
|
assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/);
|
||||||
|
assert.match(workflow, /deploy_sha:[\s\S]*allow_rollback:[\s\S]*verification_mode:/);
|
||||||
|
assert.match(workflow, /runs-on: manman-linux/);
|
||||||
|
assert.match(workflow, /group: production-mutation/);
|
||||||
|
assert.match(workflow, /branch=staging&event=push&status=success/);
|
||||||
|
assert.match(workflow, /endswith\("backend-quality-gate\.yml"\)/);
|
||||||
|
assert.match(workflow, /endswith\("release-quality-gate\.yml"\)/);
|
||||||
|
assert.match(workflow, /public staging has not accepted the requested SHA/);
|
||||||
|
assert.match(workflow, /main and staging must identify the same reviewed release/);
|
||||||
|
assert.match(workflow, /controller_sha256/);
|
||||||
|
assert.match(workflow, /sha256sum --check --status/);
|
||||||
|
assert.match(workflow, /steps\.images\.outputs\.api_image/);
|
||||||
|
assert.match(workflow, /steps\.images\.outputs\.web_image/);
|
||||||
|
assert.match(workflow, /SSH_PRIVATE_KEY_BASE64: \$\{\{ secrets\.PRODUCTION_SSH_PRIVATE_KEY \}\}/);
|
||||||
|
assert.match(workflow, /\[\[ "\$DEPLOY_HOST" == "118\.194\.235\.34" \]\]/);
|
||||||
|
assert.match(workflow, /\[\[ "\$DEPLOY_USER" == "deploy" \]\]/);
|
||||||
|
assert.match(workflow, /\[\[ "\$DEPLOY_PATH" == "\/opt\/jyotisha-production" \]\]/);
|
||||||
|
assert.match(workflow, /bash '\$incoming\/deploy\/run-production-deploy\.sh'/);
|
||||||
|
assert.doesNotMatch(workflow, /docker compose[^\n]*build|db:migrate/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("production runner validates state and migrations before switching exact images", () => {
|
||||||
|
const runner = read(productionDeployScript);
|
||||||
|
const sync = read(productionSyncScript);
|
||||||
|
|
||||||
|
assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/);
|
||||||
|
assert.match(runner, /another production mutation holds the host lock/);
|
||||||
|
assert.match(runner, /sync-production-tree\.sh/);
|
||||||
|
assertOrder(runner, [
|
||||||
|
"validate-production-env.sh",
|
||||||
|
"validate-production-database-env.sh",
|
||||||
|
"compose=(",
|
||||||
|
'"${compose[@]}" config --quiet',
|
||||||
|
"pull api web",
|
||||||
|
"up -d --no-build --pull never --wait postgres",
|
||||||
|
"--profile migration-check run --rm migration-checker",
|
||||||
|
]);
|
||||||
|
assert.match(runner, /-f deploy\/docker-compose\.production\.yml/);
|
||||||
|
assert.match(runner, /pending migrations: run Migrate Production Database/);
|
||||||
|
assert.doesNotMatch(runner, /--profile migration run --rm migrator/);
|
||||||
|
assert.match(runner, /if \[ "\$VERIFICATION_MODE" = "public" \]/);
|
||||||
|
assert.match(runner, /up -d --no-build api web/);
|
||||||
|
assert.match(runner, /up -d --no-build --force-recreate --no-deps caddy/);
|
||||||
|
assert.match(runner, /\["identity", "IDENTITY_DATABASE_URL"\]/);
|
||||||
|
assert.match(runner, /\["service", "SERVICE_DATABASE_URL"\]/);
|
||||||
|
assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/);
|
||||||
|
assert.match(runner, /production verification predicates did not converge/);
|
||||||
|
assert.match(runner, /mv -f "\$revision_file" "\$state_directory\/deployed-revision"/);
|
||||||
|
assert.match(sync, /--exclude='\/\.env\*'/);
|
||||||
|
assert.match(sync, /--exclude='\/backups\/'/);
|
||||||
|
assert.match(sync, /--exclude='\/\.state\/'/);
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
test("Gitea production schema migration is exact-SHA gated and isolated from ETL and deploy", () => {
|
||||||
|
const workflow = read(giteaProductionMigrationWorkflow);
|
||||||
|
const runner = read(productionMigrationScript);
|
||||||
|
|
||||||
|
assert.match(workflow, /^on:\n\s+workflow_dispatch:/m);
|
||||||
|
assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/);
|
||||||
|
assert.match(workflow, /\^\[0-9a-f\]\{40\}\$/);
|
||||||
|
assert.match(workflow, /main_head.*DEPLOY_SHA.*staging_head.*DEPLOY_SHA/s);
|
||||||
|
assert.match(workflow, /GITEA_SHA.*DEPLOY_SHA/);
|
||||||
|
assert.match(workflow, /STAGING_URL: \$\{\{ vars\.STAGING_URL \}\}/);
|
||||||
|
assert.match(workflow, /\[\[ "\$STAGING_URL" == "https:\/\/staging\.jyotisha\.chat" \]\]/);
|
||||||
|
assert.match(workflow, /"\$STAGING_URL\/api\/health"/);
|
||||||
|
assert.match(workflow, /observed_staging_sha.*DEPLOY_SHA/s);
|
||||||
|
assert.match(workflow, /branch=staging&event=push&status=success/);
|
||||||
|
assert.match(workflow, /endswith\("backend-quality-gate\.yml"\)/);
|
||||||
|
assert.match(workflow, /endswith\("release-quality-gate\.yml"\)/);
|
||||||
|
assert.match(workflow, /group: production-mutation/);
|
||||||
|
assert.match(workflow, /SSH_PRIVATE_KEY_BASE64: \$\{\{ secrets\.PRODUCTION_SSH_PRIVATE_KEY \}\}/);
|
||||||
|
assert.match(workflow, /PRODUCTION_KNOWN_HOSTS/);
|
||||||
|
assert.match(workflow, /\[\[ "\$DEPLOY_HOST" == "118\.194\.235\.34" \]\]/);
|
||||||
|
assert.match(workflow, /\[\[ "\$DEPLOY_USER" == "deploy" \]\]/);
|
||||||
|
assert.match(workflow, /\[\[ "\$DEPLOY_PATH" == "\/opt\/jyotisha-production" \]\]/);
|
||||||
|
assert.match(workflow, /sha256sum --check --status/);
|
||||||
|
assert.match(workflow, /recovery_reference:[\s\S]*required: true[\s\S]*type: string/);
|
||||||
|
assert.match(workflow, /recovery_created_at:[\s\S]*required: true[\s\S]*type: string/);
|
||||||
|
assert.match(workflow, /restore_verified:[\s\S]*required: true[\s\S]*default: false[\s\S]*type: boolean/);
|
||||||
|
assert.match(workflow, /\[\[ "\$RESTORE_VERIFIED" == "true" \]\]/);
|
||||||
|
assert.match(workflow, /%Y-%m-%dT%H:%M:%SZ/);
|
||||||
|
assert.match(workflow, /age > timedelta\(hours=24\)/);
|
||||||
|
assert.match(workflow, /migration files run sequentially and are not atomic as a whole/);
|
||||||
|
assert.match(workflow, /RECOVERY_REFERENCE='\$RECOVERY_REFERENCE'/);
|
||||||
|
assert.match(workflow, /RECOVERY_CREATED_AT='\$RECOVERY_CREATED_AT'/);
|
||||||
|
assert.match(workflow, /RESTORE_VERIFIED='\$RESTORE_VERIFIED'/);
|
||||||
|
assert.match(workflow, /run-production-migration\.sh/);
|
||||||
|
assert.doesNotMatch(workflow, /migrate-supabase-production|run-production-deploy|verification_mode|PRODUCTION_URL|CADDY/);
|
||||||
|
|
||||||
|
assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/);
|
||||||
|
assert.match(runner, /another production mutation holds the host lock/);
|
||||||
|
assert.match(runner, /\[ "\$DEPLOY_PATH" = "\/opt\/jyotisha-production" \]/);
|
||||||
|
assert.match(runner, /sync-production-tree\.sh/);
|
||||||
|
assert.match(runner, /validate-production-env\.sh/);
|
||||||
|
assert.match(runner, /validate-production-database-env\.sh/);
|
||||||
|
assert.match(runner, /RECOVERY_REFERENCE RECOVERY_CREATED_AT RESTORE_VERIFIED/);
|
||||||
|
assert.match(runner, /date -u -d "\$RECOVERY_CREATED_AT" \+%s/);
|
||||||
|
assert.match(runner, /recovery_age_seconds <= 24 \* 60 \* 60/);
|
||||||
|
assert.match(runner, /production recovery point must have restore_verified=true/);
|
||||||
|
assert.match(runner, /migration files run sequentially and are not atomic as a whole/);
|
||||||
|
assert.match(runner, /migration_runner_database_url="postgresql:\/\/migration_runner:/);
|
||||||
|
assert.match(runner, /role%3Dschema_owner/);
|
||||||
|
assert.match(runner, /pg_has_role\('migration_runner', 'schema_owner', 'member'\)/);
|
||||||
|
assert.match(
|
||||||
|
runner,
|
||||||
|
/printf 'SCHEMA_DATABASE_URL=%s\\n' "\$migration_runner_database_url" >"\$migration_environment"/,
|
||||||
|
);
|
||||||
|
assert.doesNotMatch(runner, /(?:awk|cat)[^\n]*\.env\.production\.database[^\n]*migration_environment/);
|
||||||
|
assert.doesNotMatch(runner, />>"\$migration_environment"/);
|
||||||
|
assert.equal((runner.match(/--profile migration-check run --rm migration-checker/g) ?? []).length, 2);
|
||||||
|
assertOrder(runner, [
|
||||||
|
"--profile migration-check run --rm migration-checker",
|
||||||
|
"--profile migration run --rm migrator",
|
||||||
|
"postcheck_status",
|
||||||
|
]);
|
||||||
|
assert.match(runner, /production migration postcheck did not converge/);
|
||||||
|
assert.doesNotMatch(runner, /migrate-supabase-production|docker-compose\.server\.yml|docker-compose\.production\.yml/);
|
||||||
|
assert.doesNotMatch(runner, /\bup\b[^\n]*(?:api|web|caddy)|Caddyfile|PRODUCTION_URL|PRODUCTION_ADMIN_URL|mv -f[^\n]*deployed-revision/);
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user