Files
Jyotisha/deploy/rehearse-production-migrations.sh
T
jesse-uxandGrok (agent loop) 09906cdccc fix(release): keep owners and grants in production recovery points
Co-Authored-By: Grok (agent loop)
2026-10-14 21:01:52 +08:00

517 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
# Rehearse pending production migrations on a disposable database.
# The product runs this on the production host, or locally with
# JYOTISHA_REHEARSAL_LOCAL=1. There is no fifth Gitea workflow for it.
# Never restores into or migrates the database named jyotisha.
#
# Host mode runs every SQL statement inside the postgres container.
# It does not call node on the host. Restore keeps the owners and grants
# stored in the backup. Migrations then run as schema_owner.
# Exit 1: refused, including when a migration removed a table or column
# grant that was present on the restored copy.
# Exit 4: service_role still lacks column SELECT on public.chart_profiles
# for the columns named by BUG-1063. Do not migrate production.
# Exit 75: another production mutation holds the host lock.
# Exit 0 prints rehearsal_ok=1 only after those checks, and only when the
# rehearsal database was removed. A failed drop is non-zero and names
# that database so it can be dropped by hand.
set -euo pipefail
set +x
umask 077
export LC_ALL=C
STOP_BEFORE="20260925030000_chart_subject_trust_and_limit.sql"
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
frontend_dir="$repo_root/frontend"
refuse() {
echo "$1" >&2
exit 1
}
database_name_from_url() {
local rest="${1#postgresql://}"
rest="${rest#*@}"
local path="${rest#*/}"
case "$path" in
*\?*)
refuse "rehearsal URL must not include query parameters"
;;
esac
printf '%s' "$path"
}
docker_cli() {
if [ "${JYOTISHA_REHEARSAL_FAKE_HOST:-}" = 1 ]; then
docker "$@"
else
sudo -n docker "$@"
fi
}
assert_rehearse_name() {
local name="$1"
[[ "$name" != "jyotisha" ]] || refuse "refusing to touch the production database"
[[ "$name" =~ ^rehearse_[0-9a-z_]+$ ]] || refuse "rehearsal database name must match rehearse_<suffix>"
}
copy_batch() {
local dest="$1" mode="$2" dir file base
local -a selected=()
mkdir -p "$dest"
for dir in "$frontend_dir/db/migrations" "$frontend_dir/supabase/migrations"; do
[ -d "$dir" ] || continue
selected=()
for file in "$dir"/*.sql; do
[ -f "$file" ] || continue
base="${file##*/}"
if [ "$mode" = before ] && [[ ! "$base" < "$STOP_BEFORE" ]]; then
continue
fi
selected+=("$file")
done
if [ "${#selected[@]}" -gt 0 ]; then
cp -p "${selected[@]}" "$dest/"
fi
done
}
count_sql() {
find "$1" -maxdepth 1 -type f -name '*.sql' | wc -l | tr -d ' '
}
print_safe_log() {
local log="$1" line
while IFS= read -r line; do
case "$line" in
applied\ *.sql|already\ applied\ *.sql)
printf '%s\n' "$line"
;;
notice\ *)
if [[ "$line" =~ ^notice\ [A-Za-z0-9_.:=[:space:]-]+$ ]]; then
printf '%s\n' "$line"
else
echo "notice_redacted=1"
fi
;;
migration\ failed:*|migration\ file\ missing:*|migration\ checksum\ mismatch:*|duplicate\ migration\ filename:*|unable\ to\ read\ migrations\ directory|database\ migration\ failed)
printf '%s\n' "$line" >&2
;;
esac
done <"$log"
}
run_node_migrator() {
local batch="$1" log="$2" status migrate_url
migrate_url="${SCHEMA_DATABASE_URL}?options=-c%20role%3Dschema_owner"
set +e
(
cd "$frontend_dir"
SCHEMA_DATABASE_URL="$migrate_url" MIGRATIONS_DIRECTORY="$batch" node scripts/db-migrate.mjs
) >"$log" 2>&1
status=$?
set -e
print_safe_log "$log"
[ "$status" -eq 0 ] || refuse "rehearsal migration failed"
}
run_node_sql() {
local kind="$1"
(
cd "$frontend_dir"
node --input-type=module - "$kind" <<'PY'
import pg from "pg";
const kind = process.argv[2];
const source = process.env.SCHEMA_DATABASE_URL;
if (!source || !source.startsWith("postgresql://")) process.exit(2);
const url = new URL(source);
const name = url.pathname.replace(/^\//, "").split("/")[0];
if (!/^rehearse_[0-9a-z_]+$/.test(name) || name === "jyotisha") process.exit(2);
const client = new pg.Client({ connectionString: source });
await client.connect();
if (kind === "count") {
const result = await client.query(
"select count(*)::text as n from public.chart_profiles where role = 'other' and (birth_time_status in ('accepted','confirmed') or active_birth_time is not null or active_birth_date is not null or active_birth_timezone_offset is not null)",
);
const count = result.rows[0]?.n ?? "";
if (!/^[0-9]+$/.test(count)) process.exit(2);
process.stdout.write(count);
} else if (kind === "archived" || kind === "self") {
const spec = kind === "archived"
? { table: "chat_sessions", column: "archived_at", whereSql: "archived_at is not null" }
: { table: "chart_profiles", column: "role", whereSql: "role = 'self'" };
const exists = await client.query(
"select exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = $1 and column_name = $2) as present",
[spec.table, spec.column],
);
let count = "absent";
if (exists.rows[0]?.present === true) {
const result = await client.query(
`select count(*)::text as n from public.${spec.table} where ${spec.whereSql}`,
);
count = result.rows[0]?.n ?? "";
}
if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2);
process.stdout.write(count);
} else if (kind === "grants") {
const result = await client.query(`
select line from (
select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name as line
from information_schema.role_table_grants
where table_schema not in ('pg_catalog', 'information_schema')
union all
select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name || '|' || column_name
from information_schema.column_privileges
where table_schema not in ('pg_catalog', 'information_schema')
) s
order by 1
`);
for (const row of result.rows) process.stdout.write(`${row.line}\n`);
} else if (kind === "bug1063") {
const columns = [
"id", "user_id", "role", "name", "birth_date", "reported_birth_time",
"birth_time_source", "birth_time_period", "declared_window_start",
"declared_window_end", "uncertainty_before_minutes", "uncertainty_after_minutes",
"latitude", "longitude", "timezone_id", "timezone_offset", "birth_place_label",
"birth_place_type", "birth_place_provider", "birth_place_provider_id", "ayanamsa",
"active_birth_time", "active_birth_date", "active_birth_timezone_offset",
"birth_time_status", "gender",
];
const result = await client.query(
`select count(*)::text as n
from unnest($1::text[]) as cols(column_name)
where not has_column_privilege('service_role', 'public.chart_profiles'::regclass, cols.column_name, 'SELECT')`,
[columns],
);
const missing = result.rows[0]?.n ?? "";
if (!/^[0-9]+$/.test(missing)) process.exit(2);
process.stdout.write(missing);
} else if (kind === "grant_create") {
await client.query(`grant connect, create on database ${name} to schema_owner`);
} else if (kind === "drop") {
await client.end();
const admin = new URL(source);
admin.pathname = "/postgres";
const dropper = new pg.Client({ connectionString: admin.toString() });
await dropper.connect();
await dropper.query(`drop database if exists ${name} with (force)`);
await dropper.end();
process.exit(0);
} else {
process.exit(2);
}
await client.end();
PY
)
}
drop_rehearse_database() {
[ "${REHEARSAL_CREATED:-}" = 1 ] || return 0
REHEARSAL_CREATED=0
local status=0
if [ "$local_mode" -eq 0 ]; then
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
-c "DROP DATABASE IF EXISTS \"${database_name}\" WITH (FORCE);" >/dev/null || status=$?
else
run_node_sql drop >/dev/null || status=$?
fi
unset POSTGRES_PASSWORD || true
return "$status"
}
say_cleanup_failed() {
echo "演练库未删除,请手动 DROP DATABASE ${database_name}" >&2
}
local_mode=0
if [ "${JYOTISHA_REHEARSAL_LOCAL:-}" = 1 ]; then
local_mode=1
fi
if [ "$local_mode" -eq 1 ]; then
: "${JYOTISHA_REHEARSAL_DATABASE_URL:?JYOTISHA_REHEARSAL_DATABASE_URL is required}"
[[ "$JYOTISHA_REHEARSAL_DATABASE_URL" == postgresql://* ]] || refuse "rehearsal URL must be a PostgreSQL URL"
case "$JYOTISHA_REHEARSAL_DATABASE_URL" in
*\?*) refuse "rehearsal URL must not include query parameters" ;;
esac
database_name="$(database_name_from_url "$JYOTISHA_REHEARSAL_DATABASE_URL")"
assert_rehearse_name "$database_name"
export SCHEMA_DATABASE_URL="$JYOTISHA_REHEARSAL_DATABASE_URL"
unset JYOTISHA_REHEARSAL_DATABASE_URL
else
: "${DEPLOY_PATH:?DEPLOY_PATH is required}"
if [ "${JYOTISHA_REHEARSAL_FAKE_HOST:-}" = 1 ]; then
case "$DEPLOY_PATH" in
/opt/jyotisha-production|/opt/jyotisha-production/*)
refuse "fake host mode cannot use the production path"
;;
esac
else
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || refuse "unexpected production path"
fi
database_name="rehearse_$(date -u +%Y%m%d%H%M%S)"
assert_rehearse_name "$database_name"
fi
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/jyotisha-rehearse.XXXXXX")"
cleanup_work() {
local status=$?
if ! drop_rehearse_database; then
say_cleanup_failed
status=1
fi
rm -rf -- "$work_dir"
exit "$status"
}
trap cleanup_work EXIT HUP INT TERM
copy_batch "$work_dir/before" before
copy_batch "$work_dir/all" all
before_count="$(count_sql "$work_dir/before")"
all_count="$(count_sql "$work_dir/all")"
if [ -e "$work_dir/before/$STOP_BEFORE" ]; then
refuse "stop file leaked into the first batch"
fi
[ -f "$work_dir/all/$STOP_BEFORE" ] || refuse "stop file is missing from the migration set"
if [ "${JYOTISHA_REHEARSAL_PLAN:-}" = 1 ]; then
trap - EXIT HUP INT TERM
rm -rf -- "$work_dir"
printf 'rehearsal_plan=1\n'
printf 'database_name=%s\n' "$database_name"
printf 'batch_before=%s\n' "$before_count"
printf 'batch_all=%s\n' "$all_count"
printf 'stop_before=%s\n' "$STOP_BEFORE"
printf 'stop_in_before=no\n'
exit 0
fi
if [ "$local_mode" -eq 0 ]; then
: "${RECOVERY_DUMP:?RECOVERY_DUMP is required}"
[ -f "$RECOVERY_DUMP" ] && [ ! -L "$RECOVERY_DUMP" ] || refuse "recovery dump is missing or unsafe"
case "$RECOVERY_DUMP" in
"$DEPLOY_PATH"/backups/*.dump.enc) ;;
*) refuse "recovery dump must be an encrypted file under the production backup directory" ;;
esac
environment_file="$DEPLOY_PATH/.env.production.database"
[ -f "$environment_file" ] && [ ! -L "$environment_file" ] || refuse "production database environment file is missing or unsafe"
state_directory="$DEPLOY_PATH/.state"
backup_directory="$DEPLOY_PATH/backups"
lock_file="$state_directory/mutation.lock"
[ -d "$state_directory" ] && [ ! -L "$state_directory" ] || refuse "production state directory is unsafe"
[ -d "$backup_directory" ] && [ ! -L "$backup_directory" ] || refuse "production backup directory is unsafe"
exec 9>"$lock_file"
if [ "${JYOTISHA_REHEARSAL_FAKE_HOST:-}" != 1 ]; then
flock -n 9 || {
echo "another production mutation holds the host lock" >&2
exit 75
}
fi
set -a
# shellcheck disable=SC1090
. "$environment_file"
set +a
: "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}"
[ "$POSTGRES_DB" = "jyotisha" ] || refuse "unexpected production database name"
usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')"
[[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || refuse "production backup disk usage must remain below 70 percent"
mapfile -t postgres_containers < <(
docker_cli ps -q \
--filter 'label=com.docker.compose.project=jyotisha-production' \
--filter 'label=com.docker.compose.service=postgres'
)
[ "${#postgres_containers[@]}" -eq 1 ] || refuse "expected exactly one running production PostgreSQL container"
postgres_container="${postgres_containers[0]}"
mapfile -t web_containers < <(
docker_cli ps -q \
--filter 'label=com.docker.compose.project=jyotisha-production' \
--filter 'label=com.docker.compose.service=web'
)
[ "${#web_containers[@]}" -eq 1 ] || refuse "expected exactly one running production web container"
web_container="${web_containers[0]}"
web_image="$(docker_cli inspect --format '{{.Image}}' "$web_container")"
[[ "$web_image" =~ ^sha256:[0-9a-f]{64}$ ]] || refuse "production web image identity is unsafe"
web_network="$(docker_cli inspect --format '{{range $name, $net := .NetworkSettings.Networks}}{{$name}}{{"\n"}}{{end}}' "$web_container" | head -n 1)"
[ -n "$web_network" ] || refuse "production web network is missing"
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
-c "CREATE DATABASE \"$database_name\";" >/dev/null
REHEARSAL_CREATED=1
set -o pipefail
openssl enc -d -aes-256-cbc -pbkdf2 \
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$RECOVERY_DUMP" |
docker_cli exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
pg_restore -U "$POSTGRES_USER" -d "$database_name" \
--exit-on-error
set +o pipefail
unset PRODUCTION_BACKUP_ENCRYPTION_KEY
# The URL is captured only into the environment, never echoed.
SCHEMA_DATABASE_URL="$(python3 - "$POSTGRES_USER" "$POSTGRES_PASSWORD" "$database_name" <<'PY'
import sys, urllib.parse
user, password, name = sys.argv[1:]
if name == "jyotisha" or not name.startswith("rehearse_"):
sys.exit(2)
print("postgresql://{user}:{password}@postgres:5432/{name}?options=-c%20role%3Dschema_owner".format(
user=urllib.parse.quote(user, safe=""),
password=urllib.parse.quote(password, safe=""),
name=name,
))
PY
)"
export SCHEMA_DATABASE_URL
run_batch() {
local batch="$1" log="$2" status
set +e
docker_cli run --rm --network "$web_network" \
-e SCHEMA_DATABASE_URL \
-e MIGRATIONS_DIRECTORY=/migrations \
-v "$batch:/migrations:ro" \
-v "$frontend_dir/scripts/db-migrate.mjs:/app/frontend/scripts/db-migrate.rehearse.mjs:ro" \
-w /app/frontend \
--entrypoint node \
"$web_image" scripts/db-migrate.rehearse.mjs >"$log" 2>&1
status=$?
set -e
return "$status"
}
else
REHEARSAL_CREATED=1
run_batch() {
run_node_migrator "$1" "$2"
return 0
}
fi
column_present() {
local table="$1" column="$2" present
present="$(docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \
"select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = '${table}' and column_name = '${column}') then 'yes' else 'no' end")"
[ "$present" = "yes" ]
}
sql_scalar() {
local kind="$1" sql
if [ "$local_mode" -eq 1 ]; then
run_node_sql "$kind"
return
fi
case "$kind" in
count)
sql="select count(*)::text from public.chart_profiles where role = 'other' and (birth_time_status in ('accepted','confirmed') or active_birth_time is not null or active_birth_date is not null or active_birth_timezone_offset is not null)"
;;
archived)
if ! column_present chat_sessions archived_at; then
printf 'absent'
return
fi
sql="select count(*)::text from public.chat_sessions where archived_at is not null"
;;
self)
if ! column_present chart_profiles role; then
printf 'absent'
return
fi
sql="select count(*)::text from public.chart_profiles where role = 'self'"
;;
*)
return 2
;;
esac
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c "$sql"
}
capture_grants() {
if [ "$local_mode" -eq 1 ]; then
run_node_sql grants
return
fi
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \
"select line from (select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name as line from information_schema.role_table_grants where table_schema not in ('pg_catalog', 'information_schema') union all select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name || '|' || column_name from information_schema.column_privileges where table_schema not in ('pg_catalog', 'information_schema')) s order by 1"
}
chart_profiles_select_missing() {
if [ "$local_mode" -eq 1 ]; then
run_node_sql bug1063
return
fi
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \
"select count(*)::text from (values ('id'),('user_id'),('role'),('name'),('birth_date'),('reported_birth_time'),('birth_time_source'),('birth_time_period'),('declared_window_start'),('declared_window_end'),('uncertainty_before_minutes'),('uncertainty_after_minutes'),('latitude'),('longitude'),('timezone_id'),('timezone_offset'),('birth_place_label'),('birth_place_type'),('birth_place_provider'),('birth_place_provider_id'),('ayanamsa'),('active_birth_time'),('active_birth_date'),('active_birth_timezone_offset'),('birth_time_status'),('gender')) as cols(column_name) where not has_column_privilege('service_role', 'public.chart_profiles'::regclass, cols.column_name, 'SELECT')"
}
grant_migration_database() {
if [ "$local_mode" -eq 1 ]; then
run_node_sql grant_create
return
fi
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 \
-c "GRANT CONNECT, CREATE ON DATABASE \"${database_name}\" TO schema_owner;" >/dev/null
}
echo "rehearsal_database=$database_name"
echo "batch_before=$before_count"
echo "stop_before=$STOP_BEFORE"
archived_before="$(sql_scalar archived)" || refuse "could not count archived sessions before migrations"
self_before="$(sql_scalar self)" || refuse "could not count self chart rows before migrations"
echo "archived_sessions_before=$archived_before"
echo "self_chart_rows_before=$self_before"
grants_before="$work_dir/grants-before.txt"
capture_grants >"$grants_before" || refuse "could not read grants before migrations"
grant_migration_database || refuse "could not grant the migration role on the rehearsal database"
set +e
run_batch "$work_dir/before" "$work_dir/before.log"
before_status=$?
set -e
if [ "$local_mode" -eq 0 ]; then
print_safe_log "$work_dir/before.log"
fi
[ "$before_status" -eq 0 ] || refuse "rehearsal migration failed"
other_rows="$(sql_scalar count)" || refuse "could not count other-person profiles before the stop file"
echo "other_person_accepted_rows=$other_rows"
if [ "$other_rows" != 0 ]; then
echo "stop: $STOP_BEFORE would change $other_rows other-person profiles; production was not changed" >&2
exit 1
fi
echo "batch_all=$all_count"
set +e
run_batch "$work_dir/all" "$work_dir/all.log"
all_status=$?
set -e
if [ "$local_mode" -eq 0 ]; then
print_safe_log "$work_dir/all.log"
fi
[ "$all_status" -eq 0 ] || refuse "rehearsal migration failed"
archived_after="$(sql_scalar archived)" || refuse "could not count archived sessions after migrations"
self_after="$(sql_scalar self)" || refuse "could not count self chart rows after migrations"
echo "archived_sessions_after=$archived_after"
echo "self_chart_rows_after=$self_after"
notice_lines=0
for log in "$work_dir/before.log" "$work_dir/all.log"; do
[ -f "$log" ] || continue
notice_lines=$((notice_lines + $(grep -c '^notice ' "$log" || true)))
done
echo "notice_lines=$notice_lines"
grants_after="$work_dir/grants-after.txt"
capture_grants >"$grants_after" || refuse "could not read grants after migrations"
revoked="$(comm -23 <(sort "$grants_before") <(sort "$grants_after") | wc -l | tr -d '[:space:]')"
echo "revoked_grants=$revoked"
if [ "$revoked" != 0 ]; then
refuse "migration revoked existing grants"
fi
if ! missing="$(chart_profiles_select_missing | tr -d '[:space:]')"; then
refuse "could not check chart_profiles column grants"
fi
[[ "$missing" =~ ^[0-9]+$ ]] || refuse "could not check chart_profiles column grants"
echo "chart_profiles_service_role_select_missing=$missing"
if [ "$missing" != 0 ]; then
echo "stop: rehearsal copy still lacks service_role column SELECT on chart_profiles after migrations; do not migrate production" >&2
exit 4
fi
echo "rehearsal_ok=1"
exit 0