From 09906cdccc428e3dc3fba730f5e5b5ff2fdfcbba Mon Sep 17 00:00:00 2001 From: jesse-ux Date: Wed, 14 Oct 2026 21:01:52 +0800 Subject: [PATCH] fix(release): keep owners and grants in production recovery points Co-Authored-By: Grok (agent loop) --- .gitea/workflows/deploy-production.yml | 7 + .../workflows/migrate-production-database.yml | 5 + deploy/rehearse-production-migrations.sh | 220 ++++++++++-------- deploy/run-production-deploy.sh | 12 + deploy/run-production-migration.sh | 10 + deploy/run-production-recovery.sh | 64 ++++- docs/BUG_HISTORY.md | 15 ++ .../production-release-runbook-20261014.md | 46 +++- ...S-release-path-production-fix2-20261014.md | 41 ++++ docs/tasks/README.md | 2 +- ...production-migration-rehearsal-20261014.md | 58 +++-- .../tests/staging-backend-workflows.test.ts | 35 ++- tests/test_rehearse_production_migrations.py | 11 + 13 files changed, 381 insertions(+), 145 deletions(-) create mode 100644 docs/tasks/PROGRESS-release-path-production-fix2-20261014.md diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml index 25f09c09..aa65a363 100644 --- a/.gitea/workflows/deploy-production.yml +++ b/.gitea/workflows/deploy-production.yml @@ -380,6 +380,13 @@ jobs: forward_verified=true fi require_current_release_heads + if [[ "$ALLOW_ROLLBACK" == true ]]; then + recovery_format="$(ssh "${ssh_options[@]}" "$remote" "awk -F= '\$1 == \"RECOVERY_FORMAT\" { print \$2 }' '$DEPLOY_PATH/.state/recovery-baseline.env' || true")" + [[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || { + echo "recovery format is below 2; dispatch Create production recovery first" >&2 + exit 1 + } + fi printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin" ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' API_IMAGE='$API_IMAGE' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' ALLOW_ROLLBACK='$ALLOW_ROLLBACK' FORWARD_REVISION_VERIFIED='$forward_verified' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' PRODUCTION_URL='$PRODUCTION_URL' PRODUCTION_ADMIN_URL='$PRODUCTION_ADMIN_URL' VERIFICATION_MODE='$VERIFICATION_MODE' bash '$incoming/deploy/run-production-deploy.sh'" require_current_release_heads diff --git a/.gitea/workflows/migrate-production-database.yml b/.gitea/workflows/migrate-production-database.yml index 84d1ca1d..87a4c1ff 100644 --- a/.gitea/workflows/migrate-production-database.yml +++ b/.gitea/workflows/migrate-production-database.yml @@ -326,6 +326,11 @@ jobs: fi require_current_release_heads printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin" + recovery_format="$(ssh "${ssh_options[@]}" "$remote" "awk -F= '\$1 == \"RECOVERY_FORMAT\" { print \$2 }' '$DEPLOY_PATH/.state/recovery-baseline.env' || true")" + [[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || { + echo "recovery format is below 2; dispatch Create production recovery first" >&2 + exit 1 + } ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' FORWARD_REVISION_VERIFIED='$forward_verified' RECOVERY_REFERENCE='$RECOVERY_REFERENCE' RECOVERY_CREATED_AT='$RECOVERY_CREATED_AT' RESTORE_VERIFIED='$RESTORE_VERIFIED' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' bash '$incoming/deploy/run-production-migration.sh'" require_current_release_heads diff --git a/deploy/rehearse-production-migrations.sh b/deploy/rehearse-production-migrations.sh index 5b5884b5..00a870dc 100755 --- a/deploy/rehearse-production-migrations.sh +++ b/deploy/rehearse-production-migrations.sh @@ -5,12 +5,16 @@ # Never restores into or migrates the database named jyotisha. # # Host mode runs every SQL statement inside the postgres container. -# It does not call node on the host. Exit 4 means: after this rehearsal -# copy was restored and migrated, service_role still cannot SELECT -# public.chart_profiles. Grants come from the backup plus migrations, -# not from a live probe of the database that is serving production. -# If removing the rehearsal database fails, the script exits non-zero -# and names that database so it can be dropped by hand. +# It does not call node on the host. Restore keeps the owners and grants +# stored in the backup. Migrations then run as schema_owner. +# Exit 1: refused, including when a migration removed a table or column +# grant that was present on the restored copy. +# Exit 4: service_role still lacks column SELECT on public.chart_profiles +# for the columns named by BUG-1063. Do not migrate production. +# Exit 75: another production mutation holds the host lock. +# Exit 0 prints rehearsal_ok=1 only after those checks, and only when the +# rehearsal database was removed. A failed drop is non-zero and names +# that database so it can be dropped by hand. set -euo pipefail set +x umask 077 @@ -98,11 +102,12 @@ print_safe_log() { } run_node_migrator() { - local batch="$1" log="$2" status + local batch="$1" log="$2" status migrate_url + migrate_url="${SCHEMA_DATABASE_URL}?options=-c%20role%3Dschema_owner" set +e ( cd "$frontend_dir" - MIGRATIONS_DIRECTORY="$batch" node scripts/db-migrate.mjs + SCHEMA_DATABASE_URL="$migrate_url" MIGRATIONS_DIRECTORY="$batch" node scripts/db-migrate.mjs ) >"$log" 2>&1 status=$? set -e @@ -131,33 +136,58 @@ if (kind === "count") { const count = result.rows[0]?.n ?? ""; if (!/^[0-9]+$/.test(count)) process.exit(2); process.stdout.write(count); -} else if (kind === "archived") { - const result = await client.query( - "select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chat_sessions' and column_name = 'archived_at') then (select count(*)::text from public.chat_sessions where archived_at is not null) else 'absent' end as n", +} else if (kind === "archived" || kind === "self") { + const spec = kind === "archived" + ? { table: "chat_sessions", column: "archived_at", whereSql: "archived_at is not null" } + : { table: "chart_profiles", column: "role", whereSql: "role = 'self'" }; + const exists = await client.query( + "select exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = $1 and column_name = $2) as present", + [spec.table, spec.column], ); - const count = result.rows[0]?.n ?? ""; - if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2); - process.stdout.write(count); -} else if (kind === "self") { - const result = await client.query( - "select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chart_profiles' and column_name = 'role') then (select count(*)::text from public.chart_profiles where role = 'self') else 'absent' end as n", - ); - const count = result.rows[0]?.n ?? ""; - if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2); - process.stdout.write(count); -} else if (kind === "smoke") { - const checks = [ - ["authenticated", "public.profiles"], - ["authenticated", "public.chat_sessions"], - ["service_role", "public.chart_profiles"], - ]; - for (const [role, table] of checks) { + let count = "absent"; + if (exists.rows[0]?.present === true) { const result = await client.query( - "select has_table_privilege($1, $2, 'SELECT') as ok", - [role, table], + `select count(*)::text as n from public.${spec.table} where ${spec.whereSql}`, ); - process.stdout.write(`${role} ${table} ${result.rows[0].ok}\n`); + count = result.rows[0]?.n ?? ""; } + if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2); + process.stdout.write(count); +} else if (kind === "grants") { + const result = await client.query(` + select line from ( + select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name as line + from information_schema.role_table_grants + where table_schema not in ('pg_catalog', 'information_schema') + union all + select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name || '|' || column_name + from information_schema.column_privileges + where table_schema not in ('pg_catalog', 'information_schema') + ) s + order by 1 + `); + for (const row of result.rows) process.stdout.write(`${row.line}\n`); +} else if (kind === "bug1063") { + const columns = [ + "id", "user_id", "role", "name", "birth_date", "reported_birth_time", + "birth_time_source", "birth_time_period", "declared_window_start", + "declared_window_end", "uncertainty_before_minutes", "uncertainty_after_minutes", + "latitude", "longitude", "timezone_id", "timezone_offset", "birth_place_label", + "birth_place_type", "birth_place_provider", "birth_place_provider_id", "ayanamsa", + "active_birth_time", "active_birth_date", "active_birth_timezone_offset", + "birth_time_status", "gender", + ]; + const result = await client.query( + `select count(*)::text as n + from unnest($1::text[]) as cols(column_name) + where not has_column_privilege('service_role', 'public.chart_profiles'::regclass, cols.column_name, 'SELECT')`, + [columns], + ); + const missing = result.rows[0]?.n ?? ""; + if (!/^[0-9]+$/.test(missing)) process.exit(2); + process.stdout.write(missing); +} else if (kind === "grant_create") { + await client.query(`grant connect, create on database ${name} to schema_owner`); } else if (kind === "drop") { await client.end(); const admin = new URL(source); @@ -313,63 +343,9 @@ if [ "$local_mode" -eq 0 ]; then -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$RECOVERY_DUMP" | docker_cli exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ pg_restore -U "$POSTGRES_USER" -d "$database_name" \ - --no-owner --exit-on-error + --exit-on-error set +o pipefail unset PRODUCTION_BACKUP_ENCRYPTION_KEY - docker_cli exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ - psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -f - <<'SQL' -DO $$ -DECLARE r record; -BEGIN - FOR r IN - SELECT n.nspname, c.relname, c.relkind - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast') - AND n.nspname NOT LIKE 'pg_%' - AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f') - LOOP - IF r.relkind = 'S' THEN - EXECUTE format('ALTER SEQUENCE %I.%I OWNER TO schema_owner', r.nspname, r.relname); - ELSIF r.relkind = 'v' THEN - EXECUTE format('ALTER VIEW %I.%I OWNER TO schema_owner', r.nspname, r.relname); - ELSIF r.relkind = 'm' THEN - EXECUTE format('ALTER MATERIALIZED VIEW %I.%I OWNER TO schema_owner', r.nspname, r.relname); - ELSE - EXECUTE format('ALTER TABLE %I.%I OWNER TO schema_owner', r.nspname, r.relname); - END IF; - END LOOP; - FOR r IN - SELECT n.nspname, p.proname, pg_get_function_identity_arguments(p.oid) AS args - FROM pg_proc p - JOIN pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') - AND n.nspname NOT LIKE 'pg_%' - LOOP - EXECUTE format('ALTER FUNCTION %I.%I(%s) OWNER TO schema_owner', r.nspname, r.proname, r.args); - END LOOP; - FOR r IN - SELECT n.nspname, t.typname - FROM pg_type t - JOIN pg_namespace n ON n.oid = t.typnamespace - WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') - AND n.nspname NOT LIKE 'pg_%' - AND t.typtype IN ('e', 'c', 'd') - LOOP - EXECUTE format('ALTER TYPE %I.%I OWNER TO schema_owner', r.nspname, r.typname); - END LOOP; - FOR r IN - SELECT n.nspname - FROM pg_namespace n - WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public') - AND n.nspname NOT LIKE 'pg_%' - LOOP - EXECUTE format('ALTER SCHEMA %I OWNER TO schema_owner', r.nspname); - END LOOP; - EXECUTE 'ALTER SCHEMA public OWNER TO schema_owner'; -END -$$; -SQL # The URL is captured only into the environment, never echoed. SCHEMA_DATABASE_URL="$(python3 - "$POSTGRES_USER" "$POSTGRES_PASSWORD" "$database_name" <<'PY' import sys, urllib.parse @@ -407,6 +383,14 @@ else } fi +column_present() { + local table="$1" column="$2" present + present="$(docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ + psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \ + "select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = '${table}' and column_name = '${column}') then 'yes' else 'no' end")" + [ "$present" = "yes" ] +} + sql_scalar() { local kind="$1" sql if [ "$local_mode" -eq 1 ]; then @@ -418,10 +402,18 @@ sql_scalar() { sql="select count(*)::text from public.chart_profiles where role = 'other' and (birth_time_status in ('accepted','confirmed') or active_birth_time is not null or active_birth_date is not null or active_birth_timezone_offset is not null)" ;; archived) - sql="select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chat_sessions' and column_name = 'archived_at') then (select count(*)::text from public.chat_sessions where archived_at is not null) else 'absent' end" + if ! column_present chat_sessions archived_at; then + printf 'absent' + return + fi + sql="select count(*)::text from public.chat_sessions where archived_at is not null" ;; self) - sql="select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chart_profiles' and column_name = 'role') then (select count(*)::text from public.chart_profiles where role = 'self') else 'absent' end" + if ! column_present chart_profiles role; then + printf 'absent' + return + fi + sql="select count(*)::text from public.chart_profiles where role = 'self'" ;; *) return 2 @@ -431,14 +423,34 @@ sql_scalar() { psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c "$sql" } -sql_smoke() { +capture_grants() { if [ "$local_mode" -eq 1 ]; then - run_node_sql smoke + run_node_sql grants return fi docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \ - "select line from (select 1 as n, 'authenticated public.profiles ' || case when has_table_privilege('authenticated', 'public.profiles', 'SELECT') then 'true' else 'false' end as line union all select 2, 'authenticated public.chat_sessions ' || case when has_table_privilege('authenticated', 'public.chat_sessions', 'SELECT') then 'true' else 'false' end union all select 3, 'service_role public.chart_profiles ' || case when has_table_privilege('service_role', 'public.chart_profiles', 'SELECT') then 'true' else 'false' end) s order by n" + "select line from (select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name as line from information_schema.role_table_grants where table_schema not in ('pg_catalog', 'information_schema') union all select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name || '|' || column_name from information_schema.column_privileges where table_schema not in ('pg_catalog', 'information_schema')) s order by 1" +} + +chart_profiles_select_missing() { + if [ "$local_mode" -eq 1 ]; then + run_node_sql bug1063 + return + fi + docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ + psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \ + "select count(*)::text from (values ('id'),('user_id'),('role'),('name'),('birth_date'),('reported_birth_time'),('birth_time_source'),('birth_time_period'),('declared_window_start'),('declared_window_end'),('uncertainty_before_minutes'),('uncertainty_after_minutes'),('latitude'),('longitude'),('timezone_id'),('timezone_offset'),('birth_place_label'),('birth_place_type'),('birth_place_provider'),('birth_place_provider_id'),('ayanamsa'),('active_birth_time'),('active_birth_date'),('active_birth_timezone_offset'),('birth_time_status'),('gender')) as cols(column_name) where not has_column_privilege('service_role', 'public.chart_profiles'::regclass, cols.column_name, 'SELECT')" +} + +grant_migration_database() { + if [ "$local_mode" -eq 1 ]; then + run_node_sql grant_create + return + fi + docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ + psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 \ + -c "GRANT CONNECT, CREATE ON DATABASE \"${database_name}\" TO schema_owner;" >/dev/null } echo "rehearsal_database=$database_name" @@ -448,6 +460,9 @@ archived_before="$(sql_scalar archived)" || refuse "could not count archived ses self_before="$(sql_scalar self)" || refuse "could not count self chart rows before migrations" echo "archived_sessions_before=$archived_before" echo "self_chart_rows_before=$self_before" +grants_before="$work_dir/grants-before.txt" +capture_grants >"$grants_before" || refuse "could not read grants before migrations" +grant_migration_database || refuse "could not grant the migration role on the rehearsal database" set +e run_batch "$work_dir/before" "$work_dir/before.log" before_status=$? @@ -481,19 +496,20 @@ for log in "$work_dir/before.log" "$work_dir/all.log"; do notice_lines=$((notice_lines + $(grep -c '^notice ' "$log" || true))) done echo "notice_lines=$notice_lines" -smoke_file="$work_dir/smoke.txt" -if ! sql_smoke >"$smoke_file"; then - refuse "rehearsal smoke could not check role privileges" +grants_after="$work_dir/grants-after.txt" +capture_grants >"$grants_after" || refuse "could not read grants after migrations" +revoked="$(comm -23 <(sort "$grants_before") <(sort "$grants_after") | wc -l | tr -d '[:space:]')" +echo "revoked_grants=$revoked" +if [ "$revoked" != 0 ]; then + refuse "migration revoked existing grants" fi -cat "$smoke_file" -if ! grep -q '^authenticated public.profiles true$' "$smoke_file"; then - refuse "web role cannot read profiles" +if ! missing="$(chart_profiles_select_missing | tr -d '[:space:]')"; then + refuse "could not check chart_profiles column grants" fi -if ! grep -q '^authenticated public.chat_sessions true$' "$smoke_file"; then - refuse "web role cannot read chat sessions" -fi -if ! grep -q '^service_role public.chart_profiles true$' "$smoke_file"; then - echo "stop: rehearsal copy still has no service_role SELECT on chart_profiles after migrations; do not migrate production" >&2 +[[ "$missing" =~ ^[0-9]+$ ]] || refuse "could not check chart_profiles column grants" +echo "chart_profiles_service_role_select_missing=$missing" +if [ "$missing" != 0 ]; then + echo "stop: rehearsal copy still lacks service_role column SELECT on chart_profiles after migrations; do not migrate production" >&2 exit 4 fi echo "rehearsal_ok=1" diff --git a/deploy/run-production-deploy.sh b/deploy/run-production-deploy.sh index 610e4444..bb43c6c5 100755 --- a/deploy/run-production-deploy.sh +++ b/deploy/run-production-deploy.sh @@ -34,6 +34,18 @@ if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then echo "invalid rollback authorization" >&2 exit 1 fi +if [ "$ALLOW_ROLLBACK" = "true" ]; then + recovery_baseline="$DEPLOY_PATH/.state/recovery-baseline.env" + [ -f "$recovery_baseline" ] && [ ! -L "$recovery_baseline" ] || { + echo "recovery format is below 2; dispatch Create production recovery first" >&2 + exit 1 + } + recovery_format="$(awk -F= '$1 == "RECOVERY_FORMAT" { print $2 }' "$recovery_baseline")" + [[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || { + echo "recovery format is below 2; dispatch Create production recovery first" >&2 + exit 1 + } +fi if [ "$VERIFICATION_MODE" != "internal" ] && [ "$VERIFICATION_MODE" != "public" ]; then echo "invalid production verification mode" >&2 exit 1 diff --git a/deploy/run-production-migration.sh b/deploy/run-production-migration.sh index 6d1a66e1..e70c20fa 100755 --- a/deploy/run-production-migration.sh +++ b/deploy/run-production-migration.sh @@ -51,6 +51,16 @@ esac echo "production recovery point must have restore_verified=true" >&2 exit 1 } +recovery_baseline="$DEPLOY_PATH/.state/recovery-baseline.env" +[ -f "$recovery_baseline" ] && [ ! -L "$recovery_baseline" ] || { + echo "recovery format is below 2; dispatch Create production recovery first" >&2 + exit 1 +} +recovery_format="$(awk -F= '$1 == "RECOVERY_FORMAT" { print $2 }' "$recovery_baseline")" +[[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || { + echo "recovery format is below 2; dispatch Create production recovery first" >&2 + exit 1 +} recovery_created_epoch="$(date -u -d "$RECOVERY_CREATED_AT" +%s 2>/dev/null)" || { echo "invalid production recovery creation time" >&2 exit 1 diff --git a/deploy/run-production-recovery.sh b/deploy/run-production-recovery.sh index 32395f35..f552bcac 100755 --- a/deploy/run-production-recovery.sh +++ b/deploy/run-production-recovery.sh @@ -88,6 +88,48 @@ set -a set +a : "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}" +privilege_sql() { + case "$1" in + table_grants) + printf '%s' "select grantee || '|' || privilege_type || '|' || table_name from information_schema.role_table_grants where table_schema = 'public' order by 1" + ;; + column_privileges) + printf '%s' "select grantee || '|' || privilege_type || '|' || table_name || '|' || column_name from information_schema.column_privileges where table_schema = 'public' order by 1" + ;; + table_owners) + printf '%s' "select c.relname || '|' || pg_get_userbyid(c.relowner) from pg_class c join pg_namespace n on n.oid = c.relnamespace where n.nspname = 'public' and c.relkind in ('r', 'p') order by 1" + ;; + *) + echo "unknown privilege check" >&2 + exit 1 + ;; + esac +} + +privilege_snapshot() { + local database="$1" kind="$2" destination="$3" + sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ + psql -U "$POSTGRES_USER" -d "$database" -At -v ON_ERROR_STOP=1 \ + -c "$(privilege_sql "$kind")" | sort >"$destination" +} + +privileges_match() { + local kind="$1" source_file restore_file status=0 + source_file="$(mktemp)" + restore_file="$(mktemp)" + if ! privilege_snapshot "$POSTGRES_DB" "$kind" "$source_file"; then + rm -f -- "$source_file" "$restore_file" + return 1 + fi + if ! privilege_snapshot "$restore_database" "$kind" "$restore_file"; then + rm -f -- "$source_file" "$restore_file" + return 1 + fi + cmp -s "$source_file" "$restore_file" || status=$? + rm -f -- "$source_file" "$restore_file" + return "$status" +} + usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')" [[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || { echo "production backup disk usage must remain below 70 percent" >&2 @@ -111,7 +153,7 @@ cleanup() { trap cleanup EXIT HUP INT TERM sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ - pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-acl | + pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom | openssl enc -aes-256-cbc -salt -pbkdf2 \ -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY >"$backup_partial" [ -s "$backup_partial" ] @@ -127,7 +169,7 @@ openssl enc -d -aes-256-cbc -pbkdf2 \ -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$backup_file" | sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ pg_restore -U "$POSTGRES_USER" -d "$restore_database" \ - --no-owner --no-acl --exit-on-error + --exit-on-error counts="$({ sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ @@ -146,6 +188,17 @@ for value in "$identity_users" "$profiles" "$credit_transactions" "$public_table } done +table_grants_match=false +column_privileges_match=false +table_owners_match=false +if privileges_match table_grants; then table_grants_match=true; fi +if privileges_match column_privileges; then column_privileges_match=true; fi +if privileges_match table_owners; then table_owners_match=true; fi +if [ "$table_grants_match" != true ] || [ "$column_privileges_match" != true ] || [ "$table_owners_match" != true ]; then + echo "restore privilege verification failed" >&2 + exit 1 +fi + sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \ psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ -c "DROP DATABASE \"$restore_database\" WITH (FORCE);" >/dev/null @@ -154,14 +207,15 @@ trap - EXIT HUP INT TERM verify_basename="${backup_basename%.dump.enc}-restore-verify.json" verify_file="$backup_directory/$verify_basename" -printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s}\n}\n' \ +printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s, "table_grants_match": %s, "column_privileges_match": %s, "table_owners_match": %s}\n}\n' \ "$backup_basename" "$backup_sha256" "$created_at" "$verified_at" \ - "$identity_users" "$profiles" "$credit_transactions" "$public_tables" >"$verify_file" + "$identity_users" "$profiles" "$credit_transactions" "$public_tables" \ + "$table_grants_match" "$column_privileges_match" "$table_owners_match" >"$verify_file" chmod 600 "$verify_file" recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}" state_tmp="$state_directory/recovery-baseline.env.tmp.$$" -printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nVERIFY_FILE=%s\n' \ +printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nRECOVERY_FORMAT=2\nVERIFY_FILE=%s\n' \ "$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp" chmod 600 "$state_tmp" mv "$state_tmp" "$state_directory/recovery-baseline.env" diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 957bb6d2..deb92593 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -36116,3 +36116,18 @@ - 相关记录:BUG-1348。 - 复发自:无。 - 修复版本:分支 `codex/release-path-production-20261014`。编号在 BUG-1347 之后。 + +## BUG-1350 | 生产恢复点不含属主与权限,整库还原后网站账号无权限 + +- 状态:fixed-pending-verify(本机 PostgreSQL 17 演练已通过;生产按钮还没点,不能标 resolved)。 +- 首次发现 / 最近更新:2026-10-14 / 2026-10-14 +- 来源:issue #74 第 2 次验收未过。任务书 `docs/tasks/TASK-release-path-production-fix2-20261014.md`。产品选 A,授权改备份工具。 +- 现象:按 8 月以来的恢复点整库还原后,网站用来查询的账号对表没有权限,站点请求会被拒绝。恢复点旁边的四个计数仍然对得上,看不出权限丢了。 +- 触发条件:用现有生产恢复点工具做备份再还原。该工具导出时丢掉属主和授权。 +- 根因:导出和还原核对都丢掉属主和授权,核对只数行。8 月的恢复点工具就是这样写的。这不是 BUG-203。那条是恢复点锁文件的所有权漂移,锁和这次的授权不是同一件事。 +- 修复:导出只保留自定义格式。还原核对比较源库和还原库的表权限、列权限、表属主,三者不一致就不写恢复点记录。记录里的格式版本是 2。部署和迁移只有在允许回退、并且这份记录的版本不低于 2 时才继续,否则提示先做新的恢复点。演练不再改类型属主,改为比对迁移有没有收回原有授权,并检查服务角色对人物表各列的读取权。新的一次性库补上迁移角色的连接和建对象权限,因为整库导出带不走数据库本身的权限。 +- 验证:本机 PostgreSQL 17 演练见 `docs/testing/production-migration-rehearsal-20261014.md`。权限三样一致,演练退出码 0,`rehearsal_ok=1`,演练库已删除,网站角色读到 1 行。生产按钮没点。 +- 防复发:工作流文本合同锁住导出不再丢掉属主和授权,并锁住格式版本 2。演练测试锁住不再改类型属主、要比对收回的授权。 +- 相关记录:BUG-203(锁文件,不是这一条)、BUG-1063(人物表列级读取权)、BUG-1348。 +- 复发自:无。 +- 修复版本:分支 `codex/release-path-production-20261014`。编号接在 BUG-1349 之后。 diff --git a/docs/operations/production-release-runbook-20261014.md b/docs/operations/production-release-runbook-20261014.md index a759bbc2..c8aae66f 100644 --- a/docs/operations/production-release-runbook-20261014.md +++ b/docs/operations/production-release-runbook-20261014.md @@ -37,7 +37,7 @@ Gitea → Jyotish Release Quality Gate → Use workflow from:`main`。不需 Gitea → Create Production Recovery Point。Use workflow from:`main`。`deploy_sha` 填 `R`。 -通过:成功结束,记下三样东西:`recovery_reference`、`recovery_created_at`、以及它已经写明还原验证通过。加密备份在生产主机的备份目录里,不要打开、不要把密钥贴出来。 +通过:成功结束,记下三样东西:`recovery_reference`、`recovery_created_at`、以及它已经写明还原验证通过。主机上的 `/opt/jyotisha-production/.state/recovery-baseline.env` 里必须有 `RECOVERY_FORMAT=2`。没有这一行,或者数字小于 2,都不要往下走:先再点一次创建恢复点。8 月以来不带权限的旧备份不能再拿来回退。加密备份在生产主机的备份目录里,不要打开、不要把密钥贴出来。 失败:停。不要迁移。 @@ -59,7 +59,7 @@ DEPLOY_PATH=/opt/jyotisha-production JYOTISHA_REHEARSAL_PLAN=1 bash deploy/rehea DEPLOY_PATH=/opt/jyotisha-production RECOVERY_DUMP=/opt/jyotisha-production/backups/<刚才那个备份文件名> bash deploy/rehearse-production-migrations.sh ``` -脚本只还原到一次性库,不会改名为 `jyotisha` 的生产库。磁盘使用率要低于 70%。另一项生产变更正在进行时会停。跑完会删掉一次性库。 +脚本只还原到一次性库,不会改名为 `jyotisha` 的生产库。新库带不走原来的建库权限,所以脚本只在这个一次性库上,把连接和建对象的权限交给迁移角色。生产库不动。磁盘使用率要低于 70%。另一项生产变更正在进行时会停。跑完会删掉一次性库。 通过,要同时满足: @@ -67,10 +67,11 @@ DEPLOY_PATH=/opt/jyotisha-production RECOVERY_DUMP=/opt/jyotisha-production/back - `archived_sessions_before` / `archived_sessions_after`:归档标记还在的会话数。迁移后应为 0。旧备份里还没有这一列时,前面那个数是 `absent`。 - `self_chart_rows_before` / `self_chart_rows_after`:人物表里「本人」那一行。迁移后应为 0。还没有 `role` 列时,前面那个数是 `absent`。 - `notice_lines`:两批迁移日志里 `notice` 的行数。标题修补和自动标题回填的数字在这些行里。生产上标题修补预期是 0;不是 0 就停,只报数字。 -- 权限冒烟三行都是 `true`:网页角色能读账户资料、能读会话列表,服务角色能读人物表。这是演练库里的授权(备份里原来的授权,加上这次迁移新给的),不是去问正在服务的那一个生产库。 -- 最后一行是 `rehearsal_ok=1`。 +- `revoked_grants=0`。这是迁移前后授权的差:迁移不能收回备份里已经有的表权限或列权限。不是 0 就停,不要点生产迁移。 +- `chart_profiles_service_role_select_missing=0`。这是服务角色对人物表每一列的读取权,缺几列这里就是几。不是 0 时脚本退出码是 4,并停住。不要点生产迁移。 +- 最后一行是 `rehearsal_ok=1`,并且命令的退出码是 0。只看见这一行但退出码不是 0,不算通过。 -服务角色读人物表若仍是 `false`,脚本退出码 4,并停住。不要点生产迁移。演练库删不掉时,脚本不再只是警告:退出码不是 0,并写出「演练库未删除,请手动 DROP DATABASE rehearse_…」。生产库还没动。把退出码和它打印的那一句原因留下来。不要复制数据库口令或备份密钥。 +演练结束后,在 postgres 容器里执行 `\l rehearse_*`,确认没有名字以 `rehearse_` 开头的库留下来。演练库删不掉时,脚本退出码不是 0,并写出「演练库未删除,请手动 DROP DATABASE rehearse_…」。只删它写出的那一个名字,不要删名为 `jyotisha` 的生产库。生产库还没动。把退出码和它打印的那一句原因留下来。不要复制数据库口令或备份密钥。 ## 6. Migrate Production Database @@ -103,16 +104,41 @@ Gitea → Deploy production。Use workflow from:`main`。 旧代码读不了新的表结构。所以不能只把程序退回去。两件事都要做,都由产品负责人点,不要让执行代理代点。 -1. 整库还原。没有单独按钮。第 4 步的 `recovery_reference` 斜杠后面就是备份文件名,例如 `production-pre-migration-20261014T120000Z.dump.enc`。文件在 `/opt/jyotisha-production/backups/` 下面。先 `sha256sum` 这个文件,和 `/opt/jyotisha-production/.state/recovery-baseline.env` 里的 `RECOVERY_SHA256` 对上,再继续。工具和创建恢复点时一样,都在 `deploy/run-production-recovery.sh` 里:`openssl` 解密,再在 postgres 容器里 `pg_restore`。这一步会清掉现在的生产库 `jyotisha` 再装回备份。先停 `api` 和 `web`,不要停 `postgres`。在 postgres 容器里删除并重建名为 `jyotisha` 的库(只动这一个),然后: +1. 整库还原。没有单独按钮。只允许 `RECOVERY_FORMAT=2` 的恢复点。先看 `/opt/jyotisha-production/.state/recovery-baseline.env`:没有 `RECOVERY_FORMAT=2` 就停,回到第 4 步再点一次创建恢复点。8 月的旧备份不能用。第 4 步的 `recovery_reference` 斜杠后面就是备份文件名,例如 `production-pre-migration-20261014T120000Z.dump.enc`。文件在 `/opt/jyotisha-production/backups/` 下面。先 `sha256sum` 这个文件,和同一份 `recovery-baseline.env` 里的 `RECOVERY_SHA256` 对上,再继续。 + +先停 `api` 和 `web`,不要停 `postgres`。口令和密钥只用主机上已有的环境文件,不要贴出来。 ```text +api_id=$(sudo -n docker ps -q --filter label=com.docker.compose.project=jyotisha-production --filter label=com.docker.compose.service=api) +web_id=$(sudo -n docker ps -q --filter label=com.docker.compose.project=jyotisha-production --filter label=com.docker.compose.service=web) +sudo -n docker stop $api_id $web_id +postgres_id=$(sudo -n docker ps -q --filter label=com.docker.compose.project=jyotisha-production --filter label=com.docker.compose.service=postgres) +sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \ + psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE jyotisha WITH (FORCE);" +sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \ + psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \ + -c "CREATE DATABASE jyotisha;" openssl enc -d -aes-256-cbc -pbkdf2 -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in <备份文件> \ - | sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" \ - pg_restore -U "$POSTGRES_USER" -d jyotisha --no-owner --no-acl --exit-on-error + | sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \ + pg_restore -U "$POSTGRES_USER" -d jyotisha --exit-on-error +sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \ + psql -U "$POSTGRES_USER" -d jyotisha -v ON_ERROR_STOP=1 \ + -c "GRANT CONNECT, CREATE ON DATABASE jyotisha TO schema_owner;" ``` -口令和密钥只用主机上已有的环境文件,不要贴出来。还原后数这四样,要和备份旁边的 `*-restore-verify.json` 里 `checks` 一致:`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表数量。对不上就停,不要部署。 +整库导出带不走数据库本身的权限。上面这一句只补回迁移角色在这个库上的连接和建对象权限,和主机最初的引导一致。不要删掉空的 `public`,备份会改它的属主,先删会让还原停住。 -2. 再点 Deploy production,`allow_rollback` 选 true,目标用发布前的生产版本 `7b620c7a`。测试环境头如果只多了文档、自己没有门禁记录,回滚用的控制器仍然是最近一次门禁通过的修订,不再因为这个被拒绝。目标版本仍必须在已经审查过的 main 历史里。 +还原后数这四样,要和备份旁边的 `*-restore-verify.json` 里 `checks` 一致:`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表数量。同一份 `checks` 里 `table_grants_match`、`column_privileges_match`、`table_owners_match` 必须都是 true。这三样是创建恢复点时,源库和还原库的表权限、列权限、表属主已经对过。对不上就停,不要部署。 + +然后再用网站账号实际读一行。没有登录身份时,行级安全会把行藏起来,看起来像没读到,但那不是权限不够。先把身份设成库里已有的一条账户,再切换角色。下面这条必须成功(退出码 0),并且能读到一行。权限不够会直接失败,失败就停,不要部署: + +```text +sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \ + psql -U "$POSTGRES_USER" -d jyotisha -v ON_ERROR_STOP=1 \ + -c "select set_config('request.jwt.claim.sub', (select id::text from public.profiles limit 1), false); set role authenticated; select 1 from public.profiles limit 1;" +``` + +2. 再点 Deploy production,`allow_rollback` 选 true,目标用发布前的生产版本 `7b620c7a`。点之前先在 Gitea 的 Actions 运行列表里确认:这个版本有一次成功的手动发布门禁(Release Quality Gate)。查不到就停,问产品负责人,不要点。测试环境头如果只多了文档、自己没有门禁记录,回滚用的控制器仍然是最近一次门禁通过的修订,不再因为这个被拒绝。目标版本仍必须在已经审查过的 main 历史里。回滚如果提示恢复点格式低于 2,同样先回到第 4 步创建新恢复点,不要改用旧备份。 还原并且部署成功之后,再按核对清单看健康检查。版本号应回到 `7b620c7a`。 diff --git a/docs/tasks/PROGRESS-release-path-production-fix2-20261014.md b/docs/tasks/PROGRESS-release-path-production-fix2-20261014.md new file mode 100644 index 00000000..248b3baf --- /dev/null +++ b/docs/tasks/PROGRESS-release-path-production-fix2-20261014.md @@ -0,0 +1,41 @@ +# PROGRESS · 发布通路修复单 2(2026-10-14) + +- 分支:`codex/release-path-production-20261014` +- issue:#74。先并入当时的 `origin/staging`(合并提交 `d13250f4`),没有改写已推历史。 +- 产品选 A:恢复点带上属主和权限,并在本机真实 PostgreSQL 17 上演练。没有点生产按钮。 +- 新记录:BUG-1350。不是 BUG-203 的复发。BUG-203 是恢复点锁文件的所有权。 + +## 做了什么 + +1. 恢复点导出只保留自定义格式。还原后比较表权限、列权限、表属主,不一致就不写恢复点记录。记录里写 `RECOVERY_FORMAT=2`。创建恢复点的按钮输出仍是原来的 7 行,那个工作流文件没有改。 +2. 部署和迁移在允许回退时,读主机上的恢复点记录。版本低于 2 就拒绝,并提示先做新的恢复点。正向部署不要求这份文件。 +3. 演练不再改类型属主。迁移前后比较授权,收回了原有授权就退出 1。人物表列级读取权缺了就退出 4。新的一次性库补上迁移角色的连接和建对象权限,因为整库导出带不走数据库本身的权限。 +4. 手册的整库还原去掉丢掉属主和授权的参数。还原后补回迁移角色在这个库上的连接和建对象权限。读网站账号之前先设登录身份,否则行级安全会把行藏起来。 + +## 和任务书不完全一样的地方 + +- 空库必须留着 `public`。备份是去改它的属主,不是重新创建它。先删掉会停在「schema public 不存在」。 +- 数归档会话时,列还不存在不能写在同一条查询里,数据库会在解析阶段就拒绝。先看列在不在,再计数。没有这一列时仍输出 `absent`。 +- 本机演练用的是 fixture 里的 PostgreSQL 17,不是生产主机上的那一份备份。生产恢复脚本要求生产目录,不能在这台电脑上原样执行。导出参数、加密方式和还原核对是按脚本重放的。 + +## 自测 + +| 命令 | 结果 | +| --- | --- | +| `bash -n` 四个生产脚本 | 退出码 0 | +| `tests/test_rehearse_production_migrations.py` | 6 通过,退出码 0 | +| `tests/test_resolve_release_revision.py` 与上一份一起 | 13 通过,退出码 0(授权那条断言是在这之后加上的,演练测试又单独跑过) | +| `tsc --noEmit -p frontend` | 退出码 0 | +| `npm run lint --prefix frontend` | 退出码 0,0 error,123 条原有 warning | +| `frontend/tests/staging-backend-workflows.test.ts` | 44 条里 39 通过、5 失败、0 取消。失败是这台 Windows 机器的环境:python 退出码 9009、两条缺 rsync、临时目录改权限被拒绝、纯文档范围脚本退出码 49。生产部署、恢复点、生产迁移合同(第 32、34、35、36、37 条)通过 | +| 本机 PostgreSQL 17 演练 | 见 `docs/testing/production-migration-rehearsal-20261014.md`。权限三样一致,`rehearsal_ok=1`,退出码 0,演练库 0 个残留,网站角色读到 1 行 | + +## 预检 + +`scripts/pre_work_check.py --remote-timeout 8 --command-timeout 45` 退出码 1。远端仓库可见,适配器正常。碎片扫描 90 秒超时,聚焦测试 45 秒超时。这是这台电脑上预检超时的老问题,没有改台账,也不写成预检通过。 + +## 没做 + +- 没有点生产按钮,没有连生产主机。 +- 没有推 `staging` 或 `main`。 +- 没有改测试环境工作流、门禁路径清单、纯文档判断脚本。 diff --git a/docs/tasks/README.md b/docs/tasks/README.md index 170245f0..3c8a8cdf 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -336,7 +336,7 @@ | `TASK-beta-main-path-fix-20261014.md` | `PROGRESS-beta-main-path-20261014.md` | **#73 第 1 次打回**:`fca62fda` 内容通过;与 #63 删除的旧报告视图有代码冲突,接受删除即可 | 待验收 | `codex/beta-main-path-20261014` | | `TASK-release-path-production-20261014.md` | `PROGRESS-release-path-production-20261014.md` | **发布通路 + 生产首发**:产品 10-14 授权改生产工作流(发布 SHA = 最近门禁成功提交 + 其后纯文档);63 个迁移清单与演练脚本、首发 runbook;生产拒绝测试验证码。Grok 不 dispatch 生产工作流 | 第 2 次打回后续修 | `codex/release-path-production-20261014`(BUG-1348、BUG-1349) | | `TASK-release-path-production-fix-20261014.md` | `PROGRESS-release-path-production-fix-20261014.md` | **#74 第 1 次打回**:发布规则成立;测试写死 Windows 路径(Linux 全红)、主机演练依赖主机 node 且会留下生产数据副本、docs-only 头时回滚被拒;另补 CHECK_RELEASE_GATE 断言与 OTP 守卫规范化 | 已修,第 2 次仍未过 | `codex/release-path-production-20261014` | -| `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2(产品选 A)**:恢复点要带属主与权限,旧格式不得回退;演练按备份里的授权比对,并在真实 PostgreSQL 17 上跑通一次 | 执行中 | `codex/release-path-production-20261014` | +| `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2(产品选 A)**:恢复点要带属主与权限,旧格式不得回退;演练按备份里的授权比对,并在真实 PostgreSQL 17 上跑通一次 | 待验收 | `codex/release-path-production-20261014` | | `TASK-beta-safety-net-20261014.md` | — | **内测安全网**:每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。**等发布通路单合入、#58 合入后开工** | 待前置 | — | | `TASK-beta-capacity-20261014.md` | — | **内测容量**:按用户限流、重计算短排队与对话重试、兑换码当邀请码、staging 10/20 并发压测。**等 #58 合入后开工,压测等 #57 合入** | 待前置 | — | | `TASK-special-lagnas-native-20261008.md` | `PROGRESS-special-lagnas-native-20261008.md` | **特殊上升点原生计算(线上缺陷)**:sync6 U6 运行时调 PyJHora,生产镜像没有 PyJHora → staging 上 Hora/Ghati/Sree/Indu/Pranapada 全 blocked;原生实现、PyJHora 只作冻结对照、快速门禁止运行时 import jhora;最先做 | 已验收(已部署 staging `795b8845`,BUG-1273) | `codex/special-lagnas-native-20261008`(未上线;部署以 `/api/health` 为准) | diff --git a/docs/testing/production-migration-rehearsal-20261014.md b/docs/testing/production-migration-rehearsal-20261014.md index 5f4b25cb..59967fb4 100644 --- a/docs/testing/production-migration-rehearsal-20261014.md +++ b/docs/testing/production-migration-rehearsal-20261014.md @@ -1,25 +1,41 @@ -# 生产迁移演练报告(模板) +# 生产迁移演练记录(2026-10-14) -产品在主机上跑完 `deploy/rehearse-production-migrations.sh` 后填写。只写行数和通过/停止。不要写姓名、出生时间、邮箱、会话正文、密钥或任何一行的内容。 +本机 Docker,镜像 `postgres:17-alpine`,和 `npm run test:db` 用的是同一个 fixture。没有连生产主机,没有点生产按钮。数据是虚构的,每种关键表一行。下面只记计数、退出码和是否一致。 -| 项 | 结果 | +## 步骤 + +1. 建库后套用生产当前版本:`frontend/db/migrations` 全部,加上 `frontend/supabase/migrations` 里文件名早于 `20260816010000` 的文件。Windows 上有 4 个同名文件,内容一致,只留一份。 +2. 用恢复脚本同款方式导出:`pg_dump --format=custom`。再用 openssl `aes-256-cbc -salt -pbkdf2` 加密,然后解密。密钥没有写进仓库。 +3. 新建空库,`pg_restore --exit-on-error`。不先删 `public`。用超级用户比较源库和还原库的表权限、列权限、表属主。 +4. 再还原一份,跑 `deploy/rehearse-production-migrations.sh` 的本地模式。 +5. 再按手册还原一份。先把登录身份设成库里已有的一条账户,再 `set role authenticated`,读 `public.profiles`。 + +## 结果 + +| 项 | 值 | | --- | --- | -| 日期 | | -| 发布修订 R | | -| 恢复点引用 | | -| 一次性库名字(必须是 `rehearse_` 开头) | | -| `other_person_accepted_rows` | 预期 0 | -| 归档会话:迁移前非空行数 / 迁移后 | 迁移后应为 0 | -| 人物表里本人镜像:迁移前 / 迁移后 | 迁移后应为 0 | -| 标题修补两条 NOTICE | 生产上预期都是 0。不是 0 就停止 | -| 自动标题回填 NOTICE | 只写数字 | -| 未知时间改成时段比较的行数 | | -| 家人整点改为已采用的行数 | | -| 时段回填的行数 | | -| 无日期证据改为已确认的行数 | | -| 网页角色读账户资料 | true 才算过 | -| 网页角色读会话列表 | true 才算过 | -| 服务角色读人物表 | true 才算过。false 时脚本退出码 4,停止,不要点生产迁移。这是演练库里的结果,不是正在服务的生产库 | -| 最终一行 | `rehearsal_ok=1` 才算演练通过 | +| 基线迁移文件 | 111 | +| 基线迁移退出码 | 0 | +| 基线已应用 | 111 | +| 源库计数(身份账户 / 账户资料 / 点数流水 / 人物表 / public 表) | 1 / 1 / 1 / 1 / 76 | +| 加密退出码 | 0 | +| 解密退出码 | 0 | +| 表权限一致 | true(871 行) | +| 列权限一致 | true(6760 行) | +| 表属主一致 | true(75 行) | +| 演练第一批文件 | 160 | +| 演练全部文件 | 175 | +| 演练退出码 | 0 | +| archived_sessions_before | absent | +| archived_sessions_after | 0 | +| self_chart_rows_before | 0 | +| self_chart_rows_after | 0 | +| other_person_accepted_rows | 0 | +| notice_lines | 161 | +| revoked_grants | 0 | +| chart_profiles_service_role_select_missing | 0 | +| rehearsal_ok | 1 | +| 结束后名字以 rehearse_ 开头的库 | 0 | +| authenticated 读到的行 | 1 | -停止时:生产库不应被修改。把脚本打印的那一句原因贴在这里,去掉任何连接地址。 +`notice_lines=161` 是这次空库演练里迁移提示的行数,不是生产上的标题修补人数。生产上仍按手册:标题修补不是 0 就停。 diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index b6a73ca8..c3fc112c 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -1151,10 +1151,14 @@ test("production deploy is manual-only and consumes the accepted staging artifac assert.match(production, /\[\[ "\$current_main" == "\$current_staging" \]\]/); assert.match(production, /bash "\$checker" --api "\$DEPLOY_SHA" "\$current_staging"/); assert.match(production, /bash deploy\/resolve-release-revision\.sh/); - // 原值: 没有断言锁住 CHECK_RELEASE_GATE=true,删掉开关测试仍绿 - // 新值: 正向部署必须带着这个开关调用解析脚本 - // 原因: 开关关掉后,正向路径不再要求跑过发布门禁 - assert.match(production, /CHECK_RELEASE_GATE=true bash deploy\/resolve-release-revision\.sh/); + // 原值: /CHECK_RELEASE_GATE=true bash deploy\/resolve-release-revision\.sh/ 同时匹配回滚那一行 + // 新值: 正向断言锚在 >>"$GITHUB_OUTPUT" 的那一次调用 + // 原因: 删掉正向那一处时,回滚行仍能让旧正则通过 + assert.match(production, /CHECK_RELEASE_GATE=true bash deploy\/resolve-release-revision\.sh >>"\$GITHUB_OUTPUT"/); + // 原值: 回滚部署不读恢复点格式 + // 新值: allow_rollback 为 true 时,主机上的恢复点格式必须 >= 2 + // 原因: 8 月以来不带权限的旧恢复点不能再用来回退 + assert.match(production, /ALLOW_ROLLBACK" == true[\s\S]*recovery format is below 2; dispatch Create production recovery first/); // 原值: 回滚用 main 头自己的 push 门禁,jq 取不到 first 就失败 // 新值: 回滚的控制器改用 RESOLVE_MODE=discover 解析出的 R;目标仍须在已审查的 main 历史里 // 原因: 测试环境头只多了文档时没有门禁记录,说明书里的回退会被拒绝 @@ -1229,6 +1233,10 @@ test("production runner validates state and migrations before switching exact im assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/); assert.match(runner, /another production mutation holds the host lock/); + // 原值: 回滚部署不读恢复点格式 + // 新值: ALLOW_ROLLBACK=true 时主机脚本也拒绝低于 2 的恢复点 + // 原因: 工作流检查被绕过时,回退仍不能使用 8 月的旧恢复点 + assert.match(runner, /\[ "\$ALLOW_ROLLBACK" = "true" \][\s\S]*recovery format is below 2; dispatch Create production recovery first/); assert.match(runner, /sync-production-tree\.sh/); assertOrder(runner, [ "validate-production-env.sh", @@ -1307,10 +1315,18 @@ test("production recovery workflow creates a verified encrypted off-site artifac assert.doesNotMatch(runner, /rm -f[^\n]*mutation\.lock|chown -R/); assert.match(runner, /another production mutation holds the host lock/); assert.match(runner, /usage_percent < 70/); - assert.match(runner, /pg_dump[\s\S]*--format=custom --no-owner --no-acl/); + // 原值: pg_dump 与 pg_restore 都带 --no-owner --no-acl + // 新值: 只保留 --format=custom,还原核对也不再丢掉属主和权限 + // 原因: 旧恢复点还原后网站账号对所有表都没有权限,四个计数却仍然对得上 + assert.match(runner, /pg_dump[\s\S]*--format=custom/); + assert.doesNotMatch(runner, /--no-owner|--no-acl/); + assert.match(runner, /table_grants_match/); + assert.match(runner, /column_privileges_match/); + assert.match(runner, /table_owners_match/); + assert.match(runner, /RECOVERY_FORMAT=2/); assert.match(runner, /openssl enc -aes-256-cbc -salt -pbkdf2/); assert.match(runner, /openssl enc -d -aes-256-cbc -pbkdf2/); - assert.match(runner, /pg_restore[\s\S]*--no-owner --no-acl --exit-on-error/); + assert.match(runner, /pg_restore[\s\S]*--exit-on-error/); assert.match(runner, /DROP DATABASE IF EXISTS[\s\S]*WITH \(FORCE\)/); assert.match(runner, /restore_database_removed/); assert.match(runner, /gitea-actions-run-\$\{RECOVERY_RUN_ID\}/); @@ -1354,6 +1370,11 @@ test("Gitea production schema migration is exact-SHA gated and isolated from ETL assert.match(workflow, /recovery_created_at:[\s\S]*required: true[\s\S]*type: string/); assert.match(workflow, /restore_verified:[\s\S]*required: true[\s\S]*default: false[\s\S]*type: boolean/); assert.match(workflow, /\[\[ "\$RESTORE_VERIFIED" == "true" \]\]/); + // 原值: 生产迁移不读恢复点格式 + // 新值: 读取 RECOVERY_FORMAT,低于 2 就拒绝并提示先创建恢复点 + // 原因: 不带权限的旧恢复点不能再当作这次迁移的退路 + assert.match(workflow, /RECOVERY_FORMAT/); + assert.match(workflow, /recovery format is below 2; dispatch Create production recovery first/); assert.match(workflow, /%Y-%m-%dT%H:%M:%SZ/); assert.match(workflow, /age > timedelta\(hours=24\)/); assert.match(workflow, /migration files run sequentially and are not atomic as a whole/); @@ -1377,6 +1398,8 @@ test("Gitea production schema migration is exact-SHA gated and isolated from ETL assert.match(runner, /date -u -d "\$RECOVERY_CREATED_AT" \+%s/); assert.match(runner, /recovery_age_seconds <= 24 \* 60 \* 60/); assert.match(runner, /production recovery point must have restore_verified=true/); + assert.match(runner, /RECOVERY_FORMAT/); + assert.match(runner, /recovery format is below 2; dispatch Create production recovery first/); assert.match(runner, /migration files run sequentially and are not atomic as a whole/); assert.match(runner, /migration_runner_database_url="postgresql:\/\/migration_runner:/); assert.match(runner, /role%3Dschema_owner/); diff --git a/tests/test_rehearse_production_migrations.py b/tests/test_rehearse_production_migrations.py index 6577fa7d..fe083442 100644 --- a/tests/test_rehearse_production_migrations.py +++ b/tests/test_rehearse_production_migrations.py @@ -227,3 +227,14 @@ def test_host_mode_does_not_call_host_node_and_failed_cleanup_is_nonzero(tmp_pat docker_text = docker_log.read_text(encoding="utf-8") assert "--entrypoint node" in docker_text assert "DROP DATABASE" in docker_text + + +def test_restore_keeps_backup_owners_and_compares_grants() -> None: + text = SCRIPT.read_text(encoding="utf-8") + assert "--no-owner" not in text + assert "ALTER TYPE" not in text + assert "typtype" not in text + assert "revoked_grants" in text + assert "chart_profiles_service_role_select_missing" in text + assert "role%3Dschema_owner" in text + assert "grant connect, create on database" in text