fix(release): keep owners and grants in production recovery points
Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
co-authored by
Grok (agent loop)
Grok (agent loop)
parent
d13250f40a
commit
09906cdccc
@@ -380,6 +380,13 @@ jobs:
|
||||
forward_verified=true
|
||||
fi
|
||||
require_current_release_heads
|
||||
if [[ "$ALLOW_ROLLBACK" == true ]]; then
|
||||
recovery_format="$(ssh "${ssh_options[@]}" "$remote" "awk -F= '\$1 == \"RECOVERY_FORMAT\" { print \$2 }' '$DEPLOY_PATH/.state/recovery-baseline.env' || true")"
|
||||
[[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || {
|
||||
echo "recovery format is below 2; dispatch Create production recovery first" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin"
|
||||
ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' API_IMAGE='$API_IMAGE' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' ALLOW_ROLLBACK='$ALLOW_ROLLBACK' FORWARD_REVISION_VERIFIED='$forward_verified' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' PRODUCTION_URL='$PRODUCTION_URL' PRODUCTION_ADMIN_URL='$PRODUCTION_ADMIN_URL' VERIFICATION_MODE='$VERIFICATION_MODE' bash '$incoming/deploy/run-production-deploy.sh'"
|
||||
require_current_release_heads
|
||||
|
||||
@@ -326,6 +326,11 @@ jobs:
|
||||
fi
|
||||
require_current_release_heads
|
||||
printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin"
|
||||
recovery_format="$(ssh "${ssh_options[@]}" "$remote" "awk -F= '\$1 == \"RECOVERY_FORMAT\" { print \$2 }' '$DEPLOY_PATH/.state/recovery-baseline.env' || true")"
|
||||
[[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || {
|
||||
echo "recovery format is below 2; dispatch Create production recovery first" >&2
|
||||
exit 1
|
||||
}
|
||||
ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' FORWARD_REVISION_VERIFIED='$forward_verified' RECOVERY_REFERENCE='$RECOVERY_REFERENCE' RECOVERY_CREATED_AT='$RECOVERY_CREATED_AT' RESTORE_VERIFIED='$RESTORE_VERIFIED' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' bash '$incoming/deploy/run-production-migration.sh'"
|
||||
require_current_release_heads
|
||||
|
||||
|
||||
@@ -5,12 +5,16 @@
|
||||
# Never restores into or migrates the database named jyotisha.
|
||||
#
|
||||
# Host mode runs every SQL statement inside the postgres container.
|
||||
# It does not call node on the host. Exit 4 means: after this rehearsal
|
||||
# copy was restored and migrated, service_role still cannot SELECT
|
||||
# public.chart_profiles. Grants come from the backup plus migrations,
|
||||
# not from a live probe of the database that is serving production.
|
||||
# If removing the rehearsal database fails, the script exits non-zero
|
||||
# and names that database so it can be dropped by hand.
|
||||
# It does not call node on the host. Restore keeps the owners and grants
|
||||
# stored in the backup. Migrations then run as schema_owner.
|
||||
# Exit 1: refused, including when a migration removed a table or column
|
||||
# grant that was present on the restored copy.
|
||||
# Exit 4: service_role still lacks column SELECT on public.chart_profiles
|
||||
# for the columns named by BUG-1063. Do not migrate production.
|
||||
# Exit 75: another production mutation holds the host lock.
|
||||
# Exit 0 prints rehearsal_ok=1 only after those checks, and only when the
|
||||
# rehearsal database was removed. A failed drop is non-zero and names
|
||||
# that database so it can be dropped by hand.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
@@ -98,11 +102,12 @@ print_safe_log() {
|
||||
}
|
||||
|
||||
run_node_migrator() {
|
||||
local batch="$1" log="$2" status
|
||||
local batch="$1" log="$2" status migrate_url
|
||||
migrate_url="${SCHEMA_DATABASE_URL}?options=-c%20role%3Dschema_owner"
|
||||
set +e
|
||||
(
|
||||
cd "$frontend_dir"
|
||||
MIGRATIONS_DIRECTORY="$batch" node scripts/db-migrate.mjs
|
||||
SCHEMA_DATABASE_URL="$migrate_url" MIGRATIONS_DIRECTORY="$batch" node scripts/db-migrate.mjs
|
||||
) >"$log" 2>&1
|
||||
status=$?
|
||||
set -e
|
||||
@@ -131,33 +136,58 @@ if (kind === "count") {
|
||||
const count = result.rows[0]?.n ?? "";
|
||||
if (!/^[0-9]+$/.test(count)) process.exit(2);
|
||||
process.stdout.write(count);
|
||||
} else if (kind === "archived") {
|
||||
const result = await client.query(
|
||||
"select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chat_sessions' and column_name = 'archived_at') then (select count(*)::text from public.chat_sessions where archived_at is not null) else 'absent' end as n",
|
||||
} else if (kind === "archived" || kind === "self") {
|
||||
const spec = kind === "archived"
|
||||
? { table: "chat_sessions", column: "archived_at", whereSql: "archived_at is not null" }
|
||||
: { table: "chart_profiles", column: "role", whereSql: "role = 'self'" };
|
||||
const exists = await client.query(
|
||||
"select exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = $1 and column_name = $2) as present",
|
||||
[spec.table, spec.column],
|
||||
);
|
||||
const count = result.rows[0]?.n ?? "";
|
||||
if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2);
|
||||
process.stdout.write(count);
|
||||
} else if (kind === "self") {
|
||||
const result = await client.query(
|
||||
"select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chart_profiles' and column_name = 'role') then (select count(*)::text from public.chart_profiles where role = 'self') else 'absent' end as n",
|
||||
);
|
||||
const count = result.rows[0]?.n ?? "";
|
||||
if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2);
|
||||
process.stdout.write(count);
|
||||
} else if (kind === "smoke") {
|
||||
const checks = [
|
||||
["authenticated", "public.profiles"],
|
||||
["authenticated", "public.chat_sessions"],
|
||||
["service_role", "public.chart_profiles"],
|
||||
];
|
||||
for (const [role, table] of checks) {
|
||||
let count = "absent";
|
||||
if (exists.rows[0]?.present === true) {
|
||||
const result = await client.query(
|
||||
"select has_table_privilege($1, $2, 'SELECT') as ok",
|
||||
[role, table],
|
||||
`select count(*)::text as n from public.${spec.table} where ${spec.whereSql}`,
|
||||
);
|
||||
process.stdout.write(`${role} ${table} ${result.rows[0].ok}\n`);
|
||||
count = result.rows[0]?.n ?? "";
|
||||
}
|
||||
if (!/^[0-9]+$/.test(count) && count !== "absent") process.exit(2);
|
||||
process.stdout.write(count);
|
||||
} else if (kind === "grants") {
|
||||
const result = await client.query(`
|
||||
select line from (
|
||||
select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name as line
|
||||
from information_schema.role_table_grants
|
||||
where table_schema not in ('pg_catalog', 'information_schema')
|
||||
union all
|
||||
select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name || '|' || column_name
|
||||
from information_schema.column_privileges
|
||||
where table_schema not in ('pg_catalog', 'information_schema')
|
||||
) s
|
||||
order by 1
|
||||
`);
|
||||
for (const row of result.rows) process.stdout.write(`${row.line}\n`);
|
||||
} else if (kind === "bug1063") {
|
||||
const columns = [
|
||||
"id", "user_id", "role", "name", "birth_date", "reported_birth_time",
|
||||
"birth_time_source", "birth_time_period", "declared_window_start",
|
||||
"declared_window_end", "uncertainty_before_minutes", "uncertainty_after_minutes",
|
||||
"latitude", "longitude", "timezone_id", "timezone_offset", "birth_place_label",
|
||||
"birth_place_type", "birth_place_provider", "birth_place_provider_id", "ayanamsa",
|
||||
"active_birth_time", "active_birth_date", "active_birth_timezone_offset",
|
||||
"birth_time_status", "gender",
|
||||
];
|
||||
const result = await client.query(
|
||||
`select count(*)::text as n
|
||||
from unnest($1::text[]) as cols(column_name)
|
||||
where not has_column_privilege('service_role', 'public.chart_profiles'::regclass, cols.column_name, 'SELECT')`,
|
||||
[columns],
|
||||
);
|
||||
const missing = result.rows[0]?.n ?? "";
|
||||
if (!/^[0-9]+$/.test(missing)) process.exit(2);
|
||||
process.stdout.write(missing);
|
||||
} else if (kind === "grant_create") {
|
||||
await client.query(`grant connect, create on database ${name} to schema_owner`);
|
||||
} else if (kind === "drop") {
|
||||
await client.end();
|
||||
const admin = new URL(source);
|
||||
@@ -313,63 +343,9 @@ if [ "$local_mode" -eq 0 ]; then
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$RECOVERY_DUMP" |
|
||||
docker_cli exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
pg_restore -U "$POSTGRES_USER" -d "$database_name" \
|
||||
--no-owner --exit-on-error
|
||||
--exit-on-error
|
||||
set +o pipefail
|
||||
unset PRODUCTION_BACKUP_ENCRYPTION_KEY
|
||||
docker_cli exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -f - <<'SQL'
|
||||
DO $$
|
||||
DECLARE r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT n.nspname, c.relname, c.relkind
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
AND n.nspname NOT LIKE 'pg_%'
|
||||
AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
|
||||
LOOP
|
||||
IF r.relkind = 'S' THEN
|
||||
EXECUTE format('ALTER SEQUENCE %I.%I OWNER TO schema_owner', r.nspname, r.relname);
|
||||
ELSIF r.relkind = 'v' THEN
|
||||
EXECUTE format('ALTER VIEW %I.%I OWNER TO schema_owner', r.nspname, r.relname);
|
||||
ELSIF r.relkind = 'm' THEN
|
||||
EXECUTE format('ALTER MATERIALIZED VIEW %I.%I OWNER TO schema_owner', r.nspname, r.relname);
|
||||
ELSE
|
||||
EXECUTE format('ALTER TABLE %I.%I OWNER TO schema_owner', r.nspname, r.relname);
|
||||
END IF;
|
||||
END LOOP;
|
||||
FOR r IN
|
||||
SELECT n.nspname, p.proname, pg_get_function_identity_arguments(p.oid) AS args
|
||||
FROM pg_proc p
|
||||
JOIN pg_namespace n ON n.oid = p.pronamespace
|
||||
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_%'
|
||||
LOOP
|
||||
EXECUTE format('ALTER FUNCTION %I.%I(%s) OWNER TO schema_owner', r.nspname, r.proname, r.args);
|
||||
END LOOP;
|
||||
FOR r IN
|
||||
SELECT n.nspname, t.typname
|
||||
FROM pg_type t
|
||||
JOIN pg_namespace n ON n.oid = t.typnamespace
|
||||
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema')
|
||||
AND n.nspname NOT LIKE 'pg_%'
|
||||
AND t.typtype IN ('e', 'c', 'd')
|
||||
LOOP
|
||||
EXECUTE format('ALTER TYPE %I.%I OWNER TO schema_owner', r.nspname, r.typname);
|
||||
END LOOP;
|
||||
FOR r IN
|
||||
SELECT n.nspname
|
||||
FROM pg_namespace n
|
||||
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'public')
|
||||
AND n.nspname NOT LIKE 'pg_%'
|
||||
LOOP
|
||||
EXECUTE format('ALTER SCHEMA %I OWNER TO schema_owner', r.nspname);
|
||||
END LOOP;
|
||||
EXECUTE 'ALTER SCHEMA public OWNER TO schema_owner';
|
||||
END
|
||||
$$;
|
||||
SQL
|
||||
# The URL is captured only into the environment, never echoed.
|
||||
SCHEMA_DATABASE_URL="$(python3 - "$POSTGRES_USER" "$POSTGRES_PASSWORD" "$database_name" <<'PY'
|
||||
import sys, urllib.parse
|
||||
@@ -407,6 +383,14 @@ else
|
||||
}
|
||||
fi
|
||||
|
||||
column_present() {
|
||||
local table="$1" column="$2" present
|
||||
present="$(docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \
|
||||
"select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = '${table}' and column_name = '${column}') then 'yes' else 'no' end")"
|
||||
[ "$present" = "yes" ]
|
||||
}
|
||||
|
||||
sql_scalar() {
|
||||
local kind="$1" sql
|
||||
if [ "$local_mode" -eq 1 ]; then
|
||||
@@ -418,10 +402,18 @@ sql_scalar() {
|
||||
sql="select count(*)::text from public.chart_profiles where role = 'other' and (birth_time_status in ('accepted','confirmed') or active_birth_time is not null or active_birth_date is not null or active_birth_timezone_offset is not null)"
|
||||
;;
|
||||
archived)
|
||||
sql="select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chat_sessions' and column_name = 'archived_at') then (select count(*)::text from public.chat_sessions where archived_at is not null) else 'absent' end"
|
||||
if ! column_present chat_sessions archived_at; then
|
||||
printf 'absent'
|
||||
return
|
||||
fi
|
||||
sql="select count(*)::text from public.chat_sessions where archived_at is not null"
|
||||
;;
|
||||
self)
|
||||
sql="select case when exists (select 1 from information_schema.columns where table_schema = 'public' and table_name = 'chart_profiles' and column_name = 'role') then (select count(*)::text from public.chart_profiles where role = 'self') else 'absent' end"
|
||||
if ! column_present chart_profiles role; then
|
||||
printf 'absent'
|
||||
return
|
||||
fi
|
||||
sql="select count(*)::text from public.chart_profiles where role = 'self'"
|
||||
;;
|
||||
*)
|
||||
return 2
|
||||
@@ -431,14 +423,34 @@ sql_scalar() {
|
||||
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c "$sql"
|
||||
}
|
||||
|
||||
sql_smoke() {
|
||||
capture_grants() {
|
||||
if [ "$local_mode" -eq 1 ]; then
|
||||
run_node_sql smoke
|
||||
run_node_sql grants
|
||||
return
|
||||
fi
|
||||
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \
|
||||
"select line from (select 1 as n, 'authenticated public.profiles ' || case when has_table_privilege('authenticated', 'public.profiles', 'SELECT') then 'true' else 'false' end as line union all select 2, 'authenticated public.chat_sessions ' || case when has_table_privilege('authenticated', 'public.chat_sessions', 'SELECT') then 'true' else 'false' end union all select 3, 'service_role public.chart_profiles ' || case when has_table_privilege('service_role', 'public.chart_profiles', 'SELECT') then 'true' else 'false' end) s order by n"
|
||||
"select line from (select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name as line from information_schema.role_table_grants where table_schema not in ('pg_catalog', 'information_schema') union all select grantee || '|' || privilege_type || '|' || table_schema || '|' || table_name || '|' || column_name from information_schema.column_privileges where table_schema not in ('pg_catalog', 'information_schema')) s order by 1"
|
||||
}
|
||||
|
||||
chart_profiles_select_missing() {
|
||||
if [ "$local_mode" -eq 1 ]; then
|
||||
run_node_sql bug1063
|
||||
return
|
||||
fi
|
||||
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 -At -c \
|
||||
"select count(*)::text from (values ('id'),('user_id'),('role'),('name'),('birth_date'),('reported_birth_time'),('birth_time_source'),('birth_time_period'),('declared_window_start'),('declared_window_end'),('uncertainty_before_minutes'),('uncertainty_after_minutes'),('latitude'),('longitude'),('timezone_id'),('timezone_offset'),('birth_place_label'),('birth_place_type'),('birth_place_provider'),('birth_place_provider_id'),('ayanamsa'),('active_birth_time'),('active_birth_date'),('active_birth_timezone_offset'),('birth_time_status'),('gender')) as cols(column_name) where not has_column_privilege('service_role', 'public.chart_profiles'::regclass, cols.column_name, 'SELECT')"
|
||||
}
|
||||
|
||||
grant_migration_database() {
|
||||
if [ "$local_mode" -eq 1 ]; then
|
||||
run_node_sql grant_create
|
||||
return
|
||||
fi
|
||||
docker_cli exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$database_name" -v ON_ERROR_STOP=1 \
|
||||
-c "GRANT CONNECT, CREATE ON DATABASE \"${database_name}\" TO schema_owner;" >/dev/null
|
||||
}
|
||||
|
||||
echo "rehearsal_database=$database_name"
|
||||
@@ -448,6 +460,9 @@ archived_before="$(sql_scalar archived)" || refuse "could not count archived ses
|
||||
self_before="$(sql_scalar self)" || refuse "could not count self chart rows before migrations"
|
||||
echo "archived_sessions_before=$archived_before"
|
||||
echo "self_chart_rows_before=$self_before"
|
||||
grants_before="$work_dir/grants-before.txt"
|
||||
capture_grants >"$grants_before" || refuse "could not read grants before migrations"
|
||||
grant_migration_database || refuse "could not grant the migration role on the rehearsal database"
|
||||
set +e
|
||||
run_batch "$work_dir/before" "$work_dir/before.log"
|
||||
before_status=$?
|
||||
@@ -481,19 +496,20 @@ for log in "$work_dir/before.log" "$work_dir/all.log"; do
|
||||
notice_lines=$((notice_lines + $(grep -c '^notice ' "$log" || true)))
|
||||
done
|
||||
echo "notice_lines=$notice_lines"
|
||||
smoke_file="$work_dir/smoke.txt"
|
||||
if ! sql_smoke >"$smoke_file"; then
|
||||
refuse "rehearsal smoke could not check role privileges"
|
||||
grants_after="$work_dir/grants-after.txt"
|
||||
capture_grants >"$grants_after" || refuse "could not read grants after migrations"
|
||||
revoked="$(comm -23 <(sort "$grants_before") <(sort "$grants_after") | wc -l | tr -d '[:space:]')"
|
||||
echo "revoked_grants=$revoked"
|
||||
if [ "$revoked" != 0 ]; then
|
||||
refuse "migration revoked existing grants"
|
||||
fi
|
||||
cat "$smoke_file"
|
||||
if ! grep -q '^authenticated public.profiles true$' "$smoke_file"; then
|
||||
refuse "web role cannot read profiles"
|
||||
if ! missing="$(chart_profiles_select_missing | tr -d '[:space:]')"; then
|
||||
refuse "could not check chart_profiles column grants"
|
||||
fi
|
||||
if ! grep -q '^authenticated public.chat_sessions true$' "$smoke_file"; then
|
||||
refuse "web role cannot read chat sessions"
|
||||
fi
|
||||
if ! grep -q '^service_role public.chart_profiles true$' "$smoke_file"; then
|
||||
echo "stop: rehearsal copy still has no service_role SELECT on chart_profiles after migrations; do not migrate production" >&2
|
||||
[[ "$missing" =~ ^[0-9]+$ ]] || refuse "could not check chart_profiles column grants"
|
||||
echo "chart_profiles_service_role_select_missing=$missing"
|
||||
if [ "$missing" != 0 ]; then
|
||||
echo "stop: rehearsal copy still lacks service_role column SELECT on chart_profiles after migrations; do not migrate production" >&2
|
||||
exit 4
|
||||
fi
|
||||
echo "rehearsal_ok=1"
|
||||
|
||||
@@ -34,6 +34,18 @@ if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then
|
||||
echo "invalid rollback authorization" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$ALLOW_ROLLBACK" = "true" ]; then
|
||||
recovery_baseline="$DEPLOY_PATH/.state/recovery-baseline.env"
|
||||
[ -f "$recovery_baseline" ] && [ ! -L "$recovery_baseline" ] || {
|
||||
echo "recovery format is below 2; dispatch Create production recovery first" >&2
|
||||
exit 1
|
||||
}
|
||||
recovery_format="$(awk -F= '$1 == "RECOVERY_FORMAT" { print $2 }' "$recovery_baseline")"
|
||||
[[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || {
|
||||
echo "recovery format is below 2; dispatch Create production recovery first" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
if [ "$VERIFICATION_MODE" != "internal" ] && [ "$VERIFICATION_MODE" != "public" ]; then
|
||||
echo "invalid production verification mode" >&2
|
||||
exit 1
|
||||
|
||||
@@ -51,6 +51,16 @@ esac
|
||||
echo "production recovery point must have restore_verified=true" >&2
|
||||
exit 1
|
||||
}
|
||||
recovery_baseline="$DEPLOY_PATH/.state/recovery-baseline.env"
|
||||
[ -f "$recovery_baseline" ] && [ ! -L "$recovery_baseline" ] || {
|
||||
echo "recovery format is below 2; dispatch Create production recovery first" >&2
|
||||
exit 1
|
||||
}
|
||||
recovery_format="$(awk -F= '$1 == "RECOVERY_FORMAT" { print $2 }' "$recovery_baseline")"
|
||||
[[ "$recovery_format" =~ ^[0-9]+$ ]] && (( recovery_format >= 2 )) || {
|
||||
echo "recovery format is below 2; dispatch Create production recovery first" >&2
|
||||
exit 1
|
||||
}
|
||||
recovery_created_epoch="$(date -u -d "$RECOVERY_CREATED_AT" +%s 2>/dev/null)" || {
|
||||
echo "invalid production recovery creation time" >&2
|
||||
exit 1
|
||||
|
||||
@@ -88,6 +88,48 @@ set -a
|
||||
set +a
|
||||
: "${POSTGRES_DB:?}" "${POSTGRES_USER:?}" "${POSTGRES_PASSWORD:?}" "${PRODUCTION_BACKUP_ENCRYPTION_KEY:?}"
|
||||
|
||||
privilege_sql() {
|
||||
case "$1" in
|
||||
table_grants)
|
||||
printf '%s' "select grantee || '|' || privilege_type || '|' || table_name from information_schema.role_table_grants where table_schema = 'public' order by 1"
|
||||
;;
|
||||
column_privileges)
|
||||
printf '%s' "select grantee || '|' || privilege_type || '|' || table_name || '|' || column_name from information_schema.column_privileges where table_schema = 'public' order by 1"
|
||||
;;
|
||||
table_owners)
|
||||
printf '%s' "select c.relname || '|' || pg_get_userbyid(c.relowner) from pg_class c join pg_namespace n on n.oid = c.relnamespace where n.nspname = 'public' and c.relkind in ('r', 'p') order by 1"
|
||||
;;
|
||||
*)
|
||||
echo "unknown privilege check" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
privilege_snapshot() {
|
||||
local database="$1" kind="$2" destination="$3"
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d "$database" -At -v ON_ERROR_STOP=1 \
|
||||
-c "$(privilege_sql "$kind")" | sort >"$destination"
|
||||
}
|
||||
|
||||
privileges_match() {
|
||||
local kind="$1" source_file restore_file status=0
|
||||
source_file="$(mktemp)"
|
||||
restore_file="$(mktemp)"
|
||||
if ! privilege_snapshot "$POSTGRES_DB" "$kind" "$source_file"; then
|
||||
rm -f -- "$source_file" "$restore_file"
|
||||
return 1
|
||||
fi
|
||||
if ! privilege_snapshot "$restore_database" "$kind" "$restore_file"; then
|
||||
rm -f -- "$source_file" "$restore_file"
|
||||
return 1
|
||||
fi
|
||||
cmp -s "$source_file" "$restore_file" || status=$?
|
||||
rm -f -- "$source_file" "$restore_file"
|
||||
return "$status"
|
||||
}
|
||||
|
||||
usage_percent="$(df -Pk "$backup_directory" | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')"
|
||||
[[ "$usage_percent" =~ ^[0-9]+$ ]] && (( usage_percent < 70 )) || {
|
||||
echo "production backup disk usage must remain below 70 percent" >&2
|
||||
@@ -111,7 +153,7 @@ cleanup() {
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-acl |
|
||||
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom |
|
||||
openssl enc -aes-256-cbc -salt -pbkdf2 \
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY >"$backup_partial"
|
||||
[ -s "$backup_partial" ]
|
||||
@@ -127,7 +169,7 @@ openssl enc -d -aes-256-cbc -pbkdf2 \
|
||||
-pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in "$backup_file" |
|
||||
sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
pg_restore -U "$POSTGRES_USER" -d "$restore_database" \
|
||||
--no-owner --no-acl --exit-on-error
|
||||
--exit-on-error
|
||||
|
||||
counts="$({
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
@@ -146,6 +188,17 @@ for value in "$identity_users" "$profiles" "$credit_transactions" "$public_table
|
||||
}
|
||||
done
|
||||
|
||||
table_grants_match=false
|
||||
column_privileges_match=false
|
||||
table_owners_match=false
|
||||
if privileges_match table_grants; then table_grants_match=true; fi
|
||||
if privileges_match column_privileges; then column_privileges_match=true; fi
|
||||
if privileges_match table_owners; then table_owners_match=true; fi
|
||||
if [ "$table_grants_match" != true ] || [ "$column_privileges_match" != true ] || [ "$table_owners_match" != true ]; then
|
||||
echo "restore privilege verification failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_container" \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "DROP DATABASE \"$restore_database\" WITH (FORCE);" >/dev/null
|
||||
@@ -154,14 +207,15 @@ trap - EXIT HUP INT TERM
|
||||
|
||||
verify_basename="${backup_basename%.dump.enc}-restore-verify.json"
|
||||
verify_file="$backup_directory/$verify_basename"
|
||||
printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s}\n}\n' \
|
||||
printf '{\n "mode": "restore_verify",\n "ok": true,\n "backup": "%s",\n "sha256": "%s",\n "created_at": "%s",\n "verified_at": "%s",\n "restore_database_removed": true,\n "checks": {"identity_users": %s, "profiles": %s, "credit_transactions": %s, "public_tables": %s, "table_grants_match": %s, "column_privileges_match": %s, "table_owners_match": %s}\n}\n' \
|
||||
"$backup_basename" "$backup_sha256" "$created_at" "$verified_at" \
|
||||
"$identity_users" "$profiles" "$credit_transactions" "$public_tables" >"$verify_file"
|
||||
"$identity_users" "$profiles" "$credit_transactions" "$public_tables" \
|
||||
"$table_grants_match" "$column_privileges_match" "$table_owners_match" >"$verify_file"
|
||||
chmod 600 "$verify_file"
|
||||
|
||||
recovery_reference="gitea-actions-run-${RECOVERY_RUN_ID}/${backup_basename}"
|
||||
state_tmp="$state_directory/recovery-baseline.env.tmp.$$"
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nVERIFY_FILE=%s\n' \
|
||||
printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nRECOVERY_FORMAT=2\nVERIFY_FILE=%s\n' \
|
||||
"$recovery_reference" "$created_at" "$backup_sha256" "$verify_basename" >"$state_tmp"
|
||||
chmod 600 "$state_tmp"
|
||||
mv "$state_tmp" "$state_directory/recovery-baseline.env"
|
||||
|
||||
@@ -36116,3 +36116,18 @@
|
||||
- 相关记录：BUG-1348。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/release-path-production-20261014`。编号在 BUG-1347 之后。
|
||||
|
||||
## BUG-1350 | 生产恢复点不含属主与权限，整库还原后网站账号无权限
|
||||
|
||||
- 状态：fixed-pending-verify（本机 PostgreSQL 17 演练已通过；生产按钮还没点，不能标 resolved）。
|
||||
- 首次发现 / 最近更新：2026-10-14 / 2026-10-14
|
||||
- 来源：issue #74 第 2 次验收未过。任务书 `docs/tasks/TASK-release-path-production-fix2-20261014.md`。产品选 A，授权改备份工具。
|
||||
- 现象：按 8 月以来的恢复点整库还原后，网站用来查询的账号对表没有权限，站点请求会被拒绝。恢复点旁边的四个计数仍然对得上，看不出权限丢了。
|
||||
- 触发条件：用现有生产恢复点工具做备份再还原。该工具导出时丢掉属主和授权。
|
||||
- 根因：导出和还原核对都丢掉属主和授权，核对只数行。8 月的恢复点工具就是这样写的。这不是 BUG-203。那条是恢复点锁文件的所有权漂移，锁和这次的授权不是同一件事。
|
||||
- 修复：导出只保留自定义格式。还原核对比较源库和还原库的表权限、列权限、表属主，三者不一致就不写恢复点记录。记录里的格式版本是 2。部署和迁移只有在允许回退、并且这份记录的版本不低于 2 时才继续，否则提示先做新的恢复点。演练不再改类型属主，改为比对迁移有没有收回原有授权，并检查服务角色对人物表各列的读取权。新的一次性库补上迁移角色的连接和建对象权限，因为整库导出带不走数据库本身的权限。
|
||||
- 验证：本机 PostgreSQL 17 演练见 `docs/testing/production-migration-rehearsal-20261014.md`。权限三样一致，演练退出码 0，`rehearsal_ok=1`，演练库已删除，网站角色读到 1 行。生产按钮没点。
|
||||
- 防复发：工作流文本合同锁住导出不再丢掉属主和授权，并锁住格式版本 2。演练测试锁住不再改类型属主、要比对收回的授权。
|
||||
- 相关记录：BUG-203（锁文件，不是这一条）、BUG-1063（人物表列级读取权）、BUG-1348。
|
||||
- 复发自：无。
|
||||
- 修复版本：分支 `codex/release-path-production-20261014`。编号接在 BUG-1349 之后。
|
||||
|
||||
@@ -37,7 +37,7 @@ Gitea → Jyotish Release Quality Gate → Use workflow from：`main`。不需
|
||||
|
||||
Gitea → Create Production Recovery Point。Use workflow from：`main`。`deploy_sha` 填 `R`。
|
||||
|
||||
通过：成功结束，记下三样东西：`recovery_reference`、`recovery_created_at`、以及它已经写明还原验证通过。加密备份在生产主机的备份目录里，不要打开、不要把密钥贴出来。
|
||||
通过：成功结束，记下三样东西：`recovery_reference`、`recovery_created_at`、以及它已经写明还原验证通过。主机上的 `/opt/jyotisha-production/.state/recovery-baseline.env` 里必须有 `RECOVERY_FORMAT=2`。没有这一行，或者数字小于 2，都不要往下走：先再点一次创建恢复点。8 月以来不带权限的旧备份不能再拿来回退。加密备份在生产主机的备份目录里，不要打开、不要把密钥贴出来。
|
||||
|
||||
失败：停。不要迁移。
|
||||
|
||||
@@ -59,7 +59,7 @@ DEPLOY_PATH=/opt/jyotisha-production JYOTISHA_REHEARSAL_PLAN=1 bash deploy/rehea
|
||||
DEPLOY_PATH=/opt/jyotisha-production RECOVERY_DUMP=/opt/jyotisha-production/backups/<刚才那个备份文件名> bash deploy/rehearse-production-migrations.sh
|
||||
```
|
||||
|
||||
脚本只还原到一次性库，不会改名为 `jyotisha` 的生产库。磁盘使用率要低于 70%。另一项生产变更正在进行时会停。跑完会删掉一次性库。
|
||||
脚本只还原到一次性库，不会改名为 `jyotisha` 的生产库。新库带不走原来的建库权限，所以脚本只在这个一次性库上，把连接和建对象的权限交给迁移角色。生产库不动。磁盘使用率要低于 70%。另一项生产变更正在进行时会停。跑完会删掉一次性库。
|
||||
|
||||
通过，要同时满足：
|
||||
|
||||
@@ -67,10 +67,11 @@ DEPLOY_PATH=/opt/jyotisha-production RECOVERY_DUMP=/opt/jyotisha-production/back
|
||||
- `archived_sessions_before` / `archived_sessions_after`：归档标记还在的会话数。迁移后应为 0。旧备份里还没有这一列时，前面那个数是 `absent`。
|
||||
- `self_chart_rows_before` / `self_chart_rows_after`：人物表里「本人」那一行。迁移后应为 0。还没有 `role` 列时，前面那个数是 `absent`。
|
||||
- `notice_lines`：两批迁移日志里 `notice` 的行数。标题修补和自动标题回填的数字在这些行里。生产上标题修补预期是 0；不是 0 就停，只报数字。
|
||||
- 权限冒烟三行都是 `true`：网页角色能读账户资料、能读会话列表，服务角色能读人物表。这是演练库里的授权（备份里原来的授权，加上这次迁移新给的），不是去问正在服务的那一个生产库。
|
||||
- 最后一行是 `rehearsal_ok=1`。
|
||||
- `revoked_grants=0`。这是迁移前后授权的差：迁移不能收回备份里已经有的表权限或列权限。不是 0 就停，不要点生产迁移。
|
||||
- `chart_profiles_service_role_select_missing=0`。这是服务角色对人物表每一列的读取权，缺几列这里就是几。不是 0 时脚本退出码是 4，并停住。不要点生产迁移。
|
||||
- 最后一行是 `rehearsal_ok=1`，并且命令的退出码是 0。只看见这一行但退出码不是 0，不算通过。
|
||||
|
||||
服务角色读人物表若仍是 `false`，脚本退出码 4，并停住。不要点生产迁移。演练库删不掉时，脚本不再只是警告：退出码不是 0，并写出「演练库未删除，请手动 DROP DATABASE rehearse_…」。生产库还没动。把退出码和它打印的那一句原因留下来。不要复制数据库口令或备份密钥。
|
||||
演练结束后，在 postgres 容器里执行 `\l rehearse_*`，确认没有名字以 `rehearse_` 开头的库留下来。演练库删不掉时，脚本退出码不是 0，并写出「演练库未删除，请手动 DROP DATABASE rehearse_…」。只删它写出的那一个名字，不要删名为 `jyotisha` 的生产库。生产库还没动。把退出码和它打印的那一句原因留下来。不要复制数据库口令或备份密钥。
|
||||
|
||||
## 6. Migrate Production Database
|
||||
|
||||
@@ -103,16 +104,41 @@ Gitea → Deploy production。Use workflow from：`main`。
|
||||
|
||||
旧代码读不了新的表结构。所以不能只把程序退回去。两件事都要做，都由产品负责人点，不要让执行代理代点。
|
||||
|
||||
1. 整库还原。没有单独按钮。第 4 步的 `recovery_reference` 斜杠后面就是备份文件名，例如 `production-pre-migration-20261014T120000Z.dump.enc`。文件在 `/opt/jyotisha-production/backups/` 下面。先 `sha256sum` 这个文件，和 `/opt/jyotisha-production/.state/recovery-baseline.env` 里的 `RECOVERY_SHA256` 对上，再继续。工具和创建恢复点时一样，都在 `deploy/run-production-recovery.sh` 里：`openssl` 解密，再在 postgres 容器里 `pg_restore`。这一步会清掉现在的生产库 `jyotisha` 再装回备份。先停 `api` 和 `web`，不要停 `postgres`。在 postgres 容器里删除并重建名为 `jyotisha` 的库（只动这一个），然后：
|
||||
1. 整库还原。没有单独按钮。只允许 `RECOVERY_FORMAT=2` 的恢复点。先看 `/opt/jyotisha-production/.state/recovery-baseline.env`：没有 `RECOVERY_FORMAT=2` 就停，回到第 4 步再点一次创建恢复点。8 月的旧备份不能用。第 4 步的 `recovery_reference` 斜杠后面就是备份文件名，例如 `production-pre-migration-20261014T120000Z.dump.enc`。文件在 `/opt/jyotisha-production/backups/` 下面。先 `sha256sum` 这个文件，和同一份 `recovery-baseline.env` 里的 `RECOVERY_SHA256` 对上，再继续。
|
||||
|
||||
先停 `api` 和 `web`，不要停 `postgres`。口令和密钥只用主机上已有的环境文件，不要贴出来。
|
||||
|
||||
```text
|
||||
api_id=$(sudo -n docker ps -q --filter label=com.docker.compose.project=jyotisha-production --filter label=com.docker.compose.service=api)
|
||||
web_id=$(sudo -n docker ps -q --filter label=com.docker.compose.project=jyotisha-production --filter label=com.docker.compose.service=web)
|
||||
sudo -n docker stop $api_id $web_id
|
||||
postgres_id=$(sudo -n docker ps -q --filter label=com.docker.compose.project=jyotisha-production --filter label=com.docker.compose.service=postgres)
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "DROP DATABASE jyotisha WITH (FORCE);"
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \
|
||||
psql -U "$POSTGRES_USER" -d postgres -v ON_ERROR_STOP=1 \
|
||||
-c "CREATE DATABASE jyotisha;"
|
||||
openssl enc -d -aes-256-cbc -pbkdf2 -pass env:PRODUCTION_BACKUP_ENCRYPTION_KEY -in <备份文件> \
|
||||
| sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" <postgres容器> \
|
||||
pg_restore -U "$POSTGRES_USER" -d jyotisha --no-owner --no-acl --exit-on-error
|
||||
| sudo -n docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \
|
||||
pg_restore -U "$POSTGRES_USER" -d jyotisha --exit-on-error
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \
|
||||
psql -U "$POSTGRES_USER" -d jyotisha -v ON_ERROR_STOP=1 \
|
||||
-c "GRANT CONNECT, CREATE ON DATABASE jyotisha TO schema_owner;"
|
||||
```
|
||||
|
||||
口令和密钥只用主机上已有的环境文件，不要贴出来。还原后数这四样，要和备份旁边的 `*-restore-verify.json` 里 `checks` 一致：`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表数量。对不上就停，不要部署。
|
||||
整库导出带不走数据库本身的权限。上面这一句只补回迁移角色在这个库上的连接和建对象权限，和主机最初的引导一致。不要删掉空的 `public`，备份会改它的属主，先删会让还原停住。
|
||||
|
||||
2. 再点 Deploy production，`allow_rollback` 选 true，目标用发布前的生产版本 `7b620c7a`。测试环境头如果只多了文档、自己没有门禁记录，回滚用的控制器仍然是最近一次门禁通过的修订，不再因为这个被拒绝。目标版本仍必须在已经审查过的 main 历史里。
|
||||
还原后数这四样，要和备份旁边的 `*-restore-verify.json` 里 `checks` 一致：`identity.users`、`public.profiles`、`public.credit_transactions`、`public` 里的表数量。同一份 `checks` 里 `table_grants_match`、`column_privileges_match`、`table_owners_match` 必须都是 true。这三样是创建恢复点时，源库和还原库的表权限、列权限、表属主已经对过。对不上就停，不要部署。
|
||||
|
||||
然后再用网站账号实际读一行。没有登录身份时，行级安全会把行藏起来，看起来像没读到，但那不是权限不够。先把身份设成库里已有的一条账户，再切换角色。下面这条必须成功（退出码 0），并且能读到一行。权限不够会直接失败，失败就停，不要部署：
|
||||
|
||||
```text
|
||||
sudo -n docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$postgres_id" \
|
||||
psql -U "$POSTGRES_USER" -d jyotisha -v ON_ERROR_STOP=1 \
|
||||
-c "select set_config('request.jwt.claim.sub', (select id::text from public.profiles limit 1), false); set role authenticated; select 1 from public.profiles limit 1;"
|
||||
```
|
||||
|
||||
2. 再点 Deploy production，`allow_rollback` 选 true，目标用发布前的生产版本 `7b620c7a`。点之前先在 Gitea 的 Actions 运行列表里确认：这个版本有一次成功的手动发布门禁（Release Quality Gate）。查不到就停，问产品负责人，不要点。测试环境头如果只多了文档、自己没有门禁记录，回滚用的控制器仍然是最近一次门禁通过的修订，不再因为这个被拒绝。目标版本仍必须在已经审查过的 main 历史里。回滚如果提示恢复点格式低于 2，同样先回到第 4 步创建新恢复点，不要改用旧备份。
|
||||
|
||||
还原并且部署成功之后，再按核对清单看健康检查。版本号应回到 `7b620c7a`。
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# PROGRESS · 发布通路修复单 2（2026-10-14）
|
||||
|
||||
- 分支：`codex/release-path-production-20261014`
|
||||
- issue：#74。先并入当时的 `origin/staging`（合并提交 `d13250f4`），没有改写已推历史。
|
||||
- 产品选 A：恢复点带上属主和权限，并在本机真实 PostgreSQL 17 上演练。没有点生产按钮。
|
||||
- 新记录：BUG-1350。不是 BUG-203 的复发。BUG-203 是恢复点锁文件的所有权。
|
||||
|
||||
## 做了什么
|
||||
|
||||
1. 恢复点导出只保留自定义格式。还原后比较表权限、列权限、表属主，不一致就不写恢复点记录。记录里写 `RECOVERY_FORMAT=2`。创建恢复点的按钮输出仍是原来的 7 行，那个工作流文件没有改。
|
||||
2. 部署和迁移在允许回退时，读主机上的恢复点记录。版本低于 2 就拒绝，并提示先做新的恢复点。正向部署不要求这份文件。
|
||||
3. 演练不再改类型属主。迁移前后比较授权，收回了原有授权就退出 1。人物表列级读取权缺了就退出 4。新的一次性库补上迁移角色的连接和建对象权限，因为整库导出带不走数据库本身的权限。
|
||||
4. 手册的整库还原去掉丢掉属主和授权的参数。还原后补回迁移角色在这个库上的连接和建对象权限。读网站账号之前先设登录身份，否则行级安全会把行藏起来。
|
||||
|
||||
## 和任务书不完全一样的地方
|
||||
|
||||
- 空库必须留着 `public`。备份是去改它的属主，不是重新创建它。先删掉会停在「schema public 不存在」。
|
||||
- 数归档会话时，列还不存在不能写在同一条查询里，数据库会在解析阶段就拒绝。先看列在不在，再计数。没有这一列时仍输出 `absent`。
|
||||
- 本机演练用的是 fixture 里的 PostgreSQL 17，不是生产主机上的那一份备份。生产恢复脚本要求生产目录，不能在这台电脑上原样执行。导出参数、加密方式和还原核对是按脚本重放的。
|
||||
|
||||
## 自测
|
||||
|
||||
| 命令 | 结果 |
|
||||
| --- | --- |
|
||||
| `bash -n` 四个生产脚本 | 退出码 0 |
|
||||
| `tests/test_rehearse_production_migrations.py` | 6 通过，退出码 0 |
|
||||
| `tests/test_resolve_release_revision.py` 与上一份一起 | 13 通过，退出码 0（授权那条断言是在这之后加上的，演练测试又单独跑过） |
|
||||
| `tsc --noEmit -p frontend` | 退出码 0 |
|
||||
| `npm run lint --prefix frontend` | 退出码 0，0 error，123 条原有 warning |
|
||||
| `frontend/tests/staging-backend-workflows.test.ts` | 44 条里 39 通过、5 失败、0 取消。失败是这台 Windows 机器的环境：python 退出码 9009、两条缺 rsync、临时目录改权限被拒绝、纯文档范围脚本退出码 49。生产部署、恢复点、生产迁移合同（第 32、34、35、36、37 条）通过 |
|
||||
| 本机 PostgreSQL 17 演练 | 见 `docs/testing/production-migration-rehearsal-20261014.md`。权限三样一致，`rehearsal_ok=1`，退出码 0，演练库 0 个残留，网站角色读到 1 行 |
|
||||
|
||||
## 预检
|
||||
|
||||
`scripts/pre_work_check.py --remote-timeout 8 --command-timeout 45` 退出码 1。远端仓库可见，适配器正常。碎片扫描 90 秒超时，聚焦测试 45 秒超时。这是这台电脑上预检超时的老问题，没有改台账，也不写成预检通过。
|
||||
|
||||
## 没做
|
||||
|
||||
- 没有点生产按钮，没有连生产主机。
|
||||
- 没有推 `staging` 或 `main`。
|
||||
- 没有改测试环境工作流、门禁路径清单、纯文档判断脚本。
|
||||
@@ -336,7 +336,7 @@
|
||||
| `TASK-beta-main-path-fix-20261014.md` | `PROGRESS-beta-main-path-20261014.md` | **#73 第 1 次打回**：`fca62fda` 内容通过；与 #63 删除的旧报告视图有代码冲突，接受删除即可 | 待验收 | `codex/beta-main-path-20261014` |
|
||||
| `TASK-release-path-production-20261014.md` | `PROGRESS-release-path-production-20261014.md` | **发布通路 + 生产首发**：产品 10-14 授权改生产工作流（发布 SHA = 最近门禁成功提交 + 其后纯文档）；63 个迁移清单与演练脚本、首发 runbook；生产拒绝测试验证码。Grok 不 dispatch 生产工作流 | 第 2 次打回后续修 | `codex/release-path-production-20261014`（BUG-1348、BUG-1349） |
|
||||
| `TASK-release-path-production-fix-20261014.md` | `PROGRESS-release-path-production-fix-20261014.md` | **#74 第 1 次打回**：发布规则成立；测试写死 Windows 路径（Linux 全红）、主机演练依赖主机 node 且会留下生产数据副本、docs-only 头时回滚被拒；另补 CHECK_RELEASE_GATE 断言与 OTP 守卫规范化 | 已修，第 2 次仍未过 | `codex/release-path-production-20261014` |
|
||||
| `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2（产品选 A）**：恢复点要带属主与权限，旧格式不得回退；演练按备份里的授权比对，并在真实 PostgreSQL 17 上跑通一次 | 执行中 | `codex/release-path-production-20261014` |
|
||||
| `TASK-release-path-production-fix2-20261014.md` | `PROGRESS-release-path-production-fix2-20261014.md` | **#74 修复单 2（产品选 A）**：恢复点要带属主与权限，旧格式不得回退；演练按备份里的授权比对，并在真实 PostgreSQL 17 上跑通一次 | 待验收 | `codex/release-path-production-20261014` |
|
||||
| `TASK-beta-safety-net-20261014.md` | — | **内测安全网**：每日备份 + 恢复演练、探活告警、本地错误汇总、日志轮转、按 requestId 查日志。**等发布通路单合入、#58 合入后开工** | 待前置 | — |
|
||||
| `TASK-beta-capacity-20261014.md` | — | **内测容量**：按用户限流、重计算短排队与对话重试、兑换码当邀请码、staging 10/20 并发压测。**等 #58 合入后开工，压测等 #57 合入** | 待前置 | — |
|
||||
| `TASK-special-lagnas-native-20261008.md` | `PROGRESS-special-lagnas-native-20261008.md` | **特殊上升点原生计算（线上缺陷）**：sync6 U6 运行时调 PyJHora，生产镜像没有 PyJHora → staging 上 Hora/Ghati/Sree/Indu/Pranapada 全 blocked；原生实现、PyJHora 只作冻结对照、快速门禁止运行时 import jhora；最先做 | 已验收（已部署 staging `795b8845`，BUG-1273） | `codex/special-lagnas-native-20261008`（未上线；部署以 `/api/health` 为准） |
|
||||
|
||||
@@ -1,25 +1,41 @@
|
||||
# 生产迁移演练报告（模板）
|
||||
# 生产迁移演练记录（2026-10-14）
|
||||
|
||||
产品在主机上跑完 `deploy/rehearse-production-migrations.sh` 后填写。只写行数和通过/停止。不要写姓名、出生时间、邮箱、会话正文、密钥或任何一行的内容。
|
||||
本机 Docker，镜像 `postgres:17-alpine`，和 `npm run test:db` 用的是同一个 fixture。没有连生产主机，没有点生产按钮。数据是虚构的，每种关键表一行。下面只记计数、退出码和是否一致。
|
||||
|
||||
| 项 | 结果 |
|
||||
## 步骤
|
||||
|
||||
1. 建库后套用生产当前版本：`frontend/db/migrations` 全部，加上 `frontend/supabase/migrations` 里文件名早于 `20260816010000` 的文件。Windows 上有 4 个同名文件，内容一致，只留一份。
|
||||
2. 用恢复脚本同款方式导出：`pg_dump --format=custom`。再用 openssl `aes-256-cbc -salt -pbkdf2` 加密，然后解密。密钥没有写进仓库。
|
||||
3. 新建空库，`pg_restore --exit-on-error`。不先删 `public`。用超级用户比较源库和还原库的表权限、列权限、表属主。
|
||||
4. 再还原一份，跑 `deploy/rehearse-production-migrations.sh` 的本地模式。
|
||||
5. 再按手册还原一份。先把登录身份设成库里已有的一条账户，再 `set role authenticated`，读 `public.profiles`。
|
||||
|
||||
## 结果
|
||||
|
||||
| 项 | 值 |
|
||||
| --- | --- |
|
||||
| 日期 | |
|
||||
| 发布修订 R | |
|
||||
| 恢复点引用 | |
|
||||
| 一次性库名字（必须是 `rehearse_` 开头） | |
|
||||
| `other_person_accepted_rows` | 预期 0 |
|
||||
| 归档会话：迁移前非空行数 / 迁移后 | 迁移后应为 0 |
|
||||
| 人物表里本人镜像：迁移前 / 迁移后 | 迁移后应为 0 |
|
||||
| 标题修补两条 NOTICE | 生产上预期都是 0。不是 0 就停止 |
|
||||
| 自动标题回填 NOTICE | 只写数字 |
|
||||
| 未知时间改成时段比较的行数 | |
|
||||
| 家人整点改为已采用的行数 | |
|
||||
| 时段回填的行数 | |
|
||||
| 无日期证据改为已确认的行数 | |
|
||||
| 网页角色读账户资料 | true 才算过 |
|
||||
| 网页角色读会话列表 | true 才算过 |
|
||||
| 服务角色读人物表 | true 才算过。false 时脚本退出码 4，停止，不要点生产迁移。这是演练库里的结果，不是正在服务的生产库 |
|
||||
| 最终一行 | `rehearsal_ok=1` 才算演练通过 |
|
||||
| 基线迁移文件 | 111 |
|
||||
| 基线迁移退出码 | 0 |
|
||||
| 基线已应用 | 111 |
|
||||
| 源库计数（身份账户 / 账户资料 / 点数流水 / 人物表 / public 表） | 1 / 1 / 1 / 1 / 76 |
|
||||
| 加密退出码 | 0 |
|
||||
| 解密退出码 | 0 |
|
||||
| 表权限一致 | true（871 行） |
|
||||
| 列权限一致 | true（6760 行） |
|
||||
| 表属主一致 | true（75 行） |
|
||||
| 演练第一批文件 | 160 |
|
||||
| 演练全部文件 | 175 |
|
||||
| 演练退出码 | 0 |
|
||||
| archived_sessions_before | absent |
|
||||
| archived_sessions_after | 0 |
|
||||
| self_chart_rows_before | 0 |
|
||||
| self_chart_rows_after | 0 |
|
||||
| other_person_accepted_rows | 0 |
|
||||
| notice_lines | 161 |
|
||||
| revoked_grants | 0 |
|
||||
| chart_profiles_service_role_select_missing | 0 |
|
||||
| rehearsal_ok | 1 |
|
||||
| 结束后名字以 rehearse_ 开头的库 | 0 |
|
||||
| authenticated 读到的行 | 1 |
|
||||
|
||||
停止时：生产库不应被修改。把脚本打印的那一句原因贴在这里，去掉任何连接地址。
|
||||
`notice_lines=161` 是这次空库演练里迁移提示的行数，不是生产上的标题修补人数。生产上仍按手册：标题修补不是 0 就停。
|
||||
|
||||
@@ -1151,10 +1151,14 @@ test("production deploy is manual-only and consumes the accepted staging artifac
|
||||
assert.match(production, /\[\[ "\$current_main" == "\$current_staging" \]\]/);
|
||||
assert.match(production, /bash "\$checker" --api "\$DEPLOY_SHA" "\$current_staging"/);
|
||||
assert.match(production, /bash deploy\/resolve-release-revision\.sh/);
|
||||
// 原值: 没有断言锁住 CHECK_RELEASE_GATE=true，删掉开关测试仍绿
|
||||
// 新值: 正向部署必须带着这个开关调用解析脚本
|
||||
// 原因: 开关关掉后，正向路径不再要求跑过发布门禁
|
||||
assert.match(production, /CHECK_RELEASE_GATE=true bash deploy\/resolve-release-revision\.sh/);
|
||||
// 原值: /CHECK_RELEASE_GATE=true bash deploy\/resolve-release-revision\.sh/ 同时匹配回滚那一行
|
||||
// 新值: 正向断言锚在 >>"$GITHUB_OUTPUT" 的那一次调用
|
||||
// 原因: 删掉正向那一处时，回滚行仍能让旧正则通过
|
||||
assert.match(production, /CHECK_RELEASE_GATE=true bash deploy\/resolve-release-revision\.sh >>"\$GITHUB_OUTPUT"/);
|
||||
// 原值: 回滚部署不读恢复点格式
|
||||
// 新值: allow_rollback 为 true 时，主机上的恢复点格式必须 >= 2
|
||||
// 原因: 8 月以来不带权限的旧恢复点不能再用来回退
|
||||
assert.match(production, /ALLOW_ROLLBACK" == true[\s\S]*recovery format is below 2; dispatch Create production recovery first/);
|
||||
// 原值: 回滚用 main 头自己的 push 门禁，jq 取不到 first 就失败
|
||||
// 新值: 回滚的控制器改用 RESOLVE_MODE=discover 解析出的 R；目标仍须在已审查的 main 历史里
|
||||
// 原因: 测试环境头只多了文档时没有门禁记录，说明书里的回退会被拒绝
|
||||
@@ -1229,6 +1233,10 @@ test("production runner validates state and migrations before switching exact im
|
||||
|
||||
assert.match(runner, /^#!\/usr\/bin\/env bash\nset -euo pipefail\nset \+x\n/);
|
||||
assert.match(runner, /another production mutation holds the host lock/);
|
||||
// 原值: 回滚部署不读恢复点格式
|
||||
// 新值: ALLOW_ROLLBACK=true 时主机脚本也拒绝低于 2 的恢复点
|
||||
// 原因: 工作流检查被绕过时，回退仍不能使用 8 月的旧恢复点
|
||||
assert.match(runner, /\[ "\$ALLOW_ROLLBACK" = "true" \][\s\S]*recovery format is below 2; dispatch Create production recovery first/);
|
||||
assert.match(runner, /sync-production-tree\.sh/);
|
||||
assertOrder(runner, [
|
||||
"validate-production-env.sh",
|
||||
@@ -1307,10 +1315,18 @@ test("production recovery workflow creates a verified encrypted off-site artifac
|
||||
assert.doesNotMatch(runner, /rm -f[^\n]*mutation\.lock|chown -R/);
|
||||
assert.match(runner, /another production mutation holds the host lock/);
|
||||
assert.match(runner, /usage_percent < 70/);
|
||||
assert.match(runner, /pg_dump[\s\S]*--format=custom --no-owner --no-acl/);
|
||||
// 原值: pg_dump 与 pg_restore 都带 --no-owner --no-acl
|
||||
// 新值: 只保留 --format=custom，还原核对也不再丢掉属主和权限
|
||||
// 原因: 旧恢复点还原后网站账号对所有表都没有权限，四个计数却仍然对得上
|
||||
assert.match(runner, /pg_dump[\s\S]*--format=custom/);
|
||||
assert.doesNotMatch(runner, /--no-owner|--no-acl/);
|
||||
assert.match(runner, /table_grants_match/);
|
||||
assert.match(runner, /column_privileges_match/);
|
||||
assert.match(runner, /table_owners_match/);
|
||||
assert.match(runner, /RECOVERY_FORMAT=2/);
|
||||
assert.match(runner, /openssl enc -aes-256-cbc -salt -pbkdf2/);
|
||||
assert.match(runner, /openssl enc -d -aes-256-cbc -pbkdf2/);
|
||||
assert.match(runner, /pg_restore[\s\S]*--no-owner --no-acl --exit-on-error/);
|
||||
assert.match(runner, /pg_restore[\s\S]*--exit-on-error/);
|
||||
assert.match(runner, /DROP DATABASE IF EXISTS[\s\S]*WITH \(FORCE\)/);
|
||||
assert.match(runner, /restore_database_removed/);
|
||||
assert.match(runner, /gitea-actions-run-\$\{RECOVERY_RUN_ID\}/);
|
||||
@@ -1354,6 +1370,11 @@ test("Gitea production schema migration is exact-SHA gated and isolated from ETL
|
||||
assert.match(workflow, /recovery_created_at:[\s\S]*required: true[\s\S]*type: string/);
|
||||
assert.match(workflow, /restore_verified:[\s\S]*required: true[\s\S]*default: false[\s\S]*type: boolean/);
|
||||
assert.match(workflow, /\[\[ "\$RESTORE_VERIFIED" == "true" \]\]/);
|
||||
// 原值: 生产迁移不读恢复点格式
|
||||
// 新值: 读取 RECOVERY_FORMAT，低于 2 就拒绝并提示先创建恢复点
|
||||
// 原因: 不带权限的旧恢复点不能再当作这次迁移的退路
|
||||
assert.match(workflow, /RECOVERY_FORMAT/);
|
||||
assert.match(workflow, /recovery format is below 2; dispatch Create production recovery first/);
|
||||
assert.match(workflow, /%Y-%m-%dT%H:%M:%SZ/);
|
||||
assert.match(workflow, /age > timedelta\(hours=24\)/);
|
||||
assert.match(workflow, /migration files run sequentially and are not atomic as a whole/);
|
||||
@@ -1377,6 +1398,8 @@ test("Gitea production schema migration is exact-SHA gated and isolated from ETL
|
||||
assert.match(runner, /date -u -d "\$RECOVERY_CREATED_AT" \+%s/);
|
||||
assert.match(runner, /recovery_age_seconds <= 24 \* 60 \* 60/);
|
||||
assert.match(runner, /production recovery point must have restore_verified=true/);
|
||||
assert.match(runner, /RECOVERY_FORMAT/);
|
||||
assert.match(runner, /recovery format is below 2; dispatch Create production recovery first/);
|
||||
assert.match(runner, /migration files run sequentially and are not atomic as a whole/);
|
||||
assert.match(runner, /migration_runner_database_url="postgresql:\/\/migration_runner:/);
|
||||
assert.match(runner, /role%3Dschema_owner/);
|
||||
|
||||
@@ -227,3 +227,14 @@ def test_host_mode_does_not_call_host_node_and_failed_cleanup_is_nonzero(tmp_pat
|
||||
docker_text = docker_log.read_text(encoding="utf-8")
|
||||
assert "--entrypoint node" in docker_text
|
||||
assert "DROP DATABASE" in docker_text
|
||||
|
||||
|
||||
def test_restore_keeps_backup_owners_and_compares_grants() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
assert "--no-owner" not in text
|
||||
assert "ALTER TYPE" not in text
|
||||
assert "typtype" not in text
|
||||
assert "revoked_grants" in text
|
||||
assert "chart_profiles_service_role_select_missing" in text
|
||||
assert "role%3Dschema_owner" in text
|
||||
assert "grant connect, create on database" in text
|
||||
|
||||
Reference in New Issue
Block a user