Jesse_Chen
201 lines
7.0 KiB
Bash
Executable File
201 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
set +x
|
|
|
|
required=(
|
|
INCOMING_PATH DEPLOY_PATH WEB_IMAGE DEPLOY_SHA EXPECTED_PREVIOUS_SHA
|
|
RECOVERY_REFERENCE RECOVERY_CREATED_AT RESTORE_VERIFIED DOCKER_CONFIG
|
|
)
|
|
case "${DOCKER_BIN:-docker}" in
|
|
docker) docker_command=(docker) ;;
|
|
"sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;;
|
|
*) echo "unsafe production Docker command" >&2; exit 1 ;;
|
|
esac
|
|
for key in "${required[@]}"; do
|
|
if [ -z "${!key:-}" ]; then
|
|
echo "required production migration input is missing: $key" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || {
|
|
echo "unsafe production migration revision" >&2
|
|
exit 1
|
|
}
|
|
image_pattern='^crpi-d1feco6itet73spp\.cn-hongkong\.personal\.cr\.aliyuncs\.com/copse/jyotisha@sha256:[0-9a-f]{64}$'
|
|
[[ "$WEB_IMAGE" =~ $image_pattern ]] || {
|
|
echo "unsafe production migration image" >&2
|
|
exit 1
|
|
}
|
|
case "$INCOMING_PATH" in
|
|
/tmp/jyotisha-production-migration.*) ;;
|
|
*) echo "unsafe incoming production migration path" >&2; exit 1 ;;
|
|
esac
|
|
[ "$DEPLOY_PATH" = "/opt/jyotisha-production" ] || {
|
|
echo "unsafe production deployment path" >&2
|
|
exit 1
|
|
}
|
|
[ "$DOCKER_CONFIG" = "$INCOMING_PATH/.docker" ] || {
|
|
echo "unsafe production Docker configuration path" >&2
|
|
exit 1
|
|
}
|
|
[[ "$RECOVERY_REFERENCE" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}$ ]] || {
|
|
echo "unsafe production recovery reference" >&2
|
|
exit 1
|
|
}
|
|
[[ "$RECOVERY_CREATED_AT" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || {
|
|
echo "unsafe production recovery creation time" >&2
|
|
exit 1
|
|
}
|
|
[ "$RESTORE_VERIFIED" = "true" ] || {
|
|
echo "production recovery point must have restore_verified=true" >&2
|
|
exit 1
|
|
}
|
|
recovery_created_epoch="$(date -u -d "$RECOVERY_CREATED_AT" +%s 2>/dev/null)" || {
|
|
echo "invalid production recovery creation time" >&2
|
|
exit 1
|
|
}
|
|
recovery_now_epoch="$(date -u +%s)"
|
|
recovery_age_seconds=$((recovery_now_epoch - recovery_created_epoch))
|
|
(( recovery_age_seconds >= 0 && recovery_age_seconds <= 24 * 60 * 60 )) || {
|
|
echo "production recovery point must be no more than 24 hours old and not in the future" >&2
|
|
exit 1
|
|
}
|
|
echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT restore_verified=true"
|
|
echo "WARNING: migration files run sequentially and are not atomic as a whole; recovery may be required after a partial migration." >&2
|
|
|
|
state_directory="$DEPLOY_PATH/.state"
|
|
install -d -m 700 "$state_directory"
|
|
exec 9>"$state_directory/mutation.lock"
|
|
flock -n 9 || {
|
|
echo "another production mutation holds the host lock" >&2
|
|
exit 75
|
|
}
|
|
|
|
current_sha="not-deployed"
|
|
if [ -f "$state_directory/deployed-revision" ]; then
|
|
current_sha="$(<"$state_directory/deployed-revision")"
|
|
else
|
|
existing_web="$("${docker_command[@]}" ps -aq \
|
|
--filter 'label=com.docker.compose.project=jyotisha-production' \
|
|
--filter 'label=com.docker.compose.service=web' | head -n 1)"
|
|
if [ -n "$existing_web" ]; then
|
|
discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
|
|
"$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)"
|
|
if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi
|
|
fi
|
|
fi
|
|
if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo "invalid deployed production revision state" >&2
|
|
exit 1
|
|
fi
|
|
[ "$current_sha" = "$EXPECTED_PREVIOUS_SHA" ] || {
|
|
echo "production revision changed while this migration was waiting" >&2
|
|
exit 1
|
|
}
|
|
[ "$current_sha" = "not-deployed" ] ||
|
|
[ "$current_sha" = "$DEPLOY_SHA" ] ||
|
|
[ "${FORWARD_REVISION_VERIFIED:-false}" = "true" ] || {
|
|
echo "forward production revision was not verified" >&2
|
|
exit 1
|
|
}
|
|
|
|
bash "$INCOMING_PATH/deploy/sync-production-tree.sh" \
|
|
"$INCOMING_PATH" "$DEPLOY_PATH"
|
|
|
|
cd "$DEPLOY_PATH"
|
|
EXPECTED_PRODUCTION_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")"
|
|
[[ "$EXPECTED_PRODUCTION_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || {
|
|
echo "production deployment owner is invalid" >&2
|
|
exit 1
|
|
}
|
|
export EXPECTED_PRODUCTION_ENV_OWNER_UID
|
|
bash deploy/validate-production-env.sh .env.production
|
|
bash deploy/validate-production-database-env.sh .env.production.database
|
|
|
|
export DATABASE_ENV_FILE='../.env.production.database'
|
|
compose=("${docker_command[@]}" compose -p jyotisha-production -f deploy/docker-compose.postgres.yml)
|
|
"${docker_command[@]}" pull "$WEB_IMAGE"
|
|
"${compose[@]}" config --quiet
|
|
"${compose[@]}" up -d --no-build --pull never --wait postgres
|
|
|
|
membership="$("${compose[@]}" exec -T postgres psql -v ON_ERROR_STOP=1 -U postgres -d jyotisha -Atc \
|
|
"select pg_has_role('migration_runner', 'schema_owner', 'member')")"
|
|
[ "$membership" = "t" ] || {
|
|
echo "migration_runner must be allowed to SET ROLE schema_owner before production migration" >&2
|
|
exit 1
|
|
}
|
|
|
|
environment_value() {
|
|
local key="$1"
|
|
local value
|
|
value="$(sed -n -E "s/^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=[[:space:]]*(.*)$/\\2/p" .env.production.database)"
|
|
case "$value" in
|
|
\"*\") value="${value:1:${#value}-2}" ;;
|
|
\'*\') value="${value:1:${#value}-2}" ;;
|
|
esac
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
percent_encode() {
|
|
local value="$1"
|
|
local encoded=""
|
|
local character hex index
|
|
LC_ALL=C
|
|
for ((index = 0; index < ${#value}; index += 1)); do
|
|
character="${value:index:1}"
|
|
case "$character" in
|
|
[a-zA-Z0-9.~_-]) encoded+="$character" ;;
|
|
*)
|
|
printf -v hex '%%%02X' "'$character"
|
|
encoded+="$hex"
|
|
;;
|
|
esac
|
|
done
|
|
printf '%s' "$encoded"
|
|
}
|
|
|
|
migration_runner_password="$(environment_value MIGRATION_RUNNER_PASSWORD)"
|
|
[ -n "$migration_runner_password" ] || {
|
|
echo "production migration runner password is missing" >&2
|
|
exit 1
|
|
}
|
|
encoded_migration_runner_password="$(percent_encode "$migration_runner_password")"
|
|
unset migration_runner_password
|
|
migration_runner_database_url="postgresql://migration_runner:${encoded_migration_runner_password}@postgres:5432/jyotisha?options=-c%20role%3Dschema_owner"
|
|
unset encoded_migration_runner_password
|
|
|
|
migration_environment="$(mktemp "$state_directory/production-migration-env.XXXXXXXXXX")"
|
|
cleanup_migration_environment() {
|
|
rm -f -- "$migration_environment"
|
|
}
|
|
trap cleanup_migration_environment EXIT
|
|
chmod 600 "$migration_environment"
|
|
printf 'SCHEMA_DATABASE_URL=%s\n' "$migration_runner_database_url" >"$migration_environment"
|
|
unset migration_runner_database_url
|
|
|
|
set +e
|
|
DATABASE_ENV_FILE="$migration_environment" \
|
|
"${compose[@]}" --profile migration-check run --rm migration-checker
|
|
precheck_status=$?
|
|
set -e
|
|
if [ "$precheck_status" -ne 0 ] && [ "$precheck_status" -ne 3 ]; then
|
|
echo "production migration precheck failed safely" >&2
|
|
exit "$precheck_status"
|
|
fi
|
|
|
|
DATABASE_ENV_FILE="$migration_environment" \
|
|
"${compose[@]}" --profile migration run --rm migrator
|
|
|
|
set +e
|
|
DATABASE_ENV_FILE="$migration_environment" \
|
|
"${compose[@]}" --profile migration-check run --rm migration-checker
|
|
postcheck_status=$?
|
|
set -e
|
|
if [ "$postcheck_status" -ne 0 ]; then
|
|
echo "production migration postcheck did not converge" >&2
|
|
exit "$postcheck_status"
|
|
fi
|
|
|
|
echo "production schema migration verified for $DEPLOY_SHA"
|