#!/usr/bin/env bash set -euo pipefail set +x required=( INCOMING_PATH DEPLOY_PATH API_IMAGE WEB_IMAGE DEPLOY_SHA EXPECTED_PREVIOUS_SHA ALLOW_ROLLBACK DOCKER_CONFIG PRODUCTION_URL PRODUCTION_ADMIN_URL VERIFICATION_MODE ) case "${DOCKER_BIN:-docker}" in docker) docker_command=(docker) ;; "sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;; *) echo "unsafe production Docker command" >&2; exit 1 ;; esac for key in "${required[@]}"; do if [ -z "${!key:-}" ]; then echo "required production deployment input is missing: $key" >&2 exit 1 fi done sha_pattern='^[0-9a-f]{40}$' digest_pattern='^[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:[1-9][0-9]{0,4})?(/[a-z0-9]+([._-][a-z0-9]+)*)+@sha256:[0-9a-f]{64}$' image_id_pattern='^sha256:[0-9a-f]{64}$' if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] || [[ ! "$API_IMAGE" =~ $digest_pattern ]] || [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then echo "unsafe production image identity" >&2 exit 1 fi api_repository="${API_IMAGE%@sha256:*}" web_repository="${WEB_IMAGE%@sha256:*}" if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then echo "invalid rollback authorization" >&2 exit 1 fi if [ "$VERIFICATION_MODE" != "internal" ] && [ "$VERIFICATION_MODE" != "public" ]; then echo "invalid production verification mode" >&2 exit 1 fi case "$INCOMING_PATH" in /tmp/jyotisha-production.*) ;; *) echo "unsafe incoming production path" >&2; exit 1 ;; esac state_directory="$DEPLOY_PATH/.state" install -d -m 700 "$state_directory" exec 9>"$state_directory/mutation.lock" flock -n 9 || { echo "another production mutation holds the host lock" >&2 exit 75 } current_sha="not-deployed" if [ -f "$state_directory/deployed-revision" ]; then current_sha="$(<"$state_directory/deployed-revision")" else existing_web="$("${docker_command[@]}" ps -aq \ --filter 'label=com.docker.compose.project=jyotisha-production' \ --filter 'label=com.docker.compose.service=web' | head -n 1)" if [ -n "$existing_web" ]; then discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)" if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi fi fi if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ $sha_pattern ]]; then echo "invalid deployed production revision state" >&2 exit 1 fi if [ "$current_sha" != "$EXPECTED_PREVIOUS_SHA" ]; then echo "production revision changed while this deployment was waiting" >&2 exit 1 fi if [ "$ALLOW_ROLLBACK" = "false" ] && [ "$current_sha" != "not-deployed" ] && [ "$current_sha" != "$DEPLOY_SHA" ] && [ "${FORWARD_REVISION_VERIFIED:-false}" != "true" ]; then echo "forward production revision was not verified" >&2 exit 1 fi container_id() { "${docker_command[@]}" ps -aq \ --filter 'label=com.docker.compose.project=jyotisha-production' \ --filter "label=com.docker.compose.service=$1" | head -n 1 } repo_digest_for_container() { local service="$1" local repository="$2" local id image_id id="$(container_id "$service")" [ -n "$id" ] || return 0 image_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")" "${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id" | awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }' } previous_api_image="$(repo_digest_for_container api "$api_repository")" previous_web_image="$(repo_digest_for_container web "$web_repository")" previous_api_id="" previous_web_id="" if [ -n "$(container_id api)" ]; then previous_api_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id api)")" fi if [ -n "$(container_id web)" ]; then previous_web_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id web)")" fi rollback_image() { local digest_ref="$1" local image_id="$2" if [[ "$digest_ref" =~ $digest_pattern ]]; then printf '%s' "$digest_ref" elif [[ "$image_id" =~ $image_id_pattern ]]; then printf '%s' "$image_id" fi } previous_api_target="$(rollback_image "$previous_api_image" "$previous_api_id")" previous_web_target="$(rollback_image "$previous_web_image" "$previous_web_id")" bash "$INCOMING_PATH/deploy/sync-production-tree.sh" \ "$INCOMING_PATH" "$DEPLOY_PATH" cd "$DEPLOY_PATH" EXPECTED_PRODUCTION_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")" [[ "$EXPECTED_PRODUCTION_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || { echo "production deployment owner is invalid" >&2 exit 1 } export EXPECTED_PRODUCTION_ENV_OWNER_UID bash deploy/validate-production-env.sh \ .env.production bash deploy/validate-production-database-env.sh .env.production.database compose=( "${docker_command[@]}" compose -p jyotisha-production --env-file .env.production -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml -f deploy/docker-compose.production.yml ) export APP_ENV_FILE='../.env.production' export DATABASE_ENV_FILE='../.env.production.database' export CADDYFILE_PATH='./Caddyfile.production.selfhosted' export SITE_ADDRESS='https://jyotisha.chat' export GITHUB_SHA="$DEPLOY_SHA" "${compose[@]}" config --quiet "${compose[@]}" pull api web "${compose[@]}" up -d --no-build --pull never --wait postgres set +e "${compose[@]}" --profile migration-check run --rm migration-checker check_status=$? set -e if [ "$check_status" -eq 3 ]; then echo "pending migrations: run Migrate Production Database for $DEPLOY_SHA" >&2 exit 3 fi if [ "$check_status" -ne 0 ]; then echo "production migration check failed safely" >&2 exit "$check_status" fi switched=false rollback() { local status=$? if [ "$switched" = "true" ] && [ -n "$previous_api_target" ] && [ -n "$previous_web_target" ] && [[ "$current_sha" =~ $sha_pattern ]]; then echo "production verification failed; restoring prior application images" >&2 rollback_services=(api web) if [ "$VERIFICATION_MODE" = "public" ]; then rollback_services+=(caddy); fi API_IMAGE="$previous_api_target" WEB_IMAGE="$previous_web_target" \ GITHUB_SHA="$current_sha" \ "${compose[@]}" up -d --no-build --remove-orphans \ "${rollback_services[@]}" || true fi exit "$status" } trap rollback ERR switched=true if [ "$VERIFICATION_MODE" = "public" ]; then "${compose[@]}" up -d --no-build --remove-orphans "${compose[@]}" up -d --no-build --force-recreate --no-deps caddy else # Before DNS cutover, do not trigger public certificate issuance for domains # that still resolve to the old production host. "${compose[@]}" up -d --no-build api web fi verify_container_image() { local service="$1" local expected_ref="$2" local id expected_id running_id repo_digests id="$(container_id "$service")" [ -n "$id" ] expected_id="$("${docker_command[@]}" image inspect --format '{{.Id}}' "$expected_ref")" running_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")" [ "$running_id" = "$expected_id" ] repo_digests="$("${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$expected_id")" grep -Fqx "$expected_ref" <<<"$repo_digests" } verify_container_image api "$API_IMAGE" verify_container_image web "$WEB_IMAGE" "${compose[@]}" exec -T \ -e EXPECTED_SHA="$DEPLOY_SHA" \ -e PRODUCTION_URL="$PRODUCTION_URL" \ -e PRODUCTION_ADMIN_URL="$PRODUCTION_ADMIN_URL" \ -e VERIFICATION_MODE="$VERIFICATION_MODE" \ web node --input-type=module <<'NODE' import { Pool } from "pg"; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); for (const [role, key] of [ ["identity", "IDENTITY_DATABASE_URL"], ["app", "APP_DATABASE_URL"], ["service", "SERVICE_DATABASE_URL"], ["admin", "ADMIN_DATABASE_URL"], ]) { const connectionString = process.env[key]; if (!connectionString) { console.error(`database readiness missing for ${role}`); process.exit(1); } const pool = new Pool({ connectionString, max: 1, connectionTimeoutMillis: 5_000 }); try { const result = await pool.query("select 1 as ready"); if (result.rows[0]?.ready !== 1) throw new Error("unexpected readiness result"); } catch (error) { console.error(`database readiness failed for ${role}`, error instanceof Error ? error.name : "query_error"); process.exit(1); } finally { await pool.end(); } } const internal = process.env.VERIFICATION_MODE === "internal"; const request = (origin, path, options = {}) => { const target = internal ? `http://127.0.0.1:3000${path}` : `${origin}${path}`; const headers = internal ? { ...options.headers, host: new URL(origin).host } : options.headers; return fetch(target, { ...options, headers }); }; let observed = {}; for (let attempt = 1; attempt <= 12; attempt += 1) { try { const login = await request(process.env.PRODUCTION_URL, "/login"); const userAdminPage = await request(process.env.PRODUCTION_URL, "/admin", { redirect: "manual" }); const userAdminApi = await request(process.env.PRODUCTION_URL, "/api/admin/session"); const adminPage = await request(process.env.PRODUCTION_ADMIN_URL, "/admin", { redirect: "manual" }); const adminApi = await request(process.env.PRODUCTION_ADMIN_URL, "/api/admin/session"); const account = await request(process.env.PRODUCTION_URL, "/api/account"); const publicHealth = await request(process.env.PRODUCTION_URL, "/api/health"); const publicBody = await publicHealth.json(); const privateHealth = await fetch("http://api:5200/api/health"); const privateBody = await privateHealth.json(); observed = { attempt, login: login.status, userAdminPage: userAdminPage.status, userAdminApi: userAdminApi.status, adminPage: adminPage.status, adminLocation: adminPage.headers.get("location"), adminApi: adminApi.status, account: account.status, publicHealth: publicHealth.status, publicSha: publicBody.deployment?.gitCommit ?? "missing", privateHealth: privateHealth.status, privateStatus: privateBody.status ?? "missing", swissephAvailable: privateBody.swisseph_available === true, verificationMode: process.env.VERIFICATION_MODE, }; if ( login.ok && userAdminPage.status === 404 && userAdminApi.status === 404 && adminPage.status === 307 && adminPage.headers.get("location") === "/login" && adminApi.status === 401 && account.status === 401 && publicHealth.ok && publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA && privateHealth.ok && privateBody.status === "ok" && privateBody.swisseph_available === true ) { process.exit(0); } } catch (error) { observed = { attempt, error: error instanceof Error ? error.name : "verification_error", }; } if (attempt < 12) await delay(5_000); } console.error("production verification predicates did not converge", JSON.stringify(observed)); process.exit(1); NODE revision_file="$state_directory/deployed-revision.tmp.$$" printf '%s\n' "$DEPLOY_SHA" >"$revision_file" chmod 600 "$revision_file" mv -f "$revision_file" "$state_directory/deployed-revision" trap - ERR printf 'previous_sha=%s\nprevious_api_image=%s\nprevious_api_id=%s\n' \ "$current_sha" "${previous_api_image:-not-deployed}" "${previous_api_id:-not-deployed}" printf 'previous_web_image=%s\nprevious_web_id=%s\nverified_sha=%s\n' \ "${previous_web_image:-not-deployed}" "${previous_web_id:-not-deployed}" "$DEPLOY_SHA"