#!/usr/bin/env bash set -euo pipefail set +x required=( INCOMING_PATH DEPLOY_PATH API_IMAGE WEB_IMAGE DEPLOY_SHA EXPECTED_PREVIOUS_SHA ALLOW_ROLLBACK DOCKER_CONFIG STAGING_URL ) case "${DOCKER_BIN:-docker}" in docker) docker_command=(docker) ;; "sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;; *) echo "unsafe staging Docker command" >&2; exit 1 ;; esac for key in "${required[@]}"; do if [ -z "${!key:-}" ]; then echo "required staging deployment input is missing: $key" >&2 exit 1 fi done sha_pattern='^[0-9a-f]{40}$' digest_pattern='^[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:[1-9][0-9]{0,4})?(/[a-z0-9]+([._-][a-z0-9]+)*)+@sha256:[0-9a-f]{64}$' image_id_pattern='^sha256:[0-9a-f]{64}$' if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] || [[ ! "$API_IMAGE" =~ $digest_pattern ]] || [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then echo "unsafe staging image identity" >&2 exit 1 fi api_repository="${API_IMAGE%@sha256:*}" web_repository="${WEB_IMAGE%@sha256:*}" if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then echo "invalid rollback authorization" >&2 exit 1 fi case "$INCOMING_PATH" in /tmp/jyotisha-staging.*) ;; *) echo "unsafe incoming staging path" >&2; exit 1 ;; esac state_directory="$DEPLOY_PATH/.state" install -d -m 700 "$state_directory" exec 9>"$state_directory/mutation.lock" flock -n 9 || { echo "another staging mutation holds the host lock" >&2 exit 75 } current_sha="not-deployed" if [ -r "$state_directory/deployed-revision" ]; then current_sha="$(<"$state_directory/deployed-revision")" else existing_web="$("${docker_command[@]}" ps -aq \ --filter 'label=com.docker.compose.project=jyotisha-staging' \ --filter 'label=com.docker.compose.service=web' | head -n 1)" if [ -n "$existing_web" ]; then discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)" if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi fi fi if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ $sha_pattern ]]; then echo "invalid deployed staging revision state" >&2 exit 1 fi if [ "$current_sha" != "$EXPECTED_PREVIOUS_SHA" ]; then echo "staging revision changed while this deployment was waiting" >&2 exit 1 fi if [ "$ALLOW_ROLLBACK" = "false" ] && [ "$current_sha" != "not-deployed" ] && [ "$current_sha" != "$DEPLOY_SHA" ] && [ "${FORWARD_REVISION_VERIFIED:-false}" != "true" ]; then echo "forward staging revision was not verified" >&2 exit 1 fi container_id() { "${docker_command[@]}" ps -aq \ --filter 'label=com.docker.compose.project=jyotisha-staging' \ --filter "label=com.docker.compose.service=$1" | head -n 1 } repo_digest_for_container() { local service="$1" local repository="$2" local id image_id id="$(container_id "$service")" [ -n "$id" ] || return 0 image_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")" "${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id" | awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }' } previous_api_image="$(repo_digest_for_container api "$api_repository")" previous_web_image="$(repo_digest_for_container web "$web_repository")" previous_api_id="" previous_web_id="" if [ -n "$(container_id api)" ]; then previous_api_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id api)")" fi if [ -n "$(container_id web)" ]; then previous_web_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id web)")" fi rollback_image() { local digest_ref="$1" local image_id="$2" if [[ "$digest_ref" =~ $digest_pattern ]]; then printf '%s' "$digest_ref" elif [[ "$image_id" =~ $image_id_pattern ]]; then printf '%s' "$image_id" fi } previous_api_target="$(rollback_image "$previous_api_image" "$previous_api_id")" previous_web_target="$(rollback_image "$previous_web_image" "$previous_web_id")" bash "$INCOMING_PATH/deploy/sync-staging-tree.sh" \ "$INCOMING_PATH" "$DEPLOY_PATH" cd "$DEPLOY_PATH" EXPECTED_STAGING_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")" [[ "$EXPECTED_STAGING_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || { echo "staging deployment owner is invalid" >&2 exit 1 } export EXPECTED_STAGING_ENV_OWNER_UID bash deploy/prepare-staging-model-provider-env.sh .env.staging bash deploy/validate-staging-env.sh \ .env.staging staging.jyotisha.chat deploy/Caddyfile.staging bash deploy/validate-staging-database-env.sh .env.staging.database compose=( "${docker_command[@]}" compose -p jyotisha-staging --env-file .env.staging -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml -f deploy/docker-compose.staging.yml ) export APP_ENV_FILE='../.env.staging' export DATABASE_ENV_FILE='../.env.staging.database' export CADDYFILE_PATH='./Caddyfile.staging' export SITE_ADDRESS='https://staging.jyotisha.chat' export GITHUB_SHA="$DEPLOY_SHA" "${compose[@]}" config --quiet "${compose[@]}" pull api web "${compose[@]}" up -d --no-build --pull never --wait postgres set +e "${compose[@]}" --profile migration-check run --rm migration-checker check_status=$? set -e if [ "$check_status" -eq 3 ]; then echo "pending migrations: run Migrate Staging Database for $DEPLOY_SHA" >&2 exit 3 fi if [ "$check_status" -ne 0 ]; then echo "staging migration check failed safely" >&2 exit "$check_status" fi switched=false rollback() { local status=$? if [ "$switched" = "true" ] && [ -n "$previous_api_target" ] && [ -n "$previous_web_target" ] && [[ "$current_sha" =~ $sha_pattern ]]; then echo "staging verification failed; restoring prior application images" >&2 API_IMAGE="$previous_api_target" WEB_IMAGE="$previous_web_target" \ GITHUB_SHA="$current_sha" \ "${compose[@]}" up -d --no-build --remove-orphans \ api web caddy || true fi exit "$status" } trap rollback ERR switched=true "${compose[@]}" up -d --no-build --remove-orphans "${compose[@]}" up -d --no-build --force-recreate --no-deps caddy verify_container_image() { local service="$1" local expected_ref="$2" local id expected_id running_id repo_digests id="$(container_id "$service")" [ -n "$id" ] expected_id="$("${docker_command[@]}" image inspect --format '{{.Id}}' "$expected_ref")" running_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")" [ "$running_id" = "$expected_id" ] repo_digests="$("${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$expected_id")" grep -Fqx "$expected_ref" <<<"$repo_digests" } verify_container_image api "$API_IMAGE" verify_container_image web "$WEB_IMAGE" "${compose[@]}" exec -T \ -e EXPECTED_SHA="$DEPLOY_SHA" -e STAGING_URL="$STAGING_URL" \ web node --input-type=module <<'NODE' const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); let observed = {}; for (let attempt = 1; attempt <= 12; attempt += 1) { try { const login = await fetch(`${process.env.STAGING_URL}/login`); const userAdminPage = await fetch(`${process.env.STAGING_URL}/admin`, { redirect: "manual" }); const userAdminApi = await fetch(`${process.env.STAGING_URL}/api/admin/session`); const adminPage = await fetch(`${process.env.ADMIN_USER_ORIGIN}/admin`, { redirect: "manual" }); const adminApi = await fetch(`${process.env.ADMIN_USER_ORIGIN}/api/admin/session`); const account = await fetch(`${process.env.STAGING_URL}/api/account`); const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`); const publicBody = await publicHealth.json(); const privateHealth = await fetch("http://api:5200/api/health"); const privateBody = await privateHealth.json(); observed = { attempt, login: login.status, userAdminPage: userAdminPage.status, userAdminApi: userAdminApi.status, adminPage: adminPage.status, adminLocation: adminPage.headers.get("location"), adminApi: adminApi.status, account: account.status, publicHealth: publicHealth.status, publicSha: publicBody.deployment?.gitCommit ?? "missing", privateHealth: privateHealth.status, privateStatus: privateBody.status ?? "missing", swissephAvailable: privateBody.swisseph_available === true, }; if ( login.ok && userAdminPage.status === 404 && userAdminApi.status === 404 && adminPage.status === 307 && adminPage.headers.get("location") === "/login" && adminApi.status === 401 && account.status === 401 && publicHealth.ok && publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA && privateHealth.ok && privateBody.status === "ok" && privateBody.swisseph_available === true ) { process.exit(0); } } catch (error) { observed = { attempt, error: error instanceof Error ? error.name : "verification_error", }; } if (attempt < 12) await delay(5_000); } console.error("staging verification predicates did not converge", JSON.stringify(observed)); process.exit(1); NODE revision_file="$state_directory/deployed-revision.tmp.$$" printf '%s\n' "$DEPLOY_SHA" >"$revision_file" chmod 600 "$revision_file" mv -f "$revision_file" "$state_directory/deployed-revision" trap - ERR printf 'previous_sha=%s\nprevious_api_image=%s\nprevious_api_id=%s\n' \ "$current_sha" "${previous_api_image:-not-deployed}" "${previous_api_id:-not-deployed}" printf 'previous_web_image=%s\nprevious_web_id=%s\nverified_sha=%s\n' \ "${previous_web_image:-not-deployed}" "${previous_web_id:-not-deployed}" "$DEPLOY_SHA"