Jyotisha/deploy/run-staging-deploy.sh

#!/usr/bin/env bash
set -euo pipefail
set +x

required=(
  INCOMING_PATH DEPLOY_PATH API_IMAGE WEB_IMAGE DEPLOY_SHA
  EXPECTED_PREVIOUS_SHA ALLOW_ROLLBACK DOCKER_CONFIG STAGING_URL
)
case "${DOCKER_BIN:-docker}" in
  docker) docker_command=(docker) ;;
  "sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;;
  *) echo "unsafe staging Docker command" >&2; exit 1 ;;
esac
for key in "${required[@]}"; do
  if [ -z "${!key:-}" ]; then
    echo "required staging deployment input is missing: $key" >&2
    exit 1
  fi
done

sha_pattern='^[0-9a-f]{40}$'
digest_pattern='^[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:[1-9][0-9]{0,4})?(/[a-z0-9]+([._-][a-z0-9]+)*)+@sha256:[0-9a-f]{64}$'
image_id_pattern='^sha256:[0-9a-f]{64}$'
if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] ||
  [[ ! "$API_IMAGE" =~ $digest_pattern ]] ||
  [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then
  echo "unsafe staging image identity" >&2
  exit 1
fi
api_repository="${API_IMAGE%@sha256:*}"
web_repository="${WEB_IMAGE%@sha256:*}"
if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then
  echo "invalid rollback authorization" >&2
  exit 1
fi
case "$INCOMING_PATH" in
  /tmp/jyotisha-staging.*) ;;
  *) echo "unsafe incoming staging path" >&2; exit 1 ;;
esac

state_directory="$DEPLOY_PATH/.state"
install -d -m 700 "$state_directory"
exec 9>"$state_directory/mutation.lock"
flock -n 9 || {
  echo "another staging mutation holds the host lock" >&2
  exit 75
}

current_sha="not-deployed"
if [ -f "$state_directory/deployed-revision" ]; then
  current_sha="$(<"$state_directory/deployed-revision")"
else
  existing_web="$("${docker_command[@]}" ps -aq \
    --filter 'label=com.docker.compose.project=jyotisha-staging' \
    --filter 'label=com.docker.compose.service=web' | head -n 1)"
  if [ -n "$existing_web" ]; then
    discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
      "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)"
    if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi
  fi
fi
if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ $sha_pattern ]]; then
  echo "invalid deployed staging revision state" >&2
  exit 1
fi
if [ "$current_sha" != "$EXPECTED_PREVIOUS_SHA" ]; then
  echo "staging revision changed while this deployment was waiting" >&2
  exit 1
fi
if [ "$ALLOW_ROLLBACK" = "false" ] &&
  [ "$current_sha" != "not-deployed" ] &&
  [ "$current_sha" != "$DEPLOY_SHA" ] &&
  [ "${FORWARD_REVISION_VERIFIED:-false}" != "true" ]; then
  echo "forward staging revision was not verified" >&2
  exit 1
fi

container_id() {
  "${docker_command[@]}" ps -aq \
    --filter 'label=com.docker.compose.project=jyotisha-staging' \
    --filter "label=com.docker.compose.service=$1" | head -n 1
}

repo_digest_for_container() {
  local service="$1"
  local repository="$2"
  local id image_id
  id="$(container_id "$service")"
  [ -n "$id" ] || return 0
  image_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")"
  "${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id" |
    awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }'
}

previous_api_image="$(repo_digest_for_container api "$api_repository")"
previous_web_image="$(repo_digest_for_container web "$web_repository")"
previous_api_id=""
previous_web_id=""
if [ -n "$(container_id api)" ]; then
  previous_api_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id api)")"
fi
if [ -n "$(container_id web)" ]; then
  previous_web_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id web)")"
fi

rollback_image() {
  local digest_ref="$1"
  local image_id="$2"
  if [[ "$digest_ref" =~ $digest_pattern ]]; then
    printf '%s' "$digest_ref"
  elif [[ "$image_id" =~ $image_id_pattern ]]; then
    printf '%s' "$image_id"
  fi
}

previous_api_target="$(rollback_image "$previous_api_image" "$previous_api_id")"
previous_web_target="$(rollback_image "$previous_web_image" "$previous_web_id")"

bash "$INCOMING_PATH/deploy/sync-staging-tree.sh" \
  "$INCOMING_PATH" "$DEPLOY_PATH"

cd "$DEPLOY_PATH"
EXPECTED_STAGING_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")"
[[ "$EXPECTED_STAGING_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || {
  echo "staging deployment owner is invalid" >&2
  exit 1
}
export EXPECTED_STAGING_ENV_OWNER_UID
bash deploy/prepare-staging-model-provider-env.sh .env.staging
bash deploy/validate-staging-env.sh \
  .env.staging staging.jyotisha.chat deploy/Caddyfile.staging
bash deploy/validate-staging-database-env.sh .env.staging.database

compose=(
  "${docker_command[@]}" compose -p jyotisha-staging --env-file .env.staging
  -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml
  -f deploy/docker-compose.staging.yml
)
export APP_ENV_FILE='../.env.staging'
export DATABASE_ENV_FILE='../.env.staging.database'
export CADDYFILE_PATH='./Caddyfile.staging'
export SITE_ADDRESS='https://staging.jyotisha.chat'
export GITHUB_SHA="$DEPLOY_SHA"

"${compose[@]}" config --quiet
"${compose[@]}" pull api web
"${compose[@]}" up -d --no-build --pull never --wait postgres

set +e
"${compose[@]}" --profile migration-check run --rm migration-checker
check_status=$?
set -e
if [ "$check_status" -eq 3 ]; then
  echo "pending migrations: run Migrate Staging Database for $DEPLOY_SHA" >&2
  exit 3
fi
if [ "$check_status" -ne 0 ]; then
  echo "staging migration check failed safely" >&2
  exit "$check_status"
fi

switched=false
rollback() {
  local status=$?
  if [ "$switched" = "true" ] &&
    [ -n "$previous_api_target" ] &&
    [ -n "$previous_web_target" ] &&
    [[ "$current_sha" =~ $sha_pattern ]]; then
    echo "staging verification failed; restoring prior application images" >&2
    API_IMAGE="$previous_api_target" WEB_IMAGE="$previous_web_target" \
      GITHUB_SHA="$current_sha" \
      "${compose[@]}" up -d --no-build --remove-orphans \
        api web caddy || true
  fi
  exit "$status"
}
trap rollback ERR

switched=true
"${compose[@]}" up -d --no-build --remove-orphans
"${compose[@]}" up -d --no-build --force-recreate --no-deps caddy

verify_container_image() {
  local service="$1"
  local expected_ref="$2"
  local id expected_id running_id repo_digests
  id="$(container_id "$service")"
  [ -n "$id" ]
  expected_id="$("${docker_command[@]}" image inspect --format '{{.Id}}' "$expected_ref")"
  running_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")"
  [ "$running_id" = "$expected_id" ]
  repo_digests="$("${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$expected_id")"
  grep -Fqx "$expected_ref" <<<"$repo_digests"
}
verify_container_image api "$API_IMAGE"
verify_container_image web "$WEB_IMAGE"

"${compose[@]}" exec -T \
  -e EXPECTED_SHA="$DEPLOY_SHA" -e STAGING_URL="$STAGING_URL" \
  web node --input-type=module <<'NODE'
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
let observed = {};
for (let attempt = 1; attempt <= 12; attempt += 1) {
  try {
    const login = await fetch(`${process.env.STAGING_URL}/login`);
    const userAdminPage = await fetch(`${process.env.STAGING_URL}/admin`, { redirect: "manual" });
    const userAdminApi = await fetch(`${process.env.STAGING_URL}/api/admin/session`);
    const adminPage = await fetch(`${process.env.ADMIN_USER_ORIGIN}/admin`, { redirect: "manual" });
    const adminApi = await fetch(`${process.env.ADMIN_USER_ORIGIN}/api/admin/session`);
    const account = await fetch(`${process.env.STAGING_URL}/api/account`);
    const publicHealth = await fetch(`${process.env.STAGING_URL}/api/health`);
    const publicBody = await publicHealth.json();
    const privateHealth = await fetch("http://api:5200/api/health");
    const privateBody = await privateHealth.json();
    observed = {
      attempt,
      login: login.status,
      userAdminPage: userAdminPage.status,
      userAdminApi: userAdminApi.status,
      adminPage: adminPage.status,
      adminLocation: adminPage.headers.get("location"),
      adminApi: adminApi.status,
      account: account.status,
      publicHealth: publicHealth.status,
      publicSha: publicBody.deployment?.gitCommit ?? "missing",
      privateHealth: privateHealth.status,
      privateStatus: privateBody.status ?? "missing",
      swissephAvailable: privateBody.swisseph_available === true,
    };
    if (
      login.ok
      && userAdminPage.status === 404
      && userAdminApi.status === 404
      && adminPage.status === 307
      && adminPage.headers.get("location") === "/login"
      && adminApi.status === 401
      && account.status === 401
      && publicHealth.ok
      && publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA
      && privateHealth.ok
      && privateBody.status === "ok"
      && privateBody.swisseph_available === true
    ) {
      process.exit(0);
    }
  } catch (error) {
    observed = {
      attempt,
      error: error instanceof Error ? error.name : "verification_error",
    };
  }
  if (attempt < 12) await delay(5_000);
}
console.error("staging verification predicates did not converge", JSON.stringify(observed));
process.exit(1);
NODE

revision_file="$state_directory/deployed-revision.tmp.$$"
printf '%s\n' "$DEPLOY_SHA" >"$revision_file"
chmod 600 "$revision_file"
mv -f "$revision_file" "$state_directory/deployed-revision"
trap - ERR

printf 'previous_sha=%s\nprevious_api_image=%s\nprevious_api_id=%s\n' \
  "$current_sha" "${previous_api_image:-not-deployed}" "${previous_api_id:-not-deployed}"
printf 'previous_web_image=%s\nprevious_web_id=%s\nverified_sha=%s\n' \
  "${previous_web_image:-not-deployed}" "${previous_web_id:-not-deployed}" "$DEPLOY_SHA"