Jyotisha/deploy/run-production-deploy.sh

#!/usr/bin/env bash
set -euo pipefail
set +x

required=(
  INCOMING_PATH DEPLOY_PATH API_IMAGE WEB_IMAGE DEPLOY_SHA
  EXPECTED_PREVIOUS_SHA ALLOW_ROLLBACK DOCKER_CONFIG PRODUCTION_URL
  PRODUCTION_ADMIN_URL VERIFICATION_MODE
)
case "${DOCKER_BIN:-docker}" in
  docker) docker_command=(docker) ;;
  "sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;;
  *) echo "unsafe production Docker command" >&2; exit 1 ;;
esac
for key in "${required[@]}"; do
  if [ -z "${!key:-}" ]; then
    echo "required production deployment input is missing: $key" >&2
    exit 1
  fi
done

sha_pattern='^[0-9a-f]{40}$'
digest_pattern='^[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:[1-9][0-9]{0,4})?(/[a-z0-9]+([._-][a-z0-9]+)*)+@sha256:[0-9a-f]{64}$'
image_id_pattern='^sha256:[0-9a-f]{64}$'
if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] ||
  [[ ! "$API_IMAGE" =~ $digest_pattern ]] ||
  [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then
  echo "unsafe production image identity" >&2
  exit 1
fi
api_repository="${API_IMAGE%@sha256:*}"
web_repository="${WEB_IMAGE%@sha256:*}"
if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then
  echo "invalid rollback authorization" >&2
  exit 1
fi
if [ "$VERIFICATION_MODE" != "internal" ] && [ "$VERIFICATION_MODE" != "public" ]; then
  echo "invalid production verification mode" >&2
  exit 1
fi
case "$INCOMING_PATH" in
  /tmp/jyotisha-production.*) ;;
  *) echo "unsafe incoming production path" >&2; exit 1 ;;
esac

state_directory="$DEPLOY_PATH/.state"
install -d -m 700 "$state_directory"
exec 9>"$state_directory/mutation.lock"
flock -n 9 || {
  echo "another production mutation holds the host lock" >&2
  exit 75
}

current_sha="not-deployed"
if [ -f "$state_directory/deployed-revision" ]; then
  current_sha="$(<"$state_directory/deployed-revision")"
else
  existing_web="$("${docker_command[@]}" ps -aq \
    --filter 'label=com.docker.compose.project=jyotisha-production' \
    --filter 'label=com.docker.compose.service=web' | head -n 1)"
  if [ -n "$existing_web" ]; then
    discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
      "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)"
    if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi
  fi
fi
if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ $sha_pattern ]]; then
  echo "invalid deployed production revision state" >&2
  exit 1
fi
if [ "$current_sha" != "$EXPECTED_PREVIOUS_SHA" ]; then
  echo "production revision changed while this deployment was waiting" >&2
  exit 1
fi
if [ "$ALLOW_ROLLBACK" = "false" ] &&
  [ "$current_sha" != "not-deployed" ] &&
  [ "$current_sha" != "$DEPLOY_SHA" ] &&
  [ "${FORWARD_REVISION_VERIFIED:-false}" != "true" ]; then
  echo "forward production revision was not verified" >&2
  exit 1
fi

container_id() {
  "${docker_command[@]}" ps -aq \
    --filter 'label=com.docker.compose.project=jyotisha-production' \
    --filter "label=com.docker.compose.service=$1" | head -n 1
}

repo_digest_for_container() {
  local service="$1"
  local repository="$2"
  local id image_id
  id="$(container_id "$service")"
  [ -n "$id" ] || return 0
  image_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")"
  "${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id" |
    awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }'
}

previous_api_image="$(repo_digest_for_container api "$api_repository")"
previous_web_image="$(repo_digest_for_container web "$web_repository")"
previous_api_id=""
previous_web_id=""
if [ -n "$(container_id api)" ]; then
  previous_api_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id api)")"
fi
if [ -n "$(container_id web)" ]; then
  previous_web_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id web)")"
fi

rollback_image() {
  local digest_ref="$1"
  local image_id="$2"
  if [[ "$digest_ref" =~ $digest_pattern ]]; then
    printf '%s' "$digest_ref"
  elif [[ "$image_id" =~ $image_id_pattern ]]; then
    printf '%s' "$image_id"
  fi
}

previous_api_target="$(rollback_image "$previous_api_image" "$previous_api_id")"
previous_web_target="$(rollback_image "$previous_web_image" "$previous_web_id")"

bash "$INCOMING_PATH/deploy/sync-production-tree.sh" \
  "$INCOMING_PATH" "$DEPLOY_PATH"

cd "$DEPLOY_PATH"
EXPECTED_PRODUCTION_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")"
[[ "$EXPECTED_PRODUCTION_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || {
  echo "production deployment owner is invalid" >&2
  exit 1
}
export EXPECTED_PRODUCTION_ENV_OWNER_UID
bash deploy/validate-production-env.sh \
  .env.production
bash deploy/validate-production-database-env.sh .env.production.database

compose=(
  "${docker_command[@]}" compose -p jyotisha-production --env-file .env.production
  -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml
  -f deploy/docker-compose.production.yml
)
export APP_ENV_FILE='../.env.production'
export DATABASE_ENV_FILE='../.env.production.database'
export CADDYFILE_PATH='./Caddyfile.production.selfhosted'
export SITE_ADDRESS='https://jyotisha.chat'
export GITHUB_SHA="$DEPLOY_SHA"

"${compose[@]}" config --quiet
"${compose[@]}" pull api web
"${compose[@]}" up -d --no-build --pull never --wait postgres

set +e
"${compose[@]}" --profile migration-check run --rm migration-checker
check_status=$?
set -e
if [ "$check_status" -eq 3 ]; then
  echo "pending migrations: run Migrate Production Database for $DEPLOY_SHA" >&2
  exit 3
fi
if [ "$check_status" -ne 0 ]; then
  echo "production migration check failed safely" >&2
  exit "$check_status"
fi

switched=false
rollback() {
  local status=$?
  if [ "$switched" = "true" ] &&
    [ -n "$previous_api_target" ] &&
    [ -n "$previous_web_target" ] &&
    [[ "$current_sha" =~ $sha_pattern ]]; then
    echo "production verification failed; restoring prior application images" >&2
    rollback_services=(api web)
    if [ "$VERIFICATION_MODE" = "public" ]; then rollback_services+=(caddy); fi
    API_IMAGE="$previous_api_target" WEB_IMAGE="$previous_web_target" \
      GITHUB_SHA="$current_sha" \
      "${compose[@]}" up -d --no-build --remove-orphans \
        "${rollback_services[@]}" || true
  fi
  exit "$status"
}
trap rollback ERR

switched=true
if [ "$VERIFICATION_MODE" = "public" ]; then
  "${compose[@]}" up -d --no-build --remove-orphans
  "${compose[@]}" up -d --no-build --force-recreate --no-deps caddy
else
  # Before DNS cutover, do not trigger public certificate issuance for domains
  # that still resolve to the old production host.
  "${compose[@]}" up -d --no-build api web
fi

verify_container_image() {
  local service="$1"
  local expected_ref="$2"
  local id expected_id running_id repo_digests
  id="$(container_id "$service")"
  [ -n "$id" ]
  expected_id="$("${docker_command[@]}" image inspect --format '{{.Id}}' "$expected_ref")"
  running_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")"
  [ "$running_id" = "$expected_id" ]
  repo_digests="$("${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$expected_id")"
  grep -Fqx "$expected_ref" <<<"$repo_digests"
}
verify_container_image api "$API_IMAGE"
verify_container_image web "$WEB_IMAGE"

"${compose[@]}" exec -T \
  -e EXPECTED_SHA="$DEPLOY_SHA" \
  -e PRODUCTION_URL="$PRODUCTION_URL" \
  -e PRODUCTION_ADMIN_URL="$PRODUCTION_ADMIN_URL" \
  -e VERIFICATION_MODE="$VERIFICATION_MODE" \
  web node --input-type=module <<'NODE'
import http from "node:http";
import { Pool } from "pg";

const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
for (const [role, key] of [
  ["identity", "IDENTITY_DATABASE_URL"],
  ["app", "APP_DATABASE_URL"],
  ["service", "SERVICE_DATABASE_URL"],
  ["admin", "ADMIN_DATABASE_URL"],
]) {
  const connectionString = process.env[key];
  if (!connectionString) {
    console.error(`database readiness missing for ${role}`);
    process.exit(1);
  }
  const pool = new Pool({ connectionString, max: 1, connectionTimeoutMillis: 5_000 });
  try {
    const result = await pool.query("select 1 as ready");
    if (result.rows[0]?.ready !== 1) throw new Error("unexpected readiness result");
  } catch (error) {
    console.error(`database readiness failed for ${role}`, error instanceof Error ? error.name : "query_error");
    process.exit(1);
  } finally {
    await pool.end();
  }
}

const internal = process.env.VERIFICATION_MODE === "internal";
const expectedUserAdminStatus = internal ? 403 : 404;
const request = (origin, path, options = {}) => {
  if (!internal) return fetch(`${origin}${path}`, options);
  return new Promise((resolve, reject) => {
    const request = http.request({
      host: "127.0.0.1",
      port: 3000,
      path,
      method: options.method ?? "GET",
      headers: { ...options.headers, Host: new URL(origin).host },
    }, (response) => {
      const chunks = [];
      response.on("data", (chunk) => chunks.push(chunk));
      response.on("end", () => {
        const status = response.statusCode ?? 0;
        resolve({
          status,
          ok: status >= 200 && status < 300,
          headers: { get: (name) => response.headers[name.toLowerCase()] ?? null },
          json: async () => JSON.parse(Buffer.concat(chunks).toString("utf8")),
        });
      });
    });
    request.on("error", reject);
    request.end();
  });
};
let observed = {};
for (let attempt = 1; attempt <= 12; attempt += 1) {
  try {
    const login = await request(process.env.PRODUCTION_URL, "/login");
    const userAdminPage = await request(process.env.PRODUCTION_URL, "/admin", { redirect: "manual" });
    const userAdminApi = await request(process.env.PRODUCTION_URL, "/api/admin/session");
    const adminPage = await request(process.env.PRODUCTION_ADMIN_URL, "/admin", { redirect: "manual" });
    const adminApi = await request(process.env.PRODUCTION_ADMIN_URL, "/api/admin/session");
    const account = await request(process.env.PRODUCTION_URL, "/api/account");
    const publicHealth = await request(process.env.PRODUCTION_URL, "/api/health");
    const publicBody = await publicHealth.json();
    const privateHealth = await fetch("http://api:5200/api/health");
    const privateBody = await privateHealth.json();
    observed = {
      attempt,
      login: login.status,
      userAdminPage: userAdminPage.status,
      userAdminApi: userAdminApi.status,
      adminPage: adminPage.status,
      adminLocation: adminPage.headers.get("location"),
      adminApi: adminApi.status,
      account: account.status,
      publicHealth: publicHealth.status,
      publicSha: publicBody.deployment?.gitCommit ?? "missing",
      privateHealth: privateHealth.status,
      privateStatus: privateBody.status ?? "missing",
      swissephAvailable: privateBody.swisseph_available === true,
      verificationMode: process.env.VERIFICATION_MODE,
    };
    if (
      login.ok
      && userAdminPage.status === expectedUserAdminStatus
      && userAdminApi.status === expectedUserAdminStatus
      && adminPage.status === 307
      && adminPage.headers.get("location") === "/login"
      && adminApi.status === 401
      && account.status === 401
      && publicHealth.ok
      && publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA
      && privateHealth.ok
      && privateBody.status === "ok"
      && privateBody.swisseph_available === true
    ) {
      process.exit(0);
    }
  } catch (error) {
    observed = {
      attempt,
      error: error instanceof Error ? error.name : "verification_error",
    };
  }
  if (attempt < 12) await delay(5_000);
}
console.error("production verification predicates did not converge", JSON.stringify(observed));
process.exit(1);
NODE

revision_file="$state_directory/deployed-revision.tmp.$$"
printf '%s\n' "$DEPLOY_SHA" >"$revision_file"
chmod 600 "$revision_file"
mv -f "$revision_file" "$state_directory/deployed-revision"
trap - ERR

printf 'previous_sha=%s\nprevious_api_image=%s\nprevious_api_id=%s\n' \
  "$current_sha" "${previous_api_image:-not-deployed}" "${previous_api_id:-not-deployed}"
printf 'previous_web_image=%s\nprevious_web_id=%s\nverified_sha=%s\n' \
  "${previous_web_image:-not-deployed}" "${previous_web_id:-not-deployed}" "$DEPLOY_SHA"