From 9b07e12b0c708c4a9d7c65ee18ac5e145502de9a Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 8 Aug 2026 21:04:12 -0400 Subject: [PATCH] ops: prepare self-hosted production migration --- .gitea/workflows/deploy-production.yml | 392 ++++++++++++++++-- .github/workflows/deploy-production.yml | 192 +-------- AGENTS.md | 17 +- deploy/Caddyfile.production.selfhosted | 19 + deploy/README.md | 166 +++----- deploy/docker-compose.production.yml | 8 + deploy/run-production-deploy.sh | 314 ++++++++++++++ deploy/sync-production-tree.sh | 27 ++ deploy/validate-production-database-env.sh | 123 ++++++ deploy/validate-production-env.sh | 158 +++++++ .../production-server-migration-2026-08.md | 195 +++++++++ frontend/tests/health-deployment.test.ts | 90 ++-- .../tests/staging-backend-workflows.test.ts | 45 +- 13 files changed, 1342 insertions(+), 404 deletions(-) create mode 100644 deploy/Caddyfile.production.selfhosted create mode 100644 deploy/docker-compose.production.yml create mode 100755 deploy/run-production-deploy.sh create mode 100755 deploy/sync-production-tree.sh create mode 100755 deploy/validate-production-database-env.sh create mode 100755 deploy/validate-production-env.sh create mode 100644 docs/operations/production-server-migration-2026-08.md diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml index 7ef96484..96cf7240 100644 --- a/.gitea/workflows/deploy-production.yml +++ b/.gitea/workflows/deploy-production.yml @@ -1,59 +1,381 @@ -name: Deploy production (manual only) +name: Deploy production on: workflow_dispatch: + inputs: + deploy_sha: + description: Exact tested 40-character staging commit SHA + required: true + type: string + allow_rollback: + description: Explicitly permit a manual rollback to an older tested SHA + required: true + default: false + type: boolean + verification_mode: + description: Use internal before DNS cutover; public after DNS and TLS converge + required: true + default: internal + type: choice + options: + - internal + - public + +permissions: + contents: read + actions: read concurrency: - group: production + group: production-mutation cancel-in-progress: false - -env: - GITEA_SHA: ${{ gitea.sha }} - DEPLOY_HOST: 103.117.123.53 - DEPLOY_PORT: '22000' - DEPLOY_USER: root - DEPLOY_PATH: /opt/jyotisha-app + queue: max jobs: deploy: - runs-on: xiaoxin + runs-on: manman-linux timeout-minutes: 30 + env: + GITEA_SHA: ${{ gitea.sha }} + GITEA_API_URL: ${{ gitea.api_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + REGISTRY_HOST: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com + IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha + DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }} + DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }} + DEPLOY_USER: ${{ vars.PRODUCTION_USER }} + DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }} + PRODUCTION_URL: ${{ vars.PRODUCTION_URL }} + PRODUCTION_ADMIN_URL: ${{ vars.PRODUCTION_ADMIN_URL }} + STAGING_URL: ${{ vars.STAGING_URL }} + PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }} steps: - - name: Checkout current Gitea revision + - name: Validate tested revision and gate run + id: revision + env: + REQUESTED_SHA: ${{ inputs.deploy_sha }} + REQUESTED_ROLLBACK: ${{ inputs.allow_rollback }} + VERIFICATION_MODE: ${{ inputs.verification_mode }} run: | set -euo pipefail - git init . - git remote remove origin 2>/dev/null || true - git remote add origin https://git.copse.top/root/Jyotisha.git - git fetch --no-tags origin "$GITEA_SHA" main - git checkout --detach --force "$GITEA_SHA" - - name: Verify runner toolchain and current main + [[ "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; } + [[ "$VERIFICATION_MODE" == internal || "$VERIFICATION_MODE" == public ]] || { echo "invalid verification_mode" >&2; exit 1; } + [[ "$STAGING_URL" == "https://staging.jyotisha.chat" ]] || { echo "unexpected staging acceptance URL" >&2; exit 1; } + allow_rollback=false + if [[ "$REQUESTED_ROLLBACK" == true ]]; then + allow_rollback=true + fi + + runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$REQUESTED_SHA&branch=staging&event=push&status=success&limit=100")" + selected_run="$(jq -cer --arg sha "$REQUESTED_SHA" ' + [.workflow_runs[] | select( + (.path | split("@")[0] | endswith("backend-quality-gate.yml")) and + .head_sha == $sha and .head_branch == "staging" and + .event == "push" and .conclusion == "success" + )] | sort_by(.id) | reverse | first + ' <<<"$runs")" + gate_run_id="$(jq -er '.id' <<<"$selected_run")" + [[ "$gate_run_id" =~ ^[0-9]+$ ]] || { echo "no successful exact-SHA production quality gate run found" >&2; exit 1; } + + read_ref_sha() { + local branch="$1" + curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" | + jq -er --arg ref "refs/heads/$branch" ' + select(type == "array" and length == 1) | + .[0] | select(.ref == $ref) | .object.sha | + select(test("^[0-9a-f]{40}$")) + ' + } + staging_head="$(read_ref_sha staging)" + controller_sha="$(read_ref_sha main)" + [[ "$controller_sha" == "$staging_head" ]] || { echo "main and staging must identify the same reviewed release" >&2; exit 1; } + if [[ "$allow_rollback" == false && "$REQUESTED_SHA" != "$staging_head" ]]; then + echo "stale production revision refused; use explicit manual rollback only when intended" >&2 + exit 1 + fi + if [[ "$allow_rollback" == true && "$REQUESTED_SHA" != "$controller_sha" ]]; then + comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$REQUESTED_SHA...$controller_sha")" + jq -e --arg base "$REQUESTED_SHA" --arg head "$controller_sha" ' + (.commits // []) as $commits | + def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha]; + def reaches($sha; $seen): + if $sha == $base then true + elif ($seen | index($sha)) != null then false + else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end; + (.total_commits | type) == "number" and + .total_commits == ($commits | length) and ($commits | length) > 0 and + ([$commits[].sha] | length == (unique | length)) and reaches($head; []) + ' <<<"$comparison" >/dev/null || { echo "rollback revision is not in reviewed main history" >&2; exit 1; } + fi + + controller_gate_run_id="$gate_run_id" + if [[ "$controller_sha" != "$REQUESTED_SHA" ]]; then + controller_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$controller_sha&branch=staging&event=push&status=success&limit=100")" + controller_run="$(jq -cer --arg sha "$controller_sha" ' + [.workflow_runs[] | select( + (.path | split("@")[0] | endswith("backend-quality-gate.yml")) and + .head_sha == $sha and .head_branch == "staging" and + .event == "push" and .conclusion == "success" + )] | sort_by(.id) | reverse | first + ' <<<"$controller_runs")" + controller_gate_run_id="$(jq -er '.id' <<<"$controller_run")" + fi + [[ "$controller_gate_run_id" =~ ^[0-9]+$ ]] + + release_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$REQUESTED_SHA&event=workflow_dispatch&status=success&limit=100")" + jq -e --arg sha "$REQUESTED_SHA" ' + any(.workflow_runs[]?; + (.path | split("@")[0] | endswith("release-quality-gate.yml")) and + .head_sha == $sha and .event == "workflow_dispatch" and .conclusion == "success" + ) + ' <<<"$release_runs" >/dev/null || { echo "no successful exact-SHA manual release quality gate found" >&2; exit 1; } + + if [[ "$allow_rollback" == false ]]; then + observed_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 30 --retry 3 --retry-all-errors \ + "$STAGING_URL/api/health" | jq -er '.deployment.gitCommit | select(test("^[0-9a-f]{40}$"))')" + [[ "$observed_staging_sha" == "$REQUESTED_SHA" ]] || { echo "public staging has not accepted the requested SHA" >&2; exit 1; } + fi + { + echo "sha=$REQUESTED_SHA" + echo "gate_run_id=$gate_run_id" + echo "controller_sha=$controller_sha" + echo "controller_gate_run_id=$controller_gate_run_id" + echo "allow_rollback=$allow_rollback" + echo "verification_mode=$VERIFICATION_MODE" + } >>"$GITHUB_OUTPUT" + + - name: Prepare pinned Node tooling + env: + NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c + NODE_TOOL_IMAGE: node:22-bookworm-slim run: | set -euo pipefail - python3 --version + if ! docker image inspect "$NODE_TOOL_SOURCE_IMAGE" >/dev/null 2>&1; then + for attempt in 1 2 3; do + if timeout 180 docker pull "$NODE_TOOL_SOURCE_IMAGE"; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "Failed to preload $NODE_TOOL_IMAGE after $attempt attempts" >&2 + exit 1 + fi + sleep $((attempt * 15)) + done + fi + docker tag "$NODE_TOOL_SOURCE_IMAGE" "$NODE_TOOL_IMAGE" + docker image inspect "$NODE_TOOL_IMAGE" >/dev/null + tool_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-node-tools.XXXXXX")" + cat > "$tool_dir/node" <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + workdir="$(pwd -P)" + exec docker run --rm \ + --user "$(id -u):$(id -g)" \ + --volume "$workdir:$workdir" \ + --workdir "$workdir" \ + --env HOME=/tmp \ + node:22-bookworm-slim "${0##*/}" "$@" + EOF + chmod 0755 "$tool_dir/node" + ln -s node "$tool_dir/npm" + test -n "${GITHUB_PATH:-}" + printf '%s\n' "$tool_dir" >> "$GITHUB_PATH" + export PATH="$tool_dir:$PATH" node --version npm --version - docker version - test "$(git rev-parse HEAD)" = "$(git ls-remote origin refs/heads/main | awk '{print $1}')" - - name: Configure pinned production SSH + + - name: Download target and controller gate artifacts env: - SSH_PRIVATE_KEY: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }} + TARGET_GATE_RUN_ID: ${{ steps.revision.outputs.gate_run_id }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + CONTROLLER_GATE_RUN_ID: ${{ steps.revision.outputs.controller_gate_run_id }} + CONTROLLER_SHA: ${{ steps.revision.outputs.controller_sha }} run: | set -euo pipefail - install -m 700 -d ~/.ssh - printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/jyotisha-production - chmod 600 ~/.ssh/jyotisha-production - printf '%s\n' '[103.117.123.53]:22000 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHQJvN2Mo3Yq8e6ZIK4P2blJ5Vjj0HbknEuk7TyjhMbO' > ~/.ssh/known_hosts - - name: Sync and rebuild reviewed revision + download_bundle() { + local run_id="$1" sha="$2" destination="$3" zip_path="$4" + local prefix artifacts selected name id attempt + prefix="staging-image-manifest-$sha-" + artifacts="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs/$run_id/artifacts?limit=100")" + selected="$(jq -cer --arg prefix "$prefix" ' + [(.artifacts // [])[] + | select(.expired == false and (.name | startswith($prefix))) + | . + {attempt: ((.name | ltrimstr($prefix)) | tonumber?)} + | select(.attempt != null and .attempt >= 1) + ] | sort_by(.attempt, .id) | reverse | first + ' <<<"$artifacts")" + name="$(jq -er '.name' <<<"$selected")" + id="$(jq -er '.id' <<<"$selected")" + attempt="${name#"$prefix"}" + [[ "$name" == "$prefix"* && "$attempt" =~ ^[1-9][0-9]*$ && "$id" =~ ^[0-9]+$ ]] + install -d -m 700 "$destination" + curl --fail --silent --show-error --location --connect-timeout 15 --max-time 120 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$id/zip" \ + --output "$zip_path" + python3 - "$zip_path" "$destination" <<'PY' + import pathlib, stat, sys, zipfile + archive = pathlib.Path(sys.argv[1]) + destination = pathlib.Path(sys.argv[2]) + allowed = {"manifest.env", "controller.tar"} + with zipfile.ZipFile(archive) as bundle: + entries = bundle.infolist() + names = [entry.filename for entry in entries] + if len(names) != len(set(names)) or not names or not set(names).issubset(allowed): + raise SystemExit("invalid production artifact bundle") + if sum(entry.file_size for entry in entries) > 3 * 1024 * 1024: + raise SystemExit("production artifact bundle is too large") + for entry in entries: + path = pathlib.PurePosixPath(entry.filename) + mode = entry.external_attr >> 16 + if path.is_absolute() or ".." in path.parts or path.name != entry.filename: + raise SystemExit("unsafe production artifact path") + if mode and not stat.S_ISREG(mode): + raise SystemExit("unsafe production artifact type") + target = destination / entry.filename + with bundle.open(entry) as source, target.open("xb") as output: + output.write(source.read()) + PY + [[ -f "$destination/manifest.env" ]] + } + rm -rf artifacts/staging-image artifacts/controller + download_bundle "$TARGET_GATE_RUN_ID" "$DEPLOY_SHA" artifacts/staging-image "${RUNNER_TEMP}/staging-target.zip" + download_bundle "$CONTROLLER_GATE_RUN_ID" "$CONTROLLER_SHA" artifacts/controller "${RUNNER_TEMP}/production-controller.zip" + + - name: Validate gate-attested controller and immutable image manifest + id: images + env: + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + CONTROLLER_SHA: ${{ steps.revision.outputs.controller_sha }} run: | set -euo pipefail - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-production -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" - rsync -az --delete --exclude='.git/' --exclude='.env.production' --exclude='frontend/node_modules/' --exclude='frontend/.next/' -e "ssh $SSH_OPTIONS" ./ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "cd '$DEPLOY_PATH' && GITHUB_SHA='$GITEA_SHA' docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build --remove-orphans" - - name: Verify production + controller_manifest=artifacts/controller/manifest.env + controller_tar=artifacts/controller/controller.tar + [[ -f "$controller_tar" ]] + [[ "$(wc -l < "$controller_manifest" | tr -d ' ')" == 4 ]] + manifest_controller_sha="$(awk -F= '$1 == "git_sha" {print $2}' "$controller_manifest")" + expected_controller_digest="$(awk -F= '$1 == "controller_sha256" {print $2}' "$controller_manifest")" + [[ "$manifest_controller_sha" == "$CONTROLLER_SHA" ]] + [[ "$expected_controller_digest" =~ ^[0-9a-f]{64}$ ]] + printf '%s %s\n' "$expected_controller_digest" "$controller_tar" | sha256sum --check --status + python3 - "$controller_tar" <<'PY' + import pathlib, sys, tarfile + archive = pathlib.Path(sys.argv[1]) + required = {"deploy/run-production-deploy.sh", "frontend/scripts/staging-image-manifest.mjs"} + with tarfile.open(archive, "r:") as bundle: + members = bundle.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)) or not required.issubset(names): + raise SystemExit("invalid production controller bundle") + if sum(member.size for member in members) > 2 * 1024 * 1024: + raise SystemExit("production controller bundle is too large") + for member in members: + path = pathlib.PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()): + raise SystemExit("unsafe production controller bundle") + PY + install -d -m 700 artifacts/controller/extracted + tar -xf "$controller_tar" -C artifacts/controller/extracted + node artifacts/controller/extracted/frontend/scripts/staging-image-manifest.mjs \ + "$controller_manifest" "$CONTROLLER_SHA" "$IMAGE_REPOSITORY" >/dev/null + node artifacts/controller/extracted/frontend/scripts/staging-image-manifest.mjs \ + artifacts/staging-image/manifest.env "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT" + + - name: Deploy exact image digests under pinned SSH identity + env: + SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + DEPLOY_SHA: ${{ steps.revision.outputs.sha }} + API_IMAGE: ${{ steps.images.outputs.api_image }} + WEB_IMAGE: ${{ steps.images.outputs.web_image }} + ALLOW_ROLLBACK: ${{ steps.revision.outputs.allow_rollback }} + VERIFICATION_MODE: ${{ steps.revision.outputs.verification_mode }} run: | set -euo pipefail - curl -fsS --retry 12 --retry-delay 5 https://jyotisha.chat/login >/dev/null - test "$(curl -sS -o /dev/null -w '%{http_code}' https://jyotisha.chat/api/account)" = 401 - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-production -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" "cd '$DEPLOY_PATH' && docker compose --env-file .env.production -f deploy/docker-compose.server.yml exec -T web node -e 'fetch(\"http://api:5200/api/health\").then(async r=>{const b=await r.json();if(!r.ok||b.status!==\"ok\"||b.swisseph_available!==true)process.exit(1)})'" + [[ "$DEPLOY_HOST" == "118.194.235.34" ]] + [[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 )) + [[ "$DEPLOY_USER" == "deploy" ]] + [[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]] + [[ "$PRODUCTION_URL" == "https://jyotisha.chat" ]] + [[ "$PRODUCTION_ADMIN_URL" == "https://admin.jyotisha.chat" ]] + [[ "$VERIFICATION_MODE" == internal || "$VERIFICATION_MODE" == public ]] + ssh_root="${RUNNER_TEMP}/production-ssh" + key_path="$ssh_root/id_ed25519" + known_hosts_path="$ssh_root/known_hosts" + incoming="" + install -m 700 -d "$ssh_root" + test -n "$SSH_PRIVATE_KEY_BASE64" + printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode > "$key_path" + printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' > "$known_hosts_path" + chmod 600 "$key_path" "$known_hosts_path" + ssh-keygen -y -f "$key_path" >/dev/null + ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path") + remote="$DEPLOY_USER@$DEPLOY_HOST" + require_current_release_heads() { + [[ "$ALLOW_ROLLBACK" == true ]] && return + current_staging="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" | + jq -er 'select(type == "array" and length == 1) | .[0] | + select(.ref == "refs/heads/staging") | .object.sha | + select(test("^[0-9a-f]{40}$"))')" + current_main="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/main" | + jq -er 'select(type == "array" and length == 1) | .[0] | + select(.ref == "refs/heads/main") | .object.sha | + select(test("^[0-9a-f]{40}$"))')" + [[ "$current_main" == "$DEPLOY_SHA" && "$current_staging" == "$DEPLOY_SHA" ]] || { + echo "main or staging advanced during deployment; refusing stale mutation" >&2 + exit 1 + } + } + cleanup() { + if [[ -n "$incoming" ]]; then + ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; sudo -n rm -rf -- '$incoming'" >/dev/null 2>&1 || true + fi + rm -rf -- "$ssh_root" + } + trap cleanup EXIT + incoming="$(ssh "${ssh_options[@]}" "$remote" "mktemp -d /tmp/jyotisha-production.XXXXXXXXXX")" + [[ "$incoming" == /tmp/jyotisha-production.* ]] + ssh "${ssh_options[@]}" "$remote" "install -d -m 700 '$incoming/.docker'" + scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" artifacts/controller/controller.tar "$remote:$incoming/controller.tar" + ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/controller.tar' -C '$incoming' && rm -f -- '$incoming/controller.tar'" + previous_sha="$(ssh "${ssh_options[@]}" "$remote" "state='$DEPLOY_PATH/.state/deployed-revision'; if [ -f \"\$state\" ]; then cat \"\$state\"; else id=\$(sudo -n docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-production' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then sudo -n docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; else printf not-deployed; fi; fi")" + [[ "$previous_sha" == not-deployed || "$previous_sha" =~ ^[0-9a-f]{40}$ ]] || exit 1 + forward_verified=false + if [[ "$previous_sha" != not-deployed && "$previous_sha" != "$DEPLOY_SHA" && "$ALLOW_ROLLBACK" != true ]]; then + comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \ + --header "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")" + jq -e --arg base "$previous_sha" --arg head "$DEPLOY_SHA" ' + (.commits // []) as $commits | + def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha]; + def reaches($sha; $seen): + if $sha == $base then true + elif ($seen | index($sha)) != null then false + else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end; + (.total_commits | type) == "number" and + .total_commits == ($commits | length) and ($commits | length) > 0 and + ([$commits[].sha] | length == (unique | length)) and reaches($head; []) + ' <<<"$comparison" >/dev/null || { echo "automatic production rollback or divergent deploy refused" >&2; exit 1; } + forward_verified=true + fi + require_current_release_heads + printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin" + ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' API_IMAGE='$API_IMAGE' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' ALLOW_ROLLBACK='$ALLOW_ROLLBACK' FORWARD_REVISION_VERIFIED='$forward_verified' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' PRODUCTION_URL='$PRODUCTION_URL' PRODUCTION_ADMIN_URL='$PRODUCTION_ADMIN_URL' VERIFICATION_MODE='$VERIFICATION_MODE' bash '$incoming/deploy/run-production-deploy.sh'" + require_current_release_heads diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 03f5fd70..5da0a860 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -1,4 +1,4 @@ -name: Deploy production +name: Production deployment moved to Gitea on: workflow_dispatch: @@ -6,193 +6,11 @@ on: permissions: contents: read -concurrency: - group: production - cancel-in-progress: false - -env: - DEPLOY_HOST: 103.117.123.53 - DEPLOY_PORT: "22000" - DEPLOY_USER: root - DEPLOY_PATH: /opt/jyotisha-app - jobs: - deploy: + retired: runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout tested revision - uses: actions/checkout@v4 - with: - ref: ${{ github.sha }} - - - name: Reject stale CI revision - id: revision + - name: Refuse deployment from the mirror run: | - tested_sha="$(git rev-parse HEAD)" - main_sha="$(git ls-remote origin refs/heads/main | awk '{print $1}')" - if [ "$tested_sha" = "$main_sha" ]; then - echo "deploy=true" >> "$GITHUB_OUTPUT" - echo "Deploying current main revision $tested_sha" - else - echo "deploy=false" >> "$GITHUB_OUTPUT" - echo "Skipping stale CI revision $tested_sha; current main is $main_sha" - fi - - - name: Configure SSH - if: steps.revision.outputs.deploy == 'true' - env: - SSH_PRIVATE_KEY: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }} - run: | - install -m 700 -d ~/.ssh - printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/jyotisha-production - chmod 600 ~/.ssh/jyotisha-production - printf '%s\n' '[103.117.123.53]:22000 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHQJvN2Mo3Yq8e6ZIK4P2blJ5Vjj0HbknEuk7TyjhMbO' > ~/.ssh/known_hosts - - - name: Sync reviewed production environment keys - if: steps.revision.outputs.deploy == 'true' - env: - GEOAPIFY_API_KEY: ${{ secrets.GEOAPIFY_API_KEY }} - VEDASTRO_API_KEY: ${{ secrets.VEDASTRO_API_KEY }} - VEDASTRO_API_ENDPOINT: ${{ secrets.VEDASTRO_API_ENDPOINT }} - VEDASTRO_ENABLE_NETWORK: ${{ secrets.VEDASTRO_ENABLE_NETWORK }} - VEDASTRO_GATEWAY_MODE: ${{ secrets.VEDASTRO_GATEWAY_MODE }} - VEDASTRO_RANGE_SCAN_NETWORK_ENABLED: ${{ secrets.VEDASTRO_RANGE_SCAN_NETWORK_ENABLED }} - VEDASTRO_TIMEOUT_SECONDS: ${{ secrets.VEDASTRO_TIMEOUT_SECONDS }} - JYOTISH_DYNAMIC_RECTIFICATION_TOKEN: ${{ secrets.JYOTISH_DYNAMIC_RECTIFICATION_TOKEN }} - run: | - set -euo pipefail - set +x - UPDATE_FILE="$RUNNER_TEMP/production-env-update.json" - REMOTE_UPDATE_FILE="$DEPLOY_PATH/.env.production.update.$GITHUB_RUN_ID" - export UPDATE_FILE - umask 077 - python3 - <<'PY' - import json - import os - from pathlib import Path - - keys = ( - "GEOAPIFY_API_KEY", - "VEDASTRO_API_KEY", - "VEDASTRO_API_ENDPOINT", - "VEDASTRO_ENABLE_NETWORK", - "VEDASTRO_GATEWAY_MODE", - "VEDASTRO_RANGE_SCAN_NETWORK_ENABLED", - "VEDASTRO_TIMEOUT_SECONDS", - "JYOTISH_DYNAMIC_RECTIFICATION_TOKEN", - ) - values = {key: os.environ.get(key, "") for key in keys} - missing = [key for key, value in values.items() if not value] - if missing: - raise SystemExit("required production environment secret is missing") - Path(os.environ["UPDATE_FILE"]).write_text( - json.dumps(values, ensure_ascii=False), - encoding="utf-8", - ) - PY - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-production -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - SCP_OPTIONS="-i $HOME/.ssh/jyotisha-production -P $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - scp $SCP_OPTIONS "$UPDATE_FILE" "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_UPDATE_FILE" - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && python3 - '$REMOTE_UPDATE_FILE'" <<'PY' - import json - import os - import re - import sys - import tempfile - from pathlib import Path - - env_path = Path(".env.production") - update_path = Path(sys.argv[1]) - try: - if not env_path.is_file(): - raise SystemExit("production environment file is missing") - updates = json.loads(update_path.read_text(encoding="utf-8")) - if not isinstance(updates, dict) or not updates: - raise SystemExit("production environment update is empty") - for key, value in updates.items(): - if not re.fullmatch(r"[A-Z][A-Z0-9_]*", key): - raise SystemExit("production environment key is invalid") - if not isinstance(value, str) or not value or "\n" in value or "\0" in value: - raise SystemExit("production environment value is invalid") - - original = env_path.read_text(encoding="utf-8").splitlines() - output = [] - written = set() - assignment = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=") - - def quote(value): - return "'" + value.replace("'", "'\"'\"'") + "'" - - for line in original: - match = assignment.match(line) - key = match.group(1) if match else None - if key not in updates: - output.append(line) - continue - if key not in written: - output.append(f"{key}={quote(updates[key])}") - written.add(key) - for key, value in updates.items(): - if key not in written: - output.append(f"{key}={quote(value)}") - - fd, temporary_name = tempfile.mkstemp( - prefix=".env.production.", - dir=str(env_path.parent), - text=True, - ) - try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write("\n".join(output) + "\n") - handle.flush() - os.fsync(handle.fileno()) - os.chmod(temporary_name, 0o600) - os.replace(temporary_name, env_path) - finally: - if os.path.exists(temporary_name): - os.unlink(temporary_name) - print(f"updated {len(updates)} production environment keys") - finally: - update_path.unlink(missing_ok=True) - PY - rm -f "$UPDATE_FILE" - - - name: Sync and rebuild - if: steps.revision.outputs.deploy == 'true' - env: - DEPLOY_GIT_SHA: ${{ github.sha }} - run: | - SSH_OPTIONS="-i $HOME/.ssh/jyotisha-production -p $DEPLOY_PORT -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20" - RSYNC_SSH="ssh $SSH_OPTIONS" - rsync -az --delete \ - --exclude='.git/' \ - --exclude='.env.production' \ - --exclude='frontend/node_modules/' \ - --exclude='frontend/.next/' \ - -e "$RSYNC_SSH" \ - ./ "$DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_PATH/" - - ssh $SSH_OPTIONS "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && GITHUB_SHA='$DEPLOY_GIT_SHA' docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build --remove-orphans" - - - name: Verify production - if: steps.revision.outputs.deploy == 'true' - env: - DEPLOY_GIT_SHA: ${{ github.sha }} - run: | - curl --fail --silent --show-error --retry 12 --retry-delay 5 https://jyotisha.chat/login >/dev/null - test "$(curl --silent --output /dev/null --write-out '%{http_code}' https://jyotisha.chat/api/account)" = "401" - deployed_sha="" - for attempt in $(seq 1 24); do - deployed_sha="$(curl --fail --silent --show-error https://jyotisha.chat/api/health | python3 -c 'import json, sys; print(json.load(sys.stdin).get("deployment", {}).get("gitCommit", ""))')" || deployed_sha="" - [ "$deployed_sha" = "$DEPLOY_GIT_SHA" ] && break - sleep 5 - done - test "$deployed_sha" = "$DEPLOY_GIT_SHA" || { echo "Production revision did not converge: expected $DEPLOY_GIT_SHA, got ${deployed_sha:-empty}" >&2; exit 1; } - ssh -i ~/.ssh/jyotisha-production -p "$DEPLOY_PORT" \ - -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=20 \ - "$DEPLOY_USER@$DEPLOY_HOST" \ - "cd '$DEPLOY_PATH' && docker compose --env-file .env.production -f deploy/docker-compose.server.yml exec -T web node -e 'fetch(\"http://api:5200/api/health\").then(async r => { const body = await r.json(); if (!r.ok || body.status !== \"ok\" || body.swisseph_available !== true) process.exit(1); console.log(JSON.stringify(body)); })'" + echo "Production deployment is controlled by .gitea/workflows/deploy-production.yml in git.copse.top." >&2 + exit 1 diff --git a/AGENTS.md b/AGENTS.md index 8b463b81..3aece1fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,21 +8,22 @@ - Production domain: `https://jyotisha.chat` - Primary source: `https://git.copse.top/root/Jyotisha.git`; GitHub upstream/mirror: `https://github.com/jesse-ux/Jyotisha.git` -- Server: Hong Kong Ubuntu 22.04 VPS, `103.117.123.53`, SSH port `22000` -- Runtime: `/opt/jyotisha-app`, Docker Compose file `deploy/docker-compose.server.yml` -- Secrets: `/opt/jyotisha-app/.env.production`; never print, copy into chat, or commit +- Migration target: Ubuntu VPS `118.194.235.34`; confirmed SSH port is a required deployment variable +- Target runtime: `/opt/jyotisha-production`, Compose project `jyotisha-production` +- Target secrets: `/opt/jyotisha-production/.env.production` and `.env.production.database`; never print, copy into chat, or commit - Public edge: Caddy only; Next.js `3000` and Python API `5200` stay Docker-private -- Managed services: Spaceship DNS, Supabase project `vtvnfqmonbfuxmqkqdlc`, external model API -- Capacity boundary: 1 vCPU / 2 GB RAM / 40 GB disk / 5 Mbps; demo and low concurrency only +- Migration source: Spaceship DNS and Supabase project `vtvnfqmonbfuxmqkqdlc`; production target uses private PostgreSQL + Better Auth +- Capacity boundary: 2 vCPU / 4 GB RAM; use digest-pinned images, bounded DB pools, and no on-host application builds +- Cutover runbook: `docs/operations/production-server-migration-2026-08.md`; old VPS/Supabase remain rollback assets until final reconciliation Deployment safety rules: 1. Run `git status --short --branch` before packaging; do not overwrite unrelated dirty files. -2. Verify `dig +short @launch1.spaceship.net A jyotisha.chat` returns `103.117.123.53` before troubleshooting Caddy certificate issuance. -3. Keep Supabase Auth Site URL and redirect URLs aligned with `https://jyotisha.chat`. +2. Before cutover, verify the authoritative DNS still matches the documented current phase; after cutover both user and admin hosts must resolve to `118.194.235.34`. +3. The target uses Better Auth with exact user/admin origins; migrated users must sign in again by OTP. 4. After deployment, verify `/login`, logged-out `/api/account` = `401`, internal `/api/health` = `200`, and `swisseph_available = true`. 5. Never expose port `5200`, `SUPABASE_SERVICE_ROLE_KEY`, model keys, user JWTs, passwords, or SSH private keys. -6. Production GitHub Actions validation, deployment, and migration workflows are manual-only. The explicitly authorized staging `Staging Backend Quality Gate` may run automatically for pull requests and pushes to `staging`, and a successful staging gate may automatically trigger `Deploy staging`; `Migrate Staging Database` remains manual-only. Run the required production validation workflows from the Actions page before manually starting production deployment; the production workflow and required secret are documented in `deploy/README.md`. +6. Production deployment is manual-only in Gitea. It consumes the exact staging-gate image digests, requires matching `main`/`staging`/public-staging SHA plus the manual release gate, and never changes DNS or imports production data. GitHub production deployment is retired. ## 1. High-Rigor Override diff --git a/deploy/Caddyfile.production.selfhosted b/deploy/Caddyfile.production.selfhosted new file mode 100644 index 00000000..a03e91b5 --- /dev/null +++ b/deploy/Caddyfile.production.selfhosted @@ -0,0 +1,19 @@ +{$SITE_ADDRESS:https://jyotisha.chat} { + encode zstd gzip + + @adminPaths path /admin /admin/* /api/admin/* + respond @adminPaths "Not found" 404 + + reverse_proxy web:3000 +} + +https://admin.jyotisha.chat { + encode zstd gzip + @root path / + redir @root /admin 308 + reverse_proxy web:3000 +} + +https://www.jyotisha.chat { + redir https://jyotisha.chat{uri} 308 +} diff --git a/deploy/README.md b/deploy/README.md index 3726b4ce..025365d8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,77 +1,78 @@ # Production deployment and maintenance -This file is the operational source of truth for the current Jyotisha demo deployment. +This file is the operational source of truth for Jyotisha deployment. The production migration is governed by `docs/operations/production-server-migration-2026-08.md`; do not change DNS or retire the old environment outside that runbook. -## Current production +## Production migration state | Item | Value | | --- | --- | | Public domain | `https://jyotisha.chat` | | DNS | Spaceship nameservers (`launch1.spaceship.net`, `launch2.spaceship.net`) | -| Server | Hong Kong VPS, Ubuntu 22.04 x86_64 | -| Public host | `103.117.123.53` | -| SSH | port `22000`, public-key authentication only | -| Capacity | 1 vCPU / 2 GB RAM / 40 GB disk / 5 Mbps | -| App directory | `/opt/jyotisha-app` | -| Environment file | `/opt/jyotisha-app/.env.production` (`0600`) | +| Current public host | Old VPS; keep as a rollback asset until reconciliation completes | +| Target host | `118.194.235.34`, Ubuntu x86_64 | +| Target SSH | dedicated `deploy` user, confirmed variable port, public-key authentication only | +| Target capacity | 2 vCPU / 4 GB RAM; no application builds on host | +| Target app directory | `/opt/jyotisha-production` | +| Target environment files | `.env.production` and `.env.production.database` (`0600`) | | Primary source repository | `https://git.copse.top/root/Jyotisha.git` | | GitHub upstream/mirror | `https://github.com/jesse-ux/Jyotisha.git` | -| Supabase project | `vtvnfqmonbfuxmqkqdlc` | +| Migration source | Supabase project `vtvnfqmonbfuxmqkqdlc` + Supabase Auth | +| Migration target | private PostgreSQL 17 + Better Auth | -This machine is suitable for a client demo and low concurrency. Supabase and the model provider stay managed externally; do not self-host them on this VPS. +This migration changes both infrastructure and persistence. It is a controlled Supabase-to-private-PostgreSQL ETL, not a database-volume copy or full-dump restore. Keep the old VPS for 7–14 days and Supabase for 14–30 days after cutover. ## Architecture ```text -Spaceship DNS - -> Caddy :80/:443 - -> web:3000 (Next.js + Mastra, Docker-private) - -> api:5200 (Python Jyotish API, Docker-private) - -> Swiss Ephemeris / local engine - -> VedAstro gateway with local fallback - -> Supabase Cloud - -> external OpenAI-compatible model API +Spaceship DNS -> Caddy :80/:443 -> web:3000 -> api:5200 + | -> local astrology engines + -> private PostgreSQL 17 + Better Auth + -> external model and mail providers ``` Only Caddy publishes host ports. Ports `3000` and `5200` must remain private. ## DNS and Supabase Auth -Spaceship resource records: +Final Spaceship resource records (apply only during the approved cutover window): ```text -A @ 103.117.123.53 +A @ 118.194.235.34 +A admin 118.194.235.34 CNAME www jyotisha.chat ``` -Supabase Authentication URL Configuration: - -```text -Site URL: https://jyotisha.chat -Redirect URLs: https://jyotisha.chat/** - https://www.jyotisha.chat/** -``` - -Before changing Caddy to the domain, verify the authoritative DNS result: +After cutover, verify both authoritative nameservers: ```bash dig +short @launch1.spaceship.net A jyotisha.chat +dig +short @launch2.spaceship.net A admin.jyotisha.chat ``` -It must return `103.117.123.53`. Caddy provisions and renews HTTPS automatically after DNS resolves. +Both user and admin hosts must return `118.194.235.34`. Caddy provisions and renews HTTPS automatically after DNS resolves. ## Production environment -`.env.production` combines the backend and frontend server variables. Required groups: +`.env.production` contains runtime-only application settings. Database bootstrap, migration, and backup credentials belong only in `.env.production.database`. ```dotenv SITE_ADDRESS=https://jyotisha.chat +APP_ENV_FILE=../.env.production +CADDYFILE_PATH=./Caddyfile.production.selfhosted JYOTISH_API_BASE=http://api:5200 GEOAPIFY_API_KEY= -NEXT_PUBLIC_SUPABASE_URL=... -NEXT_PUBLIC_SUPABASE_ANON_KEY=... -SUPABASE_SERVICE_ROLE_KEY=... +AUTH_PROVIDER=self-hosted +SELF_HOSTED_IDENTITY_ENABLED=true +AUTH_USER_ORIGIN=https://jyotisha.chat +ADMIN_USER_ORIGIN=https://admin.jyotisha.chat +IDENTITY_DATABASE_URL=postgresql://identity_runtime:@postgres:5432/jyotisha +APP_DATABASE_URL=postgresql://app_runtime:@postgres:5432/jyotisha +SERVICE_DATABASE_URL=postgresql://service_runtime:@postgres:5432/jyotisha +ADMIN_DATABASE_URL=postgresql://admin_runtime:@postgres:5432/jyotisha +BETTER_AUTH_USER_SECRET= +RESEND_API_KEY= +RESEND_FROM_EMAIL= ADMIN_EMAILS=... # Required to save/read database-backed 易支付 settings. Base64 decoding must @@ -87,10 +88,7 @@ RECTIFICATION_PRICE_CREDITS=3 # Required to encrypt/decrypt model-provider API keys stored in the admin database. # Base64 decoding must produce exactly 32 random bytes; do not reuse other keys. -# Staging migration/deploy creates this once when absent and removes legacy model API-key settings. MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY= -# OpenAI-compatible origins must be explicitly server-allowlisted. -# MODEL_PROVIDER_BASE_URL_ALLOWLIST=https://api.deepseek.com # The admin database model catalog is the only runtime model/provider source. # Provider API keys are entered in the admin UI and are never read from provider env vars. @@ -105,47 +103,37 @@ VEDASTRO_TIMEOUT_SECONDS=20 VEDASTRO_API_KEY= ``` -Never commit `.env.production`, `SUPABASE_SERVICE_ROLE_KEY`, model keys, user JWTs, SSH private keys or passwords. `NEXT_PUBLIC_SUPABASE_ANON_KEY` is intentionally public; authorization is enforced by Supabase RLS and server-side checks. +Never commit either production env file, model/payment keys, user tokens, SSH private keys, database URLs, or passwords. Do not retain Supabase runtime selectors in the target env; Supabase is migration source/rollback storage only. -After changing VedAstro variables, restart the API and verify the configuration without printing credentials: -```bash -docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build api -docker compose --env-file .env.production -f deploy/docker-compose.server.yml exec api python3 scripts/diagnose_vedastro_mode.py -``` -The report must show `mode: official_extended` and `network_enabled: true`. A missing raw response remains an upstream response boundary, not a successful external verification. +After changing runtime variables, use the exact-SHA Gitea workflow to recreate services; do not rebuild or rsync a local tree on the host. ## Connect and inspect ```bash -ssh -p 22000 root@103.117.123.53 -cd /opt/jyotisha-app -COMPOSE='docker compose --env-file .env.production -f deploy/docker-compose.server.yml' +ssh -p deploy@118.194.235.34 +cd /opt/jyotisha-production +COMPOSE='docker compose -p jyotisha-production --env-file .env.production -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml -f deploy/docker-compose.production.yml' $COMPOSE ps -$COMPOSE logs --tail=100 api web caddy +$COMPOSE logs --tail=100 postgres api web caddy free -h docker stats --no-stream ``` -The server has a persistent 2 GB `/swapfile`. UFW permits only SSH `22000/tcp`, HTTP `80/tcp`, HTTPS `443/tcp`, and the pre-existing WireGuard `51820/udp` rule. +UFW permits only the confirmed SSH port, HTTP, and HTTPS. PostgreSQL, Web, API, and the Docker API remain private. -## Manual deployment with GitHub Actions +## Manual production deployment with Gitea Actions -Production pushes and pull requests do not start GitHub Actions automatically. Run the required validation workflows from the Actions page, then manually start `.github/workflows/deploy-production.yml` for the tested branch. The deployment workflow syncs that revision with `rsync`, preserves `/opt/jyotisha-app/.env.production`, rebuilds both Docker services, and verifies the public login route, logged-out account response, and private Python health endpoint. +GitHub production deployment is retired. Production changes are released only by manually dispatching `.gitea/workflows/deploy-production.yml`. The workflow requires an exact SHA shared by `main` and `staging`, an exact-SHA staging push gate and image manifest, the manual release gate, and matching public staging health. It deploys immutable registry digests and never builds application images on the production host. -For the reviewed conversational rectification and global birthplace schema set, -run `.github/workflows/apply-production-rectification-migrations.yml` with -`operation=check` first. If the ledger and checksums are clean, rerun the same -current-`main` revision with `operation=apply`. The workflow only accepts the -four allowlisted forward migrations, applies each migration and its ledger row -in one transaction, and refuses stale revisions or checksum drift. +The `internal` verification mode is for pre-DNS checks on the new host. The `public` mode is for the same SHA after authoritative DNS and Caddy TLS converge. Neither mode imports data, applies pending migrations, or changes DNS. -Required GitHub Actions secret: +Required Gitea Actions secret: ```text -PRODUCTION_SSH_PRIVATE_KEY = dedicated production deploy private key +PRODUCTION_SSH_PRIVATE_KEY = one-line base64 of the dedicated deploy private-key file ``` -The workflow pins the VPS Ed25519 host key and serializes deployments with the `production` concurrency group. +The independently verified host key is stored in `PRODUCTION_KNOWN_HOSTS`. See the production migration runbook for all variables, data gates, and rollback boundaries. ## Staging deployment @@ -178,7 +166,7 @@ SITE_ADDRESS=https://staging.jyotisha.chat 模型供应商的 `base_url` 不再依赖域名白名单，任意公网 HTTPS origin 均可由管理员配置；部署环境不需要 `MODEL_PROVIDER_BASE_URL_ALLOWLIST`。服务端仍强制 HTTPS、禁止凭据、localhost/内网/保留地址，并在 DNS 解析、请求地址 pinning 和重定向处理上执行 SSRF 防护。 -Staging is fully self-hosted: set `AUTH_PROVIDER=self-hosted` and `SELF_HOSTED_IDENTITY_ENABLED=true`. Add the four role-specific server-only database URLs, the exact `AUTH_USER_ORIGIN=https://staging.jyotisha.chat` and `ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat`, the single `BETTER_AUTH_USER_SECRET`, and staging-only Resend settings listed in `deploy/.env.staging.identity.example`. Both hosts run the same application and Better Auth service, but cookies remain host-only; the admin host `/` redirects to `/admin`, and unauthenticated admin requests continue to `/login` on that host. Better Auth trusts only those two origins, while unknown identity hosts fail closed. Persisted `identity.users.role=admin` is the only self-hosted backend role, while `viewer` and ordinary users are denied. Browser code uses same-origin APIs; it receives neither database credentials nor Supabase keys. Production remains on Supabase and is not changed by the staging workflow. See `docs/operations/self-hosted-identity.md` for validation and rollback commands. +Staging is fully self-hosted: set `AUTH_PROVIDER=self-hosted` and `SELF_HOSTED_IDENTITY_ENABLED=true`. Add the four role-specific server-only database URLs, the exact `AUTH_USER_ORIGIN=https://staging.jyotisha.chat` and `ADMIN_USER_ORIGIN=https://admin.staging.jyotisha.chat`, the single `BETTER_AUTH_USER_SECRET`, and staging-only Resend settings listed in `deploy/.env.staging.identity.example`. Both hosts run the same application and Better Auth service, but cookies remain host-only; the admin host `/` redirects to `/admin`, and unauthenticated admin requests continue to `/login` on that host. Better Auth trusts only those two origins, while unknown identity hosts fail closed. Persisted `identity.users.role=admin` is the only self-hosted backend role, while `viewer` and ordinary users are denied. Browser code uses same-origin APIs; it receives neither database credentials nor Supabase keys. Production uses the same architecture only after the reviewed migration and cutover. See `docs/operations/self-hosted-identity.md` for identity validation. After source sync and before `up`, the workflow validates `.env.staging` mode/selectors, explicitly pins the three staging selectors against ambient shell overrides, and runs `docker compose --env-file .env.staging -f deploy/docker-compose.server.yml config --quiet`. For later manual inspections, run the same checks only after the tracked deployment files exist on the server. Do not use a manual gate run from `main` as the first publishing path: publishing requires a successful push to `staging`, while manual `Deploy staging` requires a successful gate run for the exact SHA. @@ -321,23 +309,7 @@ This disposable staging procedure does not authorize a production migration, pro ## Manual deployment fallback -If GitHub Actions is unavailable, deploy the tracked tree without copying local secrets: - -```bash -cd /Users/jesse/Downloads/Copse/astrology/yinduzhanxing -git status --short --branch -rsync -az --delete \ - --exclude='.git/' \ - --exclude='.env.production' \ - --exclude='frontend/node_modules/' \ - --exclude='frontend/.next/' \ - -e 'ssh -p 22000' \ - ./ root@103.117.123.53:/opt/jyotisha-app/ -ssh -p 22000 root@103.117.123.53 \ - 'cd /opt/jyotisha-app && docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build --remove-orphans' -``` - -The excluded `.env.production` remains only on the VPS. +There is no unreviewed rsync/build fallback for the new production. If Gitea Actions or the immutable artifact is unavailable, stop the release and restore the control plane; do not substitute a mutable image tag or copy a local working tree to production. ## Verification @@ -349,29 +321,13 @@ curl -fsS -o /dev/null -w '%{http_code}\n' https://jyotisha.chat/api/account The second command should return `401` while logged out. Verify the private Python API from inside the web container: ```bash -ssh -p 22000 root@103.117.123.53 \ - 'cd /opt/jyotisha-app && docker compose --env-file .env.production -f deploy/docker-compose.server.yml exec -T web node -e "fetch(\"http://api:5200/api/health\").then(async r=>{console.log(r.status); console.log(await r.text())})"' +ssh -p deploy@118.194.235.34 \ + 'cd /opt/jyotisha-production && docker compose -p jyotisha-production --env-file .env.production -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml -f deploy/docker-compose.production.yml exec -T web node -e "fetch(\"http://api:5200/api/health\").then(async r=>{console.log(r.status); console.log(await r.text())})"' ``` -Expected: HTTP `200`, `"status": "ok"`, and `"swisseph_available": true`. Public access to `103.117.123.53:5200` must fail. +Expected: HTTP `200`, `"status": "ok"`, and `"swisseph_available": true`. Public access to `118.194.235.34:5200` and `:5432` must fail. The deployment workflow also executes `SELECT 1` through all four runtime database roles. -Before deploying application code that depends on any new Supabase migration (columns, tables, grants, policies, or RPCs), run `cd frontend && npx supabase db push --linked`; the GitHub deployment workflow does not apply database migrations. Multi-model chat specifically requires `20260717010000_chat_session_model.sql` before the new web image is deployed. Then manually verify: OTP login, onboarding/profile persistence, per-session `model_id` persistence, code redemption, admin code generation, authenticated `/api/models` returns only sanitized public metadata, invalid model IDs are rejected before charging, each configured model can answer, the 2.5-second free undo window, streaming response, one-credit charge, refund before the first output chunk, and charged stop with partial output preserved after streaming starts. - -For the July 2026 new-user profile save fix, either run the manual GitHub Action -`Apply Supabase profile migrations` after adding `SUPABASE_DB_URL` or `DATABASE_URL` -to `/opt/jyotisha-app/.env.production`, or execute these five SQL migrations in -the Supabase SQL Editor with a project member account: - -- `20260718010000_recover_missing_profile_rows.sql` -- `20260718020000_profiles_service_role_upsert_grants.sql` -- `20260718050000_profiles_service_role_upsert_grants.sql` -- `20260718070000_profiles_service_role_upsert_id.sql` -- `20260718080000_profiles_service_role_account_upsert_selects.sql` - -Do not treat a green app deployment as proof this database step ran. If the SQL -Editor shows `You do not have access to this project`, use the correct Supabase -organization account or invite the current GitHub user to project -`vtvnfqmonbfuxmqkqdlc` before retrying. +Do not treat a green app deployment as proof of database migration or data reconciliation. The production deployment refuses pending target migrations but does not apply them. Follow the schema-first ETL and verification gates in the production migration runbook. Supabase remains the read-only rollback source until the retention and reconciliation window closes. ## Agentic birth-time rectification @@ -392,16 +348,16 @@ evidence. ```bash # Restart without rebuilding -docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d +docker compose -p jyotisha-production --env-file .env.production \ + -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml \ + -f deploy/docker-compose.production.yml up -d -# Rebuild only the web container -docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build web caddy - -# Rebuild only the Python API -docker compose --env-file .env.production -f deploy/docker-compose.server.yml up -d --build api +# Pull/deploy application images only through the exact-SHA Gitea workflow. # Follow logs -docker compose --env-file .env.production -f deploy/docker-compose.server.yml logs -f --tail=100 api web caddy +docker compose -p jyotisha-production --env-file .env.production \ + -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml \ + -f deploy/docker-compose.production.yml logs -f --tail=100 postgres api web caddy ``` ## Optional Railway deployment diff --git a/deploy/docker-compose.production.yml b/deploy/docker-compose.production.yml new file mode 100644 index 00000000..aab684bb --- /dev/null +++ b/deploy/docker-compose.production.yml @@ -0,0 +1,8 @@ +services: + web: + networks: + - default + - app + +networks: + app: diff --git a/deploy/run-production-deploy.sh b/deploy/run-production-deploy.sh new file mode 100755 index 00000000..99be8f31 --- /dev/null +++ b/deploy/run-production-deploy.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +required=( + INCOMING_PATH DEPLOY_PATH API_IMAGE WEB_IMAGE DEPLOY_SHA + EXPECTED_PREVIOUS_SHA ALLOW_ROLLBACK DOCKER_CONFIG PRODUCTION_URL + PRODUCTION_ADMIN_URL VERIFICATION_MODE +) +case "${DOCKER_BIN:-docker}" in + docker) docker_command=(docker) ;; + "sudo -n docker") docker_command=(sudo -n docker --config "$DOCKER_CONFIG") ;; + *) echo "unsafe production Docker command" >&2; exit 1 ;; +esac +for key in "${required[@]}"; do + if [ -z "${!key:-}" ]; then + echo "required production deployment input is missing: $key" >&2 + exit 1 + fi +done + +sha_pattern='^[0-9a-f]{40}$' +digest_pattern='^[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:[1-9][0-9]{0,4})?(/[a-z0-9]+([._-][a-z0-9]+)*)+@sha256:[0-9a-f]{64}$' +image_id_pattern='^sha256:[0-9a-f]{64}$' +if [[ ! "$DEPLOY_SHA" =~ $sha_pattern ]] || + [[ ! "$API_IMAGE" =~ $digest_pattern ]] || + [[ ! "$WEB_IMAGE" =~ $digest_pattern ]]; then + echo "unsafe production image identity" >&2 + exit 1 +fi +api_repository="${API_IMAGE%@sha256:*}" +web_repository="${WEB_IMAGE%@sha256:*}" +if [ "$ALLOW_ROLLBACK" != "true" ] && [ "$ALLOW_ROLLBACK" != "false" ]; then + echo "invalid rollback authorization" >&2 + exit 1 +fi +if [ "$VERIFICATION_MODE" != "internal" ] && [ "$VERIFICATION_MODE" != "public" ]; then + echo "invalid production verification mode" >&2 + exit 1 +fi +case "$INCOMING_PATH" in + /tmp/jyotisha-production.*) ;; + *) echo "unsafe incoming production path" >&2; exit 1 ;; +esac + +state_directory="$DEPLOY_PATH/.state" +install -d -m 700 "$state_directory" +exec 9>"$state_directory/mutation.lock" +flock -n 9 || { + echo "another production mutation holds the host lock" >&2 + exit 75 +} + +current_sha="not-deployed" +if [ -f "$state_directory/deployed-revision" ]; then + current_sha="$(<"$state_directory/deployed-revision")" +else + existing_web="$("${docker_command[@]}" ps -aq \ + --filter 'label=com.docker.compose.project=jyotisha-production' \ + --filter 'label=com.docker.compose.service=web' | head -n 1)" + if [ -n "$existing_web" ]; then + discovered_sha="$("${docker_command[@]}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \ + "$existing_web" | sed -n 's/^GITHUB_SHA=//p' | head -n 1)" + if [ -n "$discovered_sha" ]; then current_sha="$discovered_sha"; fi + fi +fi +if [ "$current_sha" != "not-deployed" ] && [[ ! "$current_sha" =~ $sha_pattern ]]; then + echo "invalid deployed production revision state" >&2 + exit 1 +fi +if [ "$current_sha" != "$EXPECTED_PREVIOUS_SHA" ]; then + echo "production revision changed while this deployment was waiting" >&2 + exit 1 +fi +if [ "$ALLOW_ROLLBACK" = "false" ] && + [ "$current_sha" != "not-deployed" ] && + [ "$current_sha" != "$DEPLOY_SHA" ] && + [ "${FORWARD_REVISION_VERIFIED:-false}" != "true" ]; then + echo "forward production revision was not verified" >&2 + exit 1 +fi + +container_id() { + "${docker_command[@]}" ps -aq \ + --filter 'label=com.docker.compose.project=jyotisha-production' \ + --filter "label=com.docker.compose.service=$1" | head -n 1 +} + +repo_digest_for_container() { + local service="$1" + local repository="$2" + local id image_id + id="$(container_id "$service")" + [ -n "$id" ] || return 0 + image_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")" + "${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image_id" | + awk -v prefix="$repository@sha256:" 'index($0, prefix) == 1 { print; exit }' +} + +previous_api_image="$(repo_digest_for_container api "$api_repository")" +previous_web_image="$(repo_digest_for_container web "$web_repository")" +previous_api_id="" +previous_web_id="" +if [ -n "$(container_id api)" ]; then + previous_api_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id api)")" +fi +if [ -n "$(container_id web)" ]; then + previous_web_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$(container_id web)")" +fi + +rollback_image() { + local digest_ref="$1" + local image_id="$2" + if [[ "$digest_ref" =~ $digest_pattern ]]; then + printf '%s' "$digest_ref" + elif [[ "$image_id" =~ $image_id_pattern ]]; then + printf '%s' "$image_id" + fi +} + +previous_api_target="$(rollback_image "$previous_api_image" "$previous_api_id")" +previous_web_target="$(rollback_image "$previous_web_image" "$previous_web_id")" + +bash "$INCOMING_PATH/deploy/sync-production-tree.sh" \ + "$INCOMING_PATH" "$DEPLOY_PATH" + +cd "$DEPLOY_PATH" +EXPECTED_PRODUCTION_ENV_OWNER_UID="$(stat -c '%u' "$DEPLOY_PATH" 2>/dev/null || stat -f '%u' "$DEPLOY_PATH")" +[[ "$EXPECTED_PRODUCTION_ENV_OWNER_UID" =~ ^[0-9]+$ ]] || { + echo "production deployment owner is invalid" >&2 + exit 1 +} +export EXPECTED_PRODUCTION_ENV_OWNER_UID +bash deploy/validate-production-env.sh \ + .env.production +bash deploy/validate-production-database-env.sh .env.production.database + +compose=( + "${docker_command[@]}" compose -p jyotisha-production --env-file .env.production + -f deploy/docker-compose.server.yml -f deploy/docker-compose.postgres.yml + -f deploy/docker-compose.production.yml +) +export APP_ENV_FILE='../.env.production' +export DATABASE_ENV_FILE='../.env.production.database' +export CADDYFILE_PATH='./Caddyfile.production.selfhosted' +export SITE_ADDRESS='https://jyotisha.chat' +export GITHUB_SHA="$DEPLOY_SHA" + +"${compose[@]}" config --quiet +"${compose[@]}" pull api web +"${compose[@]}" up -d --no-build --pull never --wait postgres + +set +e +"${compose[@]}" --profile migration-check run --rm migration-checker +check_status=$? +set -e +if [ "$check_status" -eq 3 ]; then + echo "pending migrations: run Migrate Production Database for $DEPLOY_SHA" >&2 + exit 3 +fi +if [ "$check_status" -ne 0 ]; then + echo "production migration check failed safely" >&2 + exit "$check_status" +fi + +switched=false +rollback() { + local status=$? + if [ "$switched" = "true" ] && + [ -n "$previous_api_target" ] && + [ -n "$previous_web_target" ] && + [[ "$current_sha" =~ $sha_pattern ]]; then + echo "production verification failed; restoring prior application images" >&2 + rollback_services=(api web) + if [ "$VERIFICATION_MODE" = "public" ]; then rollback_services+=(caddy); fi + API_IMAGE="$previous_api_target" WEB_IMAGE="$previous_web_target" \ + GITHUB_SHA="$current_sha" \ + "${compose[@]}" up -d --no-build --remove-orphans \ + "${rollback_services[@]}" || true + fi + exit "$status" +} +trap rollback ERR + +switched=true +if [ "$VERIFICATION_MODE" = "public" ]; then + "${compose[@]}" up -d --no-build --remove-orphans + "${compose[@]}" up -d --no-build --force-recreate --no-deps caddy +else + # Before DNS cutover, do not trigger public certificate issuance for domains + # that still resolve to the old production host. + "${compose[@]}" up -d --no-build api web +fi + +verify_container_image() { + local service="$1" + local expected_ref="$2" + local id expected_id running_id repo_digests + id="$(container_id "$service")" + [ -n "$id" ] + expected_id="$("${docker_command[@]}" image inspect --format '{{.Id}}' "$expected_ref")" + running_id="$("${docker_command[@]}" inspect --format '{{.Image}}' "$id")" + [ "$running_id" = "$expected_id" ] + repo_digests="$("${docker_command[@]}" image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$expected_id")" + grep -Fqx "$expected_ref" <<<"$repo_digests" +} +verify_container_image api "$API_IMAGE" +verify_container_image web "$WEB_IMAGE" + +"${compose[@]}" exec -T \ + -e EXPECTED_SHA="$DEPLOY_SHA" \ + -e PRODUCTION_URL="$PRODUCTION_URL" \ + -e PRODUCTION_ADMIN_URL="$PRODUCTION_ADMIN_URL" \ + -e VERIFICATION_MODE="$VERIFICATION_MODE" \ + web node --input-type=module <<'NODE' +import { Pool } from "pg"; + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +for (const [role, key] of [ + ["identity", "IDENTITY_DATABASE_URL"], + ["app", "APP_DATABASE_URL"], + ["service", "SERVICE_DATABASE_URL"], + ["admin", "ADMIN_DATABASE_URL"], +]) { + const connectionString = process.env[key]; + if (!connectionString) { + console.error(`database readiness missing for ${role}`); + process.exit(1); + } + const pool = new Pool({ connectionString, max: 1, connectionTimeoutMillis: 5_000 }); + try { + const result = await pool.query("select 1 as ready"); + if (result.rows[0]?.ready !== 1) throw new Error("unexpected readiness result"); + } catch (error) { + console.error(`database readiness failed for ${role}`, error instanceof Error ? error.name : "query_error"); + process.exit(1); + } finally { + await pool.end(); + } +} + +const internal = process.env.VERIFICATION_MODE === "internal"; +const request = (origin, path, options = {}) => { + const target = internal ? `http://127.0.0.1:3000${path}` : `${origin}${path}`; + const headers = internal ? { ...options.headers, host: new URL(origin).host } : options.headers; + return fetch(target, { ...options, headers }); +}; +let observed = {}; +for (let attempt = 1; attempt <= 12; attempt += 1) { + try { + const login = await request(process.env.PRODUCTION_URL, "/login"); + const userAdminPage = await request(process.env.PRODUCTION_URL, "/admin", { redirect: "manual" }); + const userAdminApi = await request(process.env.PRODUCTION_URL, "/api/admin/session"); + const adminPage = await request(process.env.PRODUCTION_ADMIN_URL, "/admin", { redirect: "manual" }); + const adminApi = await request(process.env.PRODUCTION_ADMIN_URL, "/api/admin/session"); + const account = await request(process.env.PRODUCTION_URL, "/api/account"); + const publicHealth = await request(process.env.PRODUCTION_URL, "/api/health"); + const publicBody = await publicHealth.json(); + const privateHealth = await fetch("http://api:5200/api/health"); + const privateBody = await privateHealth.json(); + observed = { + attempt, + login: login.status, + userAdminPage: userAdminPage.status, + userAdminApi: userAdminApi.status, + adminPage: adminPage.status, + adminLocation: adminPage.headers.get("location"), + adminApi: adminApi.status, + account: account.status, + publicHealth: publicHealth.status, + publicSha: publicBody.deployment?.gitCommit ?? "missing", + privateHealth: privateHealth.status, + privateStatus: privateBody.status ?? "missing", + swissephAvailable: privateBody.swisseph_available === true, + verificationMode: process.env.VERIFICATION_MODE, + }; + if ( + login.ok + && userAdminPage.status === 404 + && userAdminApi.status === 404 + && adminPage.status === 307 + && adminPage.headers.get("location") === "/login" + && adminApi.status === 401 + && account.status === 401 + && publicHealth.ok + && publicBody.deployment?.gitCommit === process.env.EXPECTED_SHA + && privateHealth.ok + && privateBody.status === "ok" + && privateBody.swisseph_available === true + ) { + process.exit(0); + } + } catch (error) { + observed = { + attempt, + error: error instanceof Error ? error.name : "verification_error", + }; + } + if (attempt < 12) await delay(5_000); +} +console.error("production verification predicates did not converge", JSON.stringify(observed)); +process.exit(1); +NODE + +revision_file="$state_directory/deployed-revision.tmp.$$" +printf '%s\n' "$DEPLOY_SHA" >"$revision_file" +chmod 600 "$revision_file" +mv -f "$revision_file" "$state_directory/deployed-revision" +trap - ERR + +printf 'previous_sha=%s\nprevious_api_image=%s\nprevious_api_id=%s\n' \ + "$current_sha" "${previous_api_image:-not-deployed}" "${previous_api_id:-not-deployed}" +printf 'previous_web_image=%s\nprevious_web_id=%s\nverified_sha=%s\n' \ + "${previous_web_image:-not-deployed}" "${previous_web_id:-not-deployed}" "$DEPLOY_SHA" diff --git a/deploy/sync-production-tree.sh b/deploy/sync-production-tree.sh new file mode 100755 index 00000000..debd6ebf --- /dev/null +++ b/deploy/sync-production-tree.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ] || [ ! -d "$1" ] || [ ! -d "$2" ]; then + echo "usage: sync-production-tree.sh SOURCE_DIRECTORY DESTINATION_DIRECTORY" >&2 + exit 1 +fi + +destination_deploy="$2/deploy" +if [ -d "$destination_deploy" ]; then + docker run --rm --pull never --network none --read-only --user 0:0 \ + --cap-drop ALL --cap-add CHOWN --security-opt no-new-privileges \ + -v "$destination_deploy:/destination" postgres:17-alpine \ + chown -R "$(id -u):$(id -g)" /destination + chmod -R u+rwX "$destination_deploy" +fi + +rsync -az --delete --no-owner --no-group \ + --exclude='/.git/' \ + --exclude='/.env*' \ + --exclude='/.docker/' \ + --exclude='/backups/' \ + --exclude='/.state/' \ + --exclude='/.incoming/' \ + --exclude='/frontend/node_modules/' \ + --exclude='/frontend/.next/' \ + "$1/" "$2/" diff --git a/deploy/validate-production-database-env.sh b/deploy/validate-production-database-env.sh new file mode 100755 index 00000000..fa2002d1 --- /dev/null +++ b/deploy/validate-production-database-env.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +ENV_FILE="${1:-.env.production.database}" + +if [ ! -e "$ENV_FILE" ]; then + echo "production database environment file is missing" >&2 + exit 1 +fi + +if [ -L "$ENV_FILE" ]; then + echo "production database environment file must not be a symlink" >&2 + exit 1 +fi + +if [ ! -f "$ENV_FILE" ]; then + echo "production database environment path must be a regular file" >&2 + exit 1 +fi + +if MODE="$(stat -c '%a' "$ENV_FILE" 2>/dev/null)"; then + : +else + MODE="$(stat -f '%Lp' "$ENV_FILE")" +fi + +if [ "$MODE" != "600" ]; then + echo "production database environment file must have mode 0600" >&2 + exit 1 +fi + +if OWNER="$(stat -c '%u' "$ENV_FILE" 2>/dev/null)"; then + : +else + OWNER="$(stat -f '%u' "$ENV_FILE")" +fi +EXPECTED_OWNER_UID="${EXPECTED_PRODUCTION_ENV_OWNER_UID:-$(id -u)}" +if [[ ! "$EXPECTED_OWNER_UID" =~ ^[0-9]+$ ]] || [ "$OWNER" != "$EXPECTED_OWNER_UID" ]; then + echo "production database environment file has an invalid owner" >&2 + exit 1 +fi + +definition_count() { + local key="$1" + grep -Ec "^[[:space:]]*(export[[:space:]]+)?${key}([[:space:]]*=|[[:space:]]*$)" "$ENV_FILE" || true +} + +environment_value() { + local key="$1" + sed -n -E "s/^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=[[:space:]]*(.*)$/\\2/p" "$ENV_FILE" +} + +is_safe_literal() { + local value="$1" + local inner + + # Required values are literal single-line values: use an unquoted token or + # matching non-empty quotes. Dotenv interpolation, comments, and malformed + # quoting are rejected rather than evaluated, so generate secrets without $. + if [ -z "$value" ] || [[ "$value" == *'$'* ]]; then + return 1 + fi + + case "$value" in + \"*\") + inner="${value:1}" + inner="${inner%?}" + [ -n "$inner" ] && [[ "$inner" != *'"'* ]] + ;; + \'*\') + inner="${value:1}" + inner="${inner%?}" + [ -n "$inner" ] && [[ "$inner" != *"'"* ]] + ;; + *\"*|*\'*) + return 1 + ;; + *[[:space:]]*|*\#*) + return 1 + ;; + esac +} + +require_once_non_empty() { + local key="$1" + local count + local value + count="$(definition_count "$key")" + value="$(environment_value "$key")" + if [ "$count" -ne 1 ] || ! is_safe_literal "$value"; then + echo "required production database literal is missing, duplicated, or ambiguous: $key" >&2 + exit 1 + fi +} + +required=( + POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD + SCHEMA_OWNER_PASSWORD IDENTITY_RUNTIME_PASSWORD APP_RUNTIME_PASSWORD + SERVICE_RUNTIME_PASSWORD ADMIN_RUNTIME_PASSWORD MIGRATION_RUNNER_PASSWORD + BACKUP_READER_PASSWORD + PRODUCTION_BACKUP_ENCRYPTION_KEY SCHEMA_DATABASE_URL +) +for key in "${required[@]}"; do + require_once_non_empty "$key" +done + +if [ "$(environment_value POSTGRES_DB)" != "jyotisha" ]; then + echo "invalid production database selector: POSTGRES_DB" >&2 + exit 1 +fi + +if [ "$(environment_value POSTGRES_USER)" != "postgres" ]; then + echo "invalid production database selector: POSTGRES_USER" >&2 + exit 1 +fi + +if ! [[ "$(environment_value SCHEMA_DATABASE_URL)" =~ ^postgresql://schema_owner:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then + echo "invalid production database selector: SCHEMA_DATABASE_URL" >&2 + exit 1 +fi + +echo "production database environment validated" diff --git a/deploy/validate-production-env.sh b/deploy/validate-production-env.sh new file mode 100755 index 00000000..dd6d928b --- /dev/null +++ b/deploy/validate-production-env.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +ENV_FILE="${1:-.env.production}" + +if [ ! -f "$ENV_FILE" ]; then + echo "production environment file is missing: $ENV_FILE" >&2 + exit 1 +fi + +if [ -L "$ENV_FILE" ]; then + echo "production environment file must not be a symlink" >&2 + exit 1 +fi + +if MODE="$(stat -c '%a' "$ENV_FILE" 2>/dev/null)"; then + : +else + MODE="$(stat -f '%Lp' "$ENV_FILE")" +fi + +if [ "$MODE" != "600" ]; then + echo "production environment file must have mode 0600" >&2 + exit 1 +fi + +if OWNER="$(stat -c '%u' "$ENV_FILE" 2>/dev/null)"; then + : +else + OWNER="$(stat -f '%u' "$ENV_FILE")" +fi +EXPECTED_OWNER_UID="${EXPECTED_PRODUCTION_ENV_OWNER_UID:-$(id -u)}" +if [[ ! "$EXPECTED_OWNER_UID" =~ ^[0-9]+$ ]] || [ "$OWNER" != "$EXPECTED_OWNER_UID" ]; then + echo "production environment file has an invalid owner" >&2 + exit 1 +fi + +require_selector() { + local key="$1" + local expected="$2" + local count + local definition_pattern + + definition_pattern="^[[:space:]]*(export[[:space:]]+)?${key}([[:space:]]*=|[[:space:]]*$)" + count="$(grep -Ec "$definition_pattern" "$ENV_FILE" || true)" + if [ "$count" -ne 1 ] || ! grep -Fqx "${key}=${expected}" "$ENV_FILE"; then + echo "invalid production selector: $key" >&2 + exit 1 + fi +} + +require_selector APP_ENV_FILE ../.env.production +require_selector CADDYFILE_PATH ./Caddyfile.production.selfhosted +require_selector SITE_ADDRESS https://jyotisha.chat +require_selector AUTH_PROVIDER self-hosted +require_selector SELF_HOSTED_IDENTITY_ENABLED true +require_selector AUTH_USER_ORIGIN https://jyotisha.chat +require_selector ADMIN_USER_ORIGIN https://admin.jyotisha.chat + +require_literal() { + local key="$1" + local minimum_length="$2" + local count value + count="$(grep -Ec "^${key}=" "$ENV_FILE" || true)" + if [ "$count" -ne 1 ]; then + echo "invalid production identity setting: $key" >&2 + exit 1 + fi + value="$(grep -E "^${key}=" "$ENV_FILE")" + value="${value#*=}" + if [ "${#value}" -lt "$minimum_length" ] || + [[ "$value" == *'$'* || "$value" == *'"'* || "$value" == *"'"* ]]; then + echo "invalid production identity setting: $key" >&2 + exit 1 + fi + LITERAL_VALUE="$value" +} + +require_literal IDENTITY_DATABASE_URL 50 +identity_database_url="$LITERAL_VALUE" +if ! [[ "$identity_database_url" =~ ^postgresql://identity_runtime:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then + echo "invalid production identity setting: IDENTITY_DATABASE_URL" >&2 + exit 1 +fi + +require_literal APP_DATABASE_URL 45 +app_database_url="$LITERAL_VALUE" +if ! [[ "$app_database_url" =~ ^postgresql://app_runtime:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then + echo "invalid production database setting: APP_DATABASE_URL" >&2 + exit 1 +fi + +require_literal SERVICE_DATABASE_URL 49 +service_database_url="$LITERAL_VALUE" +if ! [[ "$service_database_url" =~ ^postgresql://service_runtime:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then + echo "invalid production database setting: SERVICE_DATABASE_URL" >&2 + exit 1 +fi + +require_literal ADMIN_DATABASE_URL 45 +admin_database_url="$LITERAL_VALUE" +if ! [[ "$admin_database_url" =~ ^postgresql://admin_runtime:([A-Za-z0-9._~-]|%[0-9A-Fa-f]{2})+@postgres:5432/jyotisha$ ]]; then + echo "invalid production database setting: ADMIN_DATABASE_URL" >&2 + exit 1 +fi + +require_literal BETTER_AUTH_USER_SECRET 32 +require_literal RESEND_API_KEY 10 +require_literal RESEND_FROM_EMAIL 5 +if [[ "$LITERAL_VALUE" != *@* ]]; then + echo "invalid production identity setting: RESEND_FROM_EMAIL" >&2 + exit 1 +fi +require_literal ADMIN_EMAILS 3 +if [[ "$LITERAL_VALUE" != *@* ]]; then + echo "invalid production identity setting: ADMIN_EMAILS" >&2 + exit 1 +fi +require_literal EPAY_CONFIG_ENCRYPTION_KEY 44 +if [ "${#LITERAL_VALUE}" -ne 44 ] || + [[ ! "$LITERAL_VALUE" =~ ^[A-Za-z0-9+/]{43}=$ ]]; then + echo "invalid production identity setting: EPAY_CONFIG_ENCRYPTION_KEY" >&2 + exit 1 +fi +require_literal MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY 44 +if [ "${#LITERAL_VALUE}" -ne 44 ] || + [[ ! "$LITERAL_VALUE" =~ ^[A-Za-z0-9+/]{43}=$ ]]; then + echo "invalid production model provider setting: MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY" >&2 + exit 1 +fi +legacy_model_setting_pattern='^(OPENAI_API_KEY|ANTHROPIC_API_KEY|DEEPSEEK_API_KEY|LLM_API_KEY|LLM_MODELS_JSON|LLM_BASE_URL|LLM_MODEL|LLM_DEFAULT_MODEL_ID|LLM_PROVIDER_ID|MASTRA_MODEL|MODEL_PROVIDER_[A-Z0-9_]+_API_KEY)=' +if grep -Eq "$legacy_model_setting_pattern" "$ENV_FILE"; then + echo "legacy model environment settings are forbidden" >&2 + exit 1 +fi + +require_selector EPAY_CHAT_ENABLED false +require_literal JYOTISH_DYNAMIC_RECTIFICATION_TOKEN 32 + +personal_report_enabled_count="$(grep -Ec '^PERSONAL_REPORT_ENABLED=' "$ENV_FILE" || true)" +personal_report_enabled="$(grep -E '^PERSONAL_REPORT_ENABLED=' "$ENV_FILE" || true)" +personal_report_enabled="${personal_report_enabled#*=}" +if [ "$personal_report_enabled_count" -ne 1 ] || + [[ "$personal_report_enabled" != "true" && "$personal_report_enabled" != "false" ]]; then + echo "invalid production personal report setting: PERSONAL_REPORT_ENABLED" >&2 + exit 1 +fi + +personal_report_daily_limit_count="$(grep -Ec '^PERSONAL_REPORT_DAILY_LIMIT=' "$ENV_FILE" || true)" +personal_report_daily_limit="$(grep -E '^PERSONAL_REPORT_DAILY_LIMIT=' "$ENV_FILE" || true)" +personal_report_daily_limit="${personal_report_daily_limit#*=}" +if [ "$personal_report_daily_limit_count" -ne 1 ] || + [[ ! "$personal_report_daily_limit" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid production personal report setting: PERSONAL_REPORT_DAILY_LIMIT" >&2 + exit 1 +fi + +echo "production environment selectors: valid" diff --git a/docs/operations/production-server-migration-2026-08.md b/docs/operations/production-server-migration-2026-08.md new file mode 100644 index 00000000..64a78cc4 --- /dev/null +++ b/docs/operations/production-server-migration-2026-08.md @@ -0,0 +1,195 @@ +# Production migration to 118.194.235.34 + +Status: **planned; do not change DNS or retire the old production yet**. + +This runbook moves production to the current reviewed `staging` release while also changing the persistence and identity layers: + +- old runtime: VPS + Supabase PostgreSQL + Supabase Auth; +- target runtime: `118.194.235.34` + private PostgreSQL 17 + Better Auth; +- user site: `https://jyotisha.chat`; +- admin site: `https://admin.jyotisha.chat`. + +This is not a volume copy. A full Supabase dump must not be restored over the target database. + +## Release invariants + +The production workflow is manual-only and accepts a full lowercase 40-character `deploy_sha`. A normal deployment proceeds only when all of the following identify that exact SHA: + +1. current `main`; +2. current `staging`; +3. a successful push-triggered `Staging Backend Quality Gate`; +4. a successful manually triggered `Jyotish Release Quality Gate`; +5. the public staging `/api/health` deployment identity. + +The workflow consumes the exact API and Web image digests recorded by the staging gate. It does not build on the 2-core/4-GB production host, import user data, run schema migrations, or change DNS. + +## Required Gitea configuration + +Repository variables: + +| Name | Required value | +| --- | --- | +| `PRODUCTION_HOST` | `118.194.235.34` | +| `PRODUCTION_PORT` | Confirmed SSH port; do not assume `22` | +| `PRODUCTION_USER` | `deploy` | +| `PRODUCTION_PATH` | `/opt/jyotisha-production` | +| `PRODUCTION_URL` | `https://jyotisha.chat` | +| `PRODUCTION_ADMIN_URL` | `https://admin.jyotisha.chat` | +| `PRODUCTION_KNOWN_HOSTS` | Independently verified pinned host-key line | +| `STAGING_URL` | `https://staging.jyotisha.chat` | + +Repository secrets: + +- `PRODUCTION_SSH_PRIVATE_KEY`: the dedicated deploy private-key file encoded as one unwrapped base64 line; +- `REGISTRY_USERNAME` and `REGISTRY_PASSWORD`. + +Do not put the Ubuntu password, database URLs, Resend key, payment key, model-provider key, or encryption master keys in Gitea. The supplied bootstrap password must be rotated after an SSH key has been verified; it must never be committed or printed in a workflow. + +## New-server bootstrap + +Perform this interactively before any workflow dispatch: + +1. Patch Ubuntu and install Docker Engine, Compose v2, `rsync`, `curl`, `jq`, `flock`, and UFW. +2. Create a non-root `deploy` user, install a dedicated Ed25519 public key, and grant only the reviewed passwordless commands needed for Docker and deployment-tree ownership. +3. Verify a second key-only session, then disable root login and password authentication and rotate the bootstrap password. +4. Permit only the confirmed SSH port plus `80/tcp`, `443/tcp`, and `443/udp`. Do not publish `3000`, `5200`, `5432`, or the Docker API. +5. Create a 2–4 GB swap file and enable Docker log rotation. Keep at least 15 GB free before the first image pull and database import. +6. Create `/opt/jyotisha-production`, owned by `deploy`, and preload the reviewed `postgres:17-alpine` and Caddy images. PostgreSQL image upgrades are separate maintenance operations. + +Create these host-only files with owner `deploy` and mode `0600`: + +```text +/opt/jyotisha-production/.env.production +/opt/jyotisha-production/.env.production.database +``` + +The application selectors must be exact: + +```dotenv +APP_ENV_FILE=../.env.production +CADDYFILE_PATH=./Caddyfile.production.selfhosted +SITE_ADDRESS=https://jyotisha.chat +AUTH_PROVIDER=self-hosted +SELF_HOSTED_IDENTITY_ENABLED=true +AUTH_USER_ORIGIN=https://jyotisha.chat +ADMIN_USER_ORIGIN=https://admin.jyotisha.chat +EPAY_CHAT_ENABLED=false +``` + +Use distinct production credentials for PostgreSQL roles, Better Auth, Resend, backup encryption, and dynamic rectification. Preserve the existing `EPAY_CONFIG_ENCRYPTION_KEY` and `MODEL_PROVIDER_CONFIG_ENCRYPTION_KEY` only through an approved secret-to-secret transfer. If either key cannot be transferred safely, exclude its ciphertext rows from import and re-enter those settings in the new admin UI. + +## Database migration engineering gate + +Before production cutover, implement and review `frontend/scripts/migrate-supabase-production.mjs` with `--preflight`, `--apply`, and `--verify` modes. Until that tool and its fixtures pass, production data cutover is blocked. + +The tool must: + +- connect to the source using `REPEATABLE READ READ ONLY`; +- refuse a non-empty target business database; +- use explicit columns and dependency order, never `SELECT *`; +- preserve all user UUIDs and transactional primary keys; +- import Supabase `auth.users` into Better Auth without passwords, sessions, JWTs, provider tokens, or MFA secrets; +- map `banned_until`, or emit an explicit blocked-user reconciliation manifest; +- merge seed/configuration records by natural key rather than copying target-generated IDs; +- map active administrators to canonical target role codes and require at least one explicit Owner; +- run post-import reconciliation for legacy billing and retired birth-time rectification rows; +- emit only counts, state aggregates, and normalized SHA-256 manifests—not email addresses, birth data, tokens, or connection strings; +- run all target writes in a transaction and roll back on failure. + +Do not import platform schemas, source roles/grants, Supabase migration ledgers, sessions, refresh tokens, or provider tokens. Do not use a full-database `pg_restore` against the target. + +Because migrated users have no portable password/session, all sessions are invalidated and users sign in again through email OTP. Administrators re-enrol MFA. + +## Rehearsal + +Complete at least one isolated full-data rehearsal before scheduling the final window: + +1. Apply all target schema migrations to an empty rehearsal database. +2. Run migration preflight, apply, post-import reconciliation, and verify. +3. Verify source/target row counts, primary-key set hashes, normalized row hashes, credit totals, payment state totals, subscriptions, reports, consultations, and rectification records. +4. Verify one Owner exists, every active admin has a target role, and database roles remain isolated. +5. Test OTP login, historical balance and history reads, admin login/RBAC, report generation/export, payment callback rejection/idempotency, and model provider access. +6. Create an encrypted backup, restore it into a separate database, and repeat smoke checks. +7. Record the observed export/import/verification duration and use it to set the maintenance window. + +For the 2-core/4-GB host, keep database pools bounded (recommended starting maxima: identity 5, app 5, admin 3) rather than allowing three pools of 10 to consume all 30 PostgreSQL connections. + +## DNS preparation + +At least one current TTL period before cutover, reduce relevant Spaceship TTLs to `300`. Do not change record targets yet. Check both authoritative nameservers and remove any legacy `AAAA` record that points elsewhere. + +Final records are: + +| Type | Host | Value | +| --- | --- | --- | +| `A` | `@` | `118.194.235.34` | +| `A` | `admin` | `118.194.235.34` | +| `CNAME` | `www` | `jyotisha.chat` | + +Both `jyotisha.chat` and `admin.jyotisha.chat` are required. The application rejects unknown identity hosts, and the user domain intentionally hides `/admin` and `/api/admin/*`. + +## Cutover sequence + +### T-24 hours + +- Confirm the exact release SHA is deployed and accepted on staging. +- Run the manual release quality gate for that SHA. +- Confirm final backup capacity, restore rehearsal, SMTP/OTP delivery, and rollback contacts. +- Record pending payment orders and long-running jobs; choose an explicit disposition for each. +- Dispatch `Deploy production` with `verification_mode=internal` only after target schema/data preparation. This verifies the new host without depending on public DNS. + +### Maintenance freeze + +1. Set payment/package creation off and keep `EPAY_CHAT_ENABLED=false`. +2. Put old production in maintenance mode and stop Web/Agent/background writes. +3. Disable new Supabase registrations for the window. +4. Confirm source row counts stop changing. +5. Take the final encrypted source backup and one consistent source snapshot. +6. Run the production ETL once, then post-import reconciliation and verification. +7. Verify role-specific `SELECT 1` connectivity for identity, app, service, and admin roles; verify permission isolation separately. +8. Keep the new site in maintenance mode while running user/admin/report/payment smoke checks. + +Do not attempt an ad-hoc full-plus-incremental migration. Several tables lack a common `updated_at` or soft-delete contract, so an improvised delta can lose deletes, refunds, or accounting changes. + +### DNS and public verification + +1. Change the three Spaceship records only after all final data assertions pass. +2. Verify both authoritative nameservers, then public recursive resolvers. +3. Wait for Caddy certificates for both user and admin hosts. +4. Dispatch the same exact SHA with `verification_mode=public`. +5. Verify OTP login, logged-out account `401`, user-host admin paths `404`, admin-host unauthenticated behavior, health SHA, report generation/export, and one controlled payment callback test. +6. Re-enable public writes. Re-enable payment only after DNS convergence and callback verification. + +## Go/no-go assertions + +Cutover is **no-go** if any of these conditions is true: + +- `main`, `staging`, staging health, gate artifact, or requested SHA differs; +- any migration or manifest checksum is unresolved; +- source/target identity, balance, order, subscription, or report reconciliation differs; +- there is no active Owner or an active admin has no role; +- migrated banned users are not accounted for; +- a production encryption key/ciphertext decision is unresolved; +- backup restore has not been demonstrated; +- role-specific database readiness or isolation fails; +- either user/admin TLS host is unavailable; +- pending payment writes or callbacks can still reach the old writable database. + +## Rollback boundary + +Before the new database accepts real writes, rollback is: restore old DNS targets, keep the old site/Supabase authoritative, and investigate the isolated target. + +After the new database accepts real writes, a DNS-only rollback is unsafe. First stop new writes, reconcile the new PostgreSQL delta back to the chosen authority, and obtain an explicit operator decision. Otherwise post-cutover users, orders, credits, and reports can be lost. + +Keep the old VPS in maintenance/read-only mode for at least 7–14 days and retain Supabase for 14–30 days. Do not destroy either immediately after DNS cutover. + +## Workflow dispatch + +Normal release: + +1. Merge the reviewed `staging` release into `main` so both heads are the same SHA. +2. Confirm the staging push gate, public staging SHA, and manual release gate all succeeded for that SHA. +3. Open Gitea Actions → `Deploy production`. +4. Enter the exact 40-character SHA, leave `allow_rollback=false`, and choose `internal` or `public` for the current cutover phase. + +Application rollback accepts only an explicitly authorized, previously gate-attested SHA in reviewed `main` history. Database migrations and imported data are not rolled back by the application workflow. diff --git a/frontend/tests/health-deployment.test.ts b/frontend/tests/health-deployment.test.ts index 2ca95f9d..26b953fc 100644 --- a/frontend/tests/health-deployment.test.ts +++ b/frontend/tests/health-deployment.test.ts @@ -25,33 +25,6 @@ function webHealthcheckBlock(web: string) { return match[1]; } -function workflowStepBlock(workflow: string, stepName: string) { - const match = workflow.match(new RegExp(`^ - name: ${stepName}\\n([\\s\\S]*?)(?=^ - name:|(?![\\s\\S]))`, "m")); - assert.ok(match, `expected ${stepName} workflow step`); - return match[1]; -} - -function escapeRegExp(value: string) { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -const testedShaExpression = "${{ github.sha }}"; - -function assertWorkflowUsesTestedSha(workflow: string) { - const checkout = workflowStepBlock(workflow, "Checkout tested revision"); - const sync = workflowStepBlock(workflow, "Sync and rebuild"); - const verification = workflowStepBlock(workflow, "Verify production"); - const shaExpression = escapeRegExp(testedShaExpression); - - assert.match(checkout, new RegExp(`ref: ${shaExpression}`), "Checkout tested revision must check out the tested SHA"); - assert.match(sync, new RegExp(`DEPLOY_GIT_SHA: ${shaExpression}`), "Sync and rebuild must use the tested SHA"); - assert.match(sync, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/, "Sync and rebuild must inject its SHA into the web runtime"); - assert.match(verification, new RegExp(`DEPLOY_GIT_SHA: ${shaExpression}`), "Verify production must use the tested SHA"); - assert.match(verification, /curl --fail --silent --show-error https:\/\/jyotisha\.chat\/api\/health/); - assert.match(verification, /get\("deployment", \{\}\)\.get\("gitCommit"/); - assert.match(verification, /Production revision did not converge/); -} - test("health endpoint exposes deployment identity for production verification", () => { const source = readFileSync( new URL("../src/app/api/health/route.ts", import.meta.url), @@ -73,26 +46,15 @@ test("health endpoint exposes deployment identity for production verification", assert.doesNotMatch(source, /anyEnvCheck\(\["LLM_MODELS_JSON"|OPENAI_API_KEY|DEEPSEEK_API_KEY|LLM_API_KEY/); }); -test("manual production deployment passes the selected revision into the web runtime", () => { - const compose = readFileSync( - new URL("../../deploy/docker-compose.server.yml", import.meta.url), - "utf8", - ); +test("GitHub mirror cannot deploy production", () => { const workflow = readFileSync( new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8", ); - - assert.match(compose, /GITHUB_SHA: \$\{GITHUB_SHA\}/); - assert.match(workflow, /workflow_dispatch:/); - assert.doesNotMatch(workflow, /workflow_run:/); - assert.match(workflow, /DEPLOY_GIT_SHA: \$\{\{ github\.sha \}\}/); - assert.match(workflow, /GITHUB_SHA='\$DEPLOY_GIT_SHA'/); - assert.match(workflow, /get\("deployment", \{\}\)\.get\("gitCommit"/); - assert.match(workflow, /DEPLOY_GIT_SHA/); - assert.match(workflow, /Production revision did not converge/); - assert.match(workflow, /git ls-remote origin refs\/heads\/main/); - assert.match(workflow, /steps\.revision\.outputs\.deploy == 'true'/); + assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); + assert.match(workflow, /Refuse deployment from the mirror/); + assert.match(workflow, /exit 1/); + assert.doesNotMatch(workflow, /ssh|rsync|docker compose/); }); test("server compose accepts staging paths while preserving production defaults", () => { @@ -178,6 +140,21 @@ test("staging Caddy serves the same app on two exact hosts", () => { assert.doesNotMatch(caddy, /www\.jyotisha\.chat/); }); +test("self-hosted production Caddy isolates user and admin hosts", () => { + const caddy = readFileSync( + new URL("../../deploy/Caddyfile.production.selfhosted", import.meta.url), + "utf8", + ); + + assert.match(caddy, /\{\$SITE_ADDRESS:https:\/\/jyotisha\.chat\}/); + assert.match(caddy, /@adminPaths path \/admin \/admin\/\* \/api\/admin\/\*/); + assert.match(caddy, /respond @adminPaths "Not found" 404/); + assert.match(caddy, /^https:\/\/admin\.jyotisha\.chat \{$/m); + assert.match(caddy, /^https:\/\/www\.jyotisha\.chat \{$/m); + assert.match(caddy, /redir https:\/\/jyotisha\.chat\{uri\} 308/); + assert.equal((caddy.match(/reverse_proxy web:3000/g) ?? []).length, 2); +}); + test("staging deploy consumes only the isolated staging environment and tested revision", () => { const qualityGate = readFileSync( new URL("../../.github/workflows/backend-quality-gate.yml", import.meta.url), @@ -442,24 +419,17 @@ test("production traffic waits for a healthy web container and retries short rep assert.match(caddyfile, /reverse_proxy web:3000 \{\n\s+lb_try_duration 10s\n\s+lb_try_interval 250ms\n\s+\}/); }); -test("production workflow consistently uses its tested revision from checkout through verification", () => { - const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8"); +test("Gitea production verification binds the exact requested SHA", () => { + const workflow = readFileSync(new URL("../../.gitea/workflows/deploy-production.yml", import.meta.url), "utf8"); + const runner = readFileSync(new URL("../../deploy/run-production-deploy.sh", import.meta.url), "utf8"); - assertWorkflowUsesTestedSha(workflow); -}); - -test("production workflow rejects a SHA mismatch in verification", () => { - const workflow = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8"); - const verification = workflowStepBlock(workflow, "Verify production"); - const mismatchedVerification = verification.replace( - testedShaExpression, - "${{ github.event.workflow_run.head_sha || github.sha }}", - ); - - assert.throws( - () => assertWorkflowUsesTestedSha(workflow.replace(verification, mismatchedVerification)), - /Verify production must use the tested SHA/, - ); + assert.match(workflow, /REQUESTED_SHA: \$\{\{ inputs\.deploy_sha \}\}/); + assert.match(workflow, /main and staging must identify the same reviewed release/); + assert.match(workflow, /public staging has not accepted the requested SHA/); + assert.match(workflow, /DEPLOY_SHA='\$DEPLOY_SHA'/); + assert.match(runner, /publicBody\.deployment\?\.gitCommit === process\.env\.EXPECTED_SHA/); + assert.match(runner, /VERIFICATION_MODE/); + assert.match(runner, /select 1 as ready/); }); test("production API probes health rapidly while a replacement container starts", () => { diff --git a/frontend/tests/staging-backend-workflows.test.ts b/frontend/tests/staging-backend-workflows.test.ts index 4c9756b9..9d69afa1 100644 --- a/frontend/tests/staging-backend-workflows.test.ts +++ b/frontend/tests/staging-backend-workflows.test.ts @@ -58,6 +58,14 @@ const syncScript = new URL( "../../deploy/sync-staging-tree.sh", import.meta.url, ); +const productionDeployScript = new URL( + "../../deploy/run-production-deploy.sh", + import.meta.url, +); +const productionSyncScript = new URL( + "../../deploy/sync-production-tree.sh", + import.meta.url, +); function read(url: URL): string { return readFileSync(url, "utf8"); @@ -80,6 +88,7 @@ test("changed staging workflows are syntactically valid YAML", () => { giteaQualityWorkflow, giteaDeployWorkflow, giteaMigrationWorkflow, + giteaProductionWorkflow, ]) { const result = spawnSync( "python", @@ -791,20 +800,38 @@ test("run-local registry state and incoming trees are always cleaned up", () => } }); -test("production remains manual-only and separate from staging database automation", () => { - for (const production of [ - readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8"), - read(giteaProductionWorkflow), - ]) { - assert.match(production, /^on:\n\s+workflow_dispatch:/m); - assert.doesNotMatch(production, /workflow_run:|\n\s+push:/); - assert.doesNotMatch(production, /docker-compose\.postgres\.yml|db:migrate/); +test("production deploy is manual-only and consumes the accepted staging artifact", () => { + const mirror = readFileSync(new URL("../../.github/workflows/deploy-production.yml", import.meta.url), "utf8"); + const production = read(giteaProductionWorkflow); + + for (const workflow of [mirror, production]) { + assert.match(workflow, /^on:\n\s+workflow_dispatch:/m); + assert.doesNotMatch(workflow, /workflow_run:|\n\s+push:/); } + assert.match(mirror, /Production deployment is controlled by \.gitea\/workflows\/deploy-production\.yml/); + assert.match(production, /branch=staging&event=push&status=success/); + assert.match(production, /endswith\("backend-quality-gate\.yml"\)/); + assert.match(production, /endswith\("release-quality-gate\.yml"\)/); + assert.match(production, /public staging has not accepted the requested SHA/); + assert.match(production, /current_main.*DEPLOY_SHA.*current_staging.*DEPLOY_SHA/s); + assert.match(production, /verification_mode:/); + assert.match(production, /PRODUCTION_HOST/); + assert.match(production, /118\.194\.235\.34/); + assert.match(production, /PRODUCTION_SSH_PRIVATE_KEY/); + assert.match(production, /staging-image-manifest-/); + assert.match(production, /run-production-deploy\.sh/); + assert.doesNotMatch(production, /--build|root@|103\.117\.123\.53/); }); test("staging scripts pass shell syntax validation", () => { - for (const script of [deployScript, migrationScript, syncScript]) { + for (const script of [ + deployScript, + migrationScript, + syncScript, + productionDeployScript, + productionSyncScript, + ]) { const path = fileURLToPath(script); chmodSync(path, 0o755); const result = spawnSync("bash", ["-n", path], { encoding: "utf8" });