Jyotisha/.gitea/workflows/deploy-production.yml

name: Deploy production

on:
  workflow_dispatch:
    inputs:
      deploy_sha:
        description: Exact tested 40-character staging commit SHA
        required: true
        type: string
      allow_rollback:
        description: Explicitly permit a manual rollback to an older tested SHA
        required: true
        default: false
        type: boolean
      verification_mode:
        description: Use internal before DNS cutover; public after DNS and TLS converge
        required: true
        default: internal
        type: choice
        options:
          - internal
          - public

permissions:
  contents: read
  actions: read

concurrency:
  group: production-mutation
  cancel-in-progress: false
  queue: max

jobs:
  deploy:
    runs-on: xiaoxin
    timeout-minutes: 30
    env:
      GITEA_SHA: ${{ gitea.sha }}
      GITEA_API_URL: ${{ gitea.api_url }}
      GITEA_REPOSITORY: ${{ gitea.repository }}
      GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
      REGISTRY_HOST: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com
      IMAGE_REPOSITORY: crpi-d1feco6itet73spp.cn-hongkong.personal.cr.aliyuncs.com/copse/jyotisha
      DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }}
      DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }}
      DEPLOY_USER: ${{ vars.PRODUCTION_USER }}
      DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }}
      PRODUCTION_URL: ${{ vars.PRODUCTION_URL }}
      PRODUCTION_ADMIN_URL: ${{ vars.PRODUCTION_ADMIN_URL }}
      STAGING_URL: ${{ vars.STAGING_URL }}
      PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }}
    steps:
      - name: Validate tested revision and gate run
        id: revision
        env:
          REQUESTED_SHA: ${{ inputs.deploy_sha }}
          REQUESTED_ROLLBACK: ${{ inputs.allow_rollback }}
          VERIFICATION_MODE: ${{ inputs.verification_mode }}
        run: |
          set -euo pipefail
          [[ "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; }
          [[ "$VERIFICATION_MODE" == internal || "$VERIFICATION_MODE" == public ]] || { echo "invalid verification_mode" >&2; exit 1; }
          [[ "$STAGING_URL" == "https://staging.jyotisha.chat" ]] || { echo "unexpected staging acceptance URL" >&2; exit 1; }
          allow_rollback=false
          if [[ "$REQUESTED_ROLLBACK" == true ]]; then
            allow_rollback=true
          fi

          runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
            --header "Authorization: token $GITEA_TOKEN" \
            "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$REQUESTED_SHA&branch=staging&event=push&status=success&limit=100")"
          selected_run="$(jq -cer --arg sha "$REQUESTED_SHA" '
            [.workflow_runs[] | select(
              (.path | split("@")[0] | endswith("backend-quality-gate.yml")) and
              .head_sha == $sha and .head_branch == "staging" and
              .event == "push" and .conclusion == "success"
            )] | sort_by(.id) | reverse | first
          ' <<<"$runs")"
          gate_run_id="$(jq -er '.id' <<<"$selected_run")"
          [[ "$gate_run_id" =~ ^[0-9]+$ ]] || { echo "no successful exact-SHA production quality gate run found" >&2; exit 1; }

          read_ref_sha() {
            local branch="$1"
            curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" |
              jq -er --arg ref "refs/heads/$branch" '
                select(type == "array" and length == 1) |
                .[0] | select(.ref == $ref) | .object.sha |
                select(test("^[0-9a-f]{40}$"))
              '
          }
          staging_head="$(read_ref_sha staging)"
          controller_sha="$(read_ref_sha main)"
          [[ "$controller_sha" == "$staging_head" ]] || { echo "main and staging must identify the same reviewed release" >&2; exit 1; }
          if [[ "$allow_rollback" == false && "$REQUESTED_SHA" != "$staging_head" ]]; then
            echo "stale production revision refused; use explicit manual rollback only when intended" >&2
            exit 1
          fi
          if [[ "$allow_rollback" == true && "$REQUESTED_SHA" != "$controller_sha" ]]; then
            comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$REQUESTED_SHA...$controller_sha")"
            jq -e --arg base "$REQUESTED_SHA" --arg head "$controller_sha" '
              (.commits // []) as $commits |
              def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha];
              def reaches($sha; $seen):
                if $sha == $base then true
                elif ($seen | index($sha)) != null then false
                else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end;
              (.total_commits | type) == "number" and
              .total_commits == ($commits | length) and ($commits | length) > 0 and
              ([$commits[].sha] | length == (unique | length)) and reaches($head; [])
            ' <<<"$comparison" >/dev/null || { echo "rollback revision is not in reviewed main history" >&2; exit 1; }
          fi

          controller_gate_run_id="$gate_run_id"
          if [[ "$controller_sha" != "$REQUESTED_SHA" ]]; then
            controller_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$controller_sha&branch=staging&event=push&status=success&limit=100")"
            controller_run="$(jq -cer --arg sha "$controller_sha" '
              [.workflow_runs[] | select(
                (.path | split("@")[0] | endswith("backend-quality-gate.yml")) and
                .head_sha == $sha and .head_branch == "staging" and
                .event == "push" and .conclusion == "success"
              )] | sort_by(.id) | reverse | first
            ' <<<"$controller_runs")"
            controller_gate_run_id="$(jq -er '.id' <<<"$controller_run")"
          fi
          [[ "$controller_gate_run_id" =~ ^[0-9]+$ ]]

          release_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
            --header "Authorization: token $GITEA_TOKEN" \
            "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$REQUESTED_SHA&event=workflow_dispatch&status=success&limit=100")"
          jq -e --arg sha "$REQUESTED_SHA" '
            any(.workflow_runs[]?;
              (.path | split("@")[0] | endswith("release-quality-gate.yml")) and
              .head_sha == $sha and .event == "workflow_dispatch" and .conclusion == "success"
            )
          ' <<<"$release_runs" >/dev/null || { echo "no successful exact-SHA manual release quality gate found" >&2; exit 1; }

          if [[ "$allow_rollback" == false ]]; then
            observed_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 30 --retry 3 --retry-all-errors \
              "$STAGING_URL/api/health" | jq -er '.deployment.gitCommit | select(test("^[0-9a-f]{40}$"))')"
            [[ "$observed_staging_sha" == "$REQUESTED_SHA" ]] || { echo "public staging has not accepted the requested SHA" >&2; exit 1; }
          fi
          {
            echo "sha=$REQUESTED_SHA"
            echo "gate_run_id=$gate_run_id"
            echo "controller_sha=$controller_sha"
            echo "controller_gate_run_id=$controller_gate_run_id"
            echo "allow_rollback=$allow_rollback"
            echo "verification_mode=$VERIFICATION_MODE"
          } >>"$GITHUB_OUTPUT"

      - name: Prepare pinned Node tooling
        env:
          NODE_TOOL_SOURCE_IMAGE: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/library/node:22-bookworm-slim@sha256:ef343465b6a14bbdf2ab52f6e100ec0659a792464fcf72c462370d88b3df909c
          NODE_TOOL_IMAGE: node:22-bookworm-slim
        run: |
          set -euo pipefail
          if ! docker image inspect "$NODE_TOOL_SOURCE_IMAGE" >/dev/null 2>&1; then
            for attempt in 1 2 3; do
              if timeout 180 docker pull "$NODE_TOOL_SOURCE_IMAGE"; then
                break
              fi
              if [ "$attempt" -eq 3 ]; then
                echo "Failed to preload $NODE_TOOL_IMAGE after $attempt attempts" >&2
                exit 1
              fi
              sleep $((attempt * 15))
            done
          fi
          docker tag "$NODE_TOOL_SOURCE_IMAGE" "$NODE_TOOL_IMAGE"
          docker image inspect "$NODE_TOOL_IMAGE" >/dev/null
          tool_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/jyotisha-node-tools.XXXXXX")"
          cat > "$tool_dir/node" <<'EOF'
          #!/usr/bin/env bash
          set -euo pipefail
          workdir="$(pwd -P)"
          exec docker run --rm \
            --user "$(id -u):$(id -g)" \
            --volume "$workdir:$workdir" \
            --workdir "$workdir" \
            --env HOME=/tmp \
            node:22-bookworm-slim "${0##*/}" "$@"
          EOF
          chmod 0755 "$tool_dir/node"
          ln -s node "$tool_dir/npm"
          test -n "${GITHUB_PATH:-}"
          printf '%s\n' "$tool_dir" >> "$GITHUB_PATH"
          export PATH="$tool_dir:$PATH"
          node --version
          npm --version

      - name: Download target and controller gate artifacts
        env:
          TARGET_GATE_RUN_ID: ${{ steps.revision.outputs.gate_run_id }}
          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
          CONTROLLER_GATE_RUN_ID: ${{ steps.revision.outputs.controller_gate_run_id }}
          CONTROLLER_SHA: ${{ steps.revision.outputs.controller_sha }}
        run: |
          set -euo pipefail
          download_bundle() {
            local run_id="$1" sha="$2" destination="$3" zip_path="$4"
            local prefix artifacts selected name id attempt
            prefix="staging-image-manifest-$sha-"
            artifacts="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs/$run_id/artifacts?limit=100")"
            selected="$(jq -cer --arg prefix "$prefix" '
              [(.artifacts // [])[]
                | select(.expired == false and (.name | startswith($prefix)))
                | . + {attempt: ((.name | ltrimstr($prefix)) | tonumber?)}
                | select(.attempt != null and .attempt >= 1)
              ] | sort_by(.attempt, .id) | reverse | first
            ' <<<"$artifacts")"
            name="$(jq -er '.name' <<<"$selected")"
            id="$(jq -er '.id' <<<"$selected")"
            attempt="${name#"$prefix"}"
            [[ "$name" == "$prefix"* && "$attempt" =~ ^[1-9][0-9]*$ && "$id" =~ ^[0-9]+$ ]]
            install -d -m 700 "$destination"
            curl --fail --silent --show-error --location --connect-timeout 15 --max-time 120 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/artifacts/$id/zip" \
              --output "$zip_path"
            python3 - "$zip_path" "$destination" <<'PY'
          import pathlib, stat, sys, zipfile
          archive = pathlib.Path(sys.argv[1])
          destination = pathlib.Path(sys.argv[2])
          allowed = {"manifest.env", "controller.tar"}
          with zipfile.ZipFile(archive) as bundle:
              entries = bundle.infolist()
              names = [entry.filename for entry in entries]
              if len(names) != len(set(names)) or not names or not set(names).issubset(allowed):
                  raise SystemExit("invalid production artifact bundle")
              if sum(entry.file_size for entry in entries) > 3 * 1024 * 1024:
                  raise SystemExit("production artifact bundle is too large")
              for entry in entries:
                  path = pathlib.PurePosixPath(entry.filename)
                  mode = entry.external_attr >> 16
                  if path.is_absolute() or ".." in path.parts or path.name != entry.filename:
                      raise SystemExit("unsafe production artifact path")
                  if mode and not stat.S_ISREG(mode):
                      raise SystemExit("unsafe production artifact type")
                  target = destination / entry.filename
                  with bundle.open(entry) as source, target.open("xb") as output:
                      output.write(source.read())
          PY
            [[ -f "$destination/manifest.env" ]]
          }
          rm -rf artifacts/staging-image artifacts/controller
          download_bundle "$TARGET_GATE_RUN_ID" "$DEPLOY_SHA" artifacts/staging-image "${RUNNER_TEMP}/staging-target.zip"
          download_bundle "$CONTROLLER_GATE_RUN_ID" "$CONTROLLER_SHA" artifacts/controller "${RUNNER_TEMP}/production-controller.zip"

      - name: Validate gate-attested controller and immutable image manifest
        id: images
        env:
          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
          CONTROLLER_SHA: ${{ steps.revision.outputs.controller_sha }}
        run: |
          set -euo pipefail
          controller_manifest=artifacts/controller/manifest.env
          controller_tar=artifacts/controller/controller.tar
          [[ -f "$controller_tar" ]]
          [[ "$(wc -l < "$controller_manifest" | tr -d ' ')" == 4 ]]
          manifest_controller_sha="$(awk -F= '$1 == "git_sha" {print $2}' "$controller_manifest")"
          expected_controller_digest="$(awk -F= '$1 == "controller_sha256" {print $2}' "$controller_manifest")"
          [[ "$manifest_controller_sha" == "$CONTROLLER_SHA" ]]
          [[ "$expected_controller_digest" =~ ^[0-9a-f]{64}$ ]]
          printf '%s  %s\n' "$expected_controller_digest" "$controller_tar" | sha256sum --check --status
          python3 - "$controller_tar" <<'PY'
          import pathlib, sys, tarfile
          archive = pathlib.Path(sys.argv[1])
          required = {"deploy/run-production-deploy.sh", "frontend/scripts/staging-image-manifest.mjs"}
          with tarfile.open(archive, "r:") as bundle:
              members = bundle.getmembers()
              names = [member.name for member in members]
              if len(names) != len(set(names)) or not required.issubset(names):
                  raise SystemExit("invalid production controller bundle")
              if sum(member.size for member in members) > 2 * 1024 * 1024:
                  raise SystemExit("production controller bundle is too large")
              for member in members:
                  path = pathlib.PurePosixPath(member.name)
                  if path.is_absolute() or ".." in path.parts or not (member.isdir() or member.isfile()):
                      raise SystemExit("unsafe production controller bundle")
          PY
          install -d -m 700 artifacts/controller/extracted
          tar -xf "$controller_tar" -C artifacts/controller/extracted
          node artifacts/controller/extracted/frontend/scripts/staging-image-manifest.mjs \
            "$controller_manifest" "$CONTROLLER_SHA" "$IMAGE_REPOSITORY" >/dev/null
          node artifacts/controller/extracted/frontend/scripts/staging-image-manifest.mjs \
            artifacts/staging-image/manifest.env "$DEPLOY_SHA" "$IMAGE_REPOSITORY" >>"$GITHUB_OUTPUT"

      - name: Deploy exact image digests under pinned SSH identity
        env:
          SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }}
          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
          API_IMAGE: ${{ steps.images.outputs.api_image }}
          WEB_IMAGE: ${{ steps.images.outputs.web_image }}
          ALLOW_ROLLBACK: ${{ steps.revision.outputs.allow_rollback }}
          VERIFICATION_MODE: ${{ steps.revision.outputs.verification_mode }}
        run: |
          set -euo pipefail
          [[ "$DEPLOY_HOST" == "118.194.235.34" ]]
          [[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 ))
          [[ "$DEPLOY_USER" == "deploy" ]]
          [[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]]
          [[ "$PRODUCTION_URL" == "https://jyotisha.chat" ]]
          [[ "$PRODUCTION_ADMIN_URL" == "https://admin.jyotisha.chat" ]]
          [[ "$VERIFICATION_MODE" == internal || "$VERIFICATION_MODE" == public ]]
          ssh_root="${RUNNER_TEMP}/production-ssh"
          key_path="$ssh_root/id_ed25519"
          known_hosts_path="$ssh_root/known_hosts"
          incoming=""
          install -m 700 -d "$ssh_root"
          test -n "$SSH_PRIVATE_KEY_BASE64"
          printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode > "$key_path"
          printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' > "$known_hosts_path"
          chmod 600 "$key_path" "$known_hosts_path"
          ssh-keygen -y -f "$key_path" >/dev/null
          ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
          remote="$DEPLOY_USER@$DEPLOY_HOST"
          require_current_release_heads() {
            [[ "$ALLOW_ROLLBACK" == true ]] && return
            current_staging="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/staging" |
              jq -er 'select(type == "array" and length == 1) | .[0] |
                select(.ref == "refs/heads/staging") | .object.sha |
                select(test("^[0-9a-f]{40}$"))')"
            current_main="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/main" |
              jq -er 'select(type == "array" and length == 1) | .[0] |
                select(.ref == "refs/heads/main") | .object.sha |
                select(test("^[0-9a-f]{40}$"))')"
            [[ "$current_main" == "$DEPLOY_SHA" && "$current_staging" == "$DEPLOY_SHA" ]] || {
              echo "main or staging advanced during deployment; refusing stale mutation" >&2
              exit 1
            }
          }
          cleanup() {
            if [[ -n "$incoming" ]]; then
              ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; sudo -n rm -rf -- '$incoming'" >/dev/null 2>&1 || true
            fi
            rm -rf -- "$ssh_root"
          }
          trap cleanup EXIT
          incoming="$(ssh "${ssh_options[@]}" "$remote" "mktemp -d /tmp/jyotisha-production.XXXXXXXXXX")"
          [[ "$incoming" == /tmp/jyotisha-production.* ]]
          ssh "${ssh_options[@]}" "$remote" "install -d -m 700 '$incoming/.docker'"
          scp -i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path" artifacts/controller/controller.tar "$remote:$incoming/controller.tar"
          ssh "${ssh_options[@]}" "$remote" "tar -xf '$incoming/controller.tar' -C '$incoming' && rm -f -- '$incoming/controller.tar'"
          previous_sha="$(ssh "${ssh_options[@]}" "$remote" "state='$DEPLOY_PATH/.state/deployed-revision'; id=\$(sudo -n docker ps -aq --filter 'label=com.docker.compose.project=jyotisha-production' --filter 'label=com.docker.compose.service=web' | head -n 1); if [ -n \"\$id\" ]; then sudo -n docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"\$id\" | sed -n 's/^GITHUB_SHA=//p' | head -n 1; elif [ -e \"\$state\" ]; then printf state-present-without-container; else printf not-deployed; fi")"
          [[ "$previous_sha" == not-deployed || "$previous_sha" =~ ^[0-9a-f]{40}$ ]] || exit 1
          forward_verified=false
          if [[ "$previous_sha" != not-deployed && "$previous_sha" != "$DEPLOY_SHA" && "$ALLOW_ROLLBACK" != true ]]; then
            comparison="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/compare/$previous_sha...$DEPLOY_SHA")"
            jq -e --arg base "$previous_sha" --arg head "$DEPLOY_SHA" '
              (.commits // []) as $commits |
              def parents($sha): [$commits[] | select(.sha == $sha) | (.parents // [])[] | .sha];
              def reaches($sha; $seen):
                if $sha == $base then true
                elif ($seen | index($sha)) != null then false
                else any(parents($sha)[]; . as $parent | reaches($parent; $seen + [$sha])) end;
              (.total_commits | type) == "number" and
              .total_commits == ($commits | length) and ($commits | length) > 0 and
              ([$commits[].sha] | length == (unique | length)) and reaches($head; [])
            ' <<<"$comparison" >/dev/null || { echo "automatic production rollback or divergent deploy refused" >&2; exit 1; }
            forward_verified=true
          fi
          require_current_release_heads
          printf '%s' "$REGISTRY_PASSWORD" | ssh "${ssh_options[@]}" "$remote" "sudo -n docker --config '$incoming/.docker' login '$REGISTRY_HOST' --username '$REGISTRY_USERNAME' --password-stdin"
          ssh "${ssh_options[@]}" "$remote" "sudo -n env INCOMING_PATH='$incoming' DEPLOY_PATH='$DEPLOY_PATH' API_IMAGE='$API_IMAGE' WEB_IMAGE='$WEB_IMAGE' DEPLOY_SHA='$DEPLOY_SHA' EXPECTED_PREVIOUS_SHA='$previous_sha' ALLOW_ROLLBACK='$ALLOW_ROLLBACK' FORWARD_REVISION_VERIFIED='$forward_verified' DOCKER_CONFIG='$incoming/.docker' DOCKER_BIN='docker' PRODUCTION_URL='$PRODUCTION_URL' PRODUCTION_ADMIN_URL='$PRODUCTION_ADMIN_URL' VERIFICATION_MODE='$VERIFICATION_MODE' bash '$incoming/deploy/run-production-deploy.sh'"
          require_current_release_heads