Jyotisha/.gitea/workflows/create-production-recovery.yml

name: Create Production Recovery Point (manual only)

on:
  workflow_dispatch:
    inputs:
      deploy_sha:
        description: Exact accepted production release SHA this recovery point protects
        required: true
        type: string

permissions:
  contents: read
  actions: write

concurrency:
  group: production-mutation
  cancel-in-progress: false
  queue: max

jobs:
  recover:
    runs-on: manman-linux
    timeout-minutes: 30
    env:
      GITEA_SHA: ${{ gitea.sha }}
      GITEA_API_URL: ${{ gitea.api_url }}
      GITEA_REPOSITORY: ${{ gitea.repository }}
      GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
      DEPLOY_HOST: ${{ vars.PRODUCTION_HOST }}
      DEPLOY_PORT: ${{ vars.PRODUCTION_PORT }}
      DEPLOY_USER: ${{ vars.PRODUCTION_USER }}
      DEPLOY_PATH: ${{ vars.PRODUCTION_PATH }}
      STAGING_URL: ${{ vars.STAGING_URL }}
      PRODUCTION_KNOWN_HOSTS: ${{ vars.PRODUCTION_KNOWN_HOSTS }}
    steps:
      - name: Validate exact accepted release
        id: revision
        env:
          DEPLOY_SHA: ${{ inputs.deploy_sha }}
        run: |
          set -euo pipefail
          [[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "deploy_sha must be a lowercase full commit SHA" >&2; exit 1; }
          [[ "$GITEA_SHA" == "$DEPLOY_SHA" ]] || { echo "dispatch recovery from the exact main release SHA" >&2; exit 1; }
          [[ "$STAGING_URL" == "https://staging.jyotisha.chat" ]] || { echo "unexpected staging acceptance URL" >&2; exit 1; }
          read_ref_sha() {
            local branch="$1"
            curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
              --header "Authorization: token $GITEA_TOKEN" \
              "$GITEA_API_URL/repos/$GITEA_REPOSITORY/git/refs/heads/$branch" |
              jq -er --arg ref "refs/heads/$branch" '
                select(type == "array" and length == 1) |
                .[0] | select(.ref == $ref) | .object.sha |
                select(test("^[0-9a-f]{40}$"))
              '
          }
          staging_head="$(read_ref_sha staging)"
          main_head="$(read_ref_sha main)"
          [[ "$main_head" == "$DEPLOY_SHA" && "$staging_head" == "$DEPLOY_SHA" ]] || {
            echo "production recovery requires main and staging to equal deploy_sha" >&2
            exit 1
          }
          observed_staging_sha="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 30 --retry 3 --retry-all-errors \
            "$STAGING_URL/api/health" | jq -er '.deployment.gitCommit | select(test("^[0-9a-f]{40}$"))')"
          [[ "$observed_staging_sha" == "$DEPLOY_SHA" ]] || {
            echo "public staging has not accepted the requested SHA" >&2
            exit 1
          }
          release_runs="$(curl --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-all-errors \
            --header "Authorization: token $GITEA_TOKEN" \
            "$GITEA_API_URL/repos/$GITEA_REPOSITORY/actions/runs?head_sha=$DEPLOY_SHA&event=workflow_dispatch&status=success&limit=100")"
          jq -e --arg sha "$DEPLOY_SHA" '
            any(.workflow_runs[]?;
              (.path | split("@")[0] | endswith("release-quality-gate.yml")) and
              .head_sha == $sha and .event == "workflow_dispatch" and .conclusion == "success"
            )
          ' <<<"$release_runs" >/dev/null || {
            echo "no successful exact-SHA manual release quality gate found" >&2
            exit 1
          }
          echo "sha=$DEPLOY_SHA" >>"$GITHUB_OUTPUT"

      - name: Checkout exact recovery controller
        env:
          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
        run: |
          set -euo pipefail
          git init .
          git remote remove origin 2>/dev/null || true
          git remote add origin https://git.copse.top/root/Jyotisha.git
          git fetch --no-tags origin "$DEPLOY_SHA"
          git checkout --detach --force "$DEPLOY_SHA"
          [[ "$(git rev-parse HEAD)" == "$DEPLOY_SHA" ]]

      - name: Create, restore-verify, and retrieve encrypted recovery point
        id: recovery
        env:
          SSH_PRIVATE_KEY_BASE64: ${{ secrets.PRODUCTION_SSH_PRIVATE_KEY }}
          DEPLOY_SHA: ${{ steps.revision.outputs.sha }}
        run: |
          set -euo pipefail
          set +x
          [[ "$DEPLOY_HOST" == "118.194.235.34" ]]
          [[ "$DEPLOY_PORT" =~ ^[1-9][0-9]{0,4}$ ]] && (( DEPLOY_PORT <= 65535 ))
          [[ "$DEPLOY_USER" == "deploy" ]]
          [[ "$DEPLOY_PATH" == "/opt/jyotisha-production" ]]
          [[ "${GITHUB_RUN_ID:-}" =~ ^[0-9]+$ ]]
          test -n "$PRODUCTION_KNOWN_HOSTS"
          ssh_root="${RUNNER_TEMP}/production-recovery-ssh"
          key_path="$ssh_root/id_ed25519"
          known_hosts_path="$ssh_root/known_hosts"
          artifact_directory="artifacts/production-recovery"
          install -m 700 -d "$ssh_root" "$artifact_directory"
          test -n "$SSH_PRIVATE_KEY_BASE64"
          printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode >"$key_path"
          printf '%s\n' "$PRODUCTION_KNOWN_HOSTS" | tr -d '\r' >"$known_hosts_path"
          chmod 600 "$key_path" "$known_hosts_path"
          ssh-keygen -y -f "$key_path" >/dev/null
          ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
          scp_options=(-i "$key_path" -P "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
          remote="$DEPLOY_USER@$DEPLOY_HOST"
          incoming="$(ssh "${ssh_options[@]}" "$remote" 'mktemp -d /tmp/jyotisha-production-recovery.XXXXXXXXXX')"
          [[ "$incoming" == /tmp/jyotisha-production-recovery.* ]]
          cleanup() {
            ssh "${ssh_options[@]}" "$remote" "rm -rf -- '$incoming'" >/dev/null 2>&1 || true
            rm -rf -- "$ssh_root"
          }
          trap cleanup EXIT
          git show "$DEPLOY_SHA:deploy/run-production-recovery.sh" >"$ssh_root/run-production-recovery.sh"
          chmod 700 "$ssh_root/run-production-recovery.sh"
          scp "${scp_options[@]}" "$ssh_root/run-production-recovery.sh" "$remote:$incoming/run-production-recovery.sh"
          ssh "${ssh_options[@]}" "$remote" \
            "DEPLOY_PATH='$DEPLOY_PATH' RECOVERY_RUN_ID='$GITHUB_RUN_ID' bash '$incoming/run-production-recovery.sh'" \
            >"$ssh_root/recovery-output.env"
          [[ "$(wc -l <"$ssh_root/recovery-output.env" | tr -d ' ')" == 7 ]]
          recovery_reference="$(awk -F= '$1 == "RECOVERY_REFERENCE" {print $2}' "$ssh_root/recovery-output.env")"
          recovery_created_at="$(awk -F= '$1 == "RECOVERY_CREATED_AT" {print $2}' "$ssh_root/recovery-output.env")"
          recovery_verified_at="$(awk -F= '$1 == "RECOVERY_VERIFIED_AT" {print $2}' "$ssh_root/recovery-output.env")"
          recovery_sha256="$(awk -F= '$1 == "RECOVERY_SHA256" {print $2}' "$ssh_root/recovery-output.env")"
          restore_verified="$(awk -F= '$1 == "RESTORE_VERIFIED" {print $2}' "$ssh_root/recovery-output.env")"
          backup_basename="$(awk -F= '$1 == "BACKUP_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
          verify_basename="$(awk -F= '$1 == "VERIFY_BASENAME" {print $2}' "$ssh_root/recovery-output.env")"
          [[ "$backup_basename" =~ ^production-pre-migration-[0-9]{8}T[0-9]{6}Z\.dump\.enc$ ]]
          [[ "$verify_basename" == "${backup_basename%.dump.enc}-restore-verify.json" ]]
          [[ "$recovery_reference" == "gitea-actions-run-${GITHUB_RUN_ID}/${backup_basename}" ]]
          [[ "$recovery_created_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
          [[ "$recovery_verified_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
          [[ "$recovery_sha256" =~ ^[0-9a-f]{64}$ ]]
          [[ "$restore_verified" == "true" ]]
          scp "${scp_options[@]}" \
            "$remote:$DEPLOY_PATH/backups/$backup_basename" \
            "$remote:$DEPLOY_PATH/backups/$verify_basename" \
            "$artifact_directory/"
          test -s "$artifact_directory/$backup_basename"
          test -s "$artifact_directory/$verify_basename"
          printf '%s  %s\n' "$recovery_sha256" "$artifact_directory/$backup_basename" | sha256sum --check --status
          jq -e \
            --arg backup "$backup_basename" \
            --arg sha "$recovery_sha256" \
            --arg created "$recovery_created_at" \
            --arg verified "$recovery_verified_at" '
              .mode == "restore_verify" and .ok == true and
              .backup == $backup and .sha256 == $sha and
              .created_at == $created and .verified_at == $verified and
              .restore_database_removed == true and
              ([.checks.identity_users, .checks.profiles, .checks.credit_transactions, .checks.public_tables] |
                all(type == "number" and . >= 0 and floor == .))
            ' "$artifact_directory/$verify_basename" >/dev/null
          printf 'RECOVERY_REFERENCE=%s\nRECOVERY_CREATED_AT=%s\nRECOVERY_VERIFIED_AT=%s\nRECOVERY_SHA256=%s\nRESTORE_VERIFIED=true\nBACKUP_BASENAME=%s\nVERIFY_BASENAME=%s\n' \
            "$recovery_reference" "$recovery_created_at" "$recovery_verified_at" "$recovery_sha256" \
            "$backup_basename" "$verify_basename" >"$artifact_directory/attestation.env"
          chmod 600 "$artifact_directory"/*
          {
            echo "recovery_reference=$recovery_reference"
            echo "recovery_created_at=$recovery_created_at"
            echo "recovery_verified_at=$recovery_verified_at"
            echo "recovery_sha256=$recovery_sha256"
          } >>"$GITHUB_OUTPUT"
          echo "Recovery restore-verified: reference=$recovery_reference created_at=$recovery_created_at verified_at=$recovery_verified_at"

      - name: Upload encrypted off-site recovery artifact
        run: |
          set -euo pipefail
          test -n "${ACTIONS_RUNTIME_TOKEN:-}"
          test -n "${ACTIONS_RESULTS_URL:-}"
          test -n "${GITHUB_RUN_ID:-}"
          test -n "${GITHUB_REPOSITORY:-}"
          workdir="$(pwd -P)"
          docker run --rm \
            --user "$(id -u):$(id -g)" \
            --volume "$workdir:$workdir" \
            --workdir "$workdir" \
            --env HOME=/tmp \
            --env "INPUT_NAME=production-recovery-$GITHUB_RUN_ID" \
            --env INPUT_PATH=artifacts/production-recovery/ \
            --env INPUT_OVERWRITE=false \
            --env ACTIONS_RUNTIME_TOKEN \
            --env ACTIONS_RESULTS_URL \
            --env GITHUB_RUN_ID \
            --env GITHUB_REPOSITORY \
            --env "GITHUB_SHA=$GITEA_SHA" \
            --env "GITHUB_WORKSPACE=$workdir" \
            node:22-bookworm-slim \
            node -e 'process.env["INPUT_IF-NO-FILES-FOUND"]="error"; process.env["INPUT_RETENTION-DAYS"]="30"; process.env["INPUT_COMPRESSION-LEVEL"]="0"; require("./.gitea/actions/upload-artifact/dist/index.js")'

      - name: Publish recovery attestation
        env:
          RECOVERY_REFERENCE: ${{ steps.recovery.outputs.recovery_reference }}
          RECOVERY_CREATED_AT: ${{ steps.recovery.outputs.recovery_created_at }}
          RECOVERY_VERIFIED_AT: ${{ steps.recovery.outputs.recovery_verified_at }}
          RECOVERY_SHA256: ${{ steps.recovery.outputs.recovery_sha256 }}
        run: |
          set -euo pipefail
          echo "Recovery attested: reference=$RECOVERY_REFERENCE created_at=$RECOVERY_CREATED_AT verified_at=$RECOVERY_VERIFIED_AT restore_verified=true sha256=$RECOVERY_SHA256"