chore: inspect staging request access log

.gitea/workflows/diagnose-rectification-request.yml

@@ -0,0 +1,87 @@
+name: Diagnose rectification request (temporary)
+
+on:
+  push:
+    branches:
+      - codex/diagnose-rectification-request-20260827
+
+permissions:
+  contents: read
+
+jobs:
+  diagnose:
+    runs-on: xiaoxin
+    timeout-minutes: 10
+    env:
+      DEPLOY_HOST: ${{ vars.STAGING_HOST }}
+      DEPLOY_PORT: ${{ vars.STAGING_PORT }}
+      DEPLOY_USER: ${{ vars.STAGING_USER }}
+      DEPLOY_PATH: ${{ vars.STAGING_PATH }}
+      STAGING_KNOWN_HOSTS: ${{ vars.STAGING_KNOWN_HOSTS }}
+      REQUEST_HASH: dc94cae0e5cf998f4128a9f882cdf9b5
+      CASE_HASH: 6b6659f437f3296fe54c0621b56a3052
+    steps:
+      - name: Query redacted durable runtime state
+        env:
+          SSH_PRIVATE_KEY_BASE64: ${{ secrets.STAGING_SSH_PRIVATE_KEY }}
+        run: |
+          set -euo pipefail
+          ssh_root="${RUNNER_TEMP}/staging-ssh"
+          key_path="$ssh_root/id_ed25519"
+          known_hosts_path="$ssh_root/known_hosts"
+          install -m 700 -d "$ssh_root"
+          trap 'rm -rf -- "$ssh_root"' EXIT
+          test -n "$SSH_PRIVATE_KEY_BASE64"
+          printf '%s' "$SSH_PRIVATE_KEY_BASE64" | base64 --decode > "$key_path"
+          printf '%s\n' "$STAGING_KNOWN_HOSTS" | tr -d '\r' > "$known_hosts_path"
+          chmod 600 "$key_path" "$known_hosts_path"
+          ssh-keygen -y -f "$key_path" >/dev/null
+          ssh_options=(-i "$key_path" -p "$DEPLOY_PORT" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts_path")
+          remote="$DEPLOY_USER@$DEPLOY_HOST"
+          ssh "${ssh_options[@]}" "$remote" "sudo -n docker compose --project-directory '$DEPLOY_PATH' -p jyotisha-staging -f '$DEPLOY_PATH/deploy/docker-compose.postgres.yml' exec -T postgres psql -X -v ON_ERROR_STOP=1 -U postgres -d jyotisha -v request_hash='$REQUEST_HASH' -v case_hash='$CASE_HASH'" <<'SQL'
+          \pset tuples_only on
+          \pset format unaligned
+          \echo TURN
+          select concat_ws('|', t.status, t.created_at, t.updated_at, (t.successful_attempt_id is not null)::text)
+          from public.agentic_rectification_turns t
+          where md5(t.request_id::text) = :'request_hash';
+          \echo ATTEMPTS
+          select concat_ws('|', a.attempt_number::text, a.status, coalesce(a.error_code, ''), a.started_at, coalesce(a.completed_at::text, ''))
+          from public.agentic_rectification_run_attempts a
+          join public.agentic_rectification_turns t on t.id = a.turn_id
+          where md5(t.request_id::text) = :'request_hash'
+          order by a.attempt_number;
+          \echo PHASES
+          select concat_ws('|', p.sequence::text, p.phase, coalesce(p.tool_name, ''), p.created_at)
+          from public.agentic_rectification_run_phases p
+          join public.agentic_rectification_turns t on t.id = p.turn_id
+          where md5(t.request_id::text) = :'request_hash'
+          order by p.sequence;
+          \echo RECEIPTS
+          select concat_ws('|', r.tool_name, r.public_phase, r.status, coalesce(r.safe_error_code, ''), r.started_at, coalesce(r.completed_at::text, ''))
+          from public.agentic_rectification_tool_receipts r
+          join public.agentic_rectification_turns t on t.id = r.turn_id
+          where md5(t.request_id::text) = :'request_hash'
+          order by r.started_at;
+          \echo CASE_STATE
+          select concat_ws('|', c.status, c.skill_version,
+            coalesce(s.active_focus->>'intent', 'none'),
+            coalesce(s.active_focus->>'status', 'none'),
+            coalesce(s.active_focus->'expected_answer_schema'->>'probe_id', 'none'))
+          from public.agentic_rectification_cases c
+          left join public.agentic_rectification_case_conversation_summaries s on s.case_id = c.id
+          where md5(c.id::text) = :'case_hash';
+          \echo RECENT_FOCUSES
+          select concat_ws('|', f.intent, coalesce(f.target_domain, ''), coalesce(f.target_kind, ''), f.status,
+            coalesce(f.expected_answer_schema->>'probe_id', ''), f.asked_at, coalesce(f.resolved_at::text, ''))
+          from public.agentic_rectification_conversation_focuses f
+          join public.agentic_rectification_cases c on c.id = f.case_id
+          where md5(c.id::text) = :'case_hash'
+          order by f.asked_at desc
+          limit 8;
+          SQL
+          ssh "${ssh_options[@]}" "$remote" "container=\$(sudo -n docker ps -q --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=web' | head -n 1); test -n \"\$container\"; sudo -n docker logs --since '2026-08-27T10:22:45Z' --until '2026-08-27T10:26:00Z' \"\$container\" 2>&1" | grep -Ei 'rectification|stream|abort|timeout|provider|error' | sed -E 's/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/[redacted-uuid]/g; s/([0-9]{1,3}\.){3}[0-9]{1,3}/[redacted-ip]/g' | tail -n 200 || true
+          echo CADDY_ERRORS
+          ssh "${ssh_options[@]}" "$remote" "container=\$(sudo -n docker ps -q --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=caddy' | head -n 1); test -n \"\$container\"; sudo -n docker logs --since '2026-08-27T10:22:45Z' --until '2026-08-27T10:26:00Z' \"\$container\" 2>&1" | jq -r 'fromjson? | select(.level == "error") | [.ts, .logger, .msg, (.error // "")] | @tsv' | sed -E 's/([0-9]{1,3}\.){3}[0-9]{1,3}/[redacted-ip]/g' | tail -n 100 || true
+          echo CADDY_REQUEST
+          ssh "${ssh_options[@]}" "$remote" "container=\$(sudo -n docker ps -q --filter 'label=com.docker.compose.project=jyotisha-staging' --filter 'label=com.docker.compose.service=caddy' | head -n 1); test -n \"\$container\"; sudo -n docker logs --since '2026-08-27T10:22:45Z' --until '2026-08-27T10:26:00Z' \"\$container\" 2>&1" | jq -r 'fromjson? | select((.request.uri // "") | contains("/api/rectification/agent")) | [.ts, .logger, (.status // ""), (.duration // ""), (.request.method // ""), (.request.uri // ""), (.error // "")] | @tsv' | sed -E 's/([0-9]{1,3}\.){3}[0-9]{1,3}/[redacted-ip]/g' | tail -n 20 || true