research: score Jev intent variants with the previous turn

The earlier report had no previous-turn rows. This run measures V0, V1, V2, and Flash on the re-extracted corpus and records that the real sample is not representative of the simulated set. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-27 13:10:35 +08:00
parent fdb7087b19
commit 0b68fa97c4
10 changed files with 338682 additions and 99563 deletions
@@ -0,0 +1,25 @@
+"""Gitignored cache directory for the Jev intent research.
+
+The 09-19 run stored rows on a Windows path. This machine uses the repo
+`.cache/jev_intent/` directory, which is already gitignored. Set
+`JEV_INTENT_CACHE` to point at an existing cache without writing it into
+the repository.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+WINDOWS_CACHE = Path(r"G:\Ferti\Jyotisha\.cache\jev_intent")
+
+
+def cache_dir() -> Path:
+    override = os.environ.get("JEV_INTENT_CACHE")
+    if override:
+        return Path(override)
+    local = ROOT / ".cache" / "jev_intent"
+    if WINDOWS_CACHE.is_dir() and not local.exists():
+        return WINDOWS_CACHE
+    return local
@@ -26,6 +26,7 @@ from scripts.research.jev_intent_current import (  # noqa: E402
    current_model_id,
    stratified_sample,
 )
+from scripts.research.jev_intent_cache import cache_dir  # noqa: E402
 from scripts.research.jev_intent_questions import (  # noqa: E402
    JEV_MODEL,
    build_state,
@@ -35,7 +36,7 @@ from scripts.research.jev_intent_questions import (  # noqa: E402
 )

 SAMPLES_DIR = ROOT / "scripts" / "research" / "jev_intent_samples"
-CACHE_DIR = Path(r"G:\Ferti\Jyotisha\.cache\jev_intent")
+CACHE_DIR = cache_dir()
 REPORT_JSON = ROOT / "docs" / "research" / "jev_intent_2026_09_19.json"
 REPORT_MD = ROOT / "docs" / "research" / "jev_intent_2026_09_19.md"

@@ -10,6 +10,9 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from importlib.metadata import version as package_version
 from pathlib import Path
 from typing import Any, Mapping, Sequence

@@ -17,13 +20,16 @@ ROOT = Path(__file__).resolve().parents[2]
 if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

+from scripts.research.jev_intent_current import call_current_retry  # noqa: E402
 from scripts.research.jev_intent_probe import (  # noqa: E402
    CACHE_DIR,
    cache_key,
+    call_jev_retry,
    confusion_counts,
    layer_metrics,
    load_cache,
    load_jsonl,
+    save_cache,
    self_consistency,
    strip_confidence,
 )
@@ -169,15 +175,13 @@ def decide(meta: Mapping[str, Any], metrics: Mapping[str, Any]) -> dict[str, str
        blocked.append("Flash+V1 没有预测")
    if not metrics.get("flash_v0"):
        blocked.append("Flash V0 没有预测")
-    represent = metrics.get("representativeness") or {}
-    if represent.get("fail"):
-        blocked.append("来源 B 与来源 C 同层 intent 差 > 10pp")
    if blocked:
        return {
            "verdict": "缺数据",
            "blocked": "；".join(blocked),
            "reason": "上一轮变量没有测全，不能判过门。已有的 V0 数字只作对照，不代替 V1/V2。",
        }
+    represent = metrics.get("representativeness") or {}
    v0_none = ((metrics.get("jev_v0") or {}).get("by_layer") or {}).get("none") or {}
    v2_none = ((metrics.get("jev_v2_chain") or {}).get("by_layer") or {}).get("none") or {}
    v0_high = v0_none.get("high_conf_error_rate")
@@ -209,6 +213,25 @@ def decide(meta: Mapping[str, Any], metrics: Mapping[str, Any]) -> dict[str, str
        gates.append(f"intent {jev_intent:.1%} < 现行 {flash_intent:.1%} − 3pp")
    if v2_high >= v0_high:
        gates.append(f"无焦点层高置信错误 V2 {v2_high:.1%} 没有低于 V0 {v0_high:.1%}")
+    layer_names = {"choice": "点选", "collect": "采集", "none": "无焦点"}
+    gaps = []
+    for layer, cell in (represent.get("by_layer") or {}).items():
+        delta = cell.get("delta")
+        if isinstance(delta, (int, float)) and delta > 0.10:
+            gaps.append(f"{layer_names.get(layer, layer)} {delta * 100:.1f} pp")
+    if represent.get("fail"):
+        reason = (
+            "来源 B 与来源 C 同层 intent 差超过 10pp（"
+            + "，".join(gaps)
+            + "）。按任务书这一项写缺数据，不把这批真人样本外推成过门。"
+        )
+        if gates:
+            reason += "只看这批来源 B，" + "；".join(gates) + "。"
+        return {
+            "verdict": "缺数据",
+            "blocked": "来源 B 与来源 C 同层 intent 差 > 10pp",
+            "reason": reason,
+        }
    if gates:
        return {"verdict": "未过门", "blocked": "", "reason": "；".join(gates)}
    return {
@@ -244,6 +267,37 @@ def representativeness(source_b: Sequence[Mapping[str, Any]], source_c: Sequence
    return {"fail": fail, "by_layer": layers}


+def continue_summary(rows: Sequence[Mapping[str, Any]], pred_key: str) -> dict[str, Any] | None:
+    values = []
+    adopted = 0
+    for row in rows:
+        pred = row.get(pred_key) or {}
+        if pred.get("continued"):
+            adopted += 1
+        value = pred.get("continues_previous_turn")
+        if isinstance(value, (int, float)):
+            values.append(float(value))
+    if not values:
+        return None
+    ordered = sorted(values)
+    return {
+        "n": len(values),
+        "max": ordered[-1],
+        "median": ordered[len(ordered) // 2],
+        "at_or_above_0_9": sum(value >= 0.9 for value in values),
+        "adopted": adopted,
+    }
+
+
+def input_tokens(rows: Sequence[Mapping[str, Any]], pred_key: str) -> int:
+    total = 0
+    for row in rows:
+        value = (row.get(pred_key) or {}).get("input_tokens")
+        if isinstance(value, (int, float)):
+            total += int(value)
+    return total
+
+
 def build_from_samples(source_b: Sequence[dict[str, Any]], source_c: Sequence[dict[str, Any]]) -> dict[str, Any]:
    for row in list(source_b) + list(source_c):
        if "previous_turn" in row:
@@ -259,9 +313,16 @@ def build_from_samples(source_b: Sequence[dict[str, Any]], source_c: Sequence[di
        "jev_v2_gold": pack_metrics(source_b, "jev_v2_gold_1"),
        "flash_v1": strip_block(pack_metrics(source_b, "flash_v1")),
    }
-    if b_metrics["jev_v0"]:
-        b_metrics["jev_v0"]["self_consistency"] = self_consistency(source_b, "jev_v0_1", "jev_v0_2")
+    for name, left, right in (
+        ("jev_v0", "jev_v0_1", "jev_v0_2"),
+        ("jev_v1", "jev_v1_1", "jev_v1_2"),
+        ("jev_v2_chain", "jev_v2_chain_1", "jev_v2_chain_2"),
+        ("jev_v2_gold", "jev_v2_gold_1", "jev_v2_gold_2"),
+    ):
+        if b_metrics.get(name) and any(row.get(right) for row in source_b):
+            b_metrics[name]["self_consistency"] = self_consistency(source_b, left, right)
    none_b = [row for row in source_b if row.get("layer") == "none"]
+    non_echo = [row for row in source_b if row.get("gold_source") != "option_echo"]
    meta = {
        "model": JEV_MODEL,
        "sdk": "typesafe-sdk 0.7.0",
@@ -274,7 +335,15 @@ def build_from_samples(source_b: Sequence[dict[str, Any]], source_c: Sequence[di
        },
        "gold_source": {},
        "source_c_n": len(source_c),
-        "previous_decision_note": "链式与 gold 上界都未跑。09-19 文件没有 turn_id / case_id，本机没有 staging 库。",
+        "baseline": "fdb7087b",
+        "previous_decision_note": (
+            "V2 链式的 previous_decision 来自同案上一条 Jev V2 输出（链式，自喂）。"
+            "V2 gold 上界来自同案上一条人工 gold。案件第一轮 previous_turn 为 null，不喂 previous_decision。"
+        ),
+        "gold_note": (
+            "09-19 的 source_b.jsonl 不在本机，旧 157 条按 turn_id 对回 0 条。"
+            "其余 gold 为执行方按生产标注规范阅读后写入；点选回显另计。"
+        ),
    }
    gold_counts: dict[str, int] = {}
    for row in source_b:
@@ -287,11 +356,37 @@ def build_from_samples(source_b: Sequence[dict[str, Any]], source_c: Sequence[di
        "representativeness": representativeness(source_b, source_c),
        "source_b_none_confusion": {
            "jev_v0": confusion_counts(none_b, "jev_v0_1"),
+            "jev_v1": confusion_counts(none_b, "jev_v1_1"),
+            "jev_v2_chain": confusion_counts(none_b, "jev_v2_chain_1"),
            "flash_v0": confusion_counts(none_b, "flash_v0"),
        } if none_b else {},
+        "non_echo": {
+            "n": len(non_echo),
+            "jev_v0": pack_metrics(non_echo, "jev_v0_1"),
+            "jev_v2_chain": pack_metrics(non_echo, "jev_v2_chain_1"),
+            "flash_v0": strip_block(pack_metrics(non_echo, "flash_v0")),
+        },
+        "continues_chain": continue_summary(source_b, "jev_v2_chain_1"),
+        "continues_gold": continue_summary(source_b, "jev_v2_gold_1"),
    }
-    if metrics["source_c_v0"]:
+    if metrics["source_c_v0"] and any(row.get("jev_v0_2") for row in source_c):
        metrics["source_c_v0"]["self_consistency"] = self_consistency(source_c, "jev_v0_1", "jev_v0_2")
+    jev_tokens = sum(
+        input_tokens(source_b, key)
+        for key in (
+            "jev_v0_1", "jev_v0_2", "jev_v1_1", "jev_v1_2",
+            "jev_v2_chain_1", "jev_v2_chain_2", "jev_v2_gold_1", "jev_v2_gold_2",
+        )
+    )
+    source_c_tokens = input_tokens(source_c, "jev_v0_1")
+    meta["usage"] = {
+        "jev_input_tokens": jev_tokens,
+        "jev_cost_usd": round((jev_tokens / 1_000_000) * 0.042, 4),
+        "source_c_input_tokens": source_c_tokens,
+        "source_c_cost_usd": round((source_c_tokens / 1_000_000) * 0.042, 4),
+        "flash_input_tokens": input_tokens(source_b, "flash_v0") + input_tokens(source_b, "flash_v1"),
+        "response_models": ["jev-1.13.0"],
+    }
    conclusion = decide(meta, metrics)
    return {
        "meta": meta,
@@ -354,7 +449,7 @@ def write_markdown(report: Mapping[str, Any]) -> None:
        "# TypeSafe Jev 意图分类 · 上一轮 state 对照（2026-09-27）",
        "",
        f"- 任务：`docs/tasks/TASK-rectification-jev-intent-classifier-research-v2-20260927.md`",
-        f"- 基线：`origin/staging` @ `710c848b`",
+        f"- 基线：`origin/staging` @ `{meta.get('baseline') or 'fdb7087b'}`（任务书提交 `710c848b` 在其历史上）",
        f"- 模型：`{meta.get('model')}`；SDK `{meta.get('sdk')}`",
        f"- 结论：**{conclusion.get('verdict')}**。{conclusion.get('reason')}",
        "",
@@ -363,67 +458,372 @@ def write_markdown(report: Mapping[str, Any]) -> None:
        f"- 来源 B：{meta.get('source_b_n')} 条。有上一轮 {meta.get('n_with_previous')}，无上一轮 {meta.get('n_without_previous')}。",
        f"- 层：点选 {layers.get('choice', 0)} / 采集 {layers.get('collect', 0)} / 无焦点 {layers.get('none', 0)}。",
        f"- gold 来源：{json.dumps(gold, ensure_ascii=False)}。",
+        f"- {meta.get('gold_note')}",
        f"- {meta.get('previous_decision_note')}",
        "",
        "## 来源 B",
        "",
        "| 变体 | n | intent | answer_class | dated | 高置信错误 | 低置信召回 | 无焦点 unclear→answer |",
        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
-        metric_line("Jev V0（09-19 缓存）", metrics.get("jev_v0")),
-        metric_line("Flash V0（09-19 缓存，生产提示）", metrics.get("flash_v0")),
+        metric_line("Jev V0", metrics.get("jev_v0")),
+        metric_line("Flash V0（生产提示）", metrics.get("flash_v0")),
        metric_line("Jev V1", metrics.get("jev_v1")),
-        metric_line("Jev V2 链式", metrics.get("jev_v2_chain")),
+        metric_line("Jev V2 链式（自喂）", metrics.get("jev_v2_chain")),
        metric_line("Jev V2 gold 上界", metrics.get("jev_v2_gold")),
        metric_line("Flash + V1", metrics.get("flash_v1")),
        "",
-        f"Jev V0 自洽率：{pct((metrics.get('jev_v0') or {}).get('self_consistency'))}。",
+        "自洽率："
+        + "；".join(
+            f"{name} {pct((metrics.get(key) or {}).get('self_consistency'))}"
+            for name, key in (
+                ("V0", "jev_v0"),
+                ("V1", "jev_v1"),
+                ("V2 链式", "jev_v2_chain"),
+                ("V2 gold", "jev_v2_gold"),
+            )
+        )
+        + "。",
        "",
-        "## 来源 C 回归锚（只 V0）",
+        "有上一轮 / 无上一轮（intent；Jev 另给高置信错误）：",
+        "",
+        "| 变体 | 有上一轮 n | intent | 高置信错误 | 无上一轮 n | intent | 高置信错误 |",
+        "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for title, key in (
+        ("Jev V0", "jev_v0"),
+        ("Jev V1", "jev_v1"),
+        ("Jev V2 链式", "jev_v2_chain"),
+        ("Jev V2 gold", "jev_v2_gold"),
+        ("Flash V0", "flash_v0"),
+        ("Flash + V1", "flash_v1"),
+    ):
+        block = metrics.get(key) or {}
+        left = block.get("with_previous") or {}
+        right = block.get("without_previous") or {}
+        if not left and not right:
+            continue
+        lines.append(
+            f"| {title} | {left.get('n', '—')} | {pct(left.get('intent_acc'))} | {pct(left.get('high_conf_error_rate'))} | "
+            f"{right.get('n', '—')} | {pct(right.get('intent_acc'))} | {pct(right.get('high_conf_error_rate'))} |"
+        )
+    lines += [
+        "",
+        "延迟与输入 token（来源 B，第一次）：",
+        "",
+        "| 变体 | 中位 ms | P95 ms | 次均 input tok |",
+        "| --- | ---: | ---: | ---: |",
+    ]
+    for title, key in (
+        ("Jev V0", "jev_v0"),
+        ("Jev V1", "jev_v1"),
+        ("Jev V2 链式", "jev_v2_chain"),
+        ("Jev V2 gold", "jev_v2_gold"),
+        ("Flash V0", "flash_v0"),
+        ("Flash + V1", "flash_v1"),
+    ):
+        cell = ((metrics.get(key) or {}).get("all")) or {}
+        if not cell:
+            continue
+        median = cell.get("median_ms")
+        p95 = cell.get("p95_ms")
+        mean_tokens = cell.get("mean_input_tokens")
+        lines.append(
+            f"| {title} | {int(round(median)) if isinstance(median, (int, float)) else '—'} | "
+            f"{int(round(p95)) if isinstance(p95, (int, float)) else '—'} | "
+            f"{round(mean_tokens, 1) if isinstance(mean_tokens, (int, float)) else '—'} |"
+        )
+    none_v0 = ((metrics.get("jev_v0") or {}).get("by_layer") or {}).get("none") or {}
+    none_v2 = ((metrics.get("jev_v2_chain") or {}).get("by_layer") or {}).get("none") or {}
+    if none_v0 and none_v2:
+        lines += [
+            "",
+            f"无焦点层 intent：V0 {pct(none_v0.get('intent_acc'))}，V2 链式 {pct(none_v2.get('intent_acc'))}。"
+            f"高置信错误从 {pct(none_v0.get('high_conf_error_rate'))} 到 {pct(none_v2.get('high_conf_error_rate'))}。",
+        ]
+    continues = metrics.get("continues_chain") or {}
+    if continues:
+        adopted = continues.get("adopted") or 0
+        lines.append(
+            f"V2 链式承接分：中位 {continues.get('median'):.2f}，最大 {continues.get('max'):.2f}，"
+            f"≥ 0.9 的 {continues.get('at_or_above_0_9')} 条，实际沿用上一轮 intent 的 {adopted} 条。"
+        )
+    non_echo = (metrics.get("non_echo") or {}).get("jev_v2_chain") or {}
+    non_all = non_echo.get("all") or {}
+    if non_all:
+        lines += [
+            "",
+            f"去掉点选回显后，V2 链式 n={non_all.get('n')}，intent {pct(non_all.get('intent_acc'))}，"
+            f"高置信错误 {pct(non_all.get('high_conf_error_rate'))}，低置信召回 {pct(non_all.get('low_conf_recall'))}。"
+            "门槛仍按全量样本。",
+        ]
+    lines += [
+        "",
+        "## 来源 C 回归锚（只 V0，跑一次）",
        "",
    ]
    c_block = metrics.get("source_c_v0") or {}
+    layer_names = {"choice": "点选", "collect": "采集", "none": "无焦点"}
    for layer in ("choice", "collect", "none"):
        cell = (represent.get("by_layer") or {}).get(layer) or {}
        published = cell.get("published_c_intent")
        got = cell.get("source_c_intent")
        delta = cell.get("published_delta_pp")
+        gap = cell.get("delta")
        lines.append(
-            f"- {layer}：重算 {pct(got)}，09-19 公布 {pct(published)}，差 {delta if delta is None else round(delta, 2)} pp（n={cell.get('n_c')}）。"
+            f"- {layer_names[layer]}：来源 C {pct(got)}，09-19 公布 {pct(published)}，差 {delta if delta is None else round(delta, 2)} pp（n={cell.get('n_c')}）。"
+            f"来源 B {pct(cell.get('source_b_intent'))}，与来源 C 差 {None if gap is None else round(gap * 100, 1)} pp（n={cell.get('n_b')}）。"
        )
    lines += [
        f"- 来源 C 自洽率：{pct(c_block.get('self_consistency'))}。",
        "",
-        "## 无焦点层混淆（来源 B，V0）",
+        "## 无焦点层混淆（来源 B）",
        "",
        "计数来自报告 JSON，不含原文。",
        "",
    ]
-    confusion = (metrics.get("source_b_none_confusion") or {}).get("jev_v0") or {}
-    labels = confusion.get("labels") or []
-    counts = confusion.get("counts") or {}
-    if labels:
+    for title, key in (("V0", "jev_v0"), ("V1", "jev_v1"), ("V2 链式", "jev_v2_chain")):
+        confusion = (metrics.get("source_b_none_confusion") or {}).get(key) or {}
+        labels = confusion.get("labels") or []
+        counts = confusion.get("counts") or {}
+        if not labels or not any(sum((counts.get(gold) or {}).values()) for gold in labels):
+            continue
+        lines.append(f"### {title}")
+        lines.append("")
        lines.append("| gold \\ pred | " + " | ".join(labels) + " |")
        lines.append("| --- | " + " | ".join("---:" for _ in labels) + " |")
-        for gold in labels:
-            cells = [str((counts.get(gold) or {}).get(pred, 0)) for pred in labels]
-            lines.append(f"| {gold} | " + " | ".join(cells) + " |")
+        for gold_label in labels:
+            cells = [str((counts.get(gold_label) or {}).get(pred, 0)) for pred in labels]
+            lines.append(f"| {gold_label} | " + " | ".join(cells) + " |")
+        lines.append("")
+    usage = meta.get("usage") or {}
    lines += [
-        "",
        "## 写不出的项",
        "",
        conclusion.get("blocked") or "无",
        "",
-        "V1、V2、Flash+V1 要等 staging 库抽出带 `case_id` 的上一轮，并且本机有 `DEEPSEEK_API_KEY` 之后才能补。门槛不放宽。",
-        "",
    ]
+    if usage:
+        lines.append(
+            f"来源 B 的 Jev 八次合计 {usage.get('jev_input_tokens')} input tok，按 $0.042/M 约 ${usage.get('jev_cost_usd')}。"
+            f"来源 C 一次 {usage.get('source_c_input_tokens')} input tok，约 ${usage.get('source_c_cost_usd')}。"
+            f"Flash 两次合计 {usage.get('flash_input_tokens')} input tok，不按 Jev 单价计。"
+            f"响应 model：{usage.get('response_models')}。"
+        )
+        lines.append("")
    REPORT_MD.write_text("\n".join(lines) + "\n", encoding="utf-8")


+def sdk_label() -> str:
+    try:
+        return f"typesafe-sdk {package_version('typesafe-sdk')}"
+    except Exception:
+        return "typesafe-sdk 0.7.0"
+
+
+def live_cache_key(run_id: str, sample: Mapping[str, Any]) -> str:
+    decision = sample.get("previous_decision") if isinstance(sample.get("previous_decision"), dict) else {}
+    extra = f"{decision.get('intent')}:{decision.get('answer_class')}"
+    return f"{cache_key(run_id, sample)}:{extra}"
+
+
+def prepare_call(row: Mapping[str, Any], decision: Mapping[str, Any] | None) -> dict[str, Any]:
+    sample = dict(row)
+    sample.pop("previous_decision", None)
+    if decision:
+        sample["previous_decision"] = {
+            "intent": decision.get("intent"),
+            "answer_class": decision.get("answer_class"),
+        }
+    return sample
+
+
+def gold_decisions(rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any] | None]:
+    decisions: list[dict[str, Any] | None] = []
+    previous: dict[str, Any] | None = None
+    last_case = None
+    for row in rows:
+        case_id = row.get("case_id")
+        if case_id != last_case:
+            previous = None
+            last_case = case_id
+        if previous_turn_payload(row) and previous:
+            decisions.append(previous)
+        else:
+            decisions.append(None)
+        gold = row.get("gold") or {}
+        if gold.get("intent"):
+            previous = {"intent": gold.get("intent"), "answer_class": gold.get("answer_class")}
+        else:
+            previous = None
+    return decisions
+
+
+def run_independent(
+    rows: Sequence[dict[str, Any]],
+    *,
+    run_id: str,
+    cache: dict[str, Any],
+    workers: int,
+    caller,
+    decisions: Sequence[Mapping[str, Any] | None] | None = None,
+) -> None:
+    lock = threading.Lock()
+    pending: list[tuple[dict[str, Any], dict[str, Any], str]] = []
+    for index, row in enumerate(rows):
+        decision = None if decisions is None else decisions[index]
+        sample = prepare_call(row, decision)
+        key = live_cache_key(run_id, sample)
+        hit = cache.get(key)
+        if isinstance(hit, dict) and hit.get("ok"):
+            row[run_id] = hit
+        else:
+            pending.append((row, sample, key))
+    print(f"{run_id}: {len(pending)} calls, {len(rows) - len(pending)} cached", flush=True)
+    if not pending:
+        return
+    done = {"n": 0}
+
+    def work(item: tuple[dict[str, Any], dict[str, Any], str]) -> None:
+        row, sample, key = item
+        result = caller(sample)
+        with lock:
+            if result.get("ok"):
+                cache[key] = result
+            row[run_id] = result
+            done["n"] += 1
+            if done["n"] % 25 == 0 or done["n"] == len(pending):
+                save_cache(cache)
+                print(f"  {run_id} {done['n']}/{len(pending)}", flush=True)
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        list(pool.map(work, pending))
+    save_cache(cache)
+
+
+def run_chain(
+    rows: Sequence[dict[str, Any]],
+    *,
+    run_id: str,
+    cache: dict[str, Any],
+    workers: int,
+) -> None:
+    groups: list[list[dict[str, Any]]] = []
+    bucket: list[dict[str, Any]] = []
+    last_case = None
+    for row in rows:
+        case_id = row.get("case_id")
+        if bucket and case_id != last_case:
+            groups.append(bucket)
+            bucket = []
+        bucket.append(row)
+        last_case = case_id
+    if bucket:
+        groups.append(bucket)
+    lock = threading.Lock()
+    progress = {"n": 0}
+
+    def one_case(group: list[dict[str, Any]]) -> None:
+        previous: dict[str, Any] | None = None
+        for row in group:
+            decision = previous if previous_turn_payload(row) else None
+            sample = prepare_call(row, decision)
+            key = live_cache_key(run_id, sample)
+            with lock:
+                hit = cache.get(key)
+            if isinstance(hit, dict) and hit.get("ok"):
+                result = hit
+            else:
+                result = call_jev_retry(sample, variant="v2")
+                if result.get("ok"):
+                    with lock:
+                        cache[key] = result
+                        progress["n"] += 1
+                        if progress["n"] % 20 == 0:
+                            save_cache(cache)
+                            print(f"  {run_id} {progress['n']}", flush=True)
+            row[run_id] = result
+            if result.get("ok") and result.get("intent"):
+                previous = {"intent": result.get("intent"), "answer_class": result.get("answer_class")}
+            else:
+                previous = None
+
+    print(f"{run_id}: {len(groups)} cases", flush=True)
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        list(pool.map(one_case, groups))
+    save_cache(cache)
+
+
+def run_live(*, workers: int) -> dict[str, Any]:
+    source_b = [row for row in load_jsonl(OUTPUT_PATH) if isinstance(row.get("gold"), dict) and row["gold"].get("intent")]
+    source_c = load_jsonl(SAMPLES_DIR / "simulated.jsonl")
+    source_b.sort(key=lambda row: (str(row.get("case_id")), str(row.get("created_at")), str(row.get("id"))))
+    cache = load_cache()
+    decisions = gold_decisions(source_b)
+
+    def jev_v0(sample: Mapping[str, Any]) -> dict[str, Any]:
+        return call_jev_retry(sample, variant="v0")
+
+    def jev_v1(sample: Mapping[str, Any]) -> dict[str, Any]:
+        return call_jev_retry(sample, variant="v1")
+
+    def jev_v2(sample: Mapping[str, Any]) -> dict[str, Any]:
+        return call_jev_retry(sample, variant="v2")
+
+    def flash_v0(sample: Mapping[str, Any]) -> dict[str, Any]:
+        return call_current_retry(sample, variant="v0")
+
+    def flash_v1(sample: Mapping[str, Any]) -> dict[str, Any]:
+        return call_current_retry(sample, variant="v1")
+
+    run_independent(source_b, run_id="jev_v0_1", cache=cache, workers=workers, caller=jev_v0)
+    run_independent(source_b, run_id="jev_v0_2", cache=cache, workers=workers, caller=jev_v0)
+    run_independent(source_b, run_id="jev_v1_1", cache=cache, workers=workers, caller=jev_v1)
+    run_independent(source_b, run_id="jev_v1_2", cache=cache, workers=workers, caller=jev_v1)
+    run_independent(source_b, run_id="jev_v2_gold_1", cache=cache, workers=workers, caller=jev_v2, decisions=decisions)
+    run_independent(source_b, run_id="jev_v2_gold_2", cache=cache, workers=workers, caller=jev_v2, decisions=decisions)
+    run_chain(source_b, run_id="jev_v2_chain_1", cache=cache, workers=workers)
+    run_chain(source_b, run_id="jev_v2_chain_2", cache=cache, workers=workers)
+    run_independent(source_b, run_id="flash_v0", cache=cache, workers=workers, caller=flash_v0)
+    run_independent(source_b, run_id="flash_v1", cache=cache, workers=workers, caller=flash_v1)
+    run_independent(source_c, run_id="jev_v0_1", cache=cache, workers=workers, caller=jev_v0)
+    report = build_from_samples(source_b, source_c)
+    report["meta"]["sdk"] = sdk_label()
+    report["meta"]["baseline"] = "fdb7087b"
+    report["meta"]["offline"] = False
+    models = {
+        str((row.get(key) or {}).get("model"))
+        for row in list(source_b) + list(source_c)
+        for key in (
+            "jev_v0_1", "jev_v1_1", "jev_v2_chain_1", "jev_v2_gold_1",
+        )
+        if (row.get(key) or {}).get("ok") and (row.get(key) or {}).get("model")
+    }
+    if models:
+        report["meta"]["usage"]["response_models"] = sorted(models)
+        report["meta"]["model"] = sorted(models)[0] if models == {JEV_MODEL} else ", ".join(sorted(models))
+    return report
+
+
 def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--offline", action="store_true")
+    parser.add_argument("--live", action="store_true")
+    parser.add_argument("--workers", type=int, default=6)
    parser.add_argument("--from-report", action="store_true", help="recompute tables from the committed JSON rows")
    args = parser.parse_args(argv)
+    if args.live:
+        report = run_live(workers=max(1, args.workers))
+        REPORT_JSON.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+        write_markdown(report)
+        print(json.dumps({
+            "verdict": report["conclusion"]["verdict"],
+            "reason": report["conclusion"].get("reason"),
+            "blocked": report["conclusion"].get("blocked"),
+            "source_b_n": report["meta"]["source_b_n"],
+            "n_with_previous": report["meta"]["n_with_previous"],
+            "usage": report["meta"].get("usage"),
+        }, ensure_ascii=False))
+        return 0
    if args.from_report:
        if not REPORT_JSON.is_file():
            print("report json missing", file=sys.stderr)
@@ -11,6 +11,7 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Mapping, Sequence

@@ -18,9 +19,13 @@ ROOT = Path(__file__).resolve().parents[2]
 if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

-CACHE_DIR = Path(r"G:\Ferti\Jyotisha\.cache\jev_intent")
+from scripts.research.jev_intent_cache import cache_dir  # noqa: E402
+
+CACHE_DIR = cache_dir()
 LEGACY_PATH = CACHE_DIR / "source_b.jsonl"
 OUTPUT_PATH = CACHE_DIR / "source_b_v2.jsonl"
+ANSWER_CLASSES = {"yes", "weak_yes", "no", "unsure"}
+CLOSED_FOCUS = {"resolved", "declined", "skipped", "superseded"}

 EXTRACT_SQL = """
 select
@@ -32,8 +37,12 @@ select
  t.created_at,
  c.status as case_status,
  f.question_id,
+  f.intent as focus_intent,
  f.expected_answer_schema,
-  f.status as focus_status
+  f.status as focus_status,
+  f.asked_at,
+  f.resolved_at,
+  t.message_origin
 from public.agentic_rectification_turns t
 join public.agentic_rectification_cases c on c.id = t.case_id
 left join lateral (
@@ -50,6 +59,124 @@ order by t.case_id, t.created_at
 """


+def _timestamp(value: Any) -> datetime | None:
+    if value is None or value == "":
+        return None
+    text = str(value).strip().replace("Z", "+00:00")
+    try:
+        return datetime.fromisoformat(text)
+    except ValueError:
+        return None
+
+
+def _choice_copy(schema: Mapping[str, Any]) -> tuple[str, list[dict[str, str]]] | None:
+    choice = schema.get("choice") if isinstance(schema.get("choice"), dict) else schema
+    if not isinstance(choice, dict):
+        return None
+    prompt = choice.get("prompt")
+    options = choice.get("options")
+    if not isinstance(prompt, str) or not prompt.strip():
+        return None
+    if not isinstance(options, list) or len(options) != 4:
+        return None
+    cleaned: list[dict[str, str]] = []
+    for item in options:
+        if not isinstance(item, dict):
+            return None
+        key = item.get("key")
+        label = item.get("label")
+        answer = item.get("answer_class")
+        if key not in {"A", "B", "C", "D"} or not isinstance(label, str) or not label.strip():
+            return None
+        if answer not in ANSWER_CLASSES:
+            return None
+        cleaned.append({"key": str(key), "label": label.strip(), "answer_class": str(answer)})
+    if len({item["key"] for item in cleaned}) != 4:
+        return None
+    if len({item["label"] for item in cleaned}) != 4:
+        return None
+    return prompt.strip(), cleaned
+
+
+def focus_is_open(row: Mapping[str, Any]) -> bool:
+    """Production classifies the focus that is still open when the user speaks.
+
+    A focus asked earlier and already resolved, declined, skipped, or superseded
+    is not the current question. The row stays in the extract; its layer is none.
+    """
+    schema = row.get("schema")
+    if schema is None:
+        schema = row.get("expected_answer_schema")
+    if not isinstance(schema, dict) or not schema:
+        return False
+    created = _timestamp(row.get("created_at"))
+    asked = _timestamp(row.get("asked_at"))
+    resolved = _timestamp(row.get("resolved_at"))
+    if asked and created and asked > created:
+        return False
+    if resolved and created and resolved < created:
+        return False
+    status = str(row.get("focus_status") or "")
+    if status in CLOSED_FOCUS and resolved and created and resolved < created:
+        return False
+    return True
+
+
+def focus_payload(row: Mapping[str, Any]) -> tuple[dict[str, Any] | None, str, bool]:
+    schema = row.get("schema")
+    if schema is None:
+        schema = row.get("expected_answer_schema")
+    stale = isinstance(schema, dict) and bool(schema) and not focus_is_open(row)
+    if not focus_is_open(row) or not isinstance(schema, dict):
+        return None, "none", stale
+    case_status = str(row.get("case_status") or "collecting_evidence")
+    choice = _choice_copy(schema)
+    if choice:
+        prompt, options = choice
+        return {
+            "current_question": prompt,
+            "options": options,
+            "case_status": case_status,
+            "question_id": row.get("question_id"),
+        }, "choice", False
+    prompt = schema.get("prompt") if schema.get("collect") is True else None
+    if isinstance(prompt, str) and prompt.strip():
+        return {
+            "current_question": prompt.strip(),
+            "options": [],
+            "case_status": case_status,
+            "question_id": row.get("question_id"),
+        }, "collect", False
+    return None, "none", False
+
+
+def normalize_extract_row(raw: Mapping[str, Any]) -> dict[str, Any]:
+    focus, layer, stale = focus_payload(raw)
+    turn_id = str(raw.get("turn_id") or raw.get("id") or "")
+    return {
+        "id": turn_id,
+        "turn_id": turn_id,
+        "source": "B",
+        "layer": layer,
+        "case_id": str(raw.get("case_id") or "") or None,
+        "user_message": raw.get("user_message"),
+        "assistant_message": raw.get("assistant_message"),
+        "created_at": raw.get("created_at"),
+        "case_status": raw.get("case_status"),
+        "turn_status": raw.get("turn_status"),
+        "message_origin": raw.get("message_origin"),
+        "question_id": raw.get("question_id"),
+        "focus_intent": raw.get("focus_intent"),
+        "focus_status": raw.get("focus_status"),
+        "focus": focus,
+        "focus_stale": stale,
+    }
+
+
+def prepare_extract(raw_rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
+    return attach_previous_turns([normalize_extract_row(row) for row in raw_rows])
+
+
 def load_jsonl(path: Path) -> list[dict[str, Any]]:
    if not path.is_file():
        return []
@@ -185,13 +312,22 @@ def main(argv: Sequence[str] | None = None) -> int:
        return 0
    if args.raw:
        raw_rows = load_jsonl(args.raw)
-        rows = attach_previous_turns(raw_rows)
+        if raw_rows and any(key in raw_rows[0] for key in ("schema", "expected_answer_schema")):
+            rows = prepare_extract(raw_rows)
+        else:
+            rows = attach_previous_turns([dict(row) for row in raw_rows])
        counts = match_gold(rows, legacy)
        write_jsonl(args.out, rows)
+        layers: dict[str, int] = {}
+        for row in rows:
+            layer = str(row.get("layer") or "none")
+            layers[layer] = layers.get(layer, 0) + 1
        print(json.dumps({
            "out": str(args.out),
            "n": len(rows),
            "n_with_previous": sum(1 for row in rows if row.get("previous_turn")),
+            "n_stale_focus": sum(1 for row in rows if row.get("focus_stale")),
+            "layers": layers,
            "gold": counts,
        }, ensure_ascii=False))
        return 0