research(jev-intent): fix2 把来源 B 现行与高置信错误补进报告

离线从 cache 聚合，不重跑模型。无焦点层 Flash 69.7% 低于 Jev 78.8%。采集层相对门槛标不可判。

scripts/research/jev_intent_probe.py

@@ -327,6 +327,58 @@ def cache_key(run_id: str, sample: Mapping[str, Any]) -> str:
     return f"{run_id}:{sample['id']}:{digest}"
 
 
+NONE_CONFUSION_LABELS = (
+    "provide_new_evidence",
+    "stop_rectification",
+    "ask_about_result",
+    "unclear",
+    "answer_current_focus",
+)
+
+
+def attach_from_cache(
+    samples: Sequence[dict[str, Any]],
+    cache: Mapping[str, Any],
+    run_ids: Sequence[str],
+) -> dict[str, int]:
+    missing = {rid: 0 for rid in run_ids}
+    for sample in samples:
+        for rid in run_ids:
+            key = cache_key(rid, sample)
+            if key in cache:
+                sample[rid] = cache[key]
+            else:
+                missing[rid] += 1
+    return missing
+
+
+def strip_confidence(metrics: Mapping[str, Any] | None) -> dict[str, Any] | None:
+    if metrics is None:
+        return None
+    out = dict(metrics)
+    out["high_conf_error_rate"] = None
+    out["low_conf_coverage"] = None
+    out["low_conf_recall"] = None
+    return out
+
+
+def confusion_counts(
+    rows: Sequence[Mapping[str, Any]],
+    pred_key: str,
+    labels: Sequence[str] = NONE_CONFUSION_LABELS,
+) -> dict[str, Any]:
+    counts = {gold: {pred: 0 for pred in labels} for gold in labels}
+    other = 0
+    for row in rows:
+        gold = (row.get("gold") or {}).get("intent")
+        pred = (row.get(pred_key) or {}).get("intent")
+        if gold in counts and pred in counts[gold]:
+            counts[gold][pred] += 1
+        else:
+            other += 1
+    return {"labels": list(labels), "counts": counts, "other": other, "n": len(rows)}
+
+
 def run_batch(
     samples: Sequence[Mapping[str, Any]],
     *,
@@ -471,8 +523,8 @@ def write_markdown(report: Mapping[str, Any]) -> None:
             lines.append("")
             lines.append("同一样本上 Jev vs 现行（相对门槛用这一表）：")
             lines.append("")
-            lines.append("| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp |")
-            lines.append("| --- | ---: | ---: | ---: | ---: | ---: |")
+            lines.append("| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp | 判定 |")
+            lines.append("| --- | ---: | ---: | ---: | ---: | ---: | --- |")
             for layer in ("choice", "collect", "none"):
                 cell = paired.get(layer) or {}
                 def pct(value: float | None) -> str:
@@ -481,8 +533,16 @@ def write_markdown(report: Mapping[str, Any]) -> None:
                 cur = cell.get("current_intent")
                 delta = None if jev is None or cur is None else jev - cur
                 gate = None if cur is None else cur - 0.03
+                if layer == "collect":
+                    verdict = "不可判（复核与对照同源）"
+                elif jev is None or cur is None:
+                    verdict = "—"
+                elif round(jev * 100, 1) >= round((cur - 0.03) * 100, 1):
+                    verdict = "过"
+                else:
+                    verdict = "未过"
                 lines.append(
-                    f"| {layer} | {cell.get('n', 0)} | {pct(jev)} | {pct(cur)} | {pct(delta)} | {pct(gate)} |"
+                    f"| {layer} | {cell.get('n', 0)} | {pct(jev)} | {pct(cur)} | {pct(delta)} | {pct(gate)} | {verdict} |"
                 )
     else:
         lines.append(report["meta"].get("current_model_note") or "未跑现行模型。")
@@ -492,6 +552,65 @@ def write_markdown(report: Mapping[str, Any]) -> None:
         "",
         report["metrics"]["representativeness"]["note"],
         "",
+    ]
+    b_jev = report["metrics"].get("source_b")
+    b_jev_layer = report["metrics"].get("source_b_by_layer") or {}
+    b_cur = report["metrics"].get("current_source_b")
+    b_cur_layer = report["metrics"].get("current_source_b_by_layer") or {}
+    b_cons = report["metrics"].get("source_b_jev_self_consistency") or {}
+    if b_jev:
+        def pct(value: float | None) -> str:
+            return "—" if value is None else f"{value:.1%}"
+        lines += [
+            "来源 B 是真人 + 人工标注 + 线上模型三者齐备的唯一一组。现行无置信度，置信度三列为空。",
+            "",
+            "| 范围 | n | Jev intent | 现行 intent | Jev 高置信错误 | Jev 低置信召回 | Jev 自洽 |",
+            "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+        ]
+        rows_spec = [("全集", b_jev, b_cur, b_cons.get("all"))]
+        for layer in ("choice", "collect", "none"):
+            rows_spec.append((
+                layer,
+                b_jev_layer.get(layer) or {},
+                b_cur_layer.get(layer) or {},
+                b_cons.get(layer),
+            ))
+        for name, jev_m, cur_m, cons in rows_spec:
+            lines.append(
+                f"| {name} | {jev_m.get('n', 0)} | {pct(jev_m.get('intent_acc'))} | {pct((cur_m or {}).get('intent_acc'))} | "
+                f"{pct(jev_m.get('high_conf_error_rate'))} | {pct(jev_m.get('low_conf_recall'))} | {pct(cons)} |"
+            )
+        lines.append("")
+        none_cur = (b_cur_layer.get("none") or {}).get("intent_acc")
+        none_jev = (b_jev_layer.get("none") or {}).get("intent_acc")
+        if none_cur is not None and none_jev is not None:
+            lines.append(
+                f"无焦点层现行 intent {none_cur:.1%}、Jev {none_jev:.1%}。"
+                "现行更低，说明 78.8% 主要是这 33 条本身难，不是单 Jev 不行。"
+            )
+            lines.append("")
+    confusion = report["metrics"].get("source_b_none_confusion") or {}
+    if confusion:
+        lines += [
+            "无焦点层 gold × 预测混淆计数（只有计数，无原文）。预测出现 `answer_current_focus` 是因为模型把无焦点句当成在回答采集题。",
+            "",
+        ]
+        for title, key in (("gold × Jev", "jev"), ("gold × 现行", "current")):
+            table = confusion.get(key) or {}
+            labels = table.get("labels") or list(NONE_CONFUSION_LABELS)
+            counts = table.get("counts") or {}
+            lines.append(f"### {title}（n={table.get('n', 0)}）")
+            lines.append("")
+            header = "| gold \\ pred | " + " | ".join(labels) + " |"
+            sep = "| --- | " + " | ".join("---:" for _ in labels) + " |"
+            lines.append(header)
+            lines.append(sep)
+            for gold in labels:
+                row_counts = counts.get(gold) or {}
+                cells = " | ".join(str(row_counts.get(pred, 0)) for pred in labels)
+                lines.append(f"| {gold} | {cells} |")
+            lines.append("")
+    lines += [
         "## 置信度–准确率曲线与 θ",
         "",
         f"推荐 θ = {report['metrics']['theta']['recommended_theta']}。点：",
@@ -530,6 +649,25 @@ def write_markdown(report: Mapping[str, Any]) -> None:
         lost = row["lost"].replace("|", "\\|") if row["lost"] else "—"
         lines.append(f"| {row['production']} | {row['jev']} | {lost} |")
     lines += [
+        "",
+        "## 限制",
+        "",
+        "1. **复核 ≈ 生产提示，且复核模型 = 对照模型。** `REVIEW_RUBRIC` 与生产 `COLLECT_INSTRUCTIONS` 逐句对应；生成 / 复核 / 对照都是 `deepseek-flash`。进入测试集的 900 条是「Flash 用近生产提示能答对目标标签」的那 900 条，被剔的 36 条恰是 Flash 不同意的。因此来源 C 上现行 97.5% / 98.8% / 94.0% 是构造出来的上界。采集层「Jev 92.2% 未过相对门槛 94.5%」**不可当作 Jev 输给现行的证据**。",
+        "2. **采集层 intent 错例的 gold 有争议。** 来源 C 采集层 Jev intent 错例 31 条：`provide_new_evidence → answer_current_focus` 17、`unclear → answer_current_focus` 9、`stop → unclear` 4、`ask → unclear` 1。17 条 pne 几乎全是「另外 2019 年我换工作搬了家」句式，若干尾句落在当前题域，按生产提示可读成 `answer_current_focus + unsure`。9 条 unclear（「一时半会儿真捋不明白」）按「记不清 → unsure」也读得通。两类合计 ≥ 20 条，占该层 intent 错例约 2/3。gold 来自「生成目标 + Flash 复核同意」，不等于人工真值。改写示例：",
+        "   - 「另外 2019 年换过工作，感情那会儿真没细想。」gold=provide_new_evidence；可读成在回答感情采集题。",
+        "   - 「另外 2019 年搬了家，工作那摊子反而没顾上细想。」gold=provide_new_evidence；可读成在回答工作采集题。",
+        "   - 「这事我一时真说不上来。」gold=unclear；按生产提示是 unsure。",
+        "3. **语料仍不像真人。** 修复单 1 只把长度和人设写成硬红线（已过）。原单还要求按来源 B 的标点 / 语气词比例约束，未进红线：",
+        "",
+        "| 指标 | 来源 B（真人） | 来源 C（模拟） |",
+        "| --- | ---: | ---: |",
+        "| 含标点 | 13% | 98.9% |",
+        "| 含语气词（吧/呢/啊/嗯/哦/额/emm） | 4% | 25.6% |",
+        "| 含年份或月份 | 81% | 47.8% |",
+        "| 带年份句里写「2019」 | — | 236 / 429 = 55% |",
+        "| `provide_new_evidence` 里是搬家/换工作 | — | 152 / 157 |",
+        "",
+        "根因：生成脚本 `ALT_EVENT_HINTS` 给七个领域的「另一件事」全是搬家/换工作，年份未约束，模型收敛到「另外 2019 年搬过家」。这解释了模拟语料不代表真人的一部分，也解释了采集层错例为何长得一样。本单不修，留给产品决定是否再造一轮。",
         "",
         "## 回退",
         "",
@@ -570,8 +708,9 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
             if jev is None or cur is None:
                 continue
             ok = round(jev * 100, 1) >= round((cur - 0.03) * 100, 1)
+            mark = "不可判（复核与对照同源）" if layer == "collect" else ("过" if ok else "未过")
             bits.append(
-                f"{layer} Jev {jev:.1%} vs 现行 {cur:.1%}（门槛 {cur-0.03:.1%}，{'过' if ok else '未过'}）"
+                f"{layer} Jev {jev:.1%} vs 现行 {cur:.1%}（门槛 {cur-0.03:.1%}，{mark}）"
             )
         if bits:
             relative_note = " 相对 −3pp（同一样本）：" + "；".join(bits) + "。"
@@ -693,11 +832,12 @@ def main(argv: Sequence[str] | None = None) -> int:
     parser.add_argument("--sample-fraction", type=float, default=1.0)
     parser.add_argument("--current-second-fraction", type=float, default=1.0 / 3)
     parser.add_argument("--sample-seed", type=int, default=20260919)
+    parser.add_argument("--offline", action="store_true", help="aggregate from cache only, no API calls")
     args = parser.parse_args(argv)
-    if not args.current_only and not os.environ.get("TYPESAFE_API_KEY"):
+    if not args.offline and not args.current_only and not os.environ.get("TYPESAFE_API_KEY"):
         print("TYPESAFE_API_KEY missing", file=sys.stderr)
         return 2
-    if args.current_only and not os.environ.get("DEEPSEEK_API_KEY"):
+    if not args.offline and args.current_only and not os.environ.get("DEEPSEEK_API_KEY"):
         print("DEEPSEEK_API_KEY missing", file=sys.stderr)
         return 2
     synthetic = load_jsonl(SAMPLES_DIR / "synthetic.jsonl")
@@ -715,8 +855,13 @@ def main(argv: Sequence[str] | None = None) -> int:
     _merge_report_preds(samples)
     cache = load_cache()
     current_sample: list[dict[str, Any]] = []
+    second_current: list[dict[str, Any]] = []
+    if args.offline:
+        miss_c = attach_from_cache(samples, cache, ("jev_1", "jev_2", "current_1", "current_2"))
+        miss_b = attach_from_cache(source_b, cache, ("jev_1", "jev_2", "current_1"))
+        print(json.dumps({"offline_missing": {"samples": miss_c, "source_b": miss_b}}, ensure_ascii=False), flush=True)
     try:
-        if not args.current_only:
+        if not args.offline and not args.current_only:
             run_batch(samples, workers=args.workers, run_id="jev_1", cache=cache)
             save_cache(cache)
             second = samples
@@ -734,7 +879,7 @@ def main(argv: Sequence[str] | None = None) -> int:
                 run_batch(source_b, workers=args.workers, run_id="jev_2", cache=cache)
                 save_cache(cache)
         source_c_rows = [row for row in samples if row.get("source") == "C"]
-        if args.current_only or os.environ.get("DEEPSEEK_API_KEY"):
+        if not args.offline and (args.current_only or os.environ.get("DEEPSEEK_API_KEY")):
             if args.sample_fraction < 1:
                 current_sample = stratified_sample(
                     source_c_rows, fraction=args.sample_fraction, seed=args.sample_seed,
@@ -802,6 +947,38 @@ def main(argv: Sequence[str] | None = None) -> int:
         layer: layer_metrics([row for row in source_b if row.get("layer") == layer], pred_key="jev_1")
         for layer in ("choice", "collect", "none")
     } if source_b else {}
+    current_source_b = strip_confidence(
+        layer_metrics(source_b, pred_key="current_1") if source_b and any(row.get("current_1") for row in source_b) else None
+    )
+    current_source_b_by_layer = {
+        layer: strip_confidence(layer_metrics(
+            [row for row in source_b if row.get("layer") == layer and row.get("current_1")],
+            pred_key="current_1",
+        ))
+        for layer in ("choice", "collect", "none")
+    } if source_b and any(row.get("current_1") for row in source_b) else {}
+    source_b_jev_self_consistency = (
+        {
+            "all": self_consistency(source_b, "jev_1", "jev_2"),
+            **{
+                layer: self_consistency(
+                    [row for row in source_b if row.get("layer") == layer],
+                    "jev_1",
+                    "jev_2",
+                )
+                for layer in ("choice", "collect", "none")
+            },
+        }
+        if source_b else {}
+    )
+    none_b = [row for row in source_b if row.get("layer") == "none"]
+    source_b_none_confusion = (
+        {
+            "jev": confusion_counts(none_b, "jev_1"),
+            "current": confusion_counts(none_b, "current_1"),
+        }
+        if none_b else {}
+    )
     represent_fail = False
     represent_layers: dict[str, Any] = {}
     if not source_b or len(source_b) < 30:
@@ -904,6 +1081,10 @@ def main(argv: Sequence[str] | None = None) -> int:
             "source_a": source_a_metrics,
             "source_b": source_b_metrics,
             "source_b_by_layer": source_b_by_layer,
+            "current_source_b": current_source_b,
+            "current_source_b_by_layer": current_source_b_by_layer,
+            "source_b_jev_self_consistency": source_b_jev_self_consistency,
+            "source_b_none_confusion": source_b_none_confusion,
             "representativeness": {
                 "note": represent_note,
                 "fail": represent_fail,
@@ -933,6 +1114,11 @@ def main(argv: Sequence[str] | None = None) -> int:
         } for k, v in jev_metrics.items()},
         "self_consistency": jev_cons,
         "source_b_n": len(source_b),
+        "current_source_b": {
+            "n": (current_source_b or {}).get("n"),
+            "intent": (current_source_b or {}).get("intent_acc"),
+        },
+        "source_b_none_confusion_n": (source_b_none_confusion.get("jev") or {}).get("n"),
     }, ensure_ascii=False, indent=2))
     return 0