research(rectification): DeepSeek Flash 抽 33% 对照现行分类器提示词

来源 B 仍为 0。Flash 套生产提示词，采集题相对门槛未过。结论仍不可接。

scripts/research/jev_intent_probe.py

@@ -20,6 +20,11 @@ ROOT = Path(__file__).resolve().parents[2]
 if str(ROOT) not in sys.path:
     sys.path.insert(0, str(ROOT))
 
+from scripts.research.jev_intent_current import (  # noqa: E402
+    call_current_retry,
+    current_model_id,
+    stratified_sample,
+)
 from scripts.research.jev_intent_questions import (  # noqa: E402
     JEV_MODEL,
     build_state,
@@ -321,7 +326,9 @@ def run_batch(
     workers: int,
     run_id: str,
     cache: dict[str, Any],
+    call_fn=None,
 ) -> None:
+    caller = call_fn or call_jev_retry
     pending = []
     for sample in samples:
         key = f"{run_id}:{sample['id']}"
@@ -333,7 +340,7 @@ def run_batch(
         return
     print(f"{run_id}: {len(pending)} calls, {len(samples) - len(pending)} cached", flush=True)
     with ThreadPoolExecutor(max_workers=workers) as pool:
-        futures = {pool.submit(call_jev_retry, sample): sample for sample in pending}
+        futures = {pool.submit(caller, sample): sample for sample in pending}
         done = 0
         for future in as_completed(futures):
             sample = futures[future]
@@ -416,19 +423,22 @@ def write_markdown(report: Mapping[str, Any]) -> None:
             f"{m['no_vs_unsure']} | {pct(cons)} | {ms(m['median_ms'])} | {ms(m['p95_ms'])} | "
             f"{(m['mean_input_tokens'] or 0):.0f} |"
         )
+    source_a_m = report["metrics"].get("source_a")
+    if source_a_m:
+        acc = source_a_m.get("answer_class_acc")
+        acc_txt = "—" if acc is None else f"{acc:.1%}"
+        lines += [
+            "",
+            f"来源 A（测试夹具，n={source_a_m['n']}）intent {source_a_m['intent_acc']:.1%}，answer_class {acc_txt}，高置信错误 {source_a_m['high_conf_error_rate']:.1%}。",
+        ]
     lines += [
         "",
         "## 现行模型对照",
         "",
     ]
-    source_a_m = report["metrics"].get("source_a")
-    if source_a_m:
-        lines += [
-            "",
-            f"来源 A（测试夹具，n={source_a_m['n']}）intent {source_a_m['intent_acc']:.1%}，answer_class {source_a_m['answer_class_acc'] or 0:.1%}，高置信错误 {source_a_m['high_conf_error_rate']:.1%}。",
-        ]
     if report["meta"].get("current_model"):
-        lines.append(f"模型：`{report['meta']['current_model']}`。")
+        note = report["meta"].get("current_model_note") or ""
+        lines.append(f"模型：`{report['meta']['current_model']}`。{note}")
         lines.append("")
         lines.append("| 层 | n | intent | answer_class | dated | 自洽率 | 中位 ms |")
         lines.append("| --- | ---: | ---: | ---: | ---: | ---: | ---: |")
@@ -437,10 +447,30 @@ def write_markdown(report: Mapping[str, Any]) -> None:
             cons = (report["metrics"].get("current_self_consistency") or {}).get(layer)
             def pct(value: float | None) -> str:
                 return "—" if value is None else f"{value:.1%}"
+            def ms(value: float | None) -> str:
+                return "—" if value is None else f"{value:.0f}"
             lines.append(
                 f"| {layer} | {m.get('n', 0)} | {pct(m.get('intent_acc'))} | {pct(m.get('answer_class_acc'))} | "
-                f"{pct(m.get('dated_acc'))} | {pct(cons)} | {m.get('median_ms') or 0:.0f} |"
+                f"{pct(m.get('dated_acc'))} | {pct(cons)} | {ms(m.get('median_ms'))} |"
             )
+        paired = report["metrics"].get("paired_sample") or {}
+        if paired:
+            lines.append("")
+            lines.append("同一样本上 Jev vs 现行（相对门槛用这一表）：")
+            lines.append("")
+            lines.append("| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp |")
+            lines.append("| --- | ---: | ---: | ---: | ---: | ---: |")
+            for layer in ("choice", "collect", "none"):
+                cell = paired.get(layer) or {}
+                def pct(value: float | None) -> str:
+                    return "—" if value is None else f"{value:.1%}"
+                jev = cell.get("jev_intent")
+                cur = cell.get("current_intent")
+                delta = None if jev is None or cur is None else jev - cur
+                gate = None if cur is None else cur - 0.03
+                lines.append(
+                    f"| {layer} | {cell.get('n', 0)} | {pct(jev)} | {pct(cur)} | {pct(delta)} | {pct(gate)} |"
+                )
     else:
         lines.append(report["meta"].get("current_model_note") or "未跑现行模型。")
     lines += [
@@ -513,16 +543,34 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
     if high_err > 0.03:
         absolute_fail.append(f"高置信错误率 {high_err:.1%} > 3%")
     if not recall_ok:
-        absolute_fail.append(f"低置信召回 {low_recall} < 60%（错了却仍高置信）")
+        recall_txt = " / ".join(f"{v:.1%}" for v in low_recall)
+        absolute_fail.append(f"低置信召回 {recall_txt} < 60%（错了却仍高置信）")
     if choice_class is not None and choice_class < 0.80:
         absolute_fail.append(f"点选题 answer_class 准确率 {choice_class:.1%}（写库字段）")
+    paired = report["metrics"].get("paired_sample") or {}
+    relative_note = ""
+    if paired:
+        bits = []
+        for layer in ("choice", "collect", "none"):
+            cell = paired.get(layer) or {}
+            jev = cell.get("jev_intent")
+            cur = cell.get("current_intent")
+            if jev is None or cur is None:
+                continue
+            ok = round(jev * 100, 1) >= round((cur - 0.03) * 100, 1)
+            bits.append(
+                f"{layer} Jev {jev:.1%} vs 现行 {cur:.1%}（门槛 {cur-0.03:.1%}，{'过' if ok else '未过'}）"
+            )
+        if bits:
+            relative_note = " 相对 −3pp（同一样本）：" + "；".join(bits) + "。"
     if absolute_fail:
         return {
             "verdict": "不可接",
             "reason": (
                 "来源 C 上 Jev 的绝对门槛未过：" + "；".join(absolute_fail) + "。"
                 + represent
-                + (" 现行模型对照未跑，相对 −3pp 门槛无法计算。" if not current else "")
+                + relative_note
+                + (" 现行模型对照未跑，相对 −3pp 门槛无法计算。" if not current and not paired else "")
             ),
             "if_connect": (
                 "不接。现行分类器继续用会话选定的贵模型。"
@@ -530,7 +578,8 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
                 f"曲线上 θ={theta} 时高置信错误仍未清零。"
             ),
         }
-    if not current:
+    paired = report["metrics"].get("paired_sample") or {}
+    if not current and not paired:
         return {
             "verdict": "缺数据",
             "reason": (
@@ -545,11 +594,16 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
         }
     gaps = []
     for layer in ("choice", "collect", "none"):
-        jev = layers[layer]["intent_acc"]
-        cur = (current.get(layer) or {}).get("intent_acc")
+        cell = paired.get(layer) or {}
+        jev = cell.get("jev_intent")
+        cur = cell.get("current_intent")
+        if jev is None:
+            jev = layers[layer]["intent_acc"]
+        if cur is None:
+            cur = (current.get(layer) or {}).get("intent_acc")
         if cur is None:
             continue
-        gaps.append((layer, jev, cur, jev - (cur - 0.03)))
+        gaps.append((layer, jev, cur, round(jev * 100, 1) - round((cur - 0.03) * 100, 1)))
     failed = [g for g in gaps if g[3] < 0]
     if failed:
         return {
@@ -564,38 +618,130 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
     }
 
 
+def _merge_report_preds(samples: list[dict[str, Any]]) -> None:
+    if not REPORT_JSON.is_file():
+        return
+    try:
+        existing = json.loads(REPORT_JSON.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return
+    by_id = {row.get("id"): row for row in existing.get("rows") or [] if row.get("id")}
+    for sample in samples:
+        prior = by_id.get(sample.get("id")) or {}
+        for key in ("jev_1", "jev_2", "current_1", "current_2"):
+            if key not in sample and prior.get(key):
+                sample[key] = prior[key]
+
+
+def paired_layer_metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for layer in ("choice", "collect", "none"):
+        subset = [
+            row for row in rows
+            if row.get("source") == "C"
+            and row.get("layer") == layer
+            and row.get("current_1")
+            and not (row.get("current_1") or {}).get("unavailable")
+        ]
+        if not subset:
+            out[layer] = {"n": 0, "jev_intent": None, "current_intent": None}
+            continue
+        jev = layer_metrics(subset, pred_key="jev_1")
+        cur = layer_metrics(subset, pred_key="current_1")
+        out[layer] = {
+            "n": len(subset),
+            "jev_intent": jev["intent_acc"],
+            "current_intent": cur["intent_acc"],
+            "jev_answer_class": jev["answer_class_acc"],
+            "current_answer_class": cur["answer_class_acc"],
+        }
+    return out
+
+
 def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--workers", type=int, default=8)
     parser.add_argument("--limit", type=int, default=0)
     parser.add_argument("--skip-second", action="store_true")
     parser.add_argument("--second-fraction", type=float, default=1.0)
+    parser.add_argument("--current-only", action="store_true")
+    parser.add_argument("--sample-fraction", type=float, default=1.0)
+    parser.add_argument("--sample-seed", type=int, default=20260919)
     args = parser.parse_args(argv)
-    if not os.environ.get("TYPESAFE_API_KEY"):
+    if not args.current_only and not os.environ.get("TYPESAFE_API_KEY"):
         print("TYPESAFE_API_KEY missing", file=sys.stderr)
         return 2
+    if args.current_only and not os.environ.get("DEEPSEEK_API_KEY"):
+        print("DEEPSEEK_API_KEY missing", file=sys.stderr)
+        return 2
     synthetic = load_jsonl(SAMPLES_DIR / "synthetic.jsonl")
     simulated = load_jsonl(SAMPLES_DIR / "simulated.jsonl")
     source_b = load_jsonl(CACHE_DIR / "source_b.jsonl")
     samples = [row for row in synthetic + simulated if row.get("source") in {"A", "C"}]
     if args.limit:
         samples = samples[: args.limit]
+    _merge_report_preds(samples)
     cache = load_cache()
+    current_sample: list[dict[str, Any]] = []
     try:
-        run_batch(samples, workers=args.workers, run_id="jev_1", cache=cache)
-        save_cache(cache)
-        second = samples
-        if args.skip_second:
-            second = []
-        elif args.second_fraction < 1:
-            n = max(1, int(len(samples) * args.second_fraction))
-            second = [row for row in samples if row.get("source") == "C"][:n]
-        if second:
-            run_batch(second, workers=args.workers, run_id="jev_2", cache=cache)
+        if not args.current_only:
+            run_batch(samples, workers=args.workers, run_id="jev_1", cache=cache)
             save_cache(cache)
-        if source_b:
-            run_batch(source_b, workers=args.workers, run_id="jev_1", cache=cache)
+            second = samples
+            if args.skip_second:
+                second = []
+            elif args.second_fraction < 1:
+                n = max(1, int(len(samples) * args.second_fraction))
+                second = [row for row in samples if row.get("source") == "C"][:n]
+            if second:
+                run_batch(second, workers=args.workers, run_id="jev_2", cache=cache)
+                save_cache(cache)
+            if source_b:
+                run_batch(source_b, workers=args.workers, run_id="jev_1", cache=cache)
+                save_cache(cache)
+        source_c_rows = [row for row in samples if row.get("source") == "C"]
+        if args.current_only or os.environ.get("DEEPSEEK_API_KEY"):
+            if args.sample_fraction < 1:
+                current_sample = stratified_sample(
+                    source_c_rows, fraction=args.sample_fraction, seed=args.sample_seed,
+                )
+                current_sample.extend(row for row in samples if row.get("source") == "A")
+            else:
+                current_sample = list(samples)
+            run_batch(
+                current_sample, workers=args.workers, run_id="current_1",
+                cache=cache, call_fn=call_current_retry,
+            )
             save_cache(cache)
+            second_current = current_sample
+            if args.skip_second:
+                second_current = []
+            elif args.second_fraction < 1:
+                second_current = stratified_sample(
+                    [row for row in current_sample if row.get("source") == "C"],
+                    fraction=args.second_fraction,
+                    seed=args.sample_seed + 1,
+                )
+            if second_current:
+                run_batch(
+                    second_current, workers=args.workers, run_id="current_2",
+                    cache=cache, call_fn=call_current_retry,
+                )
+                save_cache(cache)
+            by_id = {row["id"]: row for row in current_sample}
+            for row in second_current:
+                dest = by_id.setdefault(row["id"], row)
+                if row.get("current_1"):
+                    dest["current_1"] = row["current_1"]
+                if row.get("current_2"):
+                    dest["current_2"] = row["current_2"]
+            for sample in samples:
+                extra = by_id.get(sample["id"])
+                if extra:
+                    if extra.get("current_1"):
+                        sample["current_1"] = extra["current_1"]
+                    if extra.get("current_2"):
+                        sample["current_2"] = extra["current_2"]
     finally:
         save_cache(cache)
 
@@ -634,16 +780,41 @@ def main(argv: Sequence[str] | None = None) -> int:
             "reviewer": "agent-rule-v1",
             "sha256": sha,
             "source_b_n": len(source_b),
-            "current_model": None,
+            "current_model": current_model_id() if any(row.get("current_1") for row in samples) else None,
             "current_model_note": (
-                "本机无会话模型目录凭据（模型 key 在数据库加密配置里）。"
-                "现行 `classifyRectificationTurnIntent` 对照未跑。"
+                (
+                    f"DeepSeek Flash 顶现行 `classifyRectificationTurnIntent` 提示词；"
+                    f"来源 C 分层随机 {args.sample_fraction:.0%}（seed {args.sample_seed}），"
+                    f"来源 A 全量。不是线上会话模型。"
+                    if any(row.get("current_1") for row in samples)
+                    else (
+                        "本机无会话模型目录凭据（模型 key 在数据库加密配置里）。"
+                        "现行 `classifyRectificationTurnIntent` 对照未跑。"
+                    )
+                )
             ),
         },
         "metrics": {
             "jev_by_layer": jev_metrics,
             "jev_self_consistency": jev_cons,
-            "current_by_layer": None,
+            "current_by_layer": (
+                {layer: layer_metrics(
+                    [row for row in samples if row.get("source") == "C" and row.get("layer") == layer and row.get("current_1")],
+                    pred_key="current_1",
+                )
+                 for layer in ("choice", "collect", "none")}
+                if any(row.get("current_1") for row in samples) else None
+            ),
+            "current_self_consistency": (
+                {layer: self_consistency(
+                    [row for row in samples if row.get("source") == "C" and row.get("layer") == layer and row.get("current_2")],
+                    "current_1",
+                    "current_2",
+                )
+                 for layer in ("choice", "collect", "none")}
+                if any(row.get("current_2") for row in samples) else None
+            ),
+            "paired_sample": paired_layer_metrics(samples) if any(row.get("current_1") for row in samples) else {},
             "source_a": source_a_metrics,
             "source_b": source_b_metrics,
             "representativeness": {"note": represent_note},