From 507ef959f519218d190574fa588fc7b92c60229b Mon Sep 17 00:00:00 2001 From: jesse-ux Date: Sat, 19 Sep 2026 12:28:43 +0800 Subject: [PATCH] =?UTF-8?q?research(jev-intent):=20fix2=20=E6=8A=8A?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=20B=20=E7=8E=B0=E8=A1=8C=E4=B8=8E=E9=AB=98?= =?UTF-8?q?=E7=BD=AE=E4=BF=A1=E9=94=99=E8=AF=AF=E8=A1=A5=E8=BF=9B=E6=8A=A5?= =?UTF-8?q?=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 离线从 cache 聚合，不重跑模型。无焦点层 Flash 69.7% 低于 Jev 78.8%。采集层相对门槛标不可判。 --- docs/research/jev_intent_2026_09_19.json | 176 ++++++++++++++- docs/research/jev_intent_2026_09_19.md | 66 +++++- ...jev-intent-classifier-research-20260919.md | 17 ++ docs/tasks/README.md | 2 +- scripts/research/jev_intent_probe.py | 202 +++++++++++++++++- tests/test_jev_intent_research.py | 10 + 6 files changed, 455 insertions(+), 18 deletions(-) diff --git a/docs/research/jev_intent_2026_09_19.json b/docs/research/jev_intent_2026_09_19.json index b95a6186..2224d264 100644 --- a/docs/research/jev_intent_2026_09_19.json +++ b/docs/research/jev_intent_2026_09_19.json @@ -244,6 +244,180 @@ "cost_usd": 0.001293768 } }, + "current_source_b": { + "n": 157, + "unavailable": 0, + "intent_acc": 0.89171974522293, + "answer_class_acc": 0.9702970297029703, + "answer_class_n": 101, + "dated_acc": 0.9363057324840764, + "all_acc": 0.8407643312101911, + "high_conf_error_rate": null, + "low_conf_coverage": null, + "low_conf_recall": null, + "no_vs_unsure": 0, + "median_ms": 685.0, + "p95_ms": 938.0, + "mean_input_tokens": 516.6496815286624, + "cost_usd": 0.0034067880000000004 + }, + "current_source_b_by_layer": { + "choice": { + "n": 17, + "unavailable": 0, + "intent_acc": 0.8823529411764706, + "answer_class_acc": 1.0, + "answer_class_n": 1, + "dated_acc": 0.8823529411764706, + "all_acc": 0.8823529411764706, + "high_conf_error_rate": null, + "low_conf_coverage": null, + "low_conf_recall": null, + "no_vs_unsure": 0, + "median_ms": 657.0, + "p95_ms": 763.0, + "mean_input_tokens": 450.5882352941176, + "cost_usd": 0.00032172 + }, + "collect": { + "n": 107, + "unavailable": 0, + "intent_acc": 0.9532710280373832, + "answer_class_acc": 0.97, + "answer_class_n": 100, + "dated_acc": 0.9439252336448598, + "all_acc": 0.8785046728971962, + "high_conf_error_rate": null, + "low_conf_coverage": null, + "low_conf_recall": null, + "no_vs_unsure": 0, + "median_ms": 689.0, + "p95_ms": 938.0, + "mean_input_tokens": 528.9345794392524, + "cost_usd": 0.0023770320000000003 + }, + "none": { + "n": 33, + "unavailable": 0, + "intent_acc": 0.696969696969697, + "answer_class_acc": null, + "answer_class_n": 0, + "dated_acc": 0.9393939393939394, + "all_acc": 0.696969696969697, + "high_conf_error_rate": null, + "low_conf_coverage": null, + "low_conf_recall": null, + "no_vs_unsure": 0, + "median_ms": 715.0, + "p95_ms": 978.0, + "mean_input_tokens": 510.8484848484849, + "cost_usd": 0.0007080360000000001 + } + }, + "source_b_jev_self_consistency": { + "all": 0.9617834394904459, + "choice": 0.9411764705882353, + "collect": 0.9906542056074766, + "none": 0.8787878787878788 + }, + "source_b_none_confusion": { + "jev": { + "labels": [ + "provide_new_evidence", + "stop_rectification", + "ask_about_result", + "unclear", + "answer_current_focus" + ], + "counts": { + "provide_new_evidence": { + "provide_new_evidence": 24, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 0, + "answer_current_focus": 0 + }, + "stop_rectification": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 0, + "answer_current_focus": 0 + }, + "ask_about_result": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 1, + "unclear": 0, + "answer_current_focus": 0 + }, + "unclear": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 1, + "answer_current_focus": 7 + }, + "answer_current_focus": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 0, + "answer_current_focus": 0 + } + }, + "other": 0, + "n": 33 + }, + "current": { + "labels": [ + "provide_new_evidence", + "stop_rectification", + "ask_about_result", + "unclear", + "answer_current_focus" + ], + "counts": { + "provide_new_evidence": { + "provide_new_evidence": 21, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 0, + "answer_current_focus": 3 + }, + "stop_rectification": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 0, + "answer_current_focus": 0 + }, + "ask_about_result": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 1, + "answer_current_focus": 0 + }, + "unclear": { + "provide_new_evidence": 1, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 2, + "answer_current_focus": 5 + }, + "answer_current_focus": { + "provide_new_evidence": 0, + "stop_rectification": 0, + "ask_about_result": 0, + "unclear": 0, + "answer_current_focus": 0 + } + }, + "other": 0, + "n": 33 + } + }, "representativeness": { "note": "来源 B 与来源 C 同层 intent 准确率：choice B 94.1% vs C 99.0%（差 4.9%，n_B=17）；collect B 94.4% vs C 92.2%（差 2.1%，n_B=107）；none B 78.8% vs C 96.0%（差 17.2%，n_B=33）。 有层差 > 10pp，结论降为缺数据。", "fail": true, @@ -101740,7 +101914,7 @@ }, "conclusion": { "verdict": "缺数据", - "reason": "来源 B 与来源 C 同层 intent 准确率差 > 10pp，模拟语料不代表真人，来源 C 门槛结论降为缺数据。来源 B 与来源 C 同层 intent 准确率：choice B 94.1% vs C 99.0%（差 4.9%，n_B=17）；collect B 94.4% vs C 92.2%（差 2.1%，n_B=107）；none B 78.8% vs C 96.0%（差 17.2%，n_B=33）。 有层差 > 10pp，结论降为缺数据。 同时来源 C 绝对门槛未过：低置信召回 23.1% / 45.5% / 25.0% < 60%（错了却仍高置信）。 相对 −3pp（同一样本）：choice Jev 99.0% vs 现行 98.8%（门槛 95.8%，过）；collect Jev 92.2% vs 现行 97.5%（门槛 94.5%，未过）；none Jev 96.0% vs 现行 94.0%（门槛 91.0%，过）。", + "reason": "来源 B 与来源 C 同层 intent 准确率差 > 10pp，模拟语料不代表真人，来源 C 门槛结论降为缺数据。来源 B 与来源 C 同层 intent 准确率：choice B 94.1% vs C 99.0%（差 4.9%，n_B=17）；collect B 94.4% vs C 92.2%（差 2.1%，n_B=107）；none B 78.8% vs C 96.0%（差 17.2%，n_B=33）。 有层差 > 10pp，结论降为缺数据。 同时来源 C 绝对门槛未过：低置信召回 23.1% / 45.5% / 25.0% < 60%（错了却仍高置信）。 相对 −3pp（同一样本）：choice Jev 99.0% vs 现行 98.8%（门槛 95.8%，过）；collect Jev 92.2% vs 现行 97.5%（门槛 94.5%，不可判（复核与对照同源））；none Jev 96.0% vs 现行 94.0%（门槛 91.0%，过）。", "if_connect": "不得上线。先补真机样本或重造更像真人的来源 C，再测。" } } diff --git a/docs/research/jev_intent_2026_09_19.md b/docs/research/jev_intent_2026_09_19.md index 5ff35d21..b9b30b75 100644 --- a/docs/research/jev_intent_2026_09_19.md +++ b/docs/research/jev_intent_2026_09_19.md @@ -15,7 +15,7 @@ ## 结论 -**缺数据**。来源 B 与来源 C 同层 intent 准确率差 > 10pp，模拟语料不代表真人，来源 C 门槛结论降为缺数据。来源 B 与来源 C 同层 intent 准确率：choice B 94.1% vs C 99.0%（差 4.9%，n_B=17）；collect B 94.4% vs C 92.2%（差 2.1%，n_B=107）；none B 78.8% vs C 96.0%（差 17.2%，n_B=33）。 有层差 > 10pp，结论降为缺数据。 同时来源 C 绝对门槛未过：低置信召回 23.1% / 45.5% / 25.0% < 60%（错了却仍高置信）。 相对 −3pp（同一样本）：choice Jev 99.0% vs 现行 98.8%（门槛 95.8%，过）；collect Jev 92.2% vs 现行 97.5%（门槛 94.5%，未过）；none Jev 96.0% vs 现行 94.0%（门槛 91.0%，过）。 +**缺数据**。来源 B 与来源 C 同层 intent 准确率差 > 10pp，模拟语料不代表真人，来源 C 门槛结论降为缺数据。来源 B 与来源 C 同层 intent 准确率：choice B 94.1% vs C 99.0%（差 4.9%，n_B=17）；collect B 94.4% vs C 92.2%（差 2.1%，n_B=107）；none B 78.8% vs C 96.0%（差 17.2%，n_B=33）。 有层差 > 10pp，结论降为缺数据。 同时来源 C 绝对门槛未过：低置信召回 23.1% / 45.5% / 25.0% < 60%（错了却仍高置信）。 相对 −3pp（同一样本）：choice Jev 99.0% vs 现行 98.8%（门槛 95.8%，过）；collect Jev 92.2% vs 现行 97.5%（门槛 94.5%，不可判（复核与对照同源））；none Jev 96.0% vs 现行 94.0%（门槛 91.0%，过）。 若接：不得上线。先补真机样本或重造更像真人的来源 C，再测。 @@ -41,18 +41,49 @@ 同一样本上 Jev vs 现行（相对门槛用这一表）： -| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp | -| --- | ---: | ---: | ---: | ---: | ---: | -| choice | 400 | 99.0% | 98.8% | 0.2% | 95.8% | -| collect | 400 | 92.2% | 97.5% | -5.2% | 94.5% | -| none | 100 | 96.0% | 94.0% | 2.0% | 91.0% | +| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp | 判定 | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| choice | 400 | 99.0% | 98.8% | 0.2% | 95.8% | 过 | +| collect | 400 | 92.2% | 97.5% | -5.2% | 94.5% | 不可判（复核与对照同源） | +| none | 100 | 96.0% | 94.0% | 2.0% | 91.0% | 过 | ## 代表性检验（来源 B vs 来源 C） -来源 B 已标注 157 条（点选 17 / 采集 107 / 无焦点 33；未标注 0），长度 P25/中位/P75/最长 = 12/21/28/172。人工标注，未用模型代标。Jev 在来源 B 上 intent：点选 94.1%、采集 94.4%、无焦点 78.8%；全集 91.1%。无焦点层差 17.2pp > 10pp。 - 来源 B 与来源 C 同层 intent 准确率：choice B 94.1% vs C 99.0%（差 4.9%，n_B=17）；collect B 94.4% vs C 92.2%（差 2.1%，n_B=107）；none B 78.8% vs C 96.0%（差 17.2%，n_B=33）。 有层差 > 10pp，结论降为缺数据。 +来源 B 是真人 + 人工标注 + 线上模型三者齐备的唯一一组。现行无置信度，置信度三列为空。 + +| 范围 | n | Jev intent | 现行 intent | Jev 高置信错误 | Jev 低置信召回 | Jev 自洽 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 全集 | 157 | 91.1% | 89.2% | 6.4% | 24.1% | 96.2% | +| choice | 17 | 94.1% | 88.2% | 0.0% | 66.7% | 94.1% | +| collect | 107 | 94.4% | 95.3% | 5.6% | 18.2% | 99.1% | +| none | 33 | 78.8% | 69.7% | 12.1% | 20.0% | 87.9% | + +无焦点层现行 intent 69.7%、Jev 78.8%。现行更低，说明 78.8% 主要是这 33 条本身难，不是单 Jev 不行。 + +无焦点层 gold × 预测混淆计数（只有计数，无原文）。预测出现 `answer_current_focus` 是因为模型把无焦点句当成在回答采集题。 + +### gold × Jev（n=33） + +| gold \ pred | provide_new_evidence | stop_rectification | ask_about_result | unclear | answer_current_focus | +| --- | ---: | ---: | ---: | ---: | ---: | +| provide_new_evidence | 24 | 0 | 0 | 0 | 0 | +| stop_rectification | 0 | 0 | 0 | 0 | 0 | +| ask_about_result | 0 | 0 | 1 | 0 | 0 | +| unclear | 0 | 0 | 0 | 1 | 7 | +| answer_current_focus | 0 | 0 | 0 | 0 | 0 | + +### gold × 现行（n=33） + +| gold \ pred | provide_new_evidence | stop_rectification | ask_about_result | unclear | answer_current_focus | +| --- | ---: | ---: | ---: | ---: | ---: | +| provide_new_evidence | 21 | 0 | 0 | 0 | 3 | +| stop_rectification | 0 | 0 | 0 | 0 | 0 | +| ask_about_result | 0 | 0 | 0 | 1 | 0 | +| unclear | 1 | 0 | 0 | 2 | 5 | +| answer_current_focus | 0 | 0 | 0 | 0 | 0 | + ## 置信度–准确率曲线与 θ 推荐 θ = 0.9。点： @@ -154,6 +185,25 @@ | 若同一句话既明确否定当前采集题又补充了新的带时间经历，intent 仍为 answer_current_focus 且 answer_class 为 no，不要改成 provide_new_evidence。 | Code keeps no + Noul true. Jev intent/Noul are independent. | Jev may split this pair; code does not re-vote intent from the Noul. | | 不要按关键词表或正则猜测，只根据当前问题与用户这句话的语义分类。 | Not sent. Jev has no keyword table in the question. | The meta-instruction is dropped (Jev answers the written question, not the intended one). | +## 限制 + +1. **复核 ≈ 生产提示，且复核模型 = 对照模型。** `REVIEW_RUBRIC` 与生产 `COLLECT_INSTRUCTIONS` 逐句对应；生成 / 复核 / 对照都是 `deepseek-flash`。进入测试集的 900 条是「Flash 用近生产提示能答对目标标签」的那 900 条，被剔的 36 条恰是 Flash 不同意的。因此来源 C 上现行 97.5% / 98.8% / 94.0% 是构造出来的上界。采集层「Jev 92.2% 未过相对门槛 94.5%」**不可当作 Jev 输给现行的证据**。 +2. **采集层 intent 错例的 gold 有争议。** 来源 C 采集层 Jev intent 错例 31 条：`provide_new_evidence → answer_current_focus` 17、`unclear → answer_current_focus` 9、`stop → unclear` 4、`ask → unclear` 1。17 条 pne 几乎全是「另外 2019 年我换工作搬了家」句式，若干尾句落在当前题域，按生产提示可读成 `answer_current_focus + unsure`。9 条 unclear（「一时半会儿真捋不明白」）按「记不清 → unsure」也读得通。两类合计 ≥ 20 条，占该层 intent 错例约 2/3。gold 来自「生成目标 + Flash 复核同意」，不等于人工真值。改写示例： + - 「另外 2019 年换过工作，感情那会儿真没细想。」gold=provide_new_evidence；可读成在回答感情采集题。 + - 「另外 2019 年搬了家，工作那摊子反而没顾上细想。」gold=provide_new_evidence；可读成在回答工作采集题。 + - 「这事我一时真说不上来。」gold=unclear；按生产提示是 unsure。 +3. **语料仍不像真人。** 修复单 1 只把长度和人设写成硬红线（已过）。原单还要求按来源 B 的标点 / 语气词比例约束，未进红线： + +| 指标 | 来源 B（真人） | 来源 C（模拟） | +| --- | ---: | ---: | +| 含标点 | 13% | 98.9% | +| 含语气词（吧/呢/啊/嗯/哦/额/emm） | 4% | 25.6% | +| 含年份或月份 | 81% | 47.8% | +| 带年份句里写「2019」 | — | 236 / 429 = 55% | +| `provide_new_evidence` 里是搬家/换工作 | — | 152 / 157 | + +根因：生成脚本 `ALT_EVENT_HINTS` 给七个领域的「另一件事」全是搬家/换工作，年份未约束，模型收敛到「另外 2019 年搬过家」。这解释了模拟语料不代表真人的一部分，也解释了采集层错例为何长得一样。本单不修，留给产品决定是否再造一轮。 + ## 回退 任何上线方案必须保留回退到现行会话模型的路径。官方限流会动态调整。 diff --git a/docs/tasks/PROGRESS-rectification-jev-intent-classifier-research-20260919.md b/docs/tasks/PROGRESS-rectification-jev-intent-classifier-research-20260919.md index 84201076..45f8f4c1 100644 --- a/docs/tasks/PROGRESS-rectification-jev-intent-classifier-research-20260919.md +++ b/docs/tasks/PROGRESS-rectification-jev-intent-classifier-research-20260919.md @@ -119,3 +119,20 @@ cache 样本 id 与 `simulated.jsonl` 一致（900/900）。unavailable = 0。 - `python -m pytest tests/test_jev_intent_research.py -q`：11 passed - 构建脚本无 `bank = {`；复核函数无 `re.search` 判标签 - 线上分类器文件未改 + +## 修复轮 2（2026-09-19，`codex/rectification-jev-intent-research-fix2-20260919`） + +任务书：`TASK-rectification-jev-intent-classifier-research-fix2-20260919.md`。**0 token**：从 `.cache/jev_intent/jev_runs_v2.json` 聚合已跑结果，未重造语料、未重跑模型。 + +补进报告的三组数（来源 B 157 条）： + +| 范围 | n | Jev intent | 现行 Flash intent | Jev 高置信错误 | Jev 低置信召回 | Jev 自洽 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 全集 | 157 | 91.1% | 89.2% | **6.4%** | 24.1% | 96.2% | +| 点选 | 17 | 94.1% | 88.2% | 0.0% | 66.7% | 94.1% | +| 采集 | 107 | 94.4% | 95.3% | 5.6% | 18.2% | 99.1% | +| 无焦点 | 33 | 78.8% | **69.7%** | **12.1%** | 20.0% | 87.9% | + +无焦点层 7 条 Jev 错全是 `unclear → answer_current_focus`。现行在无焦点上更低（69.7%），10 条错里 5 条同样是 `unclear → answer_current_focus`。说明 78.8% 主要是这 33 条难，不是单 Jev 不行。 + +采集层相对门槛改为「不可判（复核与对照同源）」。限制三节已写入报告。结论仍 **缺数据**，不上线。未立 BUG。 diff --git a/docs/tasks/README.md b/docs/tasks/README.md index 115e54cf..ef5d972e 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -234,7 +234,7 @@ | `TASK-rectification-open-collect-invite-20260914.md` | `PROGRESS-rectification-open-collect-invite-20260914.md` | **P0**：固定七条采集线问完后只说「能问的都问完了」，用户不知道还能补经历、也不知道补了有用；而两轮研究证明补带年月经历是唯一有效手段。产品拍板：交付卡照出 + 卡上给不限领域的补充邀请（先要确切日期，再退年月；举七条线之外的例子），补完必须可见生效（BUG-689） | 待验收 | `codex/rectification-open-collect-invite-20260914` | | `TASK-rectification-cluster-width-research-20260914.md` | `PROGRESS-rectification-cluster-width-research-20260914.md` | **研究单**：上一轮证明调权重改不动交付区间宽度——所有方案宽度中位数都等于整个搜索窗。先确认 sweep 的宽度口径是否含淘汰（M0），再画簇结构像（M1），最后量三个改法：放宽簇上限、按分差决定是否合并、交付区间改分位覆盖（M2）。真值覆盖率不得下降 | 待验收 | `codex/rectification-cluster-width-research-20260914` | -| `TASK-rectification-jev-intent-classifier-research-20260919.md` | `PROGRESS-rectification-jev-intent-classifier-research-20260919.md` | **研究单**：TypeSafe Jev（只做 Choice/Score/Noul 的校准判断模型，$0.042/Mtok）能否接管校正流的意图分类。产品 09-19 授权评估（推翻 09-15「分类只用贵模型」需重新拍板）。Agent 模拟校正流造 ≥900 条中文语料（标签先定、独立复核）+ 真机样本做代表性锚，量准确率 / 高置信错误率 / 低置信召回 / 延迟；只离线测，不改线上 | 修复轮已验收：结论 **缺数据** 采纳，不上线；报告补漏单 `TASK-rectification-jev-intent-classifier-research-fix2-20260919.md` 待领取 | 红线 1–8、F1/F2/F5 通过；F3/F4 部分未通过：现行模型在来源 B 157 条上跑了没报、来源 B 上 Jev 高置信错误 6.4%（无焦点 12.1%）没进 MD、无焦点层差没到标签；复核提示 ≈ 生产提示且同模型，采集层相对门槛不可判 | +| `TASK-rectification-jev-intent-classifier-research-20260919.md` | `PROGRESS-rectification-jev-intent-classifier-research-20260919.md` | **研究单**：TypeSafe Jev（只做 Choice/Score/Noul 的校准判断模型，$0.042/Mtok）能否接管校正流的意图分类。产品 09-19 授权评估（推翻 09-15「分类只用贵模型」需重新拍板）。Agent 模拟校正流造 ≥900 条中文语料（标签先定、独立复核）+ 真机样本做代表性锚，量准确率 / 高置信错误率 / 低置信召回 / 延迟；只离线测，不改线上 | **待验收**（fix2 已交，结论仍 **缺数据**） | `codex/rectification-jev-intent-research-fix2-20260919`。来源 B 现行 Flash 已入表（全集 89.2%、无焦点 69.7%）；Jev 高置信错误全集 6.4% / 无焦点 12.1%；无焦点混淆全是 unclear→answer_current_focus。采集层相对门槛标不可判。 | | `TASK-rectification-minute-resolution-research-20260914.md` | `PROGRESS-rectification-minute-resolution-research-20260914.md` | **研究单**：候选分不开的根因是打分尺度——窗口内恒定项 11.5 分 vs 随分钟变化项 2.125 分（≈5:1）。先修封存基准（v3 每例仅 3 件事且被标 invalidated）出 v4，再离线量五个改法：分盘除数、去底座、**KP 宫头子主计分（产品 09-14 拍板，推翻 BUG-325 一条红线）**、年精度事件改边际似然、聚类签名层对齐。有收益才立实现单 | 待验收 | `codex/rectification-minute-resolution-research-20260914` | diff --git a/scripts/research/jev_intent_probe.py b/scripts/research/jev_intent_probe.py index 81f92d81..bf345a1e 100644 --- a/scripts/research/jev_intent_probe.py +++ b/scripts/research/jev_intent_probe.py @@ -327,6 +327,58 @@ def cache_key(run_id: str, sample: Mapping[str, Any]) -> str: return f"{run_id}:{sample['id']}:{digest}" +NONE_CONFUSION_LABELS = ( + "provide_new_evidence", + "stop_rectification", + "ask_about_result", + "unclear", + "answer_current_focus", +) + + +def attach_from_cache( + samples: Sequence[dict[str, Any]], + cache: Mapping[str, Any], + run_ids: Sequence[str], +) -> dict[str, int]: + missing = {rid: 0 for rid in run_ids} + for sample in samples: + for rid in run_ids: + key = cache_key(rid, sample) + if key in cache: + sample[rid] = cache[key] + else: + missing[rid] += 1 + return missing + + +def strip_confidence(metrics: Mapping[str, Any] | None) -> dict[str, Any] | None: + if metrics is None: + return None + out = dict(metrics) + out["high_conf_error_rate"] = None + out["low_conf_coverage"] = None + out["low_conf_recall"] = None + return out + + +def confusion_counts( + rows: Sequence[Mapping[str, Any]], + pred_key: str, + labels: Sequence[str] = NONE_CONFUSION_LABELS, +) -> dict[str, Any]: + counts = {gold: {pred: 0 for pred in labels} for gold in labels} + other = 0 + for row in rows: + gold = (row.get("gold") or {}).get("intent") + pred = (row.get(pred_key) or {}).get("intent") + if gold in counts and pred in counts[gold]: + counts[gold][pred] += 1 + else: + other += 1 + return {"labels": list(labels), "counts": counts, "other": other, "n": len(rows)} + + def run_batch( samples: Sequence[Mapping[str, Any]], *, @@ -471,8 +523,8 @@ def write_markdown(report: Mapping[str, Any]) -> None: lines.append("") lines.append("同一样本上 Jev vs 现行（相对门槛用这一表）：") lines.append("") - lines.append("| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp |") - lines.append("| --- | ---: | ---: | ---: | ---: | ---: |") + lines.append("| 层 | n | Jev intent | 现行 intent | 差（Jev−现行） | 门槛现行−3pp | 判定 |") + lines.append("| --- | ---: | ---: | ---: | ---: | ---: | --- |") for layer in ("choice", "collect", "none"): cell = paired.get(layer) or {} def pct(value: float | None) -> str: @@ -481,8 +533,16 @@ def write_markdown(report: Mapping[str, Any]) -> None: cur = cell.get("current_intent") delta = None if jev is None or cur is None else jev - cur gate = None if cur is None else cur - 0.03 + if layer == "collect": + verdict = "不可判（复核与对照同源）" + elif jev is None or cur is None: + verdict = "—" + elif round(jev * 100, 1) >= round((cur - 0.03) * 100, 1): + verdict = "过" + else: + verdict = "未过" lines.append( - f"| {layer} | {cell.get('n', 0)} | {pct(jev)} | {pct(cur)} | {pct(delta)} | {pct(gate)} |" + f"| {layer} | {cell.get('n', 0)} | {pct(jev)} | {pct(cur)} | {pct(delta)} | {pct(gate)} | {verdict} |" ) else: lines.append(report["meta"].get("current_model_note") or "未跑现行模型。") @@ -492,6 +552,65 @@ def write_markdown(report: Mapping[str, Any]) -> None: "", report["metrics"]["representativeness"]["note"], "", + ] + b_jev = report["metrics"].get("source_b") + b_jev_layer = report["metrics"].get("source_b_by_layer") or {} + b_cur = report["metrics"].get("current_source_b") + b_cur_layer = report["metrics"].get("current_source_b_by_layer") or {} + b_cons = report["metrics"].get("source_b_jev_self_consistency") or {} + if b_jev: + def pct(value: float | None) -> str: + return "—" if value is None else f"{value:.1%}" + lines += [ + "来源 B 是真人 + 人工标注 + 线上模型三者齐备的唯一一组。现行无置信度，置信度三列为空。", + "", + "| 范围 | n | Jev intent | 现行 intent | Jev 高置信错误 | Jev 低置信召回 | Jev 自洽 |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + rows_spec = [("全集", b_jev, b_cur, b_cons.get("all"))] + for layer in ("choice", "collect", "none"): + rows_spec.append(( + layer, + b_jev_layer.get(layer) or {}, + b_cur_layer.get(layer) or {}, + b_cons.get(layer), + )) + for name, jev_m, cur_m, cons in rows_spec: + lines.append( + f"| {name} | {jev_m.get('n', 0)} | {pct(jev_m.get('intent_acc'))} | {pct((cur_m or {}).get('intent_acc'))} | " + f"{pct(jev_m.get('high_conf_error_rate'))} | {pct(jev_m.get('low_conf_recall'))} | {pct(cons)} |" + ) + lines.append("") + none_cur = (b_cur_layer.get("none") or {}).get("intent_acc") + none_jev = (b_jev_layer.get("none") or {}).get("intent_acc") + if none_cur is not None and none_jev is not None: + lines.append( + f"无焦点层现行 intent {none_cur:.1%}、Jev {none_jev:.1%}。" + "现行更低，说明 78.8% 主要是这 33 条本身难，不是单 Jev 不行。" + ) + lines.append("") + confusion = report["metrics"].get("source_b_none_confusion") or {} + if confusion: + lines += [ + "无焦点层 gold × 预测混淆计数（只有计数，无原文）。预测出现 `answer_current_focus` 是因为模型把无焦点句当成在回答采集题。", + "", + ] + for title, key in (("gold × Jev", "jev"), ("gold × 现行", "current")): + table = confusion.get(key) or {} + labels = table.get("labels") or list(NONE_CONFUSION_LABELS) + counts = table.get("counts") or {} + lines.append(f"### {title}（n={table.get('n', 0)}）") + lines.append("") + header = "| gold \\ pred | " + " | ".join(labels) + " |" + sep = "| --- | " + " | ".join("---:" for _ in labels) + " |" + lines.append(header) + lines.append(sep) + for gold in labels: + row_counts = counts.get(gold) or {} + cells = " | ".join(str(row_counts.get(pred, 0)) for pred in labels) + lines.append(f"| {gold} | {cells} |") + lines.append("") + lines += [ "## 置信度–准确率曲线与 θ", "", f"推荐 θ = {report['metrics']['theta']['recommended_theta']}。点：", @@ -530,6 +649,25 @@ def write_markdown(report: Mapping[str, Any]) -> None: lost = row["lost"].replace("|", "\\|") if row["lost"] else "—" lines.append(f"| {row['production']} | {row['jev']} | {lost} |") lines += [ + "", + "## 限制", + "", + "1. **复核 ≈ 生产提示，且复核模型 = 对照模型。** `REVIEW_RUBRIC` 与生产 `COLLECT_INSTRUCTIONS` 逐句对应；生成 / 复核 / 对照都是 `deepseek-flash`。进入测试集的 900 条是「Flash 用近生产提示能答对目标标签」的那 900 条，被剔的 36 条恰是 Flash 不同意的。因此来源 C 上现行 97.5% / 98.8% / 94.0% 是构造出来的上界。采集层「Jev 92.2% 未过相对门槛 94.5%」**不可当作 Jev 输给现行的证据**。", + "2. **采集层 intent 错例的 gold 有争议。** 来源 C 采集层 Jev intent 错例 31 条：`provide_new_evidence → answer_current_focus` 17、`unclear → answer_current_focus` 9、`stop → unclear` 4、`ask → unclear` 1。17 条 pne 几乎全是「另外 2019 年我换工作搬了家」句式，若干尾句落在当前题域，按生产提示可读成 `answer_current_focus + unsure`。9 条 unclear（「一时半会儿真捋不明白」）按「记不清 → unsure」也读得通。两类合计 ≥ 20 条，占该层 intent 错例约 2/3。gold 来自「生成目标 + Flash 复核同意」，不等于人工真值。改写示例：", + " - 「另外 2019 年换过工作，感情那会儿真没细想。」gold=provide_new_evidence；可读成在回答感情采集题。", + " - 「另外 2019 年搬了家，工作那摊子反而没顾上细想。」gold=provide_new_evidence；可读成在回答工作采集题。", + " - 「这事我一时真说不上来。」gold=unclear；按生产提示是 unsure。", + "3. **语料仍不像真人。** 修复单 1 只把长度和人设写成硬红线（已过）。原单还要求按来源 B 的标点 / 语气词比例约束，未进红线：", + "", + "| 指标 | 来源 B（真人） | 来源 C（模拟） |", + "| --- | ---: | ---: |", + "| 含标点 | 13% | 98.9% |", + "| 含语气词（吧/呢/啊/嗯/哦/额/emm） | 4% | 25.6% |", + "| 含年份或月份 | 81% | 47.8% |", + "| 带年份句里写「2019」 | — | 236 / 429 = 55% |", + "| `provide_new_evidence` 里是搬家/换工作 | — | 152 / 157 |", + "", + "根因：生成脚本 `ALT_EVENT_HINTS` 给七个领域的「另一件事」全是搬家/换工作，年份未约束，模型收敛到「另外 2019 年搬过家」。这解释了模拟语料不代表真人的一部分，也解释了采集层错例为何长得一样。本单不修，留给产品决定是否再造一轮。", "", "## 回退", "", @@ -570,8 +708,9 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]: if jev is None or cur is None: continue ok = round(jev * 100, 1) >= round((cur - 0.03) * 100, 1) + mark = "不可判（复核与对照同源）" if layer == "collect" else ("过" if ok else "未过") bits.append( - f"{layer} Jev {jev:.1%} vs 现行 {cur:.1%}（门槛 {cur-0.03:.1%}，{'过' if ok else '未过'}）" + f"{layer} Jev {jev:.1%} vs 现行 {cur:.1%}（门槛 {cur-0.03:.1%}，{mark}）" ) if bits: relative_note = " 相对 −3pp（同一样本）：" + "；".join(bits) + "。" @@ -693,11 +832,12 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--sample-fraction", type=float, default=1.0) parser.add_argument("--current-second-fraction", type=float, default=1.0 / 3) parser.add_argument("--sample-seed", type=int, default=20260919) + parser.add_argument("--offline", action="store_true", help="aggregate from cache only, no API calls") args = parser.parse_args(argv) - if not args.current_only and not os.environ.get("TYPESAFE_API_KEY"): + if not args.offline and not args.current_only and not os.environ.get("TYPESAFE_API_KEY"): print("TYPESAFE_API_KEY missing", file=sys.stderr) return 2 - if args.current_only and not os.environ.get("DEEPSEEK_API_KEY"): + if not args.offline and args.current_only and not os.environ.get("DEEPSEEK_API_KEY"): print("DEEPSEEK_API_KEY missing", file=sys.stderr) return 2 synthetic = load_jsonl(SAMPLES_DIR / "synthetic.jsonl") @@ -715,8 +855,13 @@ def main(argv: Sequence[str] | None = None) -> int: _merge_report_preds(samples) cache = load_cache() current_sample: list[dict[str, Any]] = [] + second_current: list[dict[str, Any]] = [] + if args.offline: + miss_c = attach_from_cache(samples, cache, ("jev_1", "jev_2", "current_1", "current_2")) + miss_b = attach_from_cache(source_b, cache, ("jev_1", "jev_2", "current_1")) + print(json.dumps({"offline_missing": {"samples": miss_c, "source_b": miss_b}}, ensure_ascii=False), flush=True) try: - if not args.current_only: + if not args.offline and not args.current_only: run_batch(samples, workers=args.workers, run_id="jev_1", cache=cache) save_cache(cache) second = samples @@ -734,7 +879,7 @@ def main(argv: Sequence[str] | None = None) -> int: run_batch(source_b, workers=args.workers, run_id="jev_2", cache=cache) save_cache(cache) source_c_rows = [row for row in samples if row.get("source") == "C"] - if args.current_only or os.environ.get("DEEPSEEK_API_KEY"): + if not args.offline and (args.current_only or os.environ.get("DEEPSEEK_API_KEY")): if args.sample_fraction < 1: current_sample = stratified_sample( source_c_rows, fraction=args.sample_fraction, seed=args.sample_seed, @@ -802,6 +947,38 @@ def main(argv: Sequence[str] | None = None) -> int: layer: layer_metrics([row for row in source_b if row.get("layer") == layer], pred_key="jev_1") for layer in ("choice", "collect", "none") } if source_b else {} + current_source_b = strip_confidence( + layer_metrics(source_b, pred_key="current_1") if source_b and any(row.get("current_1") for row in source_b) else None + ) + current_source_b_by_layer = { + layer: strip_confidence(layer_metrics( + [row for row in source_b if row.get("layer") == layer and row.get("current_1")], + pred_key="current_1", + )) + for layer in ("choice", "collect", "none") + } if source_b and any(row.get("current_1") for row in source_b) else {} + source_b_jev_self_consistency = ( + { + "all": self_consistency(source_b, "jev_1", "jev_2"), + **{ + layer: self_consistency( + [row for row in source_b if row.get("layer") == layer], + "jev_1", + "jev_2", + ) + for layer in ("choice", "collect", "none") + }, + } + if source_b else {} + ) + none_b = [row for row in source_b if row.get("layer") == "none"] + source_b_none_confusion = ( + { + "jev": confusion_counts(none_b, "jev_1"), + "current": confusion_counts(none_b, "current_1"), + } + if none_b else {} + ) represent_fail = False represent_layers: dict[str, Any] = {} if not source_b or len(source_b) < 30: @@ -904,6 +1081,10 @@ def main(argv: Sequence[str] | None = None) -> int: "source_a": source_a_metrics, "source_b": source_b_metrics, "source_b_by_layer": source_b_by_layer, + "current_source_b": current_source_b, + "current_source_b_by_layer": current_source_b_by_layer, + "source_b_jev_self_consistency": source_b_jev_self_consistency, + "source_b_none_confusion": source_b_none_confusion, "representativeness": { "note": represent_note, "fail": represent_fail, @@ -933,6 +1114,11 @@ def main(argv: Sequence[str] | None = None) -> int: } for k, v in jev_metrics.items()}, "self_consistency": jev_cons, "source_b_n": len(source_b), + "current_source_b": { + "n": (current_source_b or {}).get("n"), + "intent": (current_source_b or {}).get("intent_acc"), + }, + "source_b_none_confusion_n": (source_b_none_confusion.get("jev") or {}).get("n"), }, ensure_ascii=False, indent=2)) return 0 diff --git a/tests/test_jev_intent_research.py b/tests/test_jev_intent_research.py index 174d34d3..39ee1b57 100644 --- a/tests/test_jev_intent_research.py +++ b/tests/test_jev_intent_research.py @@ -132,6 +132,16 @@ class JevIntentCorpusTests(unittest.TestCase): for name in names: self.assertNotIn(name, message) + def test_report_json_includes_current_source_b(self) -> None: + path = ROOT / "docs" / "research" / "jev_intent_2026_09_19.json" + if not path.is_file(): + self.skipTest("report json not generated yet") + payload = json.loads(path.read_text(encoding="utf-8")) + current = (payload.get("metrics") or {}).get("current_source_b") + self.assertIsNotNone(current) + self.assertEqual(current["n"], payload["meta"]["source_b_n"]) + self.assertIn("source_b_none_confusion", payload["metrics"]) + if __name__ == "__main__": unittest.main()