root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

research(jev-intent): 修复轮重造语料并全量对照
Independent Staging Quality Gate / publish (push) Canceled after 0s
Details
Independent Staging Quality Gate / validate (push) Canceled after 1m37s
Details

Browse Source 
来源 C 改为 DeepSeek Flash 生成+独立复核，撤回模板拼接结论。来源 B 157 条人工标注后跑 Jev x2 与 Flash 全量对照，结论为缺数据。
This commit is contained in:
jesse-ux
2026-09-19 11:30:31 +08:00
parent 69a44fe78d
commit c2ecbc74cd
12 changed files with 95414 additions and 82588 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/research/jev_intent_probe.py
+123 -32
View File
@@ -8,6 +8,7 @@ Source B rows are aggregated and never written as raw text.
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sys
@@ -320,6 +321,12 @@ def error_portrait(rows: Sequence[Mapping[str, Any]], *, limit: int = 3) -> dict
return buckets
def cache_key(run_id: str, sample: Mapping[str, Any]) -> str:
message = str(sample.get("user_message") or "")
digest = hashlib.sha256(message.encode("utf-8")).hexdigest()[:16]
return f"{run_id}:{sample['id']}:{digest}"
def run_batch(
samples: Sequence[Mapping[str, Any]],
*,
@@ -331,7 +338,7 @@ def run_batch(
caller = call_fn or call_jev_retry
pending = []
for sample in samples:
key = f"{run_id}:{sample['id']}"
key = cache_key(run_id, sample)
if key in cache:
sample[run_id] = cache[key]
else:
@@ -346,14 +353,14 @@ def run_batch(
sample = futures[future]
result = future.result()
sample[run_id] = result
cache[f"{run_id}:{sample['id']}"] = result
cache[cache_key(run_id, sample)] = result
done += 1
if done % 50 == 0 or done == len(pending):
print(f" {run_id} {done}/{len(pending)}", flush=True)
def load_cache() -> dict[str, Any]:
path = CACHE_DIR / "jev_runs.json"
path = CACHE_DIR / "jev_runs_v2.json"
if not path.is_file():
return {}
try:
@@ -364,7 +371,7 @@ def load_cache() -> dict[str, Any]:
def save_cache(cache: dict[str, Any]) -> None:
CACHE_DIR.mkdir(parents=True, exist_ok=True)
path = CACHE_DIR / "jev_runs.json"
path = CACHE_DIR / "jev_runs_v2.json"
path.write_text(json.dumps(cache, ensure_ascii=False), encoding="utf-8")
@@ -395,9 +402,15 @@ def write_markdown(report: Mapping[str, Any]) -> None:
f"- 任务：`docs/tasks/TASK-rectification-jev-intent-classifier-research-20260919.md`",
f"- 基线：`origin/staging` @ `{report['meta']['baseline']}`",
f"- Jev 模型：`{JEV_MODEL}`（不用 jev-latest）",
f"- 生成器 / 复核：`{report['meta']['generator']}` / `{report['meta']['reviewer']}`",
f"- 真值 sha256：synthetic `{report['meta']['sha256']['synthetic']}`；simulated `{report['meta']['sha256']['simulated']}`；disputed `{report['meta']['sha256']['disputed']}`",
f"- 来源 B：{report['meta']['source_b_n']} 条（本地，不提交原文）",
f"- 来源 B：{report['meta']['source_b_n']} 条已标注（本地，不提交原文）",
"",
"## 模型",
"",
f"- 生成模型：`{report['meta']['generator']}` / 版本 `{report['meta'].get('generator_version') or report['meta']['generator']}` / 不是线上会话模型（只用于造来源 C）",
f"- 复核模型：`{report['meta']['reviewer']}` / 版本 `{report['meta'].get('reviewer_version') or report['meta']['reviewer']}` / 不是线上会话模型（只用于独立复核，看不到目标标签）",
f"- 对照模型：`{report['meta'].get('current_model') or '—'}` / 版本 `{report['meta'].get('current_model') or '—'}` / **= 线上会话模型**（DeepSeek Flash，顶生产 `classifyRectificationTurnIntent` 提示词）",
f"- Jev：`{JEV_MODEL}` / 不是线上会话模型",
"",
"## 结论",
"",
@@ -535,10 +548,9 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
low_recall = [layers[layer]["low_conf_recall"] for layer in layers if layers[layer]["low_conf_recall"] is not None]
recall_ok = bool(low_recall) and all(v is not None and v >= 0.60 for v in low_recall)
choice_class = layers.get("choice", {}).get("answer_class_acc")
if source_b_n < 30:
represent = "来源 B 不足 30 条，代表性检验只报数、不判定。"
else:
represent = report["metrics"]["representativeness"]["note"]
represent_meta = report["metrics"].get("representativeness") or {}
represent = represent_meta.get("note") or ""
represent_fail = bool(represent_meta.get("fail"))
absolute_fail: list[str] = []
if high_err > 0.03:
absolute_fail.append(f"高置信错误率 {high_err:.1%} > 3%")
@@ -563,6 +575,17 @@ def decide_verdict(report: dict[str, Any]) -> dict[str, str]:
)
if bits:
relative_note = " 相对 −3pp（同一样本）：" + "；".join(bits) + "。"
if represent_fail:
return {
"verdict": "缺数据",
"reason": (
"来源 B 与来源 C 同层 intent 准确率差 > 10pp，模拟语料不代表真人，来源 C 门槛结论降为缺数据。"
+ represent
+ ((" 同时来源 C 绝对门槛未过：" + "；".join(absolute_fail) + "。") if absolute_fail else "")
+ relative_note
),
"if_connect": "不得上线。先补真机样本或重造更像真人的来源 C，再测。",
}
if absolute_fail:
return {
"verdict": "不可接",
@@ -628,6 +651,8 @@ def _merge_report_preds(samples: list[dict[str, Any]]) -> None:
by_id = {row.get("id"): row for row in existing.get("rows") or [] if row.get("id")}
for sample in samples:
prior = by_id.get(sample.get("id")) or {}
if prior.get("user_message") != sample.get("user_message"):
continue
for key in ("jev_1", "jev_2", "current_1", "current_2"):
if key not in sample and prior.get(key):
sample[key] = prior[key]
@@ -666,6 +691,7 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.add_argument("--second-fraction", type=float, default=1.0)
parser.add_argument("--current-only", action="store_true")
parser.add_argument("--sample-fraction", type=float, default=1.0)
parser.add_argument("--current-second-fraction", type=float, default=1.0 / 3)
parser.add_argument("--sample-seed", type=int, default=20260919)
args = parser.parse_args(argv)
if not args.current_only and not os.environ.get("TYPESAFE_API_KEY"):
@@ -676,8 +702,14 @@ def main(argv: Sequence[str] | None = None) -> int:
return 2
synthetic = load_jsonl(SAMPLES_DIR / "synthetic.jsonl")
simulated = load_jsonl(SAMPLES_DIR / "simulated.jsonl")
source_b = load_jsonl(CACHE_DIR / "source_b.jsonl")
source_b_all = load_jsonl(CACHE_DIR / "source_b.jsonl")
source_b = [
row for row in source_b_all
if isinstance(row.get("gold"), dict) and row["gold"].get("intent")
]
unlabeled_b = len(source_b_all) - len(source_b)
samples = [row for row in synthetic + simulated if row.get("source") in {"A", "C"}]
sim_ids = {str(row.get("id")) for row in simulated}
if args.limit:
samples = samples[: args.limit]
_merge_report_preds(samples)
@@ -699,6 +731,8 @@ def main(argv: Sequence[str] | None = None) -> int:
if source_b:
run_batch(source_b, workers=args.workers, run_id="jev_1", cache=cache)
save_cache(cache)
run_batch(source_b, workers=args.workers, run_id="jev_2", cache=cache)
save_cache(cache)
source_c_rows = [row for row in samples if row.get("source") == "C"]
if args.current_only or os.environ.get("DEEPSEEK_API_KEY"):
if args.sample_fraction < 1:
@@ -713,9 +747,21 @@ def main(argv: Sequence[str] | None = None) -> int:
cache=cache, call_fn=call_current_retry,
)
save_cache(cache)
if source_b:
run_batch(
source_b, workers=args.workers, run_id="current_1",
cache=cache, call_fn=call_current_retry,
)
save_cache(cache)
second_current = current_sample
if args.skip_second:
second_current = []
elif args.current_second_fraction < 1:
second_current = stratified_sample(
[row for row in current_sample if row.get("source") == "C"],
fraction=args.current_second_fraction,
seed=args.sample_seed + 1,
)
elif args.second_fraction < 1:
second_current = stratified_sample(
[row for row in current_sample if row.get("source") == "C"],
@@ -752,47 +798,87 @@ def main(argv: Sequence[str] | None = None) -> int:
jev_cons = {layer: self_consistency(rows, "jev_1", "jev_2") for layer, rows in by_layer.items()}
source_a_metrics = layer_metrics(source_a, pred_key="jev_1") if source_a else None
source_b_metrics = layer_metrics(source_b, pred_key="jev_1") if source_b else None
represent_note = "来源 B 不足 30 条，代表性检验只报数、不判定。"
if source_b_metrics and source_b_metrics["n"] >= 30:
deltas = []
source_b_by_layer = {
layer: layer_metrics([row for row in source_b if row.get("layer") == layer], pred_key="jev_1")
for layer in ("choice", "collect", "none")
} if source_b else {}
represent_fail = False
represent_layers: dict[str, Any] = {}
if not source_b or len(source_b) < 30:
represent_note = f"来源 B 已标注 {len(source_b)} 条（未标注 {unlabeled_b} 不进分母），不足 30 条则只报数、不判定。"
else:
bits = []
for layer in ("choice", "collect", "none"):
# source B may not be layered the same; compare overall intent acc
pass
delta = abs((source_b_metrics["intent_acc"] or 0) - (
sum(jev_metrics[layer]["intent_acc"] * jev_metrics[layer]["n"] for layer in jev_metrics)
/ max(sum(jev_metrics[layer]["n"] for layer in jev_metrics), 1)
))
if delta > 0.10:
represent_note = f"来源 B 与来源 C 的 intent 准确率相差 {delta:.1%} > 10pp，模拟语料不代表真人，来源 C 门槛结论降为缺数据。"
b_m = source_b_by_layer.get(layer) or {}
c_m = jev_metrics.get(layer) or {}
if not b_m.get("n"):
represent_layers[layer] = {"n": 0, "delta": None}
continue
delta = abs((b_m.get("intent_acc") or 0) - (c_m.get("intent_acc") or 0))
represent_layers[layer] = {
"n": b_m.get("n"),
"source_b_intent": b_m.get("intent_acc"),
"source_c_intent": c_m.get("intent_acc"),
"delta": delta,
}
bits.append(f"{layer} B {b_m.get('intent_acc'):.1%} vs C {c_m.get('intent_acc'):.1%}（差 {delta:.1%}，n_B={b_m.get('n')}）")
if delta > 0.10:
represent_fail = True
represent_note = "来源 B 与来源 C 同层 intent 准确率：" + "；".join(bits) + "。"
if represent_fail:
represent_note += " 有层差 > 10pp，结论降为缺数据。"
else:
represent_note = f"来源 B 与来源 C intent 准确率相差 {delta:.1%}，未超过 10pp。"
represent_note += " 各层均未超过 10pp。"
sha = {
"synthetic": __import__("hashlib").sha256((SAMPLES_DIR / "synthetic.jsonl").read_bytes()).hexdigest(),
"simulated": __import__("hashlib").sha256((SAMPLES_DIR / "simulated.jsonl").read_bytes()).hexdigest(),
"disputed": __import__("hashlib").sha256((SAMPLES_DIR / "disputed.jsonl").read_bytes()).hexdigest()
if (SAMPLES_DIR / "disputed.jsonl").is_file() else "",
}
gen_name = (simulated[0].get("generator") if simulated else None) or "deepseek-flash"
rev_name = (simulated[0].get("reviewer") if simulated else None) or "deepseek-flash"
cache_ids = {
key.split(":")[1]
for key in cache
if key.startswith("jev_1:C-")
}
sim_ok = sim_ids <= cache_ids or args.current_only
print(json.dumps({
"id_check": {
"simulated": len(sim_ids),
"jev_1_cache_c": len(cache_ids),
"simulated_subset_of_cache": sim_ids <= cache_ids,
"missing": sorted(sim_ids - cache_ids)[:12],
}
}, ensure_ascii=False), flush=True)
report: dict[str, Any] = {
"meta": {
"baseline": "d9722d56",
"baseline": "69a44fe7",
"jev_model": JEV_MODEL,
"generator": "agent-template-v1",
"reviewer": "agent-rule-v1",
"generator": gen_name,
"generator_version": gen_name,
"reviewer": rev_name,
"reviewer_version": rev_name,
"sha256": sha,
"source_b_n": len(source_b),
"source_b_unlabeled": unlabeled_b,
"current_model": current_model_id() if any(row.get("current_1") for row in samples) else None,
"current_model_note": (
(
f"DeepSeek Flash 顶现行 `classifyRectificationTurnIntent` 提示词；"
f"来源 C 分层随机 {args.sample_fraction:.0%}（seed {args.sample_seed}），"
f"来源 A 全量。不是线上会话模型。"
"DeepSeek Flash = 线上会话模型，顶现行 `classifyRectificationTurnIntent` 提示词；"
f"来源 C 全量第一次"
+ (
f"，第二次分层 {args.current_second_fraction:.0%}（seed {args.sample_seed + 1}）"
if args.current_second_fraction < 1 else "，第二次全量"
)
+ "；来源 A 全量；来源 B 已标注全量一次。"
if any(row.get("current_1") for row in samples)
else (
"本机无会话模型目录凭据（模型 key 在数据库加密配置里）。"
"现行 `classifyRectificationTurnIntent` 对照未跑。"
"本机无会话模型凭据，现行对照未跑。"
)
)
),
"id_check_ok": sim_ok,
},
"metrics": {
"jev_by_layer": jev_metrics,
@@ -817,7 +903,12 @@ def main(argv: Sequence[str] | None = None) -> int:
"paired_sample": paired_layer_metrics(samples) if any(row.get("current_1") for row in samples) else {},
"source_a": source_a_metrics,
"source_b": source_b_metrics,
"representativeness": {"note": represent_note},
"source_b_by_layer": source_b_by_layer,
"representativeness": {
"note": represent_note,
"fail": represent_fail,
"layers": represent_layers,
},
"theta": curve_theta([row for row in samples if row.get("source") == "C"]),
"error_portrait": error_portrait(samples),
},