jesse-ux
c2ecbc74cd
来源 C 改为 DeepSeek Flash 生成+独立复核，撤回模板拼接结论。来源 B 157 条人工标注后跑 Jev x2 与 Flash 全量对照，结论为缺数据。
1624 lines
63 KiB
Python
1624 lines
63 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the offline Jev intent-classifier corpus (T0).
|
||
|
||
Source A: ten fixtures derived from the production classifier test file.
|
||
Source B: real staging turns, written outside git (not this script's default).
|
||
Source C: model-simulated replies. Labels are assigned first; an independent
|
||
reviewer that cannot see the target label keeps only agreements.
|
||
|
||
Does not call production `classifyRectificationTurnIntent`. Celebrity names
|
||
from the public holdout file never enter user_message.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import random
|
||
import re
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from collections import Counter
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import date
|
||
from pathlib import Path
|
||
from typing import Any, Iterable, Mapping, Sequence
|
||
from uuid import NAMESPACE_URL, uuid5
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from scripts.rectification.probe_question_contract import ( # noqa: E402
|
||
EXISTENCE_STYLE_OPTIONS,
|
||
QUALITY_STYLE_OPTIONS,
|
||
)
|
||
from scripts.research.jev_intent_questions import ( # noqa: E402
|
||
ANSWER_CLASS_VALUES,
|
||
enforce_combo,
|
||
)
|
||
|
||
HOLDOUT = ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v4.json"
|
||
SAMPLES_DIR = ROOT / "scripts" / "research" / "jev_intent_samples"
|
||
POOL_PATH = SAMPLES_DIR / "question_pool.json"
|
||
SYNTHETIC_PATH = SAMPLES_DIR / "synthetic.jsonl"
|
||
SIMULATED_PATH = SAMPLES_DIR / "simulated.jsonl"
|
||
DISPUTED_PATH = SAMPLES_DIR / "disputed.jsonl"
|
||
README_PATH = SAMPLES_DIR / "README.md"
|
||
CACHE_DIR = Path(r"G:\Ferti\Jyotisha\.cache\jev_intent")
|
||
TODAY = date(2026, 9, 19)
|
||
SEED = 20260919
|
||
PROMPT_VERSION = "gen-review-v3"
|
||
DEFAULT_MODEL = "deepseek-flash"
|
||
DEFAULT_BASE = "https://api.deepseek.com"
|
||
GENERATOR_NAME = os.environ.get("DEEPSEEK_MODEL") or DEFAULT_MODEL
|
||
REVIEWER_NAME = os.environ.get("DEEPSEEK_MODEL") or DEFAULT_MODEL
|
||
DISPUTE_RATE_LIMIT = 0.15
|
||
GEN_TEMPERATURE = 0.85
|
||
REVIEW_TEMPERATURE = 0.0
|
||
MAX_GEN_RETRIES = 3
|
||
|
||
COLLECT_PROMPTS: dict[str, str] = {
|
||
"relationship": "感情这边，还记得哪年认真在一起、分开，或结婚吗？有年份就很好。",
|
||
"career": "工作上呢，还记得哪年入职、换工作，或职责一下子变重吗？",
|
||
"family": "家里如果有结婚、添丁或住院这类事，记得大概哪年就行。",
|
||
"education": "上学这边，还记得哪年升学、转学或大考吗？",
|
||
"relocation": "有没有哪年搬家，或开始长期住在外地？",
|
||
"finance": "钱的方面，还记得哪年收入明显变过、有过大笔支出，或欠过债吗？",
|
||
"health_pressure": "身体或压力这边，还记得哪年生病、受伤，或特别难熬的一段时间吗？",
|
||
}
|
||
COLLECT_RETRY: dict[str, str] = {
|
||
"relationship": "回到感情这边——刚才说的工作我记下了，哪年认真在一起或分开还记得吗？",
|
||
"career": "回到工作这边——刚才那件我记下了，哪年入职或换工作还记得吗？",
|
||
"family": "再问一次家里：结婚、添丁或住院，大概哪年？",
|
||
"education": "回到上学这边——哪年升学、转学或大考，还记得吗？",
|
||
"relocation": "搬家或开始长期住外地，大概是哪年？",
|
||
"finance": "钱的方面再对一下：哪年收入明显变过、有过大笔支出，或欠过债？",
|
||
"health_pressure": "身体或压力这边再问一次：哪年生病、受伤，或特别难熬？",
|
||
}
|
||
TARGETED_YEAR_PROMPT = "大概哪年几月？"
|
||
SHUFFLED_CHOICE_PROMPT = "2023 年前后，工作状态是否出现明显变化？"
|
||
SHUFFLED_OPTIONS = [
|
||
{"key": "A", "label": "这段时间没有明显变化", "answer_class": "no"},
|
||
{"key": "B", "label": "记不清当时的情况", "answer_class": "unsure"},
|
||
{"key": "C", "label": "变化明显而且时间吻合", "answer_class": "yes"},
|
||
{"key": "D", "label": "有变化但程度比较弱", "answer_class": "weak_yes"},
|
||
]
|
||
COLLECT_FINANCE_PROMPT = "钱的方面，还记得哪年收入明显变过吗？"
|
||
|
||
PERSONAS = (
|
||
"terse",
|
||
"rambling",
|
||
"answer_then_event",
|
||
"deny_then_event",
|
||
"off_topic",
|
||
"impatient_stop",
|
||
"ask_result",
|
||
"dialect_netspeak",
|
||
)
|
||
|
||
QUOTAS: dict[str, list[tuple[str, str | None, bool, int]]] = {
|
||
"choice": [
|
||
("answer_current_focus", "yes", False, 45),
|
||
("answer_current_focus", "yes", True, 15),
|
||
("answer_current_focus", "weak_yes", False, 45),
|
||
("answer_current_focus", "weak_yes", True, 15),
|
||
("answer_current_focus", "no", False, 30),
|
||
("answer_current_focus", "no", True, 30),
|
||
("answer_current_focus", "unsure", False, 45),
|
||
("answer_current_focus", "unsure", True, 15),
|
||
("provide_new_evidence", None, True, 60),
|
||
("stop_rectification", None, False, 32),
|
||
("ask_about_result", None, False, 32),
|
||
("unclear", None, False, 36),
|
||
],
|
||
"collect": [
|
||
("answer_current_focus", "no", False, 80),
|
||
("answer_current_focus", "no", True, 20),
|
||
("answer_current_focus", "unsure", False, 80),
|
||
("answer_current_focus", "yes", False, 34),
|
||
("answer_current_focus", "yes", True, 16),
|
||
("answer_current_focus", "weak_yes", False, 33),
|
||
("answer_current_focus", "weak_yes", True, 17),
|
||
("provide_new_evidence", None, True, 48),
|
||
("stop_rectification", None, False, 24),
|
||
("ask_about_result", None, False, 24),
|
||
("unclear", None, False, 24),
|
||
],
|
||
"none": [
|
||
("provide_new_evidence", None, True, 50),
|
||
("stop_rectification", None, False, 15),
|
||
("ask_about_result", None, False, 15),
|
||
("unclear", None, False, 20),
|
||
],
|
||
}
|
||
|
||
PERSONA_SPECS: dict[str, str] = {
|
||
"terse": "惜字如金。句子短，像随手回一句，不用语气词堆砌。",
|
||
"rambling": "口语啰嗦，会绕半句再落到要点，但仍要把必需要素说清楚。少用「嗯/啊/emmm」。",
|
||
"answer_then_event": "先用一两句回答当前问题，再用「另外/对了」补一件别的带年份的事。",
|
||
"deny_then_event": "先明确否定当前问题，再补一件别的带年份的事。",
|
||
"off_topic": "答非所问，说一件无关的小事。不要提校正结果，不要要求停止，不要报带年份的经历。",
|
||
"impatient_stop": "不耐烦，明确说先停、不弄了、停止整套出生时间校正。不要评价当前这件事有没有发生。",
|
||
"ask_result": "追问校正结果、时间范围、候选分钟或好了没。不要回答当前这道题。",
|
||
"dialect_netspeak": "带一点口语或网络说法，但仍能读懂在说什么。不要整句都是无意义的梗。",
|
||
}
|
||
|
||
INTENT_NL: dict[str, str] = {
|
||
"answer_current_focus": "这句话是在直接回答当前这道题，不是另起一件无关的事，不是要求停止整套校正，也不是在问结果。",
|
||
"provide_new_evidence": "这句话没有回答当前这道题，只是另说了一件带大概年份或月份的经历。",
|
||
"stop_rectification": "这句话明确要求停止整个出生时间校正。说当前这件事没发生，不算停止。",
|
||
"ask_about_result": "这句话在问校正结果、时间范围、候选分钟，或好了没有。不是在答当前题。",
|
||
"unclear": "语义不清或答非所问，无法归入回答当前题、补充新经历、停止校正、询问结果。不要提到年份或月份，不要说停，不要问结果。",
|
||
}
|
||
|
||
ANSWER_NL: dict[str | None, str] = {
|
||
"yes": "明确肯定当前问题。点选题对应「明显发生 / 变化明显」；采集题必须用带年月的经历直接回答这道采集题。",
|
||
"weak_yes": "有发生或有变化，但程度弱、不太明显、印象不深。不要说完全没有，也不要说非常明显。",
|
||
"no": "明确否定当前问题：没有发生、没有变化、这方面没什么。不要再补一句「其实也有一点」。",
|
||
"unsure": "记不清、不记得、忘了、想不起来、以后再说。不要给出确定的有或没有。",
|
||
None: "不要对当前问题给出肯定、否定或记不清的态度。",
|
||
}
|
||
|
||
DATED_NL: dict[bool, str] = {
|
||
True: (
|
||
"同一句话里，除了完成上面的意图，还必须另说一件带大概年份的经历，"
|
||
"并且那件事不是对当前问题的直接回答。用「另外 / 对了 / 不过」连接。"
|
||
"必须出现四位数字年份，例如「2019年」或「2019年3月」。没有四位年份就不合格。"
|
||
),
|
||
False: (
|
||
"不要另说一件带年份或月份的新经历。"
|
||
"若当前是采集题且态度是肯定或弱肯定，可以在回答里带上这件事本身的年月，"
|
||
"那是在答当前题，不是另说一件；这种情况下也必须出现四位数字年份。"
|
||
"若不是采集题的肯定回答，不要写任何 19xx / 20xx 年份。"
|
||
),
|
||
}
|
||
|
||
ALT_EVENT_HINTS: dict[str, str] = {
|
||
"education": "工作变动或搬家",
|
||
"career": "搬家或升学",
|
||
"relocation": "换工作或家里的事",
|
||
"relationship": "工作或搬家",
|
||
"family": "工作或搬家",
|
||
"finance": "换工作或搬家",
|
||
"health_pressure": "换工作或搬家",
|
||
}
|
||
|
||
YEAR_RE = re.compile(r"(?:19|20)\d{2}")
|
||
MONTH_RE = re.compile(r"\d{1,2}\s*月")
|
||
|
||
GENERATE_SYSTEM = (
|
||
"你在模拟一位真人用户，正在做出生时间校正问答。只输出用户会打的那句话，"
|
||
"不要解释，不要加引号，不要输出 JSON。"
|
||
)
|
||
|
||
REVIEW_SYSTEM = (
|
||
"你只做出生时间校正当前轮的意图分类。不回答用户，不修改状态。"
|
||
"只输出一个 JSON 对象，不要解释。"
|
||
)
|
||
|
||
REVIEW_RUBRIC = """根据当前问题和用户回复分类。看不到任何预设标签。
|
||
|
||
intent 只能是：
|
||
- answer_current_focus：用户在回答当前问题
|
||
- provide_new_evidence：没有回答当前问题，只补充了一件带大概年或月的经历
|
||
- stop_rectification：明确要求停止整个校正；说当前这件事没发生不是停止
|
||
- ask_about_result：在问结果、范围、候选分钟或好了没
|
||
- unclear：以上都不成立，或混在一起无法单选
|
||
|
||
answer_class：
|
||
- 仅当 intent 是 answer_current_focus 时非空，否则必须为 null
|
||
- 点选题：按选项 label 的含义选 yes / weak_yes / no / unsure，不要按 A/B/C/D 位置猜
|
||
- 采集题：没有/没发生/这方面没什么 → no；记不清/不记得/忘了/想不起来/以后再说 → unsure；
|
||
用带大概年月的经历直接回答当前采集题 → yes（程度弱则为 weak_yes）
|
||
- 不要把「没有」或「记不清」标成 yes
|
||
- 采集题若既没有否定、也没有记不清、也没有带年月经历 → intent 为 unclear，answer_class 为 null
|
||
- 同一句既明确否定当前采集题又补充了新的带时间经历：intent 仍为 answer_current_focus，answer_class 为 no
|
||
|
||
has_new_dated_event：
|
||
- 仅在同一句里除了回答当前问题之外，还提供了新的、带大概时间的经历时为 true
|
||
- 单纯否定、单纯记不清、或只用带年月经历来直接回答当前采集题，都是 false
|
||
- 无当前问题时，带年月的经历 → intent 为 provide_new_evidence，has_new_dated_event 为 true
|
||
|
||
只输出：
|
||
{"intent":"...","answer_class":"yes"|"weak_yes"|"no"|"unsure"|null,"has_new_dated_event":true|false}
|
||
"""
|
||
|
||
_CACHE_LOCK = threading.Lock()
|
||
_LLM_CACHE: dict[str, Any] = {}
|
||
_USAGE = {
|
||
"generate_input_tokens": 0,
|
||
"generate_output_tokens": 0,
|
||
"review_input_tokens": 0,
|
||
"review_output_tokens": 0,
|
||
"generate_calls": 0,
|
||
"review_calls": 0,
|
||
"generate_cached": 0,
|
||
"review_cached": 0,
|
||
}
|
||
|
||
|
||
def current_model_id() -> str:
|
||
return os.environ.get("DEEPSEEK_MODEL") or DEFAULT_MODEL
|
||
|
||
|
||
def sha256_file(path: Path) -> str:
|
||
digest = hashlib.sha256()
|
||
digest.update(path.read_bytes())
|
||
return digest.hexdigest()
|
||
|
||
|
||
def write_jsonl(path: Path, rows: Sequence[Mapping[str, Any]]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
text = "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows)
|
||
path.write_text(text, encoding="utf-8")
|
||
|
||
|
||
def holdout_name_denylist() -> set[str]:
|
||
payload = json.loads(HOLDOUT.read_text(encoding="utf-8"))
|
||
names: set[str] = set()
|
||
for case in payload.get("cases") or []:
|
||
label = str(case.get("subject_label") or "").strip()
|
||
if label:
|
||
names.add(label)
|
||
for part in label.split():
|
||
if len(part) >= 3:
|
||
names.add(part)
|
||
return names
|
||
|
||
|
||
def assert_no_pii(message: str, denylist: set[str]) -> None:
|
||
for name in denylist:
|
||
if name and name in message:
|
||
raise ValueError(f"simulated reply leaked holdout name {name!r}")
|
||
|
||
|
||
def source_a_samples() -> list[dict[str, Any]]:
|
||
"""Ten fixtures from the test file's schema + the two userMessage strings."""
|
||
choice_focus = {
|
||
"kind": "choice",
|
||
"current_question": SHUFFLED_CHOICE_PROMPT,
|
||
"options": SHUFFLED_OPTIONS,
|
||
"case_status": "distinguishing_candidates",
|
||
}
|
||
collect_focus = {
|
||
"kind": "collect",
|
||
"current_question": COLLECT_FINANCE_PROMPT,
|
||
"options": [],
|
||
"case_status": "collecting_evidence",
|
||
}
|
||
rows = [
|
||
("A01", "choice", choice_focus, "没什么变化", "answer_current_focus", "no", False),
|
||
("A02", "choice", choice_focus, "记不清", "answer_current_focus", "unsure", False),
|
||
("A03", "choice", choice_focus, "变化很明显", "answer_current_focus", "yes", False),
|
||
("A04", "choice", choice_focus, "有一点变化", "answer_current_focus", "weak_yes", False),
|
||
("A05", "choice", choice_focus, "没有，不过 2019 年换过岗", "answer_current_focus", "no", True),
|
||
("A06", "choice", choice_focus, "先不弄了", "stop_rectification", None, False),
|
||
("A07", "choice", choice_focus, "结果出来了吗", "ask_about_result", None, False),
|
||
("A08", "collect", collect_focus, "没有", "answer_current_focus", "no", False),
|
||
("A09", "collect", collect_focus, "随便吧", "unclear", None, False),
|
||
("A10", "collect", collect_focus, "2016年3月入学", "answer_current_focus", "yes", False),
|
||
]
|
||
samples: list[dict[str, Any]] = []
|
||
for sid, layer, focus, message, intent, answer_class, dated in rows:
|
||
gold = enforce_combo(intent, answer_class, dated)
|
||
samples.append({
|
||
"id": sid,
|
||
"source": "A",
|
||
"layer": layer,
|
||
"persona": "test_fixture",
|
||
"focus": focus,
|
||
"user_message": message,
|
||
"gold": gold,
|
||
"generator": "test-file",
|
||
"reviewer": "manual",
|
||
})
|
||
return samples
|
||
|
||
|
||
def _hhmm(value: object) -> str | None:
|
||
text = str(value or "")[:5]
|
||
return text if len(text) == 5 and text[2] == ":" else None
|
||
|
||
|
||
def _probe_prompt(probe: Mapping[str, Any]) -> str:
|
||
family = str(probe.get("event_family") or "").replace("？", "").replace("?", "").strip()
|
||
label = str(probe.get("year_label") or "").strip()
|
||
kind = str(probe.get("choice_kind") or "existence")
|
||
meaning = str(probe.get("user_meaning") or "").strip()
|
||
if kind == "event_quality" and meaning:
|
||
from scripts.rectification.probe_question_contract import clipped_probe_label
|
||
clipped = clipped_probe_label(meaning) or meaning
|
||
if clipped.endswith("？") or clipped.endswith("?"):
|
||
return clipped
|
||
return f"{label}，有没有{family}的时候？" if label and family else clipped
|
||
if not family:
|
||
return label
|
||
if kind == "event_quality":
|
||
return f"{label}，有没有{family}的时候？" if label else f"有没有{family}的时候？"
|
||
return f"{label}，有没有{family}？" if label else f"有没有{family}？"
|
||
|
||
|
||
def _probe_options(probe: Mapping[str, Any]) -> list[dict[str, str]]:
|
||
raw = probe.get("style_options") or []
|
||
rows: list[dict[str, str]] = []
|
||
keys = ("A", "B", "C", "D")
|
||
for item in raw:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
answer_class = str(item.get("answer_class") or "")
|
||
label = str(item.get("label") or "").strip()
|
||
if answer_class not in ANSWER_CLASS_VALUES or not label:
|
||
continue
|
||
rows.append({"key": keys[len(rows)], "label": label, "answer_class": answer_class})
|
||
if len(rows) == 4:
|
||
break
|
||
if len(rows) != 4:
|
||
catalog = QUALITY_STYLE_OPTIONS if str(probe.get("choice_kind") or "") == "event_quality" else EXISTENCE_STYLE_OPTIONS
|
||
rows = [
|
||
{"key": keys[i], "label": str(item["label"]), "answer_class": str(item["answer_class"])}
|
||
for i, item in enumerate(catalog)
|
||
]
|
||
return rows
|
||
|
||
|
||
def build_question_pool(*, max_cases: int | None = None) -> dict[str, Any]:
|
||
from scripts.active_rectification_event_engine import AYANAMSA, NODE_MODE
|
||
from scripts.rectification.event_probes import discriminating_event_probes
|
||
from scripts.rectification.refinement_packet import window_scan
|
||
from scripts.rectification.scoring_service import (
|
||
build_event_contribution_matrix,
|
||
score_from_matrix,
|
||
scoreable_request,
|
||
)
|
||
from scripts.research.probe_supply_after_six import request_from_case
|
||
|
||
payload = json.loads(HOLDOUT.read_text(encoding="utf-8"))
|
||
cases = list(payload.get("cases") or [])
|
||
if max_cases is not None:
|
||
cases = cases[:max_cases]
|
||
choice_questions: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
errors: list[str] = []
|
||
for case in cases:
|
||
case_id = str(case.get("case_id") or "case")
|
||
try:
|
||
request = request_from_case(case)
|
||
request["ayanamsa"] = AYANAMSA
|
||
request["node_mode"] = NODE_MODE
|
||
scoring = scoreable_request(request)
|
||
built = build_event_contribution_matrix(scoring)
|
||
rows = score_from_matrix(scoring, built)
|
||
times = [stamp for row in rows if (stamp := _hhmm(row.get("time")))]
|
||
true_time = str(case["birth"]["time"])[:5]
|
||
probes = discriminating_event_probes(
|
||
{**request, "refresh_probes": False, "asked_probe_keys": []},
|
||
built,
|
||
scan=window_scan(built),
|
||
candidate_times=times,
|
||
representative_time=true_time,
|
||
today=TODAY,
|
||
)
|
||
except Exception as exc: # noqa: BLE001 — keep the other 19 cases
|
||
errors.append(f"{case_id}: {type(exc).__name__}")
|
||
continue
|
||
kept = 0
|
||
for probe in probes:
|
||
if not isinstance(probe, dict):
|
||
continue
|
||
prompt = _probe_prompt(probe)
|
||
options = _probe_options(probe)
|
||
key = f"{prompt}|{','.join(item['answer_class'] + ':' + item['label'] for item in options)}"
|
||
if key in seen or not prompt:
|
||
continue
|
||
seen.add(key)
|
||
choice_questions.append({
|
||
"id": str(uuid5(NAMESPACE_URL, key)),
|
||
"layer": "choice",
|
||
"current_question": prompt,
|
||
"options": options,
|
||
"domain": str(probe.get("domain") or ""),
|
||
"choice_kind": str(probe.get("choice_kind") or "existence"),
|
||
"year_label": str(probe.get("year_label") or ""),
|
||
"event_family": str(probe.get("event_family") or ""),
|
||
"case_status": "distinguishing_candidates",
|
||
})
|
||
kept += 1
|
||
print(f"pool {case_id} probes={len(probes)} unique={kept}", flush=True)
|
||
collect_questions: list[dict[str, Any]] = []
|
||
for domain, prompt in COLLECT_PROMPTS.items():
|
||
collect_questions.append({
|
||
"id": f"collect:{domain}",
|
||
"layer": "collect",
|
||
"current_question": prompt,
|
||
"options": [],
|
||
"domain": domain,
|
||
"case_status": "collecting_evidence",
|
||
})
|
||
collect_questions.append({
|
||
"id": f"collect-retry:{domain}",
|
||
"layer": "collect",
|
||
"current_question": COLLECT_RETRY[domain],
|
||
"options": [],
|
||
"domain": domain,
|
||
"case_status": "collecting_evidence",
|
||
})
|
||
collect_questions.append({
|
||
"id": "collect:targeted-year",
|
||
"layer": "collect",
|
||
"current_question": TARGETED_YEAR_PROMPT,
|
||
"options": [],
|
||
"domain": "career",
|
||
"case_status": "collecting_evidence",
|
||
})
|
||
none_questions = [{
|
||
"id": "none:open",
|
||
"layer": "none",
|
||
"current_question": "",
|
||
"options": [],
|
||
"domain": "",
|
||
"case_status": "collecting_evidence",
|
||
}]
|
||
pool = {
|
||
"choice": choice_questions,
|
||
"collect": collect_questions,
|
||
"none": none_questions,
|
||
"errors": errors,
|
||
"holdout": str(HOLDOUT.relative_to(ROOT)).replace("\\", "/"),
|
||
"generator_note": "choice prompts from discriminating_event_probes; collect copy from user-copy.ts",
|
||
}
|
||
return pool
|
||
|
||
|
||
def _option_labels(focus: Mapping[str, Any]) -> list[str]:
|
||
return [str(item.get("label") or "") for item in (focus.get("options") or []) if item.get("label")]
|
||
|
||
|
||
def gold_equal(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool:
|
||
return (
|
||
left.get("intent") == right.get("intent")
|
||
and left.get("answer_class") == right.get("answer_class")
|
||
and bool(left.get("has_new_dated_event")) == bool(right.get("has_new_dated_event"))
|
||
)
|
||
|
||
|
||
def _pick_question(pool: Mapping[str, Any], layer: str, rng: random.Random) -> dict[str, Any]:
|
||
rows = list(pool.get(layer) or [])
|
||
if not rows:
|
||
raise RuntimeError(f"empty question pool for layer={layer}")
|
||
return dict(rng.choice(rows))
|
||
|
||
|
||
def _has_year_or_month(message: str) -> bool:
|
||
return bool(YEAR_RE.search(message) or MONTH_RE.search(message))
|
||
|
||
|
||
def llm_cache_path() -> Path:
|
||
return CACHE_DIR / "llm_cache.json"
|
||
|
||
|
||
def load_llm_cache() -> None:
|
||
path = llm_cache_path()
|
||
if not path.is_file():
|
||
return
|
||
try:
|
||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError:
|
||
return
|
||
if isinstance(payload, dict):
|
||
_LLM_CACHE.update(payload)
|
||
|
||
|
||
def save_llm_cache() -> None:
|
||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||
with _CACHE_LOCK:
|
||
llm_cache_path().write_text(json.dumps(_LLM_CACHE, ensure_ascii=False), encoding="utf-8")
|
||
(CACHE_DIR / "llm_usage.json").write_text(json.dumps(_USAGE, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
|
||
def _parse_model_json(text: str) -> dict[str, Any]:
|
||
raw = (text or "").strip()
|
||
if raw.startswith("```"):
|
||
raw = raw.strip("`")
|
||
if raw.lower().startswith("json"):
|
||
raw = raw[4:]
|
||
raw = raw.strip()
|
||
start = raw.find("{")
|
||
end = raw.rfind("}")
|
||
if start < 0 or end <= start:
|
||
raise ValueError("no_json_object")
|
||
payload = json.loads(raw[start:end + 1])
|
||
if not isinstance(payload, dict):
|
||
raise ValueError("json_not_object")
|
||
return payload
|
||
|
||
|
||
def _strip_reply(text: str) -> str:
|
||
raw = (text or "").strip()
|
||
if raw.startswith("```"):
|
||
raw = raw.strip("`").strip()
|
||
if raw.lower().startswith("text"):
|
||
raw = raw[4:].strip()
|
||
if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {'"', "“", "'"}:
|
||
raw = raw[1:-1].strip()
|
||
return " ".join(raw.split())
|
||
|
||
|
||
def complete_chat(
|
||
*,
|
||
system: str,
|
||
user: str,
|
||
temperature: float,
|
||
json_mode: bool,
|
||
cache_ns: str,
|
||
max_tokens: int = 256,
|
||
timeout: float = 60.0,
|
||
) -> dict[str, Any]:
|
||
model = current_model_id()
|
||
key_payload = {
|
||
"ns": cache_ns,
|
||
"prompt_version": PROMPT_VERSION,
|
||
"model": model,
|
||
"temperature": temperature,
|
||
"json_mode": json_mode,
|
||
"system": system,
|
||
"user": user,
|
||
}
|
||
cache_key = hashlib.sha256(json.dumps(key_payload, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest()
|
||
with _CACHE_LOCK:
|
||
cached = _LLM_CACHE.get(cache_key)
|
||
if isinstance(cached, dict) and cached.get("text"):
|
||
_USAGE[f"{cache_ns}_cached"] = int(_USAGE.get(f"{cache_ns}_cached") or 0) + 1
|
||
return cached
|
||
api_key = os.environ.get("DEEPSEEK_API_KEY") or ""
|
||
if not api_key:
|
||
raise RuntimeError("DEEPSEEK_API_KEY missing")
|
||
base = (os.environ.get("DEEPSEEK_BASE_URL") or DEFAULT_BASE).rstrip("/")
|
||
body: dict[str, Any] = {
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"thinking": {"type": "disabled"},
|
||
"stream": False,
|
||
}
|
||
if json_mode:
|
||
body["response_format"] = {"type": "json_object"}
|
||
request = urllib.request.Request(
|
||
f"{base}/chat/completions",
|
||
data=json.dumps(body).encode("utf-8"),
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {api_key}",
|
||
},
|
||
method="POST",
|
||
)
|
||
delay = 1.0
|
||
last_error = "unknown"
|
||
for _attempt in range(6):
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
payload = json.loads(response.read().decode("utf-8"))
|
||
content = (((payload.get("choices") or [{}])[0].get("message") or {}).get("content")) or ""
|
||
usage = payload.get("usage") or {}
|
||
result = {
|
||
"text": content,
|
||
"model": payload.get("model") or model,
|
||
"input_tokens": usage.get("prompt_tokens") or usage.get("input_tokens") or 0,
|
||
"output_tokens": usage.get("completion_tokens") or usage.get("output_tokens") or 0,
|
||
}
|
||
with _CACHE_LOCK:
|
||
_LLM_CACHE[cache_key] = result
|
||
_USAGE[f"{cache_ns}_calls"] = int(_USAGE.get(f"{cache_ns}_calls") or 0) + 1
|
||
_USAGE[f"{cache_ns}_input_tokens"] = int(_USAGE.get(f"{cache_ns}_input_tokens") or 0) + int(result["input_tokens"] or 0)
|
||
_USAGE[f"{cache_ns}_output_tokens"] = int(_USAGE.get(f"{cache_ns}_output_tokens") or 0) + int(result["output_tokens"] or 0)
|
||
return result
|
||
except urllib.error.HTTPError as exc:
|
||
last_error = f"HTTP{exc.code}"
|
||
if exc.code in {429, 500, 502, 503, 529}:
|
||
time.sleep(delay)
|
||
delay = min(delay * 2, 20)
|
||
continue
|
||
raise
|
||
except Exception as exc: # noqa: BLE001
|
||
last_error = type(exc).__name__
|
||
time.sleep(delay)
|
||
delay = min(delay * 2, 20)
|
||
raise RuntimeError(f"chat_failed:{last_error}")
|
||
|
||
|
||
def build_generate_prompt(
|
||
question: Mapping[str, Any],
|
||
*,
|
||
intent: str,
|
||
answer_class: str | None,
|
||
dated: bool,
|
||
persona: str,
|
||
length_lo: int,
|
||
length_hi: int,
|
||
) -> str:
|
||
options = list(question.get("options") or [])
|
||
option_lines = []
|
||
for item in options:
|
||
option_lines.append(
|
||
f"- {item.get('key')}. {item.get('label')}（对应态度：{item.get('answer_class')}）"
|
||
)
|
||
option_block = "\n".join(option_lines) if option_lines else "（本题没有点选选项）"
|
||
extra = ""
|
||
if dated:
|
||
extra = f"另说的那件事请用「{ALT_EVENT_HINTS.get(str(question.get('domain') or ''), '另一件生活事')}」，不要跟当前问题同一件事。"
|
||
return f"""当前问题：
|
||
{question.get('current_question') or '（当前没有焦点问题）'}
|
||
|
||
选项：
|
||
{option_block}
|
||
|
||
你必须体现的语义（不要写出这些标签名，不要解释标签）：
|
||
- 这句话的作用：{INTENT_NL[intent]}
|
||
- 对当前问题的态度：{ANSWER_NL[answer_class]}
|
||
- 是否另说一件带年月的新经历：{DATED_NL[dated]}
|
||
{extra}
|
||
|
||
人设：{PERSONA_SPECS[persona]}
|
||
|
||
硬约束：
|
||
- 只输出用户会打的那句话
|
||
- 字数约 {length_lo}–{length_hi} 个汉字（含标点），不要明显超出
|
||
- 不得逐字复述上面的选项原文
|
||
- 不得出现真实姓名、地名、学校名、公司名、医院名、单位名
|
||
- 不要用「有的 / 发生了 / 就是那段 / 没这回事」这种套话，写得像真人随手打的
|
||
- 需要带时间时，用大概年份或年月，事件用普通生活经历
|
||
"""
|
||
|
||
|
||
def build_review_prompt(sample: Mapping[str, Any]) -> str:
|
||
focus = sample.get("focus") if isinstance(sample.get("focus"), dict) else {}
|
||
options = list(focus.get("options") or [])
|
||
option_lines = []
|
||
for item in options:
|
||
option_lines.append(f"- {item.get('key')}. {item.get('label')} (answer_class={item.get('answer_class')})")
|
||
option_block = "\n".join(option_lines) if option_lines else "（无选项）"
|
||
return f"""{REVIEW_RUBRIC}
|
||
|
||
当前问题：
|
||
{focus.get('current_question') or '（无）'}
|
||
|
||
选项：
|
||
{option_block}
|
||
|
||
用户回复：
|
||
{sample.get('user_message') or ''}
|
||
"""
|
||
|
||
|
||
def review_sample(sample: Mapping[str, Any]) -> dict[str, Any]:
|
||
"""Independent pass: model JSON label, no access to target labels."""
|
||
prompt = build_review_prompt(sample)
|
||
raw = complete_chat(
|
||
system=REVIEW_SYSTEM,
|
||
user=prompt,
|
||
temperature=REVIEW_TEMPERATURE,
|
||
json_mode=True,
|
||
cache_ns="review",
|
||
max_tokens=256,
|
||
)
|
||
parsed = _parse_model_json(raw["text"])
|
||
return enforce_combo(
|
||
parsed.get("intent"),
|
||
parsed.get("answer_class"),
|
||
parsed.get("has_new_dated_event"),
|
||
)
|
||
|
||
|
||
def _message_ok(
|
||
message: str,
|
||
*,
|
||
question: Mapping[str, Any],
|
||
intent: str,
|
||
answer_class: str | None,
|
||
dated: bool,
|
||
length_lo: int,
|
||
length_hi: int,
|
||
denylist: set[str],
|
||
) -> str | None:
|
||
if not message or len(message) < 2:
|
||
return "too_short"
|
||
try:
|
||
assert_no_pii(message, denylist)
|
||
except ValueError:
|
||
return "pii"
|
||
for label in _option_labels(question):
|
||
if label and label in message:
|
||
return "copied_option"
|
||
n = len(message)
|
||
if n < max(6, length_lo - 8) or n > length_hi + 24:
|
||
return "length"
|
||
has_date = _has_year_or_month(message)
|
||
collect_yes = (
|
||
str(question.get("layer") or "") == "collect"
|
||
and intent == "answer_current_focus"
|
||
and answer_class in {"yes", "weak_yes"}
|
||
)
|
||
if dated and not has_date:
|
||
return "missing_date"
|
||
if intent == "provide_new_evidence" and not has_date:
|
||
return "missing_date"
|
||
if collect_yes and not has_date:
|
||
return "collect_yes_needs_date"
|
||
if intent in {"stop_rectification", "ask_about_result", "unclear"} and not dated and has_date:
|
||
return "unexpected_date"
|
||
if intent == "answer_current_focus" and answer_class in {"no", "unsure"} and not dated and has_date:
|
||
return "unexpected_date"
|
||
return None
|
||
|
||
|
||
def generate_reply(
|
||
question: Mapping[str, Any],
|
||
*,
|
||
intent: str,
|
||
answer_class: str | None,
|
||
dated: bool,
|
||
persona: str,
|
||
length_lo: int,
|
||
length_hi: int,
|
||
attempt: int,
|
||
last_reason: str | None = None,
|
||
) -> str:
|
||
user = build_generate_prompt(
|
||
question,
|
||
intent=intent,
|
||
answer_class=answer_class,
|
||
dated=dated,
|
||
persona=persona,
|
||
length_lo=length_lo,
|
||
length_hi=length_hi,
|
||
)
|
||
if dated or intent == "provide_new_evidence" or (
|
||
str(question.get("layer") or "") == "collect"
|
||
and intent == "answer_current_focus"
|
||
and answer_class in {"yes", "weak_yes"}
|
||
):
|
||
user += "\n再次强调：回复里必须出现一个四位数字年份（19xx 或 20xx）。"
|
||
if attempt:
|
||
user += f"\n这是第 {attempt + 1} 次重写，请换一种说法，不要重复上次的句子。"
|
||
if last_reason in {"missing_date", "collect_yes_needs_date"}:
|
||
user += "上次缺少四位年份，这次必须写上例如 2017年3月。"
|
||
elif last_reason == "unexpected_date":
|
||
user += "上次多写了年份，这次不要出现 19xx 或 20xx。"
|
||
elif last_reason == "copied_option":
|
||
user += "上次逐字复述了选项原文，这次换自己的话。"
|
||
elif last_reason == "length":
|
||
user += "上次字数不合要求，这次严格落在指定字数区间。"
|
||
raw = complete_chat(
|
||
system=GENERATE_SYSTEM,
|
||
user=user,
|
||
temperature=GEN_TEMPERATURE,
|
||
json_mode=False,
|
||
cache_ns="generate",
|
||
max_tokens=220,
|
||
)
|
||
return _strip_reply(raw["text"])
|
||
|
||
|
||
def length_stats(messages: Iterable[str]) -> dict[str, float]:
|
||
lengths = sorted(len(item) for item in messages)
|
||
if not lengths:
|
||
return {"n": 0, "p25": 0, "median": 0, "p75": 0}
|
||
|
||
def pct(p: float) -> float:
|
||
if len(lengths) == 1:
|
||
return float(lengths[0])
|
||
idx = int(round((len(lengths) - 1) * p))
|
||
return float(lengths[idx])
|
||
|
||
return {
|
||
"n": len(lengths),
|
||
"p25": pct(0.25),
|
||
"median": pct(0.5),
|
||
"p75": pct(0.75),
|
||
"min": float(lengths[0]),
|
||
"max": float(lengths[-1]),
|
||
}
|
||
|
||
|
||
def quota_deviation(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
by_layer: dict[str, Counter[tuple[str, str | None, bool]]] = {
|
||
"choice": Counter(),
|
||
"collect": Counter(),
|
||
"none": Counter(),
|
||
}
|
||
for row in rows:
|
||
layer = str(row.get("layer") or "")
|
||
gold = row.get("gold") or {}
|
||
if layer not in by_layer:
|
||
continue
|
||
by_layer[layer][(
|
||
str(gold.get("intent")),
|
||
gold.get("answer_class"),
|
||
bool(gold.get("has_new_dated_event")),
|
||
)] += 1
|
||
report: dict[str, Any] = {}
|
||
for layer, spec in QUOTAS.items():
|
||
target_total = sum(item[3] for item in spec)
|
||
actual_total = sum(by_layer[layer].values())
|
||
cells = []
|
||
for intent, answer_class, dated, count in spec:
|
||
actual = by_layer[layer][(intent, answer_class, dated)]
|
||
share_target = count / target_total if target_total else 0
|
||
share_actual = actual / actual_total if actual_total else 0
|
||
cells.append({
|
||
"intent": intent,
|
||
"answer_class": answer_class,
|
||
"has_new_dated_event": dated,
|
||
"target": count,
|
||
"actual": actual,
|
||
"share_delta": share_actual - share_target,
|
||
})
|
||
max_delta = max((abs(cell["share_delta"]) for cell in cells), default=0.0)
|
||
report[layer] = {
|
||
"target_total": target_total,
|
||
"actual_total": actual_total,
|
||
"max_share_delta": max_delta,
|
||
"within_20pct": max_delta <= 0.20,
|
||
"cells": cells,
|
||
}
|
||
return report
|
||
|
||
|
||
def persona_stats(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
scoped = [row for row in rows if row.get("layer") in {"choice", "collect"}]
|
||
counts = Counter(str(row.get("persona") or "") for row in scoped)
|
||
terse_dialect = counts.get("terse", 0) + counts.get("dialect_netspeak", 0)
|
||
n = max(len(rows), 1)
|
||
n_scoped = max(len(scoped), 1)
|
||
return {
|
||
"choice_collect": dict(counts),
|
||
"all": dict(Counter(str(row.get("persona") or "") for row in rows)),
|
||
"each_ge_40": all(counts.get(name, 0) >= 40 for name in PERSONAS),
|
||
"terse_dialect_share_all": (Counter(str(row.get("persona") or "") for row in rows).get("terse", 0)
|
||
+ Counter(str(row.get("persona") or "") for row in rows).get("dialect_netspeak", 0)) / n,
|
||
"terse_dialect_share_choice_collect": terse_dialect / n_scoped,
|
||
"min_choice_collect": min((counts.get(name, 0) for name in PERSONAS), default=0),
|
||
}
|
||
|
||
|
||
def uniqueness_stats(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
messages = [str(row.get("user_message") or "") for row in rows]
|
||
counts = Counter(messages)
|
||
return {
|
||
"n": len(messages),
|
||
"unique": len(counts),
|
||
"max_repeat": max(counts.values()) if counts else 0,
|
||
"unique_ok": len(counts) >= 810,
|
||
"repeat_ok": (max(counts.values()) if counts else 0) <= 5,
|
||
}
|
||
|
||
|
||
def length_ok(stats: Mapping[str, Any]) -> bool:
|
||
return 8 <= float(stats.get("p25") or 0) <= 16 and 15 <= float(stats.get("median") or 0) <= 27 and 20 <= float(stats.get("p75") or 0) <= 36
|
||
|
||
|
||
def _compatible_personas(intent: str, dated: bool) -> list[str]:
|
||
names = ["rambling", "terse", "dialect_netspeak"]
|
||
if intent == "stop_rectification":
|
||
names.append("impatient_stop")
|
||
if intent == "ask_about_result":
|
||
names.append("ask_result")
|
||
if intent == "unclear":
|
||
names.append("off_topic")
|
||
if intent == "answer_current_focus" and dated:
|
||
names.append("answer_then_event")
|
||
if intent == "answer_current_focus" and dated:
|
||
names.append("deny_then_event")
|
||
return names
|
||
|
||
|
||
def _assign_persona(intent: str, dated: bool, answer_class: str | None, rng: random.Random, remaining: Counter[str]) -> str:
|
||
preferred: list[str] = []
|
||
if intent == "stop_rectification" and remaining["impatient_stop"] > 0:
|
||
preferred.append("impatient_stop")
|
||
if intent == "ask_about_result" and remaining["ask_result"] > 0:
|
||
preferred.append("ask_result")
|
||
if intent == "unclear" and remaining["off_topic"] > 0:
|
||
preferred.append("off_topic")
|
||
if intent == "answer_current_focus" and dated and answer_class == "no" and remaining["deny_then_event"] > 0:
|
||
preferred.append("deny_then_event")
|
||
if intent == "answer_current_focus" and dated and remaining["answer_then_event"] > 0:
|
||
preferred.append("answer_then_event")
|
||
if preferred:
|
||
name = preferred[0]
|
||
remaining[name] -= 1
|
||
return name
|
||
weighted: list[str] = []
|
||
for name, weight in (("rambling", 5), ("terse", 2), ("dialect_netspeak", 2)):
|
||
if remaining[name] > 0:
|
||
weighted.extend([name] * weight)
|
||
if not weighted:
|
||
weighted = [name for name in _compatible_personas(intent, dated)]
|
||
name = rng.choice(weighted)
|
||
if remaining[name] > 0:
|
||
remaining[name] -= 1
|
||
return name
|
||
|
||
|
||
def _length_bounds(persona: str, rng: random.Random) -> tuple[int, int]:
|
||
if persona == "terse":
|
||
lo = rng.randint(8, 12)
|
||
return lo, min(16, lo + rng.randint(3, 6))
|
||
if persona == "rambling":
|
||
lo = rng.randint(22, 28)
|
||
return lo, min(36, lo + rng.randint(4, 8))
|
||
if persona == "dialect_netspeak":
|
||
lo = rng.randint(12, 18)
|
||
return lo, min(30, lo + rng.randint(6, 10))
|
||
lo = rng.randint(12, 18)
|
||
return lo, min(32, lo + rng.randint(6, 12))
|
||
|
||
|
||
def plan_slots(pool: Mapping[str, Any], *, seed: int, extra_factor: float) -> list[dict[str, Any]]:
|
||
rng = random.Random(seed)
|
||
remaining = Counter({
|
||
"terse": 80,
|
||
"dialect_netspeak": 80,
|
||
"rambling": 260,
|
||
"answer_then_event": 120,
|
||
"deny_then_event": 50,
|
||
"off_topic": 48,
|
||
"impatient_stop": 48,
|
||
"ask_result": 48,
|
||
})
|
||
slots: list[dict[str, Any]] = []
|
||
counters = {"choice": 0, "collect": 0, "none": 0}
|
||
for layer, rows in QUOTAS.items():
|
||
for intent, answer_class, dated, count in rows:
|
||
need = int(round(count * extra_factor))
|
||
for _ in range(need):
|
||
counters[layer] += 1
|
||
persona = _assign_persona(intent, dated, answer_class, rng, remaining)
|
||
lo, hi = _length_bounds(persona, rng)
|
||
question = _pick_question(pool, layer, rng)
|
||
slots.append({
|
||
"id": f"C-{layer}-{counters[layer]:04d}",
|
||
"layer": layer,
|
||
"intent": intent,
|
||
"answer_class": answer_class,
|
||
"dated": dated,
|
||
"persona": persona,
|
||
"length_lo": lo,
|
||
"length_hi": hi,
|
||
"question": question,
|
||
})
|
||
rng.shuffle(slots)
|
||
return slots
|
||
|
||
|
||
def _sample_from_slot(slot: Mapping[str, Any], message: str, reviewed: Mapping[str, Any] | None) -> dict[str, Any]:
|
||
question = slot["question"]
|
||
layer = str(slot["layer"])
|
||
gold = enforce_combo(slot["intent"], slot["answer_class"], slot["dated"])
|
||
focus = {
|
||
"kind": layer if layer != "none" else None,
|
||
"current_question": str(question.get("current_question") or ""),
|
||
"options": list(question.get("options") or []),
|
||
"case_status": str(question.get("case_status") or "collecting_evidence"),
|
||
}
|
||
row = {
|
||
"id": slot["id"],
|
||
"source": "C",
|
||
"layer": layer,
|
||
"persona": slot["persona"],
|
||
"focus": focus,
|
||
"user_message": message,
|
||
"gold": gold,
|
||
"generator": GENERATOR_NAME,
|
||
"reviewer": REVIEWER_NAME,
|
||
"target_before_review": gold,
|
||
"prompt_version": PROMPT_VERSION,
|
||
}
|
||
if reviewed is not None:
|
||
row["review"] = reviewed
|
||
return row
|
||
|
||
|
||
def fill_slot(
|
||
slot: Mapping[str, Any],
|
||
*,
|
||
denylist: set[str],
|
||
) -> tuple[dict[str, Any], bool]:
|
||
last_sample: dict[str, Any] | None = None
|
||
last_reason: str | None = None
|
||
for attempt in range(MAX_GEN_RETRIES):
|
||
try:
|
||
message = generate_reply(
|
||
slot["question"],
|
||
intent=slot["intent"],
|
||
answer_class=slot["answer_class"],
|
||
dated=slot["dated"],
|
||
persona=slot["persona"],
|
||
length_lo=int(slot["length_lo"]),
|
||
length_hi=int(slot["length_hi"]),
|
||
attempt=attempt,
|
||
last_reason=last_reason,
|
||
)
|
||
except Exception as exc: # noqa: BLE001
|
||
last_sample = _sample_from_slot(slot, f"[generate_error:{type(exc).__name__}]", None)
|
||
continue
|
||
reason = _message_ok(
|
||
message,
|
||
question=slot["question"],
|
||
intent=slot["intent"],
|
||
answer_class=slot["answer_class"],
|
||
dated=slot["dated"],
|
||
length_lo=int(slot["length_lo"]),
|
||
length_hi=int(slot["length_hi"]),
|
||
denylist=denylist,
|
||
)
|
||
if reason:
|
||
last_reason = reason
|
||
last_sample = _sample_from_slot(slot, message, None)
|
||
last_sample["reject_reason"] = reason
|
||
continue
|
||
sample = _sample_from_slot(slot, message, None)
|
||
try:
|
||
reviewed = review_sample(sample)
|
||
except Exception as exc: # noqa: BLE001
|
||
sample["reject_reason"] = f"review_error:{type(exc).__name__}"
|
||
last_sample = sample
|
||
continue
|
||
sample["review"] = reviewed
|
||
if gold_equal(sample["gold"], reviewed):
|
||
return sample, True
|
||
last_sample = sample
|
||
if last_sample is None:
|
||
last_sample = _sample_from_slot(slot, "", None)
|
||
return last_sample, False
|
||
|
||
|
||
def progress_paths() -> tuple[Path, Path]:
|
||
return CACHE_DIR / "source_c_accepted.jsonl", CACHE_DIR / "source_c_disputed.jsonl"
|
||
|
||
|
||
def load_progress() -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]:
|
||
accepted_path, disputed_path = progress_paths()
|
||
accepted: dict[str, dict[str, Any]] = {}
|
||
disputed: dict[str, dict[str, Any]] = {}
|
||
for path, dest in ((accepted_path, accepted), (disputed_path, disputed)):
|
||
if not path.is_file():
|
||
continue
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
if not line.strip():
|
||
continue
|
||
row = json.loads(line)
|
||
if row.get("prompt_version") != PROMPT_VERSION:
|
||
continue
|
||
dest[str(row.get("id"))] = row
|
||
return accepted, disputed
|
||
|
||
|
||
def append_progress(path: Path, row: Mapping[str, Any]) -> None:
|
||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||
with _CACHE_LOCK:
|
||
with path.open("a", encoding="utf-8") as handle:
|
||
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
|
||
|
||
|
||
def select_simulated(accepted_rows: Sequence[Mapping[str, Any]], *, seed: int) -> list[dict[str, Any]]:
|
||
rng = random.Random(seed + 17)
|
||
style = {"terse", "dialect_netspeak"}
|
||
by_bucket: dict[tuple[str, str, str | None, bool], list[dict[str, Any]]] = {}
|
||
for sample in accepted_rows:
|
||
gold = sample.get("gold") or {}
|
||
key = (
|
||
str(sample.get("layer")),
|
||
str(gold.get("intent")),
|
||
gold.get("answer_class"),
|
||
bool(gold.get("has_new_dated_event")),
|
||
)
|
||
by_bucket.setdefault(key, []).append(dict(sample))
|
||
for bucket in by_bucket.values():
|
||
rng.shuffle(bucket)
|
||
selected: list[dict[str, Any]] = []
|
||
used_ids: set[str] = set()
|
||
persona_all: Counter[str] = Counter()
|
||
persona_cc: Counter[str] = Counter()
|
||
message_kept: Counter[str] = Counter()
|
||
|
||
def bucket_key(row: Mapping[str, Any]) -> tuple[str, str, str | None, bool]:
|
||
gold = row.get("gold") or {}
|
||
return (
|
||
str(row.get("layer")),
|
||
str(gold.get("intent")),
|
||
gold.get("answer_class"),
|
||
bool(gold.get("has_new_dated_event")),
|
||
)
|
||
|
||
def can_take(row: Mapping[str, Any], *, allow_style: bool, cap_style: bool) -> bool:
|
||
persona = str(row.get("persona") or "")
|
||
message = str(row.get("user_message") or "")
|
||
if not message or row.get("id") in used_ids:
|
||
return False
|
||
if message_kept[message] >= 5:
|
||
return False
|
||
if persona in style and not allow_style:
|
||
return False
|
||
if cap_style and persona in style and persona_all[persona] >= 80:
|
||
return False
|
||
if cap_style and persona in style and persona_all["terse"] + persona_all["dialect_netspeak"] >= 180:
|
||
return False
|
||
return True
|
||
|
||
def commit(row: Mapping[str, Any]) -> None:
|
||
selected.append(dict(row))
|
||
used_ids.add(str(row["id"]))
|
||
persona = str(row.get("persona") or "")
|
||
persona_all[persona] += 1
|
||
if row.get("layer") in {"choice", "collect"}:
|
||
persona_cc[persona] += 1
|
||
message_kept[str(row.get("user_message") or "")] += 1
|
||
|
||
def uncommit(index: int) -> dict[str, Any]:
|
||
row = selected.pop(index)
|
||
used_ids.discard(str(row["id"]))
|
||
persona = str(row.get("persona") or "")
|
||
persona_all[persona] -= 1
|
||
if row.get("layer") in {"choice", "collect"}:
|
||
persona_cc[persona] -= 1
|
||
message_kept[str(row.get("user_message") or "")] -= 1
|
||
return row
|
||
|
||
for layer, rows in QUOTAS.items():
|
||
for intent, answer_class, dated, count in rows:
|
||
bucket = list(by_bucket.get((layer, intent, answer_class, dated), []))
|
||
taken = 0
|
||
for allow_style in (False, True):
|
||
for item in bucket:
|
||
if taken >= count:
|
||
break
|
||
if can_take(item, allow_style=allow_style, cap_style=False):
|
||
commit(item)
|
||
taken += 1
|
||
layer_target = {layer: sum(cell[3] for cell in spec) for layer, spec in QUOTAS.items()}
|
||
leftovers = [row for row in accepted_rows if row.get("id") not in used_ids]
|
||
rng.shuffle(leftovers)
|
||
for allow_style in (False, True):
|
||
for item in leftovers:
|
||
if len(selected) >= 900:
|
||
break
|
||
layer = str(item.get("layer") or "none")
|
||
layer_n = sum(1 for row in selected if row.get("layer") == layer)
|
||
if layer_n >= layer_target.get(layer, 0):
|
||
continue
|
||
if can_take(item, allow_style=allow_style, cap_style=True):
|
||
commit(item)
|
||
for allow_style in (False, True):
|
||
for item in leftovers:
|
||
if len(selected) >= 900:
|
||
break
|
||
if can_take(item, allow_style=allow_style, cap_style=True):
|
||
commit(item)
|
||
|
||
def leftovers_now() -> list[dict[str, Any]]:
|
||
return [row for row in accepted_rows if row.get("id") not in used_ids]
|
||
|
||
def find_replacement(*, persona_name: str | None, non_style: bool, layer: str | None, same_bucket: tuple | None) -> dict[str, Any] | None:
|
||
for item in leftovers_now():
|
||
if message_kept[str(item.get("user_message") or "")] >= 5:
|
||
continue
|
||
if persona_name and str(item.get("persona") or "") != persona_name:
|
||
continue
|
||
if non_style and str(item.get("persona") or "") in style:
|
||
continue
|
||
if layer and str(item.get("layer") or "") != layer:
|
||
continue
|
||
if same_bucket and bucket_key(item) != same_bucket:
|
||
continue
|
||
return item
|
||
return None
|
||
|
||
for persona_name in PERSONAS:
|
||
while persona_cc[persona_name] < 40:
|
||
replacement = find_replacement(persona_name=persona_name, non_style=False, layer=None, same_bucket=None)
|
||
if replacement is None or str(replacement.get("layer") or "") not in {"choice", "collect"}:
|
||
break
|
||
victim_index = None
|
||
for index, row in enumerate(selected):
|
||
victim_persona = str(row.get("persona") or "")
|
||
if row.get("layer") not in {"choice", "collect"}:
|
||
continue
|
||
if victim_persona == persona_name:
|
||
continue
|
||
if persona_cc[victim_persona] <= 40:
|
||
continue
|
||
if str(replacement.get("layer")) != str(row.get("layer")):
|
||
continue
|
||
victim_index = index
|
||
break
|
||
if victim_index is None:
|
||
break
|
||
uncommit(victim_index)
|
||
commit(replacement)
|
||
|
||
changed = True
|
||
while changed and persona_all["terse"] + persona_all["dialect_netspeak"] > 180:
|
||
changed = False
|
||
for index, row in enumerate(list(selected)):
|
||
persona = str(row.get("persona") or "")
|
||
if persona not in style:
|
||
continue
|
||
if row.get("layer") in {"choice", "collect"} and persona_cc[persona] <= 40:
|
||
continue
|
||
replacement = (
|
||
find_replacement(persona_name=None, non_style=True, layer=str(row.get("layer")), same_bucket=bucket_key(row))
|
||
or find_replacement(persona_name=None, non_style=True, layer=str(row.get("layer")), same_bucket=None)
|
||
)
|
||
if replacement is None:
|
||
continue
|
||
uncommit(index)
|
||
commit(replacement)
|
||
changed = True
|
||
break
|
||
return selected[:900]
|
||
|
||
|
||
def generate_source_c(
|
||
pool: Mapping[str, Any],
|
||
*,
|
||
seed: int = SEED,
|
||
extra_factor: float = 1.4,
|
||
workers: int = 8,
|
||
limit: int = 0,
|
||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]:
|
||
load_llm_cache()
|
||
denylist = holdout_name_denylist()
|
||
slots = plan_slots(pool, seed=seed, extra_factor=extra_factor)
|
||
if limit:
|
||
slots = slots[:limit]
|
||
accepted_map, disputed_map = load_progress()
|
||
pending = [slot for slot in slots if slot["id"] not in accepted_map and slot["id"] not in disputed_map]
|
||
print(f"slots={len(slots)} pending={len(pending)} cached_ok={len(accepted_map)} cached_bad={len(disputed_map)}", flush=True)
|
||
accepted_path, disputed_path = progress_paths()
|
||
done = 0
|
||
if pending:
|
||
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool_ex:
|
||
futures = {pool_ex.submit(fill_slot, slot, denylist=denylist): slot for slot in pending}
|
||
for future in as_completed(futures):
|
||
slot = futures[future]
|
||
try:
|
||
sample, ok = future.result()
|
||
except Exception as exc: # noqa: BLE001
|
||
sample = _sample_from_slot(slot, f"[worker_error:{type(exc).__name__}]", None)
|
||
ok = False
|
||
if ok:
|
||
accepted_map[sample["id"]] = sample
|
||
append_progress(accepted_path, sample)
|
||
else:
|
||
disputed_map[sample["id"]] = sample
|
||
append_progress(disputed_path, sample)
|
||
done += 1
|
||
if done % 20 == 0 or done == len(pending):
|
||
save_llm_cache()
|
||
print(
|
||
f" fill {done}/{len(pending)} accepted={len(accepted_map)} disputed={len(disputed_map)}",
|
||
flush=True,
|
||
)
|
||
save_llm_cache()
|
||
selected = select_simulated(list(accepted_map.values()), seed=seed)
|
||
disputed = list(disputed_map.values())
|
||
generated = len(accepted_map) + len(disputed_map)
|
||
stats = {
|
||
"generated": generated,
|
||
"accepted_raw": len(accepted_map),
|
||
"accepted": len(selected),
|
||
"disputed": len(disputed_map),
|
||
"dispute_rate": (len(disputed_map) / max(generated, 1)),
|
||
"generator": GENERATOR_NAME,
|
||
"reviewer": REVIEWER_NAME,
|
||
"prompt_version": PROMPT_VERSION,
|
||
"seed": seed,
|
||
"usage": dict(_USAGE),
|
||
}
|
||
return selected, list(disputed_map.values()), stats
|
||
|
||
|
||
def dispute_breakdown(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
by_layer: Counter[str] = Counter()
|
||
by_intent: Counter[str] = Counter()
|
||
for row in rows:
|
||
by_layer[str(row.get("layer") or "")] += 1
|
||
gold = row.get("gold") or {}
|
||
by_intent[str(gold.get("intent") or "")] += 1
|
||
return {"by_layer": dict(by_layer), "by_intent": dict(by_intent)}
|
||
|
||
|
||
def write_readme(
|
||
*,
|
||
synthetic_sha: str,
|
||
simulated_sha: str,
|
||
disputed_sha: str,
|
||
pool_n: int,
|
||
source_c_stats: Mapping[str, Any],
|
||
quotas: Mapping[str, Any],
|
||
lengths: Mapping[str, Any],
|
||
source_b_n: int,
|
||
personas: Mapping[str, Any],
|
||
uniqueness: Mapping[str, Any],
|
||
disputes: Mapping[str, Any],
|
||
) -> None:
|
||
source_b_sql = """
|
||
select
|
||
t.id as turn_id,
|
||
t.user_message,
|
||
t.status as turn_status,
|
||
t.created_at,
|
||
c.status as case_status,
|
||
f.question_id,
|
||
f.expected_answer_schema,
|
||
f.status as focus_status
|
||
from public.agentic_rectification_turns t
|
||
join public.agentic_rectification_cases c on c.id = t.case_id
|
||
left join public.agentic_rectification_conversation_focuses f
|
||
on f.case_id = t.case_id
|
||
and f.status = 'active'
|
||
where t.user_message is not null
|
||
and length(btrim(t.user_message)) > 0
|
||
order by t.created_at desc
|
||
limit 500;
|
||
""".strip()
|
||
generate_prompt = build_generate_prompt(
|
||
{
|
||
"current_question": SHUFFLED_CHOICE_PROMPT,
|
||
"options": SHUFFLED_OPTIONS,
|
||
"domain": "career",
|
||
"layer": "choice",
|
||
},
|
||
intent="answer_current_focus",
|
||
answer_class="no",
|
||
dated=False,
|
||
persona="rambling",
|
||
length_lo=16,
|
||
length_hi=28,
|
||
)
|
||
review_prompt = build_review_prompt({
|
||
"focus": {
|
||
"current_question": SHUFFLED_CHOICE_PROMPT,
|
||
"options": SHUFFLED_OPTIONS,
|
||
},
|
||
"user_message": "（用户回复）",
|
||
})
|
||
text = f"""# Jev 意图分类样本（T0）
|
||
|
||
## 来源 A · 既有合成样本（提交）
|
||
|
||
`synthetic.jsonl`，10 条。生产测试 `frontend/tests/rectification-turn-intent-classifier.test.ts` **没有** 10 条带期望输出的 `userMessage` 夹具；本文件用该测试的 `SHUFFLED_CHOICE` 焦点、采集题 prompt，以及测试里实际出现的两句 `userMessage`（`2016年3月入学`、`随便吧`）按分类器提示词写成带标签样本。
|
||
|
||
sha256: `{synthetic_sha}`
|
||
|
||
## 来源 B · 真实会话（本地，不提交）
|
||
|
||
作用：校准锚，不是主测试集。最少 30 条；不足则代表性检验只报数不判定。
|
||
|
||
- 抽取 SQL（staging，`agentic_rectification_turns.user_message` 非空）：
|
||
|
||
```sql
|
||
{source_b_sql}
|
||
```
|
||
|
||
- 本地路径（worktree 外，已 gitignore）：`G:/Ferti/Jyotisha/.cache/jev_intent/source_b.jsonl`
|
||
- 本机抽取条数：{source_b_n}
|
||
- 真值：执行方逐条人工标注；能从后续动作反推的另存 `runtime_intent` 列。**不得提交原文。**
|
||
|
||
## 来源 C · 模型模拟语料（提交，主测试集）
|
||
|
||
`simulated.jsonl`。问题池 `question_pool.json`：20 例公开 AA holdout 走 `discriminating_event_probes` 出点选题；采集题用 `USER_COLLECT_QUESTION` / retry / `TARGETED_YEAR_PROMPT`。生成与复核各一次真实模型调用；复核看不到目标标签。
|
||
|
||
- 生成模型：`{GENERATOR_NAME}`（temperature {GEN_TEMPERATURE}，prompt_version `{PROMPT_VERSION}`）
|
||
- 复核模型：`{REVIEWER_NAME}`（temperature {REVIEW_TEMPERATURE}，同一家模型、不同提示、看不到标签）
|
||
- 生成统计：{json.dumps(source_c_stats, ensure_ascii=False)}
|
||
- 争议分层：{json.dumps(disputes, ensure_ascii=False)}
|
||
- 配额：{json.dumps({k: {'actual': v['actual_total'], 'target': v['target_total'], 'max_share_delta': v['max_share_delta'], 'ok': v['within_20pct']} for k, v in quotas.items()}, ensure_ascii=False)}
|
||
- 长度：{json.dumps(lengths, ensure_ascii=False)}
|
||
- 人设：{json.dumps(personas, ensure_ascii=False)}
|
||
- 去重：{json.dumps(uniqueness, ensure_ascii=False)}
|
||
- sha256 simulated: `{simulated_sha}`
|
||
- sha256 disputed: `{disputed_sha}`
|
||
- 问题池条数（点选题）：{pool_n}
|
||
|
||
### 生成提示原文
|
||
|
||
系统：`{GENERATE_SYSTEM}`
|
||
|
||
用户提示模板示例：
|
||
|
||
```
|
||
{generate_prompt}
|
||
```
|
||
|
||
### 复核提示原文
|
||
|
||
系统：`{REVIEW_SYSTEM}`
|
||
|
||
用户提示模板示例：
|
||
|
||
```
|
||
{review_prompt}
|
||
```
|
||
|
||
公开案例姓名不得出现在回复正文。
|
||
"""
|
||
README_PATH.write_text(text, encoding="utf-8")
|
||
|
||
|
||
def load_source_b() -> list[dict[str, Any]]:
|
||
path = Path(r"G:\Ferti\Jyotisha\.cache\jev_intent\source_b.jsonl")
|
||
if not path.is_file():
|
||
return []
|
||
rows = []
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
if line.strip():
|
||
rows.append(json.loads(line))
|
||
return rows
|
||
|
||
|
||
def _print_gate(selected: Sequence[Mapping[str, Any]], stats: Mapping[str, Any]) -> int:
|
||
quotas = quota_deviation(selected)
|
||
lengths = length_stats(str(row.get("user_message") or "") for row in selected)
|
||
personas = persona_stats(selected)
|
||
uniqueness = uniqueness_stats(selected)
|
||
print(json.dumps({
|
||
"simulated": len(selected),
|
||
"stats": stats,
|
||
"quotas": {k: {"actual": v["actual_total"], "ok": v["within_20pct"], "delta": v["max_share_delta"]} for k, v in quotas.items()},
|
||
"lengths": lengths,
|
||
"personas": personas,
|
||
"uniqueness": uniqueness,
|
||
"length_ok": length_ok(lengths),
|
||
}, ensure_ascii=False, indent=2), flush=True)
|
||
failed = []
|
||
if len(selected) < 900:
|
||
failed.append("simulated<900")
|
||
layer_n = Counter(str(row.get("layer")) for row in selected)
|
||
if not (
|
||
320 <= layer_n.get("choice", 0) <= 480
|
||
and 320 <= layer_n.get("collect", 0) <= 480
|
||
and 80 <= layer_n.get("none", 0) <= 120
|
||
):
|
||
failed.append("layer_totals")
|
||
if not all(quotas[layer]["within_20pct"] for layer in quotas):
|
||
failed.append("quota")
|
||
if not length_ok(lengths):
|
||
failed.append("length")
|
||
if not uniqueness["unique_ok"] or not uniqueness["repeat_ok"]:
|
||
failed.append("uniqueness")
|
||
if personas["min_choice_collect"] < 40:
|
||
failed.append("persona_min")
|
||
if personas["terse_dialect_share_all"] > 0.20:
|
||
failed.append("terse_dialect")
|
||
if failed:
|
||
print("FAILED constraints: " + ",".join(failed), flush=True)
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def main(argv: Sequence[str] | None = None) -> int:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--pool-only", action="store_true")
|
||
parser.add_argument("--reuse-pool", action="store_true", default=True)
|
||
parser.add_argument("--rebuild-pool", action="store_true")
|
||
parser.add_argument("--max-cases", type=int, default=None)
|
||
parser.add_argument("--seed", type=int, default=SEED)
|
||
parser.add_argument("--workers", type=int, default=8)
|
||
parser.add_argument("--limit", type=int, default=0)
|
||
parser.add_argument("--extra-factor", type=float, default=1.4)
|
||
parser.add_argument("--reselect-only", action="store_true")
|
||
args = parser.parse_args(argv)
|
||
SAMPLES_DIR.mkdir(parents=True, exist_ok=True)
|
||
reuse = args.reuse_pool and not args.rebuild_pool and POOL_PATH.is_file() and not args.pool_only
|
||
if reuse:
|
||
pool = json.loads(POOL_PATH.read_text(encoding="utf-8"))
|
||
print(f"reusing question pool choice={len(pool.get('choice') or [])}", flush=True)
|
||
else:
|
||
print("building question pool from holdout + discriminating_event_probes", flush=True)
|
||
pool = build_question_pool(max_cases=args.max_cases)
|
||
POOL_PATH.write_text(json.dumps(pool, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(
|
||
f"pool choice={len(pool['choice'])} collect={len(pool['collect'])} errors={pool['errors']}",
|
||
flush=True,
|
||
)
|
||
if args.pool_only:
|
||
return 0
|
||
if not args.reselect_only and not os.environ.get("DEEPSEEK_API_KEY"):
|
||
print("DEEPSEEK_API_KEY missing", file=sys.stderr)
|
||
return 2
|
||
synthetic = source_a_samples()
|
||
write_jsonl(SYNTHETIC_PATH, synthetic)
|
||
if args.reselect_only:
|
||
accepted_map, disputed_map = load_progress()
|
||
accepted = select_simulated(list(accepted_map.values()), seed=args.seed)
|
||
disputed = list(disputed_map.values())
|
||
usage_path = CACHE_DIR / "llm_usage.json"
|
||
usage = dict(_USAGE)
|
||
if usage_path.is_file():
|
||
try:
|
||
usage = json.loads(usage_path.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError:
|
||
pass
|
||
stats = {
|
||
"generated": len(accepted_map) + len(disputed_map),
|
||
"accepted_raw": len(accepted_map),
|
||
"accepted": len(accepted),
|
||
"disputed": len(disputed_map),
|
||
"dispute_rate": (len(disputed_map) / max(len(accepted_map) + len(disputed_map), 1)),
|
||
"generator": GENERATOR_NAME,
|
||
"reviewer": REVIEWER_NAME,
|
||
"prompt_version": PROMPT_VERSION,
|
||
"seed": args.seed,
|
||
"usage": usage,
|
||
"reselect_only": True,
|
||
}
|
||
else:
|
||
accepted, disputed, stats = generate_source_c(
|
||
pool,
|
||
seed=args.seed,
|
||
extra_factor=args.extra_factor,
|
||
workers=args.workers,
|
||
limit=args.limit,
|
||
)
|
||
if stats["dispute_rate"] > DISPUTE_RATE_LIMIT:
|
||
print(
|
||
f"dispute_rate={stats['dispute_rate']:.3f} > 0.15 (prompt_version={PROMPT_VERSION})",
|
||
flush=True,
|
||
)
|
||
write_jsonl(SIMULATED_PATH, accepted)
|
||
write_jsonl(DISPUTED_PATH, disputed)
|
||
quotas = quota_deviation(accepted)
|
||
lengths = length_stats(str(row.get("user_message") or "") for row in accepted)
|
||
personas = persona_stats(accepted)
|
||
uniqueness = uniqueness_stats(accepted)
|
||
source_b = load_source_b()
|
||
write_readme(
|
||
synthetic_sha=sha256_file(SYNTHETIC_PATH),
|
||
simulated_sha=sha256_file(SIMULATED_PATH),
|
||
disputed_sha=sha256_file(DISPUTED_PATH),
|
||
pool_n=len(pool.get("choice") or []),
|
||
source_c_stats=stats,
|
||
quotas=quotas,
|
||
lengths=lengths,
|
||
source_b_n=len(source_b),
|
||
personas=personas,
|
||
uniqueness=uniqueness,
|
||
disputes=dispute_breakdown(disputed),
|
||
)
|
||
print(json.dumps({
|
||
"synthetic": len(synthetic),
|
||
"simulated": len(accepted),
|
||
"disputed": len(disputed),
|
||
"stats": stats,
|
||
"quotas": {k: {"actual": v["actual_total"], "ok": v["within_20pct"], "delta": v["max_share_delta"]} for k, v in quotas.items()},
|
||
"lengths": lengths,
|
||
"personas": personas,
|
||
"uniqueness": uniqueness,
|
||
"source_b": len(source_b),
|
||
"sha256": {
|
||
"synthetic": sha256_file(SYNTHETIC_PATH),
|
||
"simulated": sha256_file(SIMULATED_PATH),
|
||
"disputed": sha256_file(DISPUTED_PATH),
|
||
},
|
||
}, ensure_ascii=False, indent=2))
|
||
if args.limit:
|
||
return 0
|
||
if stats["dispute_rate"] > DISPUTE_RATE_LIMIT:
|
||
print("WARN: dispute_rate still > 15%; proceeding per 让步 1, not falling back to templates", flush=True)
|
||
return _print_gate(accepted, stats)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|