973 lines
39 KiB
Python
973 lines
39 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the offline Jev intent-classifier corpus (T0).
|
||
|
||
Source A: ten fixtures derived from the production classifier test file.
|
||
Source B: real staging turns, written outside git (not this script's default).
|
||
Source C: Agent-simulated replies. Labels are assigned first; an independent
|
||
reviewer that cannot see the target label keeps only agreements.
|
||
|
||
Does not call production `classifyRectificationTurnIntent`. Celebrity names
|
||
from the public holdout file never enter user_message.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import random
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
from datetime import date
|
||
from pathlib import Path
|
||
from typing import Any, Iterable, Mapping, Sequence
|
||
from uuid import NAMESPACE_URL, uuid5
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from scripts.rectification.probe_question_contract import ( # noqa: E402
|
||
EXISTENCE_STYLE_OPTIONS,
|
||
QUALITY_STYLE_OPTIONS,
|
||
)
|
||
from scripts.research.jev_intent_questions import ( # noqa: E402
|
||
ANSWER_CLASS_VALUES,
|
||
enforce_combo,
|
||
)
|
||
|
||
HOLDOUT = ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v4.json"
|
||
SAMPLES_DIR = ROOT / "scripts" / "research" / "jev_intent_samples"
|
||
POOL_PATH = SAMPLES_DIR / "question_pool.json"
|
||
SYNTHETIC_PATH = SAMPLES_DIR / "synthetic.jsonl"
|
||
SIMULATED_PATH = SAMPLES_DIR / "simulated.jsonl"
|
||
DISPUTED_PATH = SAMPLES_DIR / "disputed.jsonl"
|
||
README_PATH = SAMPLES_DIR / "README.md"
|
||
TODAY = date(2026, 9, 19)
|
||
SEED = 20260919
|
||
GENERATOR_NAME = "agent-template-v1"
|
||
REVIEWER_NAME = "agent-rule-v1"
|
||
DISPUTE_RATE_LIMIT = 0.15
|
||
|
||
COLLECT_PROMPTS: dict[str, str] = {
|
||
"relationship": "感情这边，还记得哪年认真在一起、分开，或结婚吗？有年份就很好。",
|
||
"career": "工作上呢，还记得哪年入职、换工作，或职责一下子变重吗？",
|
||
"family": "家里如果有结婚、添丁或住院这类事，记得大概哪年就行。",
|
||
"education": "上学这边，还记得哪年升学、转学或大考吗？",
|
||
"relocation": "有没有哪年搬家，或开始长期住在外地？",
|
||
"finance": "钱的方面，还记得哪年收入明显变过、有过大笔支出，或欠过债吗？",
|
||
"health_pressure": "身体或压力这边，还记得哪年生病、受伤，或特别难熬的一段时间吗？",
|
||
}
|
||
COLLECT_RETRY: dict[str, str] = {
|
||
"relationship": "回到感情这边——刚才说的工作我记下了，哪年认真在一起或分开还记得吗？",
|
||
"career": "回到工作这边——刚才那件我记下了，哪年入职或换工作还记得吗？",
|
||
"family": "再问一次家里：结婚、添丁或住院，大概哪年？",
|
||
"education": "回到上学这边——哪年升学、转学或大考，还记得吗？",
|
||
"relocation": "搬家或开始长期住外地，大概是哪年？",
|
||
"finance": "钱的方面再对一下：哪年收入明显变过、有过大笔支出，或欠过债？",
|
||
"health_pressure": "身体或压力这边再问一次：哪年生病、受伤，或特别难熬？",
|
||
}
|
||
TARGETED_YEAR_PROMPT = "大概哪年几月？"
|
||
SHUFFLED_CHOICE_PROMPT = "2023 年前后，工作状态是否出现明显变化？"
|
||
SHUFFLED_OPTIONS = [
|
||
{"key": "A", "label": "这段时间没有明显变化", "answer_class": "no"},
|
||
{"key": "B", "label": "记不清当时的情况", "answer_class": "unsure"},
|
||
{"key": "C", "label": "变化明显而且时间吻合", "answer_class": "yes"},
|
||
{"key": "D", "label": "有变化但程度比较弱", "answer_class": "weak_yes"},
|
||
]
|
||
COLLECT_FINANCE_PROMPT = "钱的方面，还记得哪年收入明显变过吗？"
|
||
|
||
PERSONAS = (
|
||
"terse",
|
||
"rambling",
|
||
"answer_then_event",
|
||
"deny_then_event",
|
||
"off_topic",
|
||
"impatient_stop",
|
||
"ask_result",
|
||
"dialect_netspeak",
|
||
)
|
||
|
||
QUOTAS: dict[str, list[tuple[str, str | None, bool, int]]] = {
|
||
# layer -> (intent, answer_class, has_new_dated_event, count)
|
||
"choice": [
|
||
("answer_current_focus", "yes", False, 45),
|
||
("answer_current_focus", "yes", True, 15),
|
||
("answer_current_focus", "weak_yes", False, 45),
|
||
("answer_current_focus", "weak_yes", True, 15),
|
||
("answer_current_focus", "no", False, 45),
|
||
("answer_current_focus", "no", True, 15),
|
||
("answer_current_focus", "unsure", False, 45),
|
||
("answer_current_focus", "unsure", True, 15),
|
||
("provide_new_evidence", None, True, 60),
|
||
("stop_rectification", None, False, 32),
|
||
("ask_about_result", None, False, 32),
|
||
("unclear", None, False, 36),
|
||
],
|
||
"collect": [
|
||
("answer_current_focus", "no", False, 100),
|
||
("answer_current_focus", "unsure", False, 80),
|
||
("answer_current_focus", "yes", False, 34),
|
||
("answer_current_focus", "yes", True, 16),
|
||
("answer_current_focus", "weak_yes", False, 33),
|
||
("answer_current_focus", "weak_yes", True, 17),
|
||
("provide_new_evidence", None, True, 48),
|
||
("stop_rectification", None, False, 24),
|
||
("ask_about_result", None, False, 24),
|
||
("unclear", None, False, 24),
|
||
],
|
||
"none": [
|
||
("provide_new_evidence", None, True, 50),
|
||
("stop_rectification", None, False, 15),
|
||
("ask_about_result", None, False, 15),
|
||
("unclear", None, False, 20),
|
||
],
|
||
}
|
||
|
||
EVENT_STEMS: dict[str, tuple[str, ...]] = {
|
||
"education": ("升学", "转学", "大考", "换学校"),
|
||
"career": ("入职", "换工作", "职责变重", "离职"),
|
||
"relocation": ("搬家", "开始长期住外地", "换城市住"),
|
||
"relationship": ("认真在一起", "分开", "结婚"),
|
||
"family": ("家里添丁", "长辈住院", "家里有人结婚"),
|
||
"finance": ("收入明显变过", "有过大笔支出", "欠过债"),
|
||
"health_pressure": ("生病", "受伤", "特别难熬过一段时间"),
|
||
}
|
||
FILLER = (
|
||
"我想想啊就是那种说不上来特别大但也不是完全没有的感觉，",
|
||
"当时身边人也没特别提，我自己也没当回事，",
|
||
"现在回想起来细节对不上，只能说个大概，",
|
||
"别的先不说，就这一句你先记着，",
|
||
)
|
||
DIALECT = (
|
||
"嗯那会儿吧咋说呢",
|
||
"额这个嘛就那样",
|
||
"不是很能整明白",
|
||
"emmm 感觉一般般",
|
||
)
|
||
YEAR_RE = re.compile(r"(?:19|20)\d{2}")
|
||
MONTH_RE = re.compile(r"\d{1,2}\s*月")
|
||
STOP_RE = re.compile(r"(停止整套|停止校正|别搞了|不弄了|先停|不做了|结束校正|莫搞了|不玩了)")
|
||
ASK_RE = re.compile(r"(结果呢|范围呢|出结果|看盘|候选分钟|出来了吗|好了没|现在几分钟)")
|
||
NO_RE = re.compile(r"(没有|没发生|没变|无变化|这方面没什么|没这回事|没有过|那段没有)")
|
||
UNSURE_RE = re.compile(r"(记不清|不记得|忘了|想不起来|以后再说|说不清|不太记得)")
|
||
YES_RE = re.compile(r"(有的|发生了|确实有|对的|就是那段|明显有过)")
|
||
WEAK_RE = re.compile(r"(一点点|不太明显|好像有过|将就吧|有过但不算)")
|
||
EXTRA_RE = re.compile(r"(另外|不过|对了还有)")
|
||
PROVIDE_RE = re.compile(r"^(另外|对了还有|另说一件)")
|
||
|
||
|
||
def sha256_file(path: Path) -> str:
|
||
digest = hashlib.sha256()
|
||
digest.update(path.read_bytes())
|
||
return digest.hexdigest()
|
||
|
||
|
||
def write_jsonl(path: Path, rows: Sequence[Mapping[str, Any]]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
text = "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows)
|
||
path.write_text(text, encoding="utf-8")
|
||
|
||
|
||
def holdout_name_denylist() -> set[str]:
|
||
payload = json.loads(HOLDOUT.read_text(encoding="utf-8"))
|
||
names: set[str] = set()
|
||
for case in payload.get("cases") or []:
|
||
label = str(case.get("subject_label") or "").strip()
|
||
if label:
|
||
names.add(label)
|
||
for part in label.split():
|
||
if len(part) >= 3:
|
||
names.add(part)
|
||
return names
|
||
|
||
|
||
def assert_no_pii(message: str, denylist: set[str]) -> None:
|
||
for name in denylist:
|
||
if name and name in message:
|
||
raise ValueError(f"simulated reply leaked holdout name {name!r}")
|
||
|
||
|
||
def source_a_samples() -> list[dict[str, Any]]:
|
||
"""Ten fixtures from the test file's schema + the two userMessage strings."""
|
||
choice_focus = {
|
||
"kind": "choice",
|
||
"current_question": SHUFFLED_CHOICE_PROMPT,
|
||
"options": SHUFFLED_OPTIONS,
|
||
"case_status": "distinguishing_candidates",
|
||
}
|
||
collect_focus = {
|
||
"kind": "collect",
|
||
"current_question": COLLECT_FINANCE_PROMPT,
|
||
"options": [],
|
||
"case_status": "collecting_evidence",
|
||
}
|
||
rows = [
|
||
("A01", "choice", choice_focus, "没什么变化", "answer_current_focus", "no", False),
|
||
("A02", "choice", choice_focus, "记不清", "answer_current_focus", "unsure", False),
|
||
("A03", "choice", choice_focus, "变化很明显", "answer_current_focus", "yes", False),
|
||
("A04", "choice", choice_focus, "有一点变化", "answer_current_focus", "weak_yes", False),
|
||
("A05", "choice", choice_focus, "没有，不过 2019 年换过岗", "answer_current_focus", "no", True),
|
||
("A06", "choice", choice_focus, "先不弄了", "stop_rectification", None, False),
|
||
("A07", "choice", choice_focus, "结果出来了吗", "ask_about_result", None, False),
|
||
("A08", "collect", collect_focus, "没有", "answer_current_focus", "no", False),
|
||
("A09", "collect", collect_focus, "随便吧", "unclear", None, False),
|
||
("A10", "collect", collect_focus, "2016年3月入学", "answer_current_focus", "yes", False),
|
||
]
|
||
samples: list[dict[str, Any]] = []
|
||
for sid, layer, focus, message, intent, answer_class, dated in rows:
|
||
gold = enforce_combo(intent, answer_class, dated)
|
||
samples.append({
|
||
"id": sid,
|
||
"source": "A",
|
||
"layer": layer,
|
||
"persona": "test_fixture",
|
||
"focus": focus,
|
||
"user_message": message,
|
||
"gold": gold,
|
||
"generator": "test-file",
|
||
"reviewer": "manual",
|
||
})
|
||
return samples
|
||
|
||
|
||
def _hhmm(value: object) -> str | None:
|
||
text = str(value or "")[:5]
|
||
return text if len(text) == 5 and text[2] == ":" else None
|
||
|
||
|
||
def _probe_prompt(probe: Mapping[str, Any]) -> str:
|
||
family = str(probe.get("event_family") or "").replace("？", "").replace("?", "").strip()
|
||
label = str(probe.get("year_label") or "").strip()
|
||
kind = str(probe.get("choice_kind") or "existence")
|
||
meaning = str(probe.get("user_meaning") or "").strip()
|
||
if kind == "event_quality" and meaning:
|
||
from scripts.rectification.probe_question_contract import clipped_probe_label
|
||
clipped = clipped_probe_label(meaning) or meaning
|
||
if clipped.endswith("？") or clipped.endswith("?"):
|
||
return clipped
|
||
return f"{label}，有没有{family}的时候？" if label and family else clipped
|
||
if not family:
|
||
return label
|
||
if kind == "event_quality":
|
||
return f"{label}，有没有{family}的时候？" if label else f"有没有{family}的时候？"
|
||
return f"{label}，有没有{family}？" if label else f"有没有{family}？"
|
||
|
||
|
||
def _probe_options(probe: Mapping[str, Any]) -> list[dict[str, str]]:
|
||
raw = probe.get("style_options") or []
|
||
rows: list[dict[str, str]] = []
|
||
keys = ("A", "B", "C", "D")
|
||
for index, item in enumerate(raw):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
answer_class = str(item.get("answer_class") or "")
|
||
label = str(item.get("label") or "").strip()
|
||
if answer_class not in ANSWER_CLASS_VALUES or not label:
|
||
continue
|
||
rows.append({"key": keys[len(rows)], "label": label, "answer_class": answer_class})
|
||
if len(rows) == 4:
|
||
break
|
||
if len(rows) != 4:
|
||
catalog = QUALITY_STYLE_OPTIONS if str(probe.get("choice_kind") or "") == "event_quality" else EXISTENCE_STYLE_OPTIONS
|
||
rows = [
|
||
{"key": keys[i], "label": str(item["label"]), "answer_class": str(item["answer_class"])}
|
||
for i, item in enumerate(catalog)
|
||
]
|
||
return rows
|
||
|
||
|
||
def build_question_pool(*, max_cases: int | None = None) -> dict[str, Any]:
|
||
from scripts.active_rectification_event_engine import AYANAMSA, NODE_MODE
|
||
from scripts.rectification.event_probes import discriminating_event_probes
|
||
from scripts.rectification.refinement_packet import window_scan
|
||
from scripts.rectification.scoring_service import (
|
||
build_event_contribution_matrix,
|
||
score_from_matrix,
|
||
scoreable_request,
|
||
)
|
||
from scripts.research.probe_supply_after_six import request_from_case
|
||
|
||
payload = json.loads(HOLDOUT.read_text(encoding="utf-8"))
|
||
cases = list(payload.get("cases") or [])
|
||
if max_cases is not None:
|
||
cases = cases[:max_cases]
|
||
choice_questions: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
errors: list[str] = []
|
||
for case in cases:
|
||
case_id = str(case.get("case_id") or "case")
|
||
try:
|
||
request = request_from_case(case)
|
||
request["ayanamsa"] = AYANAMSA
|
||
request["node_mode"] = NODE_MODE
|
||
scoring = scoreable_request(request)
|
||
built = build_event_contribution_matrix(scoring)
|
||
rows = score_from_matrix(scoring, built)
|
||
times = [stamp for row in rows if (stamp := _hhmm(row.get("time")))]
|
||
true_time = str(case["birth"]["time"])[:5]
|
||
probes = discriminating_event_probes(
|
||
{**request, "refresh_probes": False, "asked_probe_keys": []},
|
||
built,
|
||
scan=window_scan(built),
|
||
candidate_times=times,
|
||
representative_time=true_time,
|
||
today=TODAY,
|
||
)
|
||
except Exception as exc: # noqa: BLE001 — keep the other 19 cases
|
||
errors.append(f"{case_id}: {type(exc).__name__}")
|
||
continue
|
||
kept = 0
|
||
for probe in probes:
|
||
if not isinstance(probe, dict):
|
||
continue
|
||
prompt = _probe_prompt(probe)
|
||
options = _probe_options(probe)
|
||
key = f"{prompt}|{','.join(item['answer_class'] + ':' + item['label'] for item in options)}"
|
||
if key in seen or not prompt:
|
||
continue
|
||
seen.add(key)
|
||
choice_questions.append({
|
||
"id": str(uuid5(NAMESPACE_URL, key)),
|
||
"layer": "choice",
|
||
"current_question": prompt,
|
||
"options": options,
|
||
"domain": str(probe.get("domain") or ""),
|
||
"choice_kind": str(probe.get("choice_kind") or "existence"),
|
||
"year_label": str(probe.get("year_label") or ""),
|
||
"event_family": str(probe.get("event_family") or ""),
|
||
"case_status": "distinguishing_candidates",
|
||
})
|
||
kept += 1
|
||
print(f"pool {case_id} probes={len(probes)} unique={kept}", flush=True)
|
||
collect_questions: list[dict[str, Any]] = []
|
||
for domain, prompt in COLLECT_PROMPTS.items():
|
||
collect_questions.append({
|
||
"id": f"collect:{domain}",
|
||
"layer": "collect",
|
||
"current_question": prompt,
|
||
"options": [],
|
||
"domain": domain,
|
||
"case_status": "collecting_evidence",
|
||
})
|
||
collect_questions.append({
|
||
"id": f"collect-retry:{domain}",
|
||
"layer": "collect",
|
||
"current_question": COLLECT_RETRY[domain],
|
||
"options": [],
|
||
"domain": domain,
|
||
"case_status": "collecting_evidence",
|
||
})
|
||
collect_questions.append({
|
||
"id": "collect:targeted-year",
|
||
"layer": "collect",
|
||
"current_question": TARGETED_YEAR_PROMPT,
|
||
"options": [],
|
||
"domain": "career",
|
||
"case_status": "collecting_evidence",
|
||
})
|
||
none_questions = [{
|
||
"id": "none:open",
|
||
"layer": "none",
|
||
"current_question": "",
|
||
"options": [],
|
||
"domain": "",
|
||
"case_status": "collecting_evidence",
|
||
}]
|
||
pool = {
|
||
"choice": choice_questions,
|
||
"collect": collect_questions,
|
||
"none": none_questions,
|
||
"errors": errors,
|
||
"holdout": str(HOLDOUT.relative_to(ROOT)).replace("\\", "/"),
|
||
"generator_note": "choice prompts from discriminating_event_probes; collect copy from user-copy.ts",
|
||
}
|
||
return pool
|
||
|
||
|
||
def _year(rng: random.Random) -> int:
|
||
return rng.randint(1998, 2024)
|
||
|
||
|
||
def _month(rng: random.Random) -> int:
|
||
return rng.randint(1, 12)
|
||
|
||
|
||
def _stem(domain: str, rng: random.Random) -> str:
|
||
stems = EVENT_STEMS.get(domain) or EVENT_STEMS["career"]
|
||
return rng.choice(stems)
|
||
|
||
|
||
def _dated(domain: str, rng: random.Random, *, with_month: bool = False) -> str:
|
||
year = _year(rng)
|
||
stem = _stem(domain, rng)
|
||
if with_month or rng.random() < 0.4:
|
||
return f"{year}年{_month(rng)}月{stem}"
|
||
return f"{year}年{stem}"
|
||
|
||
|
||
def _pad(text: str, rng: random.Random, target: int) -> str:
|
||
while len(text) < target:
|
||
text += rng.choice(FILLER)
|
||
if len(text) >= target:
|
||
break
|
||
return text[:target] if len(text) > target + 8 else text
|
||
|
||
|
||
def _persona_for(intent: str, dated: bool, rng: random.Random) -> str:
|
||
if not dated and rng.random() < 0.34:
|
||
return "terse"
|
||
compatible = ["terse", "rambling", "dialect_netspeak"]
|
||
if intent == "answer_current_focus" and dated:
|
||
compatible.append("answer_then_event")
|
||
compatible.append("deny_then_event")
|
||
if intent == "stop_rectification":
|
||
compatible.append("impatient_stop")
|
||
if intent == "ask_about_result":
|
||
compatible.append("ask_result")
|
||
if intent == "unclear":
|
||
compatible.append("off_topic")
|
||
return rng.choice(compatible)
|
||
|
||
|
||
def _choice_answer(answer_class: str, rng: random.Random, persona: str) -> str:
|
||
bank = {
|
||
"yes": ("有的", "发生了", "确实有", "就是那段", "明显有过"),
|
||
"weak_yes": ("一点点", "不太明显", "好像有过", "有过但不算", "将就吧"),
|
||
"no": ("没有", "没变", "没这回事", "那段没有", "没有过"),
|
||
"unsure": ("忘了", "记不清", "想不起来", "说不清", "以后再说"),
|
||
}[answer_class]
|
||
if persona == "terse":
|
||
short = [item for item in bank if len(item) <= 6]
|
||
return rng.choice(short or bank)
|
||
if persona == "dialect_netspeak":
|
||
return rng.choice(DIALECT) + rng.choice(bank)
|
||
if persona == "rambling":
|
||
return _pad(rng.choice(bank) + "，" + rng.choice(FILLER), rng, rng.randint(28, 55))
|
||
return rng.choice(bank)
|
||
|
||
|
||
def _collect_answer(answer_class: str, domain: str, rng: random.Random, persona: str) -> str:
|
||
if answer_class in {"yes", "weak_yes"}:
|
||
dated = _dated(domain, rng, with_month=answer_class == "yes")
|
||
if answer_class == "weak_yes":
|
||
dated = "大概" + dated + "吧，印象不深"
|
||
if persona == "terse":
|
||
return f"{_year(rng)}年{_stem(domain, rng)}"[:6] if len(dated) > 6 else dated
|
||
if persona == "rambling":
|
||
return _pad("有的，" + dated + "。", rng, rng.randint(30, 60))
|
||
if persona == "dialect_netspeak":
|
||
return rng.choice(DIALECT) + dated
|
||
return dated
|
||
return _choice_answer(answer_class, rng, persona)
|
||
|
||
|
||
def _extra_event(domain: str, rng: random.Random) -> str:
|
||
other = rng.choice([key for key in EVENT_STEMS if key != domain] or ["career"])
|
||
return _dated(other, rng, with_month=True)
|
||
|
||
|
||
def render_reply(
|
||
*,
|
||
layer: str,
|
||
domain: str,
|
||
intent: str,
|
||
answer_class: str | None,
|
||
dated: bool,
|
||
persona: str,
|
||
rng: random.Random,
|
||
) -> str:
|
||
if intent == "stop_rectification":
|
||
bank = ("不弄了", "先停", "别搞了", "不做了", "莫搞了")
|
||
if persona == "terse":
|
||
return rng.choice([item for item in bank if len(item) <= 6])
|
||
if persona == "impatient_stop":
|
||
return "问了这么多我先不弄了，停止整套。"
|
||
if persona == "dialect_netspeak":
|
||
return "莫搞了，这套先停。"
|
||
if persona == "rambling":
|
||
return _pad("这校正先停吧，停止整套，我今天不想继续了。", rng, 40)
|
||
return rng.choice(bank)
|
||
if intent == "ask_about_result":
|
||
bank = ("结果呢", "范围呢", "好了没", "出结果了吗", "现在几分钟")
|
||
if persona == "terse":
|
||
return rng.choice([item for item in bank if len(item) <= 6])
|
||
if persona == "ask_result":
|
||
return "我想看结果，现在几分钟了？"
|
||
if persona == "rambling":
|
||
return _pad("先不问这题了，出结果了吗，范围呢。", rng, 42)
|
||
if persona == "dialect_netspeak":
|
||
return "emmm 结果呢好了没"
|
||
return rng.choice(bank)
|
||
if intent == "unclear":
|
||
bank = ("随便吧", "看你", "嗯", "哦", "那啥", "天气真好")
|
||
if persona == "terse":
|
||
return rng.choice([item for item in bank if len(item) <= 6])
|
||
if persona == "off_topic":
|
||
return "中午吃面还是米饭，这跟校正没关系。"
|
||
if persona == "rambling":
|
||
return _pad("我也说不好你问的是啥，反正就那样吧。", rng, 36)
|
||
if persona == "dialect_netspeak":
|
||
return "hahaha 随便啦"
|
||
return rng.choice(bank)
|
||
if intent == "provide_new_evidence":
|
||
text = _extra_event(domain or "career", rng)
|
||
if persona == "terse":
|
||
return "另外" + f"{_year(rng)}年{_stem(domain or 'career', rng)}"
|
||
if persona == "rambling":
|
||
return _pad("另外说一件：" + text + "。", rng, 48)
|
||
if persona == "dialect_netspeak":
|
||
return "对了还有一件，" + text
|
||
return "另外，" + text
|
||
# answer_current_focus
|
||
if layer == "collect":
|
||
core = _collect_answer(answer_class or "no", domain or "career", rng, persona)
|
||
else:
|
||
core = _choice_answer(answer_class or "no", rng, persona)
|
||
if dated:
|
||
extra = _extra_event(domain or "career", rng)
|
||
if persona == "deny_then_event" or answer_class == "no":
|
||
core = f"{core}。不过{extra}"
|
||
else:
|
||
core = f"{core}。另外{extra}"
|
||
if persona == "rambling" and len(core) < 28:
|
||
core = _pad(core + "。", rng, rng.randint(28, 52))
|
||
if persona == "terse" and not dated:
|
||
core = core[:6]
|
||
return core.strip("。， ")
|
||
|
||
|
||
SHORT_BY_LABEL = {
|
||
("stop_rectification", None, False): ("不弄了", "先停", "别搞了"),
|
||
("ask_about_result", None, False): ("结果呢", "范围呢", "好了没"),
|
||
("unclear", None, False): ("随便吧", "看你", "嗯"),
|
||
("answer_current_focus", "no", False): ("没有", "没变", "没有过"),
|
||
("answer_current_focus", "unsure", False): ("忘了", "记不清", "说不清"),
|
||
("answer_current_focus", "yes", False): ("有的", "发生了", "确实有"),
|
||
("answer_current_focus", "weak_yes", False): ("一点点", "将就吧", "不太明显"),
|
||
}
|
||
|
||
|
||
def enforce_length_floor(rows: list[dict[str, Any]], rng: random.Random) -> None:
|
||
"""Push P25 ≤ 6 when source B is missing, without changing labels."""
|
||
stats = length_stats(str(row.get("user_message") or "") for row in rows)
|
||
if stats["p25"] <= 6:
|
||
return
|
||
candidates = [
|
||
row for row in rows
|
||
if (row["gold"]["intent"], row["gold"]["answer_class"], bool(row["gold"]["has_new_dated_event"])) in SHORT_BY_LABEL
|
||
and len(str(row.get("user_message") or "")) > 6
|
||
]
|
||
rng.shuffle(candidates)
|
||
need = max(0, int(0.26 * len(rows)) - sum(1 for row in rows if len(str(row.get("user_message") or "")) <= 6))
|
||
for row in candidates[:need]:
|
||
gold = row["gold"]
|
||
original = row["user_message"]
|
||
original_persona = row.get("persona")
|
||
row["user_message"] = rng.choice(
|
||
SHORT_BY_LABEL[(gold["intent"], gold["answer_class"], bool(gold["has_new_dated_event"]))]
|
||
)
|
||
reviewed = review_sample(row)
|
||
if gold_equal(gold, reviewed):
|
||
row["persona"] = "terse"
|
||
row["review"] = reviewed
|
||
else:
|
||
row["user_message"] = original
|
||
row["persona"] = original_persona
|
||
|
||
|
||
def _option_labels(focus: Mapping[str, Any]) -> list[str]:
|
||
return [str(item.get("label") or "") for item in (focus.get("options") or []) if item.get("label")]
|
||
|
||
|
||
def _strip_option_labels(message: str, labels: Sequence[str]) -> str:
|
||
text = message
|
||
for label in labels:
|
||
if label and label in text:
|
||
text = text.replace(label, "")
|
||
return text.strip() or message
|
||
|
||
|
||
def generate_candidate(
|
||
question: Mapping[str, Any],
|
||
*,
|
||
intent: str,
|
||
answer_class: str | None,
|
||
dated: bool,
|
||
rng: random.Random,
|
||
index: int,
|
||
) -> dict[str, Any]:
|
||
layer = str(question.get("layer") or "none")
|
||
domain = str(question.get("domain") or "career")
|
||
persona = _persona_for(intent, dated, rng)
|
||
message = render_reply(
|
||
layer=layer,
|
||
domain=domain,
|
||
intent=intent,
|
||
answer_class=answer_class,
|
||
dated=dated,
|
||
persona=persona,
|
||
rng=rng,
|
||
)
|
||
message = _strip_option_labels(message, _option_labels(question))
|
||
gold = enforce_combo(intent, answer_class, dated)
|
||
focus = {
|
||
"kind": layer if layer != "none" else None,
|
||
"current_question": str(question.get("current_question") or ""),
|
||
"options": list(question.get("options") or []),
|
||
"case_status": str(question.get("case_status") or "collecting_evidence"),
|
||
}
|
||
return {
|
||
"id": f"C-{layer}-{index:04d}",
|
||
"source": "C",
|
||
"layer": layer,
|
||
"persona": persona,
|
||
"focus": focus,
|
||
"user_message": message,
|
||
"gold": gold,
|
||
"generator": GENERATOR_NAME,
|
||
"reviewer": REVIEWER_NAME,
|
||
"target_before_review": gold,
|
||
}
|
||
|
||
|
||
def _has_dated(message: str) -> bool:
|
||
return bool(YEAR_RE.search(message) or MONTH_RE.search(message))
|
||
|
||
|
||
def review_sample(sample: Mapping[str, Any]) -> dict[str, Any]:
|
||
"""Independent pass: production-prompt rules, no access to target labels."""
|
||
message = str(sample.get("user_message") or "")
|
||
layer = str(sample.get("layer") or "none")
|
||
focus = sample.get("focus") if isinstance(sample.get("focus"), dict) else {}
|
||
question = str(focus.get("current_question") or "")
|
||
dated = _has_dated(message)
|
||
extra = bool(EXTRA_RE.search(message) and dated)
|
||
if STOP_RE.search(message) and not NO_RE.search(message):
|
||
return enforce_combo("stop_rectification", None, False)
|
||
if ASK_RE.search(message) and not YES_RE.search(message) and not NO_RE.search(message):
|
||
return enforce_combo("ask_about_result", None, False)
|
||
if layer == "none" or not question:
|
||
if dated:
|
||
return enforce_combo("provide_new_evidence", None, True)
|
||
return enforce_combo("unclear", None, False)
|
||
if PROVIDE_RE.search(message.strip()) and dated and not YES_RE.search(message) and not NO_RE.search(message) and not UNSURE_RE.search(message):
|
||
return enforce_combo("provide_new_evidence", None, True)
|
||
if layer == "collect":
|
||
if NO_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "no", extra)
|
||
if UNSURE_RE.search(message) and not dated:
|
||
return enforce_combo("answer_current_focus", "unsure", False)
|
||
if UNSURE_RE.search(message) and extra:
|
||
return enforce_combo("answer_current_focus", "unsure", True)
|
||
if dated and not PROVIDE_RE.search(message.strip()):
|
||
weak = bool(WEAK_RE.search(message) or "大概" in message)
|
||
return enforce_combo("answer_current_focus", "weak_yes" if weak else "yes", extra)
|
||
if YES_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "yes", extra)
|
||
if WEAK_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "weak_yes", extra)
|
||
return enforce_combo("unclear", None, False)
|
||
if NO_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "no", extra)
|
||
if UNSURE_RE.search(message) and not YES_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "unsure", extra)
|
||
if WEAK_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "weak_yes", extra)
|
||
if YES_RE.search(message):
|
||
return enforce_combo("answer_current_focus", "yes", extra)
|
||
if dated:
|
||
return enforce_combo("provide_new_evidence", None, True)
|
||
return enforce_combo("unclear", None, False)
|
||
|
||
|
||
def gold_equal(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool:
|
||
return (
|
||
left.get("intent") == right.get("intent")
|
||
and left.get("answer_class") == right.get("answer_class")
|
||
and bool(left.get("has_new_dated_event")) == bool(right.get("has_new_dated_event"))
|
||
)
|
||
|
||
|
||
def _pick_question(pool: Mapping[str, Any], layer: str, rng: random.Random) -> dict[str, Any]:
|
||
rows = list(pool.get(layer) or [])
|
||
if not rows:
|
||
raise RuntimeError(f"empty question pool for layer={layer}")
|
||
return dict(rng.choice(rows))
|
||
|
||
|
||
def generate_source_c(
|
||
pool: Mapping[str, Any],
|
||
*,
|
||
seed: int = SEED,
|
||
extra_factor: float = 1.35,
|
||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]:
|
||
rng = random.Random(seed)
|
||
denylist = holdout_name_denylist()
|
||
accepted: list[dict[str, Any]] = []
|
||
disputed: list[dict[str, Any]] = []
|
||
per_bucket: dict[tuple[str, str, str | None, bool], list[dict[str, Any]]] = {}
|
||
generated = 0
|
||
for layer, rows in QUOTAS.items():
|
||
for intent, answer_class, dated, count in rows:
|
||
need = int(round(count * extra_factor))
|
||
bucket: list[dict[str, Any]] = []
|
||
attempts = 0
|
||
while len(bucket) < need and attempts < need * 8:
|
||
attempts += 1
|
||
generated += 1
|
||
question = _pick_question(pool, layer, rng)
|
||
sample = generate_candidate(
|
||
question,
|
||
intent=intent,
|
||
answer_class=answer_class,
|
||
dated=dated,
|
||
rng=rng,
|
||
index=generated,
|
||
)
|
||
try:
|
||
assert_no_pii(sample["user_message"], denylist)
|
||
except ValueError:
|
||
continue
|
||
reviewed = review_sample(sample)
|
||
sample["review"] = reviewed
|
||
if gold_equal(sample["gold"], reviewed):
|
||
bucket.append(sample)
|
||
else:
|
||
disputed.append(sample)
|
||
per_bucket[(layer, intent, answer_class, dated)] = bucket
|
||
# fill quotas from accepted buckets
|
||
for layer, rows in QUOTAS.items():
|
||
for intent, answer_class, dated, count in rows:
|
||
bucket = per_bucket[(layer, intent, answer_class, dated)]
|
||
if len(bucket) < count:
|
||
# keep what we have; quota deviation is reported
|
||
accepted.extend(bucket)
|
||
else:
|
||
accepted.extend(bucket[:count])
|
||
enforce_length_floor(accepted, rng)
|
||
stats = {
|
||
"generated": generated,
|
||
"accepted": len(accepted),
|
||
"disputed": len(disputed),
|
||
"dispute_rate": (len(disputed) / max(generated, 1)),
|
||
"generator": GENERATOR_NAME,
|
||
"reviewer": REVIEWER_NAME,
|
||
"seed": seed,
|
||
}
|
||
return accepted, disputed, stats
|
||
|
||
|
||
def quota_deviation(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
by_layer: dict[str, Counter[tuple[str, str | None, bool]]] = {
|
||
"choice": Counter(),
|
||
"collect": Counter(),
|
||
"none": Counter(),
|
||
}
|
||
for row in rows:
|
||
layer = str(row.get("layer") or "")
|
||
gold = row.get("gold") or {}
|
||
if layer not in by_layer:
|
||
continue
|
||
by_layer[layer][(
|
||
str(gold.get("intent")),
|
||
gold.get("answer_class"),
|
||
bool(gold.get("has_new_dated_event")),
|
||
)] += 1
|
||
report: dict[str, Any] = {}
|
||
for layer, spec in QUOTAS.items():
|
||
target_total = sum(item[3] for item in spec)
|
||
actual_total = sum(by_layer[layer].values())
|
||
cells = []
|
||
for intent, answer_class, dated, count in spec:
|
||
actual = by_layer[layer][(intent, answer_class, dated)]
|
||
share_target = count / target_total if target_total else 0
|
||
share_actual = actual / actual_total if actual_total else 0
|
||
cells.append({
|
||
"intent": intent,
|
||
"answer_class": answer_class,
|
||
"has_new_dated_event": dated,
|
||
"target": count,
|
||
"actual": actual,
|
||
"share_delta": share_actual - share_target,
|
||
})
|
||
max_delta = max((abs(cell["share_delta"]) for cell in cells), default=0.0)
|
||
report[layer] = {
|
||
"target_total": target_total,
|
||
"actual_total": actual_total,
|
||
"max_share_delta": max_delta,
|
||
"within_20pct": max_delta <= 0.20,
|
||
"cells": cells,
|
||
}
|
||
return report
|
||
|
||
|
||
def length_stats(messages: Iterable[str]) -> dict[str, float]:
|
||
lengths = sorted(len(item) for item in messages)
|
||
if not lengths:
|
||
return {"n": 0, "p25": 0, "median": 0, "p75": 0}
|
||
def pct(p: float) -> float:
|
||
if len(lengths) == 1:
|
||
return float(lengths[0])
|
||
idx = int(round((len(lengths) - 1) * p))
|
||
return float(lengths[idx])
|
||
return {
|
||
"n": len(lengths),
|
||
"p25": pct(0.25),
|
||
"median": pct(0.5),
|
||
"p75": pct(0.75),
|
||
}
|
||
|
||
|
||
def write_readme(
|
||
*,
|
||
synthetic_sha: str,
|
||
simulated_sha: str,
|
||
disputed_sha: str,
|
||
pool_n: int,
|
||
source_c_stats: Mapping[str, Any],
|
||
quotas: Mapping[str, Any],
|
||
lengths: Mapping[str, Any],
|
||
source_b_n: int,
|
||
) -> None:
|
||
source_b_sql = """
|
||
select
|
||
t.id as turn_id,
|
||
t.user_message,
|
||
t.status as turn_status,
|
||
t.created_at,
|
||
c.status as case_status,
|
||
f.question_id,
|
||
f.expected_answer_schema,
|
||
f.status as focus_status
|
||
from public.agentic_rectification_turns t
|
||
join public.agentic_rectification_cases c on c.id = t.case_id
|
||
left join public.agentic_rectification_conversation_focuses f
|
||
on f.case_id = t.case_id
|
||
and f.status = 'active'
|
||
where t.user_message is not null
|
||
and length(btrim(t.user_message)) > 0
|
||
order by t.created_at desc
|
||
limit 500;
|
||
""".strip()
|
||
text = f"""# Jev 意图分类样本（T0）
|
||
|
||
## 来源 A · 既有合成样本（提交）
|
||
|
||
`synthetic.jsonl`，10 条。生产测试 `frontend/tests/rectification-turn-intent-classifier.test.ts` **没有** 10 条带期望输出的 `userMessage` 夹具；本文件用该测试的 `SHUFFLED_CHOICE` 焦点、采集题 prompt，以及测试里实际出现的两句 `userMessage`（`2016年3月入学`、`随便吧`）按分类器提示词写成带标签样本。
|
||
|
||
sha256: `{synthetic_sha}`
|
||
|
||
## 来源 B · 真实会话（本地，不提交）
|
||
|
||
作用：校准锚，不是主测试集。最少 30 条；不足则代表性检验只报数不判定。
|
||
|
||
- 抽取 SQL（staging，`agentic_rectification_turns.user_message` 非空）：
|
||
|
||
```sql
|
||
{source_b_sql}
|
||
```
|
||
|
||
- 本地路径（worktree 外，已 gitignore）：`G:/Ferti/Jyotisha/.cache/jev_intent/source_b.jsonl`
|
||
- 本机抽取条数：{source_b_n}（无 staging 库凭据则为 0，长度分布用任务书兜底 P25≤6 / 中位≤15 / P75≤40）
|
||
- 真值：执行方逐条人工标注；能从后续动作反推的另存 `runtime_intent` 列。**不得提交原文。**
|
||
|
||
## 来源 C · Agent 模拟语料（提交，主测试集）
|
||
|
||
`simulated.jsonl`。问题池 `question_pool.json`：20 例公开 AA holdout 走 `discriminating_event_probes` 出点选题；采集题用 `USER_COLLECT_QUESTION` / retry / `TARGETED_YEAR_PROMPT`。
|
||
|
||
- 生成器：`{GENERATOR_NAME}`（标签先定再写回复；本机无会话模型凭据，不走线上贵模型）
|
||
- 复核器：`{REVIEWER_NAME}`（不同规则、看不到目标标签；不一致进 `disputed.jsonl`，不用第三次投票）
|
||
- 生成统计：{json.dumps(source_c_stats, ensure_ascii=False)}
|
||
- 配额：{json.dumps({k: {'actual': v['actual_total'], 'target': v['target_total'], 'max_share_delta': v['max_share_delta'], 'ok': v['within_20pct']} for k, v in quotas.items()}, ensure_ascii=False)}
|
||
- 长度：{json.dumps(lengths, ensure_ascii=False)}
|
||
- sha256 simulated: `{simulated_sha}`
|
||
- sha256 disputed: `{disputed_sha}`
|
||
- 问题池条数（点选题）：{pool_n}
|
||
|
||
公开案例姓名不得出现在回复正文。
|
||
"""
|
||
README_PATH.write_text(text, encoding="utf-8")
|
||
|
||
|
||
def load_source_b() -> list[dict[str, Any]]:
|
||
path = Path(r"G:\Ferti\Jyotisha\.cache\jev_intent\source_b.jsonl")
|
||
if not path.is_file():
|
||
return []
|
||
rows = []
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
if line.strip():
|
||
rows.append(json.loads(line))
|
||
return rows
|
||
|
||
|
||
def main(argv: Sequence[str] | None = None) -> int:
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--pool-only", action="store_true")
|
||
parser.add_argument("--reuse-pool", action="store_true")
|
||
parser.add_argument("--max-cases", type=int, default=None)
|
||
parser.add_argument("--seed", type=int, default=SEED)
|
||
args = parser.parse_args(argv)
|
||
SAMPLES_DIR.mkdir(parents=True, exist_ok=True)
|
||
if args.reuse_pool and POOL_PATH.is_file() and not args.pool_only:
|
||
pool = json.loads(POOL_PATH.read_text(encoding="utf-8"))
|
||
else:
|
||
print("building question pool from holdout + discriminating_event_probes", flush=True)
|
||
pool = build_question_pool(max_cases=args.max_cases)
|
||
POOL_PATH.write_text(json.dumps(pool, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(
|
||
f"pool choice={len(pool['choice'])} collect={len(pool['collect'])} errors={pool['errors']}",
|
||
flush=True,
|
||
)
|
||
if args.pool_only:
|
||
return 0
|
||
synthetic = source_a_samples()
|
||
write_jsonl(SYNTHETIC_PATH, synthetic)
|
||
accepted, disputed, stats = generate_source_c(pool, seed=args.seed)
|
||
if stats["dispute_rate"] > DISPUTE_RATE_LIMIT:
|
||
print(
|
||
f"dispute_rate={stats['dispute_rate']:.3f} > 0.15, regenerating with extra_factor=1.8",
|
||
flush=True,
|
||
)
|
||
accepted, disputed, stats = generate_source_c(pool, seed=args.seed + 1, extra_factor=1.8)
|
||
write_jsonl(SIMULATED_PATH, accepted)
|
||
write_jsonl(DISPUTED_PATH, disputed)
|
||
quotas = quota_deviation(accepted)
|
||
lengths = length_stats(str(row.get("user_message") or "") for row in accepted)
|
||
source_b = load_source_b()
|
||
write_readme(
|
||
synthetic_sha=sha256_file(SYNTHETIC_PATH),
|
||
simulated_sha=sha256_file(SIMULATED_PATH),
|
||
disputed_sha=sha256_file(DISPUTED_PATH),
|
||
pool_n=len(pool.get("choice") or []),
|
||
source_c_stats=stats,
|
||
quotas=quotas,
|
||
lengths=lengths,
|
||
source_b_n=len(source_b),
|
||
)
|
||
print(json.dumps({
|
||
"synthetic": len(synthetic),
|
||
"simulated": len(accepted),
|
||
"disputed": len(disputed),
|
||
"stats": stats,
|
||
"quotas": {k: {"actual": v["actual_total"], "ok": v["within_20pct"], "delta": v["max_share_delta"]} for k, v in quotas.items()},
|
||
"lengths": lengths,
|
||
"source_b": len(source_b),
|
||
"sha256": {
|
||
"synthetic": sha256_file(SYNTHETIC_PATH),
|
||
"simulated": sha256_file(SIMULATED_PATH),
|
||
"disputed": sha256_file(DISPUTED_PATH),
|
||
},
|
||
}, ensure_ascii=False, indent=2))
|
||
if len(accepted) < 900:
|
||
print("FAILED: simulated < 900", flush=True)
|
||
return 1
|
||
if stats["dispute_rate"] > DISPUTE_RATE_LIMIT:
|
||
print("WARN: dispute_rate still > 15% after one repair pass; proceeding per 让步 1a", flush=True)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|