148 lines
6.2 KiB
Python
148 lines
6.2 KiB
Python
"""Offline contracts for the Jev intent-classifier research scripts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import inspect
|
|
|
|
from scripts.research.jev_intent_corpus_build import ( # noqa: E402
|
|
GENERATOR_NAME,
|
|
REVIEWER_NAME,
|
|
build_review_prompt,
|
|
holdout_name_denylist,
|
|
review_sample,
|
|
source_a_samples,
|
|
)
|
|
from scripts.research.jev_intent_questions import (
|
|
CRITERION_MAP,
|
|
INTENT_CRITERIA,
|
|
LOST_SEMANTICS,
|
|
build_questions,
|
|
build_state,
|
|
enforce_combo,
|
|
parse_jev_answers,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class JevIntentQuestionsTests(unittest.TestCase):
|
|
def test_combo_nulls_answer_class_when_not_current_focus(self) -> None:
|
|
parsed = enforce_combo("stop_rectification", "no", True)
|
|
self.assertEqual(parsed["intent"], "stop_rectification")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
self.assertTrue(parsed["has_new_dated_event"])
|
|
|
|
def test_combo_rejects_current_focus_without_class(self) -> None:
|
|
parsed = enforce_combo("answer_current_focus", None, False)
|
|
self.assertEqual(parsed["intent"], "unclear")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
|
|
def test_questions_match_advanced_json_shape(self) -> None:
|
|
sample = source_a_samples()[0]
|
|
questions = build_questions(sample)
|
|
self.assertEqual(questions["intent"]["type"], "choice")
|
|
self.assertEqual(set(questions["intent"]["criteria"]), set(INTENT_CRITERIA))
|
|
self.assertEqual(questions["answer_class"]["type"], "choice")
|
|
self.assertEqual(questions["has_new_dated_event"]["type"], "noul")
|
|
self.assertIn("true", questions["has_new_dated_event"]["criteria"])
|
|
self.assertIn("false", questions["has_new_dated_event"]["criteria"])
|
|
state = build_state(sample)
|
|
self.assertEqual(set(state), {"current_question", "options", "user_message", "case_status"})
|
|
|
|
def test_criterion_map_covers_production_prompts(self) -> None:
|
|
joined = " ".join(row["production"] for row in CRITERION_MAP)
|
|
self.assertIn("不要按 A/B/C/D", joined)
|
|
self.assertIn("通常是回答当前问题", joined)
|
|
self.assertTrue(LOST_SEMANTICS)
|
|
|
|
def test_parse_jev_answers_enforces_combo(self) -> None:
|
|
parsed = parse_jev_answers({
|
|
"intent": {"choice": "stop_rectification", "confidence": 0.9, "probabilities": {}},
|
|
"answer_class": {"choice": "no", "confidence": 0.4, "probabilities": {}},
|
|
"has_new_dated_event": {"noul": 0.1},
|
|
})
|
|
self.assertEqual(parsed["intent"], "stop_rectification")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
self.assertEqual(parsed["confidence"], 0.9)
|
|
|
|
|
|
class JevIntentCorpusTests(unittest.TestCase):
|
|
def test_source_a_has_ten_labeled_rows(self) -> None:
|
|
rows = source_a_samples()
|
|
self.assertEqual(len(rows), 10)
|
|
self.assertEqual({row["user_message"] for row in rows} & {"2016年3月入学", "随便吧"}, {"2016年3月入学", "随便吧"})
|
|
for row in rows:
|
|
self.assertIn(row["gold"]["intent"], {
|
|
"answer_current_focus", "provide_new_evidence", "stop_rectification",
|
|
"ask_about_result", "unclear",
|
|
})
|
|
|
|
def test_reviewer_does_not_read_gold(self) -> None:
|
|
sample = {
|
|
"layer": "collect",
|
|
"user_message": "没有",
|
|
"focus": {"current_question": "钱的方面，还记得哪年收入明显变过吗？", "options": []},
|
|
"gold": {"intent": "stop_rectification", "answer_class": None, "has_new_dated_event": False},
|
|
"target_before_review": {"intent": "stop_rectification"},
|
|
}
|
|
prompt = build_review_prompt(sample)
|
|
self.assertIn("没有", prompt)
|
|
self.assertIn("钱的方面，还记得哪年收入明显变过吗？", prompt)
|
|
self.assertNotIn("目标", prompt)
|
|
tail = prompt.rsplit("用户回复：", 1)[-1]
|
|
self.assertIn("没有", tail)
|
|
self.assertNotIn("stop_rectification", tail)
|
|
self.assertNotIn("template", GENERATOR_NAME.lower())
|
|
self.assertNotIn("rule", REVIEWER_NAME.lower())
|
|
|
|
def test_corpus_build_has_no_literal_word_bank(self) -> None:
|
|
src = (ROOT / "scripts" / "research" / "jev_intent_corpus_build.py").read_text(encoding="utf-8")
|
|
self.assertNotIn("bank = {", src)
|
|
self.assertNotIn('GENERATOR_NAME = "agent-template', src)
|
|
self.assertNotIn('REVIEWER_NAME = "agent-rule', src)
|
|
|
|
def test_reviewer_does_not_regex_label(self) -> None:
|
|
src = inspect.getsource(review_sample)
|
|
self.assertNotIn("re.search", src)
|
|
self.assertNotIn("re.match", src)
|
|
self.assertNotIn("STOP_RE", src)
|
|
self.assertIn("complete_chat", src)
|
|
|
|
def test_current_instructions_match_production_classifier(self) -> None:
|
|
from scripts.research.jev_intent_current import CHOICE_INSTRUCTIONS, COLLECT_INSTRUCTIONS
|
|
|
|
src = (
|
|
ROOT / "frontend" / "src" / "lib" / "rectification-agentic" / "v9" / "turn-intent-classifier.ts"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn(CHOICE_INSTRUCTIONS, src)
|
|
self.assertIn(COLLECT_INSTRUCTIONS, src)
|
|
|
|
def test_simulated_has_no_holdout_names(self) -> None:
|
|
path = ROOT / "scripts" / "research" / "jev_intent_samples" / "simulated.jsonl"
|
|
if not path.is_file():
|
|
self.skipTest("simulated.jsonl not generated yet")
|
|
names = holdout_name_denylist()
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
row = json.loads(line)
|
|
message = row["user_message"]
|
|
for name in names:
|
|
self.assertNotIn(name, message)
|
|
|
|
def test_report_json_includes_current_source_b(self) -> None:
|
|
path = ROOT / "docs" / "research" / "jev_intent_2026_09_19.json"
|
|
if not path.is_file():
|
|
self.skipTest("report json not generated yet")
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
current = (payload.get("metrics") or {}).get("current_source_b")
|
|
self.assertIsNotNone(current)
|
|
self.assertEqual(current["n"], payload["meta"]["source_b_n"])
|
|
self.assertIn("source_b_none_confusion", payload["metrics"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|