"""Offline contracts for the Jev intent-classifier research scripts.""" from __future__ import annotations import json import unittest from pathlib import Path import inspect from scripts.research.jev_intent_corpus_build import ( # noqa: E402 GENERATOR_NAME, REVIEWER_NAME, build_review_prompt, holdout_name_denylist, review_sample, source_a_samples, ) from scripts.research.jev_intent_questions import ( CRITERION_MAP, INTENT_CRITERIA, LOST_SEMANTICS, build_questions, build_state, enforce_combo, parse_jev_answers, ) ROOT = Path(__file__).resolve().parents[1] class JevIntentQuestionsTests(unittest.TestCase): def test_combo_nulls_answer_class_when_not_current_focus(self) -> None: parsed = enforce_combo("stop_rectification", "no", True) self.assertEqual(parsed["intent"], "stop_rectification") self.assertIsNone(parsed["answer_class"]) self.assertTrue(parsed["has_new_dated_event"]) def test_combo_rejects_current_focus_without_class(self) -> None: parsed = enforce_combo("answer_current_focus", None, False) self.assertEqual(parsed["intent"], "unclear") self.assertIsNone(parsed["answer_class"]) def test_questions_match_advanced_json_shape(self) -> None: sample = source_a_samples()[0] questions = build_questions(sample) self.assertEqual(questions["intent"]["type"], "choice") self.assertEqual(set(questions["intent"]["criteria"]), set(INTENT_CRITERIA)) self.assertEqual(questions["answer_class"]["type"], "choice") self.assertEqual(questions["has_new_dated_event"]["type"], "noul") self.assertIn("true", questions["has_new_dated_event"]["criteria"]) self.assertIn("false", questions["has_new_dated_event"]["criteria"]) state = build_state(sample) self.assertEqual(set(state), {"current_question", "options", "user_message", "case_status"}) def test_criterion_map_covers_production_prompts(self) -> None: joined = " ".join(row["production"] for row in CRITERION_MAP) self.assertIn("不要按 A/B/C/D", joined) self.assertIn("通常是回答当前问题", joined) self.assertTrue(LOST_SEMANTICS) def test_parse_jev_answers_enforces_combo(self) -> None: parsed = parse_jev_answers({ "intent": {"choice": "stop_rectification", "confidence": 0.9, "probabilities": {}}, "answer_class": {"choice": "no", "confidence": 0.4, "probabilities": {}}, "has_new_dated_event": {"noul": 0.1}, }) self.assertEqual(parsed["intent"], "stop_rectification") self.assertIsNone(parsed["answer_class"]) self.assertEqual(parsed["confidence"], 0.9) class JevIntentCorpusTests(unittest.TestCase): def test_source_a_has_ten_labeled_rows(self) -> None: rows = source_a_samples() self.assertEqual(len(rows), 10) self.assertEqual({row["user_message"] for row in rows} & {"2016年3月入学", "随便吧"}, {"2016年3月入学", "随便吧"}) for row in rows: self.assertIn(row["gold"]["intent"], { "answer_current_focus", "provide_new_evidence", "stop_rectification", "ask_about_result", "unclear", }) def test_reviewer_does_not_read_gold(self) -> None: sample = { "layer": "collect", "user_message": "没有", "focus": {"current_question": "钱的方面,还记得哪年收入明显变过吗?", "options": []}, "gold": {"intent": "stop_rectification", "answer_class": None, "has_new_dated_event": False}, "target_before_review": {"intent": "stop_rectification"}, } prompt = build_review_prompt(sample) self.assertIn("没有", prompt) self.assertIn("钱的方面,还记得哪年收入明显变过吗?", prompt) self.assertNotIn("目标", prompt) tail = prompt.rsplit("用户回复:", 1)[-1] self.assertIn("没有", tail) self.assertNotIn("stop_rectification", tail) self.assertNotIn("template", GENERATOR_NAME.lower()) self.assertNotIn("rule", REVIEWER_NAME.lower()) def test_corpus_build_has_no_literal_word_bank(self) -> None: src = (ROOT / "scripts" / "research" / "jev_intent_corpus_build.py").read_text(encoding="utf-8") self.assertNotIn("bank = {", src) self.assertNotIn('GENERATOR_NAME = "agent-template', src) self.assertNotIn('REVIEWER_NAME = "agent-rule', src) def test_reviewer_does_not_regex_label(self) -> None: src = inspect.getsource(review_sample) self.assertNotIn("re.search", src) self.assertNotIn("re.match", src) self.assertNotIn("STOP_RE", src) self.assertIn("complete_chat", src) def test_current_instructions_match_production_classifier(self) -> None: from scripts.research.jev_intent_current import CHOICE_INSTRUCTIONS, COLLECT_INSTRUCTIONS src = ( ROOT / "frontend" / "src" / "lib" / "rectification-agentic" / "v9" / "turn-intent-classifier.ts" ).read_text(encoding="utf-8") self.assertIn(CHOICE_INSTRUCTIONS, src) self.assertIn(COLLECT_INSTRUCTIONS, src) def test_simulated_has_no_holdout_names(self) -> None: path = ROOT / "scripts" / "research" / "jev_intent_samples" / "simulated.jsonl" if not path.is_file(): self.skipTest("simulated.jsonl not generated yet") names = holdout_name_denylist() for line in path.read_text(encoding="utf-8").splitlines(): row = json.loads(line) message = row["user_message"] for name in names: self.assertNotIn(name, message) def test_report_json_includes_current_source_b(self) -> None: path = ROOT / "docs" / "research" / "jev_intent_2026_09_19.json" if not path.is_file(): self.skipTest("report json not generated yet") payload = json.loads(path.read_text(encoding="utf-8")) current = (payload.get("metrics") or {}).get("current_source_b") self.assertIsNotNone(current) self.assertEqual(current["n"], payload["meta"]["source_b_n"]) self.assertIn("source_b_none_confusion", payload["metrics"]) if __name__ == "__main__": unittest.main()