"""Offline contracts for the Jev intent-classifier research scripts.""" from __future__ import annotations import json import unittest from pathlib import Path from scripts.research.jev_intent_corpus_build import ( # noqa: E402 holdout_name_denylist, review_sample, source_a_samples, ) from scripts.research.jev_intent_questions import ( CRITERION_MAP, INTENT_CRITERIA, LOST_SEMANTICS, build_questions, build_state, enforce_combo, parse_jev_answers, ) ROOT = Path(__file__).resolve().parents[1] class JevIntentQuestionsTests(unittest.TestCase): def test_combo_nulls_answer_class_when_not_current_focus(self) -> None: parsed = enforce_combo("stop_rectification", "no", True) self.assertEqual(parsed["intent"], "stop_rectification") self.assertIsNone(parsed["answer_class"]) self.assertTrue(parsed["has_new_dated_event"]) def test_combo_rejects_current_focus_without_class(self) -> None: parsed = enforce_combo("answer_current_focus", None, False) self.assertEqual(parsed["intent"], "unclear") self.assertIsNone(parsed["answer_class"]) def test_questions_match_advanced_json_shape(self) -> None: sample = source_a_samples()[0] questions = build_questions(sample) self.assertEqual(questions["intent"]["type"], "choice") self.assertEqual(set(questions["intent"]["criteria"]), set(INTENT_CRITERIA)) self.assertEqual(questions["answer_class"]["type"], "choice") self.assertEqual(questions["has_new_dated_event"]["type"], "noul") self.assertIn("true", questions["has_new_dated_event"]["criteria"]) self.assertIn("false", questions["has_new_dated_event"]["criteria"]) state = build_state(sample) self.assertEqual(set(state), {"current_question", "options", "user_message", "case_status"}) def test_criterion_map_covers_production_prompts(self) -> None: joined = " ".join(row["production"] for row in CRITERION_MAP) self.assertIn("不要按 A/B/C/D", joined) self.assertIn("通常是回答当前问题", joined) self.assertTrue(LOST_SEMANTICS) def test_parse_jev_answers_enforces_combo(self) -> None: parsed = parse_jev_answers({ "intent": {"choice": "stop_rectification", "confidence": 0.9, "probabilities": {}}, "answer_class": {"choice": "no", "confidence": 0.4, "probabilities": {}}, "has_new_dated_event": {"noul": 0.1}, }) self.assertEqual(parsed["intent"], "stop_rectification") self.assertIsNone(parsed["answer_class"]) self.assertEqual(parsed["confidence"], 0.9) class JevIntentCorpusTests(unittest.TestCase): def test_source_a_has_ten_labeled_rows(self) -> None: rows = source_a_samples() self.assertEqual(len(rows), 10) self.assertEqual({row["user_message"] for row in rows} & {"2016年3月入学", "随便吧"}, {"2016年3月入学", "随便吧"}) for row in rows: self.assertIn(row["gold"]["intent"], { "answer_current_focus", "provide_new_evidence", "stop_rectification", "ask_about_result", "unclear", }) def test_reviewer_does_not_read_gold(self) -> None: sample = { "layer": "collect", "user_message": "没有", "focus": {"current_question": "钱的方面,还记得哪年收入明显变过吗?", "options": []}, "gold": {"intent": "stop_rectification", "answer_class": None, "has_new_dated_event": False}, } reviewed = review_sample(sample) self.assertEqual(reviewed["intent"], "answer_current_focus") self.assertEqual(reviewed["answer_class"], "no") def test_current_instructions_match_production_classifier(self) -> None: from scripts.research.jev_intent_current import CHOICE_INSTRUCTIONS, COLLECT_INSTRUCTIONS src = ( ROOT / "frontend" / "src" / "lib" / "rectification-agentic" / "v9" / "turn-intent-classifier.ts" ).read_text(encoding="utf-8") self.assertIn(CHOICE_INSTRUCTIONS, src) self.assertIn(COLLECT_INSTRUCTIONS, src) def test_simulated_has_no_holdout_names(self) -> None: path = ROOT / "scripts" / "research" / "jev_intent_samples" / "simulated.jsonl" if not path.is_file(): self.skipTest("simulated.jsonl not generated yet") names = holdout_name_denylist() for line in path.read_text(encoding="utf-8").splitlines(): row = json.loads(line) message = row["user_message"] for name in names: self.assertNotIn(name, message) if __name__ == "__main__": unittest.main()