"""Offline contracts for the Jev intent-classifier research scripts.""" from __future__ import annotations import json import unittest from pathlib import Path import inspect from scripts.research.jev_intent_corpus_build import ( # noqa: E402 GENERATOR_NAME, REVIEWER_NAME, build_review_prompt, holdout_name_denylist, review_sample, source_a_samples, ) from scripts.research.jev_intent_questions import ( CONTINUES_PREVIOUS_TURN_INSTRUCTIONS, CRITERION_MAP, INTENT_CRITERIA, LOST_SEMANTICS, build_questions, build_state, enforce_combo, parse_jev_answers, ) from scripts.research.jev_intent_source_b_v2 import attach_previous_turns ROOT = Path(__file__).resolve().parents[1] class JevIntentQuestionsTests(unittest.TestCase): def test_combo_nulls_answer_class_when_not_current_focus(self) -> None: parsed = enforce_combo("stop_rectification", "no", True) self.assertEqual(parsed["intent"], "stop_rectification") self.assertIsNone(parsed["answer_class"]) self.assertTrue(parsed["has_new_dated_event"]) def test_combo_rejects_current_focus_without_class(self) -> None: parsed = enforce_combo("answer_current_focus", None, False) self.assertEqual(parsed["intent"], "unclear") self.assertIsNone(parsed["answer_class"]) def test_questions_match_advanced_json_shape(self) -> None: sample = source_a_samples()[0] questions = build_questions(sample) self.assertEqual(questions["intent"]["type"], "choice") self.assertEqual(set(questions["intent"]["criteria"]), set(INTENT_CRITERIA)) self.assertEqual(questions["answer_class"]["type"], "choice") self.assertEqual(questions["has_new_dated_event"]["type"], "noul") self.assertIn("true", questions["has_new_dated_event"]["criteria"]) self.assertIn("false", questions["has_new_dated_event"]["criteria"]) state = build_state(sample) self.assertEqual(set(state), {"current_question", "options", "user_message", "case_status"}) def test_criterion_map_covers_production_prompts(self) -> None: joined = " ".join(row["production"] for row in CRITERION_MAP) self.assertIn("不要按 A/B/C/D", joined) self.assertIn("通常是回答当前问题", joined) self.assertTrue(LOST_SEMANTICS) def test_parse_jev_answers_enforces_combo(self) -> None: parsed = parse_jev_answers({ "intent": {"choice": "stop_rectification", "confidence": 0.9, "probabilities": {}}, "answer_class": {"choice": "no", "confidence": 0.4, "probabilities": {}}, "has_new_dated_event": {"noul": 0.1}, }) self.assertEqual(parsed["intent"], "stop_rectification") self.assertIsNone(parsed["answer_class"]) self.assertEqual(parsed["confidence"], 0.9) class JevIntentPreviousTurnTests(unittest.TestCase): def test_v1_v2_state_shape_keeps_v0_questions(self) -> None: sample = dict(source_a_samples()[0]) sample["previous_turn"] = {"assistant_message": "上一问", "user_message": "上一句"} sample["previous_decision"] = {"intent": "unclear", "answer_class": None} v0 = build_state(sample, variant="v0") self.assertEqual(set(v0), {"current_question", "options", "user_message", "case_status"}) v1 = build_state(sample, variant="v1") self.assertEqual(v1["previous_turn"], {"assistant_message": "上一问", "user_message": "上一句"}) self.assertNotIn("previous_decision", v1) v2 = build_state(sample, variant="v2") self.assertEqual(v2["previous_decision"], {"intent": "unclear", "answer_class": None}) v0_questions = build_questions(sample) v2_questions = build_questions(sample, variant="v2") self.assertNotIn("continues_previous_turn", v0_questions) self.assertEqual(set(v0_questions["intent"]["criteria"]), set(v2_questions["intent"]["criteria"])) self.assertEqual(v2_questions["continues_previous_turn"]["type"], "noul") self.assertEqual( v2_questions["continues_previous_turn"]["instructions"], CONTINUES_PREVIOUS_TURN_INSTRUCTIONS, ) def test_continues_rule_adopts_previous_intent_only_at_threshold(self) -> None: answers = { "intent": {"choice": "answer_current_focus", "confidence": 0.95, "probabilities": {}}, "answer_class": {"choice": "yes", "confidence": 0.91, "probabilities": {}}, "has_new_dated_event": {"noul": 0.1}, "continues_previous_turn": {"noul": 0.9}, } adopted = parse_jev_answers( answers, previous_decision={"intent": "unclear", "answer_class": None}, apply_continue=True, ) self.assertEqual(adopted["intent"], "unclear") self.assertIsNone(adopted["answer_class"]) self.assertTrue(adopted["continued"]) self.assertEqual(adopted["confidence"], 0.9) kept = parse_jev_answers( {**answers, "continues_previous_turn": {"noul": 0.89}}, previous_decision={"intent": "stop_rectification", "answer_class": None}, apply_continue=True, ) self.assertEqual(kept["intent"], "answer_current_focus") self.assertEqual(kept["answer_class"], "yes") self.assertFalse(kept["continued"]) focused = parse_jev_answers( answers, previous_decision={"intent": "answer_current_focus", "answer_class": "no"}, apply_continue=True, ) self.assertEqual(focused["intent"], "answer_current_focus") self.assertEqual(focused["answer_class"], "no") def test_case_first_turn_previous_turn_is_null(self) -> None: rows = attach_previous_turns([ { "id": "first", "case_id": "case-1", "created_at": "2026-09-01T00:00:00Z", "user_message": "第一句", "assistant_message": "第一答", }, { "id": "second", "case_id": "case-1", "created_at": "2026-09-01T00:01:00Z", "user_message": "继续", "assistant_message": "第二答", }, { "id": "orphan", "user_message": "没有案件", "assistant_message": "答", }, ]) by_id = {row["id"]: row for row in rows} self.assertIsNone(by_id["first"]["previous_turn"]) self.assertEqual(by_id["first"]["previous_turn_source"], "case_first") self.assertEqual(by_id["second"]["previous_turn"], { "assistant_message": "第一答", "user_message": "第一句", }) self.assertIsNone(by_id["orphan"]["previous_turn"]) state = build_state(by_id["first"], variant="v1") self.assertIsNone(state["previous_turn"]) self.assertIn("user_message", state) def test_closed_focus_is_not_the_current_question(self) -> None: from scripts.research.jev_intent_source_b_v2 import prepare_extract rows = prepare_extract([ { "turn_id": "open-turn", "case_id": "case-1", "created_at": "2026-09-02T00:00:00+00:00", "user_message": "没有", "assistant_message": "记下了", "case_status": "collecting_evidence", "focus_status": "active", "asked_at": "2026-09-01T00:00:00+00:00", "resolved_at": None, "schema": {"collect": True, "prompt": "有没有换过工作？"}, }, { "turn_id": "after-close", "case_id": "case-1", "created_at": "2026-09-03T00:00:00+00:00", "user_message": "先看结果", "assistant_message": "还在收", "case_status": "collecting_evidence", "focus_status": "resolved", "asked_at": "2026-09-01T00:00:00+00:00", "resolved_at": "2026-09-02T12:00:00+00:00", "schema": {"collect": True, "prompt": "有没有换过工作？"}, }, ]) by_id = {row["turn_id"]: row for row in rows} self.assertEqual(by_id["open-turn"]["layer"], "collect") self.assertEqual(by_id["open-turn"]["focus"]["current_question"], "有没有换过工作？") self.assertEqual(by_id["after-close"]["layer"], "none") self.assertIsNone(by_id["after-close"]["focus"]) self.assertTrue(by_id["after-close"]["focus_stale"]) self.assertEqual(by_id["after-close"]["previous_turn"]["user_message"], "没有") class JevIntentCorpusTests(unittest.TestCase): def test_source_a_has_ten_labeled_rows(self) -> None: rows = source_a_samples() self.assertEqual(len(rows), 10) self.assertEqual({row["user_message"] for row in rows} & {"2016年3月入学", "随便吧"}, {"2016年3月入学", "随便吧"}) for row in rows: self.assertIn(row["gold"]["intent"], { "answer_current_focus", "provide_new_evidence", "stop_rectification", "ask_about_result", "unclear", }) def test_reviewer_does_not_read_gold(self) -> None: sample = { "layer": "collect", "user_message": "没有", "focus": {"current_question": "钱的方面，还记得哪年收入明显变过吗？", "options": []}, "gold": {"intent": "stop_rectification", "answer_class": None, "has_new_dated_event": False}, "target_before_review": {"intent": "stop_rectification"}, } prompt = build_review_prompt(sample) self.assertIn("没有", prompt) self.assertIn("钱的方面，还记得哪年收入明显变过吗？", prompt) self.assertNotIn("目标", prompt) tail = prompt.rsplit("用户回复：", 1)[-1] self.assertIn("没有", tail) self.assertNotIn("stop_rectification", tail) self.assertNotIn("template", GENERATOR_NAME.lower()) self.assertNotIn("rule", REVIEWER_NAME.lower()) def test_corpus_build_has_no_literal_word_bank(self) -> None: src = (ROOT / "scripts" / "research" / "jev_intent_corpus_build.py").read_text(encoding="utf-8") self.assertNotIn("bank = {", src) self.assertNotIn('GENERATOR_NAME = "agent-template', src) self.assertNotIn('REVIEWER_NAME = "agent-rule', src) def test_reviewer_does_not_regex_label(self) -> None: src = inspect.getsource(review_sample) self.assertNotIn("re.search", src) self.assertNotIn("re.match", src) self.assertNotIn("STOP_RE", src) self.assertIn("complete_chat", src) def test_current_instructions_match_production_classifier(self) -> None: from scripts.research.jev_intent_current import CHOICE_INSTRUCTIONS, COLLECT_INSTRUCTIONS src = ( ROOT / "frontend" / "src" / "lib" / "rectification-agentic" / "v9" / "turn-intent-classifier.ts" ).read_text(encoding="utf-8") self.assertIn(CHOICE_INSTRUCTIONS, src) self.assertIn(COLLECT_INSTRUCTIONS, src) def test_simulated_has_no_holdout_names(self) -> None: path = ROOT / "scripts" / "research" / "jev_intent_samples" / "simulated.jsonl" if not path.is_file(): self.skipTest("simulated.jsonl not generated yet") names = holdout_name_denylist() for line in path.read_text(encoding="utf-8").splitlines(): row = json.loads(line) message = row["user_message"] for name in names: self.assertNotIn(name, message) def test_report_json_includes_current_source_b(self) -> None: path = ROOT / "docs" / "research" / "jev_intent_2026_09_19.json" if not path.is_file(): self.skipTest("report json not generated yet") payload = json.loads(path.read_text(encoding="utf-8")) current = (payload.get("metrics") or {}).get("current_source_b") self.assertIsNotNone(current) self.assertEqual(current["n"], payload["meta"]["source_b_n"]) self.assertIn("source_b_none_confusion", payload["metrics"]) if __name__ == "__main__": unittest.main()