104 lines
4.2 KiB
Python
104 lines
4.2 KiB
Python
"""Offline contracts for the Jev intent-classifier research scripts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from scripts.research.jev_intent_corpus_build import ( # noqa: E402
|
|
holdout_name_denylist,
|
|
review_sample,
|
|
source_a_samples,
|
|
)
|
|
from scripts.research.jev_intent_questions import (
|
|
CRITERION_MAP,
|
|
INTENT_CRITERIA,
|
|
LOST_SEMANTICS,
|
|
build_questions,
|
|
build_state,
|
|
enforce_combo,
|
|
parse_jev_answers,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class JevIntentQuestionsTests(unittest.TestCase):
|
|
def test_combo_nulls_answer_class_when_not_current_focus(self) -> None:
|
|
parsed = enforce_combo("stop_rectification", "no", True)
|
|
self.assertEqual(parsed["intent"], "stop_rectification")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
self.assertTrue(parsed["has_new_dated_event"])
|
|
|
|
def test_combo_rejects_current_focus_without_class(self) -> None:
|
|
parsed = enforce_combo("answer_current_focus", None, False)
|
|
self.assertEqual(parsed["intent"], "unclear")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
|
|
def test_questions_match_advanced_json_shape(self) -> None:
|
|
sample = source_a_samples()[0]
|
|
questions = build_questions(sample)
|
|
self.assertEqual(questions["intent"]["type"], "choice")
|
|
self.assertEqual(set(questions["intent"]["criteria"]), set(INTENT_CRITERIA))
|
|
self.assertEqual(questions["answer_class"]["type"], "choice")
|
|
self.assertEqual(questions["has_new_dated_event"]["type"], "noul")
|
|
self.assertIn("true", questions["has_new_dated_event"]["criteria"])
|
|
self.assertIn("false", questions["has_new_dated_event"]["criteria"])
|
|
state = build_state(sample)
|
|
self.assertEqual(set(state), {"current_question", "options", "user_message", "case_status"})
|
|
|
|
def test_criterion_map_covers_production_prompts(self) -> None:
|
|
joined = " ".join(row["production"] for row in CRITERION_MAP)
|
|
self.assertIn("不要按 A/B/C/D", joined)
|
|
self.assertIn("通常是回答当前问题", joined)
|
|
self.assertTrue(LOST_SEMANTICS)
|
|
|
|
def test_parse_jev_answers_enforces_combo(self) -> None:
|
|
parsed = parse_jev_answers({
|
|
"intent": {"choice": "stop_rectification", "confidence": 0.9, "probabilities": {}},
|
|
"answer_class": {"choice": "no", "confidence": 0.4, "probabilities": {}},
|
|
"has_new_dated_event": {"noul": 0.1},
|
|
})
|
|
self.assertEqual(parsed["intent"], "stop_rectification")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
self.assertEqual(parsed["confidence"], 0.9)
|
|
|
|
|
|
class JevIntentCorpusTests(unittest.TestCase):
|
|
def test_source_a_has_ten_labeled_rows(self) -> None:
|
|
rows = source_a_samples()
|
|
self.assertEqual(len(rows), 10)
|
|
self.assertEqual({row["user_message"] for row in rows} & {"2016年3月入学", "随便吧"}, {"2016年3月入学", "随便吧"})
|
|
for row in rows:
|
|
self.assertIn(row["gold"]["intent"], {
|
|
"answer_current_focus", "provide_new_evidence", "stop_rectification",
|
|
"ask_about_result", "unclear",
|
|
})
|
|
|
|
def test_reviewer_does_not_read_gold(self) -> None:
|
|
sample = {
|
|
"layer": "collect",
|
|
"user_message": "没有",
|
|
"focus": {"current_question": "钱的方面，还记得哪年收入明显变过吗？", "options": []},
|
|
"gold": {"intent": "stop_rectification", "answer_class": None, "has_new_dated_event": False},
|
|
}
|
|
reviewed = review_sample(sample)
|
|
self.assertEqual(reviewed["intent"], "answer_current_focus")
|
|
self.assertEqual(reviewed["answer_class"], "no")
|
|
|
|
def test_simulated_has_no_holdout_names(self) -> None:
|
|
path = ROOT / "scripts" / "research" / "jev_intent_samples" / "simulated.jsonl"
|
|
if not path.is_file():
|
|
self.skipTest("simulated.jsonl not generated yet")
|
|
names = holdout_name_denylist()
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
row = json.loads(line)
|
|
message = row["user_message"]
|
|
for name in names:
|
|
self.assertNotIn(name, message)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|