Jesse_ChenandCursor
0b68fa97c4
The earlier report had no previous-turn rows. This run measures V0, V1, V2, and Flash on the re-extracted corpus and records that the real sample is not representative of the simulated set. Co-authored-by: Cursor <cursoragent@cursor.com>
276 lines
12 KiB
Python
276 lines
12 KiB
Python
"""Offline contracts for the Jev intent-classifier research scripts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import inspect
|
|
|
|
from scripts.research.jev_intent_corpus_build import ( # noqa: E402
|
|
GENERATOR_NAME,
|
|
REVIEWER_NAME,
|
|
build_review_prompt,
|
|
holdout_name_denylist,
|
|
review_sample,
|
|
source_a_samples,
|
|
)
|
|
from scripts.research.jev_intent_questions import (
|
|
CONTINUES_PREVIOUS_TURN_INSTRUCTIONS,
|
|
CRITERION_MAP,
|
|
INTENT_CRITERIA,
|
|
LOST_SEMANTICS,
|
|
build_questions,
|
|
build_state,
|
|
enforce_combo,
|
|
parse_jev_answers,
|
|
)
|
|
from scripts.research.jev_intent_source_b_v2 import attach_previous_turns
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class JevIntentQuestionsTests(unittest.TestCase):
|
|
def test_combo_nulls_answer_class_when_not_current_focus(self) -> None:
|
|
parsed = enforce_combo("stop_rectification", "no", True)
|
|
self.assertEqual(parsed["intent"], "stop_rectification")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
self.assertTrue(parsed["has_new_dated_event"])
|
|
|
|
def test_combo_rejects_current_focus_without_class(self) -> None:
|
|
parsed = enforce_combo("answer_current_focus", None, False)
|
|
self.assertEqual(parsed["intent"], "unclear")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
|
|
def test_questions_match_advanced_json_shape(self) -> None:
|
|
sample = source_a_samples()[0]
|
|
questions = build_questions(sample)
|
|
self.assertEqual(questions["intent"]["type"], "choice")
|
|
self.assertEqual(set(questions["intent"]["criteria"]), set(INTENT_CRITERIA))
|
|
self.assertEqual(questions["answer_class"]["type"], "choice")
|
|
self.assertEqual(questions["has_new_dated_event"]["type"], "noul")
|
|
self.assertIn("true", questions["has_new_dated_event"]["criteria"])
|
|
self.assertIn("false", questions["has_new_dated_event"]["criteria"])
|
|
state = build_state(sample)
|
|
self.assertEqual(set(state), {"current_question", "options", "user_message", "case_status"})
|
|
|
|
def test_criterion_map_covers_production_prompts(self) -> None:
|
|
joined = " ".join(row["production"] for row in CRITERION_MAP)
|
|
self.assertIn("不要按 A/B/C/D", joined)
|
|
self.assertIn("通常是回答当前问题", joined)
|
|
self.assertTrue(LOST_SEMANTICS)
|
|
|
|
def test_parse_jev_answers_enforces_combo(self) -> None:
|
|
parsed = parse_jev_answers({
|
|
"intent": {"choice": "stop_rectification", "confidence": 0.9, "probabilities": {}},
|
|
"answer_class": {"choice": "no", "confidence": 0.4, "probabilities": {}},
|
|
"has_new_dated_event": {"noul": 0.1},
|
|
})
|
|
self.assertEqual(parsed["intent"], "stop_rectification")
|
|
self.assertIsNone(parsed["answer_class"])
|
|
self.assertEqual(parsed["confidence"], 0.9)
|
|
|
|
|
|
class JevIntentPreviousTurnTests(unittest.TestCase):
|
|
def test_v1_v2_state_shape_keeps_v0_questions(self) -> None:
|
|
sample = dict(source_a_samples()[0])
|
|
sample["previous_turn"] = {"assistant_message": "上一问", "user_message": "上一句"}
|
|
sample["previous_decision"] = {"intent": "unclear", "answer_class": None}
|
|
v0 = build_state(sample, variant="v0")
|
|
self.assertEqual(set(v0), {"current_question", "options", "user_message", "case_status"})
|
|
v1 = build_state(sample, variant="v1")
|
|
self.assertEqual(v1["previous_turn"], {"assistant_message": "上一问", "user_message": "上一句"})
|
|
self.assertNotIn("previous_decision", v1)
|
|
v2 = build_state(sample, variant="v2")
|
|
self.assertEqual(v2["previous_decision"], {"intent": "unclear", "answer_class": None})
|
|
v0_questions = build_questions(sample)
|
|
v2_questions = build_questions(sample, variant="v2")
|
|
self.assertNotIn("continues_previous_turn", v0_questions)
|
|
self.assertEqual(set(v0_questions["intent"]["criteria"]), set(v2_questions["intent"]["criteria"]))
|
|
self.assertEqual(v2_questions["continues_previous_turn"]["type"], "noul")
|
|
self.assertEqual(
|
|
v2_questions["continues_previous_turn"]["instructions"],
|
|
CONTINUES_PREVIOUS_TURN_INSTRUCTIONS,
|
|
)
|
|
|
|
def test_continues_rule_adopts_previous_intent_only_at_threshold(self) -> None:
|
|
answers = {
|
|
"intent": {"choice": "answer_current_focus", "confidence": 0.95, "probabilities": {}},
|
|
"answer_class": {"choice": "yes", "confidence": 0.91, "probabilities": {}},
|
|
"has_new_dated_event": {"noul": 0.1},
|
|
"continues_previous_turn": {"noul": 0.9},
|
|
}
|
|
adopted = parse_jev_answers(
|
|
answers,
|
|
previous_decision={"intent": "unclear", "answer_class": None},
|
|
apply_continue=True,
|
|
)
|
|
self.assertEqual(adopted["intent"], "unclear")
|
|
self.assertIsNone(adopted["answer_class"])
|
|
self.assertTrue(adopted["continued"])
|
|
self.assertEqual(adopted["confidence"], 0.9)
|
|
kept = parse_jev_answers(
|
|
{**answers, "continues_previous_turn": {"noul": 0.89}},
|
|
previous_decision={"intent": "stop_rectification", "answer_class": None},
|
|
apply_continue=True,
|
|
)
|
|
self.assertEqual(kept["intent"], "answer_current_focus")
|
|
self.assertEqual(kept["answer_class"], "yes")
|
|
self.assertFalse(kept["continued"])
|
|
focused = parse_jev_answers(
|
|
answers,
|
|
previous_decision={"intent": "answer_current_focus", "answer_class": "no"},
|
|
apply_continue=True,
|
|
)
|
|
self.assertEqual(focused["intent"], "answer_current_focus")
|
|
self.assertEqual(focused["answer_class"], "no")
|
|
|
|
def test_case_first_turn_previous_turn_is_null(self) -> None:
|
|
rows = attach_previous_turns([
|
|
{
|
|
"id": "first",
|
|
"case_id": "case-1",
|
|
"created_at": "2026-09-01T00:00:00Z",
|
|
"user_message": "第一句",
|
|
"assistant_message": "第一答",
|
|
},
|
|
{
|
|
"id": "second",
|
|
"case_id": "case-1",
|
|
"created_at": "2026-09-01T00:01:00Z",
|
|
"user_message": "继续",
|
|
"assistant_message": "第二答",
|
|
},
|
|
{
|
|
"id": "orphan",
|
|
"user_message": "没有案件",
|
|
"assistant_message": "答",
|
|
},
|
|
])
|
|
by_id = {row["id"]: row for row in rows}
|
|
self.assertIsNone(by_id["first"]["previous_turn"])
|
|
self.assertEqual(by_id["first"]["previous_turn_source"], "case_first")
|
|
self.assertEqual(by_id["second"]["previous_turn"], {
|
|
"assistant_message": "第一答",
|
|
"user_message": "第一句",
|
|
})
|
|
self.assertIsNone(by_id["orphan"]["previous_turn"])
|
|
state = build_state(by_id["first"], variant="v1")
|
|
self.assertIsNone(state["previous_turn"])
|
|
self.assertIn("user_message", state)
|
|
|
|
def test_closed_focus_is_not_the_current_question(self) -> None:
|
|
from scripts.research.jev_intent_source_b_v2 import prepare_extract
|
|
|
|
rows = prepare_extract([
|
|
{
|
|
"turn_id": "open-turn",
|
|
"case_id": "case-1",
|
|
"created_at": "2026-09-02T00:00:00+00:00",
|
|
"user_message": "没有",
|
|
"assistant_message": "记下了",
|
|
"case_status": "collecting_evidence",
|
|
"focus_status": "active",
|
|
"asked_at": "2026-09-01T00:00:00+00:00",
|
|
"resolved_at": None,
|
|
"schema": {"collect": True, "prompt": "有没有换过工作？"},
|
|
},
|
|
{
|
|
"turn_id": "after-close",
|
|
"case_id": "case-1",
|
|
"created_at": "2026-09-03T00:00:00+00:00",
|
|
"user_message": "先看结果",
|
|
"assistant_message": "还在收",
|
|
"case_status": "collecting_evidence",
|
|
"focus_status": "resolved",
|
|
"asked_at": "2026-09-01T00:00:00+00:00",
|
|
"resolved_at": "2026-09-02T12:00:00+00:00",
|
|
"schema": {"collect": True, "prompt": "有没有换过工作？"},
|
|
},
|
|
])
|
|
by_id = {row["turn_id"]: row for row in rows}
|
|
self.assertEqual(by_id["open-turn"]["layer"], "collect")
|
|
self.assertEqual(by_id["open-turn"]["focus"]["current_question"], "有没有换过工作？")
|
|
self.assertEqual(by_id["after-close"]["layer"], "none")
|
|
self.assertIsNone(by_id["after-close"]["focus"])
|
|
self.assertTrue(by_id["after-close"]["focus_stale"])
|
|
self.assertEqual(by_id["after-close"]["previous_turn"]["user_message"], "没有")
|
|
|
|
|
|
class JevIntentCorpusTests(unittest.TestCase):
|
|
def test_source_a_has_ten_labeled_rows(self) -> None:
|
|
rows = source_a_samples()
|
|
self.assertEqual(len(rows), 10)
|
|
self.assertEqual({row["user_message"] for row in rows} & {"2016年3月入学", "随便吧"}, {"2016年3月入学", "随便吧"})
|
|
for row in rows:
|
|
self.assertIn(row["gold"]["intent"], {
|
|
"answer_current_focus", "provide_new_evidence", "stop_rectification",
|
|
"ask_about_result", "unclear",
|
|
})
|
|
|
|
def test_reviewer_does_not_read_gold(self) -> None:
|
|
sample = {
|
|
"layer": "collect",
|
|
"user_message": "没有",
|
|
"focus": {"current_question": "钱的方面，还记得哪年收入明显变过吗？", "options": []},
|
|
"gold": {"intent": "stop_rectification", "answer_class": None, "has_new_dated_event": False},
|
|
"target_before_review": {"intent": "stop_rectification"},
|
|
}
|
|
prompt = build_review_prompt(sample)
|
|
self.assertIn("没有", prompt)
|
|
self.assertIn("钱的方面，还记得哪年收入明显变过吗？", prompt)
|
|
self.assertNotIn("目标", prompt)
|
|
tail = prompt.rsplit("用户回复：", 1)[-1]
|
|
self.assertIn("没有", tail)
|
|
self.assertNotIn("stop_rectification", tail)
|
|
self.assertNotIn("template", GENERATOR_NAME.lower())
|
|
self.assertNotIn("rule", REVIEWER_NAME.lower())
|
|
|
|
def test_corpus_build_has_no_literal_word_bank(self) -> None:
|
|
src = (ROOT / "scripts" / "research" / "jev_intent_corpus_build.py").read_text(encoding="utf-8")
|
|
self.assertNotIn("bank = {", src)
|
|
self.assertNotIn('GENERATOR_NAME = "agent-template', src)
|
|
self.assertNotIn('REVIEWER_NAME = "agent-rule', src)
|
|
|
|
def test_reviewer_does_not_regex_label(self) -> None:
|
|
src = inspect.getsource(review_sample)
|
|
self.assertNotIn("re.search", src)
|
|
self.assertNotIn("re.match", src)
|
|
self.assertNotIn("STOP_RE", src)
|
|
self.assertIn("complete_chat", src)
|
|
|
|
def test_current_instructions_match_production_classifier(self) -> None:
|
|
from scripts.research.jev_intent_current import CHOICE_INSTRUCTIONS, COLLECT_INSTRUCTIONS
|
|
|
|
src = (
|
|
ROOT / "frontend" / "src" / "lib" / "rectification-agentic" / "v9" / "turn-intent-classifier.ts"
|
|
).read_text(encoding="utf-8")
|
|
self.assertIn(CHOICE_INSTRUCTIONS, src)
|
|
self.assertIn(COLLECT_INSTRUCTIONS, src)
|
|
|
|
def test_simulated_has_no_holdout_names(self) -> None:
|
|
path = ROOT / "scripts" / "research" / "jev_intent_samples" / "simulated.jsonl"
|
|
if not path.is_file():
|
|
self.skipTest("simulated.jsonl not generated yet")
|
|
names = holdout_name_denylist()
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
row = json.loads(line)
|
|
message = row["user_message"]
|
|
for name in names:
|
|
self.assertNotIn(name, message)
|
|
|
|
def test_report_json_includes_current_source_b(self) -> None:
|
|
path = ROOT / "docs" / "research" / "jev_intent_2026_09_19.json"
|
|
if not path.is_file():
|
|
self.skipTest("report json not generated yet")
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
current = (payload.get("metrics") or {}).get("current_source_b")
|
|
self.assertIsNotNone(current)
|
|
self.assertEqual(current["n"], payload["meta"]["source_b_n"])
|
|
self.assertIn("source_b_none_confusion", payload["metrics"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|