research: compare Jev intent state with the previous turn
V0 on the existing 157 real rows matches the 09-19 cache. The staging extract has no case linkage, so V1/V2 are unmeasured and the verdict stays 缺数据.
This commit is contained in:
@@ -17,6 +17,7 @@ from scripts.research.jev_intent_corpus_build import ( # noqa: E402
|
||||
source_a_samples,
|
||||
)
|
||||
from scripts.research.jev_intent_questions import (
|
||||
CONTINUES_PREVIOUS_TURN_INSTRUCTIONS,
|
||||
CRITERION_MAP,
|
||||
INTENT_CRITERIA,
|
||||
LOST_SEMANTICS,
|
||||
@@ -25,6 +26,7 @@ from scripts.research.jev_intent_questions import (
|
||||
enforce_combo,
|
||||
parse_jev_answers,
|
||||
)
|
||||
from scripts.research.jev_intent_source_b_v2 import attach_previous_turns
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
@@ -70,6 +72,95 @@ class JevIntentQuestionsTests(unittest.TestCase):
|
||||
self.assertEqual(parsed["confidence"], 0.9)
|
||||
|
||||
|
||||
class JevIntentPreviousTurnTests(unittest.TestCase):
|
||||
def test_v1_v2_state_shape_keeps_v0_questions(self) -> None:
|
||||
sample = dict(source_a_samples()[0])
|
||||
sample["previous_turn"] = {"assistant_message": "上一问", "user_message": "上一句"}
|
||||
sample["previous_decision"] = {"intent": "unclear", "answer_class": None}
|
||||
v0 = build_state(sample, variant="v0")
|
||||
self.assertEqual(set(v0), {"current_question", "options", "user_message", "case_status"})
|
||||
v1 = build_state(sample, variant="v1")
|
||||
self.assertEqual(v1["previous_turn"], {"assistant_message": "上一问", "user_message": "上一句"})
|
||||
self.assertNotIn("previous_decision", v1)
|
||||
v2 = build_state(sample, variant="v2")
|
||||
self.assertEqual(v2["previous_decision"], {"intent": "unclear", "answer_class": None})
|
||||
v0_questions = build_questions(sample)
|
||||
v2_questions = build_questions(sample, variant="v2")
|
||||
self.assertNotIn("continues_previous_turn", v0_questions)
|
||||
self.assertEqual(set(v0_questions["intent"]["criteria"]), set(v2_questions["intent"]["criteria"]))
|
||||
self.assertEqual(v2_questions["continues_previous_turn"]["type"], "noul")
|
||||
self.assertEqual(
|
||||
v2_questions["continues_previous_turn"]["instructions"],
|
||||
CONTINUES_PREVIOUS_TURN_INSTRUCTIONS,
|
||||
)
|
||||
|
||||
def test_continues_rule_adopts_previous_intent_only_at_threshold(self) -> None:
|
||||
answers = {
|
||||
"intent": {"choice": "answer_current_focus", "confidence": 0.95, "probabilities": {}},
|
||||
"answer_class": {"choice": "yes", "confidence": 0.91, "probabilities": {}},
|
||||
"has_new_dated_event": {"noul": 0.1},
|
||||
"continues_previous_turn": {"noul": 0.9},
|
||||
}
|
||||
adopted = parse_jev_answers(
|
||||
answers,
|
||||
previous_decision={"intent": "unclear", "answer_class": None},
|
||||
apply_continue=True,
|
||||
)
|
||||
self.assertEqual(adopted["intent"], "unclear")
|
||||
self.assertIsNone(adopted["answer_class"])
|
||||
self.assertTrue(adopted["continued"])
|
||||
self.assertEqual(adopted["confidence"], 0.9)
|
||||
kept = parse_jev_answers(
|
||||
{**answers, "continues_previous_turn": {"noul": 0.89}},
|
||||
previous_decision={"intent": "stop_rectification", "answer_class": None},
|
||||
apply_continue=True,
|
||||
)
|
||||
self.assertEqual(kept["intent"], "answer_current_focus")
|
||||
self.assertEqual(kept["answer_class"], "yes")
|
||||
self.assertFalse(kept["continued"])
|
||||
focused = parse_jev_answers(
|
||||
answers,
|
||||
previous_decision={"intent": "answer_current_focus", "answer_class": "no"},
|
||||
apply_continue=True,
|
||||
)
|
||||
self.assertEqual(focused["intent"], "answer_current_focus")
|
||||
self.assertEqual(focused["answer_class"], "no")
|
||||
|
||||
def test_case_first_turn_previous_turn_is_null(self) -> None:
|
||||
rows = attach_previous_turns([
|
||||
{
|
||||
"id": "first",
|
||||
"case_id": "case-1",
|
||||
"created_at": "2026-09-01T00:00:00Z",
|
||||
"user_message": "第一句",
|
||||
"assistant_message": "第一答",
|
||||
},
|
||||
{
|
||||
"id": "second",
|
||||
"case_id": "case-1",
|
||||
"created_at": "2026-09-01T00:01:00Z",
|
||||
"user_message": "继续",
|
||||
"assistant_message": "第二答",
|
||||
},
|
||||
{
|
||||
"id": "orphan",
|
||||
"user_message": "没有案件",
|
||||
"assistant_message": "答",
|
||||
},
|
||||
])
|
||||
by_id = {row["id"]: row for row in rows}
|
||||
self.assertIsNone(by_id["first"]["previous_turn"])
|
||||
self.assertEqual(by_id["first"]["previous_turn_source"], "case_first")
|
||||
self.assertEqual(by_id["second"]["previous_turn"], {
|
||||
"assistant_message": "第一答",
|
||||
"user_message": "第一句",
|
||||
})
|
||||
self.assertIsNone(by_id["orphan"]["previous_turn"])
|
||||
state = build_state(by_id["first"], variant="v1")
|
||||
self.assertIsNone(state["previous_turn"])
|
||||
self.assertIn("user_message", state)
|
||||
|
||||
|
||||
class JevIntentCorpusTests(unittest.TestCase):
|
||||
def test_source_a_has_ten_labeled_rows(self) -> None:
|
||||
rows = source_a_samples()
|
||||
|
||||
Reference in New Issue
Block a user