research: compare Jev intent state with the previous turn
Independent Staging Quality Gate / validate (push) Successful in 12m0s
Independent Staging Quality Gate / publish (push) Successful in 3m35s

V0 on the existing 157 real rows matches the 09-19 cache. The staging extract has no case linkage, so V1/V2 are unmeasured and the verdict stays 缺数据.
This commit is contained in:
jesse-ux
2026-09-27 11:41:53 +08:00
parent 710c848b30
commit e3bd3930e3
13 changed files with 100475 additions and 26 deletions
+91
View File
@@ -17,6 +17,7 @@ from scripts.research.jev_intent_corpus_build import ( # noqa: E402
source_a_samples,
)
from scripts.research.jev_intent_questions import (
CONTINUES_PREVIOUS_TURN_INSTRUCTIONS,
CRITERION_MAP,
INTENT_CRITERIA,
LOST_SEMANTICS,
@@ -25,6 +26,7 @@ from scripts.research.jev_intent_questions import (
enforce_combo,
parse_jev_answers,
)
from scripts.research.jev_intent_source_b_v2 import attach_previous_turns
ROOT = Path(__file__).resolve().parents[1]
@@ -70,6 +72,95 @@ class JevIntentQuestionsTests(unittest.TestCase):
self.assertEqual(parsed["confidence"], 0.9)
class JevIntentPreviousTurnTests(unittest.TestCase):
def test_v1_v2_state_shape_keeps_v0_questions(self) -> None:
sample = dict(source_a_samples()[0])
sample["previous_turn"] = {"assistant_message": "上一问", "user_message": "上一句"}
sample["previous_decision"] = {"intent": "unclear", "answer_class": None}
v0 = build_state(sample, variant="v0")
self.assertEqual(set(v0), {"current_question", "options", "user_message", "case_status"})
v1 = build_state(sample, variant="v1")
self.assertEqual(v1["previous_turn"], {"assistant_message": "上一问", "user_message": "上一句"})
self.assertNotIn("previous_decision", v1)
v2 = build_state(sample, variant="v2")
self.assertEqual(v2["previous_decision"], {"intent": "unclear", "answer_class": None})
v0_questions = build_questions(sample)
v2_questions = build_questions(sample, variant="v2")
self.assertNotIn("continues_previous_turn", v0_questions)
self.assertEqual(set(v0_questions["intent"]["criteria"]), set(v2_questions["intent"]["criteria"]))
self.assertEqual(v2_questions["continues_previous_turn"]["type"], "noul")
self.assertEqual(
v2_questions["continues_previous_turn"]["instructions"],
CONTINUES_PREVIOUS_TURN_INSTRUCTIONS,
)
def test_continues_rule_adopts_previous_intent_only_at_threshold(self) -> None:
answers = {
"intent": {"choice": "answer_current_focus", "confidence": 0.95, "probabilities": {}},
"answer_class": {"choice": "yes", "confidence": 0.91, "probabilities": {}},
"has_new_dated_event": {"noul": 0.1},
"continues_previous_turn": {"noul": 0.9},
}
adopted = parse_jev_answers(
answers,
previous_decision={"intent": "unclear", "answer_class": None},
apply_continue=True,
)
self.assertEqual(adopted["intent"], "unclear")
self.assertIsNone(adopted["answer_class"])
self.assertTrue(adopted["continued"])
self.assertEqual(adopted["confidence"], 0.9)
kept = parse_jev_answers(
{**answers, "continues_previous_turn": {"noul": 0.89}},
previous_decision={"intent": "stop_rectification", "answer_class": None},
apply_continue=True,
)
self.assertEqual(kept["intent"], "answer_current_focus")
self.assertEqual(kept["answer_class"], "yes")
self.assertFalse(kept["continued"])
focused = parse_jev_answers(
answers,
previous_decision={"intent": "answer_current_focus", "answer_class": "no"},
apply_continue=True,
)
self.assertEqual(focused["intent"], "answer_current_focus")
self.assertEqual(focused["answer_class"], "no")
def test_case_first_turn_previous_turn_is_null(self) -> None:
rows = attach_previous_turns([
{
"id": "first",
"case_id": "case-1",
"created_at": "2026-09-01T00:00:00Z",
"user_message": "第一句",
"assistant_message": "第一答",
},
{
"id": "second",
"case_id": "case-1",
"created_at": "2026-09-01T00:01:00Z",
"user_message": "继续",
"assistant_message": "第二答",
},
{
"id": "orphan",
"user_message": "没有案件",
"assistant_message": "答",
},
])
by_id = {row["id"]: row for row in rows}
self.assertIsNone(by_id["first"]["previous_turn"])
self.assertEqual(by_id["first"]["previous_turn_source"], "case_first")
self.assertEqual(by_id["second"]["previous_turn"], {
"assistant_message": "第一答",
"user_message": "第一句",
})
self.assertIsNone(by_id["orphan"]["previous_turn"])
state = build_state(by_id["first"], variant="v1")
self.assertIsNone(state["previous_turn"])
self.assertIn("user_message", state)
class JevIntentCorpusTests(unittest.TestCase):
def test_source_a_has_ten_labeled_rows(self) -> None:
rows = source_a_samples()