research: compare Jev intent state with the previous turn
V0 on the existing 157 real rows matches the 09-19 cache. The staging extract has no case linkage, so V1/V2 are unmeasured and the verdict stays 缺数据.
This commit is contained in:
@@ -55,14 +55,19 @@ def gold_of(sample: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return enforce_combo(gold.get("intent"), gold.get("answer_class"), gold.get("has_new_dated_event"))
|
||||
|
||||
|
||||
def call_jev(sample: Mapping[str, Any], *, timeout: float = 60.0) -> dict[str, Any]:
|
||||
def call_jev(
|
||||
sample: Mapping[str, Any],
|
||||
*,
|
||||
timeout: float = 60.0,
|
||||
variant: str = "v0",
|
||||
) -> dict[str, Any]:
|
||||
from typesafe_sdk import TypeSafeClient
|
||||
|
||||
started = time.perf_counter()
|
||||
with TypeSafeClient() as client:
|
||||
response = client.system_one(
|
||||
state=build_state(sample),
|
||||
questions=sdk_questions(sample),
|
||||
state=build_state(sample, variant=variant),
|
||||
questions=sdk_questions(sample, variant=variant),
|
||||
model=JEV_MODEL,
|
||||
timeout=timeout,
|
||||
)
|
||||
@@ -81,7 +86,11 @@ def call_jev(sample: Mapping[str, Any], *, timeout: float = 60.0) -> dict[str, A
|
||||
"noul": getattr(value, "noul", None),
|
||||
}
|
||||
answers = converted
|
||||
parsed = parse_jev_answers(answers)
|
||||
parsed = parse_jev_answers(
|
||||
answers,
|
||||
previous_decision=sample.get("previous_decision") if isinstance(sample.get("previous_decision"), dict) else None,
|
||||
apply_continue=variant == "v2",
|
||||
)
|
||||
usage = payload.get("usage") or {}
|
||||
return {
|
||||
"ok": True,
|
||||
@@ -95,15 +104,18 @@ def call_jev(sample: Mapping[str, Any], *, timeout: float = 60.0) -> dict[str, A
|
||||
"input_tokens": usage.get("input_tokens"),
|
||||
"output_tokens": usage.get("output_tokens"),
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"continued": parsed.get("continued"),
|
||||
"continues_previous_turn": parsed.get("continues_previous_turn"),
|
||||
"variant": variant,
|
||||
}
|
||||
|
||||
|
||||
def call_jev_retry(sample: Mapping[str, Any], *, retries: int = 4) -> dict[str, Any]:
|
||||
def call_jev_retry(sample: Mapping[str, Any], *, retries: int = 4, variant: str = "v0") -> dict[str, Any]:
|
||||
last_error = ""
|
||||
delay = 1.0
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
return call_jev(sample)
|
||||
return call_jev(sample, variant=variant)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = type(exc).__name__
|
||||
name = type(exc).__name__
|
||||
@@ -127,6 +139,9 @@ def call_jev_retry(sample: Mapping[str, Any], *, retries: int = 4) -> dict[str,
|
||||
"input_tokens": None,
|
||||
"output_tokens": None,
|
||||
"elapsed_ms": None,
|
||||
"continued": None,
|
||||
"continues_previous_turn": None,
|
||||
"variant": variant,
|
||||
"error": last_error,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user