Jesse_ChenandCursor
4e0db55f03
Engine asked_probe_keys no longer include varga split hashes that 400 the scorer, failed compares become visible and retry, user stop can still deliver a range on a stale snapshot, and holdout no longer reasks domains already in the ledger. Co-authored-by: Cursor <cursoragent@cursor.com>
332 lines
12 KiB
TypeScript
332 lines
12 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import test, { afterEach } from "node:test";
|
||
|
||
import { buildInferenceState } from "../src/lib/rectification-agentic/core/build-state.ts";
|
||
import {
|
||
ADOPT_OUTCOMES,
|
||
RECTIFICATION_TERMINATION_COPY,
|
||
} from "../src/lib/rectification-agentic/core/rectification-decision.ts";
|
||
import {
|
||
RECTIFICATION_USER_COPY,
|
||
withCompareFailedRetryNotice,
|
||
withLastSuccessfulCompareNotice,
|
||
} from "../src/lib/rectification-agentic/user-copy.ts";
|
||
import {
|
||
applyRectificationChoice,
|
||
persistNextInterviewIfIdle,
|
||
} from "../src/lib/rectification-agentic/v9/answer-choice.ts";
|
||
import { resetStaleMinuteRescoreAttemptsForTests } from "../src/lib/rectification-agentic/v9/block-scan-answer.ts";
|
||
import { STOP_ACTION } from "../src/lib/rectification-agentic/v9/choice-action.ts";
|
||
import { parseToolActivityDetail } from "../src/lib/rectification-agentic/v9/tool-service.ts";
|
||
import {
|
||
CASE_ID,
|
||
CANDIDATE_ID,
|
||
FOCUS_ID,
|
||
RESULT_ID,
|
||
SECOND_CANDIDATE_ID,
|
||
SESSION_ID,
|
||
TURN_ID,
|
||
USER_ID,
|
||
activeFocusFixture,
|
||
candidateSnapshotFixture,
|
||
computeFixture,
|
||
conversationSummaryFixture,
|
||
dossierFixture,
|
||
fakeAccounting,
|
||
receiptHandlers,
|
||
} from "./rectification-v9-test-support.ts";
|
||
|
||
const ACTION_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||
const QUESTION_ID = "question-1";
|
||
|
||
afterEach(() => {
|
||
resetStaleMinuteRescoreAttemptsForTests();
|
||
});
|
||
|
||
function scoreableEvidenceRows() {
|
||
return [
|
||
{
|
||
id: "44444444-4444-4444-8444-444444444441",
|
||
source_turn_id: TURN_ID,
|
||
subject: "self",
|
||
event_kind: "education_start",
|
||
domain: "education",
|
||
occurred_from: "2016-09-01",
|
||
occurred_to: "2016-09-30",
|
||
date_precision: "month",
|
||
summary: "education start",
|
||
status: "confirmed",
|
||
supersedes_evidence_id: null,
|
||
created_at: "2026-09-07T00:00:00.000Z",
|
||
},
|
||
{
|
||
id: "44444444-4444-4444-8444-444444444442",
|
||
source_turn_id: TURN_ID,
|
||
subject: "self",
|
||
event_kind: "career_entry",
|
||
domain: "career",
|
||
occurred_from: "2018-07-01",
|
||
occurred_to: null,
|
||
date_precision: "month",
|
||
summary: "career entry",
|
||
status: "confirmed",
|
||
supersedes_evidence_id: null,
|
||
created_at: "2026-09-07T00:00:00.000Z",
|
||
},
|
||
{
|
||
id: "44444444-4444-4444-8444-444444444443",
|
||
source_turn_id: TURN_ID,
|
||
subject: "self",
|
||
event_kind: "relationship_start",
|
||
domain: "relationship",
|
||
occurred_from: "2021-05-01",
|
||
occurred_to: null,
|
||
date_precision: "month",
|
||
summary: "relationship start",
|
||
status: "confirmed",
|
||
supersedes_evidence_id: null,
|
||
created_at: "2026-09-07T00:00:00.000Z",
|
||
},
|
||
{
|
||
id: "44444444-4444-4444-8444-444444444444",
|
||
source_turn_id: TURN_ID,
|
||
subject: "self",
|
||
event_kind: "family_event",
|
||
domain: "family",
|
||
occurred_from: "2023-03-01",
|
||
occurred_to: null,
|
||
date_precision: "month",
|
||
summary: "family event",
|
||
status: "confirmed",
|
||
supersedes_evidence_id: null,
|
||
created_at: "2026-09-07T00:00:00.000Z",
|
||
},
|
||
];
|
||
}
|
||
|
||
function staleDossier(extra: { status?: string; activeFocus?: ReturnType<typeof activeFocusFixture> | null } = {}) {
|
||
const evidence = scoreableEvidenceRows();
|
||
const inference = buildInferenceState({
|
||
range_start: "04:45",
|
||
range_end: "05:15",
|
||
candidates: [
|
||
{ id: "05:02", time: "05:02", relative_support: 58 },
|
||
{ id: "04:55", time: "04:55", relative_support: 42 },
|
||
],
|
||
events: [
|
||
{ id: "e1", domain: "education", year: 2016, precision: "month" },
|
||
{ id: "e2", domain: "career", year: 2018, precision: "month" },
|
||
{ id: "e3", domain: "relationship", year: 2021, precision: "month" },
|
||
{ id: "e4", domain: "family", year: 2023, precision: "month" },
|
||
],
|
||
probes: [],
|
||
});
|
||
return dossierFixture({
|
||
status: extra.status ?? "collecting_evidence",
|
||
evidence,
|
||
latestResult: candidateSnapshotFixture({
|
||
evidenceLedgerFingerprint: "b".repeat(64),
|
||
representativeTime: "05:02",
|
||
decisionReceipt: {
|
||
acceptance_allowed: true,
|
||
selection_allowed: true,
|
||
propose_allowed: true,
|
||
confirmation_allowed: false,
|
||
inference_state: inference,
|
||
},
|
||
}),
|
||
conversationSummary: conversationSummaryFixture({
|
||
activeFocus: extra.activeFocus === undefined
|
||
? activeFocusFixture({
|
||
questionId: QUESTION_ID,
|
||
expectedAnswerSchema: {
|
||
choice: {
|
||
prompt: "平时相处更接近哪一种？",
|
||
option_a: "照顾对方感受",
|
||
option_b: "习惯自己拿主意",
|
||
option_c: "两种都有",
|
||
option_d: "说不好",
|
||
options: [
|
||
{ key: "A", label: "照顾对方感受", answer_class: "yes" },
|
||
{ key: "B", label: "习惯自己拿主意", answer_class: "weak_yes" },
|
||
{ key: "C", label: "两种都有", answer_class: "no" },
|
||
{ key: "D", label: "说不好", answer_class: "unsure" },
|
||
],
|
||
},
|
||
probe_id: "p-d9",
|
||
semantic_key: "varga.d9.style",
|
||
scoring: true,
|
||
},
|
||
})
|
||
: extra.activeFocus,
|
||
}),
|
||
});
|
||
}
|
||
|
||
function scoreEnginePayload() {
|
||
return {
|
||
success: true,
|
||
endpoint: "rectification_v5_score",
|
||
result_id: RESULT_ID,
|
||
algorithm_version: "rectification-event-contract-v2",
|
||
event_contract_version: "rectification-event-contract-v2",
|
||
decision_policy_version: "rectification-candidate-policy-v2",
|
||
execution_ledger_version: "rectification-execution-ledger-v2",
|
||
candidate_decisions: [
|
||
{ candidate_id: CANDIDATE_ID, time: "05:02", rank: 1, relative_support: 58, tied_minute_count: 1 },
|
||
{ candidate_id: SECOND_CANDIDATE_ID, time: "04:55", rank: 2, relative_support: 42, tied_minute_count: 1 },
|
||
],
|
||
decision_receipt: {
|
||
receipt_version: "candidate-decision-receipt-v2",
|
||
contract_version: "v2",
|
||
event_contract_version: "rectification-event-contract-v2",
|
||
policy_version: "rectification-candidate-policy-v2",
|
||
decision_policy_version: "rectification-candidate-policy-v2",
|
||
display_allowed: true,
|
||
selection_allowed: true,
|
||
acceptance_allowed: true,
|
||
propose_allowed: true,
|
||
confirmation_allowed: false,
|
||
accept_allowed: true,
|
||
confirm_allowed: false,
|
||
representative_candidate_id: CANDIDATE_ID,
|
||
representative_time: "05:02",
|
||
overall_confidence: "high",
|
||
margin_percent: 16,
|
||
},
|
||
execution_ledger: [
|
||
{ ledger_version: "rectification-execution-ledger-v2", stage: "technique_layer", method: "d1-rashi", status: "executed", source: "python-engine" },
|
||
],
|
||
};
|
||
}
|
||
|
||
test("compare failure copy and receipt detail stay user-visible without PII", () => {
|
||
assert.equal(
|
||
withCompareFailedRetryNotice("这条记下了。"),
|
||
`这条记下了。\n\n${RECTIFICATION_USER_COPY.compareFailedRetry}`,
|
||
);
|
||
assert.equal(
|
||
withLastSuccessfulCompareNotice("目前范围 04:45–05:15。"),
|
||
`目前范围 04:45–05:15。\n\n${RECTIFICATION_USER_COPY.lastSuccessfulCompareRange}`,
|
||
);
|
||
const detail = parseToolActivityDetail({
|
||
result_fingerprint: JSON.stringify({
|
||
safe_error_code: "engine_request_failed",
|
||
engine_message: "asked_probe_keys[0] must be a non-empty string up to 120 characters",
|
||
}),
|
||
});
|
||
assert.equal(detail?.safe_error_code, "engine_request_failed");
|
||
assert.match(String(detail?.engine_message), /asked_probe_keys/);
|
||
const agentRun = readFileSync(new URL("../src/lib/rectification-agentic/v9/agent-run.ts", import.meta.url), "utf8");
|
||
assert.match(agentRun, /withCompareFailedRetryNotice/);
|
||
assert.match(agentRun, /rectification-compare-candidates/);
|
||
const tools = readFileSync(new URL("../src/mastra/rectification-v9-tools.ts", import.meta.url), "utf8");
|
||
assert.match(tools, /engine_message: engineMessageForReceipt/);
|
||
});
|
||
|
||
test("idle persist on a stale snapshot calls candidate score once", async () => {
|
||
let scoreCalls = 0;
|
||
const previous = globalThis.fetch;
|
||
globalThis.fetch = (async (input: RequestInfo | URL) => {
|
||
const url = String(input);
|
||
if (url.includes("/api/rectification/v5/score")) {
|
||
scoreCalls += 1;
|
||
return {
|
||
ok: true,
|
||
status: 200,
|
||
json: async () => scoreEnginePayload(),
|
||
};
|
||
}
|
||
throw new Error(`unexpected fetch ${url}`);
|
||
}) as typeof fetch;
|
||
try {
|
||
const raw = staleDossier({ activeFocus: null });
|
||
const accounting = fakeAccounting({
|
||
...receiptHandlers,
|
||
get_agentic_rectification_case_dossier: () => raw,
|
||
get_agentic_rectification_case_compute: () => computeFixture(),
|
||
persist_agentic_rectification_candidate_v2: (_fn, args) => ({
|
||
result_id: RESULT_ID,
|
||
candidates: args.p_candidates,
|
||
overall_confidence: "medium",
|
||
selection_allowed: true,
|
||
confirmation_allowed: false,
|
||
representative_time: "05:02",
|
||
evidence_ledger_fingerprint: args.p_evidence_ledger_fingerprint,
|
||
candidate_range_fingerprint: args.p_candidate_range_fingerprint,
|
||
skill_version: args.p_skill_version,
|
||
algorithm_version: args.p_algorithm_version,
|
||
event_contract_version: args.p_event_contract_version,
|
||
decision_policy_version: args.p_decision_policy_version,
|
||
decision_receipt: args.p_decision_receipt,
|
||
execution_ledger: args.p_execution_ledger,
|
||
created_at: "2026-09-07T00:00:00.000Z",
|
||
}),
|
||
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID, idempotent: false }),
|
||
});
|
||
await persistNextInterviewIfIdle({
|
||
accounting: accounting.client,
|
||
userId: USER_ID,
|
||
caseId: CASE_ID,
|
||
askedTurnId: TURN_ID,
|
||
});
|
||
assert.equal(scoreCalls, 1);
|
||
await persistNextInterviewIfIdle({
|
||
accounting: accounting.client,
|
||
userId: USER_ID,
|
||
caseId: CASE_ID,
|
||
askedTurnId: TURN_ID,
|
||
});
|
||
assert.equal(scoreCalls, 1);
|
||
} finally {
|
||
globalThis.fetch = previous;
|
||
}
|
||
});
|
||
|
||
test("STOP on a stale snapshot rescores then delivers a range", async () => {
|
||
const previous = globalThis.fetch;
|
||
globalThis.fetch = (async () => {
|
||
throw new Error("engine down");
|
||
}) as typeof fetch;
|
||
try {
|
||
const raw = staleDossier();
|
||
const accounting = fakeAccounting({
|
||
...receiptHandlers,
|
||
get_agentic_rectification_case_dossier: () => raw,
|
||
get_agentic_rectification_case_compute: () => computeFixture(),
|
||
apply_agentic_rectification_choice_action: (_fn, args) => ({
|
||
action_id: args.p_action_id,
|
||
status: "applied",
|
||
idempotent: false,
|
||
question_id: args.p_question_id,
|
||
option_id: args.p_option_id,
|
||
probe_id: "p-d9",
|
||
revision: Number(args.p_expected_revision) + 1,
|
||
source_quote: args.p_source_quote,
|
||
derived_context: args.p_derived_context,
|
||
narration: args.p_narration,
|
||
focus_status: args.p_focus_status,
|
||
}),
|
||
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID, idempotent: false }),
|
||
});
|
||
const applied = await applyRectificationChoice(accounting.client, {
|
||
userId: USER_ID,
|
||
caseId: CASE_ID,
|
||
sessionId: SESSION_ID,
|
||
actionId: ACTION_ID,
|
||
action: STOP_ACTION,
|
||
focusId: FOCUS_ID,
|
||
questionId: QUESTION_ID,
|
||
optionId: "stop",
|
||
expectedRevision: 1,
|
||
});
|
||
assert.ok(ADOPT_OUTCOMES.has(applied.nextAction.session_outcome));
|
||
assert.equal(applied.nextAction.can_adopt, true);
|
||
assert.match(applied.narration, new RegExp(RECTIFICATION_USER_COPY.lastSuccessfulCompareRange));
|
||
assert.ok(applied.narration.includes(RECTIFICATION_TERMINATION_COPY) || applied.narration.includes("范围"));
|
||
} finally {
|
||
globalThis.fetch = previous;
|
||
}
|
||
});
|