jesse-ux
b50a8f94ac
Quality gate run 2598 failed because the unwritten-evidence test still required the old persisted-question DESIGN sentence. Align the assertion with the BUG-678 hang-back contract.
376 lines
15 KiB
TypeScript
376 lines
15 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import test from "node:test";
|
||
|
||
import { RECTIFICATION_USER_COPY } from "../src/lib/rectification-agentic/user-copy.ts";
|
||
import { runV9AgentTurn, type V9AgentRunOptions } from "../src/lib/rectification-agentic/v9/agent-run.ts";
|
||
import {
|
||
batchResultFromToolChunk,
|
||
composeHostFallbackNarration,
|
||
isToolInputRejection,
|
||
publicWriteToolCompleted,
|
||
retryConstraintForAttempt,
|
||
turnExpectsEvidenceWrite,
|
||
UNWRITTEN_EVIDENCE_RETRY_CONSTRAINT,
|
||
DEFAULT_RETRY_CONSTRAINT,
|
||
} from "../src/lib/rectification-agentic/v9/host-fallback.ts";
|
||
import {
|
||
mapStreamChunkToActivity,
|
||
mapStreamChunkToPhase,
|
||
safePublicEvent,
|
||
} from "../src/lib/rectification-agentic/v9/stream-mapping.ts";
|
||
import { expectedWriteFromCollectIntent } from "../src/lib/rectification-agentic/v9/turn-intent-classifier.ts";
|
||
import { RECTIFICATION_SKILL_NAME } from "../src/lib/rectification-agentic/v9/case-status.ts";
|
||
import {
|
||
CASE_ID,
|
||
SESSION_ID,
|
||
TURN_ID,
|
||
USER_ID,
|
||
dossierFixture,
|
||
fakeAccounting,
|
||
receiptHandlers,
|
||
} from "./rectification-v9-test-support.ts";
|
||
|
||
const REJECTION = {
|
||
error: true,
|
||
message: "Tool input validation failed",
|
||
validationErrors: { errors: ["kind"], fields: { kind: "debt" } },
|
||
};
|
||
|
||
type StreamChunk = {
|
||
type: string;
|
||
payload?: Record<string, unknown>;
|
||
object?: unknown;
|
||
};
|
||
|
||
function chunk(type: string, payload?: Record<string, unknown>): StreamChunk {
|
||
return { type, ...(payload ? { payload } : {}) };
|
||
}
|
||
|
||
function readCaseThen(rest: StreamChunk[]): StreamChunk[] {
|
||
return [
|
||
chunk("start"),
|
||
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
|
||
chunk("tool-result", { toolName: "skill" }),
|
||
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
|
||
chunk("tool-result", { toolName: "rectification-read-case" }),
|
||
...rest,
|
||
];
|
||
}
|
||
|
||
function fakeAgentStream(
|
||
chunks: StreamChunk[],
|
||
sink?: unknown[],
|
||
) {
|
||
return {
|
||
stream: async (messages: unknown[]) => {
|
||
sink?.push(messages);
|
||
return {
|
||
fullStream: (async function* () {
|
||
for (const item of chunks) yield item;
|
||
})(),
|
||
totalUsage: Promise.resolve({ inputTokens: 10, outputTokens: 20 }),
|
||
};
|
||
},
|
||
getSkill: async () => ({ name: RECTIFICATION_SKILL_NAME, instructions: "skill" }),
|
||
};
|
||
}
|
||
|
||
function runOptions(overrides: Partial<V9AgentRunOptions> = {}) {
|
||
const emitted: Array<Record<string, unknown>> = [];
|
||
const billing = { reserved: 0, completed: 0, released: 0 };
|
||
const accounting = fakeAccounting({
|
||
...receiptHandlers,
|
||
get_agentic_rectification_case_dossier: () => dossierFixture(),
|
||
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
|
||
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "completed", idempotent: false }),
|
||
});
|
||
const optionsValue: V9AgentRunOptions = {
|
||
userId: USER_ID,
|
||
caseId: CASE_ID,
|
||
sessionId: SESSION_ID,
|
||
requestId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee",
|
||
action: "evidence",
|
||
message: "2018 年 3 月开始欠债",
|
||
modelName: "gpt-4o-mini",
|
||
accounting: accounting.client,
|
||
billing: {
|
||
reserve: async () => { billing.reserved += 1; return { success: true, status: 200 }; },
|
||
complete: async () => { billing.completed += 1; return true; },
|
||
release: async () => { billing.released += 1; return true; },
|
||
},
|
||
emit: (event) => { emitted.push(event as Record<string, unknown>); },
|
||
buildAgent: async () => fakeAgentStream([]) as never,
|
||
...overrides,
|
||
};
|
||
return { options: optionsValue, emitted, billing, accounting };
|
||
}
|
||
|
||
function captureDiagnostics<T>(run: () => Promise<T>): Promise<{ result: T; logs: Record<string, unknown>[] }> {
|
||
const logs: Record<string, unknown>[] = [];
|
||
const original = console.info;
|
||
console.info = (...args: unknown[]) => {
|
||
const text = typeof args[0] === "string" ? args[0] : "";
|
||
if (text.includes("RectificationRunDiagnostic")) {
|
||
logs.push(JSON.parse(text) as Record<string, unknown>);
|
||
}
|
||
};
|
||
return run().then((result) => ({ result, logs })).finally(() => {
|
||
console.info = original;
|
||
});
|
||
}
|
||
|
||
test("collect intent maps provide_new_evidence and dated current-focus to expectedWrite", () => {
|
||
assert.equal(expectedWriteFromCollectIntent({
|
||
intent: "provide_new_evidence",
|
||
answer_class: null,
|
||
}), "evidence");
|
||
assert.equal(expectedWriteFromCollectIntent({
|
||
intent: "answer_current_focus",
|
||
answer_class: "yes",
|
||
has_new_dated_event: true,
|
||
}), "evidence");
|
||
assert.equal(expectedWriteFromCollectIntent({
|
||
intent: "answer_current_focus",
|
||
answer_class: "no",
|
||
}), "none");
|
||
assert.equal(expectedWriteFromCollectIntent(null), "none");
|
||
const collectFocus = {
|
||
expectedAnswerSchema: { collect: true, prompt: "钱的方面，还记得哪年收入明显变过吗？" },
|
||
};
|
||
assert.equal(expectedWriteFromCollectIntent({
|
||
intent: "answer_current_focus",
|
||
answer_class: "yes",
|
||
}, collectFocus as never), "evidence");
|
||
});
|
||
|
||
test("runner does not use a year-utterance fallback", () => {
|
||
assert.equal(turnExpectsEvidenceWrite("evidence", "evidence"), true);
|
||
assert.equal(turnExpectsEvidenceWrite("evidence", "none"), false);
|
||
assert.equal(turnExpectsEvidenceWrite("evidence", "unknown"), false);
|
||
assert.equal(turnExpectsEvidenceWrite("evidence", undefined), false);
|
||
assert.equal(turnExpectsEvidenceWrite("read_only", "evidence"), false);
|
||
assert.equal(turnExpectsEvidenceWrite("opening", "none"), false);
|
||
});
|
||
|
||
test("retry bootstrap uses the unwritten-evidence constraint only for that error", () => {
|
||
assert.equal(retryConstraintForAttempt("evidence_not_written"), UNWRITTEN_EVIDENCE_RETRY_CONSTRAINT);
|
||
assert.equal(retryConstraintForAttempt("empty_stream"), DEFAULT_RETRY_CONSTRAINT);
|
||
assert.equal(retryConstraintForAttempt(null), DEFAULT_RETRY_CONSTRAINT);
|
||
});
|
||
|
||
test("route passes classifier expectedWrite into the agent runner", () => {
|
||
const route = readFileSync(
|
||
new URL("../src/app/api/rectification/agent/route.ts", import.meta.url),
|
||
"utf8",
|
||
);
|
||
const agentRun = readFileSync(
|
||
new URL("../src/lib/rectification-agentic/v9/agent-run.ts", import.meta.url),
|
||
"utf8",
|
||
);
|
||
assert.match(route, /classifyTurnIntentWithRetry/);
|
||
assert.match(route, /expectedWrite,/);
|
||
assert.match(route, /let expectedWrite: "evidence" \| "none" \| "unknown" = "none"/);
|
||
assert.doesNotMatch(route, /\(\?:19\|20\)\\d\{2\}/);
|
||
assert.doesNotMatch(agentRun, /\(\?:19\|20\)\\d\{2\}/);
|
||
});
|
||
|
||
test("unwritten 记下了 retries once then host-falls-back without billing", async () => {
|
||
let buildCount = 0;
|
||
const messages: unknown[] = [];
|
||
const { options, emitted, billing } = runOptions({
|
||
expectedWrite: "evidence",
|
||
buildAgent: async () => {
|
||
buildCount += 1;
|
||
return fakeAgentStream(readCaseThen([
|
||
chunk("text-delta", { text: "记下了：2018 年 3 月开始欠债。" }),
|
||
chunk("finish"),
|
||
]), messages) as never;
|
||
},
|
||
});
|
||
const { result, logs } = await captureDiagnostics(() => runV9AgentTurn(options));
|
||
assert.equal(buildCount, 2);
|
||
assert.equal(result.ok, true);
|
||
assert.equal(result.answerText, RECTIFICATION_USER_COPY.evidenceNotRecorded);
|
||
assert.ok(result.phases.includes("answer.host_fallback"));
|
||
assert.equal(result.phases.includes("billing.settled"), false);
|
||
assert.equal(emitted.some((event) => event.type === "attempt.reset"), true);
|
||
assert.equal(emitted.some((event) => event.type === "billing.settled"), false);
|
||
assert.equal(emitted.some((event) => event.type === "run.completed"), true);
|
||
assert.deepEqual(billing, { reserved: 1, completed: 0, released: 1 });
|
||
assert.equal(
|
||
emitted.some((event) => event.type === "answer.delta" && event.replace === true && event.text === ""),
|
||
true,
|
||
);
|
||
const first = logs[0];
|
||
assert.equal(first?.stateMutationCommitted, false);
|
||
assert.equal(first?.expectedWrite, "evidence");
|
||
const retryBootstrap = JSON.stringify(messages[1] ?? messages[0]);
|
||
assert.match(retryBootstrap, /rectification-record-evidence-batch/);
|
||
assert.match(retryBootstrap, /记下了/);
|
||
});
|
||
|
||
test("collect focus yes without 记下了 still retries when nothing was written", async () => {
|
||
let buildCount = 0;
|
||
const { options } = runOptions({
|
||
expectedWrite: "evidence",
|
||
buildAgent: async () => {
|
||
buildCount += 1;
|
||
return fakeAgentStream(readCaseThen([
|
||
chunk("text-delta", { text: "2018 年 3 月开始欠债。" }),
|
||
chunk("finish"),
|
||
])) as never;
|
||
},
|
||
});
|
||
const result = await runV9AgentTurn(options);
|
||
assert.equal(buildCount, 2);
|
||
assert.equal(result.answerText, RECTIFICATION_USER_COPY.evidenceNotRecorded);
|
||
});
|
||
|
||
test("read-case then batch then set-focus then 记下了 stays on the baseline path", async () => {
|
||
const { options, billing } = runOptions({
|
||
expectedWrite: "evidence",
|
||
buildAgent: async () => fakeAgentStream(readCaseThen([
|
||
chunk("tool-call", { toolName: "rectification-record-evidence-batch", args: { caseId: CASE_ID } }),
|
||
chunk("tool-result", {
|
||
toolName: "rectification-record-evidence-batch",
|
||
result: { accepted_recaps: [{ display_date_label: "2018年3月", event_phrase: "欠债" }] },
|
||
}),
|
||
chunk("tool-call", { toolName: "rectification-set-focus", args: { caseId: CASE_ID } }),
|
||
chunk("tool-result", { toolName: "rectification-set-focus" }),
|
||
chunk("text-delta", { text: "记下了：2018 年 3 月开始欠债。" }),
|
||
chunk("finish"),
|
||
])) as never,
|
||
});
|
||
const result = await runV9AgentTurn(options);
|
||
assert.equal(result.ok, true);
|
||
assert.match(result.answerText, /记下了/);
|
||
assert.equal(result.phases.includes("answer.host_fallback"), false);
|
||
assert.deepEqual(billing, { reserved: 1, completed: 1, released: 0 });
|
||
});
|
||
|
||
test("empty body after a completed batch still uses BUG-633 host fallback", async () => {
|
||
const { options, billing } = runOptions({
|
||
expectedWrite: "evidence",
|
||
buildAgent: async () => fakeAgentStream(readCaseThen([
|
||
chunk("tool-call", { toolName: "rectification-record-evidence-batch", args: { caseId: CASE_ID } }),
|
||
chunk("tool-result", {
|
||
toolName: "rectification-record-evidence-batch",
|
||
result: { accepted_recaps: [{ display_date_label: "2018年3月", event_phrase: "欠债" }] },
|
||
}),
|
||
chunk("finish"),
|
||
])) as never,
|
||
});
|
||
const result = await runV9AgentTurn(options);
|
||
assert.equal(result.ok, true);
|
||
assert.equal(result.answerText, "记下了：2018年3月 欠债。");
|
||
assert.ok(result.phases.includes("answer.host_fallback"));
|
||
assert.deepEqual(billing, { reserved: 1, completed: 1, released: 0 });
|
||
});
|
||
|
||
test("read_only action with a dated utterance does not trigger the write guard", async () => {
|
||
const { options, billing } = runOptions({
|
||
action: "read_only",
|
||
expectedWrite: "none",
|
||
message: "2018 年 3 月开始欠债",
|
||
buildAgent: async () => fakeAgentStream(readCaseThen([
|
||
chunk("text-delta", { text: "目前还在核对。" }),
|
||
chunk("finish"),
|
||
])) as never,
|
||
});
|
||
const result = await runV9AgentTurn(options);
|
||
assert.equal(result.ok, true);
|
||
assert.equal(result.answerText, "目前还在核对。");
|
||
assert.deepEqual(billing, { reserved: 1, completed: 1, released: 0 });
|
||
});
|
||
|
||
test("schema rejection envelopes fail the tool and never count as a batch recap", () => {
|
||
const rejected = chunk("tool-result", {
|
||
toolName: "rectification-record-evidence-batch",
|
||
result: REJECTION,
|
||
});
|
||
assert.equal(isToolInputRejection(REJECTION), true);
|
||
assert.deepEqual(
|
||
mapStreamChunkToActivity(rejected as never),
|
||
{
|
||
type: "tool.activity",
|
||
tool: "rectification-record-evidence-batch",
|
||
status: "failed",
|
||
code: "tool_call_rejected",
|
||
},
|
||
);
|
||
assert.equal(mapStreamChunkToPhase(rejected as never), null);
|
||
assert.equal(batchResultFromToolChunk(rejected), null);
|
||
assert.equal(composeHostFallbackNarration(REJECTION), null);
|
||
const publicEvent = safePublicEvent(mapStreamChunkToActivity(rejected as never));
|
||
assert.deepEqual(publicEvent, {
|
||
type: "tool.activity",
|
||
tool: "rectification-record-evidence-batch",
|
||
status: "failed",
|
||
code: "tool_call_rejected",
|
||
});
|
||
assert.doesNotMatch(JSON.stringify(publicEvent), /validationErrors/);
|
||
});
|
||
|
||
test("rejected batch tool-result is failed in the runner terminal status", async () => {
|
||
const { options, emitted } = runOptions({
|
||
expectedWrite: "evidence",
|
||
buildAgent: async () => fakeAgentStream(readCaseThen([
|
||
chunk("tool-call", { toolName: "rectification-record-evidence-batch", args: { caseId: CASE_ID } }),
|
||
chunk("tool-result", {
|
||
toolName: "rectification-record-evidence-batch",
|
||
result: REJECTION,
|
||
}),
|
||
chunk("text-delta", { text: "记下了：2018 年 3 月开始欠债。" }),
|
||
chunk("finish"),
|
||
])) as never,
|
||
});
|
||
const result = await runV9AgentTurn(options);
|
||
assert.equal(
|
||
emitted.some((event) => (
|
||
event.type === "tool.activity"
|
||
&& event.tool === "rectification-record-evidence-batch"
|
||
&& event.status === "failed"
|
||
&& event.code === "tool_call_rejected"
|
||
)),
|
||
true,
|
||
);
|
||
assert.equal(
|
||
emitted.some((event) => (
|
||
event.type === "tool.activity"
|
||
&& event.tool === "rectification-record-evidence-batch"
|
||
&& event.status === "completed"
|
||
)),
|
||
false,
|
||
);
|
||
assert.doesNotMatch(JSON.stringify(emitted), /validationErrors/);
|
||
const status = new Map<string, "completed" | "failed">([
|
||
["rectification-record-evidence-batch", "failed"],
|
||
]);
|
||
assert.equal(publicWriteToolCompleted(status), false);
|
||
assert.equal(result.ok, true);
|
||
assert.equal(result.answerText, RECTIFICATION_USER_COPY.evidenceNotRecorded);
|
||
});
|
||
|
||
test("chat live question only reads the last settled assistant message", () => {
|
||
const chat = readFileSync(
|
||
new URL("../src/components/rectification-agentic-chat.tsx", import.meta.url),
|
||
"utf8",
|
||
);
|
||
// 原值: messages.some((message) => role/state/question/focus_id)
|
||
// 新值: latestSettledAssistant 且 question.focus_id 对应当前焦点
|
||
// 原因: BUG-635 旧消息上的同 focus_id 不再算仍在显示
|
||
assert.match(
|
||
chat,
|
||
/const liveQuestionOnMessages = Boolean\(\s*latestSettledAssistant/,
|
||
);
|
||
assert.doesNotMatch(chat, /const liveQuestionOnMessages = messages\.some/);
|
||
const design = readFileSync(new URL("../DESIGN.md", import.meta.url), "utf8");
|
||
// 原值: DESIGN.md persisted-question 行写 “completed turn that claimed to record evidence”
|
||
// 新值: question-live 写无 asked_turn_id 的口述/选择题挂回最后一条助手消息；persisted-question 只留给没有助手消息可挂
|
||
// 原因: BUG-678 口述题也挂回消息后，已记录证据但焦点未答的完成轮不再走 persisted_question 裸行（BUG-679）
|
||
assert.match(design, /without `asked_turn_id` hangs on the last assistant message/);
|
||
assert.match(design, /only when no assistant message can carry the prompt/);
|
||
assert.doesNotMatch(design, /completed turn that claimed to record evidence/);
|
||
});
|