fix(web): isolate rectification answers from tool-step planning text

Mastra intermediate text-delta was published as answer.delta, then set-focus domain errors reset the attempt and replayed evidence. Publish only the terminal no-tool step, persist the next probe on the server, and ground batch quotes in the source turn.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-24 21:26:57 +08:00
co-authored by Cursor
parent 167bdad20d
commit fe87a9ecdb
30 changed files with 1373 additions and 237 deletions
+213 -131
View File
@@ -21,6 +21,7 @@ import {
receiptHandlers,
} from "./rectification-v9-test-support.ts";
import { RECTIFICATION_SKILL_NAME } from "../src/lib/rectification-agentic/v9/case-status.ts";
import { messageContentHash } from "../src/lib/rectification-agentic/v9/message-origin.ts";
import { safeToolErrorCode } from "../src/lib/rectification-agentic/v9/tool-service.ts";
import {
createRectificationActivityReceiptState,
@@ -91,9 +92,9 @@ test("fullStream chunks map to the allowlisted NDJSON phases only", () => {
}) as never),
{ type: "case.loaded", tool: "rectification-read-case" },
);
assert.deepEqual(
assert.equal(
mapStreamChunkToPhase(chunk("text-delta", { text: "你好" }) as never),
{ type: "answer.delta", text: "你好" },
null,
);
assert.equal(mapStreamChunkToPhase(chunk("finish") as never), null);
assert.equal(mapStreamChunkToPhase(chunk("error", { error: new Error("boom") }) as never), null);
@@ -469,8 +470,7 @@ test("answer deltas stream in order and reasoning is never forwarded", async ()
assert.equal(result.ok, true);
const deltas = emitted.filter((event) => event.type === "answer.delta");
assert.deepEqual(deltas, [
{ type: "answer.delta", text: "好的," },
{ type: "answer.delta", text: "先确认一下:" },
{ type: "answer.delta", text: "好的,先确认一下:" },
]);
assert.deepEqual(
emitted.filter((event) => event.type === "thinking.delta"),
@@ -569,7 +569,7 @@ test("Chinese process self-talk after tools is thinking, not the spoken answer",
assert.equal(result.answerText, "记下了,大约六岁入学小学。接下来你大概哪一年上的初中?");
});
test("process-only self-talk after tools is not persisted as a completed spoken answer", async () => {
test("process-only self-talk after tools is replaced by server narration, not retried", async () => {
let buildCount = 0;
const processTalk = "用户在上一轮里提供了两件带日期的经历。我需要用批量工具写入这些证据。用户";
const accounting = fakeAccounting({
@@ -582,48 +582,33 @@ test("process-only self-talk after tools is not persisted as a completed spoken
accounting: accounting.client,
buildAgent: async () => {
buildCount += 1;
return buildCount === 1
? attemptStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", {
toolName: "rectification-record-evidence-batch",
args: { caseId: CASE_ID, proposedKind: "education_start" },
}),
chunk("tool-result", { toolName: "rectification-record-evidence-batch" }),
chunk("reasoning-delta", { text: processTalk }),
chunk("finish"),
], { inputTokens: 11, outputTokens: 12 }) as never
: attemptStream(successfulAttemptChunks(), { inputTokens: 31, outputTokens: 17 }) as never;
return attemptStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", {
toolName: "rectification-record-evidence-batch",
args: { caseId: CASE_ID, proposedKind: "education_start" },
}),
chunk("tool-result", { toolName: "rectification-record-evidence-batch" }),
chunk("reasoning-delta", { text: processTalk }),
chunk("finish"),
], { inputTokens: 11, outputTokens: 12 }) as never;
},
});
const result = await runV9AgentTurn(options);
assert.equal(buildCount, 2);
assert.equal(buildCount, 1);
assert.equal(result.ok, true);
assert.equal(result.answerText, "第二次 attempt 成功");
const resetAt = emitted.findIndex((event) => event.type === "attempt.reset");
assert.ok(resetAt >= 0);
const firstAttemptAnswers = emitted.slice(0, resetAt).filter((event) => event.type === "answer.delta");
assert.deepEqual(firstAttemptAnswers, []);
const firstAttemptThinking = emitted.slice(0, resetAt)
.filter((event) => event.type === "thinking.delta")
.map((event) => event.text)
.join("");
assert.equal(firstAttemptThinking, "");
assert.match(result.answerText, /已经记下|请继续说下一件/);
assert.equal(emitted.some((event) => event.type === "attempt.reset"), false);
assert.doesNotMatch(JSON.stringify(emitted), /我需要用批量工具/);
const finalizedTurn = accounting.calls.find((call) => call.fn === "finalize_agentic_rectification_turn");
assert.equal(finalizedTurn?.args.p_assistant_message, "第二次 attempt 成功");
assert.equal(finalizedTurn?.args.p_status, "completed");
assert.deepEqual(billing, { reserved: 1, completed: 1, released: 0 });
const firstAttempt = accounting.calls.find((call) =>
call.fn === "finalize_agentic_rectification_run_attempt"
&& call.args.p_attempt_id === "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa");
assert.equal(firstAttempt?.args.p_status, "retryable");
assert.equal(firstAttempt?.args.p_error_code, "empty_stream");
});
test("a length-limited spoken answer is not billed or persisted as a completed turn", async () => {
@@ -762,7 +747,7 @@ test("browser disconnect aborts the run, finalizes retryable and releases usage"
assert.equal(billing.released, 1);
});
test("empty stream fails closed without completing billing", async () => {
test("empty stream uses server narration instead of retrying the attempt", async () => {
const { options, emitted, billing } = runOptions({
buildAgent: async () => fakeAgentStream([
chunk("start"),
@@ -774,10 +759,11 @@ test("empty stream fails closed without completing billing", async () => {
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, false);
assert.equal(result.errorCode, "empty_stream");
assert.equal(billing.released, 1);
assert.equal(emitted.some((event) => event.type === "run.completed"), false);
assert.equal(result.ok, true);
assert.match(result.answerText, /已经记下|请继续说下一件/);
assert.equal(billing.completed, 1);
assert.equal(emitted.some((event) => event.type === "attempt.reset"), false);
assert.equal(emitted.some((event) => event.type === "run.completed"), true);
});
test("legacy Skill identity fails before billing reservation", async () => {
@@ -847,7 +833,7 @@ test("execution receipts are persisted per turn (phases + tools)", async () => {
const SECOND_ATTEMPT_ID = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
type AttemptFailure = "stream_aborted" | "stream_unfinished" | "empty_stream";
type AttemptFailure = "stream_aborted" | "stream_unfinished";
function attemptStream(
chunks: StreamChunk[],
@@ -873,12 +859,10 @@ function failedAttemptChunks(errorCode: AttemptFailure): StreamChunk[] {
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", { toolName: "rectification-set-focus", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-set-focus" }),
chunk("text-delta", { text: errorCode === "empty_stream" ? " " : "失败 attempt 的半截文本" }),
chunk("text-delta", { text: "失败 attempt 的半截文本" }),
];
if (errorCode === "stream_aborted") {
chunks.push(chunk("error", { error: new Error("provider stream aborted") }));
} else if (errorCode === "empty_stream") {
chunks.push(chunk("finish"));
}
return chunks;
}
@@ -897,7 +881,7 @@ function successfulAttemptChunks(): StreamChunk[] {
];
}
for (const failureCode of ["stream_aborted", "stream_unfinished", "empty_stream"] as const) {
for (const failureCode of ["stream_aborted", "stream_unfinished"] as const) {
test(`${failureCode} attempt is isolated and the second successful attempt exclusively commits public truth`, async () => {
let buildCount = 0;
const completedUsage: Array<{ inputTokens: number; outputTokens: number; durationMs: number }> = [];
@@ -935,7 +919,7 @@ for (const failureCode of ["stream_aborted", "stream_unfinished", "empty_stream"
assert.ok(resetAt >= 0);
assert.equal(
emitted.some((event) => event.type === "answer.delta" && event.text?.includes("失败 attempt")),
failureCode !== "empty_stream",
false,
);
assert.deepEqual(
emitted.slice(resetAt + 1).filter((event) => event.type === "answer.delta"),
@@ -1010,7 +994,7 @@ for (const failureCode of ["stream_aborted", "stream_unfinished", "empty_stream"
});
}
test("a failed set-focus cannot complete a question turn even when the agent emits answer text", async () => {
test("a failed set-focus does not reset the attempt or hide the terminal answer", async () => {
let buildCount = 0;
const accounting = fakeAccounting({
...receiptHandlers,
@@ -1022,80 +1006,51 @@ test("a failed set-focus cannot complete a question turn even when the agent emi
idempotent: false,
}),
});
const failedFocusAttempt = () => attemptStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", { toolName: "rectification-set-focus", args: { caseId: CASE_ID } }),
chunk("tool-error", { toolName: "rectification-set-focus", error: new Error("invalid_focus") }),
chunk("text-delta", { text: "主问题:请确认这段经历发生在哪个月?" }),
chunk("finish"),
], { inputTokens: 41, outputTokens: 23 });
const { options, emitted, billing } = runOptions({
accounting: accounting.client,
buildAgent: async () => {
buildCount += 1;
return failedFocusAttempt() as never;
return attemptStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", { toolName: "rectification-set-focus", args: { caseId: CASE_ID } }),
chunk("tool-error", { toolName: "rectification-set-focus", error: new Error("invalid_focus") }),
chunk("text-delta", { text: "主问题:请确认这段经历发生在哪个月?" }),
chunk("finish"),
], { inputTokens: 41, outputTokens: 23 }) as never;
},
});
const result = await runV9AgentTurn(options);
assert.equal(buildCount, 2);
assert.equal(result.ok, false);
assert.equal(result.turnStatus, "retryable");
assert.equal(result.errorCode, "focus_persistence_failed");
assert.equal(result.answerText, "");
assert.deepEqual(result.toolsUsed, ["rectification-read-case", "rectification-set-focus"]);
assert.deepEqual(billing, { reserved: 1, completed: 0, released: 1 });
assert.equal(emitted.some((event) => event.type === "answer.delta"), false);
assert.equal(emitted.some((event) => event.type === "thinking.delta"), false);
assert.equal(emitted.some((event) => event.type === "attempt.reset"), true);
assert.equal(emitted.some((event) => event.type === "run.completed"), false);
assert.equal(buildCount, 1);
assert.equal(result.ok, true);
assert.equal(result.turnStatus, "completed");
assert.equal(result.answerText, "主问题:请确认这段经历发生在哪个月?");
assert.equal(emitted.some((event) => event.type === "attempt.reset"), false);
assert.deepEqual(
emitted.filter((event) => event.type === "answer.delta"),
[{ type: "answer.delta", text: "主问题:请确认这段经历发生在哪个月?" }],
);
assert.equal(
emitted.some((event) => event.type === "tool.activity"
&& (event as { tool?: string; status?: string }).tool === "rectification-set-focus"
&& (event as { tool?: string; status?: string }).status === "failed"),
&& (event as { tool?: string; status?: string }).status === "failed"
&& (event as { code?: string }).code === "duplicate_focus"),
true,
);
assert.equal(emitted.at(-1)?.type, "run.failed");
const attemptFinalizations = accounting.calls
.filter((call) => call.fn === "finalize_agentic_rectification_run_attempt")
.map((call) => ({
status: call.args.p_status,
errorCode: call.args.p_error_code,
usage: call.args.p_usage,
}));
assert.deepEqual(attemptFinalizations, [
{ status: "retryable", errorCode: "focus_persistence_failed", usage: { inputTokens: 0, outputTokens: 0 } },
{ status: "retryable", errorCode: "focus_persistence_failed", usage: { inputTokens: 0, outputTokens: 0 } },
]);
assert.equal(
accounting.calls.some((call) => call.fn === "insert_agentic_rectification_run_phase"
&& call.args.p_phase === "run.completed"),
false,
);
const finalizedTurn = accounting.calls.find((call) => call.fn === "finalize_agentic_rectification_turn");
assert.deepEqual(finalizedTurn?.args, {
p_user_id: USER_ID,
p_case_id: CASE_ID,
p_turn_id: TURN_ID,
p_attempt_id: SECOND_ATTEMPT_ID,
p_status: "retryable",
p_assistant_message: null,
p_successful_attempt_id: null,
});
assert.deepEqual(billing, { reserved: 1, completed: 1, released: 0 });
});
test("a same-attempt set-focus retry that finishes completed can commit the question turn", async () => {
test("does not retry set-focus with identical arguments", async () => {
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture(),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "completed", idempotent: false }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "failed", idempotent: false }),
});
const { options, emitted, billing } = runOptions({
accounting: accounting.client,
@@ -1116,33 +1071,10 @@ test("a same-attempt set-focus retry that finishes completed can commit the ques
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
assert.equal(result.turnStatus, "completed");
assert.equal(result.errorCode, null);
assert.deepEqual(billing, { reserved: 1, completed: 1, released: 0 });
assert.deepEqual(
emitted.filter((event) => event.type === "answer.delta"),
[{ type: "answer.delta", text: "主问题:请确认这段经历发生在哪个月?" }],
);
assert.equal(emitted.some((event) => event.type === "run.completed"), true);
let receiptState = createRectificationActivityReceiptState();
const focusStatuses: string[] = [];
for (const event of emitted) {
const activity = event as { type: string; tool?: string; status?: string };
if (activity.type !== "tool.activity" || activity.tool !== "rectification-set-focus") continue;
if (activity.status !== "started" && activity.status !== "completed" && activity.status !== "failed") continue;
focusStatuses.push(activity.status);
receiptState = reduceRectificationActivityReceipt(receiptState, {
tool: "rectification-set-focus",
status: activity.status,
});
}
assert.deepEqual(focusStatuses, ["started", "failed", "started", "completed"]);
assert.deepEqual(receiptFromRectificationActivityState(receiptState), {
steps: ["rectification-set-focus"],
methods: [],
});
assert.equal(result.ok, false);
assert.equal(result.errorCode, "repeated_tool_call");
assert.equal(emitted.some((event) => event.type === "attempt.reset"), false);
assert.deepEqual(billing, { reserved: 1, completed: 0, released: 1 });
});
test("an unclaimed V10 attempt never starts the model", async () => {
@@ -1272,3 +1204,153 @@ test("thinking-mode tool_choice rejection fails the opening turn without a secon
const attemptFinalize = accounting.calls.find((call) => call.fn === "finalize_agentic_rectification_run_attempt");
assert.equal(attemptFinalize?.args.p_error_code, "thinking_tool_choice_unsupported");
});
test("does not publish intermediate tool-step text as answer.delta", async () => {
const { options, emitted } = runOptions({
message: "2020年4月开始实习 6月转正 10月离职",
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "Let me set" }),
chunk("text-delta", { text: " _probe" }),
chunk("text-delta", { text: " _gain" }),
chunk("tool-call", {
toolName: "rectification-record-evidence-batch",
args: { caseId: CASE_ID },
}),
chunk("tool-result", { toolName: "rectification-record-evidence-batch" }),
chunk("tool-call", { toolName: "rectification-compare-candidates", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-compare-candidates" }),
chunk("text-delta", { text: "记下了,2020年4月开始实习。" }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
assert.deepEqual(
emitted.filter((event) => event.type === "answer.delta"),
[{ type: "answer.delta", text: "记下了,2020年4月开始实习。" }],
);
const publicText = JSON.stringify(emitted);
assert.doesNotMatch(publicText, /Let me set/);
assert.doesNotMatch(publicText, /_probe/);
assert.doesNotMatch(publicText, /_gain/);
assert.equal(emitted.some((event) => event.type === "thinking.delta"), false);
});
test("does not reset the whole attempt after duplicate_focus", async () => {
let buildCount = 0;
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture(),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: (_fn, args) => ({
turn_id: TURN_ID,
status: args.p_status,
idempotent: false,
}),
});
const { options, emitted } = runOptions({
accounting: accounting.client,
buildAgent: async () => {
buildCount += 1;
return attemptStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", { toolName: "rectification-record-evidence-batch", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-record-evidence-batch" }),
chunk("tool-call", { toolName: "rectification-set-focus", args: { caseId: CASE_ID } }),
chunk("tool-error", { toolName: "rectification-set-focus", error: new Error("duplicate_focus") }),
chunk("text-delta", { text: "已经记下实习,接下来对一下考试那年。" }),
chunk("finish"),
], { inputTokens: 11, outputTokens: 12 }) as never;
},
});
const result = await runV9AgentTurn(options);
assert.equal(buildCount, 1);
assert.equal(result.ok, true);
assert.equal(emitted.some((event) => event.type === "attempt.reset"), false);
assert.equal(
result.toolsUsed.filter((name) => name === "rectification-record-evidence-batch").length,
1,
);
});
test("persists message origin for every user turn", async () => {
const message = "2020年4月开始实习 6月转正 10月离职";
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture(),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "completed", idempotent: false }),
});
const { options } = runOptions({
accounting: accounting.client,
message,
messageOrigin: "typed",
clientActionId: "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "记下了,2020年4月开始实习。" }),
chunk("finish"),
]) as never,
});
await runV9AgentTurn(options);
const origin = accounting.calls.find((call) => call.fn === "record_agentic_rectification_turn_origin");
assert.deepEqual(origin?.args, {
p_user_id: USER_ID,
p_case_id: CASE_ID,
p_turn_id: TURN_ID,
p_origin: "typed",
p_client_action_id: "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
p_content_hash: messageContentHash(message),
});
});
test("does not create a 2002 user message from a 2020 assistant suggestion", async () => {
const message = "2020年4月开始实习 6月转正 10月离职";
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture(),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "completed", idempotent: false }),
});
const { options, emitted } = runOptions({
accounting: accounting.client,
message,
messageOrigin: "typed",
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "202" }),
chunk("text-delta", { text: "0" }),
chunk("text-delta", { text: " Let me ask about 2002" }),
chunk("tool-call", { toolName: "rectification-compare-candidates", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-compare-candidates" }),
chunk("text-delta", { text: "记下了,2020年4月开始实习。" }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
const origin = accounting.calls.find((call) => call.fn === "record_agentic_rectification_turn_origin");
assert.equal(origin?.args.p_origin, "typed");
assert.equal(origin?.args.p_content_hash, messageContentHash(message));
const publicText = JSON.stringify(emitted);
assert.doesNotMatch(publicText, /2002 年发生什么了/);
assert.doesNotMatch(result.answerText, /2002/);
assert.equal(result.answerText, "记下了,2020年4月开始实习。");
});