fix(web): keep thinking off the spoken consult and rectification answer

Enumerate evidence kinds so education cannot be proposed as a kind, and stream Chinese thinking on a separate channel that collapses when the reply arrives.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-21 21:30:55 +08:00
parent 56577504d8
commit 4e247c112e
27 changed files with 465 additions and 73 deletions
@@ -17,7 +17,7 @@ import {
rectificationToolActivityPhase,
} from "../src/lib/rectification-activity-labels.ts";
test("generation settings reserve visible tokens and disable thinking", () => {
test("generation settings reserve visible tokens and disable thinking by default", () => {
const settings = agentGenerationSettings({ providerId: "deepseek" });
assert.equal(AGENT_MAX_OUTPUT_TOKENS, 8192);
assert.equal(settings.modelSettings.maxOutputTokens, 8192);
@@ -25,6 +25,13 @@ test("generation settings reserve visible tokens and disable thinking", () => {
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "disabled" } });
});
test("rectification can enable a separate thinking channel without changing the visible budget", () => {
const settings = agentGenerationSettings({ providerId: "deepseek" }, { thinking: "enabled" });
assert.equal(settings.modelSettings.maxOutputTokens, 8192);
assert.deepEqual(settings.providerOptions.openai, { thinking: { type: "enabled" } });
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "enabled" } });
});
test("activity elapsed copy stays hidden until eight seconds", () => {
assert.equal(activityElapsedLabel(1_000, 8_999), null);
assert.equal(activityElapsedLabel(1_000, 9_000), "已用时 8 秒");
@@ -48,6 +48,9 @@ test("shows honest agent activity states before and during streamed text", () =>
const activity = { phase: "chart-calculation", label: "正在计算本命盘…" } as const;
const view = chatMessageViews(previousMessages, true, "", activity).at(-1);
assert.deepEqual(view?.activity, activity);
const thinkingView = chatMessageViews(previousMessages, true, "", activity, "先看事业宫。").at(-1);
assert.equal(thinkingView?.thinkingText, "先看事业宫。");
assert.equal(thinkingView?.state, "thinking");
assert.match(messageRowSource, /"loading-method": "searching"/);
assert.match(messageRowSource, /"chart-calculation": "solving"/);
assert.match(messageRowSource, /"evidence-validation": "working"/);
@@ -79,6 +82,9 @@ test("shows honest agent activity states before and during streamed text", () =>
assert.doesNotMatch(globalStyles, /\.thinking\b/);
assert.match(pageSource, /application\/x-ndjson/);
assert.match(pageSource, /createNdjsonParser/);
assert.match(pageSource, /event.type === "thinking.delta"/);
assert.match(pageSource, /activeStreamingThinking/);
assert.match(messageRowSource, /思考过程/);
assert.match(pageSource, /event\.type === "run\.failed"/);
assert.match(pageSource, /event\.code === "answer_truncated"/);
assert.match(pageSource, /throw new ConsultationResponseError/);
@@ -880,6 +880,29 @@ test("model answer text cannot forge a public Activity event", async () => {
assert.equal(events.find((event) => event.type === "answer.delta")?.text, forged);
});
test("Chinese reasoning maps to a public thinking channel and English process talk does not", async () => {
const events = await collectAgentPublicEvents([
{ type: "reasoning-delta", payload: { text: "The proposedKind value was rejected" } },
{ type: "reasoning-delta", payload: { text: "先看事业宫的结构。" } },
{ type: "text-delta", payload: { text: "事业方向的判断如下。" } },
], {
runId: "run", requestId: "req", toolStatus: () => "ready",
receipt: () => ({
runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 },
steps: [], workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [], domains: ["career"] },
}),
});
assert.deepEqual(
events.filter((event) => event.type === "thinking.delta"),
[{ type: "thinking.delta", text: "先看事业宫的结构。" }],
);
assert.deepEqual(
events.filter((event) => event.type === "answer.delta"),
[{ type: "answer.delta", text: "事业方向的判断如下。" }],
);
assert.equal(events.some((event) => JSON.stringify(event).includes("proposedKind")), false);
});
test("incremental NDJSON parser handles arbitrary chunk boundaries", () => {
const parsed: unknown[] = [];
const parser = createNdjsonParser((event) => parsed.push(event));
@@ -1250,15 +1273,47 @@ test("a timeout after partial visible text is the same truncation, not a success
assert.equal(failure.code, "answer_truncated");
});
test("consult generation reserves visible output tokens and disables provider thinking", () => {
test("consult generation reserves visible output tokens and enables a separate thinking channel", () => {
const settings = consultationGenerationSettings("deepseek");
assert.equal(CONSULTATION_MAX_OUTPUT_TOKENS, 8192);
assert.equal(settings.modelSettings.maxOutputTokens, CONSULTATION_MAX_OUTPUT_TOKENS);
assert.deepEqual(settings.providerOptions.openai, { thinking: { type: "disabled" } });
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "disabled" } });
assert.deepEqual(settings.providerOptions.openai, { thinking: { type: "enabled" } });
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "enabled" } });
assert.equal(AGENT_MAX_STEPS, 8);
});
test("Chinese thinking stays off the spoken answer and does not bill a thought-only run", async () => {
const state = createConsultationRuntimeState();
state.jyotishSkillBound = true;
state.consultationToolCallCount = 1;
state.consultationToolSuccessCount = 1;
state.consultationToolCompleted = true;
state.workflowReceipt = { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] };
async function* chunks() {
yield { type: "reasoning-delta", payload: { text: "The proposedKind value was rejected" } };
yield { type: "reasoning-delta", payload: { text: "先看事业宫的结构。" } };
yield { type: "text-delta", payload: { text: "事业方向的判断如下。" } };
}
const response = streamAgentResponse({
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
toolStatus: () => "ready", receipt: () => receipt(state),
});
const events: unknown[] = [];
const parser = createNdjsonParser((event) => events.push(event));
parser.finish(await response.text());
assert.deepEqual(
events.filter((event) => (event as { type?: string }).type === "thinking.delta"),
[{ type: "thinking.delta", text: "先看事业宫的结构。" }],
);
const answer = events
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
.map((event) => event.text)
.join("");
assert.equal(answer, "事业方向的判断如下。");
assert.doesNotMatch(JSON.stringify(events), /proposedKind/);
assert.equal(events.filter((event) => (event as { type?: string }).type === "run.completed").length, 1);
});
test("a completed run records the finish reason and the authoritative step count", async () => {
const state = createConsultationRuntimeState();
@@ -66,16 +66,18 @@ test("the model step budget and the wall-clock budget are declared as one pair",
assert.doesNotMatch(route, /AbortSignal\.timeout\(\d/);
});
test("consult streams cap visible output and disable thinking instead of sharing the token budget with hidden reasoning", () => {
test("consult streams cap visible output and enable a separate thinking channel", () => {
const settings = readFileSync(new URL("../src/lib/agent-generation-settings.ts", import.meta.url), "utf8");
assert.match(settings, /export const AGENT_MAX_OUTPUT_TOKENS = 8192;/);
assert.match(settings, /thinking: \{ type: "disabled"/);
assert.match(settings, /options\.thinking \?\? "disabled"/);
assert.match(settings, /maxOutputTokens: AGENT_MAX_OUTPUT_TOKENS/);
assert.match(tools, /function consultationGenerationSettings/);
assert.match(tools, /return agentGenerationSettings\(model\)/);
assert.match(tools, /return agentGenerationSettings\(model, \{ thinking: "enabled" \}\)/);
assert.match(tools, /AGENT_MAX_OUTPUT_TOKENS as CONSULTATION_MAX_OUTPUT_TOKENS/);
assert.match(route, /\.\.\.consultationGenerationSettings\(selectedModel\.model\)/);
assert.doesNotMatch(route, /maxOutputTokens:\s*\d/);
assert.match(stream, /thinking\.delta/);
assert.match(stream, /reasoning-delta/);
});
test("uses one runtime step append entry and no scattered hard-coded step cap", () => {
@@ -10,6 +10,10 @@ const chat = readFileSync(
new URL("../src/components/rectification-agentic-chat.tsx", import.meta.url),
"utf8",
);
const messageRow = readFileSync(
new URL("../src/components/chat-message-row.tsx", import.meta.url),
"utf8",
);
const messageActions = readFileSync(
new URL("../src/components/chat-message-actions.tsx", import.meta.url),
"utf8",
@@ -223,6 +227,8 @@ test("usage completes or releases without hiding settlement failures", () => {
assert.match(run, /billing\.complete\(/);
assert.match(run, /billing\.release\(/);
assert.match(run, /usage_settlement_failed/);
assert.match(run, /thinking: "enabled"/);
assert.match(run, /toPublicThinkingDelta/);
assert.match(route, /featureKey: "rectification"/);
assert.match(route, /rectification:case:\$\{caseId\}/);
});
@@ -293,6 +299,11 @@ test("rectification keeps receipts for the varga sentence and shows live tool pr
}
assert.match(chat, /回答未完成,已保留现有内容;本次不会扣点/);
assert.doesNotMatch(chat, /reasoning-delta|chain-of-thought/);
assert.match(chat, /event.type === "thinking.delta"/);
assert.match(messageRow, /className="message-thinking"/);
assert.match(messageRow, /思考过程/);
assert.match(messageRow, /userOpen \?\? !hasAnswer/);
assert.match(styles, /\.message-thinking-body/);
});
test("completed Agent replies restore feedback, copy and safe in-place regeneration actions", () => {
@@ -502,6 +513,7 @@ test("rectification Agent output stays natural and keeps tool execution silent",
"utf8",
);
assert.match(agent, /工具执行过程保持静默/);
assert.match(agent, /思考过程必须用简体中文/);
assert.match(agent, /本轮做了什么/);
assert.match(agent, /完成凭证完全由服务端公开 Activity\/receipt 展示/);
assert.match(agent, /禁止只说记下了、会话会保留、以后再继续/);
@@ -180,3 +180,24 @@ test("batch item schemas are independently strict and bounded", () => {
items: Array.from({ length: 13 }, () => item),
}).success, false);
});
test("evidence kind and domain schemas enumerate legal values so education is not a proposedKind", () => {
const tools = toolsUnderTest() as unknown as Record<string, ToolSchema>;
const propose = tools["rectification-propose-evidence"].inputSchema;
const batch = tools["rectification-record-evidence-batch"].inputSchema;
const validPropose = validInputs["rectification-propose-evidence"];
const validBatch = validInputs["rectification-record-evidence-batch"];
const item = (validBatch.items as Record<string, unknown>[])[0]!;
assert.equal(propose.safeParse(validPropose).success, true);
assert.equal(propose.safeParse({ ...validPropose, proposedKind: "education" }).success, false);
assert.equal(propose.safeParse({ ...validPropose, proposedKind: "education_start" }).success, true);
assert.equal(batch.safeParse({
...validBatch,
items: [{ ...item, proposedKind: "education" }],
}).success, false);
assert.equal(batch.safeParse({
...validBatch,
items: [{ ...item, proposedKind: "education_start", domain: "education" }],
}).success, true);
});
@@ -67,6 +67,8 @@ test("system prompt carries only high-priority boundaries, never the method copy
assert.match(prompt, /rectification-offer-candidates/);
assert.match(prompt, /on_user_stop/);
assert.match(prompt, /禁止只说记下了/);
assert.match(prompt, /工具执行过程保持静默/);
assert.match(prompt, /思考过程必须用简体中文/);
assert.match(prompt, /skill_verification_report/);
assert.match(prompt, /精度阶段追问不挡出牌/);
assert.match(prompt, /不得询问外貌、体质、胎记或疤痕/);
@@ -294,7 +296,7 @@ test("server-loaded Skill is bound before the provider and the first model step
assert.equal(typeof firstStep?.toolChoice, "string");
assert.equal(await observedStreamOptions.prepareStep?.({ stepNumber: 1 }), undefined);
assert.equal(observedStreamOptions.modelSettings?.maxOutputTokens, 8192);
assert.deepEqual(observedStreamOptions.providerOptions?.openai, { thinking: { type: "disabled" } });
assert.deepEqual(observedStreamOptions.providerOptions?.openai, { thinking: { type: "enabled" } });
assert.equal(emitted.filter((event) => event.type === "skill.bound").length, 1);
assert.equal(emitted.some((event) => event.type === "run.completed"), true);
assert.equal(
@@ -17,6 +17,8 @@ import {
isTerminalStatus,
} from "../src/lib/rectification-agentic/v9/case-status.ts";
import {
evidenceKindSchema,
evidenceDomainSchema,
BACKGROUND_ONLY_KINDS,
DISTINCT_KIND_GROUPS,
EVIDENCE_KINDS,
@@ -117,6 +119,10 @@ test("skill keeps Path C A/B/C/D questions and forbids unique-minute claims", ()
test("evidence model exposes the full kind/domain/precision/status sets", () => {
for (const kind of EVIDENCE_KINDS) assert.equal(isEvidenceKind(kind), true);
assert.equal(isEvidenceKind("education"), false);
assert.equal(evidenceKindSchema.safeParse("education").success, false);
assert.equal(evidenceKindSchema.safeParse("education_start").success, true);
assert.equal(evidenceDomainSchema.safeParse("education").success, true);
assert.equal(isEvidenceKind("career"), false);
assert.equal(isEvidenceDomain("career"), true);
assert.equal(isEvidenceDomain("nope"), false);
@@ -176,6 +182,7 @@ test("public receipt allowlists are exact and deny unknown values", () => {
}
assert.equal(safeActivityEvent("provider.reasoning"), null);
assert.equal(safeActivityEvent("tool.payload"), null);
assert.ok((PUBLIC_RECTIFICATION_PHASES as readonly string[]).includes("thinking.delta"));
assert.equal(PUBLIC_ACTIVITY_EVENTS.length, PUBLIC_RECTIFICATION_PHASES.length);
assert.ok(PUBLIC_RECTIFICATION_TOOLS.includes("rectification-read-case"));
assert.ok(!(PUBLIC_RECTIFICATION_TOOLS as readonly string[]).includes("rectification-scan"));
+86 -3
View File
@@ -4,6 +4,8 @@ import test from "node:test";
import {
mapStreamChunkToActivity,
mapStreamChunkToPhase,
mapStreamChunkToThinking,
toPublicThinkingDelta,
safePublicEvent,
streamToolNames,
} from "../src/lib/rectification-agentic/v9/stream-mapping.ts";
@@ -132,7 +134,7 @@ test("every public rectification tool maps its real lifecycle to public activity
assert.equal(mapStreamChunkToActivity(chunk("text-delta", { text: "x" }) as never), null);
});
test("reasoning, raw payloads, provider metadata and step internals never map", () => {
test("reasoning, raw payloads, provider metadata and step internals never map to the answer channel", () => {
assert.equal(mapStreamChunkToPhase(chunk("reasoning-start", { id: "r1" }) as never), null);
assert.equal(mapStreamChunkToPhase(chunk("reasoning-delta", { text: "内部推理" }) as never), null);
assert.equal(mapStreamChunkToPhase(chunk("reasoning-end") as never), null);
@@ -144,6 +146,36 @@ test("reasoning, raw payloads, provider metadata and step internals never map",
assert.equal(mapStreamChunkToPhase(chunk("file", { mimeType: "text/plain" }) as never), null);
});
test("Chinese thinking maps to a public thinking channel and English process talk does not", () => {
assert.deepEqual(
mapStreamChunkToThinking(chunk("reasoning-delta", { text: "先核对升学年份。" }) as never),
{ type: "thinking.delta", text: "先核对升学年份。" },
);
assert.equal(
mapStreamChunkToThinking(chunk("reasoning-delta", {
text: "The proposedKind value was rejected",
}) as never),
null,
);
assert.equal(
toPublicThinkingDelta("The proposedKind value was rejected because education is invalid"),
null,
);
assert.deepEqual(
safePublicEvent({ type: "thinking.delta", text: "先核对升学年份。" }),
{ type: "thinking.delta", text: "先核对升学年份。" },
);
assert.deepEqual(
safePublicEvent({
type: "thinking.delta",
text: "先核对升学年份。",
turnId: TURN_ID,
args: { caseId: CASE_ID },
}),
{ type: "thinking.delta", text: "先核对升学年份。" },
);
});
test("streamToolNames exposes only allowlisted rectification tools", () => {
assert.deepEqual(streamToolNames(chunk("tool-call", { toolName: "rectification-read-case" }) as never), ["rectification-read-case"]);
assert.deepEqual(streamToolNames(chunk("tool-call", { toolName: "skill" }) as never), []);
@@ -153,6 +185,7 @@ test("streamToolNames exposes only allowlisted rectification tools", () => {
test("safePublicEvent drops anything outside the allowlist", () => {
assert.deepEqual(safePublicEvent({ type: "answer.delta", text: "你好" }), { type: "answer.delta", text: "你好" });
assert.deepEqual(safePublicEvent({ type: "thinking.delta", text: "先核对升学" }), { type: "thinking.delta", text: "先核对升学" });
assert.deepEqual(safePublicEvent({ type: "attempt.reset" }), { type: "attempt.reset" });
assert.deepEqual(safePublicEvent({ type: "skill.loaded" }), { type: "skill.loaded" });
assert.deepEqual(
@@ -441,6 +474,10 @@ test("answer deltas stream in order and reasoning is never forwarded", async ()
{ type: "answer.delta", text: "好的," },
{ type: "answer.delta", text: "先确认一下:" },
]);
assert.deepEqual(
emitted.filter((event) => event.type === "thinking.delta"),
[{ type: "thinking.delta", text: "我应该先……" }],
);
assert.deepEqual(
emitted.find((event) => event.type === "run.completed"),
{ type: "run.completed", turnId: TURN_ID },
@@ -449,6 +486,50 @@ test("answer deltas stream in order and reasoning is never forwarded", async ()
assert.equal(emitted.some((event) => String(event.type).includes("raw")), false);
});
test("English tool-retry narration never becomes the spoken answer", async () => {
const { options, emitted } = runOptions({
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", {
toolName: "rectification-record-evidence-batch",
args: { caseId: CASE_ID, proposedKind: "education" },
}),
chunk("tool-error", {
toolName: "rectification-record-evidence-batch",
error: new Error("invalid_event_kind"),
}),
chunk("text-delta", {
text: "The proposedKind value was rejected. Retrying with education_start.",
}),
chunk("reasoning-delta", { text: "先改用升学开始。" }),
chunk("tool-call", {
toolName: "rectification-record-evidence-batch",
args: { caseId: CASE_ID, proposedKind: "education_start" },
}),
chunk("tool-result", { toolName: "rectification-record-evidence-batch" }),
chunk("text-delta", { text: "记下了,2016年9月上大学。" }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
assert.deepEqual(
emitted.filter((event) => event.type === "answer.delta"),
[{ type: "answer.delta", text: "记下了,2016年9月上大学。" }],
);
assert.deepEqual(
emitted.filter((event) => event.type === "thinking.delta"),
[{ type: "thinking.delta", text: "先改用升学开始。" }],
);
const publicText = JSON.stringify(emitted);
assert.doesNotMatch(publicText, /The proposedKind value was rejected/);
assert.doesNotMatch(publicText, /invalid_event_kind/);
});
test("a length-limited spoken answer is not billed or persisted as a completed turn", async () => {
const pinched = "**先看候选结构(还不能确认唯一分钟";
const accounting = fakeAccounting({
@@ -662,8 +743,9 @@ test("execution receipts are persisted per turn (phases + tools)", async () => {
assert.ok(phases.includes("answer.composed"));
assert.ok(phases.includes("billing.settled"));
assert.ok(phases.includes("run.completed"));
// answer.delta is never persisted per-delta.
// answer.delta and thinking.delta are never persisted per-delta.
assert.ok(!phases.includes("answer.delta"));
assert.ok(!phases.includes("thinking.delta"));
assert.deepEqual(result.toolsUsed, ["rectification-read-case"]);
});
@@ -872,7 +954,8 @@ test("a failed set-focus cannot complete a question turn even when the agent emi
assert.equal(result.answerText, "");
assert.deepEqual(result.toolsUsed, ["rectification-read-case", "rectification-set-focus"]);
assert.deepEqual(billing, { reserved: 1, completed: 0, released: 1 });
assert.equal(emitted.some((event) => event.type === "answer.delta"), true);
assert.equal(emitted.some((event) => event.type === "answer.delta"), false);
assert.equal(emitted.some((event) => event.type === "thinking.delta"), true);
assert.equal(emitted.some((event) => event.type === "attempt.reset"), true);
assert.equal(emitted.some((event) => event.type === "run.completed"), false);
assert.equal(