fix(consult): treat pinched answers as failed and restore reply actions

Incomplete Flash generations were billed as completed consultations. Fail
those runs, keep the partial text, and reuse the rectification like/copy/rerun
bar on ordinary chat replies.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-19 19:37:02 +08:00
parent c38f11dbd3
commit 7d667fecbf
17 changed files with 487 additions and 89 deletions
@@ -145,6 +145,10 @@ test("error normalization never records arbitrary exception messages", () => {
toAgentObservabilityErrorCode(new Error("runtime_contract_incomplete")),
"runtime_contract_incomplete",
);
assert.equal(
toAgentObservabilityErrorCode(new Error("answer_truncated")),
"answer_truncated",
);
assert.equal(
toAgentObservabilityErrorCode(new Error("/opt/internal/users/alice.json")),
"calculation_failed",
@@ -0,0 +1,11 @@
import assert from "node:assert/strict";
import test from "node:test";
import { toggleChatMessageFeedback } from "../src/components/chat-message-actions.tsx";
test("message feedback is mutually exclusive and can be cleared", () => {
assert.equal(toggleChatMessageFeedback(undefined, "up"), "up");
assert.equal(toggleChatMessageFeedback("up", "up"), undefined);
assert.equal(toggleChatMessageFeedback("up", "down"), "down");
assert.equal(toggleChatMessageFeedback("down", "up"), "up");
});
+23 -2
View File
@@ -70,8 +70,10 @@ test("shows honest agent activity states before and during streamed text", () =>
assert.doesNotMatch(globalStyles, /\.thinking\b/);
assert.match(pageSource, /application\/x-ndjson/);
assert.match(pageSource, /createNdjsonParser/);
assert.match(pageSource, /if \(event\.type === "run\.failed"\) throw new ConsultationResponseError/);
assert.match(pageSource, /if \(!runCompleted\) throw new ConsultationResponseError/);
assert.match(pageSource, /event\.type === "run\.failed"/);
assert.match(pageSource, /event\.code === "answer_truncated"/);
assert.match(pageSource, /throw new ConsultationResponseError/);
assert.match(pageSource, /if \(!runCompleted && !truncatedFailure\) throw new ConsultationResponseError/);
assert.match(pageSource, /agentExecutionReceipt = event\.receipt/);
});
@@ -95,3 +97,22 @@ test("docks the composer inside the chat panel instead of floating over content"
assert.match(globalStyles, /\.composer-wrap[^}]*bottom:\s*0/);
assert.doesNotMatch(globalStyles, /\.composer-wrap[^}]*position:\s*fixed/);
});
test("ordinary consultation replies reuse the shared Agent action bar", () => {
const actionsSource = readFileSync(new URL("../src/components/chat-message-actions.tsx", import.meta.url), "utf8");
const rectificationChat = readFileSync(
new URL("../src/components/rectification-agentic-chat.tsx", import.meta.url),
"utf8",
);
for (const label of ["赞", "踩", "复制回答", "重新生成回答"]) {
assert.match(actionsSource, new RegExp(`aria-label="${label}"`));
}
assert.match(pageSource, /<ChatMessageActions/);
assert.match(pageSource, /toggleChatMessageFeedback/);
assert.match(pageSource, /function regenerateLatestAnswer\(renderKey: string\)/);
assert.match(pageSource, /messages: session\.messages\.slice\(0, -1\)/);
assert.match(pageSource, /restoreOnFailure: session/);
assert.match(rectificationChat, /<ChatMessageActions/);
assert.match(globalStyles, /\.message-actions \{/);
assert.doesNotMatch(globalStyles, /\.rectification-message-actions \{/);
});
@@ -1,12 +1,15 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
AGENT_MAX_STEPS,
AGENT_TIMEOUT_MS,
CONSULTATION_MAX_OUTPUT_TOKENS,
mergeConsultationAnswerPolicies,
CONSULTATION_DOMAIN_WALL_CLOCK_MS,
MAX_CONSULTATION_DOMAINS,
appendConsultationRuntimeStep,
canonicalDomainPlan,
consultationGenerationSettings,
consultationModelStepTelemetry,
consultationStepBudgetReceipt,
consultationToolFailureCode,
@@ -1175,6 +1178,87 @@ test("a calculation still unanswered after the retry fails the run rather than b
assert.match(failure.message, /不会扣点/);
});
test("a length-limited answer is not billed or delivered as a completed consultation", async () => {
// Staging persisted a 253-character pinch that ended mid-heading, then treated
// the run as completed. The model had finished with reason `length`; the public
// stream still emitted `run.completed`, so the composer unlocked as if the
// reading were done.
const state = toolOnlyRunState();
const pinchedHeading = "**先看命盘结构(Lahiri岁差、均交点口径";
let completed = 0;
let failed = 0;
async function* chunks() {
yield { type: "tool-result", payload: { toolCallId: "tool-1", toolName: "run-jyotish-consultation", result: {} } };
yield { type: "text-delta", payload: { text: pinchedHeading } };
yield { type: "finish", payload: { stepResult: { reason: "length" }, output: { usage: {}, steps: [{}, {}] } } };
}
const response = streamAgentResponse({
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
toolStatus: () => "ready", receipt: () => receipt(state),
onComplete: () => { completed += 1; },
onError: () => { failed += 1; },
});
const events: unknown[] = [];
const parser = createNdjsonParser((event) => events.push(event));
parser.finish(await response.text());
assert.equal(completed, 0);
assert.equal(failed, 1);
assert.equal(state.modelFinishReason, "length");
assert.equal(events.filter((event) => (event as { type?: string }).type === "run.completed").length, 0);
const answer = events
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
.map((event) => event.text)
.join("");
assert.equal(answer, pinchedHeading);
const failure = events.find((event) => (event as { type?: string }).type === "run.failed") as {
code: string;
message: string;
};
assert.equal(failure.code, "answer_truncated");
assert.match(failure.message, /回答未完成/);
assert.match(failure.message, /不会扣点/);
assert.doesNotMatch(JSON.stringify(failure), /modelFinishReason|"length"/);
});
test("a timeout after partial visible text is the same truncation, not a successful answer", async () => {
const state = toolOnlyRunState();
const pinchedHeading = "**先看命盘结构(Lahiri岁差、均交点口径";
let completed = 0;
async function* chunks() {
yield { type: "tool-result", payload: { toolCallId: "tool-1", toolName: "run-jyotish-consultation", result: {} } };
yield { type: "text-delta", payload: { text: pinchedHeading } };
throw new DOMException("The operation was aborted due to timeout", "TimeoutError");
}
const response = streamAgentResponse({
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
toolStatus: () => "ready", receipt: () => receipt(state),
onComplete: () => { completed += 1; },
onError: () => {},
});
const events: unknown[] = [];
const parser = createNdjsonParser((event) => events.push(event));
parser.finish(await response.text());
assert.equal(completed, 0);
const answer = events
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
.map((event) => event.text)
.join("");
assert.equal(answer, pinchedHeading);
const failure = events.find((event) => (event as { type?: string }).type === "run.failed") as { code: string };
assert.equal(failure.code, "answer_truncated");
});
test("consult generation reserves visible output tokens and disables provider thinking", () => {
const settings = consultationGenerationSettings("deepseek");
assert.equal(CONSULTATION_MAX_OUTPUT_TOKENS, 8192);
assert.equal(settings.modelSettings.maxOutputTokens, CONSULTATION_MAX_OUTPUT_TOKENS);
assert.deepEqual(settings.providerOptions.openai, { thinking: { type: "disabled" } });
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "disabled" } });
assert.equal(AGENT_MAX_STEPS, 8);
});
test("a completed run records the finish reason and the authoritative step count", async () => {
const state = createConsultationRuntimeState();
@@ -133,3 +133,14 @@ test("the first default consultation title is persisted with the user question",
assert.match(sendSource, /const completedSession: ChatSession = \{[\s\S]*title: userSession\.title/);
assert.doesNotMatch(sendSource, /resolveSessionTitle\(question, reply\.title\)/);
});
test("a truncated generation keeps the partial answer and does not wait for a successful run", () => {
const stream = sendSource.slice(sendSource.indexOf('fetch("/api/consult"'));
assert.match(stream, /event\.code === "answer_truncated"/);
assert.match(stream, /truncatedFailure = event/);
assert.match(stream, /const truncatedSession: ChatSession = \{[\s\S]*role: "assistant"[\s\S]*text: reply\.text/);
assert.match(stream, /await persistSession\(truncatedSession\)/);
assert.match(stream, /setComposerNotice\(truncatedFailure\.message\)/);
assert.match(stream, /if \(!runCompleted && !truncatedFailure\) throw new ConsultationResponseError/);
assert.doesNotMatch(stream.slice(stream.indexOf("if (truncatedFailure)")), /runCompleted = true/);
});
@@ -65,6 +65,15 @@ test("the model step budget and the wall-clock budget are declared as one pair",
assert.doesNotMatch(route, /AbortSignal\.timeout\(\d/);
});
test("consult streams cap visible output and disable thinking instead of sharing the token budget with hidden reasoning", () => {
assert.match(tools, /export const CONSULTATION_MAX_OUTPUT_TOKENS = 8192;/);
assert.match(tools, /function consultationGenerationSettings/);
assert.match(tools, /thinking: \{ type: "disabled"/);
assert.match(tools, /maxOutputTokens: CONSULTATION_MAX_OUTPUT_TOKENS/);
assert.match(route, /\.\.\.consultationGenerationSettings\(selectedModel\.model\)/);
assert.doesNotMatch(route, /maxOutputTokens:\s*\d/);
});
test("uses one runtime step append entry and no scattered hard-coded step cap", () => {
assert.match(tools, /export function appendConsultationRuntimeStep/);
assert.doesNotMatch(tools, /steps\.length\s*>=\s*32/);
@@ -10,6 +10,10 @@ const chat = readFileSync(
new URL("../src/components/rectification-agentic-chat.tsx", import.meta.url),
"utf8",
);
const messageActions = readFileSync(
new URL("../src/components/chat-message-actions.tsx", import.meta.url),
"utf8",
);
const route = readFileSync(
new URL("../src/app/api/rectification/agent/route.ts", import.meta.url),
"utf8",
@@ -274,8 +278,9 @@ test("rectification activity separates live work from the receipt above the Agen
test("completed Agent replies restore feedback, copy and safe in-place regeneration actions", () => {
for (const label of ["赞", "踩", "复制回答", "重新生成回答"]) {
assert.match(chat, new RegExp(`aria-label="${label}"`));
assert.match(messageActions, new RegExp(`aria-label="${label}"`));
}
assert.match(chat, /<ChatMessageActions/);
assert.match(chat, /toggleRectificationFeedback/);
assert.match(chat, /navigator\.clipboard\.writeText\(message\.text\)/);
assert.match(chat, /\/turns\/\$\{encodeURIComponent\(message\.turnId\)\}\/regenerate/);
@@ -287,7 +292,7 @@ test("completed Agent replies restore feedback, copy and safe in-place regenerat
);
const receiptIndex = messageRender.indexOf("<CompletedActivityReceipt");
const replyIndex = messageRender.indexOf("<ChatMessageRow");
const actionsIndex = messageRender.indexOf('className="rectification-message-actions"');
const actionsIndex = messageRender.indexOf("<ChatMessageActions");
assert.ok(receiptIndex >= 0 && replyIndex >= 0 && actionsIndex >= 0);
assert.ok(receiptIndex < replyIndex && replyIndex < actionsIndex);