fix(consult): treat pinched answers as failed and restore reply actions
Incomplete Flash generations were billed as completed consultations. Fail those runs, keep the partial text, and reuse the rectification like/copy/rerun bar on ordinary chat replies. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -145,6 +145,10 @@ test("error normalization never records arbitrary exception messages", () => {
|
||||
toAgentObservabilityErrorCode(new Error("runtime_contract_incomplete")),
|
||||
"runtime_contract_incomplete",
|
||||
);
|
||||
assert.equal(
|
||||
toAgentObservabilityErrorCode(new Error("answer_truncated")),
|
||||
"answer_truncated",
|
||||
);
|
||||
assert.equal(
|
||||
toAgentObservabilityErrorCode(new Error("/opt/internal/users/alice.json")),
|
||||
"calculation_failed",
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { toggleChatMessageFeedback } from "../src/components/chat-message-actions.tsx";
|
||||
|
||||
test("message feedback is mutually exclusive and can be cleared", () => {
|
||||
assert.equal(toggleChatMessageFeedback(undefined, "up"), "up");
|
||||
assert.equal(toggleChatMessageFeedback("up", "up"), undefined);
|
||||
assert.equal(toggleChatMessageFeedback("up", "down"), "down");
|
||||
assert.equal(toggleChatMessageFeedback("down", "up"), "up");
|
||||
});
|
||||
@@ -70,8 +70,10 @@ test("shows honest agent activity states before and during streamed text", () =>
|
||||
assert.doesNotMatch(globalStyles, /\.thinking\b/);
|
||||
assert.match(pageSource, /application\/x-ndjson/);
|
||||
assert.match(pageSource, /createNdjsonParser/);
|
||||
assert.match(pageSource, /if \(event\.type === "run\.failed"\) throw new ConsultationResponseError/);
|
||||
assert.match(pageSource, /if \(!runCompleted\) throw new ConsultationResponseError/);
|
||||
assert.match(pageSource, /event\.type === "run\.failed"/);
|
||||
assert.match(pageSource, /event\.code === "answer_truncated"/);
|
||||
assert.match(pageSource, /throw new ConsultationResponseError/);
|
||||
assert.match(pageSource, /if \(!runCompleted && !truncatedFailure\) throw new ConsultationResponseError/);
|
||||
assert.match(pageSource, /agentExecutionReceipt = event\.receipt/);
|
||||
});
|
||||
|
||||
@@ -95,3 +97,22 @@ test("docks the composer inside the chat panel instead of floating over content"
|
||||
assert.match(globalStyles, /\.composer-wrap[^}]*bottom:\s*0/);
|
||||
assert.doesNotMatch(globalStyles, /\.composer-wrap[^}]*position:\s*fixed/);
|
||||
});
|
||||
|
||||
test("ordinary consultation replies reuse the shared Agent action bar", () => {
|
||||
const actionsSource = readFileSync(new URL("../src/components/chat-message-actions.tsx", import.meta.url), "utf8");
|
||||
const rectificationChat = readFileSync(
|
||||
new URL("../src/components/rectification-agentic-chat.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
for (const label of ["赞", "踩", "复制回答", "重新生成回答"]) {
|
||||
assert.match(actionsSource, new RegExp(`aria-label="${label}"`));
|
||||
}
|
||||
assert.match(pageSource, /<ChatMessageActions/);
|
||||
assert.match(pageSource, /toggleChatMessageFeedback/);
|
||||
assert.match(pageSource, /function regenerateLatestAnswer\(renderKey: string\)/);
|
||||
assert.match(pageSource, /messages: session\.messages\.slice\(0, -1\)/);
|
||||
assert.match(pageSource, /restoreOnFailure: session/);
|
||||
assert.match(rectificationChat, /<ChatMessageActions/);
|
||||
assert.match(globalStyles, /\.message-actions \{/);
|
||||
assert.doesNotMatch(globalStyles, /\.rectification-message-actions \{/);
|
||||
});
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
AGENT_MAX_STEPS,
|
||||
AGENT_TIMEOUT_MS,
|
||||
CONSULTATION_MAX_OUTPUT_TOKENS,
|
||||
mergeConsultationAnswerPolicies,
|
||||
CONSULTATION_DOMAIN_WALL_CLOCK_MS,
|
||||
MAX_CONSULTATION_DOMAINS,
|
||||
appendConsultationRuntimeStep,
|
||||
canonicalDomainPlan,
|
||||
consultationGenerationSettings,
|
||||
consultationModelStepTelemetry,
|
||||
consultationStepBudgetReceipt,
|
||||
consultationToolFailureCode,
|
||||
@@ -1175,6 +1178,87 @@ test("a calculation still unanswered after the retry fails the run rather than b
|
||||
assert.match(failure.message, /不会扣点/);
|
||||
});
|
||||
|
||||
test("a length-limited answer is not billed or delivered as a completed consultation", async () => {
|
||||
// Staging persisted a 253-character pinch that ended mid-heading, then treated
|
||||
// the run as completed. The model had finished with reason `length`; the public
|
||||
// stream still emitted `run.completed`, so the composer unlocked as if the
|
||||
// reading were done.
|
||||
const state = toolOnlyRunState();
|
||||
const pinchedHeading = "**先看命盘结构(Lahiri岁差、均交点口径";
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
async function* chunks() {
|
||||
yield { type: "tool-result", payload: { toolCallId: "tool-1", toolName: "run-jyotish-consultation", result: {} } };
|
||||
yield { type: "text-delta", payload: { text: pinchedHeading } };
|
||||
yield { type: "finish", payload: { stepResult: { reason: "length" }, output: { usage: {}, steps: [{}, {}] } } };
|
||||
}
|
||||
const response = streamAgentResponse({
|
||||
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
|
||||
toolStatus: () => "ready", receipt: () => receipt(state),
|
||||
onComplete: () => { completed += 1; },
|
||||
onError: () => { failed += 1; },
|
||||
});
|
||||
const events: unknown[] = [];
|
||||
const parser = createNdjsonParser((event) => events.push(event));
|
||||
parser.finish(await response.text());
|
||||
|
||||
assert.equal(completed, 0);
|
||||
assert.equal(failed, 1);
|
||||
assert.equal(state.modelFinishReason, "length");
|
||||
assert.equal(events.filter((event) => (event as { type?: string }).type === "run.completed").length, 0);
|
||||
const answer = events
|
||||
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
|
||||
.map((event) => event.text)
|
||||
.join("");
|
||||
assert.equal(answer, pinchedHeading);
|
||||
const failure = events.find((event) => (event as { type?: string }).type === "run.failed") as {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
assert.equal(failure.code, "answer_truncated");
|
||||
assert.match(failure.message, /回答未完成/);
|
||||
assert.match(failure.message, /不会扣点/);
|
||||
assert.doesNotMatch(JSON.stringify(failure), /modelFinishReason|"length"/);
|
||||
});
|
||||
|
||||
test("a timeout after partial visible text is the same truncation, not a successful answer", async () => {
|
||||
const state = toolOnlyRunState();
|
||||
const pinchedHeading = "**先看命盘结构(Lahiri岁差、均交点口径";
|
||||
let completed = 0;
|
||||
async function* chunks() {
|
||||
yield { type: "tool-result", payload: { toolCallId: "tool-1", toolName: "run-jyotish-consultation", result: {} } };
|
||||
yield { type: "text-delta", payload: { text: pinchedHeading } };
|
||||
throw new DOMException("The operation was aborted due to timeout", "TimeoutError");
|
||||
}
|
||||
const response = streamAgentResponse({
|
||||
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
|
||||
toolStatus: () => "ready", receipt: () => receipt(state),
|
||||
onComplete: () => { completed += 1; },
|
||||
onError: () => {},
|
||||
});
|
||||
const events: unknown[] = [];
|
||||
const parser = createNdjsonParser((event) => events.push(event));
|
||||
parser.finish(await response.text());
|
||||
|
||||
assert.equal(completed, 0);
|
||||
const answer = events
|
||||
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
|
||||
.map((event) => event.text)
|
||||
.join("");
|
||||
assert.equal(answer, pinchedHeading);
|
||||
const failure = events.find((event) => (event as { type?: string }).type === "run.failed") as { code: string };
|
||||
assert.equal(failure.code, "answer_truncated");
|
||||
});
|
||||
|
||||
test("consult generation reserves visible output tokens and disables provider thinking", () => {
|
||||
const settings = consultationGenerationSettings("deepseek");
|
||||
assert.equal(CONSULTATION_MAX_OUTPUT_TOKENS, 8192);
|
||||
assert.equal(settings.modelSettings.maxOutputTokens, CONSULTATION_MAX_OUTPUT_TOKENS);
|
||||
assert.deepEqual(settings.providerOptions.openai, { thinking: { type: "disabled" } });
|
||||
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "disabled" } });
|
||||
assert.equal(AGENT_MAX_STEPS, 8);
|
||||
});
|
||||
|
||||
|
||||
test("a completed run records the finish reason and the authoritative step count", async () => {
|
||||
const state = createConsultationRuntimeState();
|
||||
|
||||
@@ -133,3 +133,14 @@ test("the first default consultation title is persisted with the user question",
|
||||
assert.match(sendSource, /const completedSession: ChatSession = \{[\s\S]*title: userSession\.title/);
|
||||
assert.doesNotMatch(sendSource, /resolveSessionTitle\(question, reply\.title\)/);
|
||||
});
|
||||
|
||||
test("a truncated generation keeps the partial answer and does not wait for a successful run", () => {
|
||||
const stream = sendSource.slice(sendSource.indexOf('fetch("/api/consult"'));
|
||||
assert.match(stream, /event\.code === "answer_truncated"/);
|
||||
assert.match(stream, /truncatedFailure = event/);
|
||||
assert.match(stream, /const truncatedSession: ChatSession = \{[\s\S]*role: "assistant"[\s\S]*text: reply\.text/);
|
||||
assert.match(stream, /await persistSession\(truncatedSession\)/);
|
||||
assert.match(stream, /setComposerNotice\(truncatedFailure\.message\)/);
|
||||
assert.match(stream, /if \(!runCompleted && !truncatedFailure\) throw new ConsultationResponseError/);
|
||||
assert.doesNotMatch(stream.slice(stream.indexOf("if (truncatedFailure)")), /runCompleted = true/);
|
||||
});
|
||||
|
||||
@@ -65,6 +65,15 @@ test("the model step budget and the wall-clock budget are declared as one pair",
|
||||
assert.doesNotMatch(route, /AbortSignal\.timeout\(\d/);
|
||||
});
|
||||
|
||||
test("consult streams cap visible output and disable thinking instead of sharing the token budget with hidden reasoning", () => {
|
||||
assert.match(tools, /export const CONSULTATION_MAX_OUTPUT_TOKENS = 8192;/);
|
||||
assert.match(tools, /function consultationGenerationSettings/);
|
||||
assert.match(tools, /thinking: \{ type: "disabled"/);
|
||||
assert.match(tools, /maxOutputTokens: CONSULTATION_MAX_OUTPUT_TOKENS/);
|
||||
assert.match(route, /\.\.\.consultationGenerationSettings\(selectedModel\.model\)/);
|
||||
assert.doesNotMatch(route, /maxOutputTokens:\s*\d/);
|
||||
});
|
||||
|
||||
test("uses one runtime step append entry and no scattered hard-coded step cap", () => {
|
||||
assert.match(tools, /export function appendConsultationRuntimeStep/);
|
||||
assert.doesNotMatch(tools, /steps\.length\s*>=\s*32/);
|
||||
|
||||
@@ -10,6 +10,10 @@ const chat = readFileSync(
|
||||
new URL("../src/components/rectification-agentic-chat.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const messageActions = readFileSync(
|
||||
new URL("../src/components/chat-message-actions.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const route = readFileSync(
|
||||
new URL("../src/app/api/rectification/agent/route.ts", import.meta.url),
|
||||
"utf8",
|
||||
@@ -274,8 +278,9 @@ test("rectification activity separates live work from the receipt above the Agen
|
||||
|
||||
test("completed Agent replies restore feedback, copy and safe in-place regeneration actions", () => {
|
||||
for (const label of ["赞", "踩", "复制回答", "重新生成回答"]) {
|
||||
assert.match(chat, new RegExp(`aria-label="${label}"`));
|
||||
assert.match(messageActions, new RegExp(`aria-label="${label}"`));
|
||||
}
|
||||
assert.match(chat, /<ChatMessageActions/);
|
||||
assert.match(chat, /toggleRectificationFeedback/);
|
||||
assert.match(chat, /navigator\.clipboard\.writeText\(message\.text\)/);
|
||||
assert.match(chat, /\/turns\/\$\{encodeURIComponent\(message\.turnId\)\}\/regenerate/);
|
||||
@@ -287,7 +292,7 @@ test("completed Agent replies restore feedback, copy and safe in-place regenerat
|
||||
);
|
||||
const receiptIndex = messageRender.indexOf("<CompletedActivityReceipt");
|
||||
const replyIndex = messageRender.indexOf("<ChatMessageRow");
|
||||
const actionsIndex = messageRender.indexOf('className="rectification-message-actions"');
|
||||
const actionsIndex = messageRender.indexOf("<ChatMessageActions");
|
||||
assert.ok(receiptIndex >= 0 && replyIndex >= 0 && actionsIndex >= 0);
|
||||
assert.ok(receiptIndex < replyIndex && replyIndex < actionsIndex);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user