fix(consult): treat pinched answers as failed and restore reply actions
Incomplete Flash generations were billed as completed consultations. Fail those runs, keep the partial text, and reuse the rectification like/copy/rerun bar on ordinary chat replies. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,12 +1,15 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
AGENT_MAX_STEPS,
|
||||
AGENT_TIMEOUT_MS,
|
||||
CONSULTATION_MAX_OUTPUT_TOKENS,
|
||||
mergeConsultationAnswerPolicies,
|
||||
CONSULTATION_DOMAIN_WALL_CLOCK_MS,
|
||||
MAX_CONSULTATION_DOMAINS,
|
||||
appendConsultationRuntimeStep,
|
||||
canonicalDomainPlan,
|
||||
consultationGenerationSettings,
|
||||
consultationModelStepTelemetry,
|
||||
consultationStepBudgetReceipt,
|
||||
consultationToolFailureCode,
|
||||
@@ -1175,6 +1178,87 @@ test("a calculation still unanswered after the retry fails the run rather than b
|
||||
assert.match(failure.message, /不会扣点/);
|
||||
});
|
||||
|
||||
test("a length-limited answer is not billed or delivered as a completed consultation", async () => {
|
||||
// Staging persisted a 253-character pinch that ended mid-heading, then treated
|
||||
// the run as completed. The model had finished with reason `length`; the public
|
||||
// stream still emitted `run.completed`, so the composer unlocked as if the
|
||||
// reading were done.
|
||||
const state = toolOnlyRunState();
|
||||
const pinchedHeading = "**先看命盘结构(Lahiri岁差、均交点口径";
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
async function* chunks() {
|
||||
yield { type: "tool-result", payload: { toolCallId: "tool-1", toolName: "run-jyotish-consultation", result: {} } };
|
||||
yield { type: "text-delta", payload: { text: pinchedHeading } };
|
||||
yield { type: "finish", payload: { stepResult: { reason: "length" }, output: { usage: {}, steps: [{}, {}] } } };
|
||||
}
|
||||
const response = streamAgentResponse({
|
||||
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
|
||||
toolStatus: () => "ready", receipt: () => receipt(state),
|
||||
onComplete: () => { completed += 1; },
|
||||
onError: () => { failed += 1; },
|
||||
});
|
||||
const events: unknown[] = [];
|
||||
const parser = createNdjsonParser((event) => events.push(event));
|
||||
parser.finish(await response.text());
|
||||
|
||||
assert.equal(completed, 0);
|
||||
assert.equal(failed, 1);
|
||||
assert.equal(state.modelFinishReason, "length");
|
||||
assert.equal(events.filter((event) => (event as { type?: string }).type === "run.completed").length, 0);
|
||||
const answer = events
|
||||
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
|
||||
.map((event) => event.text)
|
||||
.join("");
|
||||
assert.equal(answer, pinchedHeading);
|
||||
const failure = events.find((event) => (event as { type?: string }).type === "run.failed") as {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
assert.equal(failure.code, "answer_truncated");
|
||||
assert.match(failure.message, /回答未完成/);
|
||||
assert.match(failure.message, /不会扣点/);
|
||||
assert.doesNotMatch(JSON.stringify(failure), /modelFinishReason|"length"/);
|
||||
});
|
||||
|
||||
test("a timeout after partial visible text is the same truncation, not a successful answer", async () => {
|
||||
const state = toolOnlyRunState();
|
||||
const pinchedHeading = "**先看命盘结构(Lahiri岁差、均交点口径";
|
||||
let completed = 0;
|
||||
async function* chunks() {
|
||||
yield { type: "tool-result", payload: { toolCallId: "tool-1", toolName: "run-jyotish-consultation", result: {} } };
|
||||
yield { type: "text-delta", payload: { text: pinchedHeading } };
|
||||
throw new DOMException("The operation was aborted due to timeout", "TimeoutError");
|
||||
}
|
||||
const response = streamAgentResponse({
|
||||
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
|
||||
toolStatus: () => "ready", receipt: () => receipt(state),
|
||||
onComplete: () => { completed += 1; },
|
||||
onError: () => {},
|
||||
});
|
||||
const events: unknown[] = [];
|
||||
const parser = createNdjsonParser((event) => events.push(event));
|
||||
parser.finish(await response.text());
|
||||
|
||||
assert.equal(completed, 0);
|
||||
const answer = events
|
||||
.filter((event): event is { type: string; text: string } => (event as { type?: string }).type === "answer.delta")
|
||||
.map((event) => event.text)
|
||||
.join("");
|
||||
assert.equal(answer, pinchedHeading);
|
||||
const failure = events.find((event) => (event as { type?: string }).type === "run.failed") as { code: string };
|
||||
assert.equal(failure.code, "answer_truncated");
|
||||
});
|
||||
|
||||
test("consult generation reserves visible output tokens and disables provider thinking", () => {
|
||||
const settings = consultationGenerationSettings("deepseek");
|
||||
assert.equal(CONSULTATION_MAX_OUTPUT_TOKENS, 8192);
|
||||
assert.equal(settings.modelSettings.maxOutputTokens, CONSULTATION_MAX_OUTPUT_TOKENS);
|
||||
assert.deepEqual(settings.providerOptions.openai, { thinking: { type: "disabled" } });
|
||||
assert.deepEqual(settings.providerOptions.deepseek, { thinking: { type: "disabled" } });
|
||||
assert.equal(AGENT_MAX_STEPS, 8);
|
||||
});
|
||||
|
||||
|
||||
test("a completed run records the finish reason and the authoritative step count", async () => {
|
||||
const state = createConsultationRuntimeState();
|
||||
|
||||
Reference in New Issue
Block a user