fix(consult): record a tool call the tool never received
staging 手测 run 951a841e 第一次工具调用发出 tool.failed 后重试成功,但回执里 steps 只有 skill 与那次成功的 tool,stepBudget.used 为 2——失败的那次完全不存在。 客户端看见失败过一次,回执说没有,两边都查不到为什么。 工具的 inputSchema 是 strict 的,模型参数不合法时 Mastra 在调用 execute 之前就拒了, 于是工具体内一切都没跑:调用不计数、失败步不记录、连 chart-calculation 活动事件都没 发出(这也是本次定位的证据——失败那次没有任何 activity,重试那次有)。工具无法记录 一次它从未收到的调用。叠加两处:safeToolError 把非超时非取消的错误全塌成 calculation_failed,而即使失败落进工具体的 catch,consultationWorkflowFailureCode 对非 ConsultationWorkflowError 返回 undefined、append 处又写成可选省略,于是最需要 解释的那条记录恰好是唯一没有原因的记录。 改为在流层补记:流是唯一能观测到全部工具失败的位置,无论失败在 schema 这侧还是 execute 那侧,且它持有 startedAt 因而能给出时长。tool-error 分支比对「流已见的错误数」 与「state 里已有的失败 tool 步数」,只在前者更多时补一条,工具仍记录它能看见的失败, 两者不重复计。另新增 consultationToolFailureCode,令每个错误都解析出一个码。 failureCode 刻意仍不进公开回执:白名单与「the public receipt never carries the internal failure classification」是刻意约束,workflow_rate_limited 这类后端内情不该 上线到客户端。原因走可观测日志的 toolCalls[].failureCode。客户端能看到「有一步失败」, 运维能在日志里看到为什么。 另记入 BUG-268 的线上实测值:单领域 referenceReads 两次均为 2,多领域为 0——不是 从不读方法,而是最需要方法的多领域路径一份都没打开。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import {
|
||||
canonicalDomainPlan,
|
||||
consultationModelStepTelemetry,
|
||||
consultationStepBudgetReceipt,
|
||||
consultationToolFailureCode,
|
||||
createConsultationRuntimeHooks,
|
||||
createConsultationTools,
|
||||
createConsultationRuntimeState,
|
||||
@@ -611,6 +612,75 @@ test("workflow failure codes classify transport and contract faults", () => {
|
||||
assert.equal(consultationWorkflowFailureCode(new Error("anything else")), undefined);
|
||||
});
|
||||
|
||||
test("every tool failure resolves to a code, not an absent field", () => {
|
||||
// The workflow classifier returns undefined for anything it does not own, and
|
||||
// the append site omitted the field when it was undefined, so the failures
|
||||
// raised inside the tool reached the log as the one record with no reason.
|
||||
assert.equal(consultationToolFailureCode(new ConsultationWorkflowError("workflow_rate_limited", "x")), "workflow_rate_limited");
|
||||
assert.equal(consultationToolFailureCode(new DOMException("stop", "AbortError")), "workflow_aborted");
|
||||
assert.equal(consultationToolFailureCode(new Error("invalid_consultation_domain_plan")), "invalid_domain_plan");
|
||||
assert.equal(consultationToolFailureCode(new Error("unsupported_consultation_domain")), "invalid_domain_plan");
|
||||
assert.equal(consultationToolFailureCode(new Error("anything else")), "unexpected_error");
|
||||
assert.equal(consultationToolFailureCode("not an error"), "unexpected_error");
|
||||
});
|
||||
|
||||
test("a call rejected before the tool body runs still appears in the receipt", async () => {
|
||||
// Observed on staging: the model's arguments were refused against the tool's
|
||||
// strict input schema, so `execute` never ran. The client saw tool.failed
|
||||
// while the receipt showed no failed step and the budget counted no call.
|
||||
const state = createConsultationRuntimeState();
|
||||
state.jyotishSkillLoaded = true;
|
||||
appendConsultationRuntimeStep(state, { kind: "skill", name: "jyotish-vedic-astrology", status: "completed", durationMs: 370 });
|
||||
async function* chunks() {
|
||||
yield { type: "tool-call", payload: { toolCallId: "call-1", toolName: "run-jyotish-consultation" } };
|
||||
yield { type: "tool-error", payload: { toolCallId: "call-1", toolName: "run-jyotish-consultation", error: new Error("bad arguments") } };
|
||||
}
|
||||
const response = streamAgentResponse({
|
||||
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
|
||||
toolStatus: () => "blocked",
|
||||
receipt: () => ({ ...receipt(state), steps: publicConsultationRuntimeSteps(state) }),
|
||||
onError: () => {},
|
||||
});
|
||||
const events: unknown[] = [];
|
||||
const parser = createNdjsonParser((event) => events.push(event));
|
||||
parser.finish(await response.text());
|
||||
|
||||
const failedStep = state.steps.find((step) => step.kind === "tool" && step.status === "failed");
|
||||
assert.equal(failedStep?.name, "run-jyotish-consultation");
|
||||
assert.equal(failedStep?.failureCode, "tool_call_rejected");
|
||||
assert.equal(consultationStepBudgetReceipt(state).used, 2);
|
||||
// The client learns a step failed; the classification stays server-side.
|
||||
const failed = events.find((event) => (event as { type?: string }).type === "run.failed") as {
|
||||
receipt?: { steps: Array<{ status: string; name: string }> };
|
||||
};
|
||||
assert.deepEqual(failed.receipt?.steps.map((step) => step.status), ["completed", "failed"]);
|
||||
assert.doesNotMatch(JSON.stringify(failed), /tool_call_rejected/);
|
||||
});
|
||||
|
||||
test("a failure the tool already recorded is not recorded twice", async () => {
|
||||
// The tool records what it can see, with the duration and cause it alone
|
||||
// knows. The stream must only fill the gap, never double-count.
|
||||
const state = createConsultationRuntimeState();
|
||||
state.jyotishSkillLoaded = true;
|
||||
appendConsultationRuntimeStep(state, {
|
||||
kind: "tool", name: "run-jyotish-consultation", status: "failed", durationMs: 20936, failureCode: "workflow_queue_full",
|
||||
});
|
||||
async function* chunks() {
|
||||
yield { type: "tool-call", payload: { toolCallId: "call-1", toolName: "run-jyotish-consultation" } };
|
||||
yield { type: "tool-error", payload: { toolCallId: "call-1", toolName: "run-jyotish-consultation", error: new Error("boom") } };
|
||||
}
|
||||
const response = streamAgentResponse({
|
||||
runId: "run", requestId: "req", state, stream: chunks(), requireTool: true,
|
||||
toolStatus: () => "blocked",
|
||||
receipt: () => ({ ...receipt(state), steps: publicConsultationRuntimeSteps(state) }),
|
||||
onError: () => {},
|
||||
});
|
||||
await response.text();
|
||||
|
||||
assert.equal(state.steps.filter((step) => step.status === "failed").length, 1);
|
||||
assert.equal(state.steps[0]?.failureCode, "workflow_queue_full");
|
||||
});
|
||||
|
||||
test("the public receipt never carries the internal failure classification", () => {
|
||||
const state = createConsultationRuntimeState();
|
||||
appendConsultationRuntimeStep(state, {
|
||||
|
||||
Reference in New Issue
Block a user