fix(consult): stop the model spending its step budget on invalid tool params
A staging consultation calculated the chart and then returned nothing but the ensureFinalResponseText fallback. The model had made four calls to run-jyotish-consultation, and two of them never reached a calculation: they set both domains and theme, which canonicalDomainPlan rejects at execution. The schema declared those two fields as independent optionals, the description never mentioned the constraint, and the instructions actively told the model to use theme for a single-domain retry. Each attempt therefore bought a rule the contract never stated, and because the throw happens before the step-recording try/catch, it left no trace in the receipt either. Make the constraint unrepresentable instead of enforced. The model-facing schema keeps only question and domains, so Mastra refuses the pair before the tool body runs; the description states the single-array contract, and the instruction that advertised theme is gone. canonicalDomainPlan still resolves the single-value form for callers that build a plan without that schema, and is now exported so that path has its own tests. maxSteps and the abort timeout bound the same run but were hard-coded apart. One calculation takes about 20s against a 110s budget, so time is the binding constraint and three failed calculations exhaust it whatever the step count. The budget only has to cover the longest useful shape, so it moves to 8 beside the timeout with that reasoning recorded, and the recorded step list is sized to match so an exhausted run cannot truncate its own evidence. Step exhaustion was only ever inferable by counting events, since finishReason was recorded nowhere and progressive-disclosure reads never reach the public stream. Capture it as a closed enum plus a step count, normalizing anything unrecognized, and log both as controlled fields. Neither may enter the client receipt, whose step schema is strict and would fail a successful run. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -47,6 +47,37 @@ export const billingSettlementResults = [
|
||||
export type AgentBillingSettlementResult = (typeof billingSettlementResults)[number];
|
||||
export type AgentSettlementResult = Exclude<AgentBillingSettlementResult, "not_applicable">;
|
||||
|
||||
/**
|
||||
* Closed vocabulary for why the model stopped stepping. These are the provider
|
||||
* finish reasons plus the two Mastra adds; anything else normalizes to
|
||||
* `unknown` so an unrecognized provider string can never become a log field.
|
||||
*
|
||||
* `tool-calls` on a run that produced no answer means the step budget ran out
|
||||
* while the model still wanted to call a tool, which is otherwise only
|
||||
* inferable from the recorded step list.
|
||||
*/
|
||||
export const agentModelFinishReasons = [
|
||||
"stop",
|
||||
"length",
|
||||
"content-filter",
|
||||
"tool-calls",
|
||||
"error",
|
||||
"other",
|
||||
"tripwire",
|
||||
"retry",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
export type AgentModelFinishReason = (typeof agentModelFinishReasons)[number];
|
||||
|
||||
const knownModelFinishReasons = new Set<string>(agentModelFinishReasons);
|
||||
|
||||
export function toAgentModelFinishReason(value: unknown): AgentModelFinishReason {
|
||||
return typeof value === "string" && knownModelFinishReasons.has(value)
|
||||
? value as AgentModelFinishReason
|
||||
: "unknown";
|
||||
}
|
||||
|
||||
export type AgentSettlementTelemetryOutcome = Readonly<{
|
||||
billingSettlementResult: AgentSettlementResult;
|
||||
errorCode?: string;
|
||||
@@ -97,6 +128,10 @@ export const agentObservabilityEventSchema = z.object({
|
||||
contractPhases: z.array(agentObservabilityContractPhaseSchema).max(64).optional(),
|
||||
retryCount: z.number().int().min(0).max(100).optional(),
|
||||
errorCode: machineCodeSchema.optional(),
|
||||
// Why the model stopped, and how many model steps the run consumed across
|
||||
// every attempt. Both are enum-like machine values, never provider text.
|
||||
modelFinishReason: z.enum(agentModelFinishReasons).optional(),
|
||||
modelStepCount: countSchema.optional(),
|
||||
|
||||
inputTokens: tokenCountSchema.optional(),
|
||||
outputTokens: tokenCountSchema.optional(),
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
type AgentExecutionReceipt,
|
||||
type ConsultationAgentPublicEvent,
|
||||
} from "./consultation-agent-events.ts";
|
||||
import { toAgentModelFinishReason } from "./agent-observability.ts";
|
||||
import { createVisibleTextTransformer } from "./stream-text-response.ts";
|
||||
|
||||
type Chunk = { type?: string; payload?: Record<string, unknown>; data?: unknown };
|
||||
@@ -58,6 +59,25 @@ function activity(value: unknown): ConsultationAgentPublicEvent | null {
|
||||
return { type: "activity", phase: phase.data, label: data.label.slice(0, 120) };
|
||||
}
|
||||
|
||||
/**
|
||||
* The runtime only reveals how the model loop ended through the stream: one
|
||||
* `step-finish` per model step, then a terminal `finish` carrying the reason
|
||||
* the model stopped and the authoritative step list. Without this, a run that
|
||||
* exhausted its step budget is indistinguishable from one that chose to stop,
|
||||
* because progressive-disclosure reads never reach the public event stream.
|
||||
*/
|
||||
function finishTelemetry(chunk: Chunk) {
|
||||
const payload = chunk.payload as {
|
||||
stepResult?: { reason?: unknown };
|
||||
output?: { steps?: unknown };
|
||||
} | undefined;
|
||||
const steps = payload?.output?.steps;
|
||||
return {
|
||||
reason: toAgentModelFinishReason(payload?.stepResult?.reason),
|
||||
stepCount: Array.isArray(steps) ? steps.length : null,
|
||||
};
|
||||
}
|
||||
|
||||
function safeToolError(error: unknown) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") return "cancelled" as const;
|
||||
if (error instanceof DOMException && error.name === "TimeoutError") return "timeout" as const;
|
||||
@@ -193,8 +213,17 @@ export function streamAgentResponse(options: StreamAgentResponseOptions) {
|
||||
if (/\S/.test(held)) emitted = true;
|
||||
held = "";
|
||||
};
|
||||
// A retry runs a second model loop under the same step budget, so the run
|
||||
// total accumulates while the finish reason describes the latest attempt.
|
||||
const stepCountBeforeAttempt = options.state.modelStepCount;
|
||||
for await (const chunk of readChunks(stream)) {
|
||||
for (const event of mapChunk(chunk, options, startedAt, jyotishSkillCallIds)) send(controller, event);
|
||||
if (chunk.type === "step-finish") options.state.modelStepCount += 1;
|
||||
if (chunk.type === "finish") {
|
||||
const finish = finishTelemetry(chunk);
|
||||
options.state.modelFinishReason = finish.reason;
|
||||
if (finish.stepCount !== null) options.state.modelStepCount = stepCountBeforeAttempt + finish.stepCount;
|
||||
}
|
||||
if (chunk.type === "text-delta" && typeof chunk.payload?.text === "string") {
|
||||
await outputText(visible.push(chunk.payload.text));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user