Files
Jyotisha/frontend/tests/consultation-workflow-contract.test.ts
T
Jesse_ChenandCursor c8d9ec64c3
Independent Staging Quality Gate / validate (push) Successful in 13m24s
Independent Staging Quality Gate / publish (push) Successful in 10m13s
fix(consult): stop the model spending its step budget on invalid tool params
A staging consultation calculated the chart and then returned nothing but the
ensureFinalResponseText fallback. The model had made four calls to
run-jyotish-consultation, and two of them never reached a calculation: they set
both domains and theme, which canonicalDomainPlan rejects at execution. The
schema declared those two fields as independent optionals, the description never
mentioned the constraint, and the instructions actively told the model to use
theme for a single-domain retry. Each attempt therefore bought a rule the
contract never stated, and because the throw happens before the step-recording
try/catch, it left no trace in the receipt either.

Make the constraint unrepresentable instead of enforced. The model-facing schema
keeps only question and domains, so Mastra refuses the pair before the tool body
runs; the description states the single-array contract, and the instruction that
advertised theme is gone. canonicalDomainPlan still resolves the single-value
form for callers that build a plan without that schema, and is now exported so
that path has its own tests.

maxSteps and the abort timeout bound the same run but were hard-coded apart. One
calculation takes about 20s against a 110s budget, so time is the binding
constraint and three failed calculations exhaust it whatever the step count. The
budget only has to cover the longest useful shape, so it moves to 8 beside the
timeout with that reasoning recorded, and the recorded step list is sized to
match so an exhausted run cannot truncate its own evidence.

Step exhaustion was only ever inferable by counting events, since finishReason
was recorded nowhere and progressive-disclosure reads never reach the public
stream. Capture it as a closed enum plus a step count, normalizing anything
unrecognized, and log both as controlled fields. Neither may enter the client
receipt, whose step schema is strict and would fail a successful run.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-17 15:21:11 +08:00

138 lines
8.1 KiB
TypeScript

import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8");
const reportsRoute = readFileSync(new URL("../src/app/api/reports/route.ts", import.meta.url), "utf8");
const mastra = readFileSync(new URL("../src/mastra/index.ts", import.meta.url), "utf8");
const tools = readFileSync(new URL("../src/mastra/consultation-tools.ts", import.meta.url), "utf8");
const stream = readFileSync(new URL("../src/lib/stream-agent-response.ts", import.meta.url), "utf8");
const workflow = readFileSync(new URL("../src/mastra/consultation-workflow.ts", import.meta.url), "utf8");
const plan = readFileSync(new URL("../src/lib/consultation-plan.ts", import.meta.url), "utf8");
const routeService = readFileSync(new URL("../src/lib/consultation-route-service.ts", import.meta.url), "utf8");
const pythonPlanContract = readFileSync(new URL("../../scripts/consultation_plan_contract.py", import.meta.url), "utf8");
const stagingCompose = readFileSync(new URL("../../deploy/docker-compose.staging.yml", import.meta.url), "utf8");
test("consultation plans are server-owned and bounded", () => {
assert.match(plan, /consultationPlanSchema/);
assert.match(plan, /requestedDomains/);
assert.match(plan, /requiredEvidenceCategories/);
assert.match(plan, /createConsultationPlan/);
assert.match(tools, /const userIntent = ctx\.plan\?\.userIntent \?\? input\.question/);
assert.match(tools, /input\.domains === undefined && context\.plan && context\.theme/);
assert.match(tools, /return \[context\.theme\]/);
assert.match(tools, /const domainPlan = ctx\.plan[\s\S]*createConsultationPlan\(\{/);
assert.match(tools, /userIntent,[\s\S]*theme: domain/);
assert.match(tools, /question: userIntent,[\s\S]*theme: domain/);
assert.match(tools, /plan: domainPlan/);
assert.match(workflow, /packet_version: "consultation-evidence-packet-v2"/);
assert.match(workflow, /claim_cards/);
assert.match(tools, /modelOutput: toModelOutput\(agentContext, domainPlan\)/);
assert.match(tools, /modelOutput: toModelOutput[\s\S]*ctx\.state\.consultationToolCompleted = true/);
assert.match(plan, /precisionBoundary/);
assert.match(plan, /consultationPlanMaxCreditCost = 1/);
const modelToolInput = tools.slice(tools.indexOf("const consultationToolInputSchema"), tools.indexOf("const MAX_RECORDED_STEPS"));
assert.doesNotMatch(modelToolInput, /birth|latitude|longitude|engine|script|credit/i);
const prepareCall = route.slice(route.indexOf("prepared = await prepareConsultationRoute"), route.indexOf("const modelSelection"));
assert.match(prepareCall, /beforeReserve: \(\{ consultationMode \}\) => createConsultationPlan\(\{/);
assert.ok(prepareCall.indexOf("beforeReserve:") < prepareCall.indexOf("reserve: () => reserveConsultationModel("));
assert.ok(routeService.indexOf("const preReserveResult = input.beforeReserve") < routeService.indexOf("const reservation = await input.reserve()"));
assert.match(route, /plan: prepared\.preReserveResult,[\s\S]*theme: consultationTheme/);
assert.match(route, /runConsultationWorkflow\(toolInput, \{[\s\S]*plan: prepared\.preReserveResult/);
assert.match(workflow, /plan_version: workflowRequest\.plan_version/);
assert.match(workflow, /strict_workflow_route: workflowRequest\.strictWorkflowRoute/);
assert.match(workflow, /required_layers: workflowRequest\.requiredLayers/);
assert.match(workflow, /claim_boundary: workflowRequest\.claimBoundary/);
assert.match(workflow, /precision_boundary: workflowRequest\.precision_boundary/);
assert.match(workflow, /required_evidence_categories: workflowRequest\.required_evidence_categories/);
assert.match(pythonPlanContract, /PLAN_VERSION = "consultation-plan-v2"/);
assert.match(pythonPlanContract, /server_allowlist_validated/);
assert.match(pythonPlanContract, /can_answer_precise_timing": False/);
});
test("the model step budget and the wall-clock budget are declared as one pair", () => {
assert.match(route, /const AGENT_MAX_STEPS = 8;\nconst AGENT_TIMEOUT_MS = 110_000;/);
assert.match(route, /maxSteps: AGENT_MAX_STEPS,/);
assert.match(route, /AbortSignal\.timeout\(AGENT_TIMEOUT_MS\)/);
assert.match(route, /createConsultationRuntimeState\(\{ plannedSteps: AGENT_MAX_STEPS \}\)/);
assert.doesNotMatch(route, /maxSteps: \d/);
assert.doesNotMatch(route, /AbortSignal\.timeout\(\d/);
});
test("uses one runtime step append entry and no scattered hard-coded step cap", () => {
assert.match(tools, /export function appendConsultationRuntimeStep/);
assert.doesNotMatch(tools, /steps\.length\s*>=\s*32/);
assert.doesNotMatch(stream, /state\.steps\.push/);
assert.doesNotMatch(stream, /steps\.length\s*<\s*32/);
});
test("personal consultation lets the Agent invoke the server-bound workflow tool", () => {
const agenticStart = route.indexOf("async function runAgenticConsultation");
const agenticBranch = route.slice(
agenticStart,
route.indexOf(" const { history } = parsed.data;", agenticStart),
);
assert.match(agenticBranch, /createConsultationAgentContext/);
assert.match(agenticBranch, /getJyotishAgent\(selectedModel, agentContext\)/);
assert.doesNotMatch(agenticBranch, /await runConsultationWorkflow/);
assert.doesNotMatch(agenticBranch, /JSON\.stringify\(toolInput\)/);
assert.match(tools, /\(ctx\.runWorkflow \?\? runConsultationWorkflow\)\(toolInput, \{/);
assert.match(tools, /return \{ "run-jyotish-consultation": consultationTool \};/);
assert.match(agenticBranch, /state\.workflowReceipt\?\.preciseTiming === "allowed"/);
assert.match(route, /createConsultationReplyMetadata/);
});
test("defers optional external evidence only for foreground chat", () => {
assert.match(workflow, /defer_optional_external_evidence: options\?\.foreground === true/);
assert.match(reportsRoute, /runWorkflow: \(input\) => runConsultationWorkflow\(input\)/);
assert.doesNotMatch(reportsRoute, /foreground:\s*true/);
});
test("personal Agent owns the Skill and context-bound calculation tool", () => {
const personalFactory = mastra.slice(
mastra.indexOf("export function getJyotishAgent"),
mastra.indexOf("export function getLegacyJyotishAgent"),
);
assert.match(personalFactory, /getJyotishAgent\(model: ResolvedLanguageModel, context: ConsultationAgentContext\)/);
assert.match(personalFactory, /skills: \[jyotishSkillPath\]/);
assert.match(personalFactory, /tools: createConsultationTools\(context\)/);
assert.doesNotMatch(personalFactory, /server-computed-jyotish-workflow/);
});
test("validates and emits non-sensitive workflow and execution receipts", () => {
assert.match(workflow, /consultationWorkflowResponseSchema/);
assert.match(workflow, /safeParse\(data\)/);
assert.match(workflow, /consultationWorkflowReceipt/);
assert.match(route, /agentExecutionReceipt/);
assert.match(route, /streamAgentResponse/);
});
test("carries commercial technique truth into the model contract", () => {
assert.match(workflow, /technique_truth/);
assert.match(mastra, /deterministic_claims_forbidden_for/);
assert.doesNotMatch(mastra, /AYANAM_SUGGESTIONS|AYANAM_TITLE/);
assert.doesNotMatch(mastra, /2-5 short paragraphs/);
assert.match(mastra, /reference_only/);
assert.match(mastra, /Do not use a restricted technique/);
});
test("projects consultation themes through explicit strict workflow taxonomy", () => {
const projection = readFileSync(new URL("../src/lib/consultation-workflow-request.ts", import.meta.url), "utf8");
const registry = readFileSync(new URL("../src/lib/consultation-domain-registry.ts", import.meta.url), "utf8");
assert.match(projection, /consultationDomainDefinition/);
assert.match(registry, /strictWorkflowRoute/);
assert.match(registry, /claimBoundary/);
assert.match(registry, /requiredLayers/);
assert.match(registry, /negative holdout gate/);
});
test("agentic consultation is the safe default and legacy is explicit rollback", () => {
assert.match(stagingCompose, /CONSULTATION_AGENTIC_RUNTIME: enabled/);
assert.match(route, /CONSULTATION_AGENTIC_RUNTIME\?\.trim\(\)\.toLowerCase\(\) \?\? "enabled"/);
assert.match(route, /mode === "legacy"/);
assert.match(route, /mode !== "canary"/);
});