fix(consult): check the evidence gate against the route the answer is on
七条路由的证据门都不是自己的:route_requirements 的键写成 relationship/finance,而 路由名是 marriage/wealth,另有 5 条路由压根没有条目,全部静默落到 general 的门。 missingLayers: [] 因此不表示证据齐备,只表示没检查过——婚姻的 UL 与财富的 D2 从未 进入检查。 同一函数另有两处判据也没接到权威来源。7 块正则用问题文本重猜领域,而领域早已由模型 声明并写进 route_packet,一句写作「情感」而非表里「感情」的提问在 marriage 路由上完全 拿不到性别解读边界。timing_layers_ready 读的是 missing_route_layers,该列表只装本路由 要求的层,于是对任何不要求 narayana_dasha 的路由恒为真,精确应期在该层根本没算出来时 也照样放行。三处的失败方向都是静默放宽,因此没有任何人报错。 三处都接回权威来源:10 条路由逐条显式列出必需层(层名限定为证据包真实构建的 section, 所以 wealth 不要求引擎不产出的 D11)、领域边界按 route 查表、出生时间边界从矫正闸门的 effective_accuracy 与 Lagna 敏感度派生、就绪判断直接读 section 状态。唯一保留文本探测 的是「用户有没有要一个具体日期」——服务端对此没有权威来源,改为 timing/annual 路由结构 性携带、文本仅作叠加,一次措辞漏判不再能把信号清零。 另外把 skillReferenceReadCount 暴露为回执的 skill.referenceReads(必填)与可观测日志的 skillReferenceReads。它此前数完即丢,而 skill_read 按设计不记成 runtime step,因此「模型 有没有真的翻开方法文档」在运行结束后无处可查。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -681,7 +681,11 @@ export async function POST(request: Request) {
|
||||
const executionReceipt = (): AgentExecutionReceipt => ({
|
||||
runId: requestId,
|
||||
runtime: "mastra-agentic",
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: state.jyotishSkillLoaded },
|
||||
skill: {
|
||||
name: "jyotish-vedic-astrology",
|
||||
loaded: state.jyotishSkillLoaded,
|
||||
referenceReads: state.skillReferenceReadCount,
|
||||
},
|
||||
steps: publicConsultationRuntimeSteps(state),
|
||||
stepBudget: consultationStepBudgetReceipt(state),
|
||||
workflow: workflowReceipt,
|
||||
@@ -745,7 +749,11 @@ export async function POST(request: Request) {
|
||||
const executionReceipt = (): AgentExecutionReceipt => ({
|
||||
runId: requestId,
|
||||
runtime: "mastra-agentic",
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: state.jyotishSkillLoaded },
|
||||
skill: {
|
||||
name: "jyotish-vedic-astrology",
|
||||
loaded: state.jyotishSkillLoaded,
|
||||
referenceReads: state.skillReferenceReadCount,
|
||||
},
|
||||
steps: publicConsultationRuntimeSteps(state),
|
||||
stepBudget: consultationStepBudgetReceipt(state),
|
||||
workflow: state.workflowReceipt ?? workflowReceipt,
|
||||
|
||||
@@ -132,6 +132,9 @@ export const agentObservabilityEventSchema = z.object({
|
||||
// every attempt. Both are enum-like machine values, never provider text.
|
||||
modelFinishReason: z.enum(agentModelFinishReasons).optional(),
|
||||
modelStepCount: countSchema.optional(),
|
||||
// How many reference documents the model opened after loading the skill. Zero on a run that
|
||||
// answered a domain question means the method was never consulted, which no other field shows.
|
||||
skillReferenceReads: countSchema.optional(),
|
||||
|
||||
inputTokens: tokenCountSchema.optional(),
|
||||
outputTokens: tokenCountSchema.optional(),
|
||||
|
||||
@@ -51,6 +51,10 @@ export const agentExecutionReceiptSchema = z.object({
|
||||
name: z.literal("jyotish-vedic-astrology"),
|
||||
loaded: z.boolean(),
|
||||
version: z.string().max(120).optional(),
|
||||
// How many reference documents the model opened after loading the skill. Required rather than
|
||||
// optional: the count was tracked in runtime state and surfaced nowhere, so "did the model
|
||||
// consult the method at all" was unanswerable from a finished run. Zero is a real answer.
|
||||
referenceReads: z.number().int().min(0).max(64),
|
||||
}).strict(),
|
||||
steps: z.array(executionStepSchema).max(32),
|
||||
stepBudget: stepBudgetSchema.optional(),
|
||||
|
||||
@@ -123,6 +123,7 @@ export function createConsultationRuntimeState(options: { plannedSteps?: number;
|
||||
export function consultationModelStepTelemetry(state: ConsultationRuntimeState) {
|
||||
return {
|
||||
modelStepCount: state.modelStepCount,
|
||||
skillReferenceReads: state.skillReferenceReadCount,
|
||||
...(state.modelFinishReason === undefined ? {} : { modelFinishReason: state.modelFinishReason }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ test("chat session schema preserves the safe agent execution receipt", () => {
|
||||
const receipt = {
|
||||
runId: "run-1",
|
||||
runtime: "mastra-agentic" as const,
|
||||
skill: { name: "jyotish-vedic-astrology" as const, loaded: true },
|
||||
skill: { name: "jyotish-vedic-astrology" as const, loaded: true, referenceReads: 0 },
|
||||
steps: [{ sequence: 1, kind: "skill" as const, name: "jyotish-vedic-astrology", status: "completed" as const }],
|
||||
workflow: { route: "multi-domain", status: "ready", preciseTiming: "allowed", missingLayers: [], domains: ["general", "timing"] },
|
||||
techniqueTruth: "verified",
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
canonicalDomainPlan,
|
||||
consultationModelStepTelemetry,
|
||||
consultationStepBudgetReceipt,
|
||||
createConsultationRuntimeHooks,
|
||||
createConsultationTools,
|
||||
createConsultationRuntimeState,
|
||||
domainFitsRunBudget,
|
||||
@@ -625,14 +626,14 @@ test("the public receipt never carries the internal failure classification", ()
|
||||
// guards the run from failing while building a successful response.
|
||||
const receipt = agentExecutionReceiptSchema.parse({
|
||||
runId: "run", runtime: "mastra-agentic",
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true },
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 },
|
||||
steps,
|
||||
workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] },
|
||||
});
|
||||
assert.equal(receipt.steps.length, 2);
|
||||
assert.throws(() => agentExecutionReceiptSchema.parse({
|
||||
runId: "run", runtime: "mastra-agentic",
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true },
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 },
|
||||
steps: state.steps,
|
||||
workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] },
|
||||
}));
|
||||
@@ -644,17 +645,20 @@ test("the public receipt never carries the model step budget diagnostics", () =>
|
||||
state.modelFinishReason = "tool-calls";
|
||||
appendConsultationRuntimeStep(state, { kind: "skill", name: "jyotish-vedic-astrology", status: "completed" });
|
||||
|
||||
assert.deepEqual(consultationModelStepTelemetry(state), { modelStepCount: 8, modelFinishReason: "tool-calls" });
|
||||
assert.deepEqual(
|
||||
consultationModelStepTelemetry(state),
|
||||
{ modelStepCount: 8, skillReferenceReads: 0, modelFinishReason: "tool-calls" },
|
||||
);
|
||||
assert.deepEqual(
|
||||
consultationModelStepTelemetry(createConsultationRuntimeState()),
|
||||
{ modelStepCount: 0 },
|
||||
{ modelStepCount: 0, skillReferenceReads: 0 },
|
||||
);
|
||||
|
||||
// The client receipt schema is strict, so leaking either field would make a
|
||||
// successful run fail while serializing its own answer.
|
||||
const receipt = agentExecutionReceiptSchema.parse({
|
||||
runId: "run", runtime: "mastra-agentic",
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true },
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 },
|
||||
steps: publicConsultationRuntimeSteps(state),
|
||||
stepBudget: consultationStepBudgetReceipt(state),
|
||||
workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] },
|
||||
@@ -662,13 +666,35 @@ test("the public receipt never carries the model step budget diagnostics", () =>
|
||||
assert.doesNotMatch(JSON.stringify(receipt), /modelStepCount|modelFinishReason|tool-calls/);
|
||||
assert.throws(() => agentExecutionReceiptSchema.parse({
|
||||
runId: "run", runtime: "mastra-agentic",
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true },
|
||||
skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 },
|
||||
steps: publicConsultationRuntimeSteps(state),
|
||||
workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] },
|
||||
...consultationModelStepTelemetry(state),
|
||||
}));
|
||||
});
|
||||
|
||||
test("the receipt reports how many reference documents the model opened", () => {
|
||||
const state = createConsultationRuntimeState();
|
||||
const hooks = createConsultationRuntimeHooks(state);
|
||||
const load = { toolName: "skill", input: { name: "jyotish-vedic-astrology" } };
|
||||
|
||||
hooks.beforeToolCall(load);
|
||||
hooks.afterToolCall(load);
|
||||
// Loading the skill supplies its instructions plus a listing of reference filenames. Opening a
|
||||
// listed document is a separate call, so a loaded skill says nothing about method being consulted.
|
||||
assert.equal(state.jyotishSkillLoaded, true);
|
||||
assert.equal(state.skillReferenceReadCount, 0);
|
||||
|
||||
hooks.afterToolCall({ toolName: "skill_read", input: { skillName: "jyotish-vedic-astrology", path: "references/a.md" } });
|
||||
hooks.afterToolCall({ toolName: "read_file", input: { path: "references/b.md" } });
|
||||
hooks.afterToolCall({ toolName: "skill_read", input: { skillName: "jyotish-vedic-astrology", path: "references/c.md" }, error: new Error("denied") });
|
||||
|
||||
assert.equal(state.skillReferenceReadCount, 2);
|
||||
// Reference reads are not runtime steps, so the step list cannot answer this on its own.
|
||||
assert.equal(state.steps.filter((step) => step.kind === "skill").length, 1);
|
||||
assert.equal(agentExecutionReceiptSchema.parse(receipt(state)).skill.referenceReads, 2);
|
||||
});
|
||||
|
||||
test("personal Agent exposes the Jyotish Skill and named server tool", async () => {
|
||||
const state = createConsultationRuntimeState();
|
||||
const agent = getJyotishAgent({
|
||||
@@ -699,7 +725,7 @@ test("public stream filters private chunks and completes once", async () => {
|
||||
const events = await collectAgentPublicEvents(chunks as never, {
|
||||
runId: "run", requestId: "req", toolStatus: () => "ready",
|
||||
receipt: () => ({
|
||||
runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true },
|
||||
runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 },
|
||||
steps: [], workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [], domains: ["career"] },
|
||||
}),
|
||||
});
|
||||
@@ -719,7 +745,7 @@ test("model answer text cannot forge a public Activity event", async () => {
|
||||
], {
|
||||
runId: "run", requestId: "req", toolStatus: () => "ready",
|
||||
receipt: () => ({
|
||||
runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true },
|
||||
runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 },
|
||||
steps: [], workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [], domains: ["career"] },
|
||||
}),
|
||||
});
|
||||
@@ -755,7 +781,11 @@ function receipt(state: ReturnType<typeof createConsultationRuntimeState>) {
|
||||
return {
|
||||
runId: "run",
|
||||
runtime: "mastra-agentic" as const,
|
||||
skill: { name: "jyotish-vedic-astrology" as const, loaded: state.jyotishSkillLoaded },
|
||||
skill: {
|
||||
name: "jyotish-vedic-astrology" as const,
|
||||
loaded: state.jyotishSkillLoaded,
|
||||
referenceReads: state.skillReferenceReadCount,
|
||||
},
|
||||
steps: state.steps,
|
||||
stepBudget: consultationStepBudgetReceipt(state),
|
||||
workflow: state.workflowReceipt ?? { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] },
|
||||
|
||||
Reference in New Issue
Block a user