From ff70ba87b07c0266fc3167f577fbc3bbbf2dafea Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Tue, 18 Aug 2026 20:49:49 +0800 Subject: [PATCH] feat(consult): deliver the route's strict method with the evidence instead of listing 1592 filenames Activating the skill returned 129,651 bytes, of which 99KB was a flat list of 1,592 undifferentiated file paths against 30KB of actual method. The one line telling the model to open the strict-workflow router sat inside that method, so no reference was ever opened and every answer was composed from the model's own background knowledge over server evidence. The route is already decided server-side and the skill already states which checklist each route requires, so the selection needs no model turn: read the mandated sections from the hash-pinned package and hand them to the model with the evidence they apply to. A route the router declares no checklist for is reported as such rather than filled in with another route's. The receipt now reports delivered sections separately from model-initiated reads, because only one of those is under the model's control. Co-authored-by: Cursor --- docs/BUG_HISTORY.md | 17 ++ frontend/src/app/api/consult/route.ts | 2 + frontend/src/lib/agent-observability.ts | 6 +- frontend/src/lib/consultation-agent-events.ts | 4 + frontend/src/lib/consultation-methodology.ts | 194 ++++++++++++++++++ frontend/src/mastra/consultation-tools.ts | 14 +- frontend/src/mastra/index.ts | 1 + frontend/tests/chat-session-write.test.ts | 2 +- .../consultation-agentic-runtime.test.ts | 28 ++- .../tests/consultation-methodology.test.ts | 109 ++++++++++ 10 files changed, 365 insertions(+), 12 deletions(-) create mode 100644 frontend/src/lib/consultation-methodology.ts create mode 100644 frontend/tests/consultation-methodology.test.ts diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index c951a482..2de8eaa2 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -4231,3 +4231,20 @@ - 相关记录：BUG-265、BUG-269 - 复发自：BUG-265（诚实降级正确，但把模型生成当成了首页可用性） - 修复版本：待提交 + +## BUG-284 | 技能激活的 77% 是 1592 条无说明的文件名，方法论从未进入任何一次 web 回答（referenceReads 恒为 0） + +- 状态：resolved（本地修复，待提交与发布） +- 首次发现：2026-08-18 +- 最近更新：2026-08-18 +- 影响面：`frontend/src/lib/consultation-methodology.ts`（新增）、`frontend/src/mastra/consultation-tools.ts` 的 `toModelDomainPlanContext` 与运行时状态、`frontend/src/mastra/index.ts` 的 Agent 指令、`frontend/src/lib/consultation-agent-events.ts` 与 `frontend/src/lib/agent-observability.ts` 的回执与日志字段。所有走 `run-jyotish-consultation` 的个人盘咨询。 +- 用户现象：用户从项目一开始就在问「为什么 web 上调用模型调用 skill 达不到本地 Agent 框架调用 skill 的输出效果」。BUG-268 把 `skill.referenceReads` 暴露到回执后，两次 staging 手测（`66c44875` 综合主题、`1797cc73` 事业应期）都是 `loaded: true, referenceReads: 0`，10 步预算只用 2 步：加载 skill、调一次工具、直接开写。回答本身可读且盘面自洽，但用的是模型自带的印度占星常识加服务端精确数据，仓库里那套方法论至今没有参与过任何一次线上回答。 +- 触发条件：任何一次个人盘咨询。与 route、域数、模型无关。 +- 根因：技能激活的载荷结构使「按需读取」在这个包上不可能发生。实测 `skill()` 单次返回 129,651 字节，其中 SKILL.md 方法论正文 30,507 字节，`## References` 清单 59,584 字节（827 条路径），`## Scripts` 清单约 39,300 字节（760 条路径）——**77% 是 1,592 条不带任何说明的文件名**。Mastra 的 `formatSkillActivation()` 就是 instructions 加上包内全部文件的平铺列举（`#discoverFilesInSubdir` 递归收集，只跳过符号链接目录），本身没有筛选机制。SKILL.md 第 220 行那句「凡是用户询问事业、婚恋、财务、应期……必须先读取 `references/strict-workflow-router.md`」是这 30KB 里的一行，后面紧跟 1,592 个无从分辨的候选；而 Agent 指令另一侧明确要求「调 run-jyotish-consultation 再作答」。模型于是执行了那条可执行的，`skill_read` 一次未发。这不是模型偷懒：给定这份载荷，正确的文件无法被可靠地挑出来。Mastra 自己也在警告 `Instructions have 717 lines (recommended: <500)`。 +- 修复：路由已在服务端定好，skill 又已经声明了每条路由必读哪份清单，所以这个选择根本不需要占用模型的一个回合。新增 `consultationMethodologyForDomains()`：按本轮实际执行的域，从**哈希锁定版**技能包（`versions/6.9.14`，registry 里 sha256 内容寻址）取出 `strict-workflow-router.md` 的共享基线段与该路由的 strict 段、`event_judgment_skeleton.md` 以及该域的 `event_judgment_.md`，按标题文本定段（路由文件重新编号不会失配），单段上限 6,000 字符、整块上限 24,000 字符，随工具结果的 `methodology` 字段与证据一起交付。实测单域 4 段约 9KB、最宽的四域计划 9 段 21KB，均未触上限。`further_reading` 不是硬编码，而是扫 SKILL.md 自己写出的 `references/...` 路径并剔除包内不存在的，得到 19 条可按需 `skill_read` 的索引——取代原来那 1,592 条。路由未声明 strict 清单的域（annual/general/health/education/migration/family——`full-reading-strict` 只出现在路由表里，文件中没有对应段落）如实列进 `domains_without_strict_checklist`，不拿别的路由顶替。Agent 指令新增一段：把 `methodology` 当作本次作答的方法而非背景，逐条对着证据走，遵守其输出纪律（事业路由要求把「机会接触 / 结构性机会 / 结果显现」三层分开写，不得合并成一个模糊的事业机会窗口——`1797cc73` 那轮恰好就是合并的），且这些段落已交付、不要再花回合重读。 +- 验证：新增 `frontend/tests/consultation-methodology.test.ts` 8 项：事业路由的 strict 段确实带着 `D10` 与 `Opportunity contact` 到位；无 strict 清单的域如实上报且不被顶替；多域计划每条清单只出现一次；十个域逐个与最宽计划都在体积预算内；`further_reading` 只含包内存在的 `references/` 路径且远少于包体清单；交付内容来自锁定版本号；按标题定段在下一个 `##` 处停止。`consultation-agentic-runtime.test.ts` 45 项通过，新增一项钉住回执把「服务端交付了几段」与「模型主动读了几篇」分开报。`tsc --noEmit` 与 `eslint` 清洁（5 条既有 warning 不在改动文件内）。未做的验证：没有在 staging 上比对同一问题在交付方法论前后的回答差异。 +- 待跟进：那 99KB 的文件名清单仍然每次请求都发一遍，只是不再是模型唯一的导航入口。要真正省掉它，需要把技能包的 `references` 从「软链到全部」改成只含 SKILL.md 点名的那一小批（约 20 个文件的目录视图），代价是 `skill_read` 触及范围收窄，且锁定版的 sha256 不再覆盖模型实际看到的内容——另开一轮评估。`referenceReads` 预期仍会经常为 0，这本身不再是缺陷，但它和 `methodologySections` 要一起读。 +- 防复发：「渐进式披露」只在候选集可判别时成立。把 1,592 个文件平铺给模型，等于既没有给方法也没有给索引，而 `loaded: true` 会让这件事看起来是好的——技能加载成功与方法论被采用之间没有任何因果关系，必须分别度量。另一条：当某个选择在服务端已经确定（这里是 route），就不要把它变成模型的一次工具调用；确定的事交付，不确定的事才询问。 +- 相关记录：BUG-268（把 `referenceReads` 暴露出来，这个问题才可见）、BUG-278（同为「skill 说的和 schema 收的不是一套词汇」）、BUG-279（同为服务端已算出的东西没送到模型手上） +- 复发自：无 +- 修复版本：待提交 diff --git a/frontend/src/app/api/consult/route.ts b/frontend/src/app/api/consult/route.ts index 8599d913..b9a2e7c3 100644 --- a/frontend/src/app/api/consult/route.ts +++ b/frontend/src/app/api/consult/route.ts @@ -691,6 +691,7 @@ export async function POST(request: Request) { name: "jyotish-vedic-astrology", loaded: state.jyotishSkillLoaded, referenceReads: state.skillReferenceReadCount, + methodologySections: state.methodologySectionCount, }, steps: publicConsultationRuntimeSteps(state), stepBudget: consultationStepBudgetReceipt(state), @@ -774,6 +775,7 @@ export async function POST(request: Request) { name: "jyotish-vedic-astrology", loaded: state.jyotishSkillLoaded, referenceReads: state.skillReferenceReadCount, + methodologySections: state.methodologySectionCount, }, steps: publicConsultationRuntimeSteps(state), stepBudget: consultationStepBudgetReceipt(state), diff --git a/frontend/src/lib/agent-observability.ts b/frontend/src/lib/agent-observability.ts index 9f0e1220..0ebbb293 100644 --- a/frontend/src/lib/agent-observability.ts +++ b/frontend/src/lib/agent-observability.ts @@ -132,9 +132,11 @@ export const agentObservabilityEventSchema = z.object({ // every attempt. Both are enum-like machine values, never provider text. modelFinishReason: z.enum(agentModelFinishReasons).optional(), modelStepCount: countSchema.optional(), - // How many reference documents the model opened after loading the skill. Zero on a run that - // answered a domain question means the method was never consulted, which no other field shows. + // How many reference documents the model opened after loading the skill, and how many strict-method + // sections the server delivered with the evidence. Both are needed to read the other: zero reads is + // only a gap in the answer's method if nothing was delivered either. skillReferenceReads: countSchema.optional(), + methodologySections: countSchema.optional(), inputTokens: tokenCountSchema.optional(), outputTokens: tokenCountSchema.optional(), diff --git a/frontend/src/lib/consultation-agent-events.ts b/frontend/src/lib/consultation-agent-events.ts index 7c88ed7f..0aab2e41 100644 --- a/frontend/src/lib/consultation-agent-events.ts +++ b/frontend/src/lib/consultation-agent-events.ts @@ -55,6 +55,10 @@ export const agentExecutionReceiptSchema = z.object({ // optional: the count was tracked in runtime state and surfaced nowhere, so "did the model // consult the method at all" was unanswerable from a finished run. Zero is a real answer. referenceReads: z.number().int().min(0).max(64), + // How many strict-method sections the server delivered with the evidence. Reported beside + // referenceReads rather than folded into it, because "the model went looking" and "the method + // was in front of it" are different facts and only one of them is under the model's control. + methodologySections: z.number().int().min(0).max(12), }).strict(), steps: z.array(executionStepSchema).max(32), stepBudget: stepBudgetSchema.optional(), diff --git a/frontend/src/lib/consultation-methodology.ts b/frontend/src/lib/consultation-methodology.ts new file mode 100644 index 00000000..a73af561 --- /dev/null +++ b/frontend/src/lib/consultation-methodology.ts @@ -0,0 +1,194 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { z } from "zod"; +import { + consultationDomainDefinition, + consultationDomainIds, + type ConsultationDomain, +} from "./consultation-domain-registry.ts"; +import { resolveActiveSkillPackage } from "./skill-package-registry.ts"; + +/** + * The strict checklist each consultation domain must be read against, named in + * the vocabulary of the skill's own router rather than this codebase's. + * + * Activating the skill hands the model its instructions plus a flat list of + * every file in the package: on the Jyotish package that is over 1500 paths and + * about 99KB of undifferentiated filenames, against 30KB of actual method. The + * one line telling it to open `references/strict-workflow-router.md` is inside + * that method, so in practice no reference was ever opened and every answer was + * composed from the model's own background knowledge over server evidence. + * + * The route is already decided server-side, and the skill already states which + * checklist each route requires, so the selection needs no model turn: read the + * mandated sections here and deliver them with the evidence. A checklist the + * router does not declare is reported as absent instead of substituted. + */ +const domainMethodology: Readonly> = { + career: { strictRoute: "career-timing-strict", eventJudgment: "event_judgment_career.md" }, + marriage: { strictRoute: "relationship-timing-strict", eventJudgment: "event_judgment_marriage.md" }, + wealth: { strictRoute: "wealth-timing-strict", eventJudgment: "event_judgment_wealth.md" }, + timing: { strictRoute: "event-timing-strict", eventJudgment: null }, + annual: { strictRoute: null, eventJudgment: null }, + general: { strictRoute: null, eventJudgment: null }, + health: { strictRoute: null, eventJudgment: null }, + education: { strictRoute: null, eventJudgment: null }, + migration: { strictRoute: null, eventJudgment: null }, + family: { strictRoute: null, eventJudgment: null }, +}; + +const ROUTER_FILE = "references/strict-workflow-router.md"; +const SHARED_BASELINE_HEADING = "Shared mandatory baseline"; +const SKELETON_FILE = "references/event_judgment_skeleton.md"; + +/** Per-section and whole-block ceilings, so one long section cannot crowd out the evidence. */ +export const METHODOLOGY_SECTION_MAX_CHARS = 6_000; +export const METHODOLOGY_TOTAL_MAX_CHARS = 24_000; +/** How many of the references SKILL.md names are offered as a navigable index. */ +export const METHODOLOGY_INDEX_MAX_ENTRIES = 24; + +export const consultationMethodologySectionSchema = z.object({ + title: z.string().max(200), + source: z.string().max(300), + text: z.string().max(METHODOLOGY_SECTION_MAX_CHARS), +}).strict(); + +export const consultationMethodologySchema = z.object({ + skill: z.string().max(120), + version: z.string().max(60), + /** Domains whose strict checklist the router does not declare; nothing was substituted for them. */ + domains_without_strict_checklist: z.array(z.enum(consultationDomainIds)).max(6), + sections: z.array(consultationMethodologySectionSchema).max(12), + /** The references SKILL.md itself names, for reading on purpose with skill_read. */ + further_reading: z.array(z.string().max(300)).max(METHODOLOGY_INDEX_MAX_ENTRIES), + truncated: z.boolean(), +}).strict(); + +export type ConsultationMethodology = z.infer; + +const skillPackage = resolveActiveSkillPackage("jyotish-vedic-astrology"); + +/** A published version's files never change, so each one is read from disk once per process. */ +const fileCache = new Map(); + +function packageFile(relativePath: string): string | null { + const hit = fileCache.get(relativePath); + if (hit !== undefined) return hit; + let content: string | null = null; + try { + content = readFileSync(resolve(skillPackage.resolvedPath, relativePath), "utf8"); + } catch { + content = null; + } + fileCache.set(relativePath, content); + return content; +} + +/** + * The body of one `## ...` section, found by a heading substring so the section + * numbering in the router can be renumbered without breaking the lookup. + */ +export function markdownSection(source: string, headingContains: string): string | null { + const lines = source.split("\n"); + const start = lines.findIndex((line) => line.startsWith("## ") && line.includes(headingContains)); + if (start === -1) return null; + const rest = lines.slice(start + 1); + const end = rest.findIndex((line) => line.startsWith("## ")); + const body = (end === -1 ? rest : rest.slice(0, end)).join("\n").trim(); + return body.length > 0 ? `${lines[start]}\n\n${body}` : null; +} + +function clamp(text: string) { + return text.length > METHODOLOGY_SECTION_MAX_CHARS + ? `${text.slice(0, METHODOLOGY_SECTION_MAX_CHARS - 3)}...` + : text; +} + +/** + * The reference paths SKILL.md names, in the order it names them. + * + * Derived rather than listed so that the index follows the skill: when SKILL.md + * starts pointing somewhere else, the model is offered the new pointer without + * this file being edited. Paths that are not in the package are dropped, so a + * stale pointer cannot be advertised as readable. + */ +function namedReferences(instructions: string): string[] { + const found = new Set(); + for (const match of instructions.matchAll(/references\/[\w./-]*[\w/-]/g)) { + const path = match[0].replace(/[.]+$/, ""); + if (path.includes("..") || !/\.\w+$/.test(path)) continue; + if (packageFile(path) === null) continue; + found.add(path); + if (found.size >= METHODOLOGY_INDEX_MAX_ENTRIES) break; + } + return [...found]; +} + +const planCache = new Map(); + +/** + * The strict method for one domain plan, read from the hash-pinned package. + * + * Reading the pinned version rather than the live skill directory means the + * method that shaped an answer is the method a published version states, and + * the answer stays reproducible from the registry entry alone. + */ +export function consultationMethodologyForDomains( + domains: readonly ConsultationDomain[], +): ConsultationMethodology | null { + const unique = [...new Set(domains)]; + const key = unique.join(","); + const hit = planCache.get(key); + if (hit) return hit; + + const router = packageFile(ROUTER_FILE); + const instructions = packageFile("SKILL.md"); + if (!router || !instructions) return null; + + const sections: z.infer[] = []; + let budget = METHODOLOGY_TOTAL_MAX_CHARS; + let truncated = false; + + const push = (title: string, source: string, text: string | null) => { + if (!text) return; + const bounded = clamp(text); + if (bounded.length > budget) { + truncated = true; + return; + } + budget -= bounded.length; + sections.push({ title, source, text: bounded }); + }; + + push("Shared mandatory baseline", ROUTER_FILE, markdownSection(router, SHARED_BASELINE_HEADING)); + push("Event judgment skeleton", SKELETON_FILE, packageFile(SKELETON_FILE)); + + const withoutChecklist: ConsultationDomain[] = []; + for (const domain of unique) { + const { strictRoute, eventJudgment } = domainMethodology[domain]; + if (!strictRoute) { + withoutChecklist.push(domain); + } else { + const section = markdownSection(router, strictRoute); + if (section) { + push(`${consultationDomainDefinition(domain).label} · ${strictRoute}`, ROUTER_FILE, section); + } else { + withoutChecklist.push(domain); + } + } + if (eventJudgment) { + push(`${consultationDomainDefinition(domain).label} · event judgment`, `references/${eventJudgment}`, packageFile(`references/${eventJudgment}`)); + } + } + + const methodology = consultationMethodologySchema.parse({ + skill: skillPackage.name, + version: skillPackage.version, + domains_without_strict_checklist: withoutChecklist, + sections, + further_reading: namedReferences(instructions), + truncated, + }); + planCache.set(key, methodology); + return methodology; +} diff --git a/frontend/src/mastra/consultation-tools.ts b/frontend/src/mastra/consultation-tools.ts index 47caaf9d..499a082a 100644 --- a/frontend/src/mastra/consultation-tools.ts +++ b/frontend/src/mastra/consultation-tools.ts @@ -7,6 +7,7 @@ import { type ConsultationDomain, } from "../lib/consultation-domain-registry.ts"; import { applyBirthTimeModeToWorkflowContext, type ConsultationBirthTimeMode } from "../lib/consultation-birth-time-mode.ts"; +import { consultationMethodologyForDomains } from "../lib/consultation-methodology.ts"; import type { ServerChartConsultation } from "../lib/consultation-route-service.ts"; import { createConsultationPlan, type ConsultationPlan } from "../lib/consultation-plan.ts"; import type { WorkflowReceipt } from "../lib/consultation-agent-events.ts"; @@ -90,6 +91,7 @@ export type ConsultationRuntimeStep = { export type ConsultationRuntimeState = { jyotishSkillLoaded: boolean; skillReferenceReadCount: number; + methodologySectionCount: number; consultationToolStarted: boolean; consultationToolCompleted: boolean; consultationToolCallCount: number; @@ -112,6 +114,7 @@ export function createConsultationRuntimeState(options: { plannedSteps?: number; return { jyotishSkillLoaded: false, skillReferenceReadCount: 0, + methodologySectionCount: 0, consultationToolStarted: false, consultationToolCompleted: false, consultationToolCallCount: 0, @@ -132,6 +135,7 @@ export function consultationModelStepTelemetry(state: ConsultationRuntimeState) return { modelStepCount: state.modelStepCount, skillReferenceReads: state.skillReferenceReadCount, + methodologySections: state.methodologySectionCount, ...(state.modelFinishReason === undefined ? {} : { modelFinishReason: state.modelFinishReason }), }; } @@ -451,10 +455,16 @@ function toModelDomainPlanContext( ...execution.modelOutput, })); const success = executions.every((execution) => execution.context.success); + const methodology = consultationMethodologyForDomains(domains); const plan = { success, domains, omitted_domains: [...omittedDomains], + // The skill states which checklist each route must be read against, and the + // route is already known here, so the method travels with the evidence it + // applies to instead of depending on the model opening the right file out of + // the package listing. + ...(methodology ? { methodology } : {}), }; if (consultations.length === 1 && omittedDomains.length === 0) { return { ...consultations[0], ...plan, consultations }; @@ -545,7 +555,9 @@ export function createConsultationTools(ctx: ConsultationAgentContext) { ctx.state.consultationToolCompleted = true; ctx.state.consultationToolSuccessCount += 1; appendConsultationRuntimeStep(ctx.state, { kind: "tool", name: "run-jyotish-consultation", status: "completed", durationMs: ctx.state.consultationToolDurationMs }); - return toModelDomainPlanContext(executions, omittedDomains); + const modelContext = toModelDomainPlanContext(executions, omittedDomains); + ctx.state.methodologySectionCount = modelContext.methodology?.sections.length ?? 0; + return modelContext; } catch (error) { ctx.state.consultationToolDurationMs = now() - startedAt; appendConsultationRuntimeStep(ctx.state, { diff --git a/frontend/src/mastra/index.ts b/frontend/src/mastra/index.ts index 21d2f8d7..23571116 100644 --- a/frontend/src/mastra/index.ts +++ b/frontend/src/mastra/index.ts @@ -38,6 +38,7 @@ Write in concise Simplified Chinese as a natural conversation, not a report or f For Vedic astrology questions, load the jyotish-vedic-astrology skill before deciding which calculation tool or workflow to use. Follow the skill's method and truth boundaries, but use run-jyotish-consultation for actual chart calculations instead of inventing results. Visible chat format is owned by VISIBLE VOICE above, never by the skill's report template. For questions that require a new chart claim, call run-jyotish-consultation before answering. Simple conversational follow-ups may use the existing context. Select consultation domains only through the single ordered domains array of run-jyotish-consultation, whether the question covers one domain or several; omit it to accept the domain the server already selected. At most ${MAX_CONSULTATION_DOMAINS} domains may be requested in one run, because they are calculated one after another inside a fixed time budget: list them in priority order and prefer the smallest plan that answers the question, since every extra domain takes time away from writing the answer. The server canonicalizes aliases, rejects unsupported/product domains, executes each accepted domain, and returns the actual domains in the tool context and receipt. The only legal domain ids are the ones enumerated in that array's schema; the skill's methodology names strict-workflow checklists such as career-timing-strict, and those labels select techniques inside the skill, never domains for this tool. A rejected domain plan is final for this run: correct the domains once, and never re-send the same call with extra parameters. +The tool result's methodology field is the skill's own strict checklist for the routes that actually ran, quoted from the pinned skill version. Treat it as the method for this answer, not as background: work through its mandatory modules against the evidence you were given, and obey its output discipline, including any instruction to separate kinds of claim rather than merge them into one vague statement. Those sections are already delivered, so never spend a turn re-reading them; methodology.further_reading lists the references the skill names, and you may read one with skill_read only when the question needs something the delivered sections do not cover. When methodology.domains_without_strict_checklist names a domain, the skill declares no strict checklist for it: use the shared baseline and do not imply a strict route was followed. When methodology is absent, follow the skill instructions you already loaded. The tool result always carries one top-level answer contract—status, evidence_contract, claim_cards, rectification—even when several domains ran. For a multi-domain plan that top level is the most restrictive merge of the executed domains, so obey it exactly as written and read consultations only for per-domain detail. Never treat an absent top-level field as permission to answer without a contract. When omitted_domains is non-empty, do not answer those domains and never present the reply as covering the whole plan. Stay with what was calculated. Do not announce a skipped-domain inventory or say this round was incomplete unless the user asked about coverage. Activity, progress, tool status, and execution receipts are server-owned. Never imitate data-jyotish-activity, activity events, tool-started/tool-completed messages, or receipts in the answer text. diff --git a/frontend/tests/chat-session-write.test.ts b/frontend/tests/chat-session-write.test.ts index 50704a12..e3ccfe1f 100644 --- a/frontend/tests/chat-session-write.test.ts +++ b/frontend/tests/chat-session-write.test.ts @@ -19,7 +19,7 @@ test("chat session schema preserves the safe agent execution receipt", () => { const receipt = { runId: "run-1", runtime: "mastra-agentic" as const, - skill: { name: "jyotish-vedic-astrology" as const, loaded: true, referenceReads: 0 }, + skill: { name: "jyotish-vedic-astrology" as const, loaded: true, referenceReads: 0, methodologySections: 0 }, steps: [{ sequence: 1, kind: "skill" as const, name: "jyotish-vedic-astrology", status: "completed" as const }], workflow: { route: "multi-domain", status: "ready", preciseTiming: "allowed", missingLayers: [], domains: ["general", "timing"] }, techniqueTruth: "verified", diff --git a/frontend/tests/consultation-agentic-runtime.test.ts b/frontend/tests/consultation-agentic-runtime.test.ts index 649ebe30..6bb31f8b 100644 --- a/frontend/tests/consultation-agentic-runtime.test.ts +++ b/frontend/tests/consultation-agentic-runtime.test.ts @@ -738,14 +738,14 @@ test("the public receipt never carries the internal failure classification", () // guards the run from failing while building a successful response. const receipt = agentExecutionReceiptSchema.parse({ runId: "run", runtime: "mastra-agentic", - skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 }, + skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 }, steps, workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] }, }); assert.equal(receipt.steps.length, 2); assert.throws(() => agentExecutionReceiptSchema.parse({ runId: "run", runtime: "mastra-agentic", - skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 }, + skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 }, steps: state.steps, workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] }, })); @@ -759,18 +759,18 @@ test("the public receipt never carries the model step budget diagnostics", () => assert.deepEqual( consultationModelStepTelemetry(state), - { modelStepCount: 8, skillReferenceReads: 0, modelFinishReason: "tool-calls" }, + { modelStepCount: 8, skillReferenceReads: 0, methodologySections: 0, modelFinishReason: "tool-calls" }, ); assert.deepEqual( consultationModelStepTelemetry(createConsultationRuntimeState()), - { modelStepCount: 0, skillReferenceReads: 0 }, + { modelStepCount: 0, skillReferenceReads: 0, methodologySections: 0 }, ); // The client receipt schema is strict, so leaking either field would make a // successful run fail while serializing its own answer. const receipt = agentExecutionReceiptSchema.parse({ runId: "run", runtime: "mastra-agentic", - skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 }, + skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 }, steps: publicConsultationRuntimeSteps(state), stepBudget: consultationStepBudgetReceipt(state), workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] }, @@ -778,7 +778,7 @@ test("the public receipt never carries the model step budget diagnostics", () => assert.doesNotMatch(JSON.stringify(receipt), /modelStepCount|modelFinishReason|tool-calls/); assert.throws(() => agentExecutionReceiptSchema.parse({ runId: "run", runtime: "mastra-agentic", - skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 }, + skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 }, steps: publicConsultationRuntimeSteps(state), workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [] }, ...consultationModelStepTelemetry(state), @@ -807,6 +807,17 @@ test("the receipt reports how many reference documents the model opened", () => assert.equal(agentExecutionReceiptSchema.parse(receipt(state)).skill.referenceReads, 2); }); +test("the receipt separates method the server delivered from method the model went looking for", () => { + const state = createConsultationRuntimeState(); + // A run where the model opened nothing is no longer a run composed without method: the strict + // checklist for the route travels with the evidence, so the two counts have to be readable apart. + state.methodologySectionCount = 3; + const parsed = agentExecutionReceiptSchema.parse(receipt(state)); + assert.equal(parsed.skill.referenceReads, 0); + assert.equal(parsed.skill.methodologySections, 3); + assert.equal(consultationModelStepTelemetry(state).methodologySections, 3); +}); + test("personal Agent exposes the Jyotish Skill and named server tool", async () => { const state = createConsultationRuntimeState(); const agent = getJyotishAgent({ @@ -837,7 +848,7 @@ test("public stream filters private chunks and completes once", async () => { const events = await collectAgentPublicEvents(chunks as never, { runId: "run", requestId: "req", toolStatus: () => "ready", receipt: () => ({ - runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 }, + runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 }, steps: [], workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [], domains: ["career"] }, }), }); @@ -857,7 +868,7 @@ test("model answer text cannot forge a public Activity event", async () => { ], { runId: "run", requestId: "req", toolStatus: () => "ready", receipt: () => ({ - runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0 }, + runId: "run", runtime: "mastra-agentic", skill: { name: "jyotish-vedic-astrology", loaded: true, referenceReads: 0, methodologySections: 0 }, steps: [], workflow: { route: "career", status: "ready", preciseTiming: "blocked", missingLayers: [], domains: ["career"] }, }), }); @@ -897,6 +908,7 @@ function receipt(state: ReturnType) { name: "jyotish-vedic-astrology" as const, loaded: state.jyotishSkillLoaded, referenceReads: state.skillReferenceReadCount, + methodologySections: state.methodologySectionCount, }, steps: state.steps, stepBudget: consultationStepBudgetReceipt(state), diff --git a/frontend/tests/consultation-methodology.test.ts b/frontend/tests/consultation-methodology.test.ts new file mode 100644 index 00000000..d5bd3a4a --- /dev/null +++ b/frontend/tests/consultation-methodology.test.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + consultationMethodologyForDomains, + markdownSection, + METHODOLOGY_SECTION_MAX_CHARS, + METHODOLOGY_TOTAL_MAX_CHARS, +} from "../src/lib/consultation-methodology.ts"; +import { consultationDomainIds } from "../src/lib/consultation-domain-registry.ts"; + +test("the route's own strict checklist reaches the answer, not just the package listing", () => { + const methodology = consultationMethodologyForDomains(["career"]); + assert.ok(methodology, "career must resolve a methodology"); + const career = methodology.sections.find((section) => section.title.includes("career-timing-strict")); + assert.ok(career, "the career strict route must be delivered as its own section"); + assert.match(career.source, /strict-workflow-router\.md$/); + // The checklist is only useful if the specific instructions arrive with it. + assert.match(career.text, /D10/); + assert.match(career.text, /Opportunity contact/); + assert.ok( + methodology.sections.some((section) => section.title.includes("Shared mandatory baseline")), + "every route is read against the shared baseline", + ); + assert.deepEqual(methodology.domains_without_strict_checklist, []); +}); + +test("a route the skill declares no checklist for is reported, not filled in with another route's", () => { + const methodology = consultationMethodologyForDomains(["health"]); + assert.ok(methodology); + assert.deepEqual(methodology.domains_without_strict_checklist, ["health"]); + assert.ok( + !methodology.sections.some((section) => section.title.includes("strict")), + "no strict section may be substituted for a route that has none", + ); + assert.ok( + methodology.sections.some((section) => section.title.includes("Shared mandatory baseline")), + "the baseline still applies", + ); +}); + +test("a multi-domain plan carries every executed route's checklist once", () => { + const methodology = consultationMethodologyForDomains(["career", "timing", "career"]); + assert.ok(methodology); + const titles = methodology.sections.map((section) => section.title); + assert.equal(titles.filter((title) => title.includes("career-timing-strict")).length, 1); + assert.equal(titles.filter((title) => title.includes("event-timing-strict")).length, 1); +}); + +test("no plan can spend the answer's context on method", () => { + for (const domain of consultationDomainIds) { + const methodology = consultationMethodologyForDomains([domain]); + assert.ok(methodology, `${domain} must resolve`); + const total = methodology.sections.reduce((sum, section) => sum + section.text.length, 0); + assert.ok( + total <= METHODOLOGY_TOTAL_MAX_CHARS, + `${domain} delivered ${total} chars, over the ${METHODOLOGY_TOTAL_MAX_CHARS} budget`, + ); + for (const section of methodology.sections) { + assert.ok(section.text.length <= METHODOLOGY_SECTION_MAX_CHARS); + } + } +}); + +test("the widest legal plan still fits the budget", () => { + const methodology = consultationMethodologyForDomains(["career", "marriage", "wealth", "timing"]); + assert.ok(methodology); + const total = methodology.sections.reduce((sum, section) => sum + section.text.length, 0); + assert.ok(total <= METHODOLOGY_TOTAL_MAX_CHARS, `delivered ${total} chars`); +}); + +test("further reading offers the references the skill names, and only ones that exist", () => { + const methodology = consultationMethodologyForDomains(["career"]); + assert.ok(methodology); + assert.ok(methodology.further_reading.length > 0, "the skill names references worth reading"); + assert.ok( + methodology.further_reading.includes("references/strict-workflow-router.md"), + "the router the skill points at must be reachable on purpose", + ); + for (const path of methodology.further_reading) { + assert.match(path, /^references\//); + assert.ok(!path.includes(".."), "no path may escape the package"); + } + assert.ok( + methodology.further_reading.length < 100, + "the index exists to replace the package listing, not to reproduce it", + ); +}); + +test("the delivered method is quoted from the pinned skill version", () => { + const methodology = consultationMethodologyForDomains(["career"]); + assert.ok(methodology); + assert.equal(methodology.skill, "jyotish-vedic-astrology"); + assert.match(methodology.version, /^\d+\.\d+\.\d+$/); +}); + +test("a section is found by heading text so the router may be renumbered", () => { + const source = [ + "# Doc", + "## 3. `career-timing-strict`", + "body one", + "## 4. next", + "body two", + ].join("\n"); + const section = markdownSection(source, "career-timing-strict"); + assert.ok(section); + assert.match(section, /body one/); + assert.ok(!section.includes("body two"), "a section stops at the next heading"); + assert.equal(markdownSection(source, "absent-route"), null); +});