Files
Jyotisha/frontend/tests/rectification-v9-agent.test.ts
T

506 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import assert from "node:assert/strict";
import { readFileSync, realpathSync } from "node:fs";
import { basename } from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
import {
RECTIFICATION_AGENT_HARD_STEP_LIMIT,
RECTIFICATION_AGENT_MAX_STEPS,
RECTIFICATION_AGENT_STEP_BUDGETS,
RECTIFICATION_V9_PACKAGE_PATH,
RECTIFICATION_V9_SKILL_PATH,
RECTIFICATION_V9_SKILL_NAME,
getRectificationV9Agent,
getRectificationV9RegenerationAgent,
resolveRectificationStepBudget,
} from "../src/mastra/agentic-rectification.ts";
import {
createRectificationV9ReadOnlyTools,
createRectificationV9Tools,
} from "../src/mastra/rectification-v9-tools.ts";
import { runV9AgentTurn, type V9AgentRunOptions } from "../src/lib/rectification-agentic/v9/agent-run.ts";
import { persistV9Candidate } from "../src/lib/rectification-agentic/v9/tool-service.ts";
import { RECTIFICATION_SKILL_NAME, RECTIFICATION_SKILL_VERSION } from "../src/lib/rectification-agentic/v9/case-status.ts";
import {
CASE_ID,
CANDIDATE_RANGE,
EVIDENCE_ID,
SESSION_ID,
TURN_ID,
USER_ID,
candidateSnapshotFixture,
dossierFixture,
fakeAccounting,
receiptHandlers,
RESULT_ID,
} from "./rectification-v9-test-support.ts";
const agentSource = readFileSync(
fileURLToPath(new URL("../src/mastra/agentic-rectification.ts", import.meta.url)),
"utf8",
);
test("system prompt carries only high-priority boundaries, never the method copy", () => {
const promptStart = agentSource.indexOf("const agenticRectificationInstructions");
const promptEnd = agentSource.indexOf("export function getRectificationV9Agent");
const prompt = agentSource.slice(promptStart, promptEnd);
// No gate -> scan -> score -> diagnostics orchestration in the prompt.
assert.doesNotMatch(prompt, /rectification-gate[\s\S]*rectification-scan/);
assert.doesNotMatch(prompt, /rectification-score[\s\S]*rectification-diagnostics/);
assert.doesNotMatch(prompt, /rectification-confirm[\s\S]*rectification-save-birth-time/);
assert.doesNotMatch(prompt, /80%\/60%/);
assert.doesNotMatch(prompt, /10[-]15 个事件/);
assert.doesNotMatch(prompt, /D9\/D10 类型表/);
assert.doesNotMatch(prompt, /run the required gate/);
assert.doesNotMatch(prompt, /candidate_range/);
assert.match(prompt, /jyotish-birth-time-rectification/);
// Keep the prompt short (~30 lines max).
assert.ok(prompt.split("\n").length <= 60, "instructions must stay bounded");
});
test("agent pins the dedicated rectification skill and its fixed version", () => {
assert.equal(RECTIFICATION_V9_SKILL_NAME, "jyotish-birth-time-rectification");
assert.equal(basename(RECTIFICATION_V9_SKILL_PATH), RECTIFICATION_V9_SKILL_NAME);
assert.ok(RECTIFICATION_V9_PACKAGE_PATH.endsWith("skills/jyotish-birth-time-rectification/versions/9.0.0"));
assert.notEqual(RECTIFICATION_V9_SKILL_PATH, RECTIFICATION_V9_PACKAGE_PATH);
assert.equal(realpathSync(RECTIFICATION_V9_SKILL_PATH), RECTIFICATION_V9_PACKAGE_PATH);
assert.equal(RECTIFICATION_SKILL_NAME, "jyotish-birth-time-rectification");
assert.equal(RECTIFICATION_SKILL_VERSION, "9.0.0");
});
test("step budgets are bounded per action with a hard ceiling", () => {
assert.equal(RECTIFICATION_AGENT_STEP_BUDGETS.opening, 6);
assert.equal(RECTIFICATION_AGENT_STEP_BUDGETS.evidence, 8);
assert.equal(RECTIFICATION_AGENT_STEP_BUDGETS.rescore, 12);
assert.equal(RECTIFICATION_AGENT_STEP_BUDGETS.accept, 6);
assert.equal(RECTIFICATION_AGENT_MAX_STEPS, 12);
for (const action of ["opening", "read_only", "evidence", "rescore", "accept", "confirm"]) {
const budget = resolveRectificationStepBudget(action as keyof typeof RECTIFICATION_AGENT_STEP_BUDGETS);
assert.ok(budget <= RECTIFICATION_AGENT_HARD_STEP_LIMIT, `${action} must respect the hard ceiling`);
}
assert.equal(resolveRectificationStepBudget("rescore"), 12);
});
test("no tool accepts event arrays, birth data or candidate ranges as input", () => {
const tools = createRectificationV9Tools({
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: fakeAccounting({}).client as never,
});
const toolNames = [
"rectification-read-case",
"rectification-propose-evidence",
"rectification-confirm-evidence",
"rectification-revise-evidence",
"rectification-compare-candidates",
"rectification-read-diagnostics",
"rectification-offer-candidates",
"rectification-accept-candidate",
"rectification-confirm-birth-time",
"rectification-close-case",
];
for (const name of toolNames) {
const tool = tools[name as keyof typeof tools] as unknown as { inputSchema: { safeParse(value: unknown): { success: boolean } } };
assert.ok(tool?.inputSchema, `${name} must expose an input schema`);
const malicious = {
caseId: CASE_ID,
userId: USER_ID,
birth_date: "1997-08-08",
latitude: 36.4,
longitude: 114.2,
timezone_offset: 8,
candidate_range: { start_time: "04:00", end_time: "06:00" },
events: [{ id: "e1", domain: "career", date: "2016-09" }],
scores: [10, 20],
confirmationAllowed: true,
};
const result = tool.inputSchema.safeParse(malicious);
assert.equal(result.success, false, `${name} must reject userId/birth/range/events/scores/permissions`);
}
});
type StreamChunk = {
type: string;
payload?: { toolName?: unknown; text?: unknown; args?: unknown; error?: unknown };
};
type FakeStreamResult = {
fullStream: AsyncIterable<StreamChunk>;
totalUsage?: Promise<{ inputTokens?: number; outputTokens?: number }>;
};
function chunk(type: string, payload?: Record<string, unknown>): StreamChunk {
return { type, ...(payload ? { payload } : {}) };
}
function fakeAgentStream(chunks: StreamChunk[]) {
const streamResult: FakeStreamResult = {
fullStream: (async function* () {
for (const item of chunks) yield item;
})(),
totalUsage: Promise.resolve({ inputTokens: 10, outputTokens: 20 }),
};
return {
stream: async () => streamResult,
getSkill: async () => ({ name: RECTIFICATION_SKILL_NAME, instructions: "skill" }),
};
}
function runOptions(overrides: Partial<V9AgentRunOptions> = {}): {
options: V9AgentRunOptions;
emitted: Array<{ type: string }>;
billing: { reserved: number; completed: number; released: number };
} {
const emitted: Array<{ type: string }> = [];
const billing = { reserved: 0, completed: 0, released: 0 };
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture(),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "completed", idempotent: false }),
get_agentic_rectification_turn_receipt: () => null,
});
const optionsValue: V9AgentRunOptions = {
userId: USER_ID,
caseId: CASE_ID,
sessionId: SESSION_ID,
requestId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee",
action: "evidence",
message: "2016年9月离开家去北京工作",
modelName: "gpt-4o-mini",
accounting: accounting.client,
billing: {
reserve: async () => { billing.reserved += 1; return { success: true, status: 200 }; },
complete: async () => { billing.completed += 1; return true; },
release: async () => { billing.released += 1; return true; },
},
emit: (event) => { emitted.push(event); },
buildAgent: async () => fakeAgentStream([]) as never,
...overrides,
};
return { options: optionsValue, emitted, billing };
}
test("agent receives the exact server-owned case id for tool calls", async () => {
let observedMessages: unknown[] = [];
const agent = fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "你好,我是生时校正助手。" }),
chunk("finish"),
]);
const { options } = runOptions({
action: "opening",
message: null,
buildAgent: async () => ({
...agent,
stream: async (messages: unknown[]) => {
observedMessages = messages;
return agent.stream();
},
}) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
const openingPrompt = JSON.stringify(observedMessages);
assert.match(openingPrompt, new RegExp(CASE_ID));
assert.match(openingPrompt, /不需要一次讲完所有经历/);
assert.match(openingPrompt, /日期按真实记忆提供/);
assert.match(openingPrompt, /整段只能有一个主要问题/);
assert.doesNotMatch(openingPrompt, /说明你会通过已发生的人生事件来校正出生时间/);
});
test("first turn with no real skill evidence retries once then fails without saving success", async () => {
const { options, emitted, billing } = runOptions({
accounting: fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture({ turnCount: 0, turns: [] }),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "failed", idempotent: false }),
}).client,
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "你好," }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, false);
assert.equal(result.turnStatus, "failed");
assert.equal(result.skillLoaded, false);
assert.equal(result.errorCode, "skill_not_loaded");
assert.equal(billing.released, 1, "failed first turn must release usage");
assert.equal(billing.completed, 0);
assert.equal(emitted.some((event) => event.type === "run.failed"), true);
assert.equal(emitted.some((event) => event.type === "run.completed"), false);
});
test("first turn with real skill.started/skill.loaded evidence completes and persists receipts", async () => {
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture({ turnCount: 0 }),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "completed", idempotent: false }),
});
const { options, emitted, billing } = runOptions({
accounting: accounting.client,
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "你好,我是生时校正助手。" }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
assert.equal(result.turnStatus, "completed");
assert.equal(result.skillLoaded, true);
assert.equal(billing.completed, 1);
assert.equal(billing.released, 0);
const types = emitted.map((event) => event.type);
assert.ok(types.includes("run.started"));
assert.ok(types.includes("skill.started"));
assert.ok(types.includes("skill.loaded"));
assert.ok(types.includes("case.loaded"));
assert.ok(types.includes("answer.delta"));
assert.ok(types.includes("run.completed"));
// Reasoning and raw chunks are dropped.
const accountingCalls = accounting.calls.map((call) => call.fn);
assert.ok(accountingCalls.includes("insert_agentic_rectification_run_phase"));
assert.ok(accountingCalls.includes("finalize_agentic_rectification_turn"));
});
test("framework getSkill failure is a controlled retry and then a failed turn", async () => {
const { options, billing } = runOptions({
buildAgent: async () => ({
stream: async () => ({ fullStream: (async function* () {})() }),
getSkill: async () => null,
}) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, false);
assert.equal(result.errorCode, "skill_not_loaded");
assert.equal(billing.released, 1);
});
test("distinct evidence calls in one natural turn are not mistaken for a repeated tool loop", async () => {
const firstEvidenceId = EVIDENCE_ID;
const secondEvidenceId = "88888888-8888-4888-8888-888888888888";
const { options, billing } = runOptions({
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("tool-call", {
toolName: "rectification-propose-evidence",
args: { caseId: CASE_ID, quote: "2016年9月离开家去北京开始工作", proposedKind: "career_entry" },
}),
chunk("tool-result", { toolName: "rectification-propose-evidence" }),
chunk("tool-call", { toolName: "rectification-confirm-evidence", args: { caseId: CASE_ID, evidenceId: firstEvidenceId } }),
chunk("tool-result", { toolName: "rectification-confirm-evidence" }),
chunk("tool-call", {
toolName: "rectification-propose-evidence",
args: { caseId: CASE_ID, quote: "2020年又搬到了上海", proposedKind: "relocation" },
}),
chunk("tool-result", { toolName: "rectification-propose-evidence" }),
chunk("tool-call", { toolName: "rectification-confirm-evidence", args: { caseId: CASE_ID, evidenceId: secondEvidenceId } }),
chunk("tool-result", { toolName: "rectification-confirm-evidence" }),
chunk("text-delta", { text: "这两件事已经分别纳入当前校正。" }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, true);
assert.equal(result.errorCode, null);
assert.equal(billing.completed, 1);
assert.deepEqual(result.toolsUsed.filter((name) => name.includes("evidence")), [
"rectification-propose-evidence",
"rectification-confirm-evidence",
]);
});
test("a repeated identical tool call is detected and aborts the turn", async () => {
const { options, billing } = runOptions({
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "skill", args: { name: RECTIFICATION_SKILL_NAME } }),
chunk("tool-result", { toolName: "skill" }),
...Array.from({ length: 4 }, () => chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } })),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, false);
assert.equal(result.errorCode, "repeated_tool_call");
assert.equal(billing.released, 1);
});
test("a failed opening does not let the next turn skip the real skill gate", async () => {
// The dossier has one failed turn and no completed turn: the skill gate
// must still apply, so an agent that never invokes the skill tool fails.
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture({
turnCount: 1,
turns: [{
id: TURN_ID,
role: "user",
text: "(开场)",
status: "failed",
created_at: "2026-08-12T10:00:00.000Z",
completed_at: null,
}],
}),
append_agentic_rectification_turn: () => ({ turn_id: TURN_ID }),
finalize_agentic_rectification_turn: () => ({ turn_id: TURN_ID, status: "failed", idempotent: false }),
});
const { options, billing } = runOptions({
accounting: accounting.client,
buildAgent: async () => fakeAgentStream([
chunk("start"),
chunk("tool-call", { toolName: "rectification-read-case", args: { caseId: CASE_ID } }),
chunk("tool-result", { toolName: "rectification-read-case" }),
chunk("text-delta", { text: "你好," }),
chunk("finish"),
]) as never,
});
const result = await runV9AgentTurn(options);
assert.equal(result.ok, false);
assert.equal(result.errorCode, "skill_not_loaded");
assert.equal(billing.released, 1);
});
test("same evidence + range fingerprints reuse the cached candidate snapshot", async () => {
const accounting = fakeAccounting({
persist_agentic_rectification_candidate: () => ({
...candidateSnapshotFixture(),
cached: true,
}),
});
const cached = await persistV9Candidate(accounting.client, USER_ID, CASE_ID, {
engineResultId: "engine-1",
algorithmVersion: "rectification-v5",
evidenceFingerprint: "b".repeat(64),
rangeFingerprint: "c".repeat(64),
skillVersion: "9.0.0",
candidateRange: CANDIDATE_RANGE,
candidates: [{ rank: 1, time: "05:02", relative_support: 58, tied_minute_count: 2 }],
overallConfidence: "medium",
marginPercent: 16,
selectionAllowed: true,
confirmationAllowed: false,
representativeTime: null,
});
assert.equal(cached.cached, true);
assert.equal(cached.resultId, RESULT_ID);
const call = accounting.calls.find((item) => item.fn === "persist_agentic_rectification_candidate");
assert.ok(call);
assert.equal(call.args.p_evidence_ledger_fingerprint, "b".repeat(64));
assert.equal(call.args.p_skill_version, "9.0.0");
});
test("accept-candidate requires a server-persisted result; no tool means no minute", async () => {
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture({ latestResult: null }),
accept_agentic_rectification_candidate_for_case: () => {
throw new Error("agentic_rectification_candidate_not_found");
},
});
const tools = createRectificationV9Tools({
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: accounting.client as never,
});
await assert.rejects(
(tools["rectification-accept-candidate"] as unknown as { execute(input: unknown): Promise<unknown> }).execute({
caseId: CASE_ID,
resultId: RESULT_ID,
candidateId: "05:02",
}),
(error: unknown) => error instanceof Error && error.message.includes("candidate_not_found"),
);
});
test("new evidence lets the agent choose diagnostics/compare tools autonomously", () => {
// The tool layer exposes read-diagnostics and compare-candidates; there is
// no hard-coded orchestration forcing a scan before score.
const tools = createRectificationV9Tools({
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: fakeAccounting({}).client as never,
});
assert.ok("rectification-compare-candidates" in tools);
assert.ok("rectification-read-diagnostics" in tools);
assert.ok("rectification-offer-candidates" in tools);
assert.ok(!("rectification-scan" in tools));
assert.ok(!("rectification-gate" in tools));
});
test("agent construction wires the pinned skill and the ten v9 tools", () => {
const model = {
id: "test-model",
label: "Test",
description: "",
creditCost: 1,
isDefault: true,
mode: "compatible" as const,
model: { provider: "openai", name: "gpt-4o-mini", modelId: "gpt-4o-mini" } as never,
};
const accounting = fakeAccounting({});
const agent = getRectificationV9Agent(model, {
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: accounting.client as never,
});
assert.equal(agent.id, "rectification-v9-test-model");
assert.ok(agent);
});
test("reply regeneration is a separate Jyotisha agent with only read-case access", () => {
const model = {
id: "test-model",
label: "Test",
description: "",
creditCost: 1,
isDefault: true,
mode: "compatible" as const,
model: { provider: "openai", name: "gpt-4o-mini", modelId: "gpt-4o-mini" } as never,
};
const accounting = fakeAccounting({});
const tools = createRectificationV9ReadOnlyTools({
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: accounting.client as never,
});
assert.deepEqual(Object.keys(tools), ["rectification-read-case"]);
const agent = getRectificationV9RegenerationAgent(model, {
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: accounting.client as never,
});
assert.equal(agent.id, "rectification-v9-regeneration-test-model");
assert.match(agentSource, /这不是新一轮校正/);
assert.match(agentSource, /只能使用 rectification-read-case/);
});