fix(rectification): surface dropped probes and filter tools by the decision
Independent Staging Quality Gate / validate (push) Failing after 17m9s
Independent Staging Quality Gate / publish (push) Has been skipped

Silent unrenderable discriminators, a missing question-contract golden, and a always-on tool table were hiding fail-closed drops behind the prompt wall.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-28 20:31:15 +08:00
parent 69c3e94920
commit 63b4591aae
25 changed files with 856 additions and 274 deletions
@@ -568,6 +568,7 @@ test("time-selection cards appear under the latest settled agent bubble only aft
assert.doesNotMatch(chat, /send\("message", choiceCard\?\.stop_message/);
assert.doesNotMatch(chat, /choiceCardUserMessage/);
assert.match(caseRoute, /choice_card: choiceCardFromCaseDossier/);
assert.match(caseRoute, /current_question: projectCurrentQuestion/);
assert.match(caseRoute, /overlayPublicDecision/);
assert.match(caseRoute, /interview: publicDecisionFields/);
assert.doesNotMatch(chat, /已完成验证/);
@@ -618,20 +619,12 @@ test("rectification Agent output stays natural and keeps tool execution silent",
new URL("../../skills/jyotish-birth-time-rectification/references/conversation-strategy.md", import.meta.url),
"utf8",
);
assert.match(agent, /工具执行过程保持静默/);
assert.match(agent, /思考过程必须用简体中文/);
assert.match(agent, /对用户说的话必须自己写在正文里,不要只写规划等服务器代写/);
assert.match(agent, /本轮做了什么/);
assert.match(agent, /完成凭证完全由服务端公开 Activity\/receipt 展示/);
assert.match(agent, /禁止只说记下了、会话会保留、以后再继续/);
assert.match(agent, /根据用户自然语言语义区分“当前问题没有证据”“停止整个证据收集”和“恢复继续校正”/);
assert.match(agent, /rectification-read-case 传 resume=true/);
assert.match(agent, /工具执行保持静默/);
assert.match(agent, /思考用简体中文写在思维链/);
assert.match(agent, /对用户说的话必须自己写在正文里/);
assert.doesNotMatch(route, /action === "message" && caseStatus === "paused"/);
assert.match(agent, /前者调用 rectification-resolve-focus/);
assert.match(agent, /全局停止则调用 rectification-stop-and-review/);
assert.doesNotMatch(agent, /USER_STOP_PATTERN|USER_STOP_NEGATION_PATTERN/);
assert.match(agent, /skill_verification_report/);
assert.match(agent, /D9\/D10 类型对照/);
assert.doesNotMatch(agent, /分盘句和宫位表由界面展示/);
assert.match(skill, /不得叙述读取 Skill/);
assert.match(skill, /完整回复可以(?:是)?零(?:个)?问题/);
@@ -640,7 +633,7 @@ test("rectification Agent output stays natural and keeps tool execution silent",
assert.match(strategy, /没有更多事件/);
assert.match(strategy, /(?:无需|不要求)结束、暂停或保存进度/);
assert.match(strategy, /不要一进场就出 A\/B\/C\/D/);
assert.match(agent, /不要提点选卡/);
assert.match(agent, /没有持久化选择题时,用自然语言问一件带大概年份的经历/);
});
test("clear current-turn events go through the batch evidence service", () => {
@@ -648,9 +641,7 @@ test("clear current-turn events go through the batch evidence service", () => {
new URL("../src/mastra/rectification-v9-tools.ts", import.meta.url),
"utf8",
);
assert.match(agent, /当前轮新事件一律走 rectification-record-evidence-batch/);
assert.match(agent, /rectification-confirm-evidence 只用于用户对已有 pending 明确说“对\/是”/);
assert.match(agent, /不得要求用户把已说清的事件再发一遍/);
assert.match(agent, /新事件走 rectification-record-evidence-batch/);
assert.doesNotMatch(agent, /分别调用 rectification-propose-evidence 和 rectification-confirm-evidence/);
assert.doesNotMatch(agent, /“是\/对”只能确认当前 pending draft/);
assert.match(tools, /同一轮有两件及以上可拆分事件时必须改用 rectification-record-evidence-batch/);
@@ -658,20 +649,15 @@ test("clear current-turn events go through the batch evidence service", () => {
});
test("the Agent prompt cannot offer candidates while asking for more evidence", () => {
// The hard boundary lives in the prompt; no tool input carries an
// offer_selection boolean anymore.
assert.match(agent, /不得在同一回复中一边要求继续补证据,一边提供候选采用/);
assert.match(agent, /id 不是 adopt_representative、validated_range、provisional_range 或 provisional_range_user_stopped 时不得调用 rectification-offer-candidates/);
assert.match(agent, /verify_adopted_time/);
assert.match(agent, /event_probe/);
assert.match(agent, /至少 3 条训练事件/);
assert.match(agent, /2 个领域/);
assert.match(agent, /发挥质量/);
assert.match(agent, /selection_allowed 只表示可以采用代表性时间/);
assert.doesNotMatch(agent, /offer_selection/);
assert.doesNotMatch(agent, /id 不是 adopt_representative、validated_range、provisional_range 或 provisional_range_user_stopped 时不得调用 rectification-offer-candidates/);
const tools = readFileSync(
new URL("../src/mastra/rectification-v9-tools.ts", import.meta.url),
"utf8",
);
assert.match(tools, /function agentToolKeysForDecision/);
assert.match(tools, /dropped_probes: inspectDiscriminatorProbes/);
assert.match(tools, /dropped_probes: decision.droppedProbes/);
assert.doesNotMatch(tools, /offer_selection/);
});
@@ -696,7 +682,7 @@ test("adopted time offers a consultation handoff without unique-minute copy", ()
assert.match(chat, /choiceAttachment/);
assert.match(page, /startConsultationAfterRectification/);
assert.match(page, /createSession\(modelCatalog\.defaultModelId\)/);
assert.match(agent, /start_consultation/);
assert.doesNotMatch(agent, /start_consultation/);
assert.doesNotMatch(agent, /本会话以代表性时间收口|本轮校正已收口/);
});
@@ -760,6 +760,47 @@ test("turn_decision stays inside the configured byte budget", () => {
assert.ok(!("baseline_birth_snapshot" in projection));
});
test("unrenderable focus schema stays visible as current_question, not null", () => {
const snapshot = candidateSnapshotFixture();
Object.assign(snapshot.decision_receipt, { inference_state: inferenceState() });
const dossier = parseV9CaseDossier(dossierFixture({
latestResult: snapshot,
conversationSummary: conversationSummaryFixture({
activeFocus: activeFocusFixture({
expectedAnswerSchema: { choice: { prompt: "坏题" } },
}),
}),
}));
assert.ok(dossier);
const projection = projectTurnDecision(dossier);
const currentQuestion = projection.current_question as {
unrenderable?: boolean;
reason?: string;
prompt?: string | null;
} | null;
assert.equal(currentQuestion?.unrenderable, true);
assert.equal(currentQuestion?.reason, "invalid_choice_schema");
assert.equal(currentQuestion?.prompt, null);
assert.equal(projection.current_probe, null);
});
test("collection focus without choice copy is not an unrenderable current_question", () => {
const snapshot = candidateSnapshotFixture();
Object.assign(snapshot.decision_receipt, { inference_state: inferenceState() });
const dossier = parseV9CaseDossier(dossierFixture({
latestResult: snapshot,
conversationSummary: conversationSummaryFixture({
activeFocus: activeFocusFixture({
intent: "clarify_event_date",
expectedAnswerSchema: { required: ["month"] },
}),
}),
}));
assert.ok(dossier);
const projection = projectTurnDecision(dossier);
assert.equal(projection.current_question, null);
});
test("turn_decision hides current_probe unless a valid current_question exists", () => {
const withFocus = projectTurnDecision(parseV9CaseDossier(choiceDossier())!);
assert.ok(withFocus.current_question);
@@ -1,32 +1,41 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import { inspectDiscriminatorProbes } from "../src/lib/rectification-agentic/core/candidate-contrast-packet.ts";
import {
EXISTENCE_STYLE_OPTIONS,
QUALITY_STYLE_OPTIONS,
QUESTION_CONTRACT_VERSION,
canonicalProbeQuestionContractJson,
completeStyleOptions,
isRenderableProbe,
rankDiscriminatorScore,
} from "../src/lib/rectification-agentic/v9/probe-question-contract.ts";
const GOLDEN = readFileSync(new URL("../../contracts/probe-question-v1.json", import.meta.url), "utf8");
test("existence probes complete to four answer classes without engine style_options", () => {
const completed = completeStyleOptions({ choiceKind: "existence" });
assert.deepEqual(completed, [...EXISTENCE_STYLE_OPTIONS]);
assert.equal(completed.ok, true);
assert.deepEqual(completed.ok ? completed.options : null, [...EXISTENCE_STYLE_OPTIONS]);
assert.equal(QUESTION_CONTRACT_VERSION, "probe-question-v1");
});
test("event_quality probes use quality labels and still cover unsure", () => {
const completed = completeStyleOptions({ choiceKind: "event_quality" });
assert.deepEqual(completed, [...QUALITY_STYLE_OPTIONS]);
assert.equal(completed?.some((item) => item.answer_class === "unsure"), true);
assert.equal(completed.ok, true);
assert.deepEqual(completed.ok ? completed.options : null, [...QUALITY_STYLE_OPTIONS]);
assert.equal(completed.ok && completed.options.some((item) => item.answer_class === "unsure"), true);
});
test("varga-style probes stay dynamic and fail closed without two scoring labels", () => {
assert.equal(completeStyleOptions({
const rejected = completeStyleOptions({
choiceKind: "varga_style",
styleOptions: [{ label: "巨蟹相处主动热情", answer_class: "yes" }],
}), null);
});
assert.equal(rejected.ok, false);
assert.equal(rejected.ok ? null : rejected.reason, "varga_insufficient_scoring");
const completed = completeStyleOptions({
choiceKind: "varga_style",
styleOptions: [
@@ -34,9 +43,10 @@ test("varga-style probes stay dynamic and fail closed without two scoring labels
{ label: "狮子独立强势", answer_class: "weak_yes", sign: "狮子" },
],
});
assert.equal(completed?.length, 4);
assert.equal(completed?.find((item) => item.answer_class === "no")?.label, "都不是这些特质");
assert.equal(completed?.find((item) => item.answer_class === "unsure")?.label, "这段记不清楚");
assert.equal(completed.ok, true);
assert.equal(completed.ok ? completed.options.length : 0, 4);
assert.equal(completed.ok ? completed.options.find((item) => item.answer_class === "no")?.label : null, "都不是这些特质");
assert.equal(completed.ok ? completed.options.find((item) => item.answer_class === "unsure")?.label : null, "这段记不清楚");
});
test("illegal clock or appearance copy cannot become a renderable probe", () => {
@@ -48,12 +58,13 @@ test("illegal clock or appearance copy cannot become a renderable probe", () =>
styleOptions: [
{ label: "08:12 左右发生", answer_class: "yes" },
],
}), true);
assert.equal(completeStyleOptions({
}).ok, true);
const filled = completeStyleOptions({
choiceKind: "existence",
styleOptions: [{ label: "08:12 左右发生", answer_class: "yes" }],
})?.find((item) => item.answer_class === "yes")?.label, "明确发生且时间吻合");
assert.equal(isRenderableProbe({
});
assert.equal(filled.ok ? filled.options.find((item) => item.answer_class === "yes")?.label : null, "明确发生且时间吻合");
const appearance = isRenderableProbe({
informationGain: 1.2,
candidateIds: ["05:00", "05:04"],
expectedOutcomeCount: 2,
@@ -62,7 +73,58 @@ test("illegal clock or appearance copy cannot become a renderable probe", () =>
{ label: "外貌更接近第一种", answer_class: "yes" },
{ label: "相处更独立", answer_class: "weak_yes" },
],
}), false);
});
assert.equal(appearance.ok, false);
assert.equal(appearance.ok ? null : appearance.reason, "forbidden_copy");
});
test("duplicate visible labels uniquify with an index, not answer_class", () => {
const completed = completeStyleOptions({
choiceKind: "varga_style",
styleOptions: [
{ label: "相处主动热情", answer_class: "yes" },
{ label: "相处主动热情", answer_class: "weak_yes" },
],
});
assert.equal(completed.ok, true);
const labels = completed.ok ? completed.options.map((item) => item.label) : [];
assert.equal(labels.includes("相处主动热情·2"), true);
assert.equal(labels.some((item) => item.includes("weak_yes") || item.includes("·yes")), false);
assert.equal(new Set(labels).size, 4);
});
test("unrenderable contrast probes are dropped with a reason, not selected", () => {
const inspected = inspectDiscriminatorProbes({
candidateSetVersion: "set",
vargaDifferences: [],
probes: [{
probeId: "contrast:varga.d9.a/b",
candidateSetVersion: "set",
question: "亲密关系里更接近下面哪一种相处方式?",
expectedOutcomes: [
{ outcomeId: "yes", supportsCandidateIds: ["05:00"], conflictsCandidateIds: ["05:10"] },
{ outcomeId: "weak_yes", supportsCandidateIds: ["05:10"], conflictsCandidateIds: ["05:00"] },
],
candidateSplitHash: "set:varga.d9",
informationGain: 1.2,
sourceFeatures: [],
domain: "relationship",
year: null,
semanticKey: "varga.d9.foo/bar",
choiceKind: "varga_style",
styleOptions: [
{ label: "外貌更接近第一种", answerClass: "yes" },
{ label: "外貌更接近第二种", answerClass: "weak_yes" },
],
}],
});
assert.equal(inspected.selected, null);
assert.equal(inspected.dropped[0]?.reason, "forbidden_copy");
assert.equal(inspected.dropped[0]?.semantic_key, "varga.d9.foo/bar");
});
test("TypeScript contract JSON matches the golden file byte for byte", () => {
assert.equal(canonicalProbeQuestionContractJson(), GOLDEN);
});
test("asked probes keep a novelty penalty so unused high-gain probes rank first", () => {
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
import test from "node:test";
import { PUBLIC_RECTIFICATION_TOOLS } from "../src/lib/rectification-agentic/v9/public-receipt.ts";
import { decideRectification } from "../src/lib/rectification-agentic/core/rectification-decision.ts";
import { createRectificationV9AgentTools, createRectificationV9Tools } from "../src/mastra/rectification-v9-tools.ts";
import {
CASE_ID,
@@ -204,6 +205,24 @@ test("evidence kind and domain schemas enumerate legal values so education is no
}).success, true);
});
test("agent tools omit offer-candidates when proposeAllowed is false", () => {
const tools = createRectificationV9AgentTools({
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: fakeAccounting({}).client as never,
}, decideRectification({
methodCoverageAll: false,
trainingGateOpen: false,
candidateScores: [],
}));
assert.equal("rectification-set-focus" in tools, false);
assert.equal("rectification-read-case" in tools, true);
assert.equal("rectification-offer-candidates" in tools, false);
assert.equal("rectification-accept-candidate" in tools, false);
assert.equal("rectification-confirm-birth-time" in tools, false);
});
test("agent tools omit model-driven set-focus", () => {
const tools = createRectificationV9AgentTools({
userId: USER_ID,
+14 -38
View File
@@ -46,61 +46,37 @@ test("system prompt carries only high-priority boundaries, never the method copy
const promptStart = agentSource.indexOf("const agenticRectificationInstructions");
const promptEnd = agentSource.indexOf("export function getRectificationV9Agent");
const prompt = agentSource.slice(promptStart, promptEnd);
// No gate -> scan -> score -> diagnostics orchestration in the prompt.
assert.doesNotMatch(prompt, /rectification-gate[\s\S]*rectification-scan/);
assert.doesNotMatch(prompt, /rectification-score[\s\S]*rectification-diagnostics/);
assert.doesNotMatch(prompt, /rectification-confirm[\s\S]*rectification-save-birth-time/);
assert.doesNotMatch(prompt, /10[-]15 个事件/);
assert.match(prompt, /D9\/D10 类型/);
assert.match(prompt, /80%\/60%/);
assert.match(prompt, /skill_verification_report/);
assert.doesNotMatch(prompt, /run the required gate/);
assert.doesNotMatch(prompt, /candidate_range/);
assert.match(prompt, /jyotish-birth-time-rectification/);
assert.match(prompt, /display_date_label/);
assert.match(prompt, /rectification-record-evidence-batch/);
assert.match(prompt, /不可分区间/);
assert.match(prompt, /confirmation_gate/);
assert.match(prompt, /session_outcome=adopt_representative/);
assert.doesNotMatch(prompt, /本会话以代表性时间收口|本轮校正已收口/);
assert.match(prompt, /unique_minute_path=closed_at_representative/);
assert.match(prompt, /next_user_action/);
assert.match(prompt, /rectification-offer-candidates/);
assert.match(prompt, /on_user_stop/);
assert.match(prompt, /禁止只说记下了/);
assert.match(prompt, /工具执行过程保持静默/);
assert.match(prompt, /思考过程必须用简体中文/);
assert.match(prompt, /对用户说的话必须自己写在正文里,不要只写规划等服务器代写/);
assert.match(prompt, /skill_verification_report/);
assert.match(prompt, /不要因家人或职业方法层未覆盖而改回收集/);
assert.match(prompt, /ask_candidate_discriminator/);
assert.match(prompt, /offer_provisional_range/);
assert.match(prompt, /不要再问整窗 D9\/D24/);
assert.match(prompt, /不得询问外貌、体质、胎记或疤痕/);
assert.match(prompt, /不要调用 rectification-set-focus/);
assert.match(prompt, /题干和动态选项只由选择卡展示/);
assert.match(prompt, /正文只做简短自然承接/);
assert.match(prompt, /不得另写、改写或复述题干与选项/);
assert.match(prompt, /80%\/60%/);
assert.match(prompt, /confirmation_allowed/);
assert.doesNotMatch(prompt, /本会话以代表性时间收口|本轮校正已收口/);
assert.match(prompt, /工具执行保持静默/);
assert.match(prompt, /思考用简体中文写在思维链/);
assert.match(prompt, /对用户说的话必须自己写在正文里/);
assert.match(prompt, /题干和选项只由选择卡展示/);
assert.match(prompt, /正文只自然承接/);
assert.match(prompt, /「先这样」由服务器/);
assert.doesNotMatch(prompt, /不得询问外貌、体质、胎记或疤痕/);
assert.doesNotMatch(prompt, /财务与健康只有用户主动说才问/);
assert.doesNotMatch(prompt, /id 不是 adopt_representative/);
assert.doesNotMatch(prompt, /不要调用 rectification-set-focus/);
assert.doesNotMatch(prompt, /自己写一句自然语言追问/);
assert.doesNotMatch(prompt, /运行器会把口语接到这句题干/);
assert.doesNotMatch(prompt, /运行器只在你没问/);
assert.doesNotMatch(prompt, /不得另起高考发挥/);
assert.doesNotMatch(prompt, /不得根据出生年推算高考或入学年份/);
assert.doesNotMatch(prompt, /不要再问那一件发生在哪一年/);
assert.match(prompt, /「先这样」由服务器/);
assert.match(prompt, /盘外核对(不计分)/);
assert.match(prompt, /verify_adopted_time/);
assert.match(prompt, /event_probe/);
assert.match(prompt, /至少 3 条训练事件/);
assert.match(prompt, /2 个领域/);
assert.match(prompt, /发挥质量/);
assert.doesNotMatch(prompt, /两套盘各自的前事/);
assert.doesNotMatch(prompt, /外貌、体质、胎记或疤痕可以问/);
assert.doesNotMatch(prompt, /分盘句和宫位表由界面展示/);
assert.doesNotMatch(prompt, /不是整张宫位表/);
assert.doesNotMatch(prompt, /分别 propose\+confirm/);
// Keep the prompt short (~30 lines max).
assert.ok(prompt.split("\n").length <= 60, "instructions must stay bounded");
assert.ok(prompt.split("\n").length <= 30, "instructions must stay bounded");
});
test("agent pins the dedicated rectification skill and its fixed version", () => {
@@ -261,6 +261,25 @@ test("runV9CandidateScore strictly consumes server candidate decisions and v2 re
}
});
test("runV9CandidateScore fails closed on a mismatched question contract version", async () => {
const restore = stubEngine({
...ENGINE_SCORE_RESPONSE_V2,
decision_receipt: { ...DECISION_RECEIPT, question_contract_version: "probe-question-v0" },
});
try {
await assert.rejects(
runV9CandidateScore({
baselineBirthSnapshot: SNAPSHOT,
candidateRange: RANGE,
events: toEngineEvents(EVIDENCE),
}),
(error: unknown) => error instanceof RectificationEngineError && error.code === "engine_invalid_v2_receipt",
);
} finally {
restore();
}
});
test("runV9CandidateScore fails closed without a v2 decision receipt", async () => {
const missingReceipt = Object.fromEntries(
Object.entries(ENGINE_SCORE_RESPONSE_V2).filter(([key]) => key !== "decision_receipt"),