Jesse_Chen
7de36e11d6
Independent Staging Quality Gate 2141 failed because four tests still matched the old Mastra prompt wall and treated an unrenderable focus as null. Co-authored-by: Cursor <cursoragent@cursor.com>
271 lines
11 KiB
TypeScript
271 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { readFileSync } from "node:fs";
|
|
import test from "node:test";
|
|
|
|
import {
|
|
SEALED_MINUTE_HOLDOUT,
|
|
buildConfirmationGate,
|
|
readVedastroMinuteSensitiveStatus,
|
|
sessionOutcomeFromGate,
|
|
uniqueMinutePath,
|
|
} from "../src/lib/rectification-agentic/v9/confirmation-gate.ts";
|
|
import { RECTIFICATION_POLICY } from "../src/lib/rectification-policy.ts";
|
|
import { RECTIFICATION_SKILL_VERSION } from "../src/lib/rectification-agentic/v9/case-status.ts";
|
|
import { decideRectification } from "../src/lib/rectification-agentic/core/rectification-decision.ts";
|
|
import {
|
|
createRectificationV9Tools,
|
|
latestResultToolProjection,
|
|
} from "../src/mastra/rectification-v9-tools.ts";
|
|
import { PUBLIC_RECTIFICATION_TOOLS } from "../src/lib/rectification-agentic/v9/public-receipt.ts";
|
|
import { safeToolErrorCode, RectificationToolServiceError } from "../src/lib/rectification-agentic/v9/tool-service.ts";
|
|
import {
|
|
CASE_ID,
|
|
CANDIDATE_ID,
|
|
RESULT_ID,
|
|
SECOND_CANDIDATE_ID,
|
|
TURN_ID,
|
|
USER_ID,
|
|
candidateSnapshotFixture,
|
|
dossierFixture,
|
|
fakeAccounting,
|
|
receiptHandlers,
|
|
} from "./rectification-v9-test-support.ts";
|
|
|
|
const THIRD_CANDIDATE_ID = "88888888-8888-4888-8888-888888888883";
|
|
const holdoutReport = JSON.parse(readFileSync(
|
|
new URL("../../references/real_case_calibration/minute_rectification_holdout_v2_pilot_report.json", import.meta.url),
|
|
"utf8",
|
|
)) as {
|
|
case_gate: { valid_public_aa_cases: number; minimum_public_aa_cases: number };
|
|
metrics: { top_1_rate: number; confirmation_coverage_rate: number };
|
|
};
|
|
const skill = readFileSync(
|
|
new URL("../../skills/jyotish-birth-time-rectification/SKILL.md", import.meta.url),
|
|
"utf8",
|
|
);
|
|
const candidateComparison = readFileSync(
|
|
new URL("../../skills/jyotish-birth-time-rectification/references/candidate-comparison.md", import.meta.url),
|
|
"utf8",
|
|
);
|
|
const agentSource = readFileSync(
|
|
new URL("../src/mastra/agentic-rectification.ts", import.meta.url),
|
|
"utf8",
|
|
);
|
|
|
|
const UNIQUE_MINUTE = [
|
|
{ candidateId: CANDIDATE_ID, time: "05:02", rank: 1, relativeSupport: 70, tiedMinuteCount: 1 },
|
|
{ candidateId: SECOND_CANDIDATE_ID, time: "05:03", rank: 2, relativeSupport: 30, tiedMinuteCount: 2 },
|
|
];
|
|
|
|
const PLATEAU = [
|
|
{ candidateId: CANDIDATE_ID, time: "04:45", rank: 1, relativeSupport: 40, tiedMinuteCount: 25 },
|
|
{ candidateId: SECOND_CANDIDATE_ID, time: "04:46", rank: 2, relativeSupport: 35, tiedMinuteCount: 25 },
|
|
{ candidateId: THIRD_CANDIDATE_ID, time: "04:47", rank: 3, relativeSupport: 25, tiedMinuteCount: 25 },
|
|
];
|
|
|
|
function collectDecision(candidates: readonly { time: string; relativeSupport: number }[]) {
|
|
return decideRectification({
|
|
methodCoverageAll: false,
|
|
trainingGateOpen: false,
|
|
candidateScores: candidates.map((item) => ({ time: item.time, score: item.relativeSupport })),
|
|
});
|
|
}
|
|
|
|
function blocker(
|
|
gate: ReturnType<typeof buildConfirmationGate>,
|
|
id: string,
|
|
) {
|
|
const found = gate.blockers.find((item) => item.id === id);
|
|
assert.ok(found, `missing blocker ${id}`);
|
|
return found;
|
|
}
|
|
|
|
test("sealed holdout aggregates match the v2 report and stay below the case gate", () => {
|
|
const productHoldout = JSON.parse(readFileSync(
|
|
new URL("../../references/rectification_sealed_holdout.v1.json", import.meta.url),
|
|
"utf8",
|
|
)) as typeof SEALED_MINUTE_HOLDOUT;
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.case_gate.valid_public_aa_cases);
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.case_gate.minimum_public_aa_cases);
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, holdoutReport.metrics.top_1_rate);
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, holdoutReport.metrics.confirmation_coverage_rate);
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, productHoldout.valid_public_aa_cases);
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.status, productHoldout.status);
|
|
assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases < SEALED_MINUTE_HOLDOUT.required_cases);
|
|
assert.equal(SEALED_MINUTE_HOLDOUT.status, "not_ready");
|
|
});
|
|
|
|
test("VedAstro not_evaluated is not described as fail and still forbids confirmation", () => {
|
|
const gate = buildConfirmationGate({
|
|
engineConfirmationAllowed: true,
|
|
candidates: UNIQUE_MINUTE,
|
|
decisionReceipt: {
|
|
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
|
|
},
|
|
});
|
|
const vedastro = blocker(gate, "vedastro_minute_sensitive");
|
|
const adjacent = blocker(gate, "adjacent_minutes_indistinguishable");
|
|
const holdout = blocker(gate, "public_aa_holdout");
|
|
assert.equal(readVedastroMinuteSensitiveStatus({
|
|
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
|
|
}), "not_evaluated");
|
|
assert.equal(vedastro.status, "not_evaluated");
|
|
assert.notEqual(vedastro.status, "fail");
|
|
assert.notEqual(vedastro.status, "failed");
|
|
assert.match(vedastro.user_meaning, /尚未执行/);
|
|
assert.doesNotMatch(vedastro.user_meaning, /执行失败|校验失败/);
|
|
assert.equal(adjacent.status, "passed");
|
|
assert.equal(holdout.status, "not_ready");
|
|
assert.equal(gate.confirmation_allowed, false);
|
|
assert.equal(gate.unique_minute_path, "closed_at_representative");
|
|
assert.match(holdout.user_meaning, /未达发布门槛/);
|
|
assert.match(holdout.user_meaning, /不能确认唯一分钟/);
|
|
assert.equal(uniqueMinutePath(false), "closed_at_representative");
|
|
assert.equal(uniqueMinutePath(true), "awaiting_user_consent");
|
|
});
|
|
|
|
test("a 25-minute plateau stays confirmation-blocked as an indistinguishable range", () => {
|
|
const projection = latestResultToolProjection({
|
|
resultId: RESULT_ID,
|
|
candidates: PLATEAU,
|
|
selectionAllowed: true,
|
|
confirmationAllowed: true,
|
|
representativeTime: "04:45",
|
|
selectedTime: null,
|
|
selectionKind: null,
|
|
algorithmVersion: "rectification-v5",
|
|
}, collectDecision(PLATEAU));
|
|
const gate = projection.confirmation_gate as ReturnType<typeof buildConfirmationGate>;
|
|
assert.ok(Number(projection.indistinguishable_width_minutes) >= 25);
|
|
assert.equal(projection.confirmation_allowed, false);
|
|
assert.equal(gate.confirmation_allowed, false);
|
|
assert.equal(blocker(gate, "adjacent_minutes_indistinguishable").status, "blocked");
|
|
assert.equal(
|
|
blocker(gate, "adjacent_minutes_indistinguishable").max_confirmation_width_minutes,
|
|
RECTIFICATION_POLICY.maxConfirmationWidthMinutes,
|
|
);
|
|
assert.match(blocker(gate, "adjacent_minutes_indistinguishable").user_meaning, /不可分区间|代表性候选/);
|
|
assert.equal(projection.session_outcome, "collect_evidence");
|
|
assert.equal(sessionOutcomeFromGate({
|
|
selectionAllowed: true,
|
|
confirmationAllowed: false,
|
|
}).kind, "collect_evidence");
|
|
assert.equal(sessionOutcomeFromGate({
|
|
selectionAllowed: true,
|
|
proposeAllowed: true,
|
|
confirmationAllowed: false,
|
|
interviewOpen: false,
|
|
}).kind, "adopt_representative");
|
|
assert.match(sessionOutcomeFromGate({
|
|
selectionAllowed: true,
|
|
proposeAllowed: true,
|
|
confirmationAllowed: false,
|
|
interviewOpen: false,
|
|
}).user_meaning, /可以采用代表性候选作当前排盘/);
|
|
assert.equal(sessionOutcomeFromGate({
|
|
selectionAllowed: true,
|
|
confirmationAllowed: false,
|
|
userStopped: true,
|
|
}).kind, "provisional_range_user_stopped");
|
|
assert.match(sessionOutcomeFromGate({
|
|
selectionAllowed: true,
|
|
confirmationAllowed: false,
|
|
userStopped: true,
|
|
}).user_meaning, /不是最终校正结果/);
|
|
assert.doesNotMatch(sessionOutcomeFromGate({
|
|
selectionAllowed: true,
|
|
confirmationAllowed: false,
|
|
userStopped: true,
|
|
}).user_meaning, /已完成验证/);
|
|
});
|
|
|
|
test("holdout not_ready forbids unique-minute copy and still blocks confirm", async () => {
|
|
const projection = latestResultToolProjection({
|
|
resultId: RESULT_ID,
|
|
candidates: UNIQUE_MINUTE,
|
|
selectionAllowed: true,
|
|
confirmationAllowed: true,
|
|
representativeTime: "05:02",
|
|
selectedTime: null,
|
|
selectionKind: null,
|
|
algorithmVersion: "rectification-v5",
|
|
decisionReceipt: {
|
|
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
|
|
},
|
|
}, collectDecision(UNIQUE_MINUTE));
|
|
const serialized = JSON.stringify(projection);
|
|
assert.equal(projection.confirmation_allowed, false);
|
|
assert.equal(
|
|
(projection.confirmation_gate as ReturnType<typeof buildConfirmationGate>).unique_minute_path,
|
|
"closed_at_representative",
|
|
);
|
|
assert.equal(projection.unique_minute_claim, false);
|
|
assert.doesNotMatch(serialized, /±2 分钟/);
|
|
assert.match(serialized, /candidate_range_not_birth_time_truth/);
|
|
assert.match(serialized, /不是已确认唯一出生分钟|不得写成已确认唯一出生分钟/);
|
|
assert.doesNotMatch(serialized, /obama|barack|angelina|jolie/i);
|
|
assert.match(skill, /confirmation_gate/);
|
|
assert.match(skill, /not_evaluated/);
|
|
assert.match(skill, /session_outcome=adopt_representative/);
|
|
assert.doesNotMatch(skill, /本会话以代表性时间收口|本轮校正已收口/);
|
|
assert.match(skill, /unique_minute_path=closed_at_representative/);
|
|
assert.doesNotMatch(skill, /还不能确认唯一分钟/);
|
|
assert.match(candidateComparison, /confirmation_gate/);
|
|
assert.match(agentSource, /confirmation_allowed/);
|
|
assert.match(agentSource, /不得宣称唯一出生分钟/);
|
|
assert.doesNotMatch(agentSource, /±2 分钟/);
|
|
assert.equal(PUBLIC_RECTIFICATION_TOOLS.length, 14);
|
|
assert.equal(RECTIFICATION_SKILL_VERSION, "10.0.13");
|
|
|
|
const accounting = fakeAccounting({
|
|
...receiptHandlers,
|
|
get_agentic_rectification_case_dossier: () => dossierFixture({
|
|
latestResult: {
|
|
...candidateSnapshotFixture({
|
|
confirmationAllowed: true,
|
|
selectionAllowed: true,
|
|
representativeTime: "05:02",
|
|
candidates: [
|
|
{ candidate_id: CANDIDATE_ID, rank: 1, time: "05:02", relative_support: 70, tied_minute_count: 1 },
|
|
{ candidate_id: SECOND_CANDIDATE_ID, rank: 2, time: "05:03", relative_support: 30, tied_minute_count: 2 },
|
|
],
|
|
}),
|
|
confirmation_allowed: true,
|
|
decision_receipt: {
|
|
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
|
|
},
|
|
},
|
|
}),
|
|
confirm_agentic_rectification_candidate_for_case_v2: () => ({
|
|
success: true,
|
|
saved_time: "05:02",
|
|
status: "confirmed",
|
|
result_id: RESULT_ID,
|
|
case_status: "confirmed",
|
|
idempotent: false,
|
|
}),
|
|
});
|
|
const tools = createRectificationV9Tools({
|
|
userId: USER_ID,
|
|
caseId: CASE_ID,
|
|
turnId: TURN_ID,
|
|
accounting: accounting.client as never,
|
|
});
|
|
await assert.rejects(
|
|
(tools["rectification-confirm-birth-time"] as unknown as {
|
|
execute(input: unknown): Promise<unknown>;
|
|
}).execute({
|
|
caseId: CASE_ID,
|
|
resultId: RESULT_ID,
|
|
candidateId: CANDIDATE_ID,
|
|
consentQuote: "就用05:02",
|
|
}),
|
|
(error: unknown) => error instanceof RectificationToolServiceError && error.code === "confirmation_blocked",
|
|
);
|
|
assert.equal(
|
|
accounting.calls.some((call) => call.fn === "confirm_agentic_rectification_candidate_for_case_v2"),
|
|
false,
|
|
);
|
|
assert.equal(safeToolErrorCode(new RectificationToolServiceError("confirmation_blocked")), "confirmation_blocked");
|
|
});
|