Files
Jyotisha/frontend/tests/rectification-confirmation-gate.test.ts
T
Jesse_ChenandClaude Opus 5.5 ea21743b09 fix(rectification): stop spoken collect once the training gate opens (BUG-1084..1087)
Once the discriminator training gate is open, only choice cards are asked
and the range card goes out when they are exhausted; targeted lines, their
re-ask and guided windows no longer hold the card or invite more events.
Delivery body says how many choice questions were used instead of the event
fit percent; narration names an excluded cluster instead of "range
unchanged"; a delivered turn no longer carries a collect question.

Offline replay (v4, 3 radii x 2 directions): truth in range 20/20 in every
cell; guided-window injections give the same width in truth and opposite
directions, so red line 1 was revised by product to truth-in-range only.
Skill 10.0.31 -> 10.0.32 (10.0.31 kept as deprecated for pinned cases).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017eEAG8HD3mm8gsKXgk8uU8
2026-09-29 10:06:57 +08:00

492 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import { indistinguishableWidthMinutes } from "../src/lib/rectification-agentic/v9/candidate-plateau.ts";
import {
SEALED_MINUTE_HOLDOUT,
buildConfirmationGate,
readVedastroMinuteSensitiveStatus,
sessionOutcomeFromGate,
uniqueMinutePath,
} from "../src/lib/rectification-agentic/v9/confirmation-gate.ts";
import { RECTIFICATION_POLICY } from "../src/lib/rectification-policy.ts";
import { RECTIFICATION_SKILL_VERSION } from "../src/lib/rectification-agentic/v9/case-status.ts";
import { decideRectification, publicDecisionFields } from "../src/lib/rectification-agentic/core/rectification-decision.ts";
import {
createRectificationV9Tools,
latestResultToolProjection,
} from "../src/mastra/rectification-v9-tools.ts";
import { PUBLIC_RECTIFICATION_TOOLS } from "../src/lib/rectification-agentic/v9/public-receipt.ts";
import { safeToolErrorCode, RectificationToolServiceError } from "../src/lib/rectification-agentic/v9/tool-service.ts";
import {
OPEN_ENGINE_CAPABILITY_CEILING,
CASE_ID,
CANDIDATE_ID,
RESULT_ID,
SECOND_CANDIDATE_ID,
TURN_ID,
USER_ID,
candidateSnapshotFixture,
dossierFixture,
fakeAccounting,
receiptHandlers,
} from "./rectification-v9-test-support.ts";
const THIRD_CANDIDATE_ID = "88888888-8888-4888-8888-888888888883";
const holdoutReport = JSON.parse(readFileSync(
new URL("../../references/real_case_calibration/minute_rectification_holdout_v3_report.json", import.meta.url),
"utf8",
)) as {
benchmark_id: string;
evaluated_on: string;
validation: { valid_public_aa_cases: number; minimum_public_aa_cases: number };
frozen_scoring: {
algorithm_version: string;
implementation_hash_matches: boolean;
implementation_sha256: string;
};
release_metrics: {
top_1_rate: { actual: number };
top_3_rate: { actual: number };
mean_absolute_minute_error: { actual: number };
confirmation_coverage_rate: number;
};
};
const diagnosticReport = JSON.parse(readFileSync(
new URL(
"../../references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
import.meta.url,
),
"utf8",
)) as {
trial_count: number;
is_blind_evaluation: boolean;
must_not_claim_release_gate_passed: boolean;
must_not_use_for_tuning: boolean;
frozen_scoring: {
implementation_hash_matches: boolean;
actual_sha256: string;
};
current_tree_unfrozen_diagnostic: {
actual_sha256: string;
trial_count: number;
metrics: {
top_1_rate: number;
top_3_rate: number;
mean_absolute_minute_error: number;
confirmation_coverage_rate: number;
};
metric_gates_passed: boolean;
verified_minute_claim_allowed: boolean;
};
};
// 原值:docs/research/sealed_holdout_rerun_cross_midnight_2026_09_20.json
// 新值:docs/research/sealed_holdout_rerun_midnight_anchor_2026_09_21.json
// 原因:午夜日期锚点修复的新冻结重跑另存报告;旧报告保留,确认门行为与阈值不变。
const currentTreeRerun = JSON.parse(readFileSync(
new URL("../../docs/research/sealed_holdout_rerun_midnight_anchor_2026_09_21.json", import.meta.url),
"utf8",
)) as {
frozen_record: { implementation_sha256: string };
official_valid_independent_blind: boolean;
};
type ProductHoldoutContract = typeof SEALED_MINUTE_HOLDOUT & {
metrics_produced_by: {
implementation_sha256: string;
algorithm_version: string;
implementation_hash_matches_at_replay: boolean;
source_report: string;
evaluated_on: string;
};
current_tree_scorer: {
implementation_sha256: string;
matches_metrics_scorer: boolean;
official_eval_implementation_hash_matches: boolean;
official_eval_trial_count: number;
source_report: string;
};
current_tree_unfrozen_diagnostic: {
is_blind_evaluation: boolean;
results_already_seen: boolean;
scorer_frozen: boolean;
source_audit_passed: boolean;
must_not_claim_as_release_metrics: boolean;
must_not_use_for_tuning: boolean;
report_path: string;
implementation_sha256: string;
trial_count: number;
top_1_rate: number;
top_3_rate: number;
mean_absolute_minute_error: number;
confirmation_coverage_rate: number;
metric_gates_passed: boolean;
verified_minute_claim_allowed: boolean;
};
};
const skill = readFileSync(
new URL("../../skills/jyotish-birth-time-rectification/SKILL.md", import.meta.url),
"utf8",
);
const candidateComparison = readFileSync(
new URL("../../skills/jyotish-birth-time-rectification/references/candidate-comparison.md", import.meta.url),
"utf8",
);
const agentSource = readFileSync(
new URL("../src/mastra/agentic-rectification.ts", import.meta.url),
"utf8",
);
const UNIQUE_MINUTE = [
{ candidateId: CANDIDATE_ID, time: "05:02", rank: 1, relativeSupport: 70, tiedMinuteCount: 1 },
{ candidateId: SECOND_CANDIDATE_ID, time: "05:03", rank: 2, relativeSupport: 30, tiedMinuteCount: 2 },
];
const PLATEAU = [
{ candidateId: CANDIDATE_ID, time: "04:45", rank: 1, relativeSupport: 40, tiedMinuteCount: 25 },
{ candidateId: SECOND_CANDIDATE_ID, time: "04:46", rank: 2, relativeSupport: 35, tiedMinuteCount: 25 },
{ candidateId: THIRD_CANDIDATE_ID, time: "04:47", rank: 3, relativeSupport: 25, tiedMinuteCount: 25 },
];
function collectDecision(candidates: readonly { time: string; relativeSupport: number }[]) {
return decideRectification({
engineCeiling: OPEN_ENGINE_CAPABILITY_CEILING,
methodCoverageAll: false,
trainingGateOpen: false,
candidateScores: candidates.map((item) => ({ time: item.time, score: item.relativeSupport })),
});
}
function blocker(
gate: ReturnType<typeof buildConfirmationGate>,
id: string,
) {
const found = gate.blockers.find((item) => item.id === id);
assert.ok(found, `missing blocker ${id}`);
return found;
}
test("sealed holdout aggregates match the v3 report and still block confirmation", () => {
const productHoldout = JSON.parse(readFileSync(
new URL("../../references/rectification_sealed_holdout.v1.json", import.meta.url),
"utf8",
)) as ProductHoldoutContract;
const sidecar = diagnosticReport.current_tree_unfrozen_diagnostic;
assert.equal(SEALED_MINUTE_HOLDOUT.sealed_benchmark_id, holdoutReport.benchmark_id);
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.validation.valid_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.validation.minimum_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, holdoutReport.release_metrics.top_1_rate.actual);
assert.equal(
SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate,
holdoutReport.release_metrics.confirmation_coverage_rate,
);
assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, productHoldout.valid_public_aa_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.status, productHoldout.status);
assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases >= SEALED_MINUTE_HOLDOUT.required_cases);
assert.equal(SEALED_MINUTE_HOLDOUT.status, "not_ready");
assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, 0);
assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, productHoldout.top_1_rate);
assert.notEqual(SEALED_MINUTE_HOLDOUT.top_1_rate, sidecar.metrics.top_1_rate);
assert.equal(
productHoldout.metrics_produced_by.implementation_sha256,
holdoutReport.frozen_scoring.implementation_sha256,
);
assert.equal(
productHoldout.metrics_produced_by.algorithm_version,
holdoutReport.frozen_scoring.algorithm_version,
);
assert.equal(productHoldout.metrics_produced_by.implementation_hash_matches_at_replay, true);
assert.equal(
productHoldout.metrics_produced_by.source_report,
"references/real_case_calibration/minute_rectification_holdout_v3_report.json",
);
// 原值: 历史 sidecar 哈希 / 新值: 当前冻结重跑哈希 / 原因: BUG-978 身份新鲜度,不改变确认门。
assert.equal(
productHoldout.current_tree_scorer.implementation_sha256,
currentTreeRerun.frozen_record.implementation_sha256,
);
assert.equal(
productHoldout.current_tree_scorer.source_report,
"docs/research/sealed_holdout_rerun_midnight_anchor_2026_09_21.json",
);
assert.equal(currentTreeRerun.official_valid_independent_blind, false);
assert.equal(productHoldout.current_tree_scorer.matches_metrics_scorer, false);
assert.equal(productHoldout.current_tree_scorer.official_eval_implementation_hash_matches, false);
assert.equal(productHoldout.current_tree_scorer.official_eval_trial_count, diagnosticReport.trial_count);
assert.equal(diagnosticReport.trial_count, 0);
assert.notEqual(
productHoldout.current_tree_scorer.implementation_sha256,
productHoldout.metrics_produced_by.implementation_sha256,
);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.is_blind_evaluation, false);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.results_already_seen, true);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.scorer_frozen, false);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.source_audit_passed, false);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.must_not_claim_as_release_metrics, true);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.must_not_use_for_tuning, true);
assert.equal(
productHoldout.current_tree_unfrozen_diagnostic.report_path,
"references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.top_1_rate, sidecar.metrics.top_1_rate);
assert.equal(productHoldout.current_tree_unfrozen_diagnostic.top_3_rate, sidecar.metrics.top_3_rate);
assert.equal(
productHoldout.current_tree_unfrozen_diagnostic.mean_absolute_minute_error,
sidecar.metrics.mean_absolute_minute_error,
);
const gate = buildConfirmationGate({
engineConfirmationAllowed: true,
candidates: UNIQUE_MINUTE,
decisionReceipt: {
gates: { exact_confirmation: { external_validation_status: "passed" } },
},
});
assert.equal(gate.confirmation_allowed, false);
assert.equal(blocker(gate, "public_aa_holdout").status, "not_ready");
});
test("VedAstro not_evaluated is not described as fail and still forbids confirmation", () => {
const gate = buildConfirmationGate({
engineConfirmationAllowed: true,
candidates: UNIQUE_MINUTE,
decisionReceipt: {
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
},
});
const vedastro = blocker(gate, "vedastro_minute_sensitive");
const adjacent = blocker(gate, "adjacent_minutes_indistinguishable");
const holdout = blocker(gate, "public_aa_holdout");
assert.equal(readVedastroMinuteSensitiveStatus({
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
}), "not_evaluated");
assert.equal(vedastro.status, "not_evaluated");
assert.notEqual(vedastro.status, "fail");
assert.notEqual(vedastro.status, "failed");
assert.match(vedastro.user_meaning, /尚未执行/);
assert.doesNotMatch(vedastro.user_meaning, /执行失败|校验失败/);
assert.equal(adjacent.status, "passed");
assert.equal(holdout.status, "not_ready");
assert.equal(gate.confirmation_allowed, false);
assert.equal(gate.unique_minute_path, "closed_at_representative");
assert.match(holdout.user_meaning, /未达发布门槛/);
assert.match(holdout.user_meaning, /不能确认唯一分钟/);
assert.equal(uniqueMinutePath(false), "closed_at_representative");
assert.equal(uniqueMinutePath(true), "awaiting_user_consent");
});
test("a 25-minute plateau stays confirmation-blocked as an indistinguishable range", () => {
const projection = latestResultToolProjection({
resultId: RESULT_ID,
candidates: PLATEAU,
selectionAllowed: true,
confirmationAllowed: true,
representativeTime: "04:45",
selectedTime: null,
selectionKind: null,
algorithmVersion: "rectification-v5",
}, collectDecision(PLATEAU));
const gate = projection.confirmation_gate as ReturnType<typeof buildConfirmationGate>;
assert.ok(Number(projection.engine_indistinguishable_width_minutes) >= 25);
assert.equal(projection.confirmation_allowed, false);
assert.equal(gate.confirmation_allowed, false);
assert.equal(blocker(gate, "adjacent_minutes_indistinguishable").status, "blocked");
assert.equal(
blocker(gate, "adjacent_minutes_indistinguishable").max_confirmation_width_minutes,
RECTIFICATION_POLICY.maxConfirmationWidthMinutes,
);
assert.match(blocker(gate, "adjacent_minutes_indistinguishable").user_meaning, /不可分区间|代表性候选/);
assert.equal(projection.session_outcome, "collect_evidence");
assert.equal(sessionOutcomeFromGate({
selectionAllowed: true,
confirmationAllowed: false,
}).kind, "collect_evidence");
assert.equal(sessionOutcomeFromGate({
selectionAllowed: true,
proposeAllowed: true,
confirmationAllowed: false,
interviewOpen: false,
}).kind, "adopt_representative");
assert.match(sessionOutcomeFromGate({
selectionAllowed: true,
proposeAllowed: true,
confirmationAllowed: false,
interviewOpen: false,
}).user_meaning, /可以采用代表性候选作当前排盘/);
assert.equal(sessionOutcomeFromGate({
selectionAllowed: true,
confirmationAllowed: false,
userStopped: true,
}).kind, "provisional_range_user_stopped");
// Previously asserted “不是最终校正结果”; task 6 distinguishes completed range delivery from unique-minute confirmation.
assert.match(sessionOutcomeFromGate({
selectionAllowed: true,
confirmationAllowed: false,
userStopped: true,
}).user_meaning, /交付已完成/);
assert.match(sessionOutcomeFromGate({
selectionAllowed: true,
confirmationAllowed: false,
userStopped: true,
}).user_meaning, /当前最优结果是候选时间段/);
assert.doesNotMatch(sessionOutcomeFromGate({
selectionAllowed: true,
confirmationAllowed: false,
userStopped: true,
}).user_meaning, /未完成|失败|遗憾/);
assert.doesNotMatch(sessionOutcomeFromGate({
selectionAllowed: true,
confirmationAllowed: false,
userStopped: true,
}).user_meaning, /已完成验证/);
});
test("holdout not_ready forbids unique-minute copy and still blocks confirm", async () => {
const projection = latestResultToolProjection({
resultId: RESULT_ID,
candidates: UNIQUE_MINUTE,
selectionAllowed: true,
confirmationAllowed: true,
representativeTime: "05:02",
selectedTime: null,
selectionKind: null,
algorithmVersion: "rectification-v5",
decisionReceipt: {
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
},
}, collectDecision(UNIQUE_MINUTE));
const serialized = JSON.stringify(projection);
assert.equal(projection.confirmation_allowed, false);
assert.equal(
(projection.confirmation_gate as ReturnType<typeof buildConfirmationGate>).unique_minute_path,
"closed_at_representative",
);
assert.equal(projection.unique_minute_claim, false);
assert.doesNotMatch(serialized, /±2 分钟/);
assert.match(serialized, /candidate_range_not_birth_time_truth/);
assert.match(serialized, /不是已确认唯一出生分钟|不得写成已确认唯一出生分钟/);
assert.doesNotMatch(serialized, /obama|barack|angelina|jolie/i);
assert.match(skill, /confirmation_gate/);
assert.match(skill, /not_evaluated/);
assert.match(skill, /session_outcome=adopt_representative/);
assert.doesNotMatch(skill, /本会话以代表性时间收口|本轮校正已收口/);
assert.match(skill, /unique_minute_path=closed_at_representative/);
assert.doesNotMatch(skill, /还不能确认唯一分钟/);
assert.match(candidateComparison, /confirmation_gate/);
assert.match(agentSource, /confirmation_allowed/);
assert.match(agentSource, /不得宣称唯一出生分钟/);
assert.doesNotMatch(agentSource, /±2 分钟/);
assert.equal(PUBLIC_RECTIFICATION_TOOLS.length, 14);
// 原值: "10.0.25"
// 原值: "10.0.26"
// 新值: "10.0.27"
// 原因: 出卡精度门槛 + 引导式补经历
// 原因: BUG-668 定向补事第四选项写进 Skill
// 原值: "10.0.28" / 新值: "10.0.29" / 原因: 年月阶段改口述并禁止追问本人
// 原值: "10.0.29" / 新值: "10.0.30" / 原因: D2 出卡句与 D4 卡上百分比规则写进 Skill(2026-09-26)
// 原值: "10.0.30" / 新值: "10.0.31" / 原因: 开场改大白话、题干带例子和示例回答写进 Skill OpeningPolicy(BUG-1049,2026-09-26)
// 原值: "10.0.31" / 新值: "10.0.32" / 原因: 训练门开后只问点选卡、交付正文去吻合率写进 Skill(BUG-1084/1085,2026-09-29)
assert.equal(RECTIFICATION_SKILL_VERSION, "10.0.32");
const accounting = fakeAccounting({
...receiptHandlers,
get_agentic_rectification_case_dossier: () => dossierFixture({
latestResult: {
...candidateSnapshotFixture({
confirmationAllowed: true,
selectionAllowed: true,
representativeTime: "05:02",
candidates: [
{ candidate_id: CANDIDATE_ID, rank: 1, time: "05:02", relative_support: 70, tied_minute_count: 1 },
{ candidate_id: SECOND_CANDIDATE_ID, rank: 2, time: "05:03", relative_support: 30, tied_minute_count: 2 },
],
}),
confirmation_allowed: true,
decision_receipt: {
gates: { exact_confirmation: { external_validation_status: "not_evaluated" } },
},
},
}),
confirm_agentic_rectification_candidate_for_case_v2: () => ({
success: true,
saved_time: "05:02",
status: "confirmed",
result_id: RESULT_ID,
case_status: "confirmed",
idempotent: false,
}),
});
const tools = createRectificationV9Tools({
userId: USER_ID,
caseId: CASE_ID,
turnId: TURN_ID,
accounting: accounting.client as never,
});
await assert.rejects(
(tools["rectification-confirm-birth-time"] as unknown as {
execute(input: unknown): Promise<unknown>;
}).execute({
caseId: CASE_ID,
resultId: RESULT_ID,
candidateId: CANDIDATE_ID,
consentQuote: "就用05:02",
}),
(error: unknown) => error instanceof RectificationToolServiceError && error.code === "confirmation_blocked",
);
assert.equal(
accounting.calls.some((call) => call.fn === "confirm_agentic_rectification_candidate_for_case_v2"),
false,
);
assert.equal(safeToolErrorCode(new RectificationToolServiceError("confirmation_blocked")), "confirmation_blocked");
});
test("decideFromDossier passes cluster span into the confirmation gate", () => {
const src = readFileSync(
new URL("../src/lib/rectification-agentic/v9/decision-from-dossier.ts", import.meta.url),
"utf8",
);
assert.match(src, /clusterStart: candidate\.clusterStart \?\? candidate\.cluster_start/);
assert.match(src, /clusterEnd: candidate\.clusterEnd \?\? candidate\.cluster_end/);
});
test("confirmation width uses cluster span so representative minutes cannot open the gate", () => {
const candidates = [
{ time: "04:53", rank: 1, tiedMinuteCount: 1, clusterStart: "04:50", clusterEnd: "04:55" },
{ time: "04:59", rank: 2, tiedMinuteCount: 1, clusterStart: "04:56", clusterEnd: "05:02" },
];
assert.equal(indistinguishableWidthMinutes(candidates), 13);
const gate = buildConfirmationGate({
engineConfirmationAllowed: true,
candidates,
decisionReceipt: { vedastro_minute_sensitive: { status: "passed" } },
});
assert.equal(gate.confirmation_allowed, false);
const adjacent = gate.blockers.find((item) => item.id === "adjacent_minutes_indistinguishable");
assert.equal(adjacent?.indistinguishable_width_minutes, 13);
});
test("collect_evidence keeps public can_adopt closed while selection_allowed follows the decision", () => {
const decision = decideRectification({
engineCeiling: OPEN_ENGINE_CAPABILITY_CEILING,
methodCoverageAll: false,
datedMethodCollectOpen: true,
datedEventCount: 4,
datedDomainCount: 3,
trainingGateOpen: true,
snapshotCurrent: true,
candidateScores: [
{ time: "05:06", score: 14 },
{ time: "04:53", score: 13 },
],
confirmationAllowed: false,
});
// 原值: sessionOutcome=collect_evidence,公开 can_adopt=false
// 新值: 训练门开、无带年月探针 → adopt_representative,公开 can_adopt=true
// 原因: BUG-651 带年月池空即交付;确认门仍 fail-closed
assert.equal(decision.sessionOutcome, "adopt_representative");
assert.equal(decision.canAdopt, true);
const fields = publicDecisionFields(decision);
assert.equal(fields.can_adopt, true);
assert.equal(fields.selection_allowed, decision.selectionAllowed);
});