diff --git a/frontend/tests/rectification-confirmation-gate.test.ts b/frontend/tests/rectification-confirmation-gate.test.ts index a118d80d..0c1430a2 100644 --- a/frontend/tests/rectification-confirmation-gate.test.ts +++ b/frontend/tests/rectification-confirmation-gate.test.ts @@ -33,11 +33,17 @@ import { const THIRD_CANDIDATE_ID = "88888888-8888-4888-8888-888888888883"; const holdoutReport = JSON.parse(readFileSync( - new URL("../../references/real_case_calibration/minute_rectification_holdout_v2_pilot_report.json", import.meta.url), + new URL("../../references/real_case_calibration/minute_rectification_holdout_v3_report.json", import.meta.url), "utf8", )) as { - case_gate: { valid_public_aa_cases: number; minimum_public_aa_cases: number }; - metrics: { top_1_rate: number; confirmation_coverage_rate: number }; + benchmark_id: string; + validation: { valid_public_aa_cases: number; minimum_public_aa_cases: number }; + release_metrics: { + top_1_rate: { actual: number }; + top_3_rate: { actual: number }; + mean_absolute_minute_error: { actual: number }; + confirmation_coverage_rate: number; + }; }; const skill = readFileSync( new URL("../../skills/jyotish-birth-time-rectification/SKILL.md", import.meta.url), @@ -80,19 +86,33 @@ function blocker( return found; } -test("sealed holdout aggregates match the v2 report and stay below the case gate", () => { +test("sealed holdout aggregates match the v3 report and still block confirmation", () => { const productHoldout = JSON.parse(readFileSync( new URL("../../references/rectification_sealed_holdout.v1.json", import.meta.url), "utf8", )) as typeof SEALED_MINUTE_HOLDOUT; - assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.case_gate.valid_public_aa_cases); - assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.case_gate.minimum_public_aa_cases); - assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, holdoutReport.metrics.top_1_rate); - assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, holdoutReport.metrics.confirmation_coverage_rate); + assert.equal(SEALED_MINUTE_HOLDOUT.sealed_benchmark_id, holdoutReport.benchmark_id); + assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.validation.valid_public_aa_cases); + assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.validation.minimum_public_aa_cases); + assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, holdoutReport.release_metrics.top_1_rate.actual); + assert.equal( + SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, + holdoutReport.release_metrics.confirmation_coverage_rate, + ); assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, productHoldout.valid_public_aa_cases); assert.equal(SEALED_MINUTE_HOLDOUT.status, productHoldout.status); - assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases < SEALED_MINUTE_HOLDOUT.required_cases); + assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases >= SEALED_MINUTE_HOLDOUT.required_cases); assert.equal(SEALED_MINUTE_HOLDOUT.status, "not_ready"); + assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, 0); + const gate = buildConfirmationGate({ + engineConfirmationAllowed: true, + candidates: UNIQUE_MINUTE, + decisionReceipt: { + gates: { exact_confirmation: { external_validation_status: "passed" } }, + }, + }); + assert.equal(gate.confirmation_allowed, false); + assert.equal(blocker(gate, "public_aa_holdout").status, "not_ready"); }); test("VedAstro not_evaluated is not described as fail and still forbids confirmation", () => { diff --git a/progress.md b/progress.md index b13aecae..86df79ff 100644 --- a/progress.md +++ b/progress.md @@ -1023,3 +1023,10 @@ - main/staging 同步后,migration run 30324787560 成功:应用 `20260727000000_admin_viewer_identity.sql` 与 `20260727010000_refine_admin_redemption_audit.sql`,并自动 dispatch deploy run 30324940917;该 deploy 成功,health exact SHA 为 `f950bfd...`。 - 线上 `/admin` 冒烟发现反向代理 Location 错用容器 URL `https://0.0.0.0:3000/admin/codes`;改为相对 `/admin/codes`,形成最终 SHA `5206fb2323c87c877e5d361dc64c77d0497e14dd`,同步 fast-forward 到 main/staging。 - 最终 SHA gate run 30325219367 success,自动 deploy run 30325788303 success;health 返回同一 SHA,`/admin` 307 到相对 `/admin/codes`,Refine shell 200,5 个匿名资源与匿名写请求均 401。未触发 production deploy。 + +## 2026-08-29 - C1: 密封 holdout 合同回写 v3 已评估结果,不是缺案例 + +- `references/rectification_sealed_holdout.v1.json` 此前仍是 v2 Obama 单例试点:`valid_public_aa_cases=1` / `required_cases=20` / `top_1_rate=0`。该 1/20 从未在 v3 的 20 例盲测回放后被覆盖,会让读者误以为“还缺案例”。 +- 真实状态来自 `minute_rectification_holdout_v3.json` 与 `minute_rectification_holdout_v3_report.json`(evaluated_on 2026-07-21):20 个公开 AA 案例已存在;source audit 失败(`invalidated_after_replay`);`top_1_rate=0.15`、`top_3_rate=0.25`、`mean_absolute_minute_error=6.95` 均未过门;`confirmation_coverage_rate=0.0`。 +- 合同六键已改为 v3 数字,`status` 仍为 `not_ready`。确认门保持关闭:案例数已满,但 status 非 ready 且 confirmation_coverage_rate 为 0。 +- 防复发:holdout 集一旦回放,必须把真实 status / 案例数 / 指标写回 `rectification_sealed_holdout.v1.json`;不得把合同留在上一版的计数上。 diff --git a/references/rectification_sealed_holdout.v1.json b/references/rectification_sealed_holdout.v1.json index 3a07d399..b2eecb43 100644 --- a/references/rectification_sealed_holdout.v1.json +++ b/references/rectification_sealed_holdout.v1.json @@ -1,8 +1,17 @@ { - "sealed_benchmark_id": "minute_rectification_holdout_v2", + "sealed_benchmark_id": "minute_rectification_fact_ranker_v4_holdout_v3", "status": "not_ready", - "valid_public_aa_cases": 1, + "valid_public_aa_cases": 20, "required_cases": 20, - "top_1_rate": 0, - "confirmation_coverage_rate": 0 + "top_1_rate": 0.15, + "confirmation_coverage_rate": 0.0, + "previous_pilot_id": "minute_rectification_holdout_v2", + "dataset_path": "references/real_case_calibration/minute_rectification_holdout_v3.json", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json", + "source_audit_status": "invalidated_after_replay", + "evaluated_on": "2026-07-21", + "report_status": "invalidated_after_source_audit", + "trial_count": 20, + "top_3_rate": 0.25, + "mean_absolute_minute_error": 6.95 } diff --git a/tests/test_rectification_confirmation_and.py b/tests/test_rectification_confirmation_and.py index 39501ed4..94d9e839 100644 --- a/tests/test_rectification_confirmation_and.py +++ b/tests/test_rectification_confirmation_and.py @@ -10,13 +10,15 @@ from scripts.rectification.decision_policy import ( indistinguishable_width_minutes, ) from scripts.rectification.sealed_holdout import ( - PILOT_REPORT_PATH, holdout_passed, load_sealed_minute_holdout, ) ROOT = Path(__file__).resolve().parents[1] +V3_REPORT_PATH = ( + ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json" +) def _request(*, extra_events=()): @@ -78,16 +80,18 @@ def _rows(): ] -def test_sealed_holdout_contract_matches_v2_pilot_and_stays_closed() -> None: +def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None: holdout = load_sealed_minute_holdout() - report = json.loads(PILOT_REPORT_PATH.read_text(encoding="utf-8")) + report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8")) product = json.loads((ROOT / "references" / "rectification_sealed_holdout.v1.json").read_text(encoding="utf-8")) assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"] - assert holdout["valid_public_aa_cases"] == report["case_gate"]["valid_public_aa_cases"] - assert holdout["required_cases"] == report["case_gate"]["minimum_public_aa_cases"] - assert holdout["top_1_rate"] == report["metrics"]["top_1_rate"] - assert holdout["confirmation_coverage_rate"] == report["metrics"]["confirmation_coverage_rate"] + assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"] + assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"] + assert holdout["top_1_rate"] == report["release_metrics"]["top_1_rate"]["actual"] + assert holdout["confirmation_coverage_rate"] == report["release_metrics"]["confirmation_coverage_rate"] assert holdout["status"] == "not_ready" + assert holdout["valid_public_aa_cases"] >= holdout["required_cases"] + assert holdout["confirmation_coverage_rate"] == 0.0 assert holdout_passed(holdout) is False