docs(rectification): bind sealed holdout metrics to the frozen scorer identity

The published six keys stay on the 2026-07-21 freeze. Current-tree sidecar numbers remain diagnostic and unlabeled as release metrics. Remove the unused v2 PILOT_REPORT_PATH.

Co-authored-by: Cursor <cursoragent@cursor.com>

docs/BUG_HISTORY.md

@@ -6477,6 +6477,22 @@
 - 复发自：无
 - 修复版本：待发布
 
+## BUG-427 | 密封 holdout 合同把冻结 scorer 的成绩读成当前引擎成绩
+
+- 状态：resolved
+- 首次发现：2026-08-29
+- 最近更新：2026-08-29
+- 影响面：`references/rectification_sealed_holdout.v1.json`、确认门 holdout 合同
+- 用户现象：合同已回写 v3 的 `top_1_rate=0.15`，但当前树 sidecar 诊断是 0.45。读者会把冻结 scorer 的成绩当成当前引擎发布指标。
+- 触发条件：读密封 holdout 合同、对照当前树 `implementation_sha256`，或把 sidecar 非盲诊断数字当成发布门槛。
+- 根因：合同只写了指标数字，没有绑定产生这些数字的 `implementation_sha256`，也没有标明当前树是否仍匹配、更新的非盲诊断在哪。
+- 修复：补充 `metrics_produced_by` / `current_tree_scorer` / `current_tree_unfrozen_diagnostic`。六个既有键的键名、类型、数值不动。sidecar 0.45 只记在带限定条件的诊断对象里。删除未引用的 v2 `PILOT_REPORT_PATH`。确认门保持关闭。
+- 验证：`holdout_passed()` 为 False；`test_rectification_confirmation_and` 与 `rectification-confirmation-gate` 锁定身份哈希来自报告文件、sidecar 不得写入六个键、门仍关闭。
+- 防复发：评测指标必须与产生它的 `implementation_sha256` 绑定记录。当前树哈希漂移时不得把合同数字读成当前引擎成绩。不得把非盲、结果已被看过、scorer 未冻结、source audit 未过的 sidecar 数字写入发布六键。
+- 相关记录：无
+- 复发自：无
+- 修复版本：待发布
+
 ## BUG-410 | 训练已齐仍因家人/职业方法层停在采集，Agent 只确认后截断
 
 - 状态：resolved

frontend/tests/rectification-confirmation-gate.test.ts

@@ -37,7 +37,13 @@ const holdoutReport = JSON.parse(readFileSync(
   "utf8",
 )) as {
   benchmark_id: string;
+  evaluated_on: string;
   validation: { valid_public_aa_cases: number; minimum_public_aa_cases: number };
+  frozen_scoring: {
+    algorithm_version: string;
+    implementation_hash_matches: boolean;
+    implementation_sha256: string;
+  };
   release_metrics: {
     top_1_rate: { actual: number };
     top_3_rate: { actual: number };
@@ -45,6 +51,67 @@ const holdoutReport = JSON.parse(readFileSync(
     confirmation_coverage_rate: number;
   };
 };
+const diagnosticReport = JSON.parse(readFileSync(
+  new URL(
+    "../../references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
+    import.meta.url,
+  ),
+  "utf8",
+)) as {
+  trial_count: number;
+  is_blind_evaluation: boolean;
+  must_not_claim_release_gate_passed: boolean;
+  must_not_use_for_tuning: boolean;
+  frozen_scoring: {
+    implementation_hash_matches: boolean;
+    actual_sha256: string;
+  };
+  current_tree_unfrozen_diagnostic: {
+    actual_sha256: string;
+    trial_count: number;
+    metrics: {
+      top_1_rate: number;
+      top_3_rate: number;
+      mean_absolute_minute_error: number;
+      confirmation_coverage_rate: number;
+    };
+    metric_gates_passed: boolean;
+    verified_minute_claim_allowed: boolean;
+  };
+};
+type ProductHoldoutContract = typeof SEALED_MINUTE_HOLDOUT & {
+  metrics_produced_by: {
+    implementation_sha256: string;
+    algorithm_version: string;
+    implementation_hash_matches_at_replay: boolean;
+    source_report: string;
+    evaluated_on: string;
+  };
+  current_tree_scorer: {
+    implementation_sha256: string;
+    matches_metrics_scorer: boolean;
+    official_eval_implementation_hash_matches: boolean;
+    official_eval_trial_count: number;
+    source_report: string;
+  };
+  current_tree_unfrozen_diagnostic: {
+    is_blind_evaluation: boolean;
+    results_already_seen: boolean;
+    scorer_frozen: boolean;
+    source_audit_passed: boolean;
+    must_not_claim_as_release_metrics: boolean;
+    must_not_use_for_tuning: boolean;
+    report_path: string;
+    implementation_sha256: string;
+    trial_count: number;
+    top_1_rate: number;
+    top_3_rate: number;
+    mean_absolute_minute_error: number;
+    confirmation_coverage_rate: number;
+    metric_gates_passed: boolean;
+    verified_minute_claim_allowed: boolean;
+  };
+};
 const skill = readFileSync(
   new URL("../../skills/jyotish-birth-time-rectification/SKILL.md", import.meta.url),
   "utf8",
@@ -90,7 +157,8 @@ test("sealed holdout aggregates match the v3 report and still block confirmation
   const productHoldout = JSON.parse(readFileSync(
     new URL("../../references/rectification_sealed_holdout.v1.json", import.meta.url),
     "utf8",
-  )) as typeof SEALED_MINUTE_HOLDOUT;
+  )) as ProductHoldoutContract;
+  const sidecar = diagnosticReport.current_tree_unfrozen_diagnostic;
   assert.equal(SEALED_MINUTE_HOLDOUT.sealed_benchmark_id, holdoutReport.benchmark_id);
   assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.validation.valid_public_aa_cases);
   assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.validation.minimum_public_aa_cases);
@@ -104,6 +172,49 @@ test("sealed holdout aggregates match the v3 report and still block confirmation
   assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases >= SEALED_MINUTE_HOLDOUT.required_cases);
   assert.equal(SEALED_MINUTE_HOLDOUT.status, "not_ready");
   assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, 0);
+  assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, productHoldout.top_1_rate);
+  assert.notEqual(SEALED_MINUTE_HOLDOUT.top_1_rate, sidecar.metrics.top_1_rate);
+  assert.equal(
+    productHoldout.metrics_produced_by.implementation_sha256,
+    holdoutReport.frozen_scoring.implementation_sha256,
+  );
+  assert.equal(
+    productHoldout.metrics_produced_by.algorithm_version,
+    holdoutReport.frozen_scoring.algorithm_version,
+  );
+  assert.equal(productHoldout.metrics_produced_by.implementation_hash_matches_at_replay, true);
+  assert.equal(
+    productHoldout.metrics_produced_by.source_report,
+    "references/real_case_calibration/minute_rectification_holdout_v3_report.json",
+  );
+  assert.equal(
+    productHoldout.current_tree_scorer.implementation_sha256,
+    diagnosticReport.frozen_scoring.actual_sha256,
+  );
+  assert.equal(productHoldout.current_tree_scorer.matches_metrics_scorer, false);
+  assert.equal(productHoldout.current_tree_scorer.official_eval_implementation_hash_matches, false);
+  assert.equal(productHoldout.current_tree_scorer.official_eval_trial_count, diagnosticReport.trial_count);
+  assert.equal(diagnosticReport.trial_count, 0);
+  assert.notEqual(
+    productHoldout.current_tree_scorer.implementation_sha256,
+    productHoldout.metrics_produced_by.implementation_sha256,
+  );
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.is_blind_evaluation, false);
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.results_already_seen, true);
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.scorer_frozen, false);
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.source_audit_passed, false);
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.must_not_claim_as_release_metrics, true);
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.must_not_use_for_tuning, true);
+  assert.equal(
+    productHoldout.current_tree_unfrozen_diagnostic.report_path,
+    "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
+  );
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.top_1_rate, sidecar.metrics.top_1_rate);
+  assert.equal(productHoldout.current_tree_unfrozen_diagnostic.top_3_rate, sidecar.metrics.top_3_rate);
+  assert.equal(
+    productHoldout.current_tree_unfrozen_diagnostic.mean_absolute_minute_error,
+    sidecar.metrics.mean_absolute_minute_error,
+  );
   const gate = buildConfirmationGate({
     engineConfirmationAllowed: true,
     candidates: UNIQUE_MINUTE,

progress.md

@@ -1038,3 +1038,11 @@
 - v3 `source_audit_status` 仍为 `invalidated_after_replay`。未改已发布的 `minute_rectification_holdout_v3_report.json` 数字。
 - v4 intake `day_precision_events_per_case=3` 未放宽：20 例中仅 4 例入列（`barack_obama_1961_aa_v4_holdout`、`paul_ryan_1970_aa_v4_holdout`、`sean_lennon_1975_aa_v4_holdout`、`kurt_cobain_1967_aa_v4_holdout`）。其余 16 例在补齐 review safeguards 后仍为 `insufficient_day_precision_events`；Robbins/Takamoto 校正后仍不够 3 个 day 事件。intake 保持 `production_tuning_allowed=false`、`verified_minute_claim_allowed=false`。
 - 诊断回放文件：`minute_rectification_holdout_v3_post_audit_diagnostic.json` / `_report.json`。这是 post-audit diagnostic rerun，不是盲测，不可声称 release gate 通过，不可用于调参。官方 eval runner 因当前树 scoring 文件哈希已偏离 2026-07-21 v3 freeze 而 `implementation_hash_matches=false`、trial_count=0；本任务未改 `frozen_scoring.files`。确认门仍关闭。
+
+## 2026-08-29 - D1: 合同指标绑定冻结 scorer 身份；D3: 删除失效 PILOT_REPORT_PATH
+
+- 六个既有键未改：`status=not_ready`、`valid_public_aa_cases=20`、`required_cases=20`、`top_1_rate=0.15`、`confirmation_coverage_rate=0.0`。
+- 新增 `metrics_produced_by.implementation_sha256=f41c298…`（2026-07-21 冻结 scorer，来自 v3 report）；`current_tree_scorer.implementation_sha256=99730c8…` 且 `matches_metrics_scorer=false`、官方 runner `trial_count=0`（来自 post-audit diagnostic report）。
+- sidecar `current_tree_unfrozen_diagnostic` 记录当前树非盲诊断 `top_1_rate=0.45` / `top_3_rate=0.50` / `mean_absolute_minute_error=5.8`，并标明非盲、结果已被看过、scorer 未冻结、source audit 未过，不得当作发布指标。
+- 删除 `scripts/rectification/sealed_holdout.py` 未引用的 `PILOT_REPORT_PATH`（仍指向已被 v3 取代的 v2 试点报告）。
+- 防复发：评测指标必须与产生它的 `implementation_sha256` 绑定记录。

references/rectification_sealed_holdout.v1.json

@@ -13,5 +13,36 @@
   "report_status": "invalidated_after_source_audit",
   "trial_count": 20,
   "top_3_rate": 0.25,
-  "mean_absolute_minute_error": 6.95
+  "mean_absolute_minute_error": 6.95,
+  "metrics_produced_by": {
+    "implementation_sha256": "f41c298dd6cdcebe7a93e632f7191954be7987f6012ca2a34cecb6e447fbf196",
+    "algorithm_version": "birth-time-event-fact-ranker-v4-shadow",
+    "implementation_hash_matches_at_replay": true,
+    "source_report": "references/real_case_calibration/minute_rectification_holdout_v3_report.json",
+    "evaluated_on": "2026-07-21"
+  },
+  "current_tree_scorer": {
+    "implementation_sha256": "99730c84c6434f52669a05e4d9a4a87df0a218e3237b5436b933252f2028384e",
+    "matches_metrics_scorer": false,
+    "official_eval_implementation_hash_matches": false,
+    "official_eval_trial_count": 0,
+    "source_report": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json"
+  },
+  "current_tree_unfrozen_diagnostic": {
+    "is_blind_evaluation": false,
+    "results_already_seen": true,
+    "scorer_frozen": false,
+    "source_audit_passed": false,
+    "must_not_claim_as_release_metrics": true,
+    "must_not_use_for_tuning": true,
+    "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
+    "implementation_sha256": "99730c84c6434f52669a05e4d9a4a87df0a218e3237b5436b933252f2028384e",
+    "trial_count": 20,
+    "top_1_rate": 0.45,
+    "top_3_rate": 0.5,
+    "mean_absolute_minute_error": 5.8,
+    "confirmation_coverage_rate": 0.0,
+    "metric_gates_passed": false,
+    "verified_minute_claim_allowed": false
+  }
 }

scripts/rectification/sealed_holdout.py

@@ -13,9 +13,6 @@ from typing import Any, Final
 
 ROOT = Path(__file__).resolve().parents[2]
 HOLDOUT_PATH: Final = ROOT / "references" / "rectification_sealed_holdout.v1.json"
-PILOT_REPORT_PATH: Final = (
-    ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v2_pilot_report.json"
-)
 
 
 def load_sealed_minute_holdout() -> dict[str, Any]:

tests/test_rectification_confirmation_and.py

@@ -19,6 +19,11 @@ ROOT = Path(__file__).resolve().parents[1]
 V3_REPORT_PATH = (
     ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json"
 )
+DIAGNOSTIC_REPORT_PATH = (
+    ROOT / "references" / "real_case_calibration"
+    / "minute_rectification_holdout_v3_post_audit_diagnostic_report.json"
+)
+PRODUCT_HOLDOUT_PATH = ROOT / "references" / "rectification_sealed_holdout.v1.json"
 
 
 def _request(*, extra_events=()):
@@ -83,7 +88,12 @@ def _rows():
 def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
     holdout = load_sealed_minute_holdout()
     report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8"))
-    product = json.loads((ROOT / "references" / "rectification_sealed_holdout.v1.json").read_text(encoding="utf-8"))
+    diagnostic = json.loads(DIAGNOSTIC_REPORT_PATH.read_text(encoding="utf-8"))
+    product = json.loads(PRODUCT_HOLDOUT_PATH.read_text(encoding="utf-8"))
+    sidecar = diagnostic["current_tree_unfrozen_diagnostic"]
+    produced_by = product["metrics_produced_by"]
+    current_tree = product["current_tree_scorer"]
+    unfrozen = product["current_tree_unfrozen_diagnostic"]
     assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"]
     assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"]
     assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"]
@@ -93,6 +103,42 @@ def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
     assert holdout["valid_public_aa_cases"] >= holdout["required_cases"]
     assert holdout["confirmation_coverage_rate"] == 0.0
     assert holdout_passed(holdout) is False
+    assert set(holdout) == {
+        "sealed_benchmark_id",
+        "status",
+        "valid_public_aa_cases",
+        "required_cases",
+        "top_1_rate",
+        "confirmation_coverage_rate",
+    }
+    assert produced_by["implementation_sha256"] == report["frozen_scoring"]["implementation_sha256"]
+    assert produced_by["algorithm_version"] == report["frozen_scoring"]["algorithm_version"]
+    assert produced_by["implementation_hash_matches_at_replay"] is True
+    assert produced_by["source_report"] == "references/real_case_calibration/minute_rectification_holdout_v3_report.json"
+    assert current_tree["implementation_sha256"] == diagnostic["frozen_scoring"]["actual_sha256"]
+    assert current_tree["matches_metrics_scorer"] is False
+    assert current_tree["official_eval_implementation_hash_matches"] is False
+    assert current_tree["official_eval_trial_count"] == diagnostic["trial_count"] == 0
+    assert current_tree["implementation_sha256"] != produced_by["implementation_sha256"]
+    assert unfrozen["is_blind_evaluation"] is False
+    assert unfrozen["is_blind_evaluation"] is diagnostic["is_blind_evaluation"]
+    assert unfrozen["results_already_seen"] is True
+    assert unfrozen["scorer_frozen"] is False
+    assert unfrozen["scorer_frozen"] is diagnostic["frozen_scoring"]["implementation_hash_matches"]
+    assert unfrozen["source_audit_passed"] is False
+    assert unfrozen["source_audit_passed"] is report["post_replay_source_audit"]["passed"]
+    assert unfrozen["must_not_claim_as_release_metrics"] is True
+    assert unfrozen["must_not_use_for_tuning"] is True
+    assert unfrozen["report_path"] == (
+        "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json"
+    )
+    assert unfrozen["implementation_sha256"] == sidecar["actual_sha256"]
+    assert unfrozen["trial_count"] == sidecar["trial_count"]
+    assert unfrozen["top_1_rate"] == sidecar["metrics"]["top_1_rate"]
+    assert unfrozen["top_3_rate"] == sidecar["metrics"]["top_3_rate"]
+    assert unfrozen["mean_absolute_minute_error"] == sidecar["metrics"]["mean_absolute_minute_error"]
+    assert product["top_1_rate"] != unfrozen["top_1_rate"]
+    assert holdout["top_1_rate"] != unfrozen["top_1_rate"]
 
 
 def test_vedastro_pass_rewrites_audit_but_holdout_still_blocks(monkeypatch) -> None: