diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 674204f7..547e2002 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -6477,6 +6477,22 @@ - 复发自:无 - 修复版本:待发布 +## BUG-427 | 密封 holdout 合同把冻结 scorer 的成绩读成当前引擎成绩 + +- 状态:resolved +- 首次发现:2026-08-29 +- 最近更新:2026-08-29 +- 影响面:`references/rectification_sealed_holdout.v1.json`、确认门 holdout 合同 +- 用户现象:合同已回写 v3 的 `top_1_rate=0.15`,但当前树 sidecar 诊断是 0.45。读者会把冻结 scorer 的成绩当成当前引擎发布指标。 +- 触发条件:读密封 holdout 合同、对照当前树 `implementation_sha256`,或把 sidecar 非盲诊断数字当成发布门槛。 +- 根因:合同只写了指标数字,没有绑定产生这些数字的 `implementation_sha256`,也没有标明当前树是否仍匹配、更新的非盲诊断在哪。 +- 修复:补充 `metrics_produced_by` / `current_tree_scorer` / `current_tree_unfrozen_diagnostic`。六个既有键的键名、类型、数值不动。sidecar 0.45 只记在带限定条件的诊断对象里。删除未引用的 v2 `PILOT_REPORT_PATH`。确认门保持关闭。 +- 验证:`holdout_passed()` 为 False;`test_rectification_confirmation_and` 与 `rectification-confirmation-gate` 锁定身份哈希来自报告文件、sidecar 不得写入六个键、门仍关闭。 +- 防复发:评测指标必须与产生它的 `implementation_sha256` 绑定记录。当前树哈希漂移时不得把合同数字读成当前引擎成绩。不得把非盲、结果已被看过、scorer 未冻结、source audit 未过的 sidecar 数字写入发布六键。 +- 相关记录:无 +- 复发自:无 +- 修复版本:待发布 + ## BUG-410 | 训练已齐仍因家人/职业方法层停在采集,Agent 只确认后截断 - 状态:resolved diff --git a/frontend/tests/rectification-confirmation-gate.test.ts b/frontend/tests/rectification-confirmation-gate.test.ts index 0c1430a2..a90962bc 100644 --- a/frontend/tests/rectification-confirmation-gate.test.ts +++ b/frontend/tests/rectification-confirmation-gate.test.ts @@ -37,7 +37,13 @@ const holdoutReport = JSON.parse(readFileSync( "utf8", )) as { benchmark_id: string; + evaluated_on: string; validation: { valid_public_aa_cases: number; minimum_public_aa_cases: number }; + frozen_scoring: { + algorithm_version: string; + implementation_hash_matches: boolean; + implementation_sha256: string; + }; release_metrics: { top_1_rate: { actual: number }; top_3_rate: { actual: number }; @@ -45,6 +51,67 @@ const holdoutReport = JSON.parse(readFileSync( confirmation_coverage_rate: number; }; }; +const diagnosticReport = JSON.parse(readFileSync( + new URL( + "../../references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json", + import.meta.url, + ), + "utf8", +)) as { + trial_count: number; + is_blind_evaluation: boolean; + must_not_claim_release_gate_passed: boolean; + must_not_use_for_tuning: boolean; + frozen_scoring: { + implementation_hash_matches: boolean; + actual_sha256: string; + }; + current_tree_unfrozen_diagnostic: { + actual_sha256: string; + trial_count: number; + metrics: { + top_1_rate: number; + top_3_rate: number; + mean_absolute_minute_error: number; + confirmation_coverage_rate: number; + }; + metric_gates_passed: boolean; + verified_minute_claim_allowed: boolean; + }; +}; +type ProductHoldoutContract = typeof SEALED_MINUTE_HOLDOUT & { + metrics_produced_by: { + implementation_sha256: string; + algorithm_version: string; + implementation_hash_matches_at_replay: boolean; + source_report: string; + evaluated_on: string; + }; + current_tree_scorer: { + implementation_sha256: string; + matches_metrics_scorer: boolean; + official_eval_implementation_hash_matches: boolean; + official_eval_trial_count: number; + source_report: string; + }; + current_tree_unfrozen_diagnostic: { + is_blind_evaluation: boolean; + results_already_seen: boolean; + scorer_frozen: boolean; + source_audit_passed: boolean; + must_not_claim_as_release_metrics: boolean; + must_not_use_for_tuning: boolean; + report_path: string; + implementation_sha256: string; + trial_count: number; + top_1_rate: number; + top_3_rate: number; + mean_absolute_minute_error: number; + confirmation_coverage_rate: number; + metric_gates_passed: boolean; + verified_minute_claim_allowed: boolean; + }; +}; const skill = readFileSync( new URL("../../skills/jyotish-birth-time-rectification/SKILL.md", import.meta.url), "utf8", @@ -90,7 +157,8 @@ test("sealed holdout aggregates match the v3 report and still block confirmation const productHoldout = JSON.parse(readFileSync( new URL("../../references/rectification_sealed_holdout.v1.json", import.meta.url), "utf8", - )) as typeof SEALED_MINUTE_HOLDOUT; + )) as ProductHoldoutContract; + const sidecar = diagnosticReport.current_tree_unfrozen_diagnostic; assert.equal(SEALED_MINUTE_HOLDOUT.sealed_benchmark_id, holdoutReport.benchmark_id); assert.equal(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases, holdoutReport.validation.valid_public_aa_cases); assert.equal(SEALED_MINUTE_HOLDOUT.required_cases, holdoutReport.validation.minimum_public_aa_cases); @@ -104,6 +172,49 @@ test("sealed holdout aggregates match the v3 report and still block confirmation assert.ok(SEALED_MINUTE_HOLDOUT.valid_public_aa_cases >= SEALED_MINUTE_HOLDOUT.required_cases); assert.equal(SEALED_MINUTE_HOLDOUT.status, "not_ready"); assert.equal(SEALED_MINUTE_HOLDOUT.confirmation_coverage_rate, 0); + assert.equal(SEALED_MINUTE_HOLDOUT.top_1_rate, productHoldout.top_1_rate); + assert.notEqual(SEALED_MINUTE_HOLDOUT.top_1_rate, sidecar.metrics.top_1_rate); + assert.equal( + productHoldout.metrics_produced_by.implementation_sha256, + holdoutReport.frozen_scoring.implementation_sha256, + ); + assert.equal( + productHoldout.metrics_produced_by.algorithm_version, + holdoutReport.frozen_scoring.algorithm_version, + ); + assert.equal(productHoldout.metrics_produced_by.implementation_hash_matches_at_replay, true); + assert.equal( + productHoldout.metrics_produced_by.source_report, + "references/real_case_calibration/minute_rectification_holdout_v3_report.json", + ); + assert.equal( + productHoldout.current_tree_scorer.implementation_sha256, + diagnosticReport.frozen_scoring.actual_sha256, + ); + assert.equal(productHoldout.current_tree_scorer.matches_metrics_scorer, false); + assert.equal(productHoldout.current_tree_scorer.official_eval_implementation_hash_matches, false); + assert.equal(productHoldout.current_tree_scorer.official_eval_trial_count, diagnosticReport.trial_count); + assert.equal(diagnosticReport.trial_count, 0); + assert.notEqual( + productHoldout.current_tree_scorer.implementation_sha256, + productHoldout.metrics_produced_by.implementation_sha256, + ); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.is_blind_evaluation, false); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.results_already_seen, true); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.scorer_frozen, false); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.source_audit_passed, false); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.must_not_claim_as_release_metrics, true); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.must_not_use_for_tuning, true); + assert.equal( + productHoldout.current_tree_unfrozen_diagnostic.report_path, + "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json", + ); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.top_1_rate, sidecar.metrics.top_1_rate); + assert.equal(productHoldout.current_tree_unfrozen_diagnostic.top_3_rate, sidecar.metrics.top_3_rate); + assert.equal( + productHoldout.current_tree_unfrozen_diagnostic.mean_absolute_minute_error, + sidecar.metrics.mean_absolute_minute_error, + ); const gate = buildConfirmationGate({ engineConfirmationAllowed: true, candidates: UNIQUE_MINUTE, diff --git a/progress.md b/progress.md index 270529d6..95d43b2d 100644 --- a/progress.md +++ b/progress.md @@ -1038,3 +1038,11 @@ - v3 `source_audit_status` 仍为 `invalidated_after_replay`。未改已发布的 `minute_rectification_holdout_v3_report.json` 数字。 - v4 intake `day_precision_events_per_case=3` 未放宽:20 例中仅 4 例入列(`barack_obama_1961_aa_v4_holdout`、`paul_ryan_1970_aa_v4_holdout`、`sean_lennon_1975_aa_v4_holdout`、`kurt_cobain_1967_aa_v4_holdout`)。其余 16 例在补齐 review safeguards 后仍为 `insufficient_day_precision_events`;Robbins/Takamoto 校正后仍不够 3 个 day 事件。intake 保持 `production_tuning_allowed=false`、`verified_minute_claim_allowed=false`。 - 诊断回放文件:`minute_rectification_holdout_v3_post_audit_diagnostic.json` / `_report.json`。这是 post-audit diagnostic rerun,不是盲测,不可声称 release gate 通过,不可用于调参。官方 eval runner 因当前树 scoring 文件哈希已偏离 2026-07-21 v3 freeze 而 `implementation_hash_matches=false`、trial_count=0;本任务未改 `frozen_scoring.files`。确认门仍关闭。 + +## 2026-08-29 - D1: 合同指标绑定冻结 scorer 身份;D3: 删除失效 PILOT_REPORT_PATH + +- 六个既有键未改:`status=not_ready`、`valid_public_aa_cases=20`、`required_cases=20`、`top_1_rate=0.15`、`confirmation_coverage_rate=0.0`。 +- 新增 `metrics_produced_by.implementation_sha256=f41c298…`(2026-07-21 冻结 scorer,来自 v3 report);`current_tree_scorer.implementation_sha256=99730c8…` 且 `matches_metrics_scorer=false`、官方 runner `trial_count=0`(来自 post-audit diagnostic report)。 +- sidecar `current_tree_unfrozen_diagnostic` 记录当前树非盲诊断 `top_1_rate=0.45` / `top_3_rate=0.50` / `mean_absolute_minute_error=5.8`,并标明非盲、结果已被看过、scorer 未冻结、source audit 未过,不得当作发布指标。 +- 删除 `scripts/rectification/sealed_holdout.py` 未引用的 `PILOT_REPORT_PATH`(仍指向已被 v3 取代的 v2 试点报告)。 +- 防复发:评测指标必须与产生它的 `implementation_sha256` 绑定记录。 diff --git a/references/rectification_sealed_holdout.v1.json b/references/rectification_sealed_holdout.v1.json index b2eecb43..aecebb18 100644 --- a/references/rectification_sealed_holdout.v1.json +++ b/references/rectification_sealed_holdout.v1.json @@ -13,5 +13,36 @@ "report_status": "invalidated_after_source_audit", "trial_count": 20, "top_3_rate": 0.25, - "mean_absolute_minute_error": 6.95 + "mean_absolute_minute_error": 6.95, + "metrics_produced_by": { + "implementation_sha256": "f41c298dd6cdcebe7a93e632f7191954be7987f6012ca2a34cecb6e447fbf196", + "algorithm_version": "birth-time-event-fact-ranker-v4-shadow", + "implementation_hash_matches_at_replay": true, + "source_report": "references/real_case_calibration/minute_rectification_holdout_v3_report.json", + "evaluated_on": "2026-07-21" + }, + "current_tree_scorer": { + "implementation_sha256": "99730c84c6434f52669a05e4d9a4a87df0a218e3237b5436b933252f2028384e", + "matches_metrics_scorer": false, + "official_eval_implementation_hash_matches": false, + "official_eval_trial_count": 0, + "source_report": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" + }, + "current_tree_unfrozen_diagnostic": { + "is_blind_evaluation": false, + "results_already_seen": true, + "scorer_frozen": false, + "source_audit_passed": false, + "must_not_claim_as_release_metrics": true, + "must_not_use_for_tuning": true, + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json", + "implementation_sha256": "99730c84c6434f52669a05e4d9a4a87df0a218e3237b5436b933252f2028384e", + "trial_count": 20, + "top_1_rate": 0.45, + "top_3_rate": 0.5, + "mean_absolute_minute_error": 5.8, + "confirmation_coverage_rate": 0.0, + "metric_gates_passed": false, + "verified_minute_claim_allowed": false + } } diff --git a/scripts/rectification/sealed_holdout.py b/scripts/rectification/sealed_holdout.py index e26a653a..94eafb67 100644 --- a/scripts/rectification/sealed_holdout.py +++ b/scripts/rectification/sealed_holdout.py @@ -13,9 +13,6 @@ from typing import Any, Final ROOT = Path(__file__).resolve().parents[2] HOLDOUT_PATH: Final = ROOT / "references" / "rectification_sealed_holdout.v1.json" -PILOT_REPORT_PATH: Final = ( - ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v2_pilot_report.json" -) def load_sealed_minute_holdout() -> dict[str, Any]: diff --git a/tests/test_rectification_confirmation_and.py b/tests/test_rectification_confirmation_and.py index 94d9e839..40ab2aa5 100644 --- a/tests/test_rectification_confirmation_and.py +++ b/tests/test_rectification_confirmation_and.py @@ -19,6 +19,11 @@ ROOT = Path(__file__).resolve().parents[1] V3_REPORT_PATH = ( ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json" ) +DIAGNOSTIC_REPORT_PATH = ( + ROOT / "references" / "real_case_calibration" + / "minute_rectification_holdout_v3_post_audit_diagnostic_report.json" +) +PRODUCT_HOLDOUT_PATH = ROOT / "references" / "rectification_sealed_holdout.v1.json" def _request(*, extra_events=()): @@ -83,7 +88,12 @@ def _rows(): def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None: holdout = load_sealed_minute_holdout() report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8")) - product = json.loads((ROOT / "references" / "rectification_sealed_holdout.v1.json").read_text(encoding="utf-8")) + diagnostic = json.loads(DIAGNOSTIC_REPORT_PATH.read_text(encoding="utf-8")) + product = json.loads(PRODUCT_HOLDOUT_PATH.read_text(encoding="utf-8")) + sidecar = diagnostic["current_tree_unfrozen_diagnostic"] + produced_by = product["metrics_produced_by"] + current_tree = product["current_tree_scorer"] + unfrozen = product["current_tree_unfrozen_diagnostic"] assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"] assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"] assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"] @@ -93,6 +103,42 @@ def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None: assert holdout["valid_public_aa_cases"] >= holdout["required_cases"] assert holdout["confirmation_coverage_rate"] == 0.0 assert holdout_passed(holdout) is False + assert set(holdout) == { + "sealed_benchmark_id", + "status", + "valid_public_aa_cases", + "required_cases", + "top_1_rate", + "confirmation_coverage_rate", + } + assert produced_by["implementation_sha256"] == report["frozen_scoring"]["implementation_sha256"] + assert produced_by["algorithm_version"] == report["frozen_scoring"]["algorithm_version"] + assert produced_by["implementation_hash_matches_at_replay"] is True + assert produced_by["source_report"] == "references/real_case_calibration/minute_rectification_holdout_v3_report.json" + assert current_tree["implementation_sha256"] == diagnostic["frozen_scoring"]["actual_sha256"] + assert current_tree["matches_metrics_scorer"] is False + assert current_tree["official_eval_implementation_hash_matches"] is False + assert current_tree["official_eval_trial_count"] == diagnostic["trial_count"] == 0 + assert current_tree["implementation_sha256"] != produced_by["implementation_sha256"] + assert unfrozen["is_blind_evaluation"] is False + assert unfrozen["is_blind_evaluation"] is diagnostic["is_blind_evaluation"] + assert unfrozen["results_already_seen"] is True + assert unfrozen["scorer_frozen"] is False + assert unfrozen["scorer_frozen"] is diagnostic["frozen_scoring"]["implementation_hash_matches"] + assert unfrozen["source_audit_passed"] is False + assert unfrozen["source_audit_passed"] is report["post_replay_source_audit"]["passed"] + assert unfrozen["must_not_claim_as_release_metrics"] is True + assert unfrozen["must_not_use_for_tuning"] is True + assert unfrozen["report_path"] == ( + "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" + ) + assert unfrozen["implementation_sha256"] == sidecar["actual_sha256"] + assert unfrozen["trial_count"] == sidecar["trial_count"] + assert unfrozen["top_1_rate"] == sidecar["metrics"]["top_1_rate"] + assert unfrozen["top_3_rate"] == sidecar["metrics"]["top_3_rate"] + assert unfrozen["mean_absolute_minute_error"] == sidecar["metrics"]["mean_absolute_minute_error"] + assert product["top_1_rate"] != unfrozen["top_1_rate"] + assert holdout["top_1_rate"] != unfrozen["top_1_rate"] def test_vedastro_pass_rewrites_audit_but_holdout_still_blocks(monkeypatch) -> None: