diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index 547e2002..8d4973e3 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -6493,6 +6493,22 @@ - 复发自：无 - 修复版本：待发布 +## BUG-428 | v4 intake 把已看过、迁移审阅的案例标成密封盲测材料 + +- 状态：resolved +- 首次发现：2026-08-29 +- 最近更新：2026-08-29 +- 影响面：`minute_rectification_holdout_v4_intake.json`、intake 脚本、holdout validator 人审安全条款 +- 用户现象：4 例带着 `independent_human_reviewed=true` 和 `holdout_partition=sealed_evaluation`，但 `adjudicator` 是自动化迁移，评分结果已在 v3 盲测回放和 post-audit sidecar 里被看过。 +- 触发条件：把 v3 密封集迁入 v4 intake 后，用 `case_errors(..., require_review_safeguards=True)` 当盲测资格。 +- 根因：人审安全条款只看布尔量；intake 接受该布尔量即可写成已审。曝光记录和人审来源没有一等字段。 +- 修复：4 例留在队列并标 `prior_score_exposures`、`human_review_kind=migrated_v3_record`、`frozen_before_scoring=false`、`holdout_partition=exposed_awaiting_human_rereview`。带曝光记录的案例不得过盲测校验；只有 `fresh_biography_audit` 满足人审条款。intake 不再在缺少该字段时把案例写成已审。盲测可用 0 例、已曝光待复审 4 例。生产调参与精确分钟声称仍为 false。 +- 验证：`tests/test_minute_rectification_holdout_intake.py`、`tests/test_minute_rectification_holdout_validator.py`。 +- 防复发：结果已被看过的案例不得再计入盲测。人审安全条款不得由自动化脚本自行置真。迁移来的审阅记录不满足新的传记级人工核查。 +- 相关记录：BUG-427 +- 复发自：无 +- 修复版本：待发布 + ## BUG-410 | 训练已齐仍因家人/职业方法层停在采集，Agent 只确认后截断 - 状态：resolved diff --git a/progress.md b/progress.md index 95d43b2d..7a8f9fd2 100644 --- a/progress.md +++ b/progress.md @@ -1046,3 +1046,11 @@ - sidecar `current_tree_unfrozen_diagnostic` 记录当前树非盲诊断 `top_1_rate=0.45` / `top_3_rate=0.50` / `mean_absolute_minute_error=5.8`，并标明非盲、结果已被看过、scorer 未冻结、source audit 未过，不得当作发布指标。 - 删除 `scripts/rectification/sealed_holdout.py` 未引用的 `PILOT_REPORT_PATH`（仍指向已被 v3 取代的 v2 试点报告）。 - 防复发：评测指标必须与产生它的 `implementation_sha256` 绑定记录。 + +## 2026-08-29 - D2: v4 intake 如实标注已曝光和迁移人审 + +- 4 例全部保留。每例增加 `prior_score_exposures`，指向 v3 盲测报告与 post-audit diagnostic report。 +- `frozen_before_scoring=false`，`holdout_partition=exposed_awaiting_human_rereview`，`human_review_kind=migrated_v3_record`，`independent_human_reviewed=false`。 +- 校验器：带曝光记录不得过盲测/密封用途；只有 `fresh_biography_audit` 满足人审安全条款。intake 在缺少该字段时不能把案例写成已审。 +- 队列：盲测可用 0 例，已曝光/待人工复审 4 例。`production_tuning_allowed` 与 `verified_minute_claim_allowed` 仍为 false。 +- 防复发：结果已被看过的案例不得再计入盲测；人审安全条款不得由自动化脚本自行置真。 diff --git a/references/real_case_calibration/minute_rectification_holdout_v4_intake.json b/references/real_case_calibration/minute_rectification_holdout_v4_intake.json index e02c3cc0..0490fdee 100644 --- a/references/real_case_calibration/minute_rectification_holdout_v4_intake.json +++ b/references/real_case_calibration/minute_rectification_holdout_v4_intake.json @@ -1,4 +1,5 @@ { + "blind_holdout_eligible_case_count": 0, "boundary": "This intake queue is not a frozen holdout. Promotion requires a new version, a passed source audit and a scoring identity frozen before blind replay.", "cases": [ { @@ -102,10 +103,21 @@ 5, 10 ], - "frozen_before_scoring": true, - "holdout_partition": "sealed_evaluation", - "independent_human_reviewed": true, + "frozen_before_scoring": false, + "holdout_partition": "exposed_awaiting_human_rereview", + "human_review_kind": "migrated_v3_record", + "independent_human_reviewed": false, "ingested_at": "2026-08-28T16:29:41.362489Z", + "prior_score_exposures": [ + { + "evaluation_kind": "v3_blind_replay", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json" + }, + { + "evaluation_kind": "post_audit_sidecar_diagnostic", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" + } + ], "review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.", "subject_label": "Barack Obama" }, @@ -210,10 +222,21 @@ 5, 10 ], - "frozen_before_scoring": true, - "holdout_partition": "sealed_evaluation", - "independent_human_reviewed": true, + "frozen_before_scoring": false, + "holdout_partition": "exposed_awaiting_human_rereview", + "human_review_kind": "migrated_v3_record", + "independent_human_reviewed": false, "ingested_at": "2026-08-28T16:29:41.658178Z", + "prior_score_exposures": [ + { + "evaluation_kind": "v3_blind_replay", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json" + }, + { + "evaluation_kind": "post_audit_sidecar_diagnostic", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" + } + ], "review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.", "subject_label": "Paul Ryan" }, @@ -318,10 +341,21 @@ 5, 10 ], - "frozen_before_scoring": true, - "holdout_partition": "sealed_evaluation", - "independent_human_reviewed": true, + "frozen_before_scoring": false, + "holdout_partition": "exposed_awaiting_human_rereview", + "human_review_kind": "migrated_v3_record", + "independent_human_reviewed": false, "ingested_at": "2026-08-28T16:29:41.829087Z", + "prior_score_exposures": [ + { + "evaluation_kind": "v3_blind_replay", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json" + }, + { + "evaluation_kind": "post_audit_sidecar_diagnostic", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" + } + ], "review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.", "subject_label": "Sean Lennon" }, @@ -426,14 +460,26 @@ 5, 10 ], - "frozen_before_scoring": true, - "holdout_partition": "sealed_evaluation", - "independent_human_reviewed": true, + "frozen_before_scoring": false, + "holdout_partition": "exposed_awaiting_human_rereview", + "human_review_kind": "migrated_v3_record", + "independent_human_reviewed": false, "ingested_at": "2026-08-28T16:29:41.971107Z", + "prior_score_exposures": [ + { + "evaluation_kind": "v3_blind_replay", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json" + }, + { + "evaluation_kind": "post_audit_sidecar_diagnostic", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" + } + ], "review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.", "subject_label": "Kurt Cobain" } ], + "exposed_awaiting_human_rereview_case_count": 4, "minimum_gate": { "day_precision_events_per_case": 3, "domains_per_case": 2, @@ -444,6 +490,6 @@ }, "production_tuning_allowed": false, "schema_version": "minute-rectification-holdout-v4-intake", - "status": "collecting_independently_reviewed_cases", + "status": "exposed_awaiting_human_rereview", "verified_minute_claim_allowed": false } diff --git a/scripts/minute_rectification_holdout_intake.py b/scripts/minute_rectification_holdout_intake.py index 13f88d27..5cf3ee83 100644 --- a/scripts/minute_rectification_holdout_intake.py +++ b/scripts/minute_rectification_holdout_intake.py @@ -10,9 +10,15 @@ from pathlib import Path from typing import Any try: - from scripts.minute_rectification_holdout_validator import case_errors + from scripts.minute_rectification_holdout_validator import ( + FRESH_HUMAN_REVIEW_KIND, + case_errors, + ) except ModuleNotFoundError: # pragma: no cover - direct script execution - from minute_rectification_holdout_validator import case_errors + from minute_rectification_holdout_validator import ( + FRESH_HUMAN_REVIEW_KIND, + case_errors, + ) INTAKE_SCHEMA_VERSION = "minute-rectification-holdout-v4-intake" DEFAULT_INTAKE = ( @@ -34,6 +40,13 @@ def append_case(path: Path, case: dict[str, Any]) -> dict[str, Any]: return {"appended": False, "errors": ["missing_case_id"]} if any(isinstance(existing, dict) and existing.get("case_id") == case_id for existing in cases): return {"appended": False, "errors": ["duplicate_case_id"]} + # Never auto-attest independent review. A true flag is accepted only when + # the caller already supplied a fresh biography-level human-review kind. + if ( + case.get("independent_human_reviewed") is True + and case.get("human_review_kind") != FRESH_HUMAN_REVIEW_KIND + ): + return {"appended": False, "errors": ["independent_review_requires_fresh_biography_audit"]} errors = case_errors(case, gate, require_review_safeguards=True) if errors: return {"appended": False, "errors": errors} diff --git a/scripts/minute_rectification_holdout_validator.py b/scripts/minute_rectification_holdout_validator.py index a0908862..071d55ae 100644 --- a/scripts/minute_rectification_holdout_validator.py +++ b/scripts/minute_rectification_holdout_validator.py @@ -19,6 +19,8 @@ ALLOWED_DOMAINS = { "education", "relocation", "relationship", "career", "finance", "health_pressure", } ALLOWED_PRECISIONS = {"year", "month", "day"} +FRESH_HUMAN_REVIEW_KIND = "fresh_biography_audit" +MIGRATED_HUMAN_REVIEW_KIND = "migrated_v3_record" def _is_public_url(value: Any) -> bool: @@ -143,6 +145,13 @@ def _case_errors(case: Any, gate: dict[str, Any]) -> list[str]: return errors +def _prior_score_exposures(case: Any) -> list[Any]: + raw = case.get("prior_score_exposures") if isinstance(case, dict) else None + if not isinstance(raw, list): + return [] + return [item for item in raw if item] + + def _review_safeguard_errors(case: Any, gate: dict[str, Any]) -> list[str]: """Validate safeguards required for newly admitted v4 holdout cases.""" if not isinstance(case, dict): @@ -150,6 +159,13 @@ def _review_safeguard_errors(case: Any, gate: dict[str, Any]) -> list[str]: errors: list[str] = [] if not isinstance(case.get("adjudicator"), str) or not case.get("adjudicator", "").strip(): errors.append("missing_independent_adjudicator") + if _prior_score_exposures(case): + errors.append("results_already_seen_cannot_be_blind_holdout") + review_kind = case.get("human_review_kind") + if review_kind == MIGRATED_HUMAN_REVIEW_KIND: + errors.append("migrated_review_does_not_satisfy_safeguard") + if review_kind != FRESH_HUMAN_REVIEW_KIND: + errors.append("fresh_biography_audit_not_attested") if case.get("independent_human_reviewed") is not True: errors.append("independent_review_not_attested") if case.get("frozen_before_scoring") is not True: diff --git a/tests/test_minute_rectification_holdout_intake.py b/tests/test_minute_rectification_holdout_intake.py index 4ed4abe0..ca4ee282 100644 --- a/tests/test_minute_rectification_holdout_intake.py +++ b/tests/test_minute_rectification_holdout_intake.py @@ -4,7 +4,12 @@ from copy import deepcopy from pathlib import Path from scripts.minute_rectification_holdout_intake import DEFAULT_INTAKE, append_case -from scripts.minute_rectification_holdout_validator import DEFAULT_MANIFEST, case_errors +from scripts.minute_rectification_holdout_validator import ( + DEFAULT_MANIFEST, + FRESH_HUMAN_REVIEW_KIND, + MIGRATED_HUMAN_REVIEW_KIND, + case_errors, +) V3_HOLDOUT = Path(__file__).resolve().parents[1] / "references" / "real_case_calibration" / "minute_rectification_holdout_v3.json" EXPECTED_V4_INTAKE_CASE_IDS = { @@ -20,6 +25,7 @@ def _reviewed_case() -> dict: case["case_id"] = "new-reviewed-case" case["adjudicator"] = "independent-reviewer" case["independent_human_reviewed"] = True + case["human_review_kind"] = FRESH_HUMAN_REVIEW_KIND case["frozen_before_scoring"] = True case["false_minute_commitments"] = [ { @@ -59,6 +65,19 @@ def test_intake_appends_reviewed_case_but_keeps_release_blocked(tmp_path: Path) assert data["cases"][0]["ingested_at"].endswith("Z") +def test_intake_rejects_boolean_review_without_fresh_biography_audit(tmp_path: Path) -> None: + path = tmp_path / "intake.json" + _queue(path) + case = _reviewed_case() + case["human_review_kind"] = MIGRATED_HUMAN_REVIEW_KIND + + report = append_case(path, case) + + assert report["appended"] is False + assert "independent_review_requires_fresh_biography_audit" in report["errors"] + assert json.loads(path.read_text(encoding="utf-8"))["cases"] == [] + + def test_intake_rejects_missing_review_and_commitments(tmp_path: Path) -> None: path = tmp_path / "intake.json" _queue(path) @@ -75,11 +94,65 @@ def test_intake_rejects_missing_review_and_commitments(tmp_path: Path) -> None: def test_default_intake_is_non_production_and_contains_day_precision_v3_cases() -> None: data = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8")) + gate = data["minimum_gate"] assert {case["case_id"] for case in data["cases"]} == EXPECTED_V4_INTAKE_CASE_IDS assert len(data["cases"]) == 4 assert data["production_tuning_allowed"] is False assert data["verified_minute_claim_allowed"] is False + assert data["status"] == "exposed_awaiting_human_rereview" + assert data["blind_holdout_eligible_case_count"] == 0 + assert data["exposed_awaiting_human_rereview_case_count"] == 4 + for case in data["cases"]: + assert case["prior_score_exposures"] + assert case["human_review_kind"] == MIGRATED_HUMAN_REVIEW_KIND + assert case["independent_human_reviewed"] is False + assert case["frozen_before_scoring"] is False + assert case["holdout_partition"] == "exposed_awaiting_human_rereview" + errors = case_errors(case, gate, require_review_safeguards=True) + assert "results_already_seen_cannot_be_blind_holdout" in errors + assert "migrated_review_does_not_satisfy_safeguard" in errors + assert "fresh_biography_audit_not_attested" in errors + eligible = [ + case["case_id"] + for case in data["cases"] + if not case_errors(case, gate, require_review_safeguards=True) + ] + assert eligible == [] + + +def test_exposed_case_cannot_pass_blind_holdout_safeguards() -> None: + case = _reviewed_case() + case["prior_score_exposures"] = [ + { + "evaluation_kind": "v3_blind_replay", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json", + }, + { + "evaluation_kind": "post_audit_sidecar_diagnostic", + "report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json", + }, + ] + gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"] + errors = case_errors(case, gate, require_review_safeguards=True) + assert "results_already_seen_cannot_be_blind_holdout" in errors + + +def test_migrated_human_review_does_not_satisfy_safeguards() -> None: + case = _reviewed_case() + case["human_review_kind"] = MIGRATED_HUMAN_REVIEW_KIND + case["adjudicator"] = "v3_sealed_set_post_audit_migration" + gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"] + errors = case_errors(case, gate, require_review_safeguards=True) + assert "migrated_review_does_not_satisfy_safeguard" in errors + assert "fresh_biography_audit_not_attested" in errors + + +def test_fresh_unseen_reviewed_case_still_passes_safeguards() -> None: + case = _reviewed_case() + assert "prior_score_exposures" not in case + gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"] + assert case_errors(case, gate, require_review_safeguards=True) == [] def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() -> None: @@ -90,6 +163,7 @@ def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() - payload = deepcopy(case) payload.update({ "adjudicator": "v3_sealed_set_post_audit_migration", + "human_review_kind": MIGRATED_HUMAN_REVIEW_KIND, "independent_human_reviewed": True, "frozen_before_scoring": True, "false_minute_commitments": [ @@ -108,5 +182,7 @@ def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() - assert "insufficient_day_precision_events" in rejected["iwao_takamoto_1925_aa_v4_holdout"] assert "insufficient_day_precision_events" in rejected["angelina_jolie_1975_aa_v4_holdout"] for case_id in EXPECTED_V4_INTAKE_CASE_IDS: - assert case_id not in rejected - assert all("insufficient_day_precision_events" in errors for errors in rejected.values()) + assert "migrated_review_does_not_satisfy_safeguard" in rejected[case_id] + assert "fresh_biography_audit_not_attested" in rejected[case_id] + assert "insufficient_day_precision_events" not in rejected[case_id] + assert all("insufficient_day_precision_events" in errors for case_id, errors in rejected.items() if case_id not in EXPECTED_V4_INTAKE_CASE_IDS) diff --git a/tests/test_minute_rectification_holdout_validator.py b/tests/test_minute_rectification_holdout_validator.py index 42623896..62f50351 100644 --- a/tests/test_minute_rectification_holdout_validator.py +++ b/tests/test_minute_rectification_holdout_validator.py @@ -57,6 +57,7 @@ def _add_v4_review_safeguards(manifest: dict) -> None: for case in manifest["cases"]: case["adjudicator"] = "independent-reviewer" case["independent_human_reviewed"] = True + case["human_review_kind"] = "fresh_biography_audit" case["frozen_before_scoring"] = True case["false_minute_commitments"] = [ { @@ -96,6 +97,26 @@ def test_v4_validator_rejects_unreviewed_or_uncommitted_cases(tmp_path: Path) -> assert "false_minute_commitments_do_not_match_offsets" in errors +def test_v4_safeguards_reject_exposed_or_migrated_review(tmp_path: Path) -> None: + manifest = deepcopy(_manifest()) + _add_v4_review_safeguards(manifest) + exposed = manifest["cases"][0] + exposed["prior_score_exposures"] = [ + {"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json"}, + ] + path = tmp_path / "exposed-v4.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + assert "results_already_seen_cannot_be_blind_holdout" in validate(path)["invalid_case_details"][0]["errors"] + + migrated = deepcopy(_manifest()) + _add_v4_review_safeguards(migrated) + migrated["cases"][0]["human_review_kind"] = "migrated_v3_record" + path = tmp_path / "migrated-v4.json" + path.write_text(json.dumps(migrated), encoding="utf-8") + errors = validate(path)["invalid_case_details"][0]["errors"] + assert "migrated_review_does_not_satisfy_safeguard" in errors + assert "fresh_biography_audit_not_attested" in errors + def test_validator_rejects_tuning_case_and_non_independent_event_source(tmp_path: Path) -> None: manifest = deepcopy(_manifest()) case = manifest["cases"][0]