feat: add reviewed minute holdout intake

scripts/minute_rectification_holdout_intake.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Append one independently reviewed, frozen minute-rectification holdout case."""
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from scripts.minute_rectification_holdout_validator import case_error, validate
+
+
+def append_case(path: Path, case: dict[str, Any]) -> dict[str, Any]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    gate = data.get("minimum_gate") if isinstance(data.get("minimum_gate"), dict) else {}
+    case_id = str(case.get("case_id") or "").strip()
+    if not case_id:
+        return {"appended": False, "errors": ["case_id_missing"], "validation": validate(path)}
+    if any(str(existing.get("case_id") or "") == case_id for existing in data.get("cases", []) if isinstance(existing, dict)):
+        return {"appended": False, "errors": ["case_id_duplicate"], "validation": validate(path)}
+    error = case_error(case, gate)
+    if error:
+        return {"appended": False, "errors": [error], "validation": validate(path)}
+    data.setdefault("cases", []).append({
+        **case,
+        "ingested_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+    })
+    data["status"] = "collecting_independently_reviewed_cases"
+    data["verified_minute_claim_allowed"] = False
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return {"appended": True, "errors": [], "validation": validate(path)}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("manifest", type=Path)
+    parser.add_argument("--case-json", required=True, help="One complete, independently reviewed case JSON object.")
+    args = parser.parse_args()
+    print(json.dumps(append_case(args.manifest, json.loads(args.case_json)), ensure_ascii=False, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/minute_rectification_holdout_validator.py

@@ -43,6 +43,22 @@ def _valid_negative_controls(controls: list[Any], *, minimum: int) -> bool:
     return any(offset < 0 for offset in offsets) and any(offset > 0 for offset in offsets) and len(set(offsets)) == len(offsets) and len(commitments) == len(controls)
 
 
+def case_error(case: dict[str, Any], gate: dict[str, Any]) -> str | None:
+    """Return the one blocking reason for a prospective minute-holdout case."""
+    birth = case.get("birth_source") if isinstance(case.get("birth_source"), dict) else {}
+    events = case.get("events") if isinstance(case.get("events"), list) else []
+    negatives = case.get("negative_minutes") if isinstance(case.get("negative_minutes"), list) else []
+    if not str(case.get("adjudicator") or "").strip() or case.get("independent_human_reviewed") is not True or case.get("frozen_before_scoring") is not True:
+        return "independent_review_invalid"
+    if birth.get("time_accuracy_rating") != "AA" or not birth.get("url"):
+        return "birth_source_invalid"
+    if not _valid_events(events, minimum=int(gate.get("events_per_case", 3)), birth_url=str(birth["url"])):
+        return "events_invalid"
+    if not _valid_negative_controls(negatives, minimum=int(gate.get("negative_minutes_per_case", 4))):
+        return "negative_controls_invalid"
+    return None
+
+
 def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]:
     manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
     gate = manifest.get("minimum_gate") if isinstance(manifest.get("minimum_gate"), dict) else {}
@@ -53,16 +69,10 @@ def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]:
         if not isinstance(case, dict):
             invalid.append("non_object_case")
             continue
-        birth = case.get("birth_source") if isinstance(case.get("birth_source"), dict) else {}
-        events = case.get("events") if isinstance(case.get("events"), list) else []
-        negatives = case.get("negative_minutes") if isinstance(case.get("negative_minutes"), list) else []
         case_id = str(case.get("case_id") or "unnamed_case")
-        if birth.get("time_accuracy_rating") != "AA" or not birth.get("url"):
-            invalid.append(f"{case_id}:birth_source_invalid")
-        elif not _valid_events(events, minimum=int(gate.get("events_per_case", 3)), birth_url=str(birth["url"])):
-            invalid.append(f"{case_id}:events_invalid")
-        elif not _valid_negative_controls(negatives, minimum=int(gate.get("negative_minutes_per_case", 4))):
-            invalid.append(f"{case_id}:negative_controls_invalid")
+        error = case_error(case, gate)
+        if error:
+            invalid.append(f"{case_id}:{error}")
         else:
             valid_cases += 1
     needed = int(gate.get("public_aa_cases", 20))

tests/test_minute_rectification_holdout_intake.py

@@ -0,0 +1,50 @@
+import json
+
+from scripts.minute_rectification_holdout_intake import append_case
+
+
+def _case():
+    return {
+        "case_id": "case-1",
+        "adjudicator": "independent-reviewer",
+        "independent_human_reviewed": True,
+        "frozen_before_scoring": True,
+        "birth_source": {"url": "https://birth.example/case-1", "time_accuracy_rating": "AA"},
+        "events": [
+            {"event_date": "2000-01-01", "source": {"url": "https://event.example/1"}},
+            {"event_date": "2001-01-01", "source": {"url": "https://event.example/2"}},
+            {"event_date": "2002-01-01", "source": {"url": "https://event.example/3"}},
+        ],
+        "negative_minutes": [
+            {"control_id": "a", "offset_minutes": -5, "commitment_hash": "a" * 64},
+            {"control_id": "b", "offset_minutes": -2, "commitment_hash": "b" * 64},
+            {"control_id": "c", "offset_minutes": 2, "commitment_hash": "c" * 64},
+            {"control_id": "d", "offset_minutes": 5, "commitment_hash": "d" * 64},
+        ],
+    }
+
+
+def test_intake_only_appends_a_case_that_passes_the_minute_evidence_contract(tmp_path):
+    manifest = tmp_path / "holdout.json"
+    manifest.write_text(json.dumps({
+        "benchmark_id": "test", "minimum_gate": {"public_aa_cases": 20, "events_per_case": 3, "negative_minutes_per_case": 4}, "cases": [],
+    }), encoding="utf-8")
+
+    result = append_case(manifest, _case())
+
+    assert result["appended"] is True
+    saved = json.loads(manifest.read_text(encoding="utf-8"))
+    assert saved["cases"][0]["case_id"] == "case-1"
+
+
+def test_intake_rejects_unreviewed_or_duplicate_cases(tmp_path):
+    manifest = tmp_path / "holdout.json"
+    manifest.write_text(json.dumps({
+        "benchmark_id": "test", "minimum_gate": {"public_aa_cases": 20, "events_per_case": 3, "negative_minutes_per_case": 4}, "cases": [],
+    }), encoding="utf-8")
+    invalid = _case()
+    invalid["adjudicator"] = ""
+
+    assert append_case(manifest, invalid)["appended"] is False
+    assert append_case(manifest, _case())["appended"] is True
+    assert append_case(manifest, _case())["appended"] is False

tests/test_minute_rectification_holdout_validator.py

@@ -23,6 +23,9 @@ def _case(*, offsets=(-5, -2, 2, 5), event_count=3):
     ]
     return {
         "case_id": "public-aa-case",
+        "adjudicator": "independent-reviewer",
+        "independent_human_reviewed": True,
+        "frozen_before_scoring": True,
         "birth_source": {
             "url": "https://example.test/birth-record",
             "time_accuracy_rating": "AA",
@@ -68,3 +71,14 @@ def test_validator_rejects_one_sided_or_uncommitted_false_minutes(tmp_path):
 
     assert report["status"] == "blocked_awaiting_public_aa_cases"
     assert report["invalid_cases"] == ["public-aa-case:negative_controls_invalid"]
+
+
+def test_validator_rejects_cases_without_independent_frozen_review(tmp_path):
+    path = tmp_path / "holdout.json"
+    case = _case()
+    case["independent_human_reviewed"] = False
+    path.write_text(json.dumps(_manifest(case)), encoding="utf-8")
+
+    report = validate(path)
+
+    assert report["invalid_cases"] == ["public-aa-case:independent_review_invalid"]