diff --git a/scripts/minute_rectification_holdout_intake.py b/scripts/minute_rectification_holdout_intake.py new file mode 100644 index 00000000..951d6a67 --- /dev/null +++ b/scripts/minute_rectification_holdout_intake.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Append one independently reviewed, frozen minute-rectification holdout case.""" +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from scripts.minute_rectification_holdout_validator import case_error, validate + + +def append_case(path: Path, case: dict[str, Any]) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + gate = data.get("minimum_gate") if isinstance(data.get("minimum_gate"), dict) else {} + case_id = str(case.get("case_id") or "").strip() + if not case_id: + return {"appended": False, "errors": ["case_id_missing"], "validation": validate(path)} + if any(str(existing.get("case_id") or "") == case_id for existing in data.get("cases", []) if isinstance(existing, dict)): + return {"appended": False, "errors": ["case_id_duplicate"], "validation": validate(path)} + error = case_error(case, gate) + if error: + return {"appended": False, "errors": [error], "validation": validate(path)} + data.setdefault("cases", []).append({ + **case, + "ingested_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + }) + data["status"] = "collecting_independently_reviewed_cases" + data["verified_minute_claim_allowed"] = False + path.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return {"appended": True, "errors": [], "validation": validate(path)} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("--case-json", required=True, help="One complete, independently reviewed case JSON object.") + args = parser.parse_args() + print(json.dumps(append_case(args.manifest, json.loads(args.case_json)), ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/minute_rectification_holdout_validator.py b/scripts/minute_rectification_holdout_validator.py index 120ddd37..587bdc6f 100644 --- a/scripts/minute_rectification_holdout_validator.py +++ b/scripts/minute_rectification_holdout_validator.py @@ -43,6 +43,22 @@ def _valid_negative_controls(controls: list[Any], *, minimum: int) -> bool: return any(offset < 0 for offset in offsets) and any(offset > 0 for offset in offsets) and len(set(offsets)) == len(offsets) and len(commitments) == len(controls) +def case_error(case: dict[str, Any], gate: dict[str, Any]) -> str | None: + """Return the one blocking reason for a prospective minute-holdout case.""" + birth = case.get("birth_source") if isinstance(case.get("birth_source"), dict) else {} + events = case.get("events") if isinstance(case.get("events"), list) else [] + negatives = case.get("negative_minutes") if isinstance(case.get("negative_minutes"), list) else [] + if not str(case.get("adjudicator") or "").strip() or case.get("independent_human_reviewed") is not True or case.get("frozen_before_scoring") is not True: + return "independent_review_invalid" + if birth.get("time_accuracy_rating") != "AA" or not birth.get("url"): + return "birth_source_invalid" + if not _valid_events(events, minimum=int(gate.get("events_per_case", 3)), birth_url=str(birth["url"])): + return "events_invalid" + if not _valid_negative_controls(negatives, minimum=int(gate.get("negative_minutes_per_case", 4))): + return "negative_controls_invalid" + return None + + def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) gate = manifest.get("minimum_gate") if isinstance(manifest.get("minimum_gate"), dict) else {} @@ -53,16 +69,10 @@ def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: if not isinstance(case, dict): invalid.append("non_object_case") continue - birth = case.get("birth_source") if isinstance(case.get("birth_source"), dict) else {} - events = case.get("events") if isinstance(case.get("events"), list) else [] - negatives = case.get("negative_minutes") if isinstance(case.get("negative_minutes"), list) else [] case_id = str(case.get("case_id") or "unnamed_case") - if birth.get("time_accuracy_rating") != "AA" or not birth.get("url"): - invalid.append(f"{case_id}:birth_source_invalid") - elif not _valid_events(events, minimum=int(gate.get("events_per_case", 3)), birth_url=str(birth["url"])): - invalid.append(f"{case_id}:events_invalid") - elif not _valid_negative_controls(negatives, minimum=int(gate.get("negative_minutes_per_case", 4))): - invalid.append(f"{case_id}:negative_controls_invalid") + error = case_error(case, gate) + if error: + invalid.append(f"{case_id}:{error}") else: valid_cases += 1 needed = int(gate.get("public_aa_cases", 20)) diff --git a/tests/test_minute_rectification_holdout_intake.py b/tests/test_minute_rectification_holdout_intake.py new file mode 100644 index 00000000..9d4e63f9 --- /dev/null +++ b/tests/test_minute_rectification_holdout_intake.py @@ -0,0 +1,50 @@ +import json + +from scripts.minute_rectification_holdout_intake import append_case + + +def _case(): + return { + "case_id": "case-1", + "adjudicator": "independent-reviewer", + "independent_human_reviewed": True, + "frozen_before_scoring": True, + "birth_source": {"url": "https://birth.example/case-1", "time_accuracy_rating": "AA"}, + "events": [ + {"event_date": "2000-01-01", "source": {"url": "https://event.example/1"}}, + {"event_date": "2001-01-01", "source": {"url": "https://event.example/2"}}, + {"event_date": "2002-01-01", "source": {"url": "https://event.example/3"}}, + ], + "negative_minutes": [ + {"control_id": "a", "offset_minutes": -5, "commitment_hash": "a" * 64}, + {"control_id": "b", "offset_minutes": -2, "commitment_hash": "b" * 64}, + {"control_id": "c", "offset_minutes": 2, "commitment_hash": "c" * 64}, + {"control_id": "d", "offset_minutes": 5, "commitment_hash": "d" * 64}, + ], + } + + +def test_intake_only_appends_a_case_that_passes_the_minute_evidence_contract(tmp_path): + manifest = tmp_path / "holdout.json" + manifest.write_text(json.dumps({ + "benchmark_id": "test", "minimum_gate": {"public_aa_cases": 20, "events_per_case": 3, "negative_minutes_per_case": 4}, "cases": [], + }), encoding="utf-8") + + result = append_case(manifest, _case()) + + assert result["appended"] is True + saved = json.loads(manifest.read_text(encoding="utf-8")) + assert saved["cases"][0]["case_id"] == "case-1" + + +def test_intake_rejects_unreviewed_or_duplicate_cases(tmp_path): + manifest = tmp_path / "holdout.json" + manifest.write_text(json.dumps({ + "benchmark_id": "test", "minimum_gate": {"public_aa_cases": 20, "events_per_case": 3, "negative_minutes_per_case": 4}, "cases": [], + }), encoding="utf-8") + invalid = _case() + invalid["adjudicator"] = "" + + assert append_case(manifest, invalid)["appended"] is False + assert append_case(manifest, _case())["appended"] is True + assert append_case(manifest, _case())["appended"] is False diff --git a/tests/test_minute_rectification_holdout_validator.py b/tests/test_minute_rectification_holdout_validator.py index ccedb647..ebd46073 100644 --- a/tests/test_minute_rectification_holdout_validator.py +++ b/tests/test_minute_rectification_holdout_validator.py @@ -23,6 +23,9 @@ def _case(*, offsets=(-5, -2, 2, 5), event_count=3): ] return { "case_id": "public-aa-case", + "adjudicator": "independent-reviewer", + "independent_human_reviewed": True, + "frozen_before_scoring": True, "birth_source": { "url": "https://example.test/birth-record", "time_accuracy_rating": "AA", @@ -68,3 +71,14 @@ def test_validator_rejects_one_sided_or_uncommitted_false_minutes(tmp_path): assert report["status"] == "blocked_awaiting_public_aa_cases" assert report["invalid_cases"] == ["public-aa-case:negative_controls_invalid"] + + +def test_validator_rejects_cases_without_independent_frozen_review(tmp_path): + path = tmp_path / "holdout.json" + case = _case() + case["independent_human_reviewed"] = False + path.write_text(json.dumps(_manifest(case)), encoding="utf-8") + + report = validate(path) + + assert report["invalid_cases"] == ["public-aa-case:independent_review_invalid"]