diff --git a/scripts/minute_rectification_holdout_validator.py b/scripts/minute_rectification_holdout_validator.py index 3822e38a..120ddd37 100644 --- a/scripts/minute_rectification_holdout_validator.py +++ b/scripts/minute_rectification_holdout_validator.py @@ -10,6 +10,39 @@ ROOT = Path(__file__).resolve().parents[1] DEFAULT_MANIFEST = ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v1.json" +def _valid_events(events: list[Any], *, minimum: int, birth_url: str) -> bool: + if len(events) < minimum: + return False + for event in events: + if not isinstance(event, dict): + return False + source = event.get("source") if isinstance(event.get("source"), dict) else {} + date = str(event.get("event_date") or "") + # Month/year-only biographies cannot distinguish neighbouring minutes. + if len(date) != 10 or not source.get("url") or str(source["url"]) == birth_url: + return False + return True + + +def _valid_negative_controls(controls: list[Any], *, minimum: int) -> bool: + if len(controls) < minimum: + return False + offsets: list[int] = [] + commitments: set[str] = set() + for control in controls: + if not isinstance(control, dict): + return False + offset = control.get("offset_minutes") + commitment = str(control.get("commitment_hash") or "") + if not isinstance(offset, int) or offset == 0 or len(commitment) != 64: + return False + if any(key in control for key in ("candidate_minute", "published_minute", "birth_time")): + return False + offsets.append(offset) + commitments.add(commitment) + return any(offset < 0 for offset in offsets) and any(offset > 0 for offset in offsets) and len(set(offsets)) == len(offsets) and len(commitments) == len(controls) + + def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) gate = manifest.get("minimum_gate") if isinstance(manifest.get("minimum_gate"), dict) else {} @@ -23,13 +56,15 @@ def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]: birth = case.get("birth_source") if isinstance(case.get("birth_source"), dict) else {} events = case.get("events") if isinstance(case.get("events"), list) else [] negatives = case.get("negative_minutes") if isinstance(case.get("negative_minutes"), list) else [] - required = birth.get("time_accuracy_rating") == "AA" and bool(birth.get("url")) - required = required and len(events) >= int(gate.get("events_per_case", 3)) - required = required and len(negatives) >= int(gate.get("negative_minutes_per_case", 4)) - if required: - valid_cases += 1 + case_id = str(case.get("case_id") or "unnamed_case") + if birth.get("time_accuracy_rating") != "AA" or not birth.get("url"): + invalid.append(f"{case_id}:birth_source_invalid") + elif not _valid_events(events, minimum=int(gate.get("events_per_case", 3)), birth_url=str(birth["url"])): + invalid.append(f"{case_id}:events_invalid") + elif not _valid_negative_controls(negatives, minimum=int(gate.get("negative_minutes_per_case", 4))): + invalid.append(f"{case_id}:negative_controls_invalid") else: - invalid.append(str(case.get("case_id") or "unnamed_case")) + valid_cases += 1 needed = int(gate.get("public_aa_cases", 20)) status = "ready_for_blind_replay" if valid_cases >= needed else "blocked_awaiting_public_aa_cases" return { diff --git a/tests/test_minute_rectification_holdout_validator.py b/tests/test_minute_rectification_holdout_validator.py index 455085e9..ccedb647 100644 --- a/tests/test_minute_rectification_holdout_validator.py +++ b/tests/test_minute_rectification_holdout_validator.py @@ -6,3 +6,65 @@ def test_empty_public_minute_protocol_blocks_verified_claims() -> None: assert report["status"] == "blocked_awaiting_public_aa_cases" assert report["verified_minute_claim_allowed"] is False assert report["valid_public_aa_cases"] == 0 +import hashlib +import json + +from scripts.minute_rectification_holdout_validator import validate + + +def _case(*, offsets=(-5, -2, 2, 5), event_count=3): + controls = [ + { + "control_id": f"control-{index}", + "offset_minutes": offset, + "commitment_hash": hashlib.sha256(f"fixed-control-{index}".encode()).hexdigest(), + } + for index, offset in enumerate(offsets) + ] + return { + "case_id": "public-aa-case", + "birth_source": { + "url": "https://example.test/birth-record", + "time_accuracy_rating": "AA", + }, + "events": [ + { + "event_date": f"200{index}-01-0{index + 1}", + "source": {"url": f"https://independent.example.test/event-{index}"}, + } + for index in range(event_count) + ], + "negative_minutes": controls, + } + + +def _manifest(case): + return { + "benchmark_id": "test-minute-holdout", + "minimum_gate": {"public_aa_cases": 1, "events_per_case": 3, "negative_minutes_per_case": 4}, + "boundary": "test boundary", + "cases": [case], + } + + +def test_validator_requires_independent_dated_events_and_committed_controls(tmp_path): + path = tmp_path / "holdout.json" + path.write_text(json.dumps(_manifest(_case())), encoding="utf-8") + + report = validate(path) + + assert report["status"] == "ready_for_blind_replay" + assert report["valid_public_aa_cases"] == 1 + assert report["verified_minute_claim_allowed"] is False + + +def test_validator_rejects_one_sided_or_uncommitted_false_minutes(tmp_path): + path = tmp_path / "holdout.json" + case = _case(offsets=(-5, -2, -1, -1)) + case["negative_minutes"][0].pop("commitment_hash") + path.write_text(json.dumps(_manifest(case)), encoding="utf-8") + + report = validate(path) + + assert report["status"] == "blocked_awaiting_public_aa_cases" + assert report["invalid_cases"] == ["public-aa-case:negative_controls_invalid"]