feat: add reviewed minute holdout intake

This commit is contained in:
732642856
2026-07-21 20:01:30 +08:00
parent 76a2ca7fd9
commit cfa34bac32
4 changed files with 128 additions and 9 deletions
@@ -0,0 +1,50 @@
import json
from scripts.minute_rectification_holdout_intake import append_case
def _case():
return {
"case_id": "case-1",
"adjudicator": "independent-reviewer",
"independent_human_reviewed": True,
"frozen_before_scoring": True,
"birth_source": {"url": "https://birth.example/case-1", "time_accuracy_rating": "AA"},
"events": [
{"event_date": "2000-01-01", "source": {"url": "https://event.example/1"}},
{"event_date": "2001-01-01", "source": {"url": "https://event.example/2"}},
{"event_date": "2002-01-01", "source": {"url": "https://event.example/3"}},
],
"negative_minutes": [
{"control_id": "a", "offset_minutes": -5, "commitment_hash": "a" * 64},
{"control_id": "b", "offset_minutes": -2, "commitment_hash": "b" * 64},
{"control_id": "c", "offset_minutes": 2, "commitment_hash": "c" * 64},
{"control_id": "d", "offset_minutes": 5, "commitment_hash": "d" * 64},
],
}
def test_intake_only_appends_a_case_that_passes_the_minute_evidence_contract(tmp_path):
manifest = tmp_path / "holdout.json"
manifest.write_text(json.dumps({
"benchmark_id": "test", "minimum_gate": {"public_aa_cases": 20, "events_per_case": 3, "negative_minutes_per_case": 4}, "cases": [],
}), encoding="utf-8")
result = append_case(manifest, _case())
assert result["appended"] is True
saved = json.loads(manifest.read_text(encoding="utf-8"))
assert saved["cases"][0]["case_id"] == "case-1"
def test_intake_rejects_unreviewed_or_duplicate_cases(tmp_path):
manifest = tmp_path / "holdout.json"
manifest.write_text(json.dumps({
"benchmark_id": "test", "minimum_gate": {"public_aa_cases": 20, "events_per_case": 3, "negative_minutes_per_case": 4}, "cases": [],
}), encoding="utf-8")
invalid = _case()
invalid["adjudicator"] = ""
assert append_case(manifest, invalid)["appended"] is False
assert append_case(manifest, _case())["appended"] is True
assert append_case(manifest, _case())["appended"] is False
@@ -23,6 +23,9 @@ def _case(*, offsets=(-5, -2, 2, 5), event_count=3):
]
return {
"case_id": "public-aa-case",
"adjudicator": "independent-reviewer",
"independent_human_reviewed": True,
"frozen_before_scoring": True,
"birth_source": {
"url": "https://example.test/birth-record",
"time_accuracy_rating": "AA",
@@ -68,3 +71,14 @@ def test_validator_rejects_one_sided_or_uncommitted_false_minutes(tmp_path):
assert report["status"] == "blocked_awaiting_public_aa_cases"
assert report["invalid_cases"] == ["public-aa-case:negative_controls_invalid"]
def test_validator_rejects_cases_without_independent_frozen_review(tmp_path):
path = tmp_path / "holdout.json"
case = _case()
case["independent_human_reviewed"] = False
path.write_text(json.dumps(_manifest(case)), encoding="utf-8")
report = validate(path)
assert report["invalid_cases"] == ["public-aa-case:independent_review_invalid"]