feat: add observational timing holdout

scripts/day_level_holdout_validator.py

@@ -10,13 +10,16 @@ REQUIRED={"case_id","domain","label","start","end","source_url","adjudicator","t
 
 def validate(path: Path) -> dict:
     data=json.loads(path.read_text(encoding="utf-8")); rows=data.get("annotations") or []; errors=[]
+    mode=data.get("validation_mode", "independent")
+    allowed_labels={"target_event", "no_target_event"} if mode == "independent" else {"target_event", "observational_non_target_date"}
     for i,row in enumerate(rows):
         for key in sorted(REQUIRED-set(row)): errors.append({"row":i,"field":key,"error":"missing"})
-        if row.get("label") not in {"target_event","no_target_event"}: errors.append({"row":i,"field":"label","error":"invalid"})
+        if row.get("label") not in allowed_labels: errors.append({"row":i,"field":"label","error":"invalid"})
         if not str(row.get("source_url") or "").startswith(("https://","http://")): errors.append({"row":i,"field":"source_url","error":"not_public_url"})
-    positives=sum(r.get("label")=="target_event" for r in rows); negatives=sum(r.get("label")=="no_target_event" for r in rows)
+    positives=sum(r.get("label")=="target_event" for r in rows); negatives=sum(r.get("label") in {"no_target_event", "observational_non_target_date"} for r in rows)
     gate=data.get("frozen_gate") or {}; ready=not errors and positives>=gate.get("minimum_independent_cases",20) and negatives>=gate.get("minimum_independent_negative_intervals",80)
-    return {"scope":"day_level_holdout_validation","annotation_count":len(rows),"positive_count":positives,"negative_count":negatives,"errors":errors,"status":"ready_for_blind_replay" if ready else "awaiting_independent_labels","production_tuning_allowed":False}
+    status=("observational_ready_not_independent" if ready else "awaiting_observational_labels") if mode == "observational" else ("ready_for_blind_replay" if ready else "awaiting_independent_labels")
+    return {"scope":"day_level_holdout_validation","validation_mode":mode,"annotation_count":len(rows),"positive_count":positives,"negative_count":negatives,"errors":errors,"status":status,"production_tuning_allowed":False}
 
 def main()->int:
     p=argparse.ArgumentParser(description=__doc__);p.add_argument("manifest",type=Path);a=p.parse_args();r=validate(a.manifest);print(json.dumps(r,ensure_ascii=False,indent=2,sort_keys=True));return 0

tests/test_day_level_holdout_validator.py

@@ -16,3 +16,11 @@ def test_raman_source_candidates_are_not_holdout_labels() -> None:
     assert payload["candidate_count"] >= 20
     assert all(row["status"] == "needs_independent_normalization" for row in payload["candidates"])
     assert all("day_level_label" in row["prohibited_uses"] for row in payload["candidates"])
+
+
+def test_observational_holdout_is_ready_but_not_independent() -> None:
+    report = validate(ROOT / "references/real_case_calibration/day_level_observational_holdout_v1.json")
+    assert report["status"] == "observational_ready_not_independent"
+    assert report["positive_count"] >= 20
+    assert report["negative_count"] >= 80
+    assert report["production_tuning_allowed"] is False