"""Product-facing sealed public AA holdout contract. This module does not invent cases. A human updates `references/rectification_sealed_holdout.v1.json` after a valid passing evaluation of a frozen public AA set. LOEO/LODO are not a holdout. """ from __future__ import annotations import json from pathlib import Path from typing import Any, Final ROOT = Path(__file__).resolve().parents[2] HOLDOUT_PATH: Final = ROOT / "references" / "rectification_sealed_holdout.v1.json" PILOT_REPORT_PATH: Final = ( ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v2_pilot_report.json" ) def load_sealed_minute_holdout() -> dict[str, Any]: data = json.loads(HOLDOUT_PATH.read_text(encoding="utf-8")) if not isinstance(data, dict): raise ValueError("sealed holdout contract must be an object") status = str(data.get("status") or "not_ready").strip() or "not_ready" return { "sealed_benchmark_id": str(data.get("sealed_benchmark_id") or "").strip(), "status": status if status == "ready" else "not_ready", "valid_public_aa_cases": int(data.get("valid_public_aa_cases") or 0), "required_cases": int(data.get("required_cases") or 20), "top_1_rate": float(data.get("top_1_rate") or 0), "confirmation_coverage_rate": float(data.get("confirmation_coverage_rate") or 0), } def holdout_passed(holdout: dict[str, Any] | None = None) -> bool: row = holdout or load_sealed_minute_holdout() return ( row["status"] == "ready" and int(row["valid_public_aa_cases"]) >= int(row["required_cases"]) and float(row["confirmation_coverage_rate"]) > 0 )