Jesse_Chen
55d3119c68
Keep confirmation fail-closed until the public AA set is ready, and rewrite Technique Audit from the attached VedAstro status instead of a hardcoded blocked row. Co-authored-by: Cursor <cursoragent@cursor.com>
43 lines
1.6 KiB
Python
43 lines
1.6 KiB
Python
"""Product-facing sealed public AA holdout contract.
|
|
|
|
This module does not invent cases. A human updates
|
|
`references/rectification_sealed_holdout.v1.json` after a valid passing
|
|
evaluation of a frozen public AA set. LOEO/LODO are not a holdout.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Final
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
HOLDOUT_PATH: Final = ROOT / "references" / "rectification_sealed_holdout.v1.json"
|
|
PILOT_REPORT_PATH: Final = (
|
|
ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v2_pilot_report.json"
|
|
)
|
|
|
|
|
|
def load_sealed_minute_holdout() -> dict[str, Any]:
|
|
data = json.loads(HOLDOUT_PATH.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise ValueError("sealed holdout contract must be an object")
|
|
status = str(data.get("status") or "not_ready").strip() or "not_ready"
|
|
return {
|
|
"sealed_benchmark_id": str(data.get("sealed_benchmark_id") or "").strip(),
|
|
"status": status if status == "ready" else "not_ready",
|
|
"valid_public_aa_cases": int(data.get("valid_public_aa_cases") or 0),
|
|
"required_cases": int(data.get("required_cases") or 20),
|
|
"top_1_rate": float(data.get("top_1_rate") or 0),
|
|
"confirmation_coverage_rate": float(data.get("confirmation_coverage_rate") or 0),
|
|
}
|
|
|
|
|
|
def holdout_passed(holdout: dict[str, Any] | None = None) -> bool:
|
|
row = holdout or load_sealed_minute_holdout()
|
|
return (
|
|
row["status"] == "ready"
|
|
and int(row["valid_public_aa_cases"]) >= int(row["required_cases"])
|
|
and float(row["confirmation_coverage_rate"]) > 0
|
|
)
|