root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

docs(rectification): record v3 sealed holdout as evaluated not missing

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-29 00:25:26 +08:00
parent fe0afdaddb
commit 9d80498316
4 changed files with 60 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_rectification_confirmation_and.py
+11 -7
View File
@@ -10,13 +10,15 @@ from scripts.rectification.decision_policy import (
indistinguishable_width_minutes,
)
from scripts.rectification.sealed_holdout import (
PILOT_REPORT_PATH,
holdout_passed,
load_sealed_minute_holdout,
)
ROOT = Path(__file__).resolve().parents[1]
V3_REPORT_PATH = (
ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json"
)
def _request(*, extra_events=()):
@@ -78,16 +80,18 @@ def _rows():
]
def test_sealed_holdout_contract_matches_v2_pilot_and_stays_closed() -> None:
def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
holdout = load_sealed_minute_holdout()
report = json.loads(PILOT_REPORT_PATH.read_text(encoding="utf-8"))
report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8"))
product = json.loads((ROOT / "references" / "rectification_sealed_holdout.v1.json").read_text(encoding="utf-8"))
assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"]
assert holdout["valid_public_aa_cases"] == report["case_gate"]["valid_public_aa_cases"]
assert holdout["required_cases"] == report["case_gate"]["minimum_public_aa_cases"]
assert holdout["top_1_rate"] == report["metrics"]["top_1_rate"]
assert holdout["confirmation_coverage_rate"] == report["metrics"]["confirmation_coverage_rate"]
assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"]
assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"]
assert holdout["top_1_rate"] == report["release_metrics"]["top_1_rate"]["actual"]
assert holdout["confirmation_coverage_rate"] == report["release_metrics"]["confirmation_coverage_rate"]
assert holdout["status"] == "not_ready"
assert holdout["valid_public_aa_cases"] >= holdout["required_cases"]
assert holdout["confirmation_coverage_rate"] == 0.0
assert holdout_passed(holdout) is False