Jesse_Chen
65 lines
2.6 KiB
Python
65 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate public cases that may be used for development but never for holdout claims."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from scripts.minute_rectification_holdout_validator import ROOT, _case_errors
|
|
|
|
DEFAULT_MANIFEST = ROOT / "references" / "real_case_calibration" / "minute_rectification_development_v1.json"
|
|
|
|
|
|
def validate(manifest_path: Path = DEFAULT_MANIFEST) -> dict[str, Any]:
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
gate = manifest.get("minimum_case_contract") or {}
|
|
invalid: list[dict[str, Any]] = []
|
|
valid = 0
|
|
seen: set[str] = set()
|
|
for raw_case in manifest.get("cases", []):
|
|
case = deepcopy(raw_case) if isinstance(raw_case, dict) else raw_case
|
|
errors: list[str] = []
|
|
if isinstance(case, dict):
|
|
if case.get("development_partition") != "tuning_and_diagnostics":
|
|
errors.append("case_not_in_development_partition")
|
|
if case.get("allowed_for_tuning") is not True:
|
|
errors.append("case_not_allowed_for_tuning")
|
|
if case.get("excluded_from_holdout") is not True:
|
|
errors.append("case_not_excluded_from_holdout")
|
|
case["holdout_partition"] = "sealed_evaluation"
|
|
case["excluded_from_tuning"] = True
|
|
errors.extend(_case_errors(case, gate))
|
|
errors = [
|
|
error for error in errors
|
|
if error not in {"case_not_in_sealed_evaluation_partition", "case_not_excluded_from_tuning"}
|
|
]
|
|
case_id = raw_case.get("case_id") if isinstance(raw_case, dict) else "non_object_case"
|
|
if case_id in seen:
|
|
errors.append("duplicate_case_id")
|
|
if isinstance(case_id, str):
|
|
seen.add(case_id)
|
|
if errors:
|
|
invalid.append({"case_id": case_id, "errors": sorted(set(errors))})
|
|
else:
|
|
valid += 1
|
|
schema_valid = manifest.get("schema_version") == "minute-rectification-development-v1"
|
|
return {
|
|
"scope": "minute_rectification_development_validation",
|
|
"status": "ready_for_development" if schema_valid and not invalid and valid else "blocked",
|
|
"valid_development_cases": valid,
|
|
"invalid_cases": invalid,
|
|
"excluded_from_holdout": True,
|
|
"may_open_release_gate": False,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print(json.dumps(validate(), ensure_ascii=False, indent=2, sort_keys=True))
|