Jesse_ChenandClaude Fable 5.1
54e63fe46e
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017eEAG8HD3mm8gsKXgk8uU8
150 lines
7.0 KiB
Python
150 lines
7.0 KiB
Python
"""Schema and provenance checks for the holdout v5 open evaluation set.
|
|
|
|
Public Rodden-AA biographies only. The dataset is research data; these tests do not
|
|
touch scoring and never read private material.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CAL = ROOT / "references" / "real_case_calibration"
|
|
V4 = CAL / "minute_rectification_holdout_v4.json"
|
|
V5 = CAL / "minute_rectification_holdout_v5.json"
|
|
ROSTER = CAL / "holdout_v5_roster.json"
|
|
EVENTS = CAL / "minute_rectification_holdout_v5_events.json"
|
|
|
|
DOMAINS = {"education", "relocation", "relationship", "career", "finance", "health_pressure", "family", "appearance", "occupation"}
|
|
DATE_RE = {"year": re.compile(r"^\d{4}$"), "month": re.compile(r"^\d{4}-\d{2}$"), "day": re.compile(r"^\d{4}-\d{2}-\d{2}$")}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def v5() -> dict:
|
|
return json.loads(V5.read_text(encoding="utf-8"))
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def v4() -> dict:
|
|
return json.loads(V4.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_gate_counts(v5: dict) -> None:
|
|
gate = v5["minimum_gate"]
|
|
assert gate == {"public_aa_cases": 60, "events_per_case": 7, "domains_per_case": 4}
|
|
assert len(v5["cases"]) >= gate["public_aa_cases"]
|
|
for case in v5["cases"]:
|
|
events = case["events"]
|
|
domains = {event["domain"] for event in events}
|
|
assert len(events) >= gate["events_per_case"], case["case_id"]
|
|
assert len(domains) >= gate["domains_per_case"], case["case_id"]
|
|
|
|
|
|
def test_calculation_profile_matches_v4(v5: dict, v4: dict) -> None:
|
|
assert v5["ayanamsa"] == v4["ayanamsa"] == "raman"
|
|
assert v5["node_mode"] == v4["node_mode"] == "mean"
|
|
assert v5["candidate_radii_minutes"] == [10, 30, 60]
|
|
assert v5["truth_hidden_from_ranker"] is False
|
|
assert v5["minute_step"] == 2
|
|
|
|
|
|
def test_v4_cases_are_inherited_verbatim(v5: dict, v4: dict) -> None:
|
|
by_id = {case["case_id"]: case for case in v5["cases"]}
|
|
for original in v4["cases"]:
|
|
inherited = by_id[original["case_id"]]
|
|
assert inherited["inherited_from_v4"] is True
|
|
assert inherited["birth"] == original["birth"]
|
|
assert inherited["events"] == original["events"]
|
|
assert inherited["true_minute"] == original["true_minute"]
|
|
assert inherited["false_minute_offsets"] == original["false_minute_offsets"]
|
|
assert sum(1 for case in v5["cases"] if case["inherited_from_v4"]) == len(v4["cases"])
|
|
|
|
|
|
def test_case_ids_unique_and_birth_records_complete(v5: dict) -> None:
|
|
ids = [case["case_id"] for case in v5["cases"]]
|
|
assert len(ids) == len(set(ids))
|
|
for case in v5["cases"]:
|
|
birth = case["birth"]
|
|
assert re.match(r"^\d{4}-\d{2}-\d{2}$", birth["date"]), case["case_id"]
|
|
assert re.match(r"^\d{2}:\d{2}$", birth["time"]), case["case_id"]
|
|
assert isinstance(birth["latitude"], (int, float)) and -90 <= birth["latitude"] <= 90
|
|
assert isinstance(birth["longitude"], (int, float)) and -180 <= birth["longitude"] <= 180
|
|
assert isinstance(birth["timezone_offset"], (int, float))
|
|
assert birth["source"]["rodden_rating"] == "AA", case["case_id"]
|
|
# v4 rows cite Astro-Databank's adbvip pages; v5 rows cite the wiki record page.
|
|
assert birth["source"]["url"].startswith("https://www.astro.com/"), case["case_id"]
|
|
assert birth["source"]["publisher"] == "Astro-Databank", case["case_id"]
|
|
assert case["true_minute"] == birth["time"]
|
|
|
|
|
|
def test_new_cases_carry_checks_and_quoted_events(v5: dict) -> None:
|
|
new_cases = [case for case in v5["cases"] if not case["inherited_from_v4"]]
|
|
assert new_cases, "v5 must add cases beyond v4"
|
|
for case in new_cases:
|
|
checks = case["checks"]
|
|
assert checks["timezone_agrees"] is True, case["case_id"]
|
|
assert checks["timezone_rule"] in {"zoneinfo_match", "standard_time_kept_dst_disagreement", "lmt_from_birthplace_longitude"}
|
|
assert checks["place_distance_km"] is not None and checks["place_distance_km"] <= 60, case["case_id"]
|
|
assert case["birth"]["source"]["verified_via"]["url"].startswith("https://"), case["case_id"]
|
|
for event in case["events"]:
|
|
source = event["source"]
|
|
assert source["independent_of_birth_source"] is True
|
|
assert source["url"].startswith("https://en.wikipedia.org/wiki/")
|
|
assert len(source["quote"]) >= 20, event["id"]
|
|
|
|
|
|
def test_events_domains_dates_and_precision(v5: dict) -> None:
|
|
for case in v5["cases"]:
|
|
seen: set[str] = set()
|
|
for event in case["events"]:
|
|
assert event["domain"] in DOMAINS, event["id"]
|
|
assert event["precision"] in DATE_RE, event["id"]
|
|
assert DATE_RE[event["precision"]].match(str(event["date"])), (event["id"], event["date"])
|
|
assert event["id"] not in seen
|
|
seen.add(event["id"])
|
|
assert int(str(event["date"])[:4]) >= int(case["birth"]["date"][:4]), event["id"]
|
|
# replay harnesses evaluate at 2026-09-14; the scoring contract rejects later events
|
|
assert str(event["date"])[:4] < "2026" or (event["precision"] != "year" and str(event["date"]) <= "2026-09-14"), event["id"]
|
|
|
|
|
|
def test_strata_and_summary_agree(v5: dict) -> None:
|
|
cases = v5["cases"]
|
|
summary = v5["summary"]
|
|
assert summary["cases"] == len(cases)
|
|
assert summary["events_total"] == sum(len(case["events"]) for case in cases)
|
|
assert summary["southern_hemisphere"] == sum(1 for case in cases if case["birth"]["latitude"] < 0)
|
|
assert summary["night_birth_22_to_02"] == sum(1 for case in cases if case["strata"]["night_birth_22_to_02"])
|
|
assert summary["utc_plus_8"] == sum(1 for case in cases if abs(case["birth"]["timezone_offset"] - 8.0) < 0.01)
|
|
assert len(summary["decades"]) >= 3
|
|
assert set(summary["latitude_bands"]) == {"lt25", "25to45", "gt45"}
|
|
assert summary["southern_hemisphere"] >= 8
|
|
assert summary["night_birth_22_to_02"] >= 6
|
|
precision = Counter(event["precision"] for case in cases for event in case["events"])
|
|
assert precision == Counter(summary["events_by_precision"])
|
|
|
|
|
|
def test_roster_and_events_files_match_dataset(v5: dict) -> None:
|
|
roster = json.loads(ROSTER.read_text(encoding="utf-8"))
|
|
events = json.loads(EVENTS.read_text(encoding="utf-8"))
|
|
new_ids = {case["case_id"] for case in v5["cases"] if not case["inherited_from_v4"]}
|
|
assert {row["case_id"] for row in roster["cases"]} == new_ids
|
|
assert set(events) == new_ids
|
|
for case in v5["cases"]:
|
|
if case["inherited_from_v4"]:
|
|
continue
|
|
assert sorted(events[case["case_id"]]["events"], key=lambda e: (e["date"], e["id"])) == case["events"]
|
|
|
|
|
|
def test_build_is_deterministic(tmp_path: Path) -> None:
|
|
from scripts.research import holdout_v5_build as build
|
|
|
|
payload, problems = build.build(build.load(V4), build.load(ROSTER), build.load(EVENTS))
|
|
assert problems == []
|
|
rendered = json.dumps(payload, ensure_ascii=False, indent=1) + "\n"
|
|
assert rendered == V5.read_text(encoding="utf-8")
|