diff --git a/references/real_case_calibration/minute_rectification_public_aa_source_audit_v1.json b/references/real_case_calibration/minute_rectification_public_aa_source_audit_v1.json new file mode 100644 index 00000000..9c928a29 --- /dev/null +++ b/references/real_case_calibration/minute_rectification_public_aa_source_audit_v1.json @@ -0,0 +1,180 @@ +{ + "additional_public_aa_cases_required": 3, + "boundary": "Source discovery is not holdout validation. Cases require independent dated events and committed two-sided false-minute controls before blind replay.", + "cases": [ + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_03_14.htm", + "case_ids": [ + "albert_ii_accession_2005" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Albert II, Prince of Monaco" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_04.htm", + "case_ids": [ + "jolie_marriage_2014" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Angelina Jolie" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_08_04.htm", + "case_ids": [ + "obama_election_2008" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Barack Obama" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_14.htm", + "case_ids": [ + "boy_george_grammy_1984" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Boy George" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_02_27.htm", + "case_ids": [ + "chelsea_clinton_marriage_2010" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Chelsea Clinton" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/astro-databank/Trump,_Donald", + "case_ids": [ + "trump_inauguration_2017" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Donald Trump" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_07_06.htm", + "case_ids": [ + "kahlo_bus_injury_1925", + "kahlo_marriage_1929" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Frida Kahlo" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_08_29.htm", + "case_ids": [ + "bergman_oscar_1945" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Ingrid Bergman" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_02_11.htm", + "case_ids": [ + "aniston_emmy_2002" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Jennifer Aniston" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_09.htm", + "case_ids": [ + "depp_marriage_2015" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Johnny Depp" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/astro-databank/DiCaprio,_Leonardo", + "case_ids": [ + "dicaprio_oscar_2016" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Leonardo DiCaprio" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/astro-databank/Markle,_Meghan", + "case_ids": [ + "markle_marriage_2018" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Meghan Markle" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_22.htm", + "case_ids": [ + "streep_oscar_1983" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Meryl Streep" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/astro-databank/Winfrey,_Oprah", + "case_ids": [ + "winfrey_oprah_show_launch_1986" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Oprah Winfrey" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_10_20.htm", + "case_ids": [ + "snoop_marriage_1997" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Snoop Dogg" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_02_24.htm", + "case_ids": [ + "jobs_iphone_2007" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "Steve Jobs" + }, + { + "additional_dated_events_required": 2, + "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_21.htm", + "case_ids": [ + "william_marriage_2011" + ], + "existing_dated_event_count": 1, + "negative_controls_required": 4, + "subject": "William, Prince of Wales" + } + ], + "minimum_public_aa_cases": 20, + "public_aa_case_count": 17, + "scope": "minute_rectification_public_aa_source_audit" +} diff --git a/scripts/minute_rectification_source_audit.py b/scripts/minute_rectification_source_audit.py new file mode 100644 index 00000000..640b0f3b --- /dev/null +++ b/scripts/minute_rectification_source_audit.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Audit reusable public AA cases before adding them to the minute holdout.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_SOURCES = ( + ROOT / "references" / "real_case_calibration" / "replay_manifest.json", + ROOT / "references" / "real_case_calibration" / "replay_manifest_holdout_v2.json", + ROOT / "references" / "real_case_calibration" / "replay_manifest_probe3_v2.json", + ROOT / "references" / "real_case_calibration" / "public_context_manifest.json", +) + + +def _events(case: dict[str, Any]) -> list[dict[str, Any]]: + raw = case.get("events") if isinstance(case.get("events"), list) else case.get("event_outcomes") + return [event for event in raw or [] if isinstance(event, dict)] + + +def build_source_audit(paths: list[Path] | tuple[Path, ...] = DEFAULT_SOURCES) -> dict[str, Any]: + entries: dict[str, dict[str, Any]] = {} + for path in paths: + data = json.loads(path.read_text(encoding="utf-8")) + for case in data.get("cases", []): + if not isinstance(case, dict): + continue + subject = case.get("subject") if isinstance(case.get("subject"), dict) else case + birth = subject.get("birth_source") if isinstance(subject.get("birth_source"), dict) else {} + source_url = str(birth.get("url") or "") + if birth.get("time_accuracy_rating") != "AA" or not source_url: + continue + name = str(subject.get("name") or case.get("case_id") or "unnamed") + entry = entries.setdefault(source_url, { + "subject": name, + "birth_source_url": source_url, + "case_ids": [], + "dated_event_dates": set(), + }) + entry["case_ids"].append(str(case.get("case_id") or name)) + for event in _events(case): + date = str(event.get("event_date") or event.get("date") or "") + if len(date) == 10: + entry["dated_event_dates"].add(date) + cases = [] + for entry in entries.values(): + event_count = len(entry["dated_event_dates"]) + cases.append({ + "subject": entry["subject"], + "birth_source_url": entry["birth_source_url"], + "case_ids": sorted(entry["case_ids"]), + "existing_dated_event_count": event_count, + "additional_dated_events_required": max(0, 3 - event_count), + "negative_controls_required": 4, + }) + cases.sort(key=lambda case: (case["additional_dated_events_required"], case["subject"])) + return { + "scope": "minute_rectification_public_aa_source_audit", + "public_aa_case_count": len(cases), + "minimum_public_aa_cases": 20, + "additional_public_aa_cases_required": max(0, 20 - len(cases)), + "cases": cases, + "boundary": "Source discovery is not holdout validation. Cases require independent dated events and committed two-sided false-minute controls before blind replay.", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sources", nargs="*", type=Path, default=list(DEFAULT_SOURCES)) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + result = build_source_audit(args.sources) + text = json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + if args.output: + args.output.write_text(text + "\n", encoding="utf-8") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_minute_rectification_source_audit.py b/tests/test_minute_rectification_source_audit.py new file mode 100644 index 00000000..004c8da9 --- /dev/null +++ b/tests/test_minute_rectification_source_audit.py @@ -0,0 +1,44 @@ +import json +from pathlib import Path + +from scripts.minute_rectification_source_audit import build_source_audit + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_source_audit_deduplicates_public_aa_cases_and_reports_missing_evidence(tmp_path): + manifest = tmp_path / "cases.json" + manifest.write_text(json.dumps({ + "cases": [ + { + "case_id": "case-a", + "subject": {"name": "A", "birth_source": {"url": "https://birth.example/a", "time_accuracy_rating": "AA"}}, + "event_outcomes": [{"event_date": "2000-01-01"}], + }, + { + "case_id": "case-a-duplicate", + "subject": {"name": "A", "birth_source": {"url": "https://birth.example/a", "time_accuracy_rating": "AA"}}, + "event_outcomes": [{"event_date": "2001-01-01"}], + }, + { + "case_id": "case-b", + "subject": {"name": "B", "birth_source": {"url": "https://birth.example/b", "time_accuracy_rating": "A"}}, + }, + ], + }), encoding="utf-8") + + audit = build_source_audit([manifest]) + + assert audit["public_aa_case_count"] == 1 + assert audit["cases"][0]["subject"] == "A" + assert audit["cases"][0]["additional_dated_events_required"] == 1 + assert audit["cases"][0]["negative_controls_required"] == 4 + + +def test_frozen_source_audit_matches_the_reusable_public_case_manifests(): + artifact = json.loads((ROOT / "references/real_case_calibration/minute_rectification_public_aa_source_audit_v1.json").read_text(encoding="utf-8")) + + assert artifact == build_source_audit() + assert artifact["public_aa_case_count"] == 17 + assert artifact["additional_public_aa_cases_required"] == 3