feat: audit public minute holdout sources

2026-07-21 19:32:47 +08:00
parent 0fb5fefe4b
commit 76a2ca7fd9
3 changed files with 309 additions and 0 deletions
@@ -0,0 +1,180 @@
+{
+  "additional_public_aa_cases_required": 3,
+  "boundary": "Source discovery is not holdout validation. Cases require independent dated events and committed two-sided false-minute controls before blind replay.",
+  "cases": [
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_03_14.htm",
+      "case_ids": [
+        "albert_ii_accession_2005"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Albert II, Prince of Monaco"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_04.htm",
+      "case_ids": [
+        "jolie_marriage_2014"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Angelina Jolie"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_08_04.htm",
+      "case_ids": [
+        "obama_election_2008"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Barack Obama"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_14.htm",
+      "case_ids": [
+        "boy_george_grammy_1984"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Boy George"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_02_27.htm",
+      "case_ids": [
+        "chelsea_clinton_marriage_2010"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Chelsea Clinton"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/astro-databank/Trump,_Donald",
+      "case_ids": [
+        "trump_inauguration_2017"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Donald Trump"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_07_06.htm",
+      "case_ids": [
+        "kahlo_bus_injury_1925",
+        "kahlo_marriage_1929"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Frida Kahlo"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_08_29.htm",
+      "case_ids": [
+        "bergman_oscar_1945"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Ingrid Bergman"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_02_11.htm",
+      "case_ids": [
+        "aniston_emmy_2002"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Jennifer Aniston"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_09.htm",
+      "case_ids": [
+        "depp_marriage_2015"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Johnny Depp"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/astro-databank/DiCaprio,_Leonardo",
+      "case_ids": [
+        "dicaprio_oscar_2016"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Leonardo DiCaprio"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/astro-databank/Markle,_Meghan",
+      "case_ids": [
+        "markle_marriage_2018"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Meghan Markle"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_22.htm",
+      "case_ids": [
+        "streep_oscar_1983"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Meryl Streep"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/astro-databank/Winfrey,_Oprah",
+      "case_ids": [
+        "winfrey_oprah_show_launch_1986"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Oprah Winfrey"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_10_20.htm",
+      "case_ids": [
+        "snoop_marriage_1997"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Snoop Dogg"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_02_24.htm",
+      "case_ids": [
+        "jobs_iphone_2007"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "Steve Jobs"
+    },
+    {
+      "additional_dated_events_required": 2,
+      "birth_source_url": "https://www.astro.com/adbvip/adbvip_06_21.htm",
+      "case_ids": [
+        "william_marriage_2011"
+      ],
+      "existing_dated_event_count": 1,
+      "negative_controls_required": 4,
+      "subject": "William, Prince of Wales"
+    }
+  ],
+  "minimum_public_aa_cases": 20,
+  "public_aa_case_count": 17,
+  "scope": "minute_rectification_public_aa_source_audit"
+}
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Audit reusable public AA cases before adding them to the minute holdout."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_SOURCES = (
+    ROOT / "references" / "real_case_calibration" / "replay_manifest.json",
+    ROOT / "references" / "real_case_calibration" / "replay_manifest_holdout_v2.json",
+    ROOT / "references" / "real_case_calibration" / "replay_manifest_probe3_v2.json",
+    ROOT / "references" / "real_case_calibration" / "public_context_manifest.json",
+)
+
+
+def _events(case: dict[str, Any]) -> list[dict[str, Any]]:
+    raw = case.get("events") if isinstance(case.get("events"), list) else case.get("event_outcomes")
+    return [event for event in raw or [] if isinstance(event, dict)]
+
+
+def build_source_audit(paths: list[Path] | tuple[Path, ...] = DEFAULT_SOURCES) -> dict[str, Any]:
+    entries: dict[str, dict[str, Any]] = {}
+    for path in paths:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        for case in data.get("cases", []):
+            if not isinstance(case, dict):
+                continue
+            subject = case.get("subject") if isinstance(case.get("subject"), dict) else case
+            birth = subject.get("birth_source") if isinstance(subject.get("birth_source"), dict) else {}
+            source_url = str(birth.get("url") or "")
+            if birth.get("time_accuracy_rating") != "AA" or not source_url:
+                continue
+            name = str(subject.get("name") or case.get("case_id") or "unnamed")
+            entry = entries.setdefault(source_url, {
+                "subject": name,
+                "birth_source_url": source_url,
+                "case_ids": [],
+                "dated_event_dates": set(),
+            })
+            entry["case_ids"].append(str(case.get("case_id") or name))
+            for event in _events(case):
+                date = str(event.get("event_date") or event.get("date") or "")
+                if len(date) == 10:
+                    entry["dated_event_dates"].add(date)
+    cases = []
+    for entry in entries.values():
+        event_count = len(entry["dated_event_dates"])
+        cases.append({
+            "subject": entry["subject"],
+            "birth_source_url": entry["birth_source_url"],
+            "case_ids": sorted(entry["case_ids"]),
+            "existing_dated_event_count": event_count,
+            "additional_dated_events_required": max(0, 3 - event_count),
+            "negative_controls_required": 4,
+        })
+    cases.sort(key=lambda case: (case["additional_dated_events_required"], case["subject"]))
+    return {
+        "scope": "minute_rectification_public_aa_source_audit",
+        "public_aa_case_count": len(cases),
+        "minimum_public_aa_cases": 20,
+        "additional_public_aa_cases_required": max(0, 20 - len(cases)),
+        "cases": cases,
+        "boundary": "Source discovery is not holdout validation. Cases require independent dated events and committed two-sided false-minute controls before blind replay.",
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("sources", nargs="*", type=Path, default=list(DEFAULT_SOURCES))
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+    result = build_source_audit(args.sources)
+    text = json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)
+    if args.output:
+        args.output.write_text(text + "\n", encoding="utf-8")
+    print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,44 @@
+import json
+from pathlib import Path
+
+from scripts.minute_rectification_source_audit import build_source_audit
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_source_audit_deduplicates_public_aa_cases_and_reports_missing_evidence(tmp_path):
+    manifest = tmp_path / "cases.json"
+    manifest.write_text(json.dumps({
+        "cases": [
+            {
+                "case_id": "case-a",
+                "subject": {"name": "A", "birth_source": {"url": "https://birth.example/a", "time_accuracy_rating": "AA"}},
+                "event_outcomes": [{"event_date": "2000-01-01"}],
+            },
+            {
+                "case_id": "case-a-duplicate",
+                "subject": {"name": "A", "birth_source": {"url": "https://birth.example/a", "time_accuracy_rating": "AA"}},
+                "event_outcomes": [{"event_date": "2001-01-01"}],
+            },
+            {
+                "case_id": "case-b",
+                "subject": {"name": "B", "birth_source": {"url": "https://birth.example/b", "time_accuracy_rating": "A"}},
+            },
+        ],
+    }), encoding="utf-8")
+
+    audit = build_source_audit([manifest])
+
+    assert audit["public_aa_case_count"] == 1
+    assert audit["cases"][0]["subject"] == "A"
+    assert audit["cases"][0]["additional_dated_events_required"] == 1
+    assert audit["cases"][0]["negative_controls_required"] == 4
+
+
+def test_frozen_source_audit_matches_the_reusable_public_case_manifests():
+    artifact = json.loads((ROOT / "references/real_case_calibration/minute_rectification_public_aa_source_audit_v1.json").read_text(encoding="utf-8"))
+
+    assert artifact == build_source_audit()
+    assert artifact["public_aa_case_count"] == 17
+    assert artifact["additional_public_aa_cases_required"] == 3