Jyotisha/scripts/public_real_case_negative_controls.py

#!/usr/bin/env python3
"""Rank known public event dates against deterministic non-target control dates."""

from __future__ import annotations

import argparse
import copy
import json
import statistics
from datetime import date, timedelta
from pathlib import Path
from typing import Any, Iterable

from scripts.public_real_case_benchmark import clear_engine_cache, replay_case

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_OFFSETS = (-120, -90, -60, -30, 30, 60, 90, 120)


def generate_control_dates(event_date: str, offsets: Iterable[int] = DEFAULT_OFFSETS) -> list[str]:
    target = date.fromisoformat(event_date)
    return [(target + timedelta(days=int(offset))).isoformat() for offset in offsets if int(offset) != 0]


def rank_positive_against_controls(positive_score: int, control_scores: list[int]) -> dict[str, Any]:
    rank = 1 + sum(score >= positive_score for score in control_scores)
    max_control = max(control_scores) if control_scores else None
    return {
        "positive_score": positive_score,
        "positive_rank": rank,
        "candidate_count": len(control_scores) + 1,
        "reciprocal_rank": 1 / rank,
        "top_1": rank == 1,
        "top_3": rank <= 3,
        "max_control_score": max_control,
        "score_margin": positive_score - max_control if max_control is not None else None,
    }


def summarize_negative_control_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
    controls = [control for row in rows for control in row.get("controls") or [] if not control.get("blocked")]
    rankings = [row["ranking"] for row in rows if row.get("ranking")]
    margins = [item["score_margin"] for item in rankings if item["score_margin"] is not None]
    return {
        "case_count": len(rows),
        "ranked_case_count": len(rankings),
        "control_date_count": len(controls),
        "control_activation_rate": sum((control.get("score") or 0) >= 4 for control in controls) / len(controls) if controls else None,
        "control_strong_activation_rate": sum((control.get("score") or 0) >= 7 for control in controls) / len(controls) if controls else None,
        "positive_top_1_rate": sum(item["top_1"] for item in rankings) / len(rankings) if rankings else None,
        "positive_top_3_rate": sum(item["top_3"] for item in rankings) / len(rankings) if rankings else None,
        "mean_reciprocal_rank": statistics.mean(item["reciprocal_rank"] for item in rankings) if rankings else None,
        "mean_score_margin": statistics.mean(margins) if margins else None,
        "balanced_accuracy": None,
        "balanced_accuracy_blocked_reason": "controls_are_non_target_dates_not_independently_adjudicated_all-domain_non_events",
    }


def build_report(manifest: dict[str, Any], offsets: Iterable[int] = DEFAULT_OFFSETS) -> dict[str, Any]:
    clear_engine_cache()
    rows = []
    for case in manifest.get("cases") or []:
        event = case["event_outcomes"][0]
        positive = replay_case(case, rule_version="v2_1")
        controls = []
        for control_date in generate_control_dates(event["event_date"], offsets):
            control_case = copy.deepcopy(case)
            control_event = control_case["event_outcomes"][0]
            control_event["event_date"] = control_date
            control_event["outcome"] = f"non_target_control_date_for:{event['outcome']}"
            result = replay_case(control_case, rule_version="v2_1")
            controls.append({
                "date": control_date,
                "score": result.get("score"),
                "result_class": result.get("result_class"),
                "blocked": bool(result.get("blocked")),
                "blocked_reason": result.get("blocked_reason"),
            })
        control_scores = [int(item["score"]) for item in controls if not item["blocked"] and item.get("score") is not None]
        ranking = None
        if not positive.get("blocked") and positive.get("score") is not None:
            ranking = rank_positive_against_controls(int(positive["score"]), control_scores)
        rows.append({
            "case_id": case["case_id"],
            "name": case["subject"]["name"],
            "domain": event["domain"],
            "positive_date": event["event_date"],
            "positive": positive,
            "controls": controls,
            "ranking": ranking,
        })
    return {
        "benchmark_id": "public_real_case_negative_control_pilot_2026_07_11",
        "rule_version": "v2_1",
        "control_offsets_days": list(offsets),
        "summary": summarize_negative_control_rows(rows),
        "boundary": (
            "Controls are dates without the exact recorded target outcome. They may contain other life events. "
            "This pilot measures date ranking and false domain activation, not scientific causal validity."
        ),
        "cases": rows,
    }


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", default="references/real_case_calibration/replay_manifest_probe3_v2.json")
    parser.add_argument("--output")
    args = parser.parse_args()
    manifest = json.loads((ROOT / args.manifest).read_text(encoding="utf-8"))
    report = build_report(manifest)
    payload = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
    if args.output:
        output_path = ROOT / args.output
        output_path.parent.mkdir(parents=True, exist_ok=True)
        output_path.write_text(payload, encoding="utf-8")
    print(payload, end="")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())