diff --git a/.gitignore b/.gitignore index c2675f67..5d614461 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,8 @@ __pycache__/ *.sqlite .agents/ venv_vedastro/ -scratch/local/ -scratch/test_*.json -scratch/test_*.txt +/scratch/ +/.serena/ test_hooks.json hermes_memory.db/ *.egg-info/ diff --git a/references/real_case_calibration/catalog.schema.json b/references/real_case_calibration/catalog.schema.json index cc579bda..1377f301 100644 --- a/references/real_case_calibration/catalog.schema.json +++ b/references/real_case_calibration/catalog.schema.json @@ -2,138 +2,86 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Real Case Calibration Catalog", "type": "object", - "required": [ - "case_id", - "source", - "chart_signature", - "event_outcomes", - "similarity", - "replay" - ], + "required": ["case_id", "subject", "source", "chart_signature", "event_outcomes", "similarity", "replay"], "properties": { - "case_id": { - "type": "string" - }, - "source": { + "case_id": {"type": "string"}, + "subject": { "type": "object", - "required": [ - "url", - "source_grade", - "license_or_quote_boundary" - ], + "required": ["name", "year", "month", "day", "hour", "minute", "lat", "lon", "tz", "node_mode", "birth_source"], "properties": { - "url": { - "type": "string" - }, - "source_grade": { - "type": "string", - "enum": [ - "primary", - "verified_secondary", - "forum_claim", - "unverified" - ] - }, - "license_or_quote_boundary": { - "type": "string" - } - } - }, - "chart_signature": { - "type": "object", - "properties": { - "lagna": { - "type": "string" - }, - "moon_sign": { - "type": "string" - }, - "d9_lagna": { - "type": "string" - }, - "ul": { - "type": "string" - }, - "a7": { - "type": "string" - }, - "a10": { - "type": "string" - }, - "notable_yogas": { - "type": "array", - "items": { - "type": "string" + "name": {"type": "string"}, + "year": {"type": "integer"}, + "month": {"type": "integer", "minimum": 1, "maximum": 12}, + "day": {"type": "integer", "minimum": 1, "maximum": 31}, + "hour": {"type": "integer", "minimum": 0, "maximum": 23}, + "minute": {"type": "integer", "minimum": 0, "maximum": 59}, + "lat": {"type": "number"}, + "lon": {"type": "number"}, + "tz": {"type": "number"}, + "node_mode": {"enum": ["mean", "true"]}, + "birth_source": { + "type": "object", + "required": ["url", "source_grade", "time_accuracy_rating", "evidence_basis"], + "properties": { + "url": {"type": "string"}, + "source_grade": {"enum": ["primary", "verified_secondary"]}, + "time_accuracy_rating": {"enum": ["A", "AA"]}, + "evidence_basis": {"type": "string"} } } } }, + "source": { + "type": "object", + "required": ["url", "source_grade", "license_or_quote_boundary"], + "properties": { + "url": {"type": "string"}, + "source_grade": {"enum": ["primary", "verified_secondary", "forum_claim", "unverified"]}, + "license_or_quote_boundary": {"type": "string"} + } + }, + "chart_signature": {"type": "object"}, "event_outcomes": { "type": "array", + "minItems": 1, "items": { "type": "object", - "required": [ - "event_type", - "event_date", - "outcome" - ], + "required": ["event_type", "event_date", "domain", "expected_label", "outcome", "source"], "properties": { - "event_type": { - "type": "string" - }, - "event_date": { - "type": "string" - }, - "outcome": { - "type": "string" - }, - "source_excerpt_note": { - "type": "string" + "event_type": {"type": "string"}, + "event_date": {"type": "string"}, + "domain": {"enum": ["career", "marriage"]}, + "expected_label": {"enum": ["career_status", "legal_marriage"]}, + "outcome": {"type": "string"}, + "source_excerpt_note": {"type": "string"}, + "source": { + "type": "object", + "required": ["url", "source_grade"], + "properties": { + "url": {"type": "string"}, + "source_grade": {"enum": ["primary", "verified_secondary"]} + } } } } }, "similarity": { "type": "object", + "required": ["score", "matching_factors", "dissimilar_factors"], "properties": { - "score": { - "type": "number" - }, - "matching_factors": { - "type": "array", - "items": { - "type": "string" - } - }, - "dissimilar_factors": { - "type": "array", - "items": { - "type": "string" - } - } + "score": {"type": "number"}, + "matching_factors": {"type": "array", "items": {"type": "string"}}, + "dissimilar_factors": {"type": "array", "items": {"type": "string"}} } }, "replay": { "type": "object", + "required": ["outcome_replay_status", "do_not_use_for_prediction"], "properties": { - "outcome_replay_status": { - "type": "string", - "enum": [ - "not_started", - "blocked", - "partial", - "complete" - ] - }, - "conflict_notes": { - "type": "array", - "items": { - "type": "string" - } - }, - "do_not_use_for_prediction": { - "type": "boolean" - } + "outcome_replay_status": {"enum": ["pending", "replayed", "blocked"]}, + "do_not_use_for_prediction": {"type": "boolean"}, + "report_path": {"type": "string"}, + "conflict_notes": {"type": "array", "items": {"type": "string"}} } } } diff --git a/references/real_case_calibration/replay_manifest.json b/references/real_case_calibration/replay_manifest.json index 42ad725a..c323cb8a 100644 --- a/references/real_case_calibration/replay_manifest.json +++ b/references/real_case_calibration/replay_manifest.json @@ -1,8 +1,144 @@ { - "schema_version": "1.0", - "status": "contract_ready_no_cases", + "schema_version": "2.0", + "status": "ready", "case_schema": "references/real_case_calibration/catalog.schema.json", - "cases": [], - "blocked_reason": "no_structured_outcome_replay_cases_imported", - "runtime_boundary": "This manifest defines the replay import contract only. No real-case outcome replay is complete until structured cases are imported and validated." + "selection_policy": { + "birth_time_minimum": "Rodden A", + "event_source_minimum": "verified_secondary", + "domains": {"career": 5, "marriage": 5}, + "privacy": "public_figures_only_no_user_birth_data" + }, + "cases": [ + { + "case_id": "jobs_iphone_2007", + "subject": { + "name": "Steve Jobs", "year": 1955, "month": 2, "day": 24, "hour": 19, "minute": 15, + "lat": 37.7833, "lon": -122.4167, "tz": -8.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_02_24.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "BC/BR in hand"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_02_24.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "career_breakthrough", "event_date": "2007-01-09", "domain": "career", "expected_label": "career_status", "outcome": "Apple publicly introduced the iPhone", "source_excerpt_note": "Apple Newsroom dates the announcement to 9 January 2007.", "source": {"url": "https://www.apple.com/newsroom/2007/01/09Apple-Reinvents-the-Phone-with-iPhone/", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "obama_election_2008", + "subject": { + "name": "Barack Obama", "year": 1961, "month": 8, "day": 4, "hour": 19, "minute": 24, + "lat": 21.3, "lon": -157.8667, "tz": -10.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_08_04.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "BC/BR in hand"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_08_04.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "career_status", "event_date": "2008-11-04", "domain": "career", "expected_label": "career_status", "outcome": "Won the United States presidential election", "source_excerpt_note": "Federal Elections 2008 records the presidential general election.", "source": {"url": "https://www.fec.gov/introduction-campaign-finance/election-results-and-voting-information/federal-elections-2008/", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "schwarzenegger_governor_2003", + "subject": { + "name": "Arnold Schwarzenegger", "year": 1947, "month": 7, "day": 30, "hour": 4, "minute": 10, + "lat": 47.0833, "lon": 15.45, "tz": 2.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_07_30.htm", "source_grade": "verified_secondary", "time_accuracy_rating": "A", "evidence_basis": "from memory"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_07_30.htm", "source_grade": "verified_secondary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "career_status", "event_date": "2003-10-07", "domain": "career", "expected_label": "career_status", "outcome": "Won the California gubernatorial recall election", "source_excerpt_note": "California Secretary of State Statement of Vote dates the election to 7 October 2003.", "source": {"url": "https://elections.cdn.sos.ca.gov/sov/2003-special/sov-complete.pdf", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "streep_oscar_1983", + "subject": { + "name": "Meryl Streep", "year": 1949, "month": 6, "day": 22, "hour": 8, "minute": 5, + "lat": 40.7333, "lon": -74.3667, "tz": -4.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_06_22.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "BC/BR in hand"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_06_22.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "career_award", "event_date": "1983-04-11", "domain": "career", "expected_label": "career_status", "outcome": "Won Best Actress for Sophie's Choice", "source_excerpt_note": "The 55th Academy Awards ceremony occurred on 11 April 1983.", "source": {"url": "https://www.oscars.org/oscars/ceremonies/1983", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "aniston_emmy_2002", + "subject": { + "name": "Jennifer Aniston", "year": 1969, "month": 2, "day": 11, "hour": 22, "minute": 22, + "lat": 34.05, "lon": -118.25, "tz": -8.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_02_11.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "quoted BC/BR"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_02_11.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "career_award", "event_date": "2002-09-22", "domain": "career", "expected_label": "career_status", "outcome": "Won the Primetime Emmy for lead actress in a comedy series", "source_excerpt_note": "Television Academy lists the 2002 category result.", "source": {"url": "https://www.televisionacademy.com/awards/nominees-winners/2002/outstanding-lead-actress-in-a-comedy-series", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "william_marriage_2011", + "subject": { + "name": "William, Prince of Wales", "year": 1982, "month": 6, "day": 21, "hour": 21, "minute": 3, + "lat": 51.5333, "lon": -0.2, "tz": 1.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_06_21.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "BC/BR in hand"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_06_21.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "legal_marriage", "event_date": "2011-04-29", "domain": "marriage", "expected_label": "legal_marriage", "outcome": "Married Catherine Middleton", "source_excerpt_note": "Royal Family records the wedding on 29 April 2011.", "source": {"url": "https://www.royal.uk/wedding-prince-william-and-miss-catherine-middleton", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "jolie_marriage_2014", + "subject": { + "name": "Angelina Jolie", "year": 1975, "month": 6, "day": 4, "hour": 9, "minute": 9, + "lat": 34.0961, "lon": -118.2944, "tz": -7.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_06_04.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "quoted BC/BR"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_06_04.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "legal_marriage", "event_date": "2014-08-23", "domain": "marriage", "expected_label": "legal_marriage", "outcome": "Married Brad Pitt", "source_excerpt_note": "Public biography records the private wedding on 23 August 2014.", "source": {"url": "https://en.wikipedia.org/wiki/Angelina_Jolie", "source_grade": "verified_secondary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "kahlo_marriage_1929", + "subject": { + "name": "Frida Kahlo", "year": 1907, "month": 7, "day": 6, "hour": 8, "minute": 30, + "lat": 19.3333, "lon": -99.1667, "tz": -6.6111, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_07_06.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "BC/BR in hand"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_07_06.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending", "timezone_note": "Astro-Databank LMT m99w10"}, + "event_outcomes": [{"event_type": "legal_marriage", "event_date": "1929-08-21", "domain": "marriage", "expected_label": "legal_marriage", "outcome": "Married Diego Rivera in a civil ceremony", "source_excerpt_note": "Biography cites the Coyoacan civil ceremony on 21 August 1929.", "source": {"url": "https://en.wikipedia.org/wiki/Frida_Kahlo", "source_grade": "verified_secondary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "snoop_marriage_1997", + "subject": { + "name": "Snoop Dogg", "year": 1971, "month": 10, "day": 20, "hour": 18, "minute": 20, + "lat": 33.7667, "lon": -118.1833, "tz": -7.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_10_20.htm", "source_grade": "primary", "time_accuracy_rating": "AA", "evidence_basis": "BC/BR in hand"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_10_20.htm", "source_grade": "primary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "legal_marriage", "event_date": "1997-06-14", "domain": "marriage", "expected_label": "legal_marriage", "outcome": "Married Shante Taylor", "source_excerpt_note": "Public biography records the marriage on 14 June 1997.", "source": {"url": "https://en.wikipedia.org/wiki/Snoop_Dogg", "source_grade": "verified_secondary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + }, + { + "case_id": "disney_marriage_1925", + "subject": { + "name": "Walt Disney", "year": 1901, "month": 12, "day": 5, "hour": 0, "minute": 35, + "lat": 41.85, "lon": -87.65, "tz": -6.0, "node_mode": "mean", + "birth_source": {"url": "https://www.astro.com/adbvip/adbvip_12_05.htm", "source_grade": "verified_secondary", "time_accuracy_rating": "A", "evidence_basis": "from memory"} + }, + "source": {"url": "https://www.astro.com/adbvip/adbvip_12_05.htm", "source_grade": "verified_secondary", "license_or_quote_boundary": "facts_and_short_summary_only"}, + "chart_signature": {"benchmark_role": "blind_outcome_replay", "external_oracle_status": "pending"}, + "event_outcomes": [{"event_type": "legal_marriage", "event_date": "1925-07-13", "domain": "marriage", "expected_label": "legal_marriage", "outcome": "Married Lillian Bounds", "source_excerpt_note": "Walt Disney Family Museum dates the marriage to 13 July 1925.", "source": {"url": "https://www.waltdisney.org/blog/who-did-walt-disney-marry", "source_grade": "primary"}}], + "similarity": {"score": 0.0, "matching_factors": ["not_user_similarity_benchmark"], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": false, "report_path": "docs/benchmark/public_real_case_benchmark_2026_07_11.json"} + } + ], + "runtime_boundary": "Positive-event replay only. It measures technical activation recall on known dated events; it does not establish specificity, causal validity, or scientific predictive accuracy. External JHora/PyJHora/VedAstro raw parity remains separately audited." } diff --git a/scripts/muntha.py b/scripts/muntha.py index 1c3b5636..bc95102c 100644 --- a/scripts/muntha.py +++ b/scripts/muntha.py @@ -15,7 +15,7 @@ Muntha 是 Tajika 年运盘（Varshaphala）中的核心指标， 注意：不同流派对 Muntha 计算公式有微小差异。 本实现采用最广泛接受的方法。 """ -from typing import Dict, Optional +from typing import Dict, List, Optional from datetime import datetime, timedelta diff --git a/scripts/public_real_case_benchmark.py b/scripts/public_real_case_benchmark.py new file mode 100644 index 00000000..ad272f44 --- /dev/null +++ b/scripts/public_real_case_benchmark.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Replay research-grade public events through the local Jyotish evidence stack.""" + +from __future__ import annotations + +import argparse +import copy +import json +import subprocess +import sys +from datetime import date +from pathlib import Path +from typing import Any + +from scripts.functional_benefics import derive_functional_benefic_malefic +from scripts.narayana_dasha import narayana_dasha_full_report + + +ROOT = Path(__file__).resolve().parents[1] +ENGINE = ROOT / "scripts" / "jyotish_engine.py" +SIGNS = [ + "Aries", "Taurus", "Gemini", "Cancer", "Leo", "Virgo", + "Libra", "Scorpio", "Sagittarius", "Capricorn", "Aquarius", "Pisces", +] +EVENT_HOUSES = {"career": [10, 6, 9, 11], "marriage": [7, 2, 11, 5]} +EVENT_KARAKAS = {"career": {"Sun", "Saturn", "Mercury"}, "marriage": {"Venus", "Jupiter"}} +PRIMARY_HOUSE = {"career": 10, "marriage": 7} +EXPECTED_LABEL = {"career": "career_status", "marriage": "legal_marriage"} +SIGN_LORDS = { + "Aries": "Mars", "Taurus": "Venus", "Gemini": "Mercury", "Cancer": "Moon", + "Leo": "Sun", "Virgo": "Mercury", "Libra": "Venus", "Scorpio": "Mars", + "Sagittarius": "Jupiter", "Capricorn": "Saturn", "Aquarius": "Saturn", "Pisces": "Jupiter", +} +_ENGINE_JSON_CACHE: dict[str, dict[str, Any]] = {} + + +def clear_engine_cache() -> None: + _ENGINE_JSON_CACHE.clear() + + +def summarize_results(rows: list[dict[str, Any]]) -> dict[str, Any]: + total = len(rows) + blocked = sum(bool(row.get("blocked")) for row in rows) + evaluated = total - blocked + hits = sum(row.get("result_class") in {"strong_hit", "weak_hit"} for row in rows if not row.get("blocked")) + exact = sum(bool(row.get("matched_expected_label")) for row in rows if not row.get("blocked")) + activation_rate = hits / evaluated if evaluated else None + strong_rate = exact / evaluated if evaluated else None + return { + "total_events": total, + "evaluated_events": evaluated, + "strong_hits": sum(row.get("result_class") == "strong_hit" for row in rows), + "weak_hits": sum(row.get("result_class") == "weak_hit" for row in rows), + "misses": sum(row.get("result_class") == "miss" for row in rows), + "blocked_events": blocked, + "known_event_activation_rate": activation_rate, + "strong_activation_rate": strong_rate, + "positive_event_recall": activation_rate, + "positive_event_recall_deprecated": True, + "exact_label_rate": strong_rate, + "exact_label_rate_deprecated": True, + "blocked_rate": blocked / total if total else None, + "balanced_accuracy": None, + "balanced_accuracy_blocked_reason": "no_verified_negative_control_dates", + } + + +def promotion_decision(v1: dict[str, Any], v2: dict[str, Any]) -> dict[str, Any]: + if int(v2.get("blocked_events") or 0) > int(v1.get("blocked_events") or 0): + return {"promote": False, "reason": "v2_increased_blocked_events"} + recall1 = v1.get("positive_event_recall") + recall2 = v2.get("positive_event_recall") + exact1 = v1.get("exact_label_rate") + exact2 = v2.get("exact_label_rate") + if None in {recall1, recall2, exact1, exact2}: + return {"promote": False, "reason": "comparison_metric_missing"} + improved = recall2 >= recall1 and exact2 >= exact1 and (recall2 > recall1 or exact2 > exact1) + return {"promote": improved, "reason": "holdout_metrics_improved" if improved else "no_holdout_improvement"} + + +def compare_reports(v1: dict[str, Any], v2: dict[str, Any]) -> dict[str, Any]: + """Compare frozen rule versions without reinterpreting holdout outcomes.""" + v1_cases = {row["case_id"]: row for row in v1.get("cases") or []} + v2_cases = {row["case_id"]: row for row in v2.get("cases") or []} + deltas = [] + for case_id in sorted(v1_cases.keys() & v2_cases.keys()): + before = v1_cases[case_id] + after = v2_cases[case_id] + before_signals = set(before.get("signals") or []) + deltas.append({ + "case_id": case_id, + "v1_score": before.get("score"), + "v2_score": after.get("score"), + "score_delta": (after.get("score") or 0) - (before.get("score") or 0), + "v1_result_class": before.get("result_class"), + "v2_result_class": after.get("result_class"), + "added_signals": sorted(set(after.get("signals") or []) - before_signals), + }) + return { + "benchmark_id": "public_real_case_holdout_comparison_2026_07_11", + "boundary": "Blind positive-event holdout comparison; no negative controls and no scientific accuracy claim.", + "v1_summary": v1.get("summary") or {}, + "v2_summary": v2.get("summary") or {}, + "promotion": promotion_decision(v1.get("summary") or {}, v2.get("summary") or {}), + "case_deltas": deltas, + } + + +def combine_reports(reports: list[dict[str, Any]], promotion: dict[str, Any]) -> dict[str, Any]: + rows = [row for report in reports for row in report.get("cases") or []] + return { + "benchmark_id": "public_real_case_20_case_closure_2026_07_11", + "rule_version": "v2", + "method": { + "cohorts": ["batch1_discovery_10", "frozen_holdout_10"], + "selection": "Rodden A/AA public figures with independently dated public career or legal-marriage events", + "score_thresholds": {"strong_hit": ">=7", "weak_hit": "4-6", "miss": "<4"}, + }, + "summary": summarize_results(rows), + "domain_summaries": { + domain: summarize_results([row for row in rows if row.get("domain") == domain]) + for domain in ("career", "marriage") + }, + "holdout_promotion": promotion, + "boundary": "Twenty positive public events; no negative controls, specificity estimate, or scientific accuracy claim.", + "technique_audit": [ + {"technique": "D1 + Functional Benefic/Malefic", "status": "used", "scope": "20/20"}, + {"technique": "D9/UL/Darakaraka", "status": "used", "scope": "10 marriage events"}, + {"technique": "D10/A10/Amatyakaraka", "status": "used", "scope": "10 career events"}, + {"technique": "Vimshottari MD/AD", "status": "used", "scope": "20/20"}, + {"technique": "Narayana Dasha", "status": "used", "scope": "20/20"}, + {"technique": "Double Transit PAC", "status": "used", "scope": "20/20"}, + {"technique": "Rahu/Ketu dispositor", "status": "used", "scope": "v2 scoring"}, + {"technique": "Vimshottari PD/PrAD", "status": "partial", "reason": "ratio expansion available but not externally validated or scored"}, + {"technique": "Tajika/Varshaphala/Muntha", "status": "partial", "reason": "local annual layer remains simplified and external oracle closure is incomplete"}, + {"technique": "KP exact cusp/significators", "status": "partial", "reason": "current local KP house layer uses sign-center approximation rather than exact cusps"}, + {"technique": "VedAstro official raw", "status": "blocked", "reason": "official_snapshot_budget_exhausted"}, + {"technique": "PyJHora/JHora/jyotishganit parity", "status": "blocked", "reason": "external canonical raw comparison incomplete"}, + {"technique": "MEVG / Global Web Evidence", "status": "used", "scope": "20 public birth/event source pairs"}, + {"technique": "Real Case Calibration", "status": "used", "scope": "10 discovery + 10 frozen holdout"}, + {"technique": "Negative controls", "status": "blocked", "reason": "no verified non-event dates"}, + ], + "technique_debt": { + "vimshottari_pd_prad": "available_ratio_expansion_not_scored_or_externally_validated", + "tajika_varshaphala_muntha": "available_experimental_not_scored_due_simplified_year_lord_and_oracle_gap", + "kp_cusp_significators": "partial_not_scored_house_centers_are_not_precise_cusps", + "annual_transit_to_arudha_or_ul": "untested_candidate_layer", + "negative_control_dates": "missing_blocks_balanced_accuracy", + }, + "cases": rows, + } + + +def node_dispositor_bonus( + active_lords: set[str], + domain: str, + chart: dict[str, Any], + roles: dict[str, Any], +) -> tuple[int, list[str]]: + event_houses = set(EVENT_HOUSES[domain]) + score = 0 + signals: list[str] = [] + planets = chart.get("planets") or {} + for node in sorted(active_lords & {"Rahu", "Ketu"}): + node_sign = (planets.get(node) or {}).get("sign") + dispositor = SIGN_LORDS.get(node_sign) + if not dispositor: + continue + if set((roles.get("owned_houses") or {}).get(dispositor) or []) & event_houses: + score += 1 + signals.append(f"{node}_dispositor_{dispositor}_owns_event_house") + occupied = (planets.get(dispositor) or {}).get("house") + if occupied in event_houses: + score += 1 + signals.append(f"{node}_dispositor_{dispositor}_occupies_event_house:{occupied}") + return score, signals + + +def _house_from_sign(ascendant: str, target: str) -> int | None: + if ascendant not in SIGNS or target not in SIGNS: + return None + return (SIGNS.index(target) - SIGNS.index(ascendant)) % 12 + 1 + + +def varga_and_karaka_bonus( + active_lords: set[str], + domain: str, + varga: dict[str, Any], + jaimini: dict[str, Any], +) -> tuple[int, list[str]]: + chart_key = "D10_Dasamsa" if domain == "career" else "D9_Navamsa" + chart = ((varga.get("divisional_charts") or {}).get(chart_key) or {}) + ascendant = chart.get("ascendant") + primary_house = PRIMARY_HOUSE[domain] + primary_sign = SIGNS[(SIGNS.index(ascendant) + primary_house - 1) % 12] if ascendant in SIGNS else None + primary_lord = SIGN_LORDS.get(primary_sign) + lagna_lord = SIGN_LORDS.get(ascendant) + score = 0 + signals: list[str] = [] + label = "D10" if domain == "career" else "D9" + for lord in sorted(active_lords): + if lord == lagna_lord: + score += 1 + signals.append(f"active_dasha_matches_{label}_Lagna_lord:{lord}") + if lord == primary_lord: + score += 1 + signals.append(f"active_dasha_matches_{label}_{primary_house}L:{lord}") + lord_sign = (chart.get(lord) or {}).get("sign") + if _house_from_sign(ascendant, lord_sign) == primary_house: + score += 1 + signals.append(f"active_dasha_occupies_{label}_house_{primary_house}:{lord}") + karaka_name = "Amatyakaraka" if domain == "career" else "Darakaraka" + karaka_planet = ((((jaimini.get("chara_karaka_7") or {}).get("karaka_table") or {}).get(karaka_name) or {}).get("planet")) + if karaka_planet in active_lords: + score += 1 + signals.append(f"active_dasha_matches_{karaka_name}:{karaka_planet}") + return score, signals + + +def _engine_json(command: str, subject: dict[str, Any], *extra: str, timeout: int = 30) -> dict[str, Any]: + cache_key = json.dumps( + {"command": command, "subject": subject, "extra": extra}, + sort_keys=True, + ensure_ascii=True, + default=str, + ) + if cache_key in _ENGINE_JSON_CACHE: + return copy.deepcopy(_ENGINE_JSON_CACHE[cache_key]) + args = [ + sys.executable, str(ENGINE), command, + "--year", str(subject["year"]), "--month", str(subject["month"]), + "--day", str(subject["day"]), "--hour", str(subject["hour"]), + "--minute", str(subject["minute"]), "--lat", str(subject["lat"]), + "--lon", str(subject["lon"]), "--tz", str(subject["tz"]), + "--node-mode", str(subject.get("node_mode", "mean")), + *extra, + ] + completed = subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout) + payload = json.loads(completed.stdout) + _ENGINE_JSON_CACHE[cache_key] = payload + return copy.deepcopy(payload) + + +def _find_dasha(dasha: dict[str, Any], event_date: str) -> tuple[str | None, str | None]: + target = date.fromisoformat(event_date) + for md in dasha.get("timeline") or []: + if date.fromisoformat(md["start"][:10]) <= target < date.fromisoformat(md["end"][:10]): + for ad in md.get("antardasha_timeline") or []: + if date.fromisoformat(ad["start"][:10]) <= target < date.fromisoformat(ad["end"][:10]): + return md.get("lord"), ad.get("lord") + return md.get("lord"), None + return None, None + + +def _planet_score(planet: str | None, event_houses: set[int], chart: dict[str, Any], roles: dict[str, Any], karakas: set[str]) -> tuple[int, list[str]]: + if not planet: + return 0, [] + score = 0 + signals: list[str] = [] + owned = set((roles.get("owned_houses") or {}).get(planet) or []) + occupied = (chart.get("planets") or {}).get(planet, {}).get("house") + owned_hits = sorted(owned & event_houses) + if owned_hits: + score += 2 + signals.append(f"{planet}_owns_event_houses:{owned_hits}") + if occupied in event_houses: + score += 1 + signals.append(f"{planet}_occupies_event_house:{occupied}") + if planet in karakas: + score += 1 + signals.append(f"{planet}_domain_karaka") + return score, signals + + +def score_active_dasha_lords( + lords: list[str | None], + event_houses: set[int], + chart: dict[str, Any], + roles: dict[str, Any], + karakas: set[str], +) -> tuple[int, list[str]]: + score = 0 + signals: list[str] = [] + for lord in dict.fromkeys(lord for lord in lords if lord): + points, lord_signals = _planet_score(lord, event_houses, chart, roles, karakas) + score += points + signals.extend(lord_signals) + return score, signals + + +def _transit_json(event_date: str, subject: dict[str, Any]) -> dict[str, Any]: + target = date.fromisoformat(event_date) + command = [ + sys.executable, str(ENGINE), "transit", + "--year", str(target.year), "--month", str(target.month), "--day", str(target.day), + "--planet", "Jupiter,Saturn", "--tz", str(subject["tz"]), + "--node-mode", str(subject.get("node_mode", "mean")), + ] + completed = subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True, timeout=30) + return json.loads(completed.stdout) + + +def ashtakavarga_audit(domain: str, packet: dict[str, Any], transit: dict[str, Any]) -> dict[str, Any]: + event_houses = EVENT_HOUSES[domain] + sav = packet.get("sav") or {} + bav = packet.get("bav") or {} + event_house_sav = { + str(house): (packet.get("house_scores") or {}).get(f"house_{house}") + for house in event_houses + } + transit_support = {} + for planet in ("Jupiter", "Saturn"): + sign = ((transit.get("planets") or {}).get(planet) or {}).get("sign") + sign_index = SIGNS.index(sign) if sign in SIGNS else None + bindus = ((bav.get(planet) or {}).get("bindus") or []) + transit_support[planet] = { + "sign": sign, + "sav": (sav.get("scores") or {}).get(sign), + "bav": bindus[sign_index] if sign_index is not None and sign_index < len(bindus) else None, + } + return { + "status": "used_non_scoring", + "scoring_effect": 0, + "method": packet.get("method"), + "version": packet.get("version"), + "sav_total": sav.get("total"), + "sav_valid": sav.get("valid"), + "all_bav_valid": packet.get("all_bav_valid"), + "event_house_sav": event_house_sav, + "transit_support": transit_support, + "settings": { + "ayanamsa": transit.get("ayanamsa"), + "node_mode": transit.get("node_mode"), + }, + "boundary": "Audit evidence only. SAV/BAV does not change V2.1 event scores until a fresh holdout validates it.", + } + + +def _narayana_at_event(subject: dict[str, Any], chart: dict[str, Any], event_date: str) -> dict[str, Any]: + asc_sign = chart["ascendant"]["sign"] + asc_idx = SIGNS.index(asc_sign) + planet_lons = {name: data["degree"] for name, data in chart["planets"].items() if "degree" in data} + born = date(subject["year"], subject["month"], subject["day"]) + target = date.fromisoformat(event_date) + age = (target - born).days / 365.2425 + report = narayana_dasha_full_report(asc_idx, planet_lons, current_age=age, birth_year=subject["year"]) + return report.get("current_dasha") or {} + + +def _arudha_lord(jaimini: dict[str, Any], domain: str) -> str | None: + arudha = jaimini.get("arudha_padas") or {} + if domain == "career": + return ((arudha.get("padas") or {}).get("A10") or {}).get("lord") + return (arudha.get("upapada") or {}).get("lord") + + +def _double_transit_score(packet: dict[str, Any]) -> tuple[int, list[str]]: + strengths = [row.get("strength") for row in packet.get("double_transit") or []] + if "strong" in strengths: + return 2, ["double_transit_pac_strong"] + if strengths: + return 1, ["double_transit_pac_present"] + return 0, [] + + +def replay_case(case: dict[str, Any], rule_version: str = "v1") -> dict[str, Any]: + subject = case["subject"] + event = case["event_outcomes"][0] + domain = event["domain"] + event_houses = set(EVENT_HOUSES[domain]) + try: + chart = _engine_json("chart", subject) + dasha = _engine_json("dasha", subject, "--years", "100") + varga = _engine_json("varga", subject, "--d10" if domain == "career" else "--d9") + jaimini = _engine_json("jaimini", subject) + pac = _engine_json( + "double-transit-pac", subject, + "--date", event["event_date"], "--house", str(PRIMARY_HOUSE[domain]), + ) + except (subprocess.SubprocessError, json.JSONDecodeError, KeyError, ValueError) as exc: + return { + "case_id": case["case_id"], "name": subject["name"], "domain": domain, + "event_date": event["event_date"], "blocked": True, "result_class": "blocked", + "matched_expected_label": False, "blocked_reason": f"{type(exc).__name__}: {exc}", + } + + roles = derive_functional_benefic_malefic(chart["ascendant"]["sign"]) + md, ad = _find_dasha(dasha, event["event_date"]) + score = 0 + signals: list[str] = [] + if rule_version == "v2_1": + score, signals = score_active_dasha_lords([md, ad], event_houses, chart, roles, EVENT_KARAKAS[domain]) + else: + for lord in (md, ad): + points, lord_signals = _planet_score(lord, event_houses, chart, roles, EVENT_KARAKAS[domain]) + score += points + signals.extend(lord_signals) + + active_lords = {lord for lord in (md, ad) if lord} + if rule_version in {"v2", "v2_1"}: + node_points, node_signals = node_dispositor_bonus(active_lords, domain, chart, roles) + varga_points, varga_signals = varga_and_karaka_bonus(active_lords, domain, varga, jaimini) + score += node_points + varga_points + signals.extend(node_signals) + signals.extend(varga_signals) + + arudha_lord = _arudha_lord(jaimini, domain) + if arudha_lord in {md, ad}: + score += 1 + signals.append(f"active_dasha_matches_{'A10' if domain == 'career' else 'UL'}_lord:{arudha_lord}") + + narayana = _narayana_at_event(subject, chart, event["event_date"]) + narayana_md = narayana.get("md") or {} + event_sign = SIGNS[(SIGNS.index(chart["ascendant"]["sign"]) + PRIMARY_HOUSE[domain] - 1) % 12] + if narayana_md.get("sign") == event_sign: + score += 2 + signals.append(f"narayana_activates_primary_event_sign:{event_sign}") + narayana_lord = narayana_md.get("lord") + if set((roles.get("owned_houses") or {}).get(narayana_lord) or []) & event_houses: + score += 1 + signals.append(f"narayana_lord_owns_event_house:{narayana_lord}") + + pac_points, pac_signals = _double_transit_score(pac) + score += pac_points + signals.extend(pac_signals) + + ashtakavarga = {"status": "not_run", "scoring_effect": 0} + if rule_version == "v2_1": + try: + ashtakavarga_packet = _engine_json("ashtakavarga", subject) + transit_packet = _transit_json(event["event_date"], subject) + ashtakavarga = ashtakavarga_audit(domain, ashtakavarga_packet, transit_packet) + except (subprocess.SubprocessError, json.JSONDecodeError, KeyError, ValueError) as exc: + ashtakavarga = { + "status": "blocked", + "scoring_effect": 0, + "blocked_reason": f"{type(exc).__name__}: {exc}", + } + + if score >= 7: + result_class = "strong_hit" + actual_label = EXPECTED_LABEL[domain] + elif score >= 4: + result_class = "weak_hit" + actual_label = "domain_activation" + else: + result_class = "miss" + actual_label = None + + return { + "case_id": case["case_id"], + "name": subject["name"], + "domain": domain, + "event_date": event["event_date"], + "outcome": event["outcome"], + "birth_time_rating": subject["birth_source"]["time_accuracy_rating"], + "rule_version": rule_version, + "blocked": False, + "result_class": result_class, + "score": score, + "expected_label": EXPECTED_LABEL[domain], + "actual_label": actual_label, + "matched_expected_label": actual_label == EXPECTED_LABEL[domain], + "signals": signals, + "evidence": { + "ascendant": chart["ascendant"], + "vimshottari": {"mahadasha": md, "antardasha": ad}, + "narayana": narayana, + "functional_benefic_malefic": roles, + "domain_varga": varga, + "arudha_lord": arudha_lord, + "double_transit_pac": pac, + "ashtakavarga_audit": ashtakavarga, + "birth_source": subject["birth_source"], + "event_source": event["source"], + }, + } + + +def build_report( + manifest: dict[str, Any], + strict_probe_blocked_reason: str | None = None, + rule_version: str = "v1", +) -> dict[str, Any]: + rows = [replay_case(case, rule_version=rule_version) for case in manifest.get("cases") or []] + return { + "benchmark_id": "public_real_case_benchmark_2026_07_11", + "rule_version": rule_version, + "method": { + "selection": "Rodden A/AA public figures with independently dated public events", + "pre_registered_layers": ["D1", "D9_or_D10", "UL_or_A10", "Functional Benefic/Malefic", "Vimshottari MD/AD", "Narayana Dasha", "Double Transit PAC"] + (["Rahu/Ketu dispositor", "D9/D10 Lagna and primary-house lord", "Amatyakaraka/Darakaraka"] if rule_version in {"v2", "v2_1"} else []) + (["SAV/BAV non-scoring audit", "deduplicated MD/AD lord scoring"] if rule_version == "v2_1" else []), + "score_thresholds": {"strong_hit": ">=7", "weak_hit": "4-6", "miss": "<4"}, + "boundary": "Positive-event technical activation replay; not scientific predictive accuracy.", + }, + "summary": summarize_results(rows), + "strict_workflow_batch": { + "status": "blocked" if strict_probe_blocked_reason else "not_run", + "blocked_reason": strict_probe_blocked_reason, + }, + "external_oracle_boundary": { + "VedAstro": "diagnostic_only_unless_official_raw_present", + "PyJHora": "blocked_or_benchmark_only_until_dependency_available", + "JHora": "manual_oracle_not_automated", + "jyotishganit": "parity_contract_separate_from_this_event_replay", + }, + "cases": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", default="references/real_case_calibration/replay_manifest.json") + parser.add_argument("--output") + parser.add_argument("--strict-probe-blocked-reason") + parser.add_argument("--rule-version", choices=["v1", "v2", "v2_1", "compare"], default="v1") + parser.add_argument("--comparison-v1") + parser.add_argument("--comparison-v2") + args = parser.parse_args() + manifest = json.loads((ROOT / args.manifest).read_text(encoding="utf-8")) + if args.rule_version == "compare": + if not args.comparison_v1 or not args.comparison_v2: + parser.error("compare requires --comparison-v1 and --comparison-v2 to avoid duplicate engine replay") + v1 = json.loads((ROOT / args.comparison_v1).read_text(encoding="utf-8")) + v2 = json.loads((ROOT / args.comparison_v2).read_text(encoding="utf-8")) + report = compare_reports(v1, v2) + else: + report = build_report(manifest, args.strict_probe_blocked_reason, rule_version=args.rule_version) + payload = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if args.output: + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(payload, encoding="utf-8") + print(payload, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/public_real_case_negative_controls.py b/scripts/public_real_case_negative_controls.py new file mode 100644 index 00000000..dd1e0b00 --- /dev/null +++ b/scripts/public_real_case_negative_controls.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Rank known public event dates against deterministic non-target control dates.""" + +from __future__ import annotations + +import argparse +import copy +import json +import statistics +from datetime import date, timedelta +from pathlib import Path +from typing import Any, Iterable + +from scripts.public_real_case_benchmark import clear_engine_cache, replay_case + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_OFFSETS = (-120, -90, -60, -30, 30, 60, 90, 120) + + +def generate_control_dates(event_date: str, offsets: Iterable[int] = DEFAULT_OFFSETS) -> list[str]: + target = date.fromisoformat(event_date) + return [(target + timedelta(days=int(offset))).isoformat() for offset in offsets if int(offset) != 0] + + +def rank_positive_against_controls(positive_score: int, control_scores: list[int]) -> dict[str, Any]: + rank = 1 + sum(score >= positive_score for score in control_scores) + max_control = max(control_scores) if control_scores else None + return { + "positive_score": positive_score, + "positive_rank": rank, + "candidate_count": len(control_scores) + 1, + "reciprocal_rank": 1 / rank, + "top_1": rank == 1, + "top_3": rank <= 3, + "max_control_score": max_control, + "score_margin": positive_score - max_control if max_control is not None else None, + } + + +def summarize_negative_control_rows(rows: list[dict[str, Any]]) -> dict[str, Any]: + controls = [control for row in rows for control in row.get("controls") or [] if not control.get("blocked")] + rankings = [row["ranking"] for row in rows if row.get("ranking")] + margins = [item["score_margin"] for item in rankings if item["score_margin"] is not None] + return { + "case_count": len(rows), + "ranked_case_count": len(rankings), + "control_date_count": len(controls), + "control_activation_rate": sum((control.get("score") or 0) >= 4 for control in controls) / len(controls) if controls else None, + "control_strong_activation_rate": sum((control.get("score") or 0) >= 7 for control in controls) / len(controls) if controls else None, + "positive_top_1_rate": sum(item["top_1"] for item in rankings) / len(rankings) if rankings else None, + "positive_top_3_rate": sum(item["top_3"] for item in rankings) / len(rankings) if rankings else None, + "mean_reciprocal_rank": statistics.mean(item["reciprocal_rank"] for item in rankings) if rankings else None, + "mean_score_margin": statistics.mean(margins) if margins else None, + "balanced_accuracy": None, + "balanced_accuracy_blocked_reason": "controls_are_non_target_dates_not_independently_adjudicated_all-domain_non_events", + } + + +def build_report(manifest: dict[str, Any], offsets: Iterable[int] = DEFAULT_OFFSETS) -> dict[str, Any]: + clear_engine_cache() + rows = [] + for case in manifest.get("cases") or []: + event = case["event_outcomes"][0] + positive = replay_case(case, rule_version="v2_1") + controls = [] + for control_date in generate_control_dates(event["event_date"], offsets): + control_case = copy.deepcopy(case) + control_event = control_case["event_outcomes"][0] + control_event["event_date"] = control_date + control_event["outcome"] = f"non_target_control_date_for:{event['outcome']}" + result = replay_case(control_case, rule_version="v2_1") + controls.append({ + "date": control_date, + "score": result.get("score"), + "result_class": result.get("result_class"), + "blocked": bool(result.get("blocked")), + "blocked_reason": result.get("blocked_reason"), + }) + control_scores = [int(item["score"]) for item in controls if not item["blocked"] and item.get("score") is not None] + ranking = None + if not positive.get("blocked") and positive.get("score") is not None: + ranking = rank_positive_against_controls(int(positive["score"]), control_scores) + rows.append({ + "case_id": case["case_id"], + "name": case["subject"]["name"], + "domain": event["domain"], + "positive_date": event["event_date"], + "positive": positive, + "controls": controls, + "ranking": ranking, + }) + return { + "benchmark_id": "public_real_case_negative_control_pilot_2026_07_11", + "rule_version": "v2_1", + "control_offsets_days": list(offsets), + "summary": summarize_negative_control_rows(rows), + "boundary": ( + "Controls are dates without the exact recorded target outcome. They may contain other life events. " + "This pilot measures date ranking and false domain activation, not scientific causal validity." + ), + "cases": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", default="references/real_case_calibration/replay_manifest_probe3_v2.json") + parser.add_argument("--output") + args = parser.parse_args() + manifest = json.loads((ROOT / args.manifest).read_text(encoding="utf-8")) + report = build_report(manifest) + payload = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if args.output: + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(payload, encoding="utf-8") + print(payload, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/real_case_replay_validator.py b/scripts/real_case_replay_validator.py index b48fb509..8ec41ad2 100644 --- a/scripts/real_case_replay_validator.py +++ b/scripts/real_case_replay_validator.py @@ -11,6 +11,7 @@ from typing import Any CASE_REQUIRED_FIELDS = { "case_id", + "subject", "source", "chart_signature", "event_outcomes", @@ -18,7 +19,18 @@ CASE_REQUIRED_FIELDS = { "replay", } SOURCE_REQUIRED_FIELDS = {"url", "source_grade", "license_or_quote_boundary"} -EVENT_REQUIRED_FIELDS = {"event_type", "event_date", "outcome"} +SUBJECT_REQUIRED_FIELDS = { + "name", "year", "month", "day", "hour", "minute", "lat", "lon", "tz", + "node_mode", "birth_source", +} +BIRTH_SOURCE_REQUIRED_FIELDS = { + "url", "source_grade", "time_accuracy_rating", "evidence_basis", +} +EVENT_REQUIRED_FIELDS = { + "event_type", "event_date", "domain", "expected_label", "outcome", "source", +} +EVENT_SOURCE_REQUIRED_FIELDS = {"url", "source_grade"} +ALLOWED_BIRTH_TIME_RATINGS = {"A", "AA"} def _missing(mapping: dict[str, Any], required: set[str]) -> list[str]: @@ -40,6 +52,26 @@ def _case_errors(case: Any, index: int) -> list[dict[str, Any]]: elif "source" in case: errors.append({"case_id": case.get("case_id"), "field": "source", "error": "not_object"}) + subject = case.get("subject") + if isinstance(subject, dict): + for field in _missing(subject, SUBJECT_REQUIRED_FIELDS): + errors.append({"case_id": case.get("case_id"), "field": f"subject.{field}", "error": "missing"}) + birth_source = subject.get("birth_source") + if isinstance(birth_source, dict): + for field in _missing(birth_source, BIRTH_SOURCE_REQUIRED_FIELDS): + errors.append({"case_id": case.get("case_id"), "field": f"subject.birth_source.{field}", "error": "missing"}) + rating = birth_source.get("time_accuracy_rating") + if rating not in ALLOWED_BIRTH_TIME_RATINGS: + errors.append({ + "case_id": case.get("case_id"), + "field": "subject.birth_source.time_accuracy_rating", + "error": "birth_time_rating_below_A", + }) + elif "birth_source" in subject: + errors.append({"case_id": case.get("case_id"), "field": "subject.birth_source", "error": "not_object"}) + elif "subject" in case: + errors.append({"case_id": case.get("case_id"), "field": "subject", "error": "not_object"}) + events = case.get("event_outcomes") if isinstance(events, list): if not events: @@ -50,6 +82,12 @@ def _case_errors(case: Any, index: int) -> list[dict[str, Any]]: continue for field in _missing(event, EVENT_REQUIRED_FIELDS): errors.append({"case_id": case.get("case_id"), "field": f"event_outcomes[{event_index}].{field}", "error": "missing"}) + event_source = event.get("source") + if isinstance(event_source, dict): + for field in _missing(event_source, EVENT_SOURCE_REQUIRED_FIELDS): + errors.append({"case_id": case.get("case_id"), "field": f"event_outcomes[{event_index}].source.{field}", "error": "missing"}) + elif "source" in event: + errors.append({"case_id": case.get("case_id"), "field": f"event_outcomes[{event_index}].source", "error": "not_object"}) elif "event_outcomes" in case: errors.append({"case_id": case.get("case_id"), "field": "event_outcomes", "error": "not_array"}) @@ -72,12 +110,24 @@ def validate_manifest(path: str | Path) -> dict[str, Any]: errors: list[dict[str, Any]] = [] replay_ready_count = 0 + domain_counts: dict[str, int] = {} + birth_time_ratings: dict[str, int] = {} for index, case in enumerate(cases): case_errors = _case_errors(case, index) errors.extend(case_errors) replay = case.get("replay") if isinstance(case, dict) else {} if not case_errors and isinstance(replay, dict) and replay.get("outcome_replay_status") == "replayed": replay_ready_count += 1 + if isinstance(case, dict): + subject = case.get("subject") or {} + birth_source = subject.get("birth_source") if isinstance(subject, dict) else {} + rating = birth_source.get("time_accuracy_rating") if isinstance(birth_source, dict) else None + if isinstance(rating, str): + birth_time_ratings[rating] = birth_time_ratings.get(rating, 0) + 1 + for event in case.get("event_outcomes") or []: + if isinstance(event, dict) and isinstance(event.get("domain"), str): + domain = event["domain"] + domain_counts[domain] = domain_counts.get(domain, 0) + 1 if errors: status = "invalid" @@ -99,6 +149,8 @@ def validate_manifest(path: str | Path) -> dict[str, Any]: "case_schema": manifest.get("case_schema"), "case_count": len(cases), "replay_ready_count": replay_ready_count, + "domain_counts": dict(sorted(domain_counts.items())), + "birth_time_ratings": dict(sorted(birth_time_ratings.items())), "blocked_reason": blocked_reason, "errors": errors, "runtime_boundary": manifest.get("runtime_boundary", ""), diff --git a/scripts/unified_consultation_orchestrator.py b/scripts/unified_consultation_orchestrator.py index c3d85c0b..7bbeed85 100644 --- a/scripts/unified_consultation_orchestrator.py +++ b/scripts/unified_consultation_orchestrator.py @@ -4,6 +4,7 @@ from __future__ import annotations +import json from dataclasses import dataclass from pathlib import Path from typing import Any @@ -479,6 +480,109 @@ class UnifiedConsultationOrchestrator: } replay_manifest_path = Path(__file__).resolve().parents[1] / "references/real_case_calibration/replay_manifest.json" replay_manifest = validate_real_case_replay_manifest(replay_manifest_path) + holdout_manifest_path = Path(__file__).resolve().parents[1] / "references/real_case_calibration/replay_manifest_holdout_v2.json" + holdout_manifest = validate_real_case_replay_manifest(holdout_manifest_path) + benchmark_path = Path(__file__).resolve().parents[1] / "docs/benchmark/public_real_case_20_case_closure_2026_07_11.json" + if benchmark_path.exists(): + benchmark_payload = json.loads(benchmark_path.read_text(encoding="utf-8")) + public_outcome_benchmark = { + "status": "used", + "path": "docs/benchmark/public_real_case_20_case_closure_2026_07_11.json", + "summary": benchmark_payload.get("summary") or {}, + "method": benchmark_payload.get("method") or {}, + "strict_workflow_batch": benchmark_payload.get("strict_workflow_batch") or {}, + "holdout_promotion": benchmark_payload.get("holdout_promotion") or {}, + "technique_debt": benchmark_payload.get("technique_debt") or {}, + } + else: + public_outcome_benchmark = { + "status": "blocked", + "path": "docs/benchmark/public_real_case_20_case_closure_2026_07_11.json", + "blocked_reason": "public_outcome_benchmark_missing", + } + supplemental_path = Path(__file__).resolve().parents[1] / "docs/benchmark/public_real_case_probe3_v2_2026_07_11.json" + combined_observation_path = Path(__file__).resolve().parents[1] / "docs/benchmark/public_real_case_23_case_observation_2026_07_11.json" + if supplemental_path.exists() and combined_observation_path.exists(): + supplemental_payload = json.loads(supplemental_path.read_text(encoding="utf-8")) + combined_payload = json.loads(combined_observation_path.read_text(encoding="utf-8")) + supplemental_public_probe = { + "status": "used", + "path": "docs/benchmark/public_real_case_probe3_v2_2026_07_11.json", + "summary": supplemental_payload.get("summary") or {}, + "combined_observation": combined_payload.get("summary") or {}, + "boundary": "Three-case independent probe is contradictory generalization evidence, not a promotion or accuracy estimate.", + } + else: + supplemental_public_probe = { + "status": "blocked", + "blocked_reason": "supplemental_public_probe_missing", + } + corrected_v21_path = Path(__file__).resolve().parents[1] / "docs/benchmark/public_real_case_23_case_v21_corrected_observation_2026_07_11.json" + if corrected_v21_path.exists(): + corrected_payload = json.loads(corrected_v21_path.read_text(encoding="utf-8")) + corrected_v21_observation = { + "status": "used", + "path": "docs/benchmark/public_real_case_23_case_v21_corrected_observation_2026_07_11.json", + "summary": corrected_payload.get("summary") or {}, + "domain_summaries": corrected_payload.get("domain_summaries") or {}, + "ashtakavarga_audit_status": corrected_payload.get("ashtakavarga_audit_status"), + "ashtakavarga_descriptive": corrected_payload.get("ashtakavarga_descriptive") or {}, + "boundary": corrected_payload.get("boundary"), + } + else: + corrected_v21_observation = { + "status": "blocked", + "blocked_reason": "corrected_v21_observation_missing", + } + negative_control_path = Path(__file__).resolve().parents[1] / "docs/benchmark/public_real_case_negative_control_pilot_2026_07_11.json" + if negative_control_path.exists(): + negative_payload = json.loads(negative_control_path.read_text(encoding="utf-8")) + negative_summary = negative_payload.get("summary") or {} + negative_control_pilot = { + "status": "used", + "path": "docs/benchmark/public_real_case_negative_control_pilot_2026_07_11.json", + "summary": negative_summary, + "boundary": negative_payload.get("boundary"), + } + else: + negative_control_pilot = { + "status": "blocked", + "blocked_reason": "negative_control_pilot_missing", + } + negative_summary = {} + annual_control_path = Path(__file__).resolve().parents[1] / "docs/benchmark/public_real_case_annual_control_pilot_2026_07_11.json" + if annual_control_path.exists(): + annual_payload = json.loads(annual_control_path.read_text(encoding="utf-8")) + annual_control_pilot = { + "status": "used", + "path": "docs/benchmark/public_real_case_annual_control_pilot_2026_07_11.json", + "summary": annual_payload.get("summary") or {}, + "boundary": annual_payload.get("boundary"), + } + else: + annual_control_pilot = { + "status": "blocked", + "blocked_reason": "annual_control_pilot_missing", + } + if negative_control_pilot.get("status") == "used" and annual_control_pilot.get("status") == "used": + timing_precision_gate = { + "status": "blocked", + "maximum_supported_precision": "unvalidated_broad_window", + "blocked_claims": ["exact_day", "exact_month_from_current_replay_score"], + "domain_support": {"career": "blocked", "marriage": "partial_candidate"}, + "reason": "near_and_annual_control_rankings_below_gate", + "observed_positive_top_1_rate": negative_summary.get("positive_top_1_rate"), + "observed_positive_top_3_rate": negative_summary.get("positive_top_3_rate"), + "annual_positive_top_1_rate": (annual_control_pilot.get("summary") or {}).get("positive_top_1_rate"), + } + else: + timing_precision_gate = { + "status": "blocked", + "maximum_supported_precision": "unvalidated_broad_window", + "blocked_claims": ["exact_day", "exact_month_from_current_replay_score"], + "domain_support": {"career": "blocked", "marriage": "partial_candidate"}, + "reason": "control_pilot_missing", + } candidate_refs = case_index_by_domain.get(route, []) packet = machine_evidence_packet if isinstance(machine_evidence_packet, dict) else {} sections = packet.get("sections") if isinstance(packet.get("sections"), dict) else {} @@ -532,16 +636,24 @@ class UnifiedConsultationOrchestrator: "status": "partial_scored" if scored_candidates else "catalog_available_matching_not_run", "batch_id": "real_case_studies_batch1", "route": route, - "source_roots": ["references/real_case_studies", "docs/benchmark"], + "source_roots": ["references/real_case_studies", "references/real_case_calibration", "docs/benchmark"], "case_index_by_domain": case_index_by_domain, "required_replay_schema": "references/real_case_calibration/catalog.schema.json", "outcome_replay_manifest": replay_manifest, + "holdout_replay_manifest": holdout_manifest, + "public_outcome_benchmark": public_outcome_benchmark, + "supplemental_public_probe": supplemental_public_probe, + "corrected_v21_observation": corrected_v21_observation, + "negative_control_pilot": negative_control_pilot, + "annual_control_pilot": annual_control_pilot, + "timing_precision_gate": timing_precision_gate, "candidate_refs": list(candidate_refs), "scored_candidates": scored_candidates, "reference_grade": scored_candidates[0]["reference_grade"] if scored_candidates else "ungraded_until_similarity_scored", "boundary": ( - "Local case catalog has route, evidence-section, and timing-evidence scoring only; concrete event " - "outcome matching must run before a case can be used as complete calibration evidence." + "The public benchmark replays twenty dated outcomes, including a frozen ten-case holdout, but it contains positive events only. It can " + "measure activation recall, not specificity or scientific predictive accuracy; user-chart " + "similarity still requires separate structured matching." ), } @@ -586,6 +698,8 @@ class UnifiedConsultationOrchestrator: blocked_items.append("vedastro_official_raw_archive_manifest_missing") case_packet = real_case_calibration if isinstance(real_case_calibration, dict) else {} case_status = case_packet.get("status") or "required_not_satisfied" + timing_precision = case_packet.get("timing_precision_gate") if isinstance(case_packet.get("timing_precision_gate"), dict) else {} + timing_precision_status = timing_precision.get("status") or "blocked" functional_packet = packet.get("functional_benefic_malefic") if isinstance(packet.get("functional_benefic_malefic"), dict) else {} functional_status = functional_packet.get("status") or "blocked" if functional_status != "used": @@ -594,6 +708,8 @@ class UnifiedConsultationOrchestrator: blocked_items.append("real_case_calibration_not_yet_materialized") elif case_status != "complete": blocked_items.append("real_case_calibration_partial") + if timing_precision_status != "pass": + blocked_items.append("timing_precision_gate_blocked") cross_system_arbitration = build_cross_system_arbitration( route_packet=route_packet, jyotish_evidence=packet, @@ -657,6 +773,15 @@ class UnifiedConsultationOrchestrator: "used": bool(case_packet), "effect_on_confidence": "partial_reference_only_until_outcome_replay" if case_status != "complete" else "supports_calibration", }, + { + "technique": "Timing Precision Gate", + "status": timing_precision_status, + "used": bool(timing_precision), + "maximum_supported_precision": timing_precision.get("maximum_supported_precision", "unvalidated_broad_window"), + "blocked_claims": timing_precision.get("blocked_claims", ["exact_day", "exact_month_from_current_replay_score"]), + "domain_support": timing_precision.get("domain_support", {}), + "effect_on_confidence": "blocks_false_precision_until_control_date_rankings_pass", + }, { "technique": "Functional Benefic/Malefic", "status": functional_status, @@ -727,6 +852,7 @@ class UnifiedConsultationOrchestrator: "Blind Technical Mode", "MEVG / Global Web Evidence", "Real Case Calibration", + "Timing Precision Gate", "Functional Benefic/Malefic", ], "status": "blocked" if blocked_items else "pass", diff --git a/tests/test_muntha_module.py b/tests/test_muntha_module.py new file mode 100644 index 00000000..8f4be13d --- /dev/null +++ b/tests/test_muntha_module.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from scripts.muntha import calc_muntha_from_sun_sign + + +def test_standalone_muntha_module_imports_and_calculates() -> None: + result = calc_muntha_from_sun_sign(0, 12) + + assert result["muntha_sign"] == 11 + assert result["muntha_lord"] == "Jupiter" diff --git a/tests/test_public_real_case_benchmark.py b/tests/test_public_real_case_benchmark.py new file mode 100644 index 00000000..644bf5c9 --- /dev/null +++ b/tests/test_public_real_case_benchmark.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.public_real_case_benchmark import ( + _engine_json, + ashtakavarga_audit, + clear_engine_cache, + combine_reports, + compare_reports, + node_dispositor_bonus, + promotion_decision, + score_active_dasha_lords, + summarize_results, + varga_and_karaka_bonus, +) + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_summary_reports_positive_recall_without_inventing_specificity() -> None: + summary = summarize_results( + [ + {"result_class": "strong_hit", "matched_expected_label": True, "blocked": False}, + {"result_class": "weak_hit", "matched_expected_label": False, "blocked": False}, + {"result_class": "miss", "matched_expected_label": False, "blocked": False}, + {"result_class": "blocked", "matched_expected_label": False, "blocked": True}, + ] + ) + assert summary["positive_event_recall"] == 2 / 3 + assert summary["exact_label_rate"] == 1 / 3 + assert summary["blocked_rate"] == 1 / 4 + assert summary["balanced_accuracy"] is None + assert summary["balanced_accuracy_blocked_reason"] == "no_verified_negative_control_dates" + assert summary["known_event_activation_rate"] == 2 / 3 + assert summary["strong_activation_rate"] == 1 / 3 + assert summary["positive_event_recall_deprecated"] is True + assert summary["exact_label_rate_deprecated"] is True + + +def test_v21_deduplicates_same_md_ad_lord() -> None: + chart = {"planets": {"Jupiter": {"house": 10}}} + roles = {"owned_houses": {"Jupiter": [9, 11]}} + + score, signals = score_active_dasha_lords( + ["Jupiter", "Jupiter"], {6, 9, 10, 11}, chart, roles, {"Sun", "Saturn", "Mercury"} + ) + + assert score == 3 + assert signals.count("Jupiter_owns_event_houses:[9, 11]") == 1 + assert signals.count("Jupiter_occupies_event_house:10") == 1 + + +def test_ashtakavarga_audit_reports_event_houses_and_transit_bav_without_scoring() -> None: + audit = ashtakavarga_audit( + "marriage", + { + "method": "Ashtakavarga", + "version": "2.1", + "sav": {"total": 337, "valid": True, "scores": {"Libra": 30, "Sagittarius": 27}}, + "all_bav_valid": True, + "house_scores": { + "house_2": {"sign": "Leo", "sav_score": 24}, + "house_5": {"sign": "Scorpio", "sav_score": 29}, + "house_7": {"sign": "Capricorn", "sav_score": 31}, + "house_11": {"sign": "Taurus", "sav_score": 32}, + }, + "bav": { + "Jupiter": {"bindus": [0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0], "total": 56, "valid": True}, + "Saturn": {"bindus": [0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0], "total": 39, "valid": True}, + }, + }, + { + "ayanamsa": 24.1, + "node_mode": "mean", + "planets": {"Jupiter": {"sign": "Libra"}, "Saturn": {"sign": "Sagittarius"}}, + }, + ) + + assert audit["status"] == "used_non_scoring" + assert audit["sav_total"] == 337 + assert audit["event_house_sav"]["7"]["sav_score"] == 31 + assert audit["transit_support"]["Jupiter"] == {"sign": "Libra", "sav": 30, "bav": 5} + assert audit["transit_support"]["Saturn"] == {"sign": "Sagittarius", "sav": 27, "bav": 4} + + +def test_engine_json_cache_reuses_identical_subject_command(monkeypatch) -> None: + calls = [] + + class Completed: + stdout = '{"ok": true}' + + def fake_run(*args, **kwargs): + calls.append((args, kwargs)) + return Completed() + + clear_engine_cache() + monkeypatch.setattr("scripts.public_real_case_benchmark.subprocess.run", fake_run) + subject = {"year": 2000, "month": 1, "day": 1, "hour": 12, "minute": 0, "lat": 0, "lon": 0, "tz": 0} + + assert _engine_json("chart", subject) == {"ok": True} + assert _engine_json("chart", subject) == {"ok": True} + assert len(calls) == 1 + clear_engine_cache() + + +def test_committed_report_matches_the_ten_case_manifest() -> None: + manifest = json.loads((ROOT / "references/real_case_calibration/replay_manifest.json").read_text(encoding="utf-8")) + report = json.loads((ROOT / "docs/benchmark/public_real_case_benchmark_2026_07_11.json").read_text(encoding="utf-8")) + assert {case["case_id"] for case in manifest["cases"]} == {case["case_id"] for case in report["cases"]} + assert report["summary"]["total_events"] == 10 + assert report["summary"]["positive_event_recall"] == 0.8 + assert report["summary"]["balanced_accuracy"] is None + + +def test_v2_node_dispositor_adds_general_event_house_support() -> None: + chart = { + "planets": { + "Rahu": {"sign": "Taurus", "house": 3}, + "Venus": {"sign": "Capricorn", "house": 10}, + } + } + roles = {"owned_houses": {"Venus": [3, 10]}} + score, signals = node_dispositor_bonus({"Rahu"}, "career", chart, roles) + assert score == 2 + assert "Rahu_dispositor_Venus_owns_event_house" in signals + assert "Rahu_dispositor_Venus_occupies_event_house:10" in signals + + +def test_v2_varga_and_chara_karaka_support_is_domain_specific() -> None: + varga = { + "divisional_charts": { + "D10_Dasamsa": { + "ascendant": "Taurus", + "Saturn": {"sign": "Aquarius"}, + } + } + } + jaimini = { + "chara_karaka_7": { + "karaka_table": {"Amatyakaraka": {"planet": "Saturn"}} + } + } + score, signals = varga_and_karaka_bonus({"Saturn"}, "career", varga, jaimini) + assert score == 3 + assert "active_dasha_matches_D10_10L:Saturn" in signals + assert "active_dasha_occupies_D10_house_10:Saturn" in signals + assert "active_dasha_matches_Amatyakaraka:Saturn" in signals + + +def test_v2_promotion_requires_holdout_improvement_without_more_blocking() -> None: + promoted = promotion_decision( + {"positive_event_recall": 0.6, "exact_label_rate": 0.2, "blocked_events": 0}, + {"positive_event_recall": 0.8, "exact_label_rate": 0.4, "blocked_events": 0}, + ) + assert promoted["promote"] is True + blocked = promotion_decision( + {"positive_event_recall": 0.6, "exact_label_rate": 0.2, "blocked_events": 0}, + {"positive_event_recall": 0.8, "exact_label_rate": 0.4, "blocked_events": 1}, + ) + assert blocked["promote"] is False + assert blocked["reason"] == "v2_increased_blocked_events" + + +def test_compare_reports_keeps_case_level_deltas_auditable() -> None: + v1 = { + "rule_version": "v1", + "summary": {"positive_event_recall": 0.5, "exact_label_rate": 0.0, "blocked_events": 0}, + "cases": [{"case_id": "case-a", "score": 3, "result_class": "miss", "signals": ["base"]}], + } + v2 = { + "rule_version": "v2", + "summary": {"positive_event_recall": 1.0, "exact_label_rate": 1.0, "blocked_events": 0}, + "cases": [{"case_id": "case-a", "score": 7, "result_class": "strong_hit", "signals": ["base", "new"]}], + } + + comparison = compare_reports(v1, v2) + + assert comparison["promotion"]["promote"] is True + assert comparison["case_deltas"] == [ + { + "case_id": "case-a", + "v1_score": 3, + "v2_score": 7, + "score_delta": 4, + "v1_result_class": "miss", + "v2_result_class": "strong_hit", + "added_signals": ["new"], + } + ] + + +def test_combine_reports_recomputes_twenty_case_summary() -> None: + batch1 = {"cases": [{"case_id": "a", "result_class": "strong_hit", "matched_expected_label": True, "blocked": False}]} + holdout = {"cases": [{"case_id": "b", "result_class": "miss", "matched_expected_label": False, "blocked": False}]} + + combined = combine_reports([batch1, holdout], {"promote": True, "reason": "holdout_metrics_improved"}) + + assert combined["summary"]["total_events"] == 2 + assert combined["summary"]["positive_event_recall"] == 0.5 + assert combined["holdout_promotion"]["promote"] is True + assert [row["case_id"] for row in combined["cases"]] == ["a", "b"] diff --git a/tests/test_public_release_privacy_scan.py b/tests/test_public_release_privacy_scan.py index 24f9d15d..95e6b53f 100644 --- a/tests/test_public_release_privacy_scan.py +++ b/tests/test_public_release_privacy_scan.py @@ -29,3 +29,9 @@ def test_public_release_privacy_scan_supports_unpacked_zip_without_git(tmp_path: assert [path.name for path in iter_release_files(tmp_path)] == ["INSTALL.md"] report = build_report(tmp_path) assert report["status"] == "pass", report["findings"] + + +def test_private_workspace_directories_are_gitignored() -> None: + gitignore = (Path(__file__).resolve().parents[1] / ".gitignore").read_text(encoding="utf-8").splitlines() + assert "/scratch/" in gitignore + assert "/.serena/" in gitignore diff --git a/tests/test_real_case_negative_controls.py b/tests/test_real_case_negative_controls.py new file mode 100644 index 00000000..2eb8183d --- /dev/null +++ b/tests/test_real_case_negative_controls.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from scripts.public_real_case_negative_controls import ( + generate_control_dates, + rank_positive_against_controls, + summarize_negative_control_rows, +) + + +def test_generate_control_dates_uses_fixed_offsets_without_positive_date() -> None: + dates = generate_control_dates("2018-05-19", offsets=(-60, -30, 30, 60)) + + assert dates == ["2018-03-20", "2018-04-19", "2018-06-18", "2018-07-18"] + assert "2018-05-19" not in dates + + +def test_rank_positive_uses_conservative_tie_ordering() -> None: + result = rank_positive_against_controls(5, [7, 5, 4]) + + assert result == { + "positive_score": 5, + "positive_rank": 3, + "candidate_count": 4, + "reciprocal_rank": 1 / 3, + "top_1": False, + "top_3": True, + "max_control_score": 7, + "score_margin": -2, + } + + +def test_negative_control_summary_reports_false_activations() -> None: + summary = summarize_negative_control_rows( + [ + { + "ranking": {"top_1": True, "top_3": True, "reciprocal_rank": 1.0, "score_margin": 2}, + "controls": [{"score": 2}, {"score": 3}], + }, + { + "ranking": {"top_1": False, "top_3": True, "reciprocal_rank": 0.5, "score_margin": -1}, + "controls": [{"score": 4}, {"score": 7}], + }, + ] + ) + + assert summary["case_count"] == 2 + assert summary["control_date_count"] == 4 + assert summary["control_activation_rate"] == 0.5 + assert summary["control_strong_activation_rate"] == 0.25 + assert summary["positive_top_1_rate"] == 0.5 + assert summary["positive_top_3_rate"] == 1.0 + assert summary["mean_reciprocal_rank"] == 0.75 + assert summary["mean_score_margin"] == 0.5 diff --git a/tests/test_real_case_replay_validator.py b/tests/test_real_case_replay_validator.py index 85bb107c..fd9e6a24 100644 --- a/tests/test_real_case_replay_validator.py +++ b/tests/test_real_case_replay_validator.py @@ -9,13 +9,50 @@ from scripts.unified_consultation_orchestrator import UnifiedConsultationOrchest ROOT = Path(__file__).resolve().parents[1] -def test_real_case_replay_manifest_blocks_when_no_cases_are_imported() -> None: +def test_real_case_replay_manifest_contains_ten_research_grade_cases() -> None: result = validate_manifest(ROOT / "references/real_case_calibration/replay_manifest.json") - assert result["status"] == "blocked" - assert result["case_count"] == 0 - assert result["replay_ready_count"] == 0 - assert result["blocked_reason"] == "no_structured_outcome_replay_cases_imported" + assert result["status"] == "pass" + assert result["case_count"] == 10 + assert result["replay_ready_count"] == 10 + assert result["domain_counts"] == {"career": 5, "marriage": 5} + assert result["birth_time_ratings"] == {"A": 2, "AA": 8} + + +def test_holdout_manifest_contains_ten_new_balanced_cases() -> None: + batch1_path = ROOT / "references/real_case_calibration/replay_manifest.json" + holdout_path = ROOT / "references/real_case_calibration/replay_manifest_holdout_v2.json" + result = validate_manifest(holdout_path) + assert result["status"] == "pass" + assert result["case_count"] == 10 + assert result["domain_counts"] == {"career": 5, "marriage": 5} + assert result["birth_time_ratings"] == {"A": 5, "AA": 5} + batch1 = json.loads(batch1_path.read_text(encoding="utf-8")) + holdout = json.loads(holdout_path.read_text(encoding="utf-8")) + assert {case["subject"]["name"] for case in batch1["cases"]}.isdisjoint( + {case["subject"]["name"] for case in holdout["cases"]} + ) + + +def test_three_case_probe_is_aa_and_disjoint_from_prior_twenty() -> None: + probe_path = ROOT / "references/real_case_calibration/replay_manifest_probe3_v2.json" + result = validate_manifest(probe_path) + + assert result["status"] == "pass" + assert result["case_count"] == 3 + assert result["replay_ready_count"] == 3 + assert result["domain_counts"] == {"career": 2, "marriage": 1} + assert result["birth_time_ratings"] == {"AA": 3} + + prior_names = set() + for path in ( + ROOT / "references/real_case_calibration/replay_manifest.json", + ROOT / "references/real_case_calibration/replay_manifest_holdout_v2.json", + ): + payload = json.loads(path.read_text(encoding="utf-8")) + prior_names.update(case["subject"]["name"] for case in payload["cases"]) + probe = json.loads(probe_path.read_text(encoding="utf-8")) + assert prior_names.isdisjoint(case["subject"]["name"] for case in probe["cases"]) def test_real_case_replay_validator_accepts_one_structured_case(tmp_path: Path) -> None: @@ -26,6 +63,24 @@ def test_real_case_replay_validator_accepts_one_structured_case(tmp_path: Path) "cases": [ { "case_id": "public_case_001", + "subject": { + "name": "Public Case", + "year": 1970, + "month": 1, + "day": 1, + "hour": 12, + "minute": 0, + "lat": 0.0, + "lon": 0.0, + "tz": 0.0, + "node_mode": "mean", + "birth_source": { + "url": "https://example.com/birth-record", + "source_grade": "primary", + "time_accuracy_rating": "AA", + "evidence_basis": "birth_record_in_hand", + }, + }, "source": { "url": "https://example.com/public-case", "source_grade": "verified_secondary", @@ -36,7 +91,13 @@ def test_real_case_replay_validator_accepts_one_structured_case(tmp_path: Path) { "event_type": "career_breakthrough", "event_date": "2000-01", + "domain": "career", + "expected_label": "career_status", "outcome": "public_success", + "source": { + "url": "https://example.com/event", + "source_grade": "verified_secondary", + }, } ], "similarity": { @@ -61,6 +122,62 @@ def test_real_case_replay_validator_accepts_one_structured_case(tmp_path: Path) assert result["replay_ready_count"] == 1 +def test_real_case_replay_validator_rejects_low_accuracy_birth_time_and_unsourced_event(tmp_path: Path) -> None: + manifest = { + "schema_version": "2.0", + "status": "ready", + "case_schema": "references/real_case_calibration/catalog.schema.json", + "cases": [ + { + "case_id": "weak_case", + "subject": { + "name": "Weak Case", + "year": 1970, + "month": 1, + "day": 1, + "hour": 12, + "minute": 0, + "lat": 0.0, + "lon": 0.0, + "tz": 0.0, + "node_mode": "mean", + "birth_source": { + "url": "https://example.com/birth", + "source_grade": "unverified", + "time_accuracy_rating": "DD", + "evidence_basis": "conflicting_times", + }, + }, + "source": { + "url": "https://example.com/case", + "source_grade": "unverified", + "license_or_quote_boundary": "summary_only", + }, + "chart_signature": {}, + "event_outcomes": [ + { + "event_type": "legal_marriage", + "event_date": "2000-01-01", + "domain": "marriage", + "expected_label": "legal_marriage", + "outcome": "married", + } + ], + "similarity": {"score": 0.0, "matching_factors": [], "dissimilar_factors": []}, + "replay": {"outcome_replay_status": "replayed", "do_not_use_for_prediction": False}, + } + ], + } + path = tmp_path / "replay_manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + result = validate_manifest(path) + assert result["status"] == "invalid" + assert {error["error"] for error in result["errors"]} >= { + "birth_time_rating_below_A", + "missing", + } + + def test_orchestrator_exposes_real_case_replay_manifest_status() -> None: orchestrator = UnifiedConsultationOrchestrator() route = {"question_type": "career", "primary_theme": "career"} @@ -68,6 +185,56 @@ def test_orchestrator_exposes_real_case_replay_manifest_status() -> None: packet = orchestrator.real_case_calibration_catalog(route_packet=route, machine_evidence_packet={}) replay = packet["outcome_replay_manifest"] - assert replay["status"] == "blocked" - assert replay["case_count"] == 0 + assert replay["status"] == "pass" + assert replay["case_count"] == 10 + holdout = packet["holdout_replay_manifest"] + assert holdout["status"] == "pass" + assert holdout["case_count"] == 10 + benchmark = packet["public_outcome_benchmark"] + assert benchmark["status"] == "used" + assert benchmark["summary"]["total_events"] == 20 + assert benchmark["summary"]["positive_event_recall"] == 0.8 + assert benchmark["summary"]["exact_label_rate"] == 0.4 + assert benchmark["summary"]["balanced_accuracy"] is None + assert benchmark["holdout_promotion"] == {"promote": True, "reason": "holdout_metrics_improved"} + supplemental = packet["supplemental_public_probe"] + assert supplemental["status"] == "used" + assert supplemental["summary"]["total_events"] == 3 + assert supplemental["summary"]["positive_event_recall"] == 1 / 3 + assert supplemental["combined_observation"]["total_events"] == 23 + assert supplemental["combined_observation"]["positive_event_recall"] == 17 / 23 + corrected = packet["corrected_v21_observation"] + assert corrected["status"] == "used" + assert corrected["summary"]["total_events"] == 23 + assert corrected["summary"]["positive_event_recall_deprecated"] is True + assert corrected["ashtakavarga_audit_status"] == "used_non_scoring" + negative = packet["negative_control_pilot"] + assert negative["status"] == "used" + assert negative["summary"]["control_date_count"] == 24 + assert negative["summary"]["positive_top_1_rate"] == 0.0 + assert negative["summary"]["positive_top_3_rate"] == 0.0 + annual = packet["annual_control_pilot"] + assert annual["status"] == "used" + assert annual["summary"]["control_date_count"] == 12 + assert annual["summary"]["positive_top_1_rate"] == 1 / 3 + timing_gate = packet["timing_precision_gate"] + assert timing_gate["status"] == "blocked" + assert timing_gate["maximum_supported_precision"] == "unvalidated_broad_window" + assert timing_gate["blocked_claims"] == ["exact_day", "exact_month_from_current_replay_score"] + assert timing_gate["domain_support"] == {"career": "blocked", "marriage": "partial_candidate"} + runtime_log = orchestrator.runtime_evidence_log( + surface="api_web", + entry_mode="direct_chart", + route_packet=route, + executed_steps=["compute_chart"], + skipped_steps=[], + real_case_calibration=packet, + ) + assert "timing_precision_gate_blocked" in runtime_log["quality_gate"]["blocked_items"] + timing_row = next( + row for row in runtime_log["quality_gate"]["technique_audit_table"] + if row["technique"] == "Timing Precision Gate" + ) + assert timing_row["status"] == "blocked" + assert timing_row["maximum_supported_precision"] == "unvalidated_broad_window" assert packet["required_replay_schema"] == "references/real_case_calibration/catalog.schema.json" diff --git a/tests/test_unified_consultation_orchestrator.py b/tests/test_unified_consultation_orchestrator.py index d73bab02..865bec22 100644 --- a/tests/test_unified_consultation_orchestrator.py +++ b/tests/test_unified_consultation_orchestrator.py @@ -198,9 +198,10 @@ def test_runtime_evidence_log_exposes_blind_packet_case_and_quality_gate_contrac "Cross-System Arbitration", "Evidence Packet", "Blind Technical Mode", - "MEVG / Global Web Evidence", - "Real Case Calibration", - "Functional Benefic/Malefic", + "MEVG / Global Web Evidence", + "Real Case Calibration", + "Timing Precision Gate", + "Functional Benefic/Malefic", ] engines = log["external_engine_cross_validation"]["engines"] assert engines["VedAstro"]["status"] == "local_fallback"