root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
52bfbb2ac5574c309902bd6d875c475539cf4fac
Jyotisha/scripts/research/scoring_research.py
T

616 lines
35 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Offline research runner for TASK-rectification-scoring-research-20260929.
R-0 research scorer reconciled per minute against the production scorer, then
per-(candidate, event) rule features recorded (eleven production rules,
auxiliary rules, and unscored extras: KP cusp sub/star lords, Pranapada /
Hora / Ghati / Bhava lagna, D60).
R-A likelihood-ratio weights (Laplace α = 1) per radius, leave-one-CASE-out
and full-fit; variants A1 (production features, reward only), A2 (+extras,
reward only), A3 (+extras, naive-Bayes present/absent, negative allowed);
six-probe replay, engine top1, calibration curve, robustness (±7-day and
±1–3-month date shifts, 1–2 flipped answers).
R-B absence evidence: gap years inside a domain's spoken span answered "no"
at 0.5 / 1.0 / 2.0 points (a card is 2.0), plus "user omitted 1–2 events"
robustness.
R-C precision follow-up: with all day events degraded to year ("spoken"
form), how many events would split candidates if the month were known,
and the gain from asking 1 / 2 of them (as a prior recompute and as an
answered probe).
Every verdict field is written as ``pending_v5``: the task brief allows a
verdict only on the v5 open set. v4 numbers are debugging references.
Run:
PYTHONHASHSEED=0 python3 scripts/research/scoring_research.py --limit 2 --radii 10 --no-write
PYTHONHASHSEED=0 python3 scripts/research/scoring_research.py
"""
from __future__ import annotations
import argparse
import gc
import json
import pickle
import sys
import time as clock_module
import traceback
from collections import defaultdict
from pathlib import Path
from random import Random
from typing import Any, Sequence
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.active_rectification_event_engine import AYANAMSA, NODE_MODE, compute_candidate_static_contexts # noqa: E402
from scripts.rectification.scoring_service import build_event_contribution_matrix, score_from_matrix, scoreable_request # noqa: E402
from scripts.research.minute_resolution_sweep import MINUTE_STEP, scoring_request_for # noqa: E402
from scripts.research.offline_research_20260926_lib import exhaustive_flip_sets, flipped_answers, sample_flip_sets # noqa: E402
from scripts.research.precision_gate_lib import gate_verdict, jitter_day_events # noqa: E402
from scripts.research.probe_supply_after_six import ASK_COUNT, optimal_answer # noqa: E402
from scripts.active_rectification_events import precision_weight # noqa: E402
import scripts.research.scoring_research_lib as lib # noqa: E402
HOLDOUT_V4 = ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v4.json"
OUT_DIR = ROOT / "docs" / "research"
RADII = (10, 30, 60)
PRIORS = ("raw", "percent")
ABSENCE_POINTS = (0.5, 1.0, 2.0) # score points per "no"; a card is 2.0
CARD_POINTS = 2.0
OMIT_REPEATS = 5
SHIFT_REPEATS = 3
FLIP_REPEATS = 10
SEED = 20260929
ABSENCE_SOURCE = "absence_research"
FOLLOWUP_SOURCE = "precision_followup_research"
# ---------------------------------------------------------------------------
# per-case data
# ---------------------------------------------------------------------------
class CaseData:
def __init__(self, case: dict[str, Any], radius: int, cache_dir: Path | None) -> None:
self.case = case
self.case_id = str(case["case_id"])
self.radius = radius
self.true_time = str(case["birth"]["time"])[:5]
self.request = scoring_request_for({**case, "candidate_radius_minutes": radius}, radius)
self.contexts = self._contexts(cache_dir)
self.times = [ctx["candidate_at"].strftime("%H:%M") for ctx in self.contexts]
self.truth_times = lib.truth_cluster_times(self.contexts, self.true_time)
self.d60 = lib.d60_charts(self.contexts)
# production
prod_built = build_event_contribution_matrix(self.request, static_contexts=self.contexts)
self.prod_rows = score_from_matrix(self.request, prod_built)
self.prod_built = prod_built
self.probes = lib.production_probes(self.request, prod_built, self.times, self.true_time)
self.prod_public = lib.public_for(self.prod_rows, self.contexts)
# research recording pass (must reconcile to production)
self.store = lib.FeatureStore()
built = build_event_contribution_matrix(
self.request, row_provider=lib.make_recording_provider(self.contexts, self.store, self.d60),
static_contexts=self.contexts,
)
rows = score_from_matrix(self.request, built)
self.reconciliation = lib.reconcile_rows(self.prod_rows, rows)
self.training_ids = lib.training_event_ids(self.request)
self.representatives = lib.representatives_for(self.contexts)
self.events_by_id = {str(e["id"]): e for e in self.request["events"]}
def _contexts(self, cache_dir: Path | None) -> list[dict[str, Any]]:
if cache_dir is not None:
path = cache_dir / f"{self.case_id}_{self.radius}.pkl"
if path.exists():
with path.open("rb") as handle:
return pickle.load(handle)
contexts = compute_candidate_static_contexts(self.request)
if cache_dir is not None:
cache_dir.mkdir(parents=True, exist_ok=True)
with (cache_dir / f"{self.case_id}_{self.radius}.pkl").open("wb") as handle:
pickle.dump(contexts, handle)
return contexts
# features -------------------------------------------------------------
def examples(self, *, include_extras: bool) -> list[tuple[dict[str, float], bool]]:
out: list[tuple[dict[str, float], bool]] = []
truth = set(self.truth_times)
for matrix in self.feature_matrices(include_extras=include_extras).values():
for time in self.times:
out.append((matrix.get(time, {}), time in truth))
return out
def feature_matrices(self, *, include_extras: bool) -> dict[str, dict[str, dict[str, float]]]:
cache = self.__dict__.setdefault("_fm_cache", {})
if include_extras not in cache:
cache[include_extras] = {
event_id: lib.event_feature_matrix(self.store, event_id, self.times, include_extras=include_extras)
for event_id in self.training_ids
}
return cache[include_extras]
def proxy_rows(self, table: lib.LRTable, variant: str) -> list[dict[str, Any]]:
"""Linear proxy of `weighted_rows` for fitting scale / temperature (no matrix build)."""
include_extras, allow_negative = lib.VARIANT_SPEC[variant]
universe = [n for n in sorted(table.present) if (include_extras or not lib.is_extra_feature(n))]
matrices = self.feature_matrices(include_extras=include_extras)
rows = []
for time in self.times:
total = 0.0
for event_id, matrix in matrices.items():
weight = precision_weight(str(self.events_by_id[event_id]["precision"]))
total += weight * lib.weighted_points(matrix.get(time, {}), table, allow_negative=allow_negative, universe=universe)
rows.append({"time": time, "score": round(total, 6)})
return rows
# scoring with a weight table ------------------------------------------
def weighted_rows(
self, table: lib.LRTable, variant: str, scale: float,
store: lib.FeatureStore | None = None, request: dict[str, Any] | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
request = request or self.request
built = build_event_contribution_matrix(
request,
row_provider=lib.make_weighted_provider(store or self.store, table, variant=variant, scale=scale),
static_contexts=self.contexts,
)
return score_from_matrix(request, built), built
def rescored_store(self, events: Sequence[dict[str, Any]]) -> tuple[lib.FeatureStore, dict[str, Any], list[dict[str, Any]]]:
"""Re-record features for an altered event list (date shifts): (store, request, production rows)."""
request = scoring_request_for({**self.case, "events": list(events), "candidate_radius_minutes": self.radius}, self.radius)
store = lib.FeatureStore()
built = build_event_contribution_matrix(
request, row_provider=lib.make_recording_provider(self.contexts, store, self.d60), static_contexts=self.contexts,
)
return store, request, score_from_matrix(request, built)
def production_rows_for(self, events: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
request = scoring_request_for({**self.case, "events": list(events), "candidate_radius_minutes": self.radius}, self.radius)
built = build_event_contribution_matrix(request, static_contexts=self.contexts)
return score_from_matrix(request, built)
def evaluate(data: CaseData, rows: Sequence[dict[str, Any]], *, softmax_prior: bool, pre_answers=(), answers=None, temperature: float = 1.0, probes=None) -> dict[str, dict[str, Any]]:
public = lib.public_for(rows, data.contexts)
out: dict[str, dict[str, Any]] = {}
for mode in PRIORS:
prior = lib.prior_for(public, mode=mode, softmax=softmax_prior, temperature=temperature)
result = lib.replay(
public=public, prior=prior, probes=data.probes if probes is None else probes, true_time=data.true_time,
window_times=data.times, pre_answers=pre_answers, answers=answers,
)
result["engine_top1"] = lib.engine_top1(public, data.true_time)
out[mode] = result
return out
# ---------------------------------------------------------------------------
# R-A
# ---------------------------------------------------------------------------
def fit_prior(training: Sequence[CaseData], table: lib.LRTable, variant: str) -> tuple[float, float]:
"""(scale, temperature) from training cases only.
scale: research within-case score range matched to the production median
range (so the raw-prior replay's ±2 / 8-lead constants mean the same);
temperature: single Platt temperature for the percent (softmax) prior.
Training-case scores use the linear proxy (`CaseData.proxy_rows`: feature
weights × precision weight, no kind factor / transition proximity) so a
fold costs O(n) dictionary sums instead of n contribution-matrix builds.
"""
prod = [lib.within_case_range(d.prod_rows) for d in training]
rows_by_case = {d.case_id: d.proxy_rows(table, variant) for d in training}
research = [lib.within_case_range(rows) for rows in rows_by_case.values()]
p_med, r_med = lib.median(prod), lib.median(research)
scale = round(p_med / r_med, 6) if p_med and r_med else 1.0
publics = [
(lib.public_for([{**r, "score": float(r["score"]) * scale} for r in rows_by_case[d.case_id]], d.contexts), d.truth_times)
for d in training
]
return scale, lib.fit_temperature(publics)
def run_ra(by_radius: dict[int, list[CaseData]], *, fast: bool, probes_per_variant: bool = False) -> dict[str, Any]:
out: dict[str, Any] = {"variants": list(lib.VARIANTS), "alpha": lib.LAPLACE_ALPHA, "probes_per_variant": probes_per_variant, "radii": {}}
for radius, cases in sorted(by_radius.items()):
block: dict[str, Any] = {"n": len(cases), "baseline": {}, "full_fit": {}, "loo": {}, "robustness": {}, "calibration": {}, "lr_table": {}, "noise_features": {}}
baseline_rows = []
for data in cases:
row = evaluate(data, data.prod_rows, softmax_prior=False)
baseline_rows.append({"case_id": data.case_id, **{f"{m}": row[m] for m in PRIORS}})
block["baseline"] = {m: lib.summarize([r[m] for r in baseline_rows]) for m in PRIORS}
block["per_case"] = {"baseline": [{"case_id": r["case_id"], **{m: {k: r[m][k] for k in ("top1", "coverage", "width", "engine_top1")} for m in PRIORS}} for r in baseline_rows]}
per_case_examples = {
include: {d.case_id: d.examples(include_extras=include) for d in cases} for include in (False, True)
}
altered_stores: dict[tuple[str, str, int], tuple[lib.FeatureStore, dict[str, Any]]] = {}
for variant in lib.VARIANTS:
include_extras, _neg = lib.VARIANT_SPEC[variant]
examples = per_case_examples[include_extras]
# full fit -------------------------------------------------------
full_table = lib.estimate_lr([ex for rows in examples.values() for ex in rows])
full_scale, full_temperature = fit_prior(cases, full_table, variant)
full_rows = []
for data in cases:
rows, _ = data.weighted_rows(full_table, variant, full_scale)
full_rows.append(evaluate(data, rows, softmax_prior=True, temperature=full_temperature))
block["full_fit"][variant] = {
"scale": full_scale,
"temperature": full_temperature,
**{m: lib.summarize([r[m] for r in full_rows]) for m in PRIORS},
}
if variant == "A3":
block["lr_table"] = {
name: {
"log_lr_present": full_table.present[name],
"log_lr_absent": full_table.absent[name],
**full_table.support[name],
"extra": lib.is_extra_feature(name),
}
for name in sorted(full_table.present)
}
ci = lib.bootstrap_lr(examples, resamples=50 if fast else 200)
for name, interval in ci.items():
block["lr_table"].setdefault(name, {}).update(interval)
block["noise_features"] = sorted(
name for name, row in block["lr_table"].items()
if row.get("p05") is not None and row["p05"] <= 0.0 <= row["p95"]
)
# leave-one-case-out -------------------------------------------
loo_rows = []
robustness = defaultdict(list)
calibration_points: list[tuple[float, bool]] = []
temperatures: list[float] = []
for held in cases:
training = [d for d in cases if d.case_id != held.case_id]
table = lib.estimate_lr([ex for d in training for ex in examples[d.case_id]])
scale, temperature = fit_prior(training, table, variant)
temperatures.append(temperature)
rows, built_v = held.weighted_rows(table, variant, scale)
result = evaluate(held, rows, softmax_prior=True, temperature=temperature)
if probes_per_variant:
own = lib.production_probes(held.request, built_v, held.times, held.true_time)
result["own_probes"] = evaluate(held, rows, softmax_prior=True, temperature=temperature, probes=own)
result["own_probes_count"] = len(own)
result["case_id"] = held.case_id
loo_rows.append(result)
if variant == "A3":
public = lib.public_for(rows, held.contexts)
posterior = lib.percent_softmax({str(r["time"])[:5]: float(r["score"]) for r in public}, temperature)
truth = set(held.truth_times)
for rep in public:
stamp = str(rep["time"])[:5]
members = {str(t)[:5] for t in (rep.get("cluster_times") or [stamp])}
calibration_points.append((posterior.get(stamp, 0.0) / 100.0, bool(members & truth)))
# robustness: date shifts (features re-recorded with fold weights)
if not fast or variant == "A3":
events = list(held.case.get("events") or [])
shifted_sets = {
"jitter_7d": [jitter_day_events(events, 7, case_id=held.case_id)],
"shift_1_3_months": [
lib.shift_day_events_by_months(events, seed=f"{SEED}:{held.case_id}:{radius}:{k}")
for k in range(1 if fast else SHIFT_REPEATS)
],
}
for label, variants_of_events in shifted_sets.items():
for index, altered in enumerate(variants_of_events):
cache_key = (held.case_id, label, index)
if cache_key not in altered_stores:
altered_stores[cache_key] = held.rescored_store(altered)[:2]
store, request_alt = altered_stores[cache_key]
rows_alt, _ = held.weighted_rows(table, variant, scale, store=store, request=request_alt)
robustness[label].append(evaluate(held, rows_alt, softmax_prior=True, temperature=temperature))
# answer flips on the LOO rows
asked = list(held.probes)[:ASK_COUNT]
optimal = [optimal_answer(p, held.true_time) for p in asked]
for k in (1, 2):
sets = exhaustive_flip_sets(optimal, k) if k == 1 else sample_flip_sets(
optimal, k, 3 if fast else FLIP_REPEATS, case_id=held.case_id, radius=radius,
)
for picked in sets:
robustness[f"flip_{k}"].append(evaluate(
held, rows, softmax_prior=True, answers=flipped_answers(optimal, picked), temperature=temperature,
))
block["loo"][variant] = {m: lib.summarize([r[m] for r in loo_rows]) for m in PRIORS}
block["loo"][variant]["temperatures"] = sorted(set(temperatures))
block["loo"][variant]["gate_verdict"] = {
m: gate_verdict(block["baseline"][m], block["loo"][variant][m]) for m in PRIORS
}
block["loo"][variant]["hard_line_1"] = {
m: {
"coverage_count": sum(1 for r in loo_rows if r[m]["coverage"]),
"baseline_coverage_count": sum(1 for r in baseline_rows if r[m]["coverage"]),
"top1": block["loo"][variant][m]["top1"],
"baseline_top1": block["baseline"][m]["top1"],
"pass": (sum(1 for r in loo_rows if r[m]["coverage"]) >= sum(1 for r in baseline_rows if r[m]["coverage"])
and float(block["loo"][variant][m]["top1"]) + 1e-9 >= float(block["baseline"][m]["top1"])),
"note": "truth and opposite cells coincide under D1 (nothing is injected after the six probes)",
}
for m in PRIORS
}
if probes_per_variant:
block["loo"][variant]["own_probes"] = {m: lib.summarize([r["own_probes"][m] for r in loo_rows]) for m in PRIORS}
block["loo"][variant]["own_probes"]["mean_count"] = round(sum(r["own_probes_count"] for r in loo_rows) / max(len(loo_rows), 1), 2)
block["per_case"][variant] = [{"case_id": r["case_id"], **{m: {k: r[m][k] for k in ("top1", "coverage", "width", "engine_top1")} for m in PRIORS}} for r in loo_rows]
block["loo"][variant]["verdict"] = "pending_v5"
if robustness:
block["robustness"][variant] = {
label: {m: lib.summarize([r[m] for r in rows_]) for m in PRIORS}
for label, rows_ in robustness.items()
}
if variant == "A3":
block["calibration"] = lib.calibration_bins(calibration_points)
out["radii"][str(radius)] = block
return out
# ---------------------------------------------------------------------------
# R-B
# ---------------------------------------------------------------------------
def absence_probes(data: CaseData, events: Sequence[dict[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, list[int]]]:
gaps = lib.absent_years(events)
probes: list[dict[str, Any]] = []
for domain, years in gaps.items():
for year in years:
probe = lib.split_probe(
data.representatives, birth_date=str(data.request["birth_date"]), domain=domain,
year=year, month=None, source=ABSENCE_SOURCE,
)
if probe is not None:
probes.append(probe)
return probes, gaps
def run_rb(by_radius: dict[int, list[CaseData]], *, fast: bool) -> dict[str, Any]:
out: dict[str, Any] = {"points": list(ABSENCE_POINTS), "card_points": CARD_POINTS, "omit_repeats": OMIT_REPEATS, "radii": {}}
for radius, cases in sorted(by_radius.items()):
rows_by_arm: dict[str, list[dict[str, Any]]] = defaultdict(list)
gap_stats = []
omit_rows: dict[str, list[dict[str, Any]]] = defaultdict(list)
for data in cases:
training = [data.events_by_id[i] for i in data.training_ids]
spoken = [{**e, "date": e.get("date_start"), "domain": e["domain"]} for e in training]
probes, gaps = absence_probes(data, spoken)
gap_stats.append({
"case_id": data.case_id, "absent_years": sum(len(v) for v in gaps.values()),
"domains_with_gaps": len(gaps), "absence_probes": len(probes),
})
base = evaluate(data, data.prod_rows, softmax_prior=False)
rows_by_arm["B0"].append(base)
for points in ABSENCE_POINTS:
pre = [(probe, "no", points / CARD_POINTS) for probe in probes]
rows_by_arm[f"absence@{points}"].append(evaluate(data, data.prod_rows, softmax_prior=False, pre_answers=pre))
# omission robustness: user did not mention 1–2 of the training events
for k in (1, 2):
for rep in range(1 if fast else OMIT_REPEATS):
kept = lib.drop_events(spoken, k, seed=f"{SEED}:{data.case_id}:{radius}:{k}:{rep}")
probes_k, _ = absence_probes(data, kept)
for points in ABSENCE_POINTS:
pre = [(probe, "no", points / CARD_POINTS) for probe in probes_k]
omit_rows[f"omit_{k}@{points}"].append(evaluate(data, data.prod_rows, softmax_prior=False, pre_answers=pre))
block: dict[str, Any] = {
"n": len(cases),
"gap_stats": {
"mean_absent_years": round(sum(g["absent_years"] for g in gap_stats) / len(gap_stats), 2),
"mean_absence_probes": round(sum(g["absence_probes"] for g in gap_stats) / len(gap_stats), 2),
"cases_with_probes": sum(1 for g in gap_stats if g["absence_probes"]),
},
"arms": {arm: {m: lib.summarize([r[m] for r in rows]) for m in PRIORS} for arm, rows in rows_by_arm.items()},
"omission": {arm: {m: lib.summarize([r[m] for r in rows]) for m in PRIORS} for arm, rows in omit_rows.items()},
}
for arm in block["arms"]:
if arm == "B0":
continue
block["arms"][arm]["gate_verdict"] = {m: gate_verdict(block["arms"]["B0"][m], block["arms"][arm][m]) for m in PRIORS}
block["arms"][arm]["hard_line_1"] = {m: {
"coverage_count": sum(1 for r in rows_by_arm[arm] if r[m]["coverage"]),
"baseline_coverage_count": sum(1 for r in rows_by_arm["B0"] if r[m]["coverage"]),
"pass": sum(1 for r in rows_by_arm[arm] if r[m]["coverage"]) >= sum(1 for r in rows_by_arm["B0"] if r[m]["coverage"])
and float(block["arms"][arm][m]["top1"]) + 1e-9 >= float(block["arms"]["B0"][m]["top1"]),
} for m in PRIORS}
block["arms"][arm]["verdict"] = "pending_v5"
out["radii"][str(radius)] = block
return out
# ---------------------------------------------------------------------------
# R-C
# ---------------------------------------------------------------------------
def run_rc(by_radius: dict[int, list[CaseData]], *, fast: bool) -> dict[str, Any]:
out: dict[str, Any] = {"radii": {}}
for radius, cases in sorted(by_radius.items()):
rows_by_arm: dict[str, list[dict[str, Any]]] = defaultdict(list)
askable_counts = []
for data in cases:
originals = {str(e["id"]): e for e in data.case.get("events") or []}
raw_events = list(data.case.get("events") or [])
spoken = lib.degrade_to_year(raw_events)
spoken_rows = data.production_rows_for(spoken)
def keep(arm: str, result: dict[str, dict[str, Any]]) -> None:
rows_by_arm[arm].append({"case_id": data.case_id, **result})
keep("B0_true_precision", evaluate(data, data.prod_rows, softmax_prior=False))
keep("C0_spoken", evaluate(data, spoken_rows, softmax_prior=False))
# which spoken (year) events would split candidates if the month were known?
training = set(data.training_ids)
id_map = {str(e["id"]): str(rid) for rid, e in zip(
[str(x["id"]) for x in data.request["events"]], data.case.get("events") or [],
)}
candidates = []
for event in raw_events:
if str(event.get("precision")) not in {"day", "month"}:
continue
request_id = id_map.get(str(event["id"]))
if request_id not in training:
continue
year, month = int(str(event["date"])[:4]), int(str(event["date"])[5:7])
probe = lib.split_probe(
data.representatives, birth_date=str(data.request["birth_date"]), domain=str(event["domain"]),
year=year, month=month, source=FOLLOWUP_SOURCE,
)
if probe is None:
continue
candidates.append((float(probe.get("information_gain") or 0), str(event["id"]), probe))
candidates.sort(key=lambda item: (-item[0], item[1]))
askable_counts.append(len(candidates))
for k in (1, 2):
picked = candidates[:k]
if len(picked) < k:
continue
ids = {item[1] for item in picked}
upgraded = lib.upgrade_to_month(spoken, ids, originals)
upgraded_rows = data.production_rows_for(upgraded)
pre = [(item[2], "yes", 1.0) for item in picked]
keep(f"C{k}_prior", evaluate(data, upgraded_rows, softmax_prior=False))
keep(f"C{k}_probe", evaluate(data, spoken_rows, softmax_prior=False, pre_answers=pre))
keep(f"C{k}_both", evaluate(data, upgraded_rows, softmax_prior=False, pre_answers=pre))
by_case = {arm: {r["case_id"]: r for r in rows} for arm, rows in rows_by_arm.items()}
block: dict[str, Any] = {
"n": len(cases),
"askable_per_case": {
"mean": round(sum(askable_counts) / len(askable_counts), 2) if askable_counts else None,
"distribution": {str(k): askable_counts.count(k) for k in sorted(set(askable_counts))},
"cases_with_at_least_one": sum(1 for c in askable_counts if c >= 1),
"cases_with_at_least_two": sum(1 for c in askable_counts if c >= 2),
},
"arms": {arm: {m: lib.summarize([r[m] for r in rows]) for m in PRIORS} for arm, rows in rows_by_arm.items()},
"per_case": {arm: [{"case_id": r["case_id"], **{m: {k: r[m][k] for k in ("top1", "coverage", "width", "engine_top1")} for m in PRIORS}} for r in rows] for arm, rows in rows_by_arm.items()},
}
for arm, rows in rows_by_arm.items():
if arm.startswith("C0") or arm.startswith("B0"):
continue
ids = [r["case_id"] for r in rows]
subset_c0 = [by_case["C0_spoken"][cid] for cid in ids]
subset_b0 = [by_case["B0_true_precision"][cid] for cid in ids]
block["arms"][arm]["subset_C0"] = {m: lib.summarize([r[m] for r in subset_c0]) for m in PRIORS}
block["arms"][arm]["subset_B0"] = {m: lib.summarize([r[m] for r in subset_b0]) for m in PRIORS}
block["arms"][arm]["gate_verdict_vs_subset_C0"] = {m: gate_verdict(block["arms"][arm]["subset_C0"][m], block["arms"][arm][m]) for m in PRIORS}
block["arms"][arm]["hard_line_1_vs_subset_C0"] = {m: {
"coverage_count": sum(1 for r in rows if r[m]["coverage"]),
"subset_C0_coverage_count": sum(1 for r in subset_c0 if r[m]["coverage"]),
"pass": sum(1 for r in rows if r[m]["coverage"]) >= sum(1 for r in subset_c0 if r[m]["coverage"])
and float(block["arms"][arm][m]["top1"]) + 1e-9 >= float(block["arms"][arm]["subset_C0"][m]["top1"]),
} for m in PRIORS}
block["arms"][arm]["gate_verdict_vs_C0"] = {m: gate_verdict(block["arms"]["C0_spoken"][m], block["arms"][arm][m]) for m in PRIORS}
block["arms"][arm]["verdict"] = "pending_v5"
out["radii"][str(radius)] = block
return out
# ---------------------------------------------------------------------------
# main
# ---------------------------------------------------------------------------
def load_cases(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
payload = json.loads(path.read_text(encoding="utf-8"))
return payload, list(payload["cases"])
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False, indent=1, sort_keys=True) + "\n", encoding="utf-8")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--holdout", default=str(HOLDOUT_V4))
parser.add_argument("--dataset-label", default="v4")
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--radii", nargs="+", type=int, default=list(RADII))
parser.add_argument("--parts", nargs="+", default=["ra", "rb", "rc"])
parser.add_argument("--cache-dir", default="")
parser.add_argument("--fast", action="store_true", help="fewer repeats (development)")
parser.add_argument("--no-write", action="store_true")
parser.add_argument("--out-suffix", default="2026_09_29")
parser.add_argument("--probes-per-variant", action="store_true", help="also replay LOO variants with probes generated from their own matrix")
args = parser.parse_args()
started = clock_module.time()
holdout_path = Path(args.holdout)
meta, cases = load_cases(holdout_path)
if args.limit:
cases = cases[: args.limit]
cache_dir = Path(args.cache_dir) if args.cache_dir else None
errors: list[dict[str, Any]] = []
reconciliation: list[dict[str, Any]] = []
parts = {"ra": "lr", "rb": "absence", "rc": "precision_followup"}
radii_results: dict[str, dict[str, dict[str, Any]]] = {key: {} for key in parts.values()}
part_meta: dict[str, dict[str, Any]] = {}
# One radius at a time: 77 cases x 3 radii of static contexts do not fit in memory together.
for radius in args.radii:
loaded: list[CaseData] = []
for case in cases:
try:
data = CaseData(case, radius, cache_dir)
except Exception as exc: # noqa: BLE001
errors.append({"case_id": case.get("case_id"), "radius": radius, "error": f"{type(exc).__name__}: {exc}", "trace": traceback.format_exc()})
continue
reconciliation.append({"case_id": data.case_id, "radius": radius, **data.reconciliation})
loaded.append(data)
print(f"loaded radius {radius}: {len(loaded)} cases ({clock_module.time() - started:.0f}s)", flush=True)
if any(row["changed"] for row in reconciliation if row["radius"] == radius):
print(json.dumps({"UNRECONCILED": [row for row in reconciliation if row["changed"]]}, ensure_ascii=False, indent=1))
return 2
by_radius = {radius: loaded}
runners = {
"ra": lambda: run_ra(by_radius, fast=args.fast, probes_per_variant=args.probes_per_variant),
"rb": lambda: run_rb(by_radius, fast=args.fast),
"rc": lambda: run_rc(by_radius, fast=args.fast),
}
for part, key in parts.items():
if part not in args.parts:
continue
block = runners[part]()
radii_results[key].update(block.pop("radii"))
part_meta[key] = block
print(f"{part.upper()} radius {radius} done {clock_module.time() - started:.0f}s", flush=True)
del loaded, by_radius
gc.collect()
header = {
"generated_for": "TASK-rectification-scoring-research-20260929",
"dataset": args.dataset_label,
"holdout": str(holdout_path.relative_to(ROOT)) if holdout_path.is_relative_to(ROOT) else str(holdout_path),
"case_count": len(cases),
"radii": list(args.radii),
"minute_step": MINUTE_STEP,
"ayanamsa": AYANAMSA,
"node_mode": NODE_MODE,
"ask_count": ASK_COUNT,
"probe_today": lib.TODAY.isoformat(),
"probes": "production G0 probes generated once per case/radius from the production matrix; reused for every arm (own_probes = regenerated from the variant matrix)",
"open_set_not_blind": True,
"verdict": "pending_v5" if args.dataset_label != "v5" else "see_report",
"reconciliation": {"cases": len(reconciliation), "unreconciled": sum(1 for r in reconciliation if r["changed"]), "rows": reconciliation},
"errors": errors,
"seed": SEED,
"fast": bool(args.fast),
}
results: dict[str, dict[str, Any]] = {}
for part, key in parts.items():
if part in args.parts:
results[key] = {**header, "part": {"ra": "R-A", "rb": "R-B", "rc": "R-C"}[part], **part_meta.get(key, {}), "radii": radii_results[key]}
for key, payload in results.items():
if not args.no_write:
write_json(OUT_DIR / f"scoring_research_{key}_{args.out_suffix}.json", payload)
brief: dict[str, Any] = {"reconciliation_unreconciled": header["reconciliation"]["unreconciled"], "errors": len(errors), "elapsed_seconds": round(clock_module.time() - started, 1)}
for key, payload in results.items():
brief[key] = {}
for radius, block in payload["radii"].items():
if key == "lr":
brief[key][radius] = {
"baseline": {m: (block["baseline"][m]["top1"], block["baseline"][m]["coverage"], block["baseline"][m]["width_median"], block["baseline"][m]["engine_top1"]) for m in PRIORS},
**{v: {m: (block["loo"][v][m]["top1"], block["loo"][v][m]["coverage"], block["loo"][v][m]["width_median"], block["loo"][v][m]["engine_top1"]) for m in PRIORS} for v in lib.VARIANTS},
}
else:
brief[key][radius] = {arm: {m: (s[m]["top1"], s[m]["coverage"], s[m]["width_median"]) for m in PRIORS} for arm, s in block["arms"].items()}
print(json.dumps(brief, ensure_ascii=False, indent=1))
return 0 if not errors else 1
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink