"""Offline segment-oriented rectification research for BUG-1105. The production candidate scorer is only used as an observation source. This module never changes production defaults. A segment is a maximal contiguous run of equal divisional ascendant sign; equal signs separated by another sign remain different segments. """ from __future__ import annotations from collections import defaultdict from dataclasses import dataclass from datetime import date from typing import Any, Iterable, Sequence from scripts.active_rectification_event_engine import compute_candidate_static_contexts from scripts.rectification.refinement_packet import window_scan from scripts.rectification.scoring_service import build_event_contribution_matrix, score_from_matrix from scripts.research.minute_resolution_sweep import scoring_request_for from scripts.research.probe_supply_after_six import ASK_COUNT, apply_answer, optimal_answer from scripts.research.scoring_research_lib import ( public_for, replay, reconcile_rows, score_map, truth_cluster_times, ) from scripts.research.cluster_width_lib import SEPARATION_LEAD, delivery_from_public, still_valid_public from scripts.rectification.event_probes import discriminating_event_probes VARGA_PREFIXES = ("D1", "D9", "D10") RADII = (10, 15, 30, 60) THRESHOLDS = (0.5, 0.6, 0.7, 0.8, 0.9) TODAY = date(2026, 9, 16) def clock(stamp: str) -> int: hour, minute = str(stamp)[:5].split(":") return int(hour) * 60 + int(minute) def sign_value(row: dict[str, Any], prefix: str) -> Any: value = row.get(prefix) if isinstance(value, dict): return value.get("sign_idx", value.get("sign")) return value def segment_rows(rows: Sequence[dict[str, Any]], prefix: str) -> list[dict[str, Any]]: """Return maximal sampled runs without merging a non-contiguous sign. Clock time is cyclic here: 23:59 followed by 00:00 is one minute apart. A missing sample (for example 23:59 followed by 00:01) still starts a new run. This deliberately uses the observation order rather than a set of signs, so A-B-A produces three separately numbered segments. """ segments: list[dict[str, Any]] = [] current: dict[str, Any] | None = None for row in rows: stamp = str(row.get("time") or row.get("stamp") or "")[:5] value = sign_value(row, prefix) if not stamp or value is None: current = None continue minute = clock(stamp) contiguous = ( current is not None and (minute - int(current["last_minute"])) % 1440 == 1 ) if current is None or current["value"] != value or not contiguous: current = { "segment_id": len(segments), "varga": prefix, "value": value, "start": stamp, "end": stamp, "times": [stamp], "last_minute": minute, } segments.append(current) else: current["end"] = stamp current["times"].append(stamp) current["last_minute"] = minute for segment in segments: segment.pop("last_minute", None) return segments def segment_members(rows: Sequence[dict[str, Any]], prefix: str) -> list[list[str]]: return [list(item["times"]) for item in segment_rows(rows, prefix)] def row_signature(context: dict[str, Any], prefix: str) -> int | str | None: if prefix == "D1": value = context.get("ascendant_index") else: charts = context.get("varga_charts") or {} value = ((charts.get(prefix) or {}).get("Ascendant") or {}).get("sign_idx") return value if isinstance(value, (int, str)) else None def contexts_to_rows(contexts: Sequence[dict[str, Any]], prefixes: Iterable[str] = VARGA_PREFIXES) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for context in contexts: feature = context.get("feature") stamp = feature.get("time") if isinstance(feature, dict) else None if not stamp: stamp = context["candidate_at"].strftime("%H:%M") rows.append({"time": str(stamp)[:5], **{p: row_signature(context, p) for p in prefixes}}) return rows def truth_segment(rows: Sequence[dict[str, Any]], prefix: str, truth_time: str) -> dict[str, Any] | None: for segment in segment_rows(rows, prefix): if truth_time[:5] in segment["times"]: return segment return None def unique_sign_count(rows: Sequence[dict[str, Any]], prefix: str) -> int: return len({sign_value(row, prefix) for row in rows if sign_value(row, prefix) is not None}) def unique_segment_count(rows: Sequence[dict[str, Any]], prefix: str) -> int: return len(segment_rows(rows, prefix)) def window_payload(case: dict[str, Any], radius: int) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: """Build the two-minute scoring grid and an independent one-minute scan grid.""" scoring_request = scoring_request_for({**case, "candidate_radius_minutes": radius}, radius) scoring_request["minute_step"] = 2 scoring_contexts = compute_candidate_static_contexts(scoring_request) scan_request = scoring_request_for({**case, "candidate_radius_minutes": radius}, radius) scan_request["minute_step"] = 1 scan_contexts = compute_candidate_static_contexts(scan_request) return scoring_request, scoring_contexts, contexts_to_rows(scan_contexts) def _probe_payload(request: dict[str, Any], built: dict[str, Any], times: Sequence[str], true_time: str) -> list[dict[str, Any]]: return discriminating_event_probes( {**request, "refresh_probes": False, "asked_probe_keys": []}, built, scan=window_scan(built), candidate_times=list(times), representative_time=true_time, today=TODAY, ) def replay_state(rows: Sequence[dict[str, Any]], contexts: Sequence[dict[str, Any]], request: dict[str, Any], true_time: str, *, probes: Sequence[dict[str, Any]] | None = None, answers: Sequence[str | None] | None = None) -> dict[str, Any]: times = [str(row["time"])[:5] for row in rows] public = public_for(rows, contexts) reps = [str(row["time"])[:5] for row in public] scores = {stamp: float(row.get("score") or 0) for stamp, row in ((str(item["time"])[:5], item) for item in public)} conflicts = {stamp: 0 for stamp in reps} eliminated: set[str] = set() actual_probes = list(probes) if probes is not None else _probe_payload(request, {"static_contexts": list(contexts), "rows": list(rows)}, times, true_time) given = list(answers) if answers is not None else [optimal_answer(p, true_time) for p in actual_probes[:ASK_COUNT]] for probe, answer in zip(actual_probes[:ASK_COUNT], given): if answer in {"yes", "no", "weak_yes"}: scores, conflicts, eliminated = apply_answer(scores, conflicts, eliminated, probe, answer, reps) posterior = [{**row, "score": scores.get(str(row["time"])[:5], row.get("score") or 0)} for row in public] valid = still_valid_public(posterior, scores, eliminated, lead=SEPARATION_LEAD) return { "result": {"questions": sum(answer is not None for answer in given)}, "public": public, "posterior": posterior, "valid": valid, "scores": scores, "eliminated": eliminated, "probes": actual_probes, "delivery": delivery_from_public(valid), } def native_case(case: dict[str, Any], radius: int, *, do_reconcile: bool = False) -> dict[str, Any]: request, contexts, chart_rows = window_payload(case, radius) true_time = str(case["birth"]["time"])[:5] built = build_event_contribution_matrix(request, static_contexts=contexts) rows = score_from_matrix(request, built) times = [str(row["time"])[:5] for row in rows] probes = _probe_payload(request, built, times, true_time) state = replay_state(rows, contexts, request, true_time, probes=probes) reconciliation = {"status": "not_run"} if do_reconcile: from scripts.research.scoring_research_lib import FeatureStore, d60_charts, make_recording_provider store = FeatureStore() recording = build_event_contribution_matrix(request, row_provider=make_recording_provider(contexts, store, d60_charts(contexts)), static_contexts=contexts) recording_rows = score_from_matrix(request, recording) reconciliation = reconcile_rows(rows, recording_rows) return {"case_id": str(case["case_id"]), "radius": radius, "true_time": true_time, "request": request, "contexts": contexts, "chart_rows": chart_rows, "rows": rows, "probes": probes, "state": state, "reconciliation": reconciliation} def valid_minute_scores( state: dict[str, Any], chart_rows: Sequence[dict[str, Any]], scores: dict[str, float] | None = None, ) -> dict[str, float]: """Project representative posterior scores onto each minute in its cluster.""" effective_scores = scores if scores is not None else state["scores"] output: dict[str, float] = {} for row in state["posterior"]: representative = str(row["time"])[:5] value = float(effective_scores.get(representative, row.get("score") or 0)) for stamp in row.get("cluster_times") or [representative]: output[str(stamp)[:5]] = value return output def segment_metrics(state: dict[str, Any], chart_rows: Sequence[dict[str, Any]], prefix: str, true_time: str, mode: str) -> dict[str, Any]: segments = segment_rows(chart_rows, prefix) scores = {str(key)[:5]: float(value) for key, value in state["scores"].items()} if mode == "percent": total = sum(max(value, 0.0) for value in scores.values()) scores = ( {key: max(value, 0.0) / total * 100.0 for key, value in scores.items()} if total > 0 else {key: 0.0 for key in scores} ) minute_scores = valid_minute_scores(state, chart_rows, scores) valid_times = {str(t)[:5] for row in state["valid"] for t in (row.get("cluster_times") or [row.get("time")])} truth = truth_segment(chart_rows, prefix, true_time) qualities: list[float] = [] for segment in segments: values = [minute_scores.get(t, 0.0) for t in segment["times"] if t in valid_times] if mode == "uniform": quality = float(len(values)) elif mode == "percent": quality = sum(max(v, 0.0) for v in values) else: quality = sum(values) qualities.append(quality) total = sum(qualities) share = max(qualities) / total if qualities and total > 0 else None leaders = [i for i, value in enumerate(qualities) if share is not None and abs(value - max(qualities)) <= 1e-9] truth_id = truth["segment_id"] if truth else None retained = truth_id is not None and truth_id in {segment["segment_id"] for segment in segments if any(t in valid_times for t in segment["times"])} correct = truth_id is not None and truth_id in leaders valid_segment_count = sum(1 for segment in segments if any(t in valid_times for t in segment["times"])) return { "prefix": prefix, "mode": mode, "segment_count_window": len(segments), "valid_segment_count": valid_segment_count, "truth_segment_id": truth_id, "truth_retained": bool(retained), "top_segment_correct": bool(correct), "top_segment_tie": len(leaders) > 1, "top_segment_ids": leaders, "top_share": None if share is None else round(share, 8), "segment_qualities": [round(v, 8) for v in qualities], } def threshold_scan(rows: Sequence[dict[str, Any]], thresholds: Sequence[float] = THRESHOLDS) -> dict[str, Any]: out: dict[str, Any] = {} for threshold in thresholds: eligible = [row for row in rows if row.get("top_share") is not None and float(row["top_share"]) >= threshold] out[str(threshold)] = {"n": len(eligible), "denominator": len(rows), "coverage": round(len(eligible) / len(rows), 8) if rows else None, "accuracy": round(sum(bool(row.get("top_segment_correct")) for row in eligible) / len(eligible), 8) if eligible else None, "truth_retained": round(sum(bool(row.get("truth_retained")) for row in eligible) / len(eligible), 8) if eligible else None} return out def choose_loo_threshold(training: Sequence[dict[str, Any]], thresholds: Sequence[float] = THRESHOLDS, minimum: int = 5) -> float | None: if not training: return None candidates = [] required = min(minimum, len(training)) for threshold in thresholds: eligible = [row for row in training if row.get("top_share") is not None and float(row["top_share"]) >= threshold] if len(eligible) < required or not eligible: continue accuracy = sum(bool(row.get("top_segment_correct")) for row in eligible) / len(eligible) retained = sum(bool(row.get("truth_retained")) for row in eligible) / len(eligible) candidates.append((accuracy, retained, len(eligible), -float(threshold), float(threshold))) if not candidates: return None return max(candidates)[-1] def segment_probe_score(probe: dict[str, Any], chart_rows: Sequence[dict[str, Any]], prefix: str, minute_weights: dict[str, float]) -> float: segments = segment_rows(chart_rows, prefix) segment_by_time = {t: segment["segment_id"] for segment in segments for t in segment["times"]} yes = {str(t)[:5] for item in probe.get("expected_outcomes") or [] if item.get("answer_class") in {"yes", "weak_yes"} for t in item.get("supports") or []} no = {str(t)[:5] for item in probe.get("expected_outcomes") or [] if item.get("answer_class") == "no" for t in item.get("supports") or []} totals: dict[int, float] = defaultdict(float) for stamp in yes | no: if stamp in segment_by_time: totals[segment_by_time[stamp]] += max(minute_weights.get(stamp, 0.0), 0.0) if len(totals) < 2: return 0.0 values = sorted(totals.values(), reverse=True) return round(values[0] - values[1], 8) def reorder_probes_by_segments(probes: Sequence[dict[str, Any]], chart_rows: Sequence[dict[str, Any]], prefix: str, minute_weights: dict[str, float]) -> list[dict[str, Any]]: return sorted(enumerate(probes), key=lambda item: (-segment_probe_score(item[1], chart_rows, prefix, minute_weights), item[0])) and [item[1] for item in sorted(enumerate(probes), key=lambda item: (-segment_probe_score(item[1], chart_rows, prefix, minute_weights), item[0]))] def strategy_prefixes(domain: str) -> tuple[str, ...]: if domain == "career": return ("D1", "D10") if domain == "relationship": return ("D1", "D9") return VARGA_PREFIXES