refactor: rebuild birth time rectification agent

scripts/rectification/__init__.py

@@ -0,0 +1 @@
+"""Single source of truth for V5 birth-time rectification scoring and diagnostics."""

scripts/rectification/api_service.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import Any
+from uuid import NAMESPACE_URL, uuid5
+
+from scripts.rectification.candidate_feature_service import build_candidate_feature_snapshot
+from scripts.rectification.contracts import RectificationRequest
+from scripts.rectification.diagnostics_service import run_diagnostics
+from scripts.rectification.scoring_service import (
+    ALGORITHM_VERSION,
+    build_event_contribution_matrix,
+    calculation_spec,
+    score_from_matrix,
+    sha256,
+)
+
+
+def candidate_features(request: RectificationRequest) -> dict[str, Any]:
+    spec = calculation_spec(request)
+    spec_hash = sha256(spec)
+    return {
+        "algorithm_version": ALGORITHM_VERSION,
+        "calculation_spec": spec,
+        "calculation_spec_hash": spec_hash,
+        "candidate_feature_snapshot": build_candidate_feature_snapshot(request, spec_hash),
+        "can_confirm_exact_minute": False,
+    }
+
+
+def score_candidates(request: RectificationRequest) -> dict[str, Any]:
+    built = build_event_contribution_matrix(request)
+    rows = score_from_matrix(request, built)
+    spec = calculation_spec(request)
+    spec_hash = sha256(spec)
+    diagnostics = run_diagnostics(request, rows, built)
+    fingerprint = sha256(request)
+    return {
+        "result_id": str(uuid5(NAMESPACE_URL, f"{ALGORITHM_VERSION}:{fingerprint}")),
+        "algorithm_version": ALGORITHM_VERSION,
+        "calculation_spec": spec,
+        "calculation_spec_hash": spec_hash,
+        "candidate_scores": [{
+            "time": row["time"],
+            "score": row["score"],
+            "supporting_event_ids": [item["event_id"] for item in row["evidence"] if item["points"] > 0],
+            "conflicting_event_ids": [item["event_id"] for item in row["evidence"] if item["points"] < 0],
+        } for row in rows],
+        "event_contribution_matrix": built["matrix"],
+        "candidate_feature_snapshot": build_candidate_feature_snapshot(request, spec_hash, built.get("static_contexts")),
+        "diagnostics": diagnostics,
+        "robustness": {
+            "neighbor_support_minutes": diagnostics["neighbor_support_minutes"],
+            "leave_one_out_retention_rate": diagnostics["leave_one_event_out_retention_rate"],
+            "leave_one_domain_out_retention_rate": diagnostics["leave_one_domain_out_retention_rate"],
+            "date_sensitivity_retention_rate": diagnostics["date_sensitivity_retention_rate"],
+        },
+        "missing_layers": built["missing_layers"],
+        "can_confirm_exact_minute": False,
+    }
+
+
+def diagnostics(request: RectificationRequest) -> dict[str, Any]:
+    scored = score_candidates(request)
+    return {
+        "result_id": scored["result_id"],
+        "algorithm_version": scored["algorithm_version"],
+        "calculation_spec_hash": scored["calculation_spec_hash"],
+        "diagnostics": scored["diagnostics"],
+        "missing_layers": scored["missing_layers"],
+        "can_confirm_exact_minute": False,
+    }

scripts/rectification/candidate_feature_service.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from scripts.active_rectification_event_engine import compute_candidate_static_contexts
+from scripts.rectification.contracts import RectificationRequest
+from scripts.rectification.scoring_service import ALGORITHM_VERSION, sha256
+
+
+def build_candidate_feature_snapshot(
+    request: RectificationRequest,
+    calculation_spec_hash: str,
+    static_contexts: Sequence[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    contexts = list(static_contexts) if static_contexts is not None else compute_candidate_static_contexts(request)
+    features = [context["feature"] for context in contexts]
+    return {
+        "calculation_spec_hash": calculation_spec_hash,
+        "algorithm_version": ALGORITHM_VERSION,
+        "candidate_count": len(features),
+        "feature_hash": sha256(features),
+        "features": features,
+    }

scripts/rectification/contracts.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import math
+import re
+import uuid
+from datetime import date
+from typing import Any, Literal, NotRequired, TypedDict, cast
+
+DatePrecision = Literal["day", "month", "quarter", "year", "range"]
+
+SCOREABLE_EVENT_KINDS: dict[str, frozenset[str]] = {
+    "education": frozenset({"education_milestone"}),
+    "relocation": frozenset({"relocation"}),
+    "relationship": frozenset({"relationship_start", "relationship_end", "relationship_change"}),
+    "career": frozenset({"career_change"}),
+    "finance": frozenset({"finance_change"}),
+    "health_pressure": frozenset({"self_health_event"}),
+}
+DATE_PRECISIONS = frozenset({"day", "month", "quarter", "year", "range"})
+_REQUEST_FIELDS = frozenset({"birth_date", "start_time", "end_time", "lat", "lon", "tz", "events"})
+_EVENT_FIELDS = frozenset({"id", "domain", "event_kind", "date_start", "date_end", "precision", "summary"})
+_CLOCK = re.compile(r"(?:[01]\d|2[0-3]):[0-5]\d\Z")
+
+
+class LifeEvent(TypedDict):
+    id: str
+    domain: str
+    event_kind: str
+    date_start: str
+    date_end: str
+    precision: DatePrecision
+    summary: NotRequired[str]
+
+
+class RectificationRequest(TypedDict):
+    birth_date: str
+    start_time: str
+    end_time: str
+    lat: float
+    lon: float
+    tz: float
+    events: list[LifeEvent]
+
+
+JsonObject = dict[str, Any]
+
+
+def _bounded_number(body: dict[str, Any], name: str, minimum: float, maximum: float) -> float:
+    value = body.get(name)
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(float(value)):
+        raise ValueError(f"{name} must be a finite number")
+    result = float(value)
+    if not minimum <= result <= maximum:
+        raise ValueError(f"{name} must be between {minimum:g} and {maximum:g}")
+    return result
+
+
+def _calendar_date(value: Any, label: str) -> date:
+    if not isinstance(value, str):
+        raise ValueError(f"{label} must be a valid YYYY-MM-DD value")
+    try:
+        return date.fromisoformat(value)
+    except ValueError as exc:
+        raise ValueError(f"{label} must be a valid YYYY-MM-DD value") from exc
+
+
+def normalize_rectification_request(body: Any, *, today: date | None = None) -> RectificationRequest:
+    if not isinstance(body, dict):
+        raise ValueError("request body must be an object")
+    unsupported = sorted(set(body) - _REQUEST_FIELDS)
+    if unsupported:
+        raise ValueError(f"unsupported rectification field: {unsupported[0]}")
+
+    birth_day = _calendar_date(body.get("birth_date"), "birth_date")
+    start_time, end_time = body.get("start_time"), body.get("end_time")
+    if not isinstance(start_time, str) or not _CLOCK.fullmatch(start_time):
+        raise ValueError("start_time must be HH:MM")
+    if not isinstance(end_time, str) or not _CLOCK.fullmatch(end_time):
+        raise ValueError("end_time must be HH:MM")
+    if start_time > end_time:
+        raise ValueError("start_time must not exceed end_time")
+
+    events = body.get("events")
+    if not isinstance(events, list) or not 1 <= len(events) <= 100:
+        raise ValueError("events must contain between 1 and 100 items")
+    upper_date = today or date.today()
+    cleaned_events: list[LifeEvent] = []
+    for index, raw_event in enumerate(events):
+        if not isinstance(raw_event, dict):
+            raise ValueError(f"events[{index}] must be an object")
+        unsupported_event_fields = sorted(set(raw_event) - _EVENT_FIELDS)
+        if unsupported_event_fields:
+            raise ValueError(f"events[{index}] contains unsupported field: {unsupported_event_fields[0]}")
+        try:
+            event_id = str(uuid.UUID(str(raw_event.get("id") or "")))
+        except (ValueError, AttributeError) as exc:
+            raise ValueError(f"events[{index}].id must be a UUID") from exc
+        domain, event_kind = raw_event.get("domain"), raw_event.get("event_kind")
+        if domain not in SCOREABLE_EVENT_KINDS:
+            raise ValueError(f"events[{index}].domain is not scoreable")
+        if event_kind not in SCOREABLE_EVENT_KINDS[cast(str, domain)]:
+            raise ValueError(f"events[{index}].event_kind does not match domain")
+        precision = raw_event.get("precision")
+        if precision not in DATE_PRECISIONS:
+            raise ValueError(f"events[{index}].precision is invalid")
+        start_day = _calendar_date(raw_event.get("date_start"), f"events[{index}].date_start")
+        end_day = _calendar_date(raw_event.get("date_end"), f"events[{index}].date_end")
+        if start_day > end_day:
+            raise ValueError(f"events[{index}].date_start must not exceed date_end")
+        if start_day < birth_day or end_day > upper_date:
+            raise ValueError(f"events[{index}] dates must be between birth_date and today")
+        summary = raw_event.get("summary", "")
+        if not isinstance(summary, str) or len(summary) > 1_000:
+            raise ValueError(f"events[{index}].summary must be a string up to 1000 characters")
+        cleaned_events.append({
+            "id": event_id,
+            "domain": cast(str, domain),
+            "event_kind": cast(str, event_kind),
+            "date_start": start_day.isoformat(),
+            "date_end": end_day.isoformat(),
+            "precision": cast(DatePrecision, precision),
+            "summary": summary.strip(),
+        })
+
+    return {
+        "birth_date": birth_day.isoformat(),
+        "start_time": start_time,
+        "end_time": end_time,
+        "lat": _bounded_number(body, "lat", -90, 90),
+        "lon": _bounded_number(body, "lon", -180, 180),
+        "tz": _bounded_number(body, "tz", -14, 14),
+        "events": cleaned_events,
+    }

scripts/rectification/diagnostics_service.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from statistics import variance
+from typing import Any, Sequence
+
+from scripts.active_rectification_events import CandidateScoreRow
+from scripts.rectification.contracts import RectificationRequest
+
+
+def _winner(rows: Sequence[CandidateScoreRow]) -> str | None:
+    return max(rows, key=lambda row: row["score"])["time"] if rows else None
+
+
+def _primary_cluster(rows: Sequence[CandidateScoreRow], relative_floor: float = .97) -> list[str]:
+    if not rows:
+        return []
+    peak = max(row["score"] for row in rows)
+    floor = peak * relative_floor if peak >= 0 else peak / relative_floor
+    selected = [row["time"] for row in rows if row["score"] >= floor]
+    if not selected:
+        return []
+    groups: list[list[str]] = []
+    for current in selected:
+        minute = lambda value: int(value[:2]) * 60 + int(value[3:])
+        if groups and minute(current) - minute(groups[-1][-1]) == 1:
+            groups[-1].append(current)
+        else:
+            groups.append([current])
+    return max(groups, key=lambda group: (max(next(row["score"] for row in rows if row["time"] == time) for time in group), len(group)))
+
+
+def _subtract(rows: Sequence[CandidateScoreRow], removed_ids: set[str]) -> list[CandidateScoreRow]:
+    return [{**row, "score": round(row["score"] - sum(item["points"] for item in row["evidence"] if item["event_id"] in removed_ids), 4)} for row in rows]
+
+
+def run_diagnostics(request: RectificationRequest, rows: list[CandidateScoreRow], built: dict[str, Any]) -> dict[str, Any]:
+    primary = set(_primary_cluster(rows))
+    event_runs = []
+    domain_runs = []
+    event_domain = {event["id"]: event["domain"] for event in request["events"]}
+    for event in request["events"]:
+        winner = _winner(_subtract(rows, {event["id"]}))
+        event_runs.append({"removed_event_id": event["id"], "winner": winner, "retained": winner in primary})
+    by_domain: dict[str, set[str]] = defaultdict(set)
+    for event_id, domain in event_domain.items():
+        by_domain[domain].add(event_id)
+    for domain, event_ids in by_domain.items():
+        winner = _winner(_subtract(rows, event_ids))
+        domain_runs.append({"removed_domain": domain, "winner": winner, "retained": winner in primary})
+    top = sorted(rows, key=lambda row: row["score"], reverse=True)
+    top_score = top[0]["score"] if top else 0
+    secondary = next((row for row in top if row["time"] not in primary), None)
+    margin = 0 if not secondary else max(0, (top_score - secondary["score"]) / max(abs(top_score), 1e-9) * 100)
+    positive_total = sum(max(row["score"], 0) for row in rows)
+    primary_mass = sum(max(row["score"], 0) for row in rows if row["time"] in primary)
+    date_items = []
+    for item in built["date_sensitivity"]:
+        date_items.append({
+            **{key: value for key, value in item.items() if key != "sample_winners"},
+            "candidate_cluster_retention_rate": sum(winner in primary for winner in item["sample_winners"]) / len(item["sample_winners"]),
+        })
+    layers: dict[str, float] = defaultdict(float)
+    for event_id, candidates in built["matrix"].items():
+        for contribution in candidates.values():
+            for layer in contribution["technique_layers"]:
+                layers[layer] += abs(contribution["points"])
+    clusters = [_primary_cluster(rows)]
+    candidate_splits = []
+    if secondary and clusters[0]:
+        candidate_splits.append({
+            "left_cluster": {"start": clusters[0][0], "end": clusters[0][-1]},
+            "right_cluster": {"start": secondary["time"], "end": secondary["time"]},
+            "technique_layers": [name for name, _ in sorted(layers.items(), key=lambda item: item[1], reverse=True)[:8]],
+            "event_ids": [item["event_id"] for item in secondary["evidence"] if item["points"] != 0],
+        })
+    return {
+        "primary_cluster_retention_rate": 1.0 if primary else 0.0,
+        "leave_one_event_out_retention_rate": sum(item["retained"] for item in event_runs) / len(event_runs) if event_runs else 0.0,
+        "leave_one_domain_out_retention_rate": sum(item["retained"] for item in domain_runs) / len(domain_runs) if domain_runs else 0.0,
+        "date_sensitivity_retention_rate": sum(item["candidate_cluster_retention_rate"] for item in date_items) / len(date_items) if date_items else 0.0,
+        "neighbor_support_minutes": len(primary),
+        "primary_secondary_margin_percent": round(min(margin, 100), 4),
+        "cluster_mass_ratio": primary_mass / positive_total if positive_total else 0.0,
+        "unstable_event_ids": [item["removed_event_id"] for item in event_runs if not item["retained"]],
+        "most_discriminating_layers": [name for name, _ in sorted(layers.items(), key=lambda item: item[1], reverse=True)[:12]],
+        "event_date_sensitivity": date_items,
+        "candidate_splits": candidate_splits,
+        "leave_one_event_out": event_runs,
+        "leave_one_domain_out": domain_runs,
+    }

scripts/rectification/scoring_service.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections import defaultdict
+from datetime import date, timedelta
+from functools import lru_cache
+from typing import Any, Callable, Sequence
+
+from scripts.active_rectification_event_engine import compute_candidate_static_contexts, compute_event_candidate_rows
+from scripts.active_rectification_events import CandidateScoreRow
+from scripts.rectification.contracts import LifeEvent, RectificationRequest
+
+ALGORITHM_VERSION = "rectification-v5-matrix-scoring-1"
+INPUT_CONTRACT_VERSION = "rectification-calculation-spec-v4"
+
+
+def _parse(value: str) -> date:
+    return date.fromisoformat(value)
+
+
+def _iso(value: date) -> str:
+    return value.isoformat()
+
+
+def _month_end(value: date) -> date:
+    next_month = value.replace(day=28) + timedelta(days=4)
+    return next_month - timedelta(days=next_month.day)
+
+
+def _even_dates(start: date, end: date, count: int) -> list[date]:
+    if count <= 1 or start == end:
+        return [start]
+    span = (end - start).days
+    return sorted({start + timedelta(days=round(span * index / (count - 1))) for index in range(count)})
+
+
+def sample_event_dates(event: LifeEvent) -> list[str]:
+    start, end = _parse(event["date_start"]), _parse(event["date_end"])
+    precision = event["precision"]
+    if start > end:
+        raise ValueError("invalid_event_date_range")
+    if precision == "day" or start == end:
+        return [_iso(start)]
+    if precision == "month":
+        middle = start.replace(day=min(15, _month_end(start).day))
+        return sorted({_iso(start), _iso(middle), _iso(end)})
+    if precision == "quarter":
+        values: list[date] = []
+        cursor = start.replace(day=15)
+        while cursor <= end and len(values) < 3:
+            values.append(cursor)
+            cursor = (cursor.replace(day=28) + timedelta(days=4)).replace(day=15)
+        return [_iso(item) for item in values] or [_iso(start)]
+    if precision == "year":
+        return [_iso(start.replace(month=month, day=15)) for month in range(1, 13)]
+    return [_iso(item) for item in _even_dates(start, end, 12)]
+
+
+def _legacy_request(request: RectificationRequest, event: LifeEvent, sampled_date: str) -> dict[str, Any]:
+    return {
+        "birth_date": request["birth_date"],
+        "start_time": request["start_time"],
+        "end_time": request["end_time"],
+        "lat": request["lat"],
+        "lon": request["lon"],
+        "tz": request["tz"],
+        "events": [{
+            "id": event["id"], "domain": event["domain"], "date": sampled_date,
+            "precision": "day", "summary": event.get("summary", ""),
+        }],
+    }
+
+
+def _canonical(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
+
+
+@lru_cache(maxsize=4096)
+def _cached_rows(serialized: str) -> tuple[CandidateScoreRow, ...]:
+    return tuple(compute_event_candidate_rows(json.loads(serialized)))
+
+
+def build_event_contribution_matrix(
+    request: RectificationRequest,
+    row_provider: Callable[[dict[str, Any]], Sequence[CandidateScoreRow]] | None = None,
+) -> dict[str, Any]:
+    static_contexts = None if row_provider is not None else compute_candidate_static_contexts(request)
+    provider = row_provider or (lambda value: compute_event_candidate_rows(value, static_contexts=static_contexts))
+    matrix: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict)
+    missing_layers: set[str] = set()
+    date_sensitivity: list[dict[str, Any]] = []
+    candidate_grid: list[str] | None = None
+    for event in request["events"]:
+        samples = sample_event_dates(event)
+        sample_rows = [list(provider(_legacy_request(request, event, sampled))) for sampled in samples]
+        grids = [[row["time"] for row in rows] for rows in sample_rows]
+        if any(grid != grids[0] for grid in grids[1:]) or (candidate_grid is not None and grids[0] != candidate_grid):
+            raise ValueError("candidate_grid_mismatch")
+        candidate_grid = grids[0]
+        winners = []
+        for rows in sample_rows:
+            winners.append(max(rows, key=lambda row: row["score"])["time"])
+            missing_layers.update(layer for row in rows for layer in row["missing_layers"])
+        for index, candidate_time in enumerate(candidate_grid):
+            evidences = [rows[index]["evidence"][0] for rows in sample_rows]
+            points = [float(item["points"]) for item in evidences]
+            matrix[event["id"]][candidate_time] = {
+                "points": round(sum(points) / len(points), 4),
+                "rule_ids": sorted({rule for item in evidences for rule in item["rule_ids"]}),
+                "technique_layers": sorted({rule.split(":", 1)[0] for item in evidences for rule in item["rule_ids"]}),
+            }
+        winner = max(set(winners), key=winners.count)
+        mean = sum(matrix[event["id"]][time]["points"] for time in candidate_grid) / len(candidate_grid)
+        variance = sum((matrix[event["id"]][time]["points"] - mean) ** 2 for time in candidate_grid) / len(candidate_grid)
+        date_sensitivity.append({
+            "event_id": event["id"],
+            "declared_date_range": {"start": event["date_start"], "end": event["date_end"], "precision": event["precision"]},
+            "sample_dates": samples,
+            "winner_retention_rate": winners.count(winner) / len(winners),
+            "score_variance": round(variance, 6),
+            "sample_winners": winners,
+        })
+    return {
+        "candidate_times": candidate_grid or [],
+        "matrix": dict(matrix),
+        "date_sensitivity": date_sensitivity,
+        "missing_layers": sorted(missing_layers),
+        "static_contexts": static_contexts,
+    }
+
+
+def score_from_matrix(request: RectificationRequest, built: dict[str, Any]) -> list[CandidateScoreRow]:
+    rows: list[CandidateScoreRow] = []
+    for candidate_time in built["candidate_times"]:
+        evidence = []
+        for event in request["events"]:
+            contribution = built["matrix"][event["id"]][candidate_time]
+            evidence.append({
+                "event_id": event["id"], "domain": event["domain"], "candidate_time": candidate_time,
+                "rule_ids": contribution["rule_ids"], "points": contribution["points"],
+            })
+        rows.append({
+            "time": candidate_time,
+            "score": round(sum(item["points"] for item in evidence), 4),
+            "evidence": evidence,
+            "missing_layers": built["missing_layers"],
+        })
+    return rows
+
+
+def calculation_spec(request: RectificationRequest) -> dict[str, Any]:
+    return {
+        "version": INPUT_CONTRACT_VERSION,
+        "birthDate": request["birth_date"],
+        "candidateRange": {"start": request["start_time"], "end": request["end_time"]},
+        "latitude": request["lat"], "longitude": request["lon"], "timezoneOffsetHours": request["tz"],
+        "ayanamsa": "lahiri", "nodeMode": "mean", "minuteStep": 1,
+    }
+
+
+def sha256(value: Any) -> str:
+    return hashlib.sha256(_canonical(value).encode()).hexdigest()