fix(rectification): keep proportional prior after holdout calibration (BUG-560)
Implement offset and softmax relative-support scales and a public holdout calibration script, but leave the default proportional because no scheme passed the coverage-and-width gate. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
@@ -313,20 +313,18 @@ def _quantized_score(row: CandidateScoreRow) -> Decimal:
|
||||
return _decimal(row.get("score")).quantize(SCORE_QUANTUM, rounding=ROUND_HALF_UP)
|
||||
|
||||
|
||||
def _relative_support(scores: Sequence[Decimal]) -> list[int]:
|
||||
if not scores:
|
||||
def _distribute_percent(weights: Sequence[Decimal]) -> list[int]:
|
||||
if not weights:
|
||||
return []
|
||||
weights = [max(score, Decimal(0)) for score in scores]
|
||||
total = sum(weights, Decimal(0))
|
||||
if total == 0:
|
||||
base, remainder = divmod(100, len(scores))
|
||||
return [base + (1 if index < remainder else 0) for index in range(len(scores))]
|
||||
|
||||
base, remainder = divmod(100, len(weights))
|
||||
return [base + (1 if index < remainder else 0) for index in range(len(weights))]
|
||||
exact = [weight * Decimal(100) / total for weight in weights]
|
||||
floors = [int(value.to_integral_value(rounding=ROUND_FLOOR)) for value in exact]
|
||||
remaining = 100 - sum(floors)
|
||||
order = sorted(
|
||||
range(len(scores)),
|
||||
range(len(weights)),
|
||||
key=lambda index: (-(exact[index] - Decimal(floors[index])), index),
|
||||
)
|
||||
for index in order[:remaining]:
|
||||
@@ -334,18 +332,69 @@ def _relative_support(scores: Sequence[Decimal]) -> list[int]:
|
||||
return floors
|
||||
|
||||
|
||||
def _relative_support_proportional(scores: Sequence[Decimal]) -> list[int]:
|
||||
return _distribute_percent([max(score, Decimal(0)) for score in scores])
|
||||
|
||||
|
||||
def _relative_support_offset(scores: Sequence[Decimal], floor: Decimal) -> list[int]:
|
||||
return _distribute_percent([max(score - floor, Decimal(0)) for score in scores])
|
||||
|
||||
|
||||
def _relative_support_softmax(scores: Sequence[Decimal], temperature: Decimal) -> list[int]:
|
||||
from math import exp
|
||||
if not scores:
|
||||
return []
|
||||
peak = max(scores)
|
||||
temp = temperature if temperature > 0 else Decimal("0.5")
|
||||
weights = [Decimal(str(exp(float((score - peak) / temp)))) for score in scores]
|
||||
return _distribute_percent(weights)
|
||||
|
||||
|
||||
RELATIVE_SUPPORT_MODE = "proportional"
|
||||
RELATIVE_SUPPORT_TEMPERATURE = Decimal("0.5")
|
||||
|
||||
|
||||
def _relative_support(
|
||||
scores: Sequence[Decimal],
|
||||
*,
|
||||
floor: Decimal | None = None,
|
||||
mode: str | None = None,
|
||||
temperature: Decimal | None = None,
|
||||
) -> list[int]:
|
||||
if not scores:
|
||||
return []
|
||||
selected = mode or RELATIVE_SUPPORT_MODE
|
||||
if selected == "softmax":
|
||||
return _relative_support_softmax(
|
||||
scores,
|
||||
temperature if temperature is not None else RELATIVE_SUPPORT_TEMPERATURE,
|
||||
)
|
||||
if selected == "offset" and floor is not None:
|
||||
return _relative_support_offset(scores, floor)
|
||||
return _relative_support_proportional(scores)
|
||||
|
||||
|
||||
def build_candidate_decisions(
|
||||
rows: Sequence[CandidateScoreRow],
|
||||
*,
|
||||
result_id: str,
|
||||
static_contexts: Sequence[dict[str, Any]] | None = None,
|
||||
support_mode: str | None = None,
|
||||
temperature: Decimal | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
ranked = sorted(rows, key=lambda row: (-_quantized_score(row), row["time"]))
|
||||
public_rows = select_signature_representatives(ranked, static_contexts)
|
||||
if not public_rows:
|
||||
return []
|
||||
supports = _relative_support([_quantized_score(row) for row in public_rows])
|
||||
public_scores = [_quantized_score(row) for row in public_rows]
|
||||
all_scores = [_quantized_score(row) for row in ranked]
|
||||
floor = min(all_scores) if all_scores else Decimal(0)
|
||||
supports = _relative_support(
|
||||
public_scores,
|
||||
floor=floor,
|
||||
mode=support_mode,
|
||||
temperature=temperature,
|
||||
)
|
||||
decisions = []
|
||||
for index, row in enumerate(public_rows):
|
||||
score = _quantized_score(row)
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare relative_support scales on public minute-rectification cases.
|
||||
|
||||
Reads development cases for inspection and the sealed holdout for the
|
||||
enablement gate. Scores each window once, then rebuilds public candidate
|
||||
decisions under proportional / offset / softmax temperatures.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import traceback
|
||||
from calendar import monthrange
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from scripts.rectification.candidate_contrast import cluster_contexts_by_signature, context_time
|
||||
from scripts.rectification.contracts import normalize_rectification_request
|
||||
from scripts.rectification.decision_policy import build_candidate_decisions
|
||||
from scripts.rectification.scoring_service import (
|
||||
build_event_contribution_matrix,
|
||||
score_from_matrix,
|
||||
scoreable_request,
|
||||
)
|
||||
|
||||
DEVELOPMENT_MANIFEST = ROOT / "references" / "real_case_calibration" / "minute_rectification_development_v1.json"
|
||||
HOLDOUT_MANIFEST = ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3.json"
|
||||
LEAD = 8
|
||||
TODAY = date(2026, 9, 6)
|
||||
KIND_BY_DOMAIN = {
|
||||
"education": "education_milestone",
|
||||
"career": "career_change",
|
||||
"relationship": "relationship_commitment",
|
||||
"relocation": "relocation",
|
||||
"health_pressure": "self_health_event",
|
||||
"health": "self_health_event",
|
||||
"finance": "finance_change",
|
||||
"family": "family_event",
|
||||
}
|
||||
SCHEMES: tuple[dict[str, Any], ...] = (
|
||||
{"id": "proportional", "mode": "proportional", "temperature": None},
|
||||
{"id": "offset", "mode": "offset", "temperature": None},
|
||||
{"id": "softmax_0.25", "mode": "softmax", "temperature": Decimal("0.25")},
|
||||
{"id": "softmax_0.5", "mode": "softmax", "temperature": Decimal("0.5")},
|
||||
{"id": "softmax_1", "mode": "softmax", "temperature": Decimal("1")},
|
||||
{"id": "softmax_2", "mode": "softmax", "temperature": Decimal("2")},
|
||||
)
|
||||
|
||||
|
||||
def _clock(value: str) -> int:
|
||||
return int(value[:2]) * 60 + int(value[3:5])
|
||||
|
||||
|
||||
def _hhmm_from_minutes(value: int) -> str:
|
||||
wrapped = value % 1440
|
||||
return f"{wrapped // 60:02d}:{wrapped % 60:02d}"
|
||||
|
||||
|
||||
def _shift_clock(value: str, delta: int) -> str:
|
||||
return _hhmm_from_minutes(_clock(value) + delta)
|
||||
|
||||
|
||||
def _expand_date(raw: str, precision: str) -> tuple[str, str]:
|
||||
text = str(raw or "").strip()
|
||||
if precision == "day":
|
||||
day = date.fromisoformat(text)
|
||||
return day.isoformat(), day.isoformat()
|
||||
if precision == "month":
|
||||
month = date.fromisoformat(f"{text}-01") if len(text) == 7 else date.fromisoformat(text[:10]).replace(day=1)
|
||||
last = monthrange(month.year, month.month)[1]
|
||||
return month.isoformat(), month.replace(day=last).isoformat()
|
||||
year = int(text[:4])
|
||||
return f"{year}-01-01", f"{year}-12-31"
|
||||
|
||||
|
||||
def _event_kind(event: dict[str, Any]) -> str:
|
||||
domain = str(event.get("domain") or "")
|
||||
description = str(event.get("description") or "").lower()
|
||||
if domain == "relationship" and not any(token in description for token in ("married", "wedding", "wife", "husband")):
|
||||
return "relationship_start"
|
||||
return KIND_BY_DOMAIN[domain]
|
||||
|
||||
|
||||
def _request_from_case(case: dict[str, Any]) -> dict[str, Any]:
|
||||
birth = case["birth"]
|
||||
true_time = str(birth["time"])[:5]
|
||||
radius = int(case.get("candidate_radius_minutes") or 10)
|
||||
events = []
|
||||
for event in case.get("events") or []:
|
||||
precision = str(event.get("precision") or "year")
|
||||
start, end = _expand_date(str(event.get("date") or ""), precision)
|
||||
domain = str(event.get("domain") or "")
|
||||
events.append({
|
||||
"id": str(uuid5(NAMESPACE_URL, str(event.get("id") or ""))),
|
||||
"domain": domain,
|
||||
"event_kind": _event_kind(event),
|
||||
"date_start": start,
|
||||
"date_end": end,
|
||||
"precision": precision,
|
||||
"summary": str(event.get("description") or event.get("id") or domain)[:200],
|
||||
})
|
||||
return normalize_rectification_request({
|
||||
"birth_date": str(birth["date"]),
|
||||
"start_time": _shift_clock(true_time, -radius),
|
||||
"end_time": _shift_clock(true_time, radius),
|
||||
"lat": float(birth["latitude"]),
|
||||
"lon": float(birth["longitude"]),
|
||||
"tz": float(birth["timezone_offset"]),
|
||||
"events": events,
|
||||
}, today=TODAY)
|
||||
|
||||
|
||||
def _cluster_bounds(time: str, clusters: list[dict[str, Any]]) -> tuple[str, str]:
|
||||
for cluster in clusters:
|
||||
times = [str(item)[:5] for item in cluster.get("times") or [] if str(item)]
|
||||
if time not in times:
|
||||
continue
|
||||
ordered = sorted(times, key=_clock)
|
||||
return ordered[0], ordered[-1]
|
||||
return time, time
|
||||
|
||||
|
||||
def _union_metrics(
|
||||
decisions: list[dict[str, Any]],
|
||||
*,
|
||||
true_time: str,
|
||||
start_time: str,
|
||||
clusters: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
if not decisions:
|
||||
return {"width": None, "covers": False, "peak": 0, "valid_count": 0}
|
||||
peak = max(int(item["relative_support"]) for item in decisions)
|
||||
valid = [item for item in decisions if peak - int(item["relative_support"]) < LEAD]
|
||||
origin = _clock(start_time)
|
||||
points: list[int] = []
|
||||
for item in valid:
|
||||
time = str(item["time"])[:5]
|
||||
lo, hi = _cluster_bounds(time, clusters)
|
||||
for stamp in (time, lo, hi):
|
||||
points.append((_clock(stamp) - origin) % 1440)
|
||||
width = (max(points) - min(points) + 1) if points else None
|
||||
true_offset = (_clock(true_time) - origin) % 1440
|
||||
covers = width is not None and min(points) <= true_offset <= max(points)
|
||||
return {
|
||||
"width": width,
|
||||
"covers": covers,
|
||||
"peak": peak,
|
||||
"valid_count": len(valid),
|
||||
"supports": [int(item["relative_support"]) for item in decisions],
|
||||
}
|
||||
|
||||
|
||||
def _score_case(case: dict[str, Any]) -> dict[str, Any]:
|
||||
request = _request_from_case(case)
|
||||
scoring = scoreable_request(request)
|
||||
built = build_event_contribution_matrix(scoring)
|
||||
rows = score_from_matrix(scoring, built)
|
||||
contexts = built.get("static_contexts") if isinstance(built.get("static_contexts"), list) else None
|
||||
clusters = cluster_contexts_by_signature(
|
||||
[item for item in (contexts or []) if isinstance(item, dict) and context_time(item)]
|
||||
) if contexts else []
|
||||
true_time = str(case["birth"]["time"])[:5]
|
||||
schemes = {}
|
||||
for scheme in SCHEMES:
|
||||
decisions = build_candidate_decisions(
|
||||
rows,
|
||||
result_id=str(uuid5(NAMESPACE_URL, case["case_id"])),
|
||||
static_contexts=contexts,
|
||||
support_mode=scheme["mode"],
|
||||
temperature=scheme["temperature"],
|
||||
)
|
||||
schemes[scheme["id"]] = _union_metrics(
|
||||
decisions,
|
||||
true_time=true_time,
|
||||
start_time=request["start_time"],
|
||||
clusters=clusters,
|
||||
)
|
||||
return {
|
||||
"case_id": case["case_id"],
|
||||
"radius": int(case.get("candidate_radius_minutes") or 10),
|
||||
"schemes": schemes,
|
||||
}
|
||||
|
||||
|
||||
def _summarize(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
table: dict[str, Any] = {}
|
||||
for scheme in SCHEMES:
|
||||
widths = [int(item["schemes"][scheme["id"]]["width"]) for item in rows if item["schemes"][scheme["id"]]["width"] is not None]
|
||||
covered = sum(1 for item in rows if item["schemes"][scheme["id"]]["covers"])
|
||||
table[scheme["id"]] = {
|
||||
"n": len(rows),
|
||||
"coverage": covered,
|
||||
"median_width": statistics.median(widths) if widths else None,
|
||||
"mean_width": round(statistics.mean(widths), 2) if widths else None,
|
||||
}
|
||||
return table
|
||||
|
||||
|
||||
def _pick_scheme(holdout: dict[str, Any]) -> dict[str, Any]:
|
||||
baseline = holdout["proportional"]
|
||||
old_coverage = int(baseline["coverage"])
|
||||
old_median = baseline["median_width"]
|
||||
passing = []
|
||||
for scheme_id, row in holdout.items():
|
||||
if scheme_id == "proportional":
|
||||
continue
|
||||
median = row["median_width"]
|
||||
if median is None or old_median is None:
|
||||
continue
|
||||
if int(row["coverage"]) >= old_coverage - 1 and median < old_median:
|
||||
passing.append((scheme_id, row))
|
||||
if not passing:
|
||||
return {"id": "proportional", "enabled": False, "reason": "no_scheme_passed_holdout_gate"}
|
||||
preferred = [item for item in passing if item[0] == "offset"]
|
||||
chosen = preferred[0] if preferred else min(passing, key=lambda item: (item[1]["median_width"], -item[1]["coverage"]))
|
||||
return {"id": chosen[0], "enabled": True, "reason": "passed_holdout_gate"}
|
||||
|
||||
|
||||
def _load_cases(path: Path, *, holdout_only: bool) -> list[dict[str, Any]]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
cases = list(payload.get("cases") or [])
|
||||
if holdout_only:
|
||||
cases = [item for item in cases if item.get("excluded_from_tuning") is True]
|
||||
return cases
|
||||
|
||||
|
||||
def run(limit: int | None = None) -> dict[str, Any]:
|
||||
development_cases = _load_cases(DEVELOPMENT_MANIFEST, holdout_only=False)
|
||||
holdout_cases = _load_cases(HOLDOUT_MANIFEST, holdout_only=True)
|
||||
if limit is not None:
|
||||
development_cases = development_cases[:limit]
|
||||
holdout_cases = holdout_cases[:limit]
|
||||
development_rows = [_score_case(item) for item in development_cases]
|
||||
holdout_rows = [_score_case(item) for item in holdout_cases]
|
||||
development = _summarize(development_rows)
|
||||
holdout = _summarize(holdout_rows)
|
||||
return {
|
||||
"lead": LEAD,
|
||||
"development": development,
|
||||
"holdout": holdout,
|
||||
"selection": _pick_scheme(holdout),
|
||||
"development_rows": development_rows,
|
||||
"holdout_rows": holdout_rows,
|
||||
}
|
||||
|
||||
|
||||
def _print_table(title: str, table: dict[str, Any]) -> None:
|
||||
print(title)
|
||||
print(f"{'scheme':<16} {'coverage':>10} {'median':>10} {'mean':>10}")
|
||||
for scheme_id, row in table.items():
|
||||
print(
|
||||
f"{scheme_id:<16} "
|
||||
f"{row['coverage']}/{row['n']:<7} "
|
||||
f"{str(row['median_width']):>10} "
|
||||
f"{str(row['mean_width']):>10}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--limit", type=int, default=None)
|
||||
parser.add_argument("--json-out", type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
report = run(args.limit)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
_print_table("development", report["development"])
|
||||
print()
|
||||
_print_table("holdout", report["holdout"])
|
||||
print()
|
||||
print("selection", json.dumps(report["selection"], ensure_ascii=False))
|
||||
if args.json_out:
|
||||
args.json_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.json_out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user