research(rectification): add reported-offset evaluation and frozen rerun integrity

Preserve closed confirmation gates and previously-exposed dataset boundaries. Add auditable 900-trial sensitivity results, current scorer freshness checks, and the v5 collection protocol.

Co-Authored-By: Claude Code <noreply@anthropic.com>

tests/test_rectification_confirmation_and.py

@@ -93,6 +93,7 @@ def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
     sidecar = diagnostic["current_tree_unfrozen_diagnostic"]
     produced_by = product["metrics_produced_by"]
     current_tree = product["current_tree_scorer"]
+    rerun = json.loads((ROOT / current_tree["source_report"]).read_text(encoding="utf-8"))
     unfrozen = product["current_tree_unfrozen_diagnostic"]
     assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"]
     assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"]
@@ -115,7 +116,10 @@ def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
     assert produced_by["algorithm_version"] == report["frozen_scoring"]["algorithm_version"]
     assert produced_by["implementation_hash_matches_at_replay"] is True
     assert produced_by["source_report"] == "references/real_case_calibration/minute_rectification_holdout_v3_report.json"
-    assert current_tree["implementation_sha256"] == diagnostic["frozen_scoring"]["actual_sha256"]
+    assert current_tree["implementation_sha256"] == rerun["frozen_record"]["implementation_sha256"]
+    assert current_tree["fixed_protocol_rerun_hash_matches"] is rerun["implementation_hash_matches_at_replay"] is True
+    assert current_tree["fixed_protocol_rerun_trial_count"] == rerun["trial_count"] == 20
+    assert rerun["official_valid_independent_blind"] is False
     assert current_tree["matches_metrics_scorer"] is False
     assert current_tree["official_eval_implementation_hash_matches"] is False
     assert current_tree["official_eval_trial_count"] == diagnostic["trial_count"] == 0

tests/test_rectification_validation_integrity_gate.py

@@ -0,0 +1,16 @@
+"""Collect research integrity regressions in the existing rectification quick gate."""
+
+from tests.test_reported_offset_research import (  # noqa: F401
+    test_cross_midnight_real_engine_scores_match_candidate_date_replay,
+    test_offset_beyond_radius_has_no_truth_candidate,
+    test_opaque_tie_break_is_independent_of_truth_and_input_order,
+    test_recorded_specification_and_all_prespecified_cells,
+    test_shifted_window_preserves_dates_across_midnight,
+    test_zero_offset_centres_on_truth_and_has_complete_grid,
+)
+from tests.test_sealed_holdout_contract_freshness import (  # noqa: F401
+    test_changed_frozen_identity_fails_before_any_replay,
+    test_contract_tracks_actual_current_scorer_and_dataset_audit,
+    test_fixed_protocol_rerun_is_auditable_but_never_independent_blind,
+    test_frozen_record_matches_dataset_scorer_and_evaluator_bytes,
+)

tests/test_reported_offset_research.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta
+
+import pytest
+
+from scripts.research import reported_offset_sweep as sweep
+from scripts.research.sealed_holdout_rerun import DATASET, opaque_order
+
+
+def public_case():
+    return json.loads(DATASET.read_text(encoding="utf-8"))["cases"][0]
+
+
+def test_zero_offset_centres_on_truth_and_has_complete_grid():
+    case = public_case()
+    truth = datetime.fromisoformat(f"{case['birth']['date']}T{case['birth']['time']}:00")
+    request, candidates = sweep.shifted_window(case, 0, 15)
+    assert candidates[15] == truth
+    assert len(candidates) == 31
+    assert request["start_time"] == candidates[0].strftime("%H:%M")
+    assert request["end_time"] == candidates[-1].strftime("%H:%M")
+    assert truth in candidates
+    assert not any("truth" in key or "offset" in key for key in request)
+
+
+@pytest.mark.parametrize("offset", [-30, -20, 20, 30])
+def test_offset_beyond_radius_has_no_truth_candidate(offset):
+    case = public_case()
+    truth = datetime.fromisoformat(f"{case['birth']['date']}T{case['birth']['time']}:00")
+    _, candidates = sweep.shifted_window(case, offset, 15)
+    assert truth not in candidates
+    rows = [{"time": moment.strftime("%H:%M"), "score": 1} for moment in candidates]
+    metrics = sweep.reveal_metrics(rows, candidates, candidates, truth, "test", "test")
+    assert metrics["truth_in_window"] is False
+    assert metrics["true_rank"] is None
+    assert metrics["delivery_covers_truth"] is False
+
+
+def test_opaque_tie_break_is_independent_of_truth_and_input_order():
+    # Explicitly synthetic unit inputs, not a fabricated engine-contract fixture.
+    moments = [datetime(2000, 1, 1, 12) + timedelta(minutes=i) for i in range(3)]
+    rows = [{"time": moment.strftime("%H:%M"), "score": 1} for moment in moments]
+    ordered = opaque_order("test", "synthetic", rows)
+    assert ordered == opaque_order("test", "synthetic", list(reversed(rows)))
+    winner = next(moment for moment in moments if moment.strftime("%H:%M") == ordered[0]["time"])
+    for truth in moments:
+        metrics = sweep.reveal_metrics(rows, moments, moments, truth, "test", "synthetic")
+        assert metrics["top_1_hit"] is (truth == winner)
+        assert metrics["true_rank"] == 1 + next(i for i, row in enumerate(ordered) if row["time"] == truth.strftime("%H:%M"))
+
+
+@pytest.mark.parametrize("clock,offset", [("00:02", -5), ("23:58", 5)])
+def test_shifted_window_preserves_dates_across_midnight(clock, offset):
+    case = public_case()
+    # Synthetic boundary mutation of the public case; no real user's birth data.
+    case["birth"] = {**case["birth"], "time": clock}
+    request, candidates = sweep.shifted_window(case, offset, 15)
+    truth = datetime.fromisoformat(f"{case['birth']['date']}T{clock}:00")
+    assert candidates[15] == truth + timedelta(minutes=offset)
+    assert request["birth_date"] == candidates[0].date().isoformat()
+    rows = [{"time": moment.strftime("%H:%M"), "score": int(moment == truth)} for moment in candidates]
+    metrics = sweep.reveal_metrics(rows, candidates, candidates, truth, "test", "synthetic")
+    assert metrics["top_1_minute_error"] == 0
+    assert metrics["truth_in_window"] is True
+    assert metrics["delivery_covers_truth"] is True
+    assert metrics["delivery_width_minutes"] == 31
+
+
+def test_cross_midnight_real_engine_scores_match_candidate_date_replay():
+    case = json.loads(DATASET.read_text(encoding="utf-8"))["cases"][5]
+    request, candidates = sweep.shifted_window(case, 0, 60)
+    assert len({candidate.date() for candidate in candidates}) == 2
+    contexts = sweep.compute_candidate_static_contexts(request, candidates=candidates)
+    grouped = sweep.score_window(request, contexts)
+    expected = []
+    for context in contexts:
+        dated = {**request, "birth_date": context["candidate_at"].date().isoformat()}
+        built = sweep.build_event_contribution_matrix(dated, static_contexts=[context])
+        expected.extend(sweep.score_from_matrix(dated, built))
+    assert grouped == expected
+    old_matrix = sweep.build_event_contribution_matrix(request, static_contexts=contexts)
+    old_rows = sweep.score_from_matrix(request, old_matrix)
+    assert old_rows != expected
+
+
+def test_recorded_specification_and_all_prespecified_cells():
+    report = json.loads((sweep.ROOT / "docs/research/reported_offset_2026_09_20.json").read_text(encoding="utf-8"))
+    spec = report["specification"]
+    assert spec["ayanamsa"] == "raman"
+    assert spec["node_mode"] == "mean"
+    assert spec["radii_minutes"] == list(sweep.RADII)
+    assert spec["offsets_minutes"] == list(sweep.OFFSETS)
+    assert spec["minute_step"] == 1
+    assert spec["implementation_sha256_prefix"] == spec["implementation_sha256"][:16]
+    assert spec["dataset_sha256"] == sweep.file_sha256(DATASET)
+    assert spec["evaluator_sha256"] == sweep.file_sha256(sweep.ROOT / "scripts/research/reported_offset_sweep.py")
+    assert spec["production_scoring_sha256"] == sweep.implementation_sha256(spec["production_scoring_files"])
+    assert spec["research_implementation_sha256"] == sweep.implementation_sha256(spec["research_files"])
+    assert spec["replay_revision"] == "candidate_date_grouped_v2"
+    assert spec["truth_hidden_from_ranker"] is True
+    assert spec["is_blind_evaluation"] is False
+    assert report["trial_count"] == 20 * len(sweep.RADII) * len(sweep.OFFSETS)
+    assert len(report["summary"]) == len(sweep.RADII) * len(sweep.OFFSETS)
+    assert report["summary"] == sweep.summarize(report["trials"], sweep.RADII, sweep.OFFSETS)
+    assert {(row["case_ordinal"], row["radius_minutes"], row["offset_minutes"]) for row in report["trials"]} == {
+        (case, radius, offset) for case in range(1, 21) for radius in sweep.RADII for offset in sweep.OFFSETS
+    }
+    assert report["widening_geometry"]["radius_120_scored"] is False
+    for trial in report["trials"]:
+        assert not ({"predicted_time", "birth", "latitude", "longitude"} & trial.keys())
+    for row in report["summary"]:
+        expected = float(abs(row["offset_minutes"]) <= row["radius_minutes"])
+        assert row["truth_in_window_rate"] == expected
+        assert row["delivery_coverage_rate"] <= expected
+        assert row["top_1_rate"] <= expected

tests/test_sealed_holdout_contract_freshness.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.minute_rectification_blind_eval import implementation_sha256, summarize_trials
+from scripts.rectification.sealed_holdout import holdout_passed, load_sealed_minute_holdout
+from scripts.research.sealed_holdout_rerun import DATASET, FREEZE, REPORT, file_sha256, freeze_record, run
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def read(path):
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def test_contract_tracks_actual_current_scorer_and_dataset_audit():
+    dataset = read(DATASET)
+    contract = read(ROOT / "references/rectification_sealed_holdout.v1.json")
+    actual_hash = implementation_sha256(dataset["frozen_scoring"]["files"])
+    assert contract["current_tree_scorer"]["implementation_sha256"] == actual_hash
+    assert contract["source_audit_status"] == dataset["source_audit_status"]
+    assert contract["evaluated_on"] == read(REPORT)["evaluated_on"]
+    assert contract["status"] == "not_ready"
+    assert contract["confirmation_coverage_rate"] == 0.0
+    assert holdout_passed(load_sealed_minute_holdout()) is False
+
+
+def test_frozen_record_matches_dataset_scorer_and_evaluator_bytes():
+    frozen = read(FREEZE)
+    actual = freeze_record()
+    assert {key: value for key, value in frozen.items() if key != "frozen_at_utc"} == {
+        key: value for key, value in actual.items() if key != "frozen_at_utc"
+    }
+    assert frozen["dataset_sha256"] == file_sha256(DATASET)
+    assert len(frozen["files"]) == 12
+    assert read(DATASET)["frozen_scoring"]["implementation_sha256"] == frozen["historical_frozen_sha256"]
+
+
+def test_fixed_protocol_rerun_is_auditable_but_never_independent_blind():
+    report = read(REPORT)
+    contract = read(ROOT / "references/rectification_sealed_holdout.v1.json")
+    scorer = contract["current_tree_scorer"]
+    rerun = contract["current_tree_fixed_protocol_rerun"]
+    assert report["frozen_record"] == read(FREEZE)
+    assert report["trial_count"] == len(report["trials"]) == 20
+    assert report["excluded_cases"] == []
+    aggregate = summarize_trials(report["trials"], read(DATASET)["release_metrics"])
+    assert report["metrics"] == aggregate["metrics"] == scorer["metrics"]
+    assert report["metric_gates_passed"] == aggregate["metric_gates_passed"]
+    assert scorer["source_report"] == REPORT.relative_to(ROOT).as_posix()
+    assert scorer["implementation_sha256"] == report["frozen_record"]["implementation_sha256"]
+    assert scorer["fixed_protocol_rerun_trial_count"] == report["trial_count"]
+    assert scorer["fixed_protocol_rerun_hash_matches"] is True
+    assert scorer["official_eval_trial_count"] == report["official_blind_trial_count"] == 0
+    assert scorer["official_eval_implementation_hash_matches"] is False
+    for item in (rerun, report):
+        assert item["official_valid_independent_blind"] is False
+        assert item["is_blind_evaluation"] is False
+        assert item["truth_hidden_from_ranker"] is True
+        assert item["results_previously_seen"] is True
+        assert item["verified_minute_claim_allowed"] is False
+    assert rerun["must_not_claim_as_release_metrics"] is True
+    assert all(row["event_count"] == 3 for row in report["trials"])
+    # Never persist actual candidate/truth times or coordinates in report artifacts.
+    for row in report["trials"]:
+        assert not ({"predicted_time", "published_truth_revealed_after_ranking", "latitude", "longitude"} & row.keys())
+
+
+def test_changed_frozen_identity_fails_before_any_replay(tmp_path, monkeypatch):
+    frozen = read(FREEZE)
+    frozen["implementation_sha256"] = "0" * 64
+    path = tmp_path / "bad-freeze.json"
+    path.write_text(json.dumps(frozen), encoding="utf-8")
+    def unexpected(*args, **kwargs):
+        raise AssertionError("must reject identity before scoring")
+    monkeypatch.setattr("scripts.research.sealed_holdout_rerun.build_feature_fact_rows", unexpected)
+    with pytest.raises(ValueError, match="frozen_record_mismatch:implementation_sha256"):
+        run(path)