root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

research(rectification): add reported-offset evaluation and frozen rerun integrity
Independent Staging Quality Gate / validate (push) Successful in 12m7s
Details
Independent Staging Quality Gate / publish (push) Successful in 3m46s
Details

Browse Source 
Preserve closed confirmation gates and previously-exposed dataset boundaries. Add auditable 900-trial sensitivity results, current scorer freshness checks, and the v5 collection protocol.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
jesse-ux
2026-09-20 12:03:52 +08:00
co-authored by Claude Code
parent 5049e7847b
commit 932f2fffba
22 changed files with 14134 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/research/reported_offset_sweep.py
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Reported-centre sensitivity on exposed public cases, without oracle answers."""
from __future__ import annotations
import argparse
import json
import sys
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.active_rectification_event_engine import (
AYANAMSA, NODE_MODE, compute_candidate_static_contexts,
)
from scripts.minute_rectification_blind_eval import _opaque_winner, implementation_sha256
from scripts.minute_rectification_holdout_validator import validate
from scripts.rectification.candidate_contrast import select_signature_representatives
from scripts.rectification.scoring_service import build_event_contribution_matrix, score_from_matrix
from scripts.research.cluster_width_lib import SEPARATION_LEAD, still_valid_public
from scripts.research.probe_supply_after_six import request_from_case
from scripts.research.sealed_holdout_rerun import DATASET, file_sha256, opaque_order
OFFSETS = (-30, -20, -15, -10, -8, -5, -3, 0, 3, 5, 8, 10, 15, 20, 30)
RADII = (15, 30, 60)
MINUTE_STEP = 1
PRODUCTION_FILES = [
"scripts/rectification/scoring_service.py",
"scripts/rectification/dasha_transition_proximity.py",
"scripts/rectification/candidate_contrast.py",
"scripts/rectification/case_holdout.py",
"scripts/rectification/contracts.py",
"scripts/rectification/event_probes.py",
]
RESEARCH_FILES = [
"scripts/research/reported_offset_sweep.py",
"scripts/research/cluster_width_lib.py",
"scripts/research/probe_supply_after_six.py",
"scripts/research/sealed_holdout_rerun.py",
"scripts/minute_rectification_blind_eval.py",
]
def shifted_window(case: dict[str, Any], offset: int, radius: int) -> tuple[dict[str, Any], list[datetime]]:
if radius not in (*RADII, 120):
raise ValueError("unsupported_radius")
true_at = datetime.fromisoformat(f"{case['birth']['date']}T{case['birth']['time']}:00")
reported_at = true_at + timedelta(minutes=offset)
start = reported_at - timedelta(minutes=radius)
end = reported_at + timedelta(minutes=radius)
candidates = [start + timedelta(minutes=i) for i in range(0, radius * 2 + 1, MINUTE_STEP)]
# Reuse only the established event normalization, then replace the window.
# The ranker sees neither the truth label nor the simulated offset.
request = request_from_case(case)
request.update({
"birth_date": start.date().isoformat(),
"start_time": start.strftime("%H:%M"),
"end_time": end.strftime("%H:%M"),
"minute_step": MINUTE_STEP,
"ayanamsa": AYANAMSA,
"node_mode": NODE_MODE,
})
return request, candidates
def score_window(request: dict[str, Any], contexts: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Keep auxiliary transition anchors on each candidate's actual date.
The production matrix's transition-proximity helper accepts one birth date
per call, unlike the static chart layer which reads candidate_at. Grouping
is only an offline adapter; it does not change the production scorer.
"""
by_date: dict[str, list[dict[str, Any]]] = {}
for context in contexts:
by_date.setdefault(context["candidate_at"].date().isoformat(), []).append(context)
by_time = {}
for candidate_date, group in by_date.items():
dated_request = {**request, "birth_date": candidate_date}
built = build_event_contribution_matrix(dated_request, static_contexts=group)
by_time.update({row["time"]: row for row in score_from_matrix(dated_request, built)})
return [by_time[context["candidate_at"].strftime("%H:%M")] for context in contexts]
def delivery_moments(public: list[dict[str, Any]], candidates: list[datetime]) -> list[datetime]:
"""Initial delivery envelope in date-aware order; no truth-based narrowing."""
valid = still_valid_public(public, {}, lead=SEPARATION_LEAD)
clocks = {str(value)[:5] for row in valid for value in (row.get("cluster_times") or [row["time"]])}
return [candidate for candidate in candidates if candidate.strftime("%H:%M") in clocks]
def reveal_metrics(
rows: list[dict[str, Any]], candidates: list[datetime], delivery: list[datetime],
truth: datetime, benchmark_id: str, case_id: str,
) -> dict[str, Any]:
ordered = opaque_order(benchmark_id, case_id, rows)
predicted_clock = _opaque_winner(benchmark_id, case_id, rows)
by_clock = {candidate.strftime("%H:%M"): candidate for candidate in candidates}
if len(by_clock) != len(candidates):
raise ValueError("ambiguous_candidate_clock")
predicted = by_clock[predicted_clock]
within = min(candidates) <= truth <= max(candidates)
in_candidates = truth in candidates
rank = next((i for i, row in enumerate(ordered, 1) if by_clock[row["time"]] == truth), None)
covered = bool(delivery) and min(delivery) <= truth <= max(delivery)
return {
"truth_in_window": within,
"truth_in_candidates": in_candidates,
"true_rank": rank,
"top_1_hit": predicted == truth,
"top_1_minute_error": abs((predicted - truth).total_seconds()) / 60,
"delivery_covers_truth": covered,
"candidate_count": len(candidates),
"delivery_width_minutes": int((max(delivery) - min(delivery)).total_seconds() / 60) + 1 if delivery else 0,
}
def summarize(trials: list[dict[str, Any]], radii: tuple[int, ...], offsets: tuple[int, ...]) -> list[dict[str, Any]]:
result = []
for radius in radii:
for offset in offsets:
group = [row for row in trials if row["radius_minutes"] == radius and row["offset_minutes"] == offset]
count = len(group)
result.append({
"radius_minutes": radius, "offset_minutes": offset, "trial_count": count,
**{key: round(sum(row[source] for row in group) / count, 4) if count else None for key, source in (
("truth_in_window_rate", "truth_in_window"),
("top_1_rate", "top_1_hit"),
("delivery_coverage_rate", "delivery_covers_truth"),
("mean_absolute_minute_error", "top_1_minute_error"),
)},
})
return result
def run(dataset: Path = DATASET, radii: tuple[int, ...] = RADII, offsets: tuple[int, ...] = OFFSETS) -> dict[str, Any]:
manifest = json.loads(dataset.read_text(encoding="utf-8"))
frozen_files = manifest["frozen_scoring"]["files"]
scoring_files = sorted(set(frozen_files + PRODUCTION_FILES))
starting_hash = implementation_sha256(scoring_files)
validation = validate(dataset)
invalid = validation["invalid_cases"]
trials = []
for index, case in enumerate(manifest["cases"], 1):
if case["case_id"] in invalid:
continue
# Reuse static chart calculations, not window-dependent scores or ranks.
contexts_by_moment: dict[datetime, dict[str, Any]] = {}
for radius in radii:
for offset in offsets:
request, candidates = shifted_window(case, offset, radius)
missing = [moment for moment in candidates if moment not in contexts_by_moment]
if missing:
contexts_by_moment.update(zip(missing, compute_candidate_static_contexts(request, candidates=missing), strict=True))
contexts = [contexts_by_moment[moment] for moment in candidates]
rows = score_window(request, contexts)
public = select_signature_representatives(rows, contexts)
delivery = delivery_moments(public, candidates)
# All scores and the delivery envelope are locked before reveal.
truth = datetime.fromisoformat(f"{case['birth']['date']}T{case['birth']['time']}:00")
trials.append({
"case_ordinal": index, "offset_minutes": offset, "radius_minutes": radius,
**reveal_metrics(rows, candidates, delivery, truth, manifest["benchmark_id"], case["case_id"]),
})
if implementation_sha256(scoring_files) != starting_hash:
raise ValueError("scorer_changed_during_sweep")
return {
"scope": "reported_offset_sensitivity_not_product_accuracy",
"specification": {
"ayanamsa": AYANAMSA, "node_mode": NODE_MODE,
"radii_minutes": list(radii), "offsets_minutes": list(offsets), "minute_step": MINUTE_STEP,
"dataset": dataset.relative_to(ROOT).as_posix(), "dataset_sha256": file_sha256(dataset),
"dataset_benchmark_id": manifest["benchmark_id"],
"implementation_sha256": implementation_sha256(frozen_files),
"implementation_sha256_prefix": implementation_sha256(frozen_files)[:16],
"production_scoring_files": scoring_files, "production_scoring_sha256": starting_hash,
"research_files": RESEARCH_FILES,
"research_implementation_sha256": implementation_sha256(RESEARCH_FILES),
"hash_scope": "explicit_identity_file_sets_not_a_transitive_dependency_lock",
"scorer": "native_event_contribution_matrix_not_shadow_fact_ranker",
"delivery": "initial_signature_clusters_peak_gap_lt_8_envelope_no_answers_no_elimination",
"rank": "score_desc_then_sha256(benchmark_id:case_id:candidate_time)",
"grid": "every_minute_inclusive_not_production_two_minute_sampling",
"cross_midnight": "date_aware_candidates_distance_and_matrix_grouped_by_candidate_date",
"evaluator_sha256": file_sha256(Path(__file__)),
"replay_revision": "candidate_date_grouped_v2",
"supersedes": "initial_sweep_invalidated_cross_midnight_transition_anchor",
"is_blind_evaluation": False, "truth_hidden_from_ranker": True,
"results_previously_seen": True, "must_not_use_for_tuning": True,
},
"excluded_cases": invalid, "case_count": validation["valid_public_aa_cases"],
"trial_count": len(trials), "trials": trials,
"summary": summarize(trials, radii, offsets),
"widening_geometry": {
"scored_radii": list(radii),
"radius_120_scored": 120 in radii,
"truth_in_window_condition": "abs(reported_offset_minutes) <= radius_minutes",
"radius_15_first_integer_minute_outside": 16,
"all_tested_offsets_within_radius_30_60_120": max(map(abs, offsets)) <= 30,
"boundary": "Geometry rescues candidate inclusion only, not ranking or delivery coverage; no population frequency without a reported-offset distribution.",
},
}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
print(json.dumps(run(), ensure_ascii=False, indent=2))
scripts/research/sealed_holdout_rerun.py
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Freeze and replay the exposed v3 corpus; never claim a fresh blind holdout."""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[2]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.active_rectification_event_engine import AYANAMSA, NODE_MODE
from scripts.minute_rectification_blind_eval import (
_candidate_moments, _clock_distance, _opaque_winner, _request,
implementation_sha256, summarize_trials,
)
from scripts.minute_rectification_fact_blind_eval_v4 import _would_confirm
from scripts.minute_rectification_fact_ranker_v4 import (
ALGORITHM_VERSION, rank_fact_rows, score_fact_ranker_v4,
)
from scripts.minute_rectification_feature_facts_v4 import build_feature_fact_rows
from scripts.minute_rectification_holdout_validator import validate
DATASET = ROOT / "references/real_case_calibration/minute_rectification_holdout_v3.json"
FREEZE = ROOT / "docs/research/sealed_holdout_rerun_2026_09_20.freeze.json"
REPORT = ROOT / "docs/research/sealed_holdout_rerun_2026_09_20.json"
def file_sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def opaque_order(benchmark_id: str, case_id: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Extend the existing opaque winner to a total, truth-independent ranking."""
return sorted(rows, key=lambda row: (
-row["score"],
hashlib.sha256(f"{benchmark_id}:{case_id}:{row['time']}".encode()).hexdigest(),
))
def freeze_record(dataset: Path = DATASET) -> dict[str, Any]:
manifest = json.loads(dataset.read_text(encoding="utf-8"))
files = manifest["frozen_scoring"]["files"]
return {
"record_version": "exposed-v3-fixed-protocol-rerun-v1",
"frozen_at_utc": datetime.now(timezone.utc).isoformat(),
"dataset_path": dataset.relative_to(ROOT).as_posix(),
"dataset_sha256": file_sha256(dataset),
"algorithm_version": ALGORITHM_VERSION,
"implementation_sha256": implementation_sha256(files),
"files": files,
"historical_frozen_sha256": manifest["frozen_scoring"]["implementation_sha256"],
"evaluator_sha256": file_sha256(Path(__file__)),
"ayanamsa": AYANAMSA,
"node_mode": NODE_MODE,
"candidate_radius_minutes": sorted({case["candidate_radius_minutes"] for case in manifest["cases"]}),
"minute_step": 1,
"release_metrics": manifest["release_metrics"],
"results_previously_seen": True,
"official_valid_independent_blind": False,
"must_not_use_for_tuning": True,
"tie_breaker": manifest["frozen_scoring"]["tie_breaker"],
"metric_rank_definition": "competition_rank_1_plus_strictly_higher_scores_legacy_protocol",
"extra_metric_rank_definition": "score_desc_then_existing_opaque_sha256_total_order",
}
def run(freeze_path: Path = FREEZE, dataset: Path = DATASET) -> dict[str, Any]:
frozen = json.loads(freeze_path.read_text(encoding="utf-8"))
actual = freeze_record(dataset)
for key in actual:
if key != "frozen_at_utc" and actual[key] != frozen.get(key):
raise ValueError(f"frozen_record_mismatch:{key}")
manifest = json.loads(dataset.read_text(encoding="utf-8"))
validation = validate(dataset)
invalid = validation["invalid_cases"]
trials = []
for index, case in enumerate(manifest["cases"], 1):
if case["case_id"] in invalid:
continue
request = _request(case, case["events"])
candidates = _candidate_moments(case)
facts = build_feature_fact_rows(request, candidates=candidates)
rows, _ = rank_fact_rows(facts, request["events"])
result = score_fact_ranker_v4(facts, request["events"])
predicted = _opaque_winner(manifest["benchmark_id"], case["case_id"], rows)
ordered = opaque_order(manifest["benchmark_id"], case["case_id"], rows)
sparse_request = _request(case, case["events"][:1])
sparse_facts = build_feature_fact_rows(sparse_request, candidates=candidates)
sparse_result = score_fact_ranker_v4(sparse_facts, sparse_request["events"])
# Truth is revealed only after both full and sparse ranking/decisions.
truth = case["birth"]["time"]
truth_score = next(row["score"] for row in rows if row["time"] == truth)
would_confirm = _would_confirm(result)
trials.append({
"case_ordinal": index,
"candidate_count": len(rows),
"event_count": len(request["events"]),
"true_rank": 1 + sum(row["score"] > truth_score for row in rows),
"opaque_true_rank": next(i for i, row in enumerate(ordered, 1) if row["time"] == truth),
"minute_error": _clock_distance(predicted, truth),
"would_confirm": would_confirm,
"false_confirmation": would_confirm and predicted != truth,
"insufficient_evidence_rejected": not _would_confirm(sparse_result),
"full_trial_reasons": result["reasons"],
"sparse_trial_reasons": sparse_result["reasons"],
})
aggregate = summarize_trials(trials, manifest["release_metrics"])
count = len(trials)
return {
"scope": "fixed_protocol_previously_exposed_v3_rerun",
"evaluated_on": datetime.now(timezone.utc).date().isoformat(),
"frozen_record": frozen,
"implementation_hash_matches_at_replay": True,
"dataset_hash_matches_at_replay": True,
"source_audit_status": manifest["source_audit_status"],
"validation_status": validation["status"],
"valid_public_aa_cases": validation["valid_public_aa_cases"],
"excluded_cases": invalid,
"trial_count": count,
"trials": trials,
**aggregate,
"opaque_exact_top_1_rate": sum(row["opaque_true_rank"] == 1 for row in trials) / count if count else None,
"opaque_exact_top_3_rate": sum(row["opaque_true_rank"] <= 3 for row in trials) / count if count else None,
"official_valid_independent_blind": False,
"official_blind_trial_count": 0,
"is_blind_evaluation": False,
"truth_hidden_from_ranker": True,
"results_previously_seen": True,
"verified_minute_claim_allowed": False,
"status": "blocked_independent_blind_evidence",
"boundary": "Three-event low-information protocol, not a mathematical accuracy lower bound and not representative of real sessions. Historical v3/v4 exposure cannot be undone by freezing today's scorer. No tuning or release claims.",
}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--freeze", action="store_true", help="Create a new record before replay; never overwrite an existing record")
parser.add_argument("--freeze-path", type=Path, default=FREEZE)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
if args.freeze:
with args.freeze_path.open("x", encoding="utf-8", newline="\n") as handle:
json.dump(freeze_record(), handle, ensure_ascii=False, indent=2)
handle.write("\n")
print(args.freeze_path.relative_to(ROOT).as_posix())
else:
print(json.dumps(run(args.freeze_path), ensure_ascii=False, indent=2))