Jesse_ChenandClaude Opus 5.5
79f6ac5297
Sync8 changed ashtakavarga.py, dasha_analyzer.py and narayana_dasha.py, which are in the frozen scoring identity. Sealed (20) and reported-offset (900) replays match the scoring_perf_2026_10_13 records trial for trial, so ERR-110 stores a new record. Prior records are kept byte-for-byte. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HwMNZEzdzmrbPhuRNmZe3C
245 lines
12 KiB
Python
245 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Freeze and replay the exposed v3 corpus; never claim a fresh blind holdout."""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import sys
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from scripts.active_rectification_event_engine import AYANAMSA, NODE_MODE
|
||
from scripts.minute_rectification_blind_eval import (
|
||
_candidate_moments, _clock_distance, _opaque_winner, _request,
|
||
implementation_sha256, summarize_trials,
|
||
)
|
||
from scripts.minute_rectification_fact_blind_eval_v4 import _would_confirm
|
||
from scripts.minute_rectification_fact_ranker_v4 import (
|
||
ALGORITHM_VERSION, rank_fact_rows, score_fact_ranker_v4,
|
||
)
|
||
from scripts.minute_rectification_feature_facts_v4 import build_feature_fact_rows
|
||
from scripts.minute_rectification_holdout_validator import validate
|
||
|
||
DATASET = ROOT / "references/real_case_calibration/minute_rectification_holdout_v3.json"
|
||
# 原值: docs/research/sealed_holdout_rerun_scoring_perf_2026_10_13.{freeze.json,json}
|
||
# 新值: docs/research/sealed_holdout_rerun_upstream_sync8_2026_10_18.{freeze.json,json}
|
||
# 原因: sync8（BUG-1380～1389）改了 ashtakavarga / dasha_analyzer / narayana_dasha，输出不变，ERR-110 另存新记录
|
||
FREEZE = ROOT / "docs/research/sealed_holdout_rerun_upstream_sync8_2026_10_18.freeze.json"
|
||
REPORT = ROOT / "docs/research/sealed_holdout_rerun_upstream_sync8_2026_10_18.json"
|
||
LEGACY_REPORT = ROOT / "docs/research/sealed_holdout_rerun_2026_09_20.json"
|
||
ARCHIVE = ROOT / "docs/research/history/rectification_pre_cross_midnight_2026_09_20"
|
||
PRODUCTION_FILES = [
|
||
"scripts/rectification/scoring_service.py",
|
||
"scripts/rectification/dasha_transition_proximity.py",
|
||
"scripts/rectification/candidate_contrast.py",
|
||
"scripts/rectification/case_holdout.py",
|
||
"scripts/rectification/contracts.py",
|
||
"scripts/rectification/event_probes.py",
|
||
"scripts/rectification/candidate_window.py",
|
||
"scripts/rectification/decision_policy.py",
|
||
"scripts/rectification/refinement_packet.py",
|
||
"scripts/rectification/api_service.py",
|
||
# BUG-1181: functional roles feed *_functional_*_auxiliary scoring rules;
|
||
# 57782aea changed scores without tripping this identity.
|
||
"scripts/functional_benefics.py",
|
||
]
|
||
RESEARCH_FILES = [
|
||
"scripts/research/reported_offset_sweep.py",
|
||
"scripts/research/cluster_width_lib.py",
|
||
"scripts/research/probe_supply_after_six.py",
|
||
"scripts/research/sealed_holdout_rerun.py",
|
||
"scripts/minute_rectification_blind_eval.py",
|
||
"scripts/minute_rectification_fact_blind_eval_v4.py",
|
||
"scripts/minute_rectification_holdout_validator.py",
|
||
]
|
||
|
||
|
||
def file_sha256(path: Path) -> str:
|
||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
||
|
||
def opaque_order(benchmark_id: str, case_id: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
"""Extend the existing opaque winner to a total, truth-independent ranking."""
|
||
return sorted(rows, key=lambda row: (
|
||
-row["score"],
|
||
hashlib.sha256(f"{benchmark_id}:{case_id}:{row['time']}".encode()).hexdigest(),
|
||
))
|
||
|
||
|
||
def implementation_identity(dataset: Path = DATASET) -> dict[str, Any]:
|
||
"""Bind legacy scorer, changed production files and both replay adapters.
|
||
|
||
The production identity is contextual for the shadow rerun, not a claim
|
||
that the shadow scorer executes transition proximity. These are explicit
|
||
audited file sets, not a transitive dependency or environment lock.
|
||
"""
|
||
legacy_files = json.loads(dataset.read_text(encoding="utf-8"))["frozen_scoring"]["files"]
|
||
production_files = sorted(set(legacy_files + PRODUCTION_FILES))
|
||
return {
|
||
"historical_artifacts_manifest_sha256": file_sha256(ARCHIVE / "manifest.json"),
|
||
"production_scoring_files": production_files,
|
||
"production_scoring_sha256": implementation_sha256(production_files),
|
||
"research_files": RESEARCH_FILES,
|
||
"research_implementation_sha256": implementation_sha256(RESEARCH_FILES),
|
||
"file_sha256": {path: file_sha256(ROOT / path) for path in sorted(set(production_files + RESEARCH_FILES))},
|
||
"hash_scope": "explicit_identity_file_sets_not_a_transitive_dependency_lock",
|
||
}
|
||
|
||
|
||
def verify_frozen_record(frozen: dict[str, Any], actual: dict[str, Any]) -> None:
|
||
for key in actual:
|
||
if key != "frozen_at_utc" and actual[key] != frozen.get(key):
|
||
raise ValueError(f"frozen_record_mismatch:{key}")
|
||
if set(frozen) != set(actual):
|
||
raise ValueError("frozen_record_mismatch:keys")
|
||
|
||
|
||
def historical_comparison(report_path: Path, trials: list[dict[str, Any]], keys: tuple[str, ...]) -> dict[str, Any]:
|
||
"""Keep every before/after row, including unchanged rows and failures."""
|
||
history_path = ARCHIVE / report_path.relative_to(ROOT)
|
||
archive_manifest = json.loads((ARCHIVE / "manifest.json").read_text(encoding="utf-8"))
|
||
entry = next(item for item in archive_manifest["files"] if item["source_path"] == report_path.relative_to(ROOT).as_posix())
|
||
if file_sha256(history_path) != entry["sha256"]:
|
||
raise ValueError("historical_report_bytes_changed")
|
||
previous = json.loads(history_path.read_text(encoding="utf-8"))
|
||
identity = lambda row: tuple(row[key] for key in keys)
|
||
old = {identity(row): row for row in previous["trials"]}
|
||
new = {identity(row): row for row in trials}
|
||
if len(old) != len(previous["trials"]) or len(new) != len(trials) or old.keys() != new.keys():
|
||
raise ValueError("historical_comparison_trial_identity_mismatch")
|
||
comparisons = [{
|
||
**{key: row[key] for key in keys},
|
||
"before": old[identity(row)], "after": row,
|
||
"changed_fields": sorted(key for key in set(row) | set(old[identity(row)])
|
||
if row.get(key) != old[identity(row)].get(key)),
|
||
} for row in trials]
|
||
return {
|
||
"source_report": history_path.relative_to(ROOT).as_posix(),
|
||
"source_report_sha256": file_sha256(history_path),
|
||
"reason": "BUG-981 new production identity and native single-matrix sweep; prior grouped sweep/shadow metrics not presumed incorrect",
|
||
"trial_count": len(comparisons),
|
||
"changed_trial_count": sum(bool(row["changed_fields"]) for row in comparisons),
|
||
"trials": comparisons,
|
||
}
|
||
|
||
|
||
def freeze_record(dataset: Path = DATASET) -> dict[str, Any]:
|
||
manifest = json.loads(dataset.read_text(encoding="utf-8"))
|
||
files = manifest["frozen_scoring"]["files"]
|
||
return {
|
||
"record_version": "exposed-v3-fixed-protocol-cross-midnight-rerun-v2",
|
||
"extended_identity": implementation_identity(dataset),
|
||
"production_identity_scope": "context_only_shadow_scorer_does_not_call_transition_proximity",
|
||
"frozen_at_utc": datetime.now(timezone.utc).isoformat(),
|
||
"dataset_path": dataset.relative_to(ROOT).as_posix(),
|
||
"dataset_sha256": file_sha256(dataset),
|
||
"algorithm_version": ALGORITHM_VERSION,
|
||
"implementation_sha256": implementation_sha256(files),
|
||
"files": files,
|
||
"historical_frozen_sha256": manifest["frozen_scoring"]["implementation_sha256"],
|
||
"evaluator_sha256": file_sha256(Path(__file__)),
|
||
"ayanamsa": AYANAMSA,
|
||
"node_mode": NODE_MODE,
|
||
"candidate_radius_minutes": sorted({case["candidate_radius_minutes"] for case in manifest["cases"]}),
|
||
"minute_step": 1,
|
||
"release_metrics": manifest["release_metrics"],
|
||
"results_previously_seen": True,
|
||
"official_valid_independent_blind": False,
|
||
"must_not_use_for_tuning": True,
|
||
"tie_breaker": manifest["frozen_scoring"]["tie_breaker"],
|
||
"metric_rank_definition": "competition_rank_1_plus_strictly_higher_scores_legacy_protocol",
|
||
"extra_metric_rank_definition": "score_desc_then_existing_opaque_sha256_total_order",
|
||
}
|
||
|
||
|
||
def run(freeze_path: Path = FREEZE, dataset: Path = DATASET) -> dict[str, Any]:
|
||
frozen = json.loads(freeze_path.read_text(encoding="utf-8"))
|
||
actual = freeze_record(dataset)
|
||
verify_frozen_record(frozen, actual)
|
||
replay_started_at = datetime.now(timezone.utc).isoformat()
|
||
manifest = json.loads(dataset.read_text(encoding="utf-8"))
|
||
validation = validate(dataset)
|
||
invalid = validation["invalid_cases"]
|
||
trials = []
|
||
for index, case in enumerate(manifest["cases"], 1):
|
||
if case["case_id"] in invalid:
|
||
continue
|
||
request = _request(case, case["events"])
|
||
candidates = _candidate_moments(case)
|
||
facts = build_feature_fact_rows(request, candidates=candidates)
|
||
rows, _ = rank_fact_rows(facts, request["events"])
|
||
result = score_fact_ranker_v4(facts, request["events"])
|
||
predicted = _opaque_winner(manifest["benchmark_id"], case["case_id"], rows)
|
||
ordered = opaque_order(manifest["benchmark_id"], case["case_id"], rows)
|
||
sparse_request = _request(case, case["events"][:1])
|
||
sparse_facts = build_feature_fact_rows(sparse_request, candidates=candidates)
|
||
sparse_result = score_fact_ranker_v4(sparse_facts, sparse_request["events"])
|
||
# Truth is revealed only after both full and sparse ranking/decisions.
|
||
truth = case["birth"]["time"]
|
||
truth_score = next(row["score"] for row in rows if row["time"] == truth)
|
||
would_confirm = _would_confirm(result)
|
||
trials.append({
|
||
"case_ordinal": index,
|
||
"candidate_count": len(rows),
|
||
"event_count": len(request["events"]),
|
||
"true_rank": 1 + sum(row["score"] > truth_score for row in rows),
|
||
"opaque_true_rank": next(i for i, row in enumerate(ordered, 1) if row["time"] == truth),
|
||
"minute_error": _clock_distance(predicted, truth),
|
||
"would_confirm": would_confirm,
|
||
"false_confirmation": would_confirm and predicted != truth,
|
||
"insufficient_evidence_rejected": not _would_confirm(sparse_result),
|
||
"full_trial_reasons": result["reasons"],
|
||
"sparse_trial_reasons": sparse_result["reasons"],
|
||
})
|
||
verify_frozen_record(frozen, freeze_record(dataset))
|
||
aggregate = summarize_trials(trials, manifest["release_metrics"])
|
||
count = len(trials)
|
||
return {
|
||
"scope": "fixed_protocol_previously_exposed_v3_rerun",
|
||
"replay_started_at_utc": replay_started_at,
|
||
"replay_finished_at_utc": datetime.now(timezone.utc).isoformat(),
|
||
"historical_comparison": historical_comparison(LEGACY_REPORT, trials, ("case_ordinal",)),
|
||
"evaluated_on": datetime.now(timezone.utc).date().isoformat(),
|
||
"frozen_record": frozen,
|
||
"implementation_hash_matches_at_replay": True,
|
||
"dataset_hash_matches_at_replay": True,
|
||
"source_audit_status": manifest["source_audit_status"],
|
||
"validation_status": validation["status"],
|
||
"valid_public_aa_cases": validation["valid_public_aa_cases"],
|
||
"excluded_cases": invalid,
|
||
"trial_count": count,
|
||
"trials": trials,
|
||
**aggregate,
|
||
"opaque_exact_top_1_rate": sum(row["opaque_true_rank"] == 1 for row in trials) / count if count else None,
|
||
"opaque_exact_top_3_rate": sum(row["opaque_true_rank"] <= 3 for row in trials) / count if count else None,
|
||
"official_valid_independent_blind": False,
|
||
"official_blind_trial_count": 0,
|
||
"is_blind_evaluation": False,
|
||
"truth_hidden_from_ranker": True,
|
||
"results_previously_seen": True,
|
||
"verified_minute_claim_allowed": False,
|
||
"status": "blocked_independent_blind_evidence",
|
||
"boundary": "Three-event low-information protocol, not a mathematical accuracy lower bound and not representative of real sessions. Historical v3/v4 exposure cannot be undone by freezing today's scorer. No tuning or release claims.",
|
||
}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--freeze", action="store_true", help="Create a new record before replay; never overwrite an existing record")
|
||
parser.add_argument("--freeze-path", type=Path, default=FREEZE)
|
||
parser.add_argument("--json", action="store_true")
|
||
args = parser.parse_args()
|
||
if args.freeze:
|
||
with args.freeze_path.open("x", encoding="utf-8", newline="\n") as handle:
|
||
json.dump(freeze_record(), handle, ensure_ascii=False, indent=2)
|
||
handle.write("\n")
|
||
print(args.freeze_path.relative_to(ROOT).as_posix())
|
||
else:
|
||
print(json.dumps(run(args.freeze_path), ensure_ascii=False, indent=2))
|