jesse-uxandClaude Code
932f2fffba
Preserve closed confirmation gates and previously-exposed dataset boundaries. Add auditable 900-trial sensitivity results, current scorer freshness checks, and the v5 collection protocol. Co-Authored-By: Claude Code <noreply@anthropic.com>
82 lines
4.0 KiB
Python
82 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from scripts.minute_rectification_blind_eval import implementation_sha256, summarize_trials
|
|
from scripts.rectification.sealed_holdout import holdout_passed, load_sealed_minute_holdout
|
|
from scripts.research.sealed_holdout_rerun import DATASET, FREEZE, REPORT, file_sha256, freeze_record, run
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def read(path):
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def test_contract_tracks_actual_current_scorer_and_dataset_audit():
|
|
dataset = read(DATASET)
|
|
contract = read(ROOT / "references/rectification_sealed_holdout.v1.json")
|
|
actual_hash = implementation_sha256(dataset["frozen_scoring"]["files"])
|
|
assert contract["current_tree_scorer"]["implementation_sha256"] == actual_hash
|
|
assert contract["source_audit_status"] == dataset["source_audit_status"]
|
|
assert contract["evaluated_on"] == read(REPORT)["evaluated_on"]
|
|
assert contract["status"] == "not_ready"
|
|
assert contract["confirmation_coverage_rate"] == 0.0
|
|
assert holdout_passed(load_sealed_minute_holdout()) is False
|
|
|
|
|
|
def test_frozen_record_matches_dataset_scorer_and_evaluator_bytes():
|
|
frozen = read(FREEZE)
|
|
actual = freeze_record()
|
|
assert {key: value for key, value in frozen.items() if key != "frozen_at_utc"} == {
|
|
key: value for key, value in actual.items() if key != "frozen_at_utc"
|
|
}
|
|
assert frozen["dataset_sha256"] == file_sha256(DATASET)
|
|
assert len(frozen["files"]) == 12
|
|
assert read(DATASET)["frozen_scoring"]["implementation_sha256"] == frozen["historical_frozen_sha256"]
|
|
|
|
|
|
def test_fixed_protocol_rerun_is_auditable_but_never_independent_blind():
|
|
report = read(REPORT)
|
|
contract = read(ROOT / "references/rectification_sealed_holdout.v1.json")
|
|
scorer = contract["current_tree_scorer"]
|
|
rerun = contract["current_tree_fixed_protocol_rerun"]
|
|
assert report["frozen_record"] == read(FREEZE)
|
|
assert report["trial_count"] == len(report["trials"]) == 20
|
|
assert report["excluded_cases"] == []
|
|
aggregate = summarize_trials(report["trials"], read(DATASET)["release_metrics"])
|
|
assert report["metrics"] == aggregate["metrics"] == scorer["metrics"]
|
|
assert report["metric_gates_passed"] == aggregate["metric_gates_passed"]
|
|
assert scorer["source_report"] == REPORT.relative_to(ROOT).as_posix()
|
|
assert scorer["implementation_sha256"] == report["frozen_record"]["implementation_sha256"]
|
|
assert scorer["fixed_protocol_rerun_trial_count"] == report["trial_count"]
|
|
assert scorer["fixed_protocol_rerun_hash_matches"] is True
|
|
assert scorer["official_eval_trial_count"] == report["official_blind_trial_count"] == 0
|
|
assert scorer["official_eval_implementation_hash_matches"] is False
|
|
for item in (rerun, report):
|
|
assert item["official_valid_independent_blind"] is False
|
|
assert item["is_blind_evaluation"] is False
|
|
assert item["truth_hidden_from_ranker"] is True
|
|
assert item["results_previously_seen"] is True
|
|
assert item["verified_minute_claim_allowed"] is False
|
|
assert rerun["must_not_claim_as_release_metrics"] is True
|
|
assert all(row["event_count"] == 3 for row in report["trials"])
|
|
# Never persist actual candidate/truth times or coordinates in report artifacts.
|
|
for row in report["trials"]:
|
|
assert not ({"predicted_time", "published_truth_revealed_after_ranking", "latitude", "longitude"} & row.keys())
|
|
|
|
|
|
def test_changed_frozen_identity_fails_before_any_replay(tmp_path, monkeypatch):
|
|
frozen = read(FREEZE)
|
|
frozen["implementation_sha256"] = "0" * 64
|
|
path = tmp_path / "bad-freeze.json"
|
|
path.write_text(json.dumps(frozen), encoding="utf-8")
|
|
def unexpected(*args, **kwargs):
|
|
raise AssertionError("must reject identity before scoring")
|
|
monkeypatch.setattr("scripts.research.sealed_holdout_rerun.build_feature_fact_rows", unexpected)
|
|
with pytest.raises(ValueError, match="frozen_record_mismatch:implementation_sha256"):
|
|
run(path)
|