diff --git a/references/oracle/evidence_packet_index_2026_07_19.json b/references/oracle/evidence_packet_index_2026_07_19.json index f25851fd..e5c07b9f 100644 --- a/references/oracle/evidence_packet_index_2026_07_19.json +++ b/references/oracle/evidence_packet_index_2026_07_19.json @@ -5,7 +5,7 @@ "production_tuning_allowed": false, "boundary": "Index of current governance packets only. Raw oracle artifacts remain in references/oracle/artifacts and are not all duplicated here.", "summary": { - "packet_count": 66, + "packet_count": 67, "blocked_or_partial_count": 40, "human_review_required_count": 1 }, @@ -537,6 +537,14 @@ "claim_status": "partial", "consumer_policy": "research_to_commercial_contract_only", "claim_boundary": "Classifies D1-D60 sources for names/use notes only; generic-only Dn remains hidden until formula/source/oracle gates close." + }, + { + "packet_id": "day_level_holdout_readiness_ledger", + "path": "references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json", + "domain": "timing_holdout", + "claim_status": "blocked", + "consumer_policy": "human_review_required", + "claim_boundary": "Reports frozen-label gap for day-level timing holdout; current pilot candidates are not independent labels and cannot unlock day/month claims." } ] } diff --git a/references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json b/references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json new file mode 100644 index 00000000..69fbf969 --- /dev/null +++ b/references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json @@ -0,0 +1,25 @@ +{ + "blocked_reason": "Pilot candidates are not independent frozen labels. Public positive event sources cannot be used as negative intervals without human absence adjudication.", + "boundary": "No day/month timing claim can be promoted until the frozen positive and negative thresholds are met before blind replay.", + "claim_status": "blocked", + "created_at": "2026-07-19", + "current": { + "candidate_annotation_count": 9, + "frozen_final_count": 0, + "frozen_negative_count": 0, + "frozen_positive_count": 0 + }, + "production_tuning_allowed": false, + "remaining": { + "negative_needed": 80, + "positive_needed": 20 + }, + "required": { + "minimum_frozen_negative": 80, + "minimum_frozen_positive": 20 + }, + "scope": "day_level_holdout_readiness_ledger", + "source_packet": "references/real_case_calibration/day_level_holdout_v3_human_annotation_packet_2026_07_19.json", + "status": "awaiting_independent_labels", + "truth_matrix_allowed": false +} diff --git a/scripts/day_level_holdout_readiness_ledger.py b/scripts/day_level_holdout_readiness_ledger.py new file mode 100644 index 00000000..ac25a3bf --- /dev/null +++ b/scripts/day_level_holdout_readiness_ledger.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Report day-level timing holdout readiness without promoting unlabeled data.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +PACKET = ROOT / "references/real_case_calibration/day_level_holdout_v3_human_annotation_packet_2026_07_19.json" +MIN_POSITIVE = 20 +MIN_NEGATIVE = 80 + + +def build() -> dict[str, Any]: + packet = json.loads(PACKET.read_text(encoding="utf-8")) + annotations = packet["annotations"] + frozen_final = [ + row + for row in annotations + if row.get("final_label") + and row.get("independent_human_reviewed") is True + and row.get("frozen_before_scoring") is True + ] + positive = sum(1 for row in frozen_final if row["final_label"] == "target_event") + negative = sum(1 for row in frozen_final if row["final_label"] == "no_target_event") + return { + "scope": "day_level_holdout_readiness_ledger", + "created_at": "2026-07-19", + "status": "awaiting_independent_labels", + "claim_status": "blocked", + "production_tuning_allowed": False, + "truth_matrix_allowed": False, + "source_packet": str(PACKET.relative_to(ROOT)), + "current": { + "candidate_annotation_count": len(annotations), + "frozen_final_count": len(frozen_final), + "frozen_positive_count": positive, + "frozen_negative_count": negative, + }, + "required": { + "minimum_frozen_positive": MIN_POSITIVE, + "minimum_frozen_negative": MIN_NEGATIVE, + }, + "remaining": { + "positive_needed": max(0, MIN_POSITIVE - positive), + "negative_needed": max(0, MIN_NEGATIVE - negative), + }, + "blocked_reason": ( + "Pilot candidates are not independent frozen labels. Public positive " + "event sources cannot be used as negative intervals without human " + "absence adjudication." + ), + "boundary": ( + "No day/month timing claim can be promoted until the frozen positive " + "and negative thresholds are met before blind replay." + ), + } + + +def main() -> int: + print(json.dumps(build(), ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_claim_audit_runtime_gate.py b/tests/test_claim_audit_runtime_gate.py new file mode 100644 index 00000000..a5f21f41 --- /dev/null +++ b/tests/test_claim_audit_runtime_gate.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from pathlib import Path + +from scripts.claim_audit_runtime_gate import evaluate_claim + +ROOT = Path(__file__).resolve().parents[1] +INDEX = ROOT / "references/oracle/evidence_packet_index_2026_07_19.json" + + +def test_claim_gate_blocks_verified_claims_for_blocked_domain() -> None: + result = evaluate_claim(INDEX, "timing_holdout", "verified_precise_prediction") + assert result["decision"] == "block" + assert result["allowed_claim_status"] == "exploratory_unvalidated" + assert result["production_tuning_allowed"] is False + assert set(result["blocking_packets"]) == { + "day_level_human_annotation_packet", + "day_level_holdout_readiness_ledger", + } + + +def test_claim_gate_degrades_partial_domains() -> None: + result = evaluate_claim(INDEX, "shadbala_ashtakavarga", "complete_absolute_truth") + assert result["decision"] == "degrade" + assert result["allowed_claim_status"] == "partial_method_variant" + assert result["blocking_packets"] == ["xalen_shadbala_av_delta_report"] + assert "formulas/units/school variants remain open" in result["boundaries"][0] + + +def test_claim_gate_allows_ready_contract_but_not_prediction_truth() -> None: + result = evaluate_claim(INDEX, "profile_schema", "ready_contract") + assert result["decision"] == "allow" + assert result["allowed_claim_status"] == "ready_contract" + assert result["blocking_packets"] == [] + + +def test_claim_gate_blocks_unknown_domain() -> None: + result = evaluate_claim(INDEX, "unknown_domain", "complete") + assert result["decision"] == "block" + assert result["allowed_claim_status"] == "blocked_unknown_domain" diff --git a/tests/test_day_level_holdout_readiness_ledger.py b/tests/test_day_level_holdout_readiness_ledger.py new file mode 100644 index 00000000..0ca585d8 --- /dev/null +++ b/tests/test_day_level_holdout_readiness_ledger.py @@ -0,0 +1,43 @@ +import json +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +ARTIFACT = ROOT / "references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json" +INDEX = ROOT / "references/oracle/evidence_packet_index_2026_07_19.json" + + +def test_holdout_readiness_script_keeps_unlabeled_pilot_blocked(): + data = json.loads( + subprocess.check_output( + ["python3", "scripts/day_level_holdout_readiness_ledger.py"], + cwd=ROOT, + text=True, + ) + ) + assert data["scope"] == "day_level_holdout_readiness_ledger" + assert data["claim_status"] == "blocked" + assert data["production_tuning_allowed"] is False + assert data["truth_matrix_allowed"] is False + assert data["current"]["candidate_annotation_count"] == 9 + assert data["current"]["frozen_final_count"] == 0 + + +def test_holdout_readiness_counts_real_remaining_frozen_labels(): + data = json.loads(ARTIFACT.read_text(encoding="utf-8")) + assert data["remaining"] == {"positive_needed": 20, "negative_needed": 80} + assert data["required"] == { + "minimum_frozen_positive": 20, + "minimum_frozen_negative": 80, + } + assert "cannot be used as negative intervals" in data["blocked_reason"] + assert "No day/month timing claim" in data["boundary"] + + +def test_evidence_index_registers_holdout_readiness(): + packets = { + row["packet_id"]: row + for row in json.loads(INDEX.read_text(encoding="utf-8"))["packets"] + } + assert packets["day_level_holdout_readiness_ledger"]["claim_status"] == "blocked"