feat: sync day level holdout readiness ledger

2026-07-19 23:30:46 +08:00
parent d30263f78b
commit 198c0763a6
5 changed files with 186 additions and 1 deletions
@@ -5,7 +5,7 @@
  "production_tuning_allowed": false,
  "boundary": "Index of current governance packets only. Raw oracle artifacts remain in references/oracle/artifacts and are not all duplicated here.",
  "summary": {
-    "packet_count": 66,
+    "packet_count": 67,
    "blocked_or_partial_count": 40,
    "human_review_required_count": 1
  },
@@ -537,6 +537,14 @@
      "claim_status": "partial",
      "consumer_policy": "research_to_commercial_contract_only",
      "claim_boundary": "Classifies D1-D60 sources for names/use notes only; generic-only Dn remains hidden until formula/source/oracle gates close."
+    },
+    {
+      "packet_id": "day_level_holdout_readiness_ledger",
+      "path": "references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json",
+      "domain": "timing_holdout",
+      "claim_status": "blocked",
+      "consumer_policy": "human_review_required",
+      "claim_boundary": "Reports frozen-label gap for day-level timing holdout; current pilot candidates are not independent labels and cannot unlock day/month claims."
    }
  ]
 }
@@ -0,0 +1,25 @@
+{
+  "blocked_reason": "Pilot candidates are not independent frozen labels. Public positive event sources cannot be used as negative intervals without human absence adjudication.",
+  "boundary": "No day/month timing claim can be promoted until the frozen positive and negative thresholds are met before blind replay.",
+  "claim_status": "blocked",
+  "created_at": "2026-07-19",
+  "current": {
+    "candidate_annotation_count": 9,
+    "frozen_final_count": 0,
+    "frozen_negative_count": 0,
+    "frozen_positive_count": 0
+  },
+  "production_tuning_allowed": false,
+  "remaining": {
+    "negative_needed": 80,
+    "positive_needed": 20
+  },
+  "required": {
+    "minimum_frozen_negative": 80,
+    "minimum_frozen_positive": 20
+  },
+  "scope": "day_level_holdout_readiness_ledger",
+  "source_packet": "references/real_case_calibration/day_level_holdout_v3_human_annotation_packet_2026_07_19.json",
+  "status": "awaiting_independent_labels",
+  "truth_matrix_allowed": false
+}
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Report day-level timing holdout readiness without promoting unlabeled data."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PACKET = ROOT / "references/real_case_calibration/day_level_holdout_v3_human_annotation_packet_2026_07_19.json"
+MIN_POSITIVE = 20
+MIN_NEGATIVE = 80
+
+
+def build() -> dict[str, Any]:
+    packet = json.loads(PACKET.read_text(encoding="utf-8"))
+    annotations = packet["annotations"]
+    frozen_final = [
+        row
+        for row in annotations
+        if row.get("final_label")
+        and row.get("independent_human_reviewed") is True
+        and row.get("frozen_before_scoring") is True
+    ]
+    positive = sum(1 for row in frozen_final if row["final_label"] == "target_event")
+    negative = sum(1 for row in frozen_final if row["final_label"] == "no_target_event")
+    return {
+        "scope": "day_level_holdout_readiness_ledger",
+        "created_at": "2026-07-19",
+        "status": "awaiting_independent_labels",
+        "claim_status": "blocked",
+        "production_tuning_allowed": False,
+        "truth_matrix_allowed": False,
+        "source_packet": str(PACKET.relative_to(ROOT)),
+        "current": {
+            "candidate_annotation_count": len(annotations),
+            "frozen_final_count": len(frozen_final),
+            "frozen_positive_count": positive,
+            "frozen_negative_count": negative,
+        },
+        "required": {
+            "minimum_frozen_positive": MIN_POSITIVE,
+            "minimum_frozen_negative": MIN_NEGATIVE,
+        },
+        "remaining": {
+            "positive_needed": max(0, MIN_POSITIVE - positive),
+            "negative_needed": max(0, MIN_NEGATIVE - negative),
+        },
+        "blocked_reason": (
+            "Pilot candidates are not independent frozen labels. Public positive "
+            "event sources cannot be used as negative intervals without human "
+            "absence adjudication."
+        ),
+        "boundary": (
+            "No day/month timing claim can be promoted until the frozen positive "
+            "and negative thresholds are met before blind replay."
+        ),
+    }
+
+
+def main() -> int:
+    print(json.dumps(build(), ensure_ascii=False, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from scripts.claim_audit_runtime_gate import evaluate_claim
+
+ROOT = Path(__file__).resolve().parents[1]
+INDEX = ROOT / "references/oracle/evidence_packet_index_2026_07_19.json"
+
+
+def test_claim_gate_blocks_verified_claims_for_blocked_domain() -> None:
+    result = evaluate_claim(INDEX, "timing_holdout", "verified_precise_prediction")
+    assert result["decision"] == "block"
+    assert result["allowed_claim_status"] == "exploratory_unvalidated"
+    assert result["production_tuning_allowed"] is False
+    assert set(result["blocking_packets"]) == {
+        "day_level_human_annotation_packet",
+        "day_level_holdout_readiness_ledger",
+    }
+
+
+def test_claim_gate_degrades_partial_domains() -> None:
+    result = evaluate_claim(INDEX, "shadbala_ashtakavarga", "complete_absolute_truth")
+    assert result["decision"] == "degrade"
+    assert result["allowed_claim_status"] == "partial_method_variant"
+    assert result["blocking_packets"] == ["xalen_shadbala_av_delta_report"]
+    assert "formulas/units/school variants remain open" in result["boundaries"][0]
+
+
+def test_claim_gate_allows_ready_contract_but_not_prediction_truth() -> None:
+    result = evaluate_claim(INDEX, "profile_schema", "ready_contract")
+    assert result["decision"] == "allow"
+    assert result["allowed_claim_status"] == "ready_contract"
+    assert result["blocking_packets"] == []
+
+
+def test_claim_gate_blocks_unknown_domain() -> None:
+    result = evaluate_claim(INDEX, "unknown_domain", "complete")
+    assert result["decision"] == "block"
+    assert result["allowed_claim_status"] == "blocked_unknown_domain"
@@ -0,0 +1,43 @@
+import json
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ARTIFACT = ROOT / "references/real_case_calibration/day_level_holdout_readiness_ledger_2026_07_19.json"
+INDEX = ROOT / "references/oracle/evidence_packet_index_2026_07_19.json"
+
+
+def test_holdout_readiness_script_keeps_unlabeled_pilot_blocked():
+    data = json.loads(
+        subprocess.check_output(
+            ["python3", "scripts/day_level_holdout_readiness_ledger.py"],
+            cwd=ROOT,
+            text=True,
+        )
+    )
+    assert data["scope"] == "day_level_holdout_readiness_ledger"
+    assert data["claim_status"] == "blocked"
+    assert data["production_tuning_allowed"] is False
+    assert data["truth_matrix_allowed"] is False
+    assert data["current"]["candidate_annotation_count"] == 9
+    assert data["current"]["frozen_final_count"] == 0
+
+
+def test_holdout_readiness_counts_real_remaining_frozen_labels():
+    data = json.loads(ARTIFACT.read_text(encoding="utf-8"))
+    assert data["remaining"] == {"positive_needed": 20, "negative_needed": 80}
+    assert data["required"] == {
+        "minimum_frozen_positive": 20,
+        "minimum_frozen_negative": 80,
+    }
+    assert "cannot be used as negative intervals" in data["blocked_reason"]
+    assert "No day/month timing claim" in data["boundary"]
+
+
+def test_evidence_index_registers_holdout_readiness():
+    packets = {
+        row["packet_id"]: row
+        for row in json.loads(INDEX.read_text(encoding="utf-8"))["packets"]
+    }
+    assert packets["day_level_holdout_readiness_ledger"]["claim_status"] == "blocked"