feat: sync blind timing holdout gate

docs/research/day_level_holdout_labeling_guide_zh.md

@@ -0,0 +1,76 @@
+# 日级应期 holdout 人工标签指南
+
+目标：给研究仓提供真实、独立、可复验的正/负样本。你不需要会编程，只需要提供来源清楚的事实材料。
+
+## 需要收集什么
+
+每条标签只回答一个问题：
+
+某人在某个日期区间，某类事件是否发生？
+
+可用标签：
+
+- `target_event`：事件发生了。
+- `no_target_event`：有公开资料支持该区间没有发生这个目标事件。
+
+## 优先领域
+
+1. 事业：任命、创办公司、上市、获奖、重大作品发布。
+2. 婚恋：结婚、离婚、订婚、公开伴侣关系变化。
+3. 财富：上市、重大融资、破产、重大资产事件。
+
+暂不优先健康/死亡，噪音和伦理风险高。
+
+## 合格来源
+
+优先：
+
+- 官方 biography / timeline
+- Britannica / Nobel / company official timeline
+- 出版传记中可核对页码或章节的时间线
+- IMDb / MusicBrainz / company history 等结构化公开资料
+
+不合格：
+
+- “没搜到新闻所以没发生”
+- ChatGPT 生成内容
+- 无来源论坛故事
+- 已被本项目观察过的旧控制日期
+- 模糊说法：“那一年很平静”
+
+## 最小可用规模
+
+pilot 阶段：
+
+- 3 个公开人物
+- 每人 1 个领域
+- 每人 1 个正样本窗口
+- 每人 2 个负样本窗口
+
+正式升级门槛：
+
+- 至少 20 个独立案例
+- 至少 80 个独立负样本区间
+- 标签冻结后才允许评分
+
+## 填写方式
+
+生成空模板：
+
+```bash
+python3 scripts/day_level_holdout_template.py --output /tmp/holdout_annotation_template.json
+```
+
+把公开来源、日期区间、事件说明填进去，再交给 intake：
+
+```bash
+python3 scripts/day_level_negative_holdout_intake.py references/real_case_calibration/day_level_holdout_v3_preregistration.json --row-json '{"case_id":"..."}'
+```
+
+## 结论边界
+
+没有真实独立负样本前：
+
+- 可以输出候选日期排序；
+- 可以说明触发信号；
+- 不能说“精确日期预测已验证”。

scripts/day_level_holdout_template.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Create a blank independent day-level timing holdout annotation template."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+TEMPLATE = {
+    "case_id": "",
+    "subject": {
+        "name": "",
+        "public_profile_url": "",
+        "birth_time_rating": "AA/A only preferred",
+    },
+    "domain": "career|marriage|wealth|health",
+    "label": "target_event|no_target_event",
+    "start": "YYYY-MM-DD",
+    "end": "YYYY-MM-DD",
+    "event_description": "",
+    "event_absent_assertion": "",
+    "source_url": "https://",
+    "source_quote_or_summary": "",
+    "adjudicator": "",
+    "time_uncertainty_days": 0,
+    "independent_human_reviewed": True,
+    "frozen_before_scoring": True,
+    "source_path": "",
+    "notes": "",
+}
+
+
+def build_template() -> dict:
+    return {
+        "template_type": "day_level_holdout_annotation_v3",
+        "instructions": [
+            "Use target_event for known dated events.",
+            "Use no_target_event only when a public source supports that the target event did not occur in the interval.",
+            "Do not use old control dates or rows observed before preregistration for tuning.",
+            "Freeze labels before running timing_ranker_blind_eval.py.",
+        ],
+        "annotation": TEMPLATE,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+    text = json.dumps(build_template(), ensure_ascii=False, indent=2) + "\n"
+    if args.output:
+        args.output.write_text(text, encoding="utf-8")
+    else:
+        print(text, end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/day_level_holdout_validator.py

@@ -11,11 +11,14 @@ REQUIRED={"case_id","domain","label","start","end","source_url","adjudicator","t
 def validate(path: Path) -> dict:
     data=json.loads(path.read_text(encoding="utf-8")); rows=data.get("annotations") or []; errors=[]
     mode=data.get("validation_mode", "independent")
+    prohibited=set(data.get("prohibited_tuning_data") or [])
     allowed_labels={"target_event", "no_target_event"} if mode == "independent" else {"target_event", "observational_non_target_date"}
     for i,row in enumerate(rows):
         for key in sorted(REQUIRED-set(row)): errors.append({"row":i,"field":key,"error":"missing"})
         if row.get("label") not in allowed_labels: errors.append({"row":i,"field":"label","error":"invalid"})
         if not str(row.get("source_url") or "").startswith(("https://","http://")): errors.append({"row":i,"field":"source_url","error":"not_public_url"})
+        if mode == "independent" and row.get("independent_human_reviewed") is not True: errors.append({"row":i,"field":"independent_human_reviewed","error":"not_independently_human_reviewed"})
+        if row.get("source_path") in prohibited: errors.append({"row":i,"field":"source_path","error":"prohibited_tuning_source"})
     positives=sum(r.get("label")=="target_event" for r in rows); negatives=sum(r.get("label") in {"no_target_event", "observational_non_target_date"} for r in rows)
     gate=data.get("frozen_gate") or {}; ready=not errors and positives>=gate.get("minimum_independent_cases",20) and negatives>=gate.get("minimum_independent_negative_intervals",80)
     status=("observational_ready_not_independent" if ready else "awaiting_observational_labels") if mode == "observational" else ("ready_for_blind_replay" if ready else "awaiting_independent_labels")

scripts/timing_ranker_blind_eval.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Evaluate frozen day/month timing candidates against independent holdout labels."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from scripts.day_level_holdout_validator import validate
+
+
+def _load(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _key(row: dict) -> tuple[str, str, str]:
+    return (str(row.get("case_id") or ""), str(row.get("start") or ""), str(row.get("end") or ""))
+
+
+def evaluate(manifest_path: Path, candidates_path: Path) -> dict:
+    validation = validate(manifest_path)
+    manifest = _load(manifest_path)
+    candidates = sorted(
+        (_load(candidates_path).get("candidate_windows") or []),
+        key=lambda row: float(row.get("score") or 0),
+        reverse=True,
+    )
+    labels = {_key(row): row.get("label") for row in manifest.get("annotations") or []}
+    ranked = [{**row, "rank": index + 1, "label": labels.get(_key(row))} for index, row in enumerate(candidates)]
+    positives = [row for row in ranked if row.get("label") == "target_event"]
+    negatives = [row for row in ranked if row.get("label") == "no_target_event"]
+    top_3_positive = sum(1 for row in positives if row["rank"] <= 3)
+    positive_top_3_rate = top_3_positive / len(positives) if positives else 0.0
+    min_positive_score = min((float(row.get("score") or 0) for row in positives), default=0.0)
+    false_positive_negatives = sum(1 for row in negatives if float(row.get("score") or 0) >= min_positive_score)
+    specificity = 1 - (false_positive_negatives / len(negatives)) if negatives else 0.0
+    gate = manifest.get("frozen_gate") or {}
+    blockers = []
+    if validation["status"] != "ready_for_blind_replay":
+        blockers.append("holdout_not_ready")
+    if positive_top_3_rate < gate.get("minimum_positive_top_3_rate", 1):
+        blockers.append("positive_top_3_rate_below_gate")
+    if specificity < gate.get("minimum_specificity", 1):
+        blockers.append("specificity_below_gate")
+    passed = not blockers
+    return {
+        "scope": "timing_ranker_blind_eval",
+        "status": "pass" if passed else "blocked",
+        "claim_status": "calibrated_day_level" if passed else "exploratory_unvalidated",
+        "production_tuning_allowed": bool(passed),
+        "positive_count": len(positives),
+        "negative_count": len(negatives),
+        "positive_top_3_rate": positive_top_3_rate,
+        "specificity": specificity,
+        "blockers": blockers,
+        "validation": validation,
+        "ranked_windows": ranked,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("manifest", type=Path)
+    parser.add_argument("candidates", type=Path)
+    args = parser.parse_args()
+    print(json.dumps(evaluate(args.manifest, args.candidates), ensure_ascii=False, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_day_level_holdout_template.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from scripts.day_level_holdout_template import build_template
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_holdout_template_is_nontechnical_and_validator_compatible_shape() -> None:
+    template = build_template()
+    annotation = template["annotation"]
+
+    for token in [
+        "case_id",
+        "subject",
+        "domain",
+        "label",
+        "start",
+        "end",
+        "event_absent_assertion",
+        "source_url",
+        "adjudicator",
+        "independent_human_reviewed",
+        "frozen_before_scoring",
+    ]:
+        assert token in annotation
+
+    assert "Do not use old control dates" in " ".join(template["instructions"])
+    assert annotation["source_url"] == "https://"
+    assert annotation["independent_human_reviewed"] is True
+
+
+def test_holdout_template_cli_writes_json(tmp_path: Path) -> None:
+    output = tmp_path / "template.json"
+
+    subprocess.run(
+        [sys.executable, str(ROOT / "scripts" / "day_level_holdout_template.py"), "--output", str(output)],
+        check=True,
+        text=True,
+    )
+
+    data = json.loads(output.read_text(encoding="utf-8"))
+    assert data["template_type"] == "day_level_holdout_annotation_v3"
+    assert data["annotation"]["label"] == "target_event|no_target_event"

tests/test_timing_ranker_blind_eval.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from scripts.day_level_holdout_validator import validate
+from scripts.day_level_negative_holdout_intake import append_annotation
+from scripts.timing_ranker_blind_eval import evaluate
+
+
+def test_holdout_validator_rejects_non_independent_and_prohibited_rows(tmp_path: Path) -> None:
+    manifest = {
+        "prohibited_tuning_data": ["old_controls.json"],
+        "frozen_gate": {"minimum_independent_cases": 1, "minimum_independent_negative_intervals": 1},
+        "annotations": [
+            {
+                "case_id": "case-1",
+                "domain": "career",
+                "label": "no_target_event",
+                "start": "2020-01-01",
+                "end": "2020-01-07",
+                "source_url": "https://example.org/timeline",
+                "adjudicator": "same_person",
+                "time_uncertainty_days": 0,
+                "independent_human_reviewed": False,
+                "source_path": "old_controls.json",
+            }
+        ],
+    }
+    path = tmp_path / "holdout.json"
+    path.write_text(json.dumps(manifest), encoding="utf-8")
+
+    report = validate(path)
+
+    assert report["status"] == "awaiting_independent_labels"
+    assert report["production_tuning_allowed"] is False
+    assert {error["error"] for error in report["errors"]} >= {
+        "not_independently_human_reviewed",
+        "prohibited_tuning_source",
+    }
+
+
+def test_blind_eval_requires_positive_windows_to_rank_above_negative_windows(tmp_path: Path) -> None:
+    manifest = {
+        "frozen_gate": {
+            "minimum_positive_top_3_rate": 0.6,
+            "minimum_specificity": 0.6,
+            "minimum_independent_cases": 1,
+            "minimum_independent_negative_intervals": 1,
+        },
+        "annotations": [
+            {
+                "case_id": "case-1",
+                "domain": "career",
+                "label": "target_event",
+                "start": "2020-01-01",
+                "end": "2020-01-07",
+                "source_url": "https://example.org/event",
+                "adjudicator": "reviewer-a",
+                "time_uncertainty_days": 0,
+                "independent_human_reviewed": True,
+            },
+            {
+                "case_id": "case-1-neg",
+                "domain": "career",
+                "label": "no_target_event",
+                "start": "2020-02-01",
+                "end": "2020-02-07",
+                "source_url": "https://example.org/non-event",
+                "adjudicator": "reviewer-b",
+                "time_uncertainty_days": 0,
+                "independent_human_reviewed": True,
+            },
+        ],
+    }
+    candidates = {
+        "candidate_windows": [
+            {"case_id": "case-1", "start": "2020-01-01", "end": "2020-01-07", "score": 0.90},
+            {"case_id": "case-1-neg", "start": "2020-02-01", "end": "2020-02-07", "score": 0.20},
+        ]
+    }
+    manifest_path = tmp_path / "holdout.json"
+    candidates_path = tmp_path / "candidates.json"
+    manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+    candidates_path.write_text(json.dumps(candidates), encoding="utf-8")
+
+    report = evaluate(manifest_path, candidates_path)
+
+    assert report["status"] == "pass"
+    assert report["claim_status"] == "calibrated_day_level"
+    assert report["production_tuning_allowed"] is True
+    assert report["positive_top_3_rate"] == 1.0
+    assert report["specificity"] == 1.0
+
+
+def test_blind_eval_blocks_when_negative_scores_outrank_positive_scores(tmp_path: Path) -> None:
+    manifest = {
+        "frozen_gate": {
+            "minimum_positive_top_3_rate": 0.6,
+            "minimum_specificity": 0.6,
+            "minimum_independent_cases": 1,
+            "minimum_independent_negative_intervals": 1,
+        },
+        "annotations": [
+            {
+                "case_id": "positive",
+                "domain": "marriage",
+                "label": "target_event",
+                "start": "2020-01-01",
+                "end": "2020-01-07",
+                "source_url": "https://example.org/event",
+                "adjudicator": "reviewer-a",
+                "time_uncertainty_days": 0,
+                "independent_human_reviewed": True,
+            },
+            {
+                "case_id": "negative",
+                "domain": "marriage",
+                "label": "no_target_event",
+                "start": "2020-02-01",
+                "end": "2020-02-07",
+                "source_url": "https://example.org/non-event",
+                "adjudicator": "reviewer-b",
+                "time_uncertainty_days": 0,
+                "independent_human_reviewed": True,
+            },
+        ],
+    }
+    candidates = {
+        "candidate_windows": [
+            {"case_id": "positive", "start": "2020-01-01", "end": "2020-01-07", "score": 0.10},
+            {"case_id": "negative", "start": "2020-02-01", "end": "2020-02-07", "score": 0.95},
+        ]
+    }
+    manifest_path = tmp_path / "holdout.json"
+    candidates_path = tmp_path / "candidates.json"
+    manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+    candidates_path.write_text(json.dumps(candidates), encoding="utf-8")
+
+    report = evaluate(manifest_path, candidates_path)
+
+    assert report["status"] == "blocked"
+    assert report["claim_status"] == "exploratory_unvalidated"
+    assert report["production_tuning_allowed"] is False
+    assert "specificity_below_gate" in report["blockers"]
+
+
+def test_negative_holdout_intake_appends_valid_independent_label(tmp_path: Path) -> None:
+    path = tmp_path / "holdout.json"
+    path.write_text(json.dumps({"annotations": [], "prohibited_tuning_data": []}), encoding="utf-8")
+
+    report = append_annotation(
+        path,
+        {
+            "case_id": "case-2-neg",
+            "domain": "career",
+            "label": "no_target_event",
+            "start": "2020-03-01",
+            "end": "2020-03-31",
+            "source_url": "https://example.org/biography",
+            "adjudicator": "reviewer-c",
+            "time_uncertainty_days": 0,
+            "independent_human_reviewed": True,
+        },
+    )
+
+    saved = json.loads(path.read_text(encoding="utf-8"))
+    assert report["appended"] is True
+    assert saved["annotations"][0]["case_id"] == "case-2-neg"
+    assert saved["annotations"][0]["frozen_before_scoring"] is True
+
+
+def test_negative_holdout_intake_rejects_non_independent_label(tmp_path: Path) -> None:
+    path = tmp_path / "holdout.json"
+    path.write_text(json.dumps({"annotations": []}), encoding="utf-8")
+
+    report = append_annotation(
+        path,
+        {
+            "case_id": "bad",
+            "domain": "career",
+            "label": "no_target_event",
+            "start": "2020-03-01",
+            "end": "2020-03-31",
+            "source_url": "https://example.org/biography",
+            "adjudicator": "reviewer-c",
+            "time_uncertainty_days": 0,
+            "independent_human_reviewed": False,
+        },
+    )
+
+    assert report["appended"] is False
+    assert report["errors"][0]["error"] == "not_independently_human_reviewed"