diff --git a/docs/research/day_level_holdout_labeling_guide_zh.md b/docs/research/day_level_holdout_labeling_guide_zh.md new file mode 100644 index 00000000..b397012b --- /dev/null +++ b/docs/research/day_level_holdout_labeling_guide_zh.md @@ -0,0 +1,76 @@ +# 日级应期 holdout 人工标签指南 + +目标：给研究仓提供真实、独立、可复验的正/负样本。你不需要会编程，只需要提供来源清楚的事实材料。 + +## 需要收集什么 + +每条标签只回答一个问题： + +某人在某个日期区间，某类事件是否发生？ + +可用标签： + +- `target_event`：事件发生了。 +- `no_target_event`：有公开资料支持该区间没有发生这个目标事件。 + +## 优先领域 + +1. 事业：任命、创办公司、上市、获奖、重大作品发布。 +2. 婚恋：结婚、离婚、订婚、公开伴侣关系变化。 +3. 财富：上市、重大融资、破产、重大资产事件。 + +暂不优先健康/死亡，噪音和伦理风险高。 + +## 合格来源 + +优先： + +- 官方 biography / timeline +- Britannica / Nobel / company official timeline +- 出版传记中可核对页码或章节的时间线 +- IMDb / MusicBrainz / company history 等结构化公开资料 + +不合格： + +- “没搜到新闻所以没发生” +- ChatGPT 生成内容 +- 无来源论坛故事 +- 已被本项目观察过的旧控制日期 +- 模糊说法：“那一年很平静” + +## 最小可用规模 + +pilot 阶段： + +- 3 个公开人物 +- 每人 1 个领域 +- 每人 1 个正样本窗口 +- 每人 2 个负样本窗口 + +正式升级门槛： + +- 至少 20 个独立案例 +- 至少 80 个独立负样本区间 +- 标签冻结后才允许评分 + +## 填写方式 + +生成空模板： + +```bash +python3 scripts/day_level_holdout_template.py --output /tmp/holdout_annotation_template.json +``` + +把公开来源、日期区间、事件说明填进去，再交给 intake： + +```bash +python3 scripts/day_level_negative_holdout_intake.py references/real_case_calibration/day_level_holdout_v3_preregistration.json --row-json '{"case_id":"..."}' +``` + +## 结论边界 + +没有真实独立负样本前： + +- 可以输出候选日期排序； +- 可以说明触发信号； +- 不能说“精确日期预测已验证”。 diff --git a/scripts/day_level_holdout_template.py b/scripts/day_level_holdout_template.py new file mode 100644 index 00000000..6c77749c --- /dev/null +++ b/scripts/day_level_holdout_template.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Create a blank independent day-level timing holdout annotation template.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +TEMPLATE = { + "case_id": "", + "subject": { + "name": "", + "public_profile_url": "", + "birth_time_rating": "AA/A only preferred", + }, + "domain": "career|marriage|wealth|health", + "label": "target_event|no_target_event", + "start": "YYYY-MM-DD", + "end": "YYYY-MM-DD", + "event_description": "", + "event_absent_assertion": "", + "source_url": "https://", + "source_quote_or_summary": "", + "adjudicator": "", + "time_uncertainty_days": 0, + "independent_human_reviewed": True, + "frozen_before_scoring": True, + "source_path": "", + "notes": "", +} + + +def build_template() -> dict: + return { + "template_type": "day_level_holdout_annotation_v3", + "instructions": [ + "Use target_event for known dated events.", + "Use no_target_event only when a public source supports that the target event did not occur in the interval.", + "Do not use old control dates or rows observed before preregistration for tuning.", + "Freeze labels before running timing_ranker_blind_eval.py.", + ], + "annotation": TEMPLATE, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + text = json.dumps(build_template(), ensure_ascii=False, indent=2) + "\n" + if args.output: + args.output.write_text(text, encoding="utf-8") + else: + print(text, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/day_level_holdout_validator.py b/scripts/day_level_holdout_validator.py index 788fbfb6..2063404e 100644 --- a/scripts/day_level_holdout_validator.py +++ b/scripts/day_level_holdout_validator.py @@ -11,11 +11,14 @@ REQUIRED={"case_id","domain","label","start","end","source_url","adjudicator","t def validate(path: Path) -> dict: data=json.loads(path.read_text(encoding="utf-8")); rows=data.get("annotations") or []; errors=[] mode=data.get("validation_mode", "independent") + prohibited=set(data.get("prohibited_tuning_data") or []) allowed_labels={"target_event", "no_target_event"} if mode == "independent" else {"target_event", "observational_non_target_date"} for i,row in enumerate(rows): for key in sorted(REQUIRED-set(row)): errors.append({"row":i,"field":key,"error":"missing"}) if row.get("label") not in allowed_labels: errors.append({"row":i,"field":"label","error":"invalid"}) if not str(row.get("source_url") or "").startswith(("https://","http://")): errors.append({"row":i,"field":"source_url","error":"not_public_url"}) + if mode == "independent" and row.get("independent_human_reviewed") is not True: errors.append({"row":i,"field":"independent_human_reviewed","error":"not_independently_human_reviewed"}) + if row.get("source_path") in prohibited: errors.append({"row":i,"field":"source_path","error":"prohibited_tuning_source"}) positives=sum(r.get("label")=="target_event" for r in rows); negatives=sum(r.get("label") in {"no_target_event", "observational_non_target_date"} for r in rows) gate=data.get("frozen_gate") or {}; ready=not errors and positives>=gate.get("minimum_independent_cases",20) and negatives>=gate.get("minimum_independent_negative_intervals",80) status=("observational_ready_not_independent" if ready else "awaiting_observational_labels") if mode == "observational" else ("ready_for_blind_replay" if ready else "awaiting_independent_labels") diff --git a/scripts/timing_ranker_blind_eval.py b/scripts/timing_ranker_blind_eval.py new file mode 100644 index 00000000..aa97a0d9 --- /dev/null +++ b/scripts/timing_ranker_blind_eval.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Evaluate frozen day/month timing candidates against independent holdout labels.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from scripts.day_level_holdout_validator import validate + + +def _load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _key(row: dict) -> tuple[str, str, str]: + return (str(row.get("case_id") or ""), str(row.get("start") or ""), str(row.get("end") or "")) + + +def evaluate(manifest_path: Path, candidates_path: Path) -> dict: + validation = validate(manifest_path) + manifest = _load(manifest_path) + candidates = sorted( + (_load(candidates_path).get("candidate_windows") or []), + key=lambda row: float(row.get("score") or 0), + reverse=True, + ) + labels = {_key(row): row.get("label") for row in manifest.get("annotations") or []} + ranked = [{**row, "rank": index + 1, "label": labels.get(_key(row))} for index, row in enumerate(candidates)] + positives = [row for row in ranked if row.get("label") == "target_event"] + negatives = [row for row in ranked if row.get("label") == "no_target_event"] + top_3_positive = sum(1 for row in positives if row["rank"] <= 3) + positive_top_3_rate = top_3_positive / len(positives) if positives else 0.0 + min_positive_score = min((float(row.get("score") or 0) for row in positives), default=0.0) + false_positive_negatives = sum(1 for row in negatives if float(row.get("score") or 0) >= min_positive_score) + specificity = 1 - (false_positive_negatives / len(negatives)) if negatives else 0.0 + gate = manifest.get("frozen_gate") or {} + blockers = [] + if validation["status"] != "ready_for_blind_replay": + blockers.append("holdout_not_ready") + if positive_top_3_rate < gate.get("minimum_positive_top_3_rate", 1): + blockers.append("positive_top_3_rate_below_gate") + if specificity < gate.get("minimum_specificity", 1): + blockers.append("specificity_below_gate") + passed = not blockers + return { + "scope": "timing_ranker_blind_eval", + "status": "pass" if passed else "blocked", + "claim_status": "calibrated_day_level" if passed else "exploratory_unvalidated", + "production_tuning_allowed": bool(passed), + "positive_count": len(positives), + "negative_count": len(negatives), + "positive_top_3_rate": positive_top_3_rate, + "specificity": specificity, + "blockers": blockers, + "validation": validation, + "ranked_windows": ranked, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("candidates", type=Path) + args = parser.parse_args() + print(json.dumps(evaluate(args.manifest, args.candidates), ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_day_level_holdout_template.py b/tests/test_day_level_holdout_template.py new file mode 100644 index 00000000..aad25713 --- /dev/null +++ b/tests/test_day_level_holdout_template.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from scripts.day_level_holdout_template import build_template + +ROOT = Path(__file__).resolve().parents[1] + + +def test_holdout_template_is_nontechnical_and_validator_compatible_shape() -> None: + template = build_template() + annotation = template["annotation"] + + for token in [ + "case_id", + "subject", + "domain", + "label", + "start", + "end", + "event_absent_assertion", + "source_url", + "adjudicator", + "independent_human_reviewed", + "frozen_before_scoring", + ]: + assert token in annotation + + assert "Do not use old control dates" in " ".join(template["instructions"]) + assert annotation["source_url"] == "https://" + assert annotation["independent_human_reviewed"] is True + + +def test_holdout_template_cli_writes_json(tmp_path: Path) -> None: + output = tmp_path / "template.json" + + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "day_level_holdout_template.py"), "--output", str(output)], + check=True, + text=True, + ) + + data = json.loads(output.read_text(encoding="utf-8")) + assert data["template_type"] == "day_level_holdout_annotation_v3" + assert data["annotation"]["label"] == "target_event|no_target_event" diff --git a/tests/test_timing_ranker_blind_eval.py b/tests/test_timing_ranker_blind_eval.py new file mode 100644 index 00000000..e9dcef0c --- /dev/null +++ b/tests/test_timing_ranker_blind_eval.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from scripts.day_level_holdout_validator import validate +from scripts.day_level_negative_holdout_intake import append_annotation +from scripts.timing_ranker_blind_eval import evaluate + + +def test_holdout_validator_rejects_non_independent_and_prohibited_rows(tmp_path: Path) -> None: + manifest = { + "prohibited_tuning_data": ["old_controls.json"], + "frozen_gate": {"minimum_independent_cases": 1, "minimum_independent_negative_intervals": 1}, + "annotations": [ + { + "case_id": "case-1", + "domain": "career", + "label": "no_target_event", + "start": "2020-01-01", + "end": "2020-01-07", + "source_url": "https://example.org/timeline", + "adjudicator": "same_person", + "time_uncertainty_days": 0, + "independent_human_reviewed": False, + "source_path": "old_controls.json", + } + ], + } + path = tmp_path / "holdout.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + + report = validate(path) + + assert report["status"] == "awaiting_independent_labels" + assert report["production_tuning_allowed"] is False + assert {error["error"] for error in report["errors"]} >= { + "not_independently_human_reviewed", + "prohibited_tuning_source", + } + + +def test_blind_eval_requires_positive_windows_to_rank_above_negative_windows(tmp_path: Path) -> None: + manifest = { + "frozen_gate": { + "minimum_positive_top_3_rate": 0.6, + "minimum_specificity": 0.6, + "minimum_independent_cases": 1, + "minimum_independent_negative_intervals": 1, + }, + "annotations": [ + { + "case_id": "case-1", + "domain": "career", + "label": "target_event", + "start": "2020-01-01", + "end": "2020-01-07", + "source_url": "https://example.org/event", + "adjudicator": "reviewer-a", + "time_uncertainty_days": 0, + "independent_human_reviewed": True, + }, + { + "case_id": "case-1-neg", + "domain": "career", + "label": "no_target_event", + "start": "2020-02-01", + "end": "2020-02-07", + "source_url": "https://example.org/non-event", + "adjudicator": "reviewer-b", + "time_uncertainty_days": 0, + "independent_human_reviewed": True, + }, + ], + } + candidates = { + "candidate_windows": [ + {"case_id": "case-1", "start": "2020-01-01", "end": "2020-01-07", "score": 0.90}, + {"case_id": "case-1-neg", "start": "2020-02-01", "end": "2020-02-07", "score": 0.20}, + ] + } + manifest_path = tmp_path / "holdout.json" + candidates_path = tmp_path / "candidates.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + candidates_path.write_text(json.dumps(candidates), encoding="utf-8") + + report = evaluate(manifest_path, candidates_path) + + assert report["status"] == "pass" + assert report["claim_status"] == "calibrated_day_level" + assert report["production_tuning_allowed"] is True + assert report["positive_top_3_rate"] == 1.0 + assert report["specificity"] == 1.0 + + +def test_blind_eval_blocks_when_negative_scores_outrank_positive_scores(tmp_path: Path) -> None: + manifest = { + "frozen_gate": { + "minimum_positive_top_3_rate": 0.6, + "minimum_specificity": 0.6, + "minimum_independent_cases": 1, + "minimum_independent_negative_intervals": 1, + }, + "annotations": [ + { + "case_id": "positive", + "domain": "marriage", + "label": "target_event", + "start": "2020-01-01", + "end": "2020-01-07", + "source_url": "https://example.org/event", + "adjudicator": "reviewer-a", + "time_uncertainty_days": 0, + "independent_human_reviewed": True, + }, + { + "case_id": "negative", + "domain": "marriage", + "label": "no_target_event", + "start": "2020-02-01", + "end": "2020-02-07", + "source_url": "https://example.org/non-event", + "adjudicator": "reviewer-b", + "time_uncertainty_days": 0, + "independent_human_reviewed": True, + }, + ], + } + candidates = { + "candidate_windows": [ + {"case_id": "positive", "start": "2020-01-01", "end": "2020-01-07", "score": 0.10}, + {"case_id": "negative", "start": "2020-02-01", "end": "2020-02-07", "score": 0.95}, + ] + } + manifest_path = tmp_path / "holdout.json" + candidates_path = tmp_path / "candidates.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + candidates_path.write_text(json.dumps(candidates), encoding="utf-8") + + report = evaluate(manifest_path, candidates_path) + + assert report["status"] == "blocked" + assert report["claim_status"] == "exploratory_unvalidated" + assert report["production_tuning_allowed"] is False + assert "specificity_below_gate" in report["blockers"] + + +def test_negative_holdout_intake_appends_valid_independent_label(tmp_path: Path) -> None: + path = tmp_path / "holdout.json" + path.write_text(json.dumps({"annotations": [], "prohibited_tuning_data": []}), encoding="utf-8") + + report = append_annotation( + path, + { + "case_id": "case-2-neg", + "domain": "career", + "label": "no_target_event", + "start": "2020-03-01", + "end": "2020-03-31", + "source_url": "https://example.org/biography", + "adjudicator": "reviewer-c", + "time_uncertainty_days": 0, + "independent_human_reviewed": True, + }, + ) + + saved = json.loads(path.read_text(encoding="utf-8")) + assert report["appended"] is True + assert saved["annotations"][0]["case_id"] == "case-2-neg" + assert saved["annotations"][0]["frozen_before_scoring"] is True + + +def test_negative_holdout_intake_rejects_non_independent_label(tmp_path: Path) -> None: + path = tmp_path / "holdout.json" + path.write_text(json.dumps({"annotations": []}), encoding="utf-8") + + report = append_annotation( + path, + { + "case_id": "bad", + "domain": "career", + "label": "no_target_event", + "start": "2020-03-01", + "end": "2020-03-31", + "source_url": "https://example.org/biography", + "adjudicator": "reviewer-c", + "time_uncertainty_days": 0, + "independent_human_reviewed": False, + }, + ) + + assert report["appended"] is False + assert report["errors"][0]["error"] == "not_independently_human_reviewed"