from __future__ import annotations import json from pathlib import Path from scripts.rectification.decision_policy import ( apply_vedastro_minute_sensitive_to_receipt, build_candidate_decisions, build_decision_receipt, indistinguishable_width_minutes, ) from scripts.rectification.sealed_holdout import ( holdout_passed, load_sealed_minute_holdout, ) ROOT = Path(__file__).resolve().parents[1] V3_REPORT_PATH = ( ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json" ) DIAGNOSTIC_REPORT_PATH = ( ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_post_audit_diagnostic_report.json" ) PRODUCT_HOLDOUT_PATH = ROOT / "references" / "rectification_sealed_holdout.v1.json" def _request(*, extra_events=()): events = [ { "id": "00000000-0000-4000-8000-000000000001", "domain": "career", "summary": "入职", "event_kind": "career_entry", "precision": "day", "date_start": "2016-09-15", "date_end": "2016-09-15", }, { "id": "00000000-0000-4000-8000-000000000002", "domain": "relationship", "summary": "开始一段关系", "event_kind": "relationship_start", "precision": "day", "date_start": "2018-03-01", "date_end": "2018-03-01", }, { "id": "00000000-0000-4000-8000-000000000003", "domain": "education", "summary": "毕业", "event_kind": "education_completion", "precision": "day", "date_start": "2015-06-01", "date_end": "2015-06-01", }, { "id": "00000000-0000-4000-8000-000000000010", "domain": "career", "summary": "职责变化", "event_kind": "career_change", "precision": "day", "date_start": "2019-04-01", "date_end": "2019-04-01", }, *extra_events, ] return {"events": events} def _diagnostics(): return { "leave_one_event_out_retention_rate": 1, "leave_one_domain_out_retention_rate": 1, "date_sensitivity_retention_rate": 1, "primary_secondary_margin_percent": 50, } def _rows(): return [ {"time": "05:13", "score": 20, "evidence": [], "missing_layers": []}, {"time": "05:14", "score": 8, "evidence": [], "missing_layers": []}, ] def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None: holdout = load_sealed_minute_holdout() report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8")) diagnostic = json.loads(DIAGNOSTIC_REPORT_PATH.read_text(encoding="utf-8")) product = json.loads(PRODUCT_HOLDOUT_PATH.read_text(encoding="utf-8")) sidecar = diagnostic["current_tree_unfrozen_diagnostic"] produced_by = product["metrics_produced_by"] current_tree = product["current_tree_scorer"] unfrozen = product["current_tree_unfrozen_diagnostic"] assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"] assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"] assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"] assert holdout["top_1_rate"] == report["release_metrics"]["top_1_rate"]["actual"] assert holdout["confirmation_coverage_rate"] == report["release_metrics"]["confirmation_coverage_rate"] assert holdout["status"] == "not_ready" assert holdout["valid_public_aa_cases"] >= holdout["required_cases"] assert holdout["confirmation_coverage_rate"] == 0.0 assert holdout_passed(holdout) is False assert set(holdout) == { "sealed_benchmark_id", "status", "valid_public_aa_cases", "required_cases", "top_1_rate", "confirmation_coverage_rate", } assert produced_by["implementation_sha256"] == report["frozen_scoring"]["implementation_sha256"] assert produced_by["algorithm_version"] == report["frozen_scoring"]["algorithm_version"] assert produced_by["implementation_hash_matches_at_replay"] is True assert produced_by["source_report"] == "references/real_case_calibration/minute_rectification_holdout_v3_report.json" assert current_tree["implementation_sha256"] == diagnostic["frozen_scoring"]["actual_sha256"] assert current_tree["matches_metrics_scorer"] is False assert current_tree["official_eval_implementation_hash_matches"] is False assert current_tree["official_eval_trial_count"] == diagnostic["trial_count"] == 0 assert current_tree["implementation_sha256"] != produced_by["implementation_sha256"] assert unfrozen["is_blind_evaluation"] is False assert unfrozen["is_blind_evaluation"] is diagnostic["is_blind_evaluation"] assert unfrozen["results_already_seen"] is True assert unfrozen["scorer_frozen"] is False assert unfrozen["scorer_frozen"] is diagnostic["frozen_scoring"]["implementation_hash_matches"] assert unfrozen["source_audit_passed"] is False assert unfrozen["source_audit_passed"] is report["post_replay_source_audit"]["passed"] assert unfrozen["must_not_claim_as_release_metrics"] is True assert unfrozen["must_not_use_for_tuning"] is True assert unfrozen["report_path"] == ( "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json" ) assert unfrozen["implementation_sha256"] == sidecar["actual_sha256"] assert unfrozen["trial_count"] == sidecar["trial_count"] assert unfrozen["top_1_rate"] == sidecar["metrics"]["top_1_rate"] assert unfrozen["top_3_rate"] == sidecar["metrics"]["top_3_rate"] assert unfrozen["mean_absolute_minute_error"] == sidecar["metrics"]["mean_absolute_minute_error"] assert product["top_1_rate"] != unfrozen["top_1_rate"] assert holdout["top_1_rate"] != unfrozen["top_1_rate"] def test_vedastro_pass_rewrites_audit_but_holdout_still_blocks(monkeypatch) -> None: extra = [{ "id": "00000000-0000-4000-8000-000000000004", "domain": "family", "summary": "家人变化", "event_kind": "family_event", "precision": "day", "date_start": "2020-01-01", "date_end": "2020-01-01", }] decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt(_request(extra_events=extra), decisions, {"missing_layers": [], "matrix": {}}, _diagnostics()) assert receipt["gates"]["exact_confirmation"]["engine_granted"] is True apply_vedastro_minute_sensitive_to_receipt(receipt, "passed", summary={"discriminated": True}) audit = {row["technique"]: row for row in receipt["technique_audit_table"]} assert audit["VedAstro 分钟级校验"]["status"] == "executed" assert "未调用不等于失败" not in audit["VedAstro 分钟级校验"]["note"] assert audit["唯一分钟确认"]["status"] == "blocked" assert receipt["confirmation_allowed"] is False assert receipt["gates"]["exact_confirmation"]["holdout"]["status"] == "not_ready" def test_confirmation_allowed_only_when_holdout_ready(monkeypatch) -> None: extra = [{ "id": "00000000-0000-4000-8000-000000000004", "domain": "family", "summary": "家人变化", "event_kind": "family_event", "precision": "day", "date_start": "2020-01-01", "date_end": "2020-01-01", }] monkeypatch.setattr( "scripts.rectification.decision_policy.load_sealed_minute_holdout", lambda: { "sealed_benchmark_id": "minute_rectification_holdout_v2", "status": "ready", "valid_public_aa_cases": 20, "required_cases": 20, "top_1_rate": 0.7, "confirmation_coverage_rate": 0.2, }, ) monkeypatch.setattr("scripts.rectification.decision_policy.holdout_passed", lambda holdout=None: True) decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt(_request(extra_events=extra), decisions, {"missing_layers": [], "matrix": {}}, _diagnostics()) apply_vedastro_minute_sensitive_to_receipt(receipt, "passed", summary={"discriminated": True}) assert receipt["confirmation_allowed"] is True assert receipt["confirm_allowed"] is True audit = {row["technique"]: row for row in receipt["technique_audit_table"]} assert audit["唯一分钟确认"]["status"] == "executed" def test_kp_only_missing_does_not_block_propose_or_engine_grant() -> None: extra = [{ "id": "00000000-0000-4000-8000-000000000004", "domain": "family", "summary": "家人变化", "event_kind": "family_event", "precision": "day", "date_start": "2020-01-01", "date_end": "2020-01-01", }] decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt( _request(extra_events=extra), decisions, {"missing_layers": ["KP_cusps"], "matrix": {}}, _diagnostics(), ) assert receipt["gates"]["required_layers"]["passed"] is True assert receipt["gates"]["required_layers"]["missing_layers"] == [] assert receipt["gates"]["required_layers"]["skipped_by_policy"] == ["KP_cusps"] assert receipt["propose_allowed"] is True assert receipt["gates"]["exact_confirmation"]["engine_granted"] is True assert "missing_mandatory_layers" not in receipt["confirmation_reasons"] def test_four_events_two_domains_selects_but_does_not_propose() -> None: events = [ { "id": "00000000-0000-4000-8000-000000000001", "domain": "education", "summary": "入学", "event_kind": "education_start", "precision": "month", "date_start": "2016-09-01", "date_end": "2016-09-30", }, { "id": "00000000-0000-4000-8000-000000000002", "domain": "education", "summary": "毕业", "event_kind": "education_completion", "precision": "month", "date_start": "2020-06-01", "date_end": "2020-06-30", }, { "id": "00000000-0000-4000-8000-000000000003", "domain": "relationship", "summary": "相识", "event_kind": "relationship_start", "precision": "day", "date_start": "2024-05-01", "date_end": "2024-05-01", }, { "id": "00000000-0000-4000-8000-000000000004", "domain": "relationship", "summary": "分手", "event_kind": "relationship_end", "precision": "day", "date_start": "2024-08-08", "date_end": "2024-08-08", }, ] decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt( {"events": events}, decisions, {"missing_layers": ["KP_cusps"], "matrix": {}}, { "leave_one_event_out_retention_rate": 1, "leave_one_domain_out_retention_rate": 0.5, "date_sensitivity_retention_rate": 1, "primary_secondary_margin_percent": 10.27, }, ) assert receipt["selection_allowed"] is True assert receipt["propose_allowed"] is False assert receipt["gates"]["exact_confirmation"]["engine_granted"] is False assert "insufficient_confirmation_domains" in receipt["confirmation_reasons"] assert "insufficient_diagnostic_stability" in receipt["confirmation_reasons"] def test_d9_missing_still_blocks_required_layers() -> None: extra = [{ "id": "00000000-0000-4000-8000-000000000004", "domain": "family", "summary": "家人变化", "event_kind": "family_event", "precision": "day", "date_start": "2020-01-01", "date_end": "2020-01-01", }] decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt( _request(extra_events=extra), decisions, {"missing_layers": ["KP_cusps", "D9"], "matrix": {}}, _diagnostics(), ) assert receipt["gates"]["required_layers"]["passed"] is False assert receipt["gates"]["required_layers"]["missing_layers"] == ["D9"] assert receipt["propose_allowed"] is False assert "missing_mandatory_layers" in receipt["confirmation_reasons"] def test_timeout_status_stays_not_evaluated_not_fail() -> None: decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt(_request(), decisions, {"missing_layers": [], "matrix": {}}, _diagnostics()) apply_vedastro_minute_sensitive_to_receipt( receipt, "not_evaluated", summary={"reason": "vedastro_minute_snapshot_timeout"}, ) exact = receipt["gates"]["exact_confirmation"] assert exact["external_validation_status"] == "not_evaluated" assert exact["external_validation_status"] != "failed" assert receipt["confirmation_allowed"] is False note = next(row["note"] for row in receipt["technique_audit_table"] if row["technique"] == "VedAstro 分钟级校验") assert "不等于失败" in note def test_indistinguishable_width_matches_public_span() -> None: width = indistinguishable_width_minutes([ {"time": "04:45", "rank": 1, "tied_minute_count": 25}, {"time": "04:46", "rank": 2, "tied_minute_count": 25}, {"time": "04:47", "rank": 3, "tied_minute_count": 25}, ]) assert width >= 25 def test_fourteen_minute_tied_cluster_proposes_but_does_not_confirm() -> None: extra = [{ "id": "00000000-0000-4000-8000-000000000004", "domain": "family", "summary": "家人变化", "event_kind": "family_event", "precision": "day", "date_start": "2020-01-01", "date_end": "2020-01-01", }] rows = [ { "time": f"{15 + ((50 + offset) // 60):02d}:{(50 + offset) % 60:02d}", "score": 20, "evidence": [], "missing_layers": [], } for offset in range(14) ] decisions = build_candidate_decisions(rows, result_id="00000000-0000-4000-8000-000000000099") receipt = build_decision_receipt( _request(extra_events=extra), decisions, {"missing_layers": ["KP_cusps"], "matrix": {}}, _diagnostics(), ) exact = receipt["gates"]["exact_confirmation"] assert receipt["selection_allowed"] is True assert receipt["propose_allowed"] is True assert receipt["gates"]["unique_top"]["passed"] is False assert exact["indistinguishable_width_minutes"] >= 14 assert exact["adjacent_passed"] is False assert exact["engine_granted"] is False assert receipt["confirmation_allowed"] is False assert exact["unique_minute_path"] == "closed_at_representative" assert "adjacent_minutes_indistinguishable" in receipt["confirmation_reasons"] note = next( row["note"] for row in receipt["technique_audit_table"] if row["technique"] == "唯一分钟确认" ) assert "本会话以代表性时间收口" in note