Jesse_ChenandClaude Opus 5.5
758fee954b
Product decision 2026-10-02 (TASK-upstream-sync5 R1): a time range is offered
only with at least 4 dated, primary-scoreable events covering 3 domains,
counted on all of them (training + reserved holdout). Was 3 training events /
2 domains in three TS copies and the Python acceptance gate while the policy
file already said 4/3.
- One definition: references/rectification_policy.v1.json
(minConfirmationEvents / minConfirmationDomains). TS core/types MIN_DATED_*,
rectification-decision MIN_STANDALONE_*, evidence-model MIN_ACCEPTANCE_*,
the convergence evaluator and the post-inference trainingGateOpen all read
it; Python decision_policy MIN_ACCEPTANCE_* alias MIN_CONFIRMATION_*.
- Python receipt counts all scoreable events / domains for event_quality and
domain_diversity; decision policy identity v3 -> v4 (candidate UUIDs carry
it). Candidate scores unchanged (77 v5 cases A/B identical), so the
algorithm stays rectification-v5-matrix-scoring-10.
- Memoization golden v3 written by write_golden; v2 frozen by sha256 with a
test that its scores equal v3 and only the receipt policy moved.
- Collect gap copy names the exact gap ("再来两件……其中至少一件不是……")
instead of always "再来一件"; VOICE.md updated. Legacy life-events form copy
4/3 as well.
- 30 frontend test files, 4 Python tests: fixtures extended to the same
scenario at 4/3, or assertions changed with 原值/新值/原因 notes.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N4f2nya58RoRu4yEmJgRGE
404 lines
17 KiB
Python
404 lines
17 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
from scripts.rectification.decision_policy import (
|
||
apply_vedastro_minute_sensitive_to_receipt,
|
||
build_candidate_decisions,
|
||
build_decision_receipt,
|
||
indistinguishable_width_minutes,
|
||
)
|
||
from scripts.rectification.sealed_holdout import (
|
||
holdout_passed,
|
||
load_sealed_minute_holdout,
|
||
)
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
V3_REPORT_PATH = (
|
||
ROOT / "references" / "real_case_calibration" / "minute_rectification_holdout_v3_report.json"
|
||
)
|
||
DIAGNOSTIC_REPORT_PATH = (
|
||
ROOT / "references" / "real_case_calibration"
|
||
/ "minute_rectification_holdout_v3_post_audit_diagnostic_report.json"
|
||
)
|
||
PRODUCT_HOLDOUT_PATH = ROOT / "references" / "rectification_sealed_holdout.v1.json"
|
||
|
||
|
||
def _request(*, extra_events=()):
|
||
events = [
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000001",
|
||
"domain": "career",
|
||
"summary": "入职",
|
||
"event_kind": "career_entry",
|
||
"precision": "day",
|
||
"date_start": "2016-09-15",
|
||
"date_end": "2016-09-15",
|
||
},
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000002",
|
||
"domain": "relationship",
|
||
"summary": "开始一段关系",
|
||
"event_kind": "relationship_start",
|
||
"precision": "day",
|
||
"date_start": "2018-03-01",
|
||
"date_end": "2018-03-01",
|
||
},
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000003",
|
||
"domain": "education",
|
||
"summary": "毕业",
|
||
"event_kind": "education_completion",
|
||
"precision": "day",
|
||
"date_start": "2015-06-01",
|
||
"date_end": "2015-06-01",
|
||
},
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000010",
|
||
"domain": "career",
|
||
"summary": "职责变化",
|
||
"event_kind": "career_change",
|
||
"precision": "day",
|
||
"date_start": "2019-04-01",
|
||
"date_end": "2019-04-01",
|
||
},
|
||
*extra_events,
|
||
]
|
||
return {"events": events}
|
||
|
||
|
||
def _diagnostics():
|
||
return {
|
||
"leave_one_event_out_retention_rate": 1,
|
||
"leave_one_domain_out_retention_rate": 1,
|
||
"date_sensitivity_retention_rate": 1,
|
||
"primary_secondary_margin_percent": 50,
|
||
}
|
||
|
||
|
||
def _rows():
|
||
return [
|
||
{"time": "05:13", "score": 20, "evidence": [], "missing_layers": []},
|
||
{"time": "05:14", "score": 8, "evidence": [], "missing_layers": []},
|
||
]
|
||
|
||
|
||
def test_sealed_holdout_contract_matches_v3_report_and_stays_closed() -> None:
|
||
holdout = load_sealed_minute_holdout()
|
||
report = json.loads(V3_REPORT_PATH.read_text(encoding="utf-8"))
|
||
diagnostic = json.loads(DIAGNOSTIC_REPORT_PATH.read_text(encoding="utf-8"))
|
||
product = json.loads(PRODUCT_HOLDOUT_PATH.read_text(encoding="utf-8"))
|
||
sidecar = diagnostic["current_tree_unfrozen_diagnostic"]
|
||
produced_by = product["metrics_produced_by"]
|
||
current_tree = product["current_tree_scorer"]
|
||
rerun = json.loads((ROOT / current_tree["source_report"]).read_text(encoding="utf-8"))
|
||
unfrozen = product["current_tree_unfrozen_diagnostic"]
|
||
assert holdout["sealed_benchmark_id"] == report["benchmark_id"] == product["sealed_benchmark_id"]
|
||
assert holdout["valid_public_aa_cases"] == report["validation"]["valid_public_aa_cases"]
|
||
assert holdout["required_cases"] == report["validation"]["minimum_public_aa_cases"]
|
||
assert holdout["top_1_rate"] == report["release_metrics"]["top_1_rate"]["actual"]
|
||
assert holdout["confirmation_coverage_rate"] == report["release_metrics"]["confirmation_coverage_rate"]
|
||
assert holdout["status"] == "not_ready"
|
||
assert holdout["valid_public_aa_cases"] >= holdout["required_cases"]
|
||
assert holdout["confirmation_coverage_rate"] == 0.0
|
||
assert holdout_passed(holdout) is False
|
||
assert set(holdout) == {
|
||
"sealed_benchmark_id",
|
||
"status",
|
||
"valid_public_aa_cases",
|
||
"required_cases",
|
||
"top_1_rate",
|
||
"confirmation_coverage_rate",
|
||
}
|
||
assert produced_by["implementation_sha256"] == report["frozen_scoring"]["implementation_sha256"]
|
||
assert produced_by["algorithm_version"] == report["frozen_scoring"]["algorithm_version"]
|
||
assert produced_by["implementation_hash_matches_at_replay"] is True
|
||
assert produced_by["source_report"] == "references/real_case_calibration/minute_rectification_holdout_v3_report.json"
|
||
assert current_tree["implementation_sha256"] == rerun["frozen_record"]["implementation_sha256"]
|
||
assert current_tree["extended_identity"] == rerun["frozen_record"]["extended_identity"]
|
||
assert "scripts/rectification/dasha_transition_proximity.py" in current_tree["extended_identity"]["production_scoring_files"]
|
||
assert product["current_tree_fixed_protocol_rerun"]["extended_identity"] == current_tree["extended_identity"]
|
||
assert current_tree["fixed_protocol_rerun_hash_matches"] is rerun["implementation_hash_matches_at_replay"] is True
|
||
assert current_tree["fixed_protocol_rerun_trial_count"] == rerun["trial_count"] == 20
|
||
assert rerun["official_valid_independent_blind"] is False
|
||
assert current_tree["matches_metrics_scorer"] is False
|
||
assert current_tree["official_eval_implementation_hash_matches"] is False
|
||
assert current_tree["official_eval_trial_count"] == diagnostic["trial_count"] == 0
|
||
assert current_tree["implementation_sha256"] != produced_by["implementation_sha256"]
|
||
assert unfrozen["is_blind_evaluation"] is False
|
||
assert unfrozen["is_blind_evaluation"] is diagnostic["is_blind_evaluation"]
|
||
assert unfrozen["results_already_seen"] is True
|
||
assert unfrozen["scorer_frozen"] is False
|
||
assert unfrozen["scorer_frozen"] is diagnostic["frozen_scoring"]["implementation_hash_matches"]
|
||
assert unfrozen["source_audit_passed"] is False
|
||
assert unfrozen["source_audit_passed"] is report["post_replay_source_audit"]["passed"]
|
||
assert unfrozen["must_not_claim_as_release_metrics"] is True
|
||
assert unfrozen["must_not_use_for_tuning"] is True
|
||
assert unfrozen["report_path"] == (
|
||
"references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json"
|
||
)
|
||
assert unfrozen["implementation_sha256"] == sidecar["actual_sha256"]
|
||
assert unfrozen["trial_count"] == sidecar["trial_count"]
|
||
assert unfrozen["top_1_rate"] == sidecar["metrics"]["top_1_rate"]
|
||
assert unfrozen["top_3_rate"] == sidecar["metrics"]["top_3_rate"]
|
||
assert unfrozen["mean_absolute_minute_error"] == sidecar["metrics"]["mean_absolute_minute_error"]
|
||
assert product["top_1_rate"] != unfrozen["top_1_rate"]
|
||
assert holdout["top_1_rate"] != unfrozen["top_1_rate"]
|
||
|
||
|
||
def test_vedastro_pass_rewrites_audit_but_holdout_still_blocks(monkeypatch) -> None:
|
||
extra = [{
|
||
"id": "00000000-0000-4000-8000-000000000004",
|
||
"domain": "family",
|
||
"summary": "家人变化",
|
||
"event_kind": "family_event",
|
||
"precision": "day",
|
||
"date_start": "2020-01-01",
|
||
"date_end": "2020-01-01",
|
||
}]
|
||
decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(_request(extra_events=extra), decisions, {"missing_layers": [], "matrix": {}}, _diagnostics())
|
||
assert receipt["gates"]["exact_confirmation"]["engine_granted"] is True
|
||
apply_vedastro_minute_sensitive_to_receipt(receipt, "passed", summary={"discriminated": True})
|
||
audit = {row["technique"]: row for row in receipt["technique_audit_table"]}
|
||
assert audit["VedAstro 分钟级校验"]["status"] == "executed"
|
||
assert "未调用不等于失败" not in audit["VedAstro 分钟级校验"]["note"]
|
||
assert audit["唯一分钟确认"]["status"] == "blocked"
|
||
assert receipt["confirmation_allowed"] is False
|
||
assert receipt["gates"]["exact_confirmation"]["holdout"]["status"] == "not_ready"
|
||
|
||
|
||
def test_confirmation_allowed_only_when_holdout_ready(monkeypatch) -> None:
|
||
extra = [{
|
||
"id": "00000000-0000-4000-8000-000000000004",
|
||
"domain": "family",
|
||
"summary": "家人变化",
|
||
"event_kind": "family_event",
|
||
"precision": "day",
|
||
"date_start": "2020-01-01",
|
||
"date_end": "2020-01-01",
|
||
}]
|
||
monkeypatch.setattr(
|
||
"scripts.rectification.decision_policy.load_sealed_minute_holdout",
|
||
lambda: {
|
||
"sealed_benchmark_id": "minute_rectification_holdout_v2",
|
||
"status": "ready",
|
||
"valid_public_aa_cases": 20,
|
||
"required_cases": 20,
|
||
"top_1_rate": 0.7,
|
||
"confirmation_coverage_rate": 0.2,
|
||
},
|
||
)
|
||
monkeypatch.setattr("scripts.rectification.decision_policy.holdout_passed", lambda holdout=None: True)
|
||
decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(_request(extra_events=extra), decisions, {"missing_layers": [], "matrix": {}}, _diagnostics())
|
||
apply_vedastro_minute_sensitive_to_receipt(receipt, "passed", summary={"discriminated": True})
|
||
assert receipt["confirmation_allowed"] is True
|
||
assert receipt["confirm_allowed"] is True
|
||
audit = {row["technique"]: row for row in receipt["technique_audit_table"]}
|
||
assert audit["唯一分钟确认"]["status"] == "executed"
|
||
|
||
|
||
def test_kp_only_missing_does_not_block_propose_or_engine_grant() -> None:
|
||
extra = [{
|
||
"id": "00000000-0000-4000-8000-000000000004",
|
||
"domain": "family",
|
||
"summary": "家人变化",
|
||
"event_kind": "family_event",
|
||
"precision": "day",
|
||
"date_start": "2020-01-01",
|
||
"date_end": "2020-01-01",
|
||
}]
|
||
decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(
|
||
_request(extra_events=extra),
|
||
decisions,
|
||
{"missing_layers": ["KP_cusps"], "matrix": {}},
|
||
_diagnostics(),
|
||
)
|
||
assert receipt["gates"]["required_layers"]["passed"] is True
|
||
assert receipt["gates"]["required_layers"]["missing_layers"] == []
|
||
assert receipt["gates"]["required_layers"]["skipped_by_policy"] == ["KP_cusps"]
|
||
assert receipt["propose_allowed"] is True
|
||
assert receipt["gates"]["exact_confirmation"]["engine_granted"] is True
|
||
assert "missing_mandatory_layers" not in receipt["confirmation_reasons"]
|
||
|
||
|
||
def test_four_events_two_domains_neither_selects_nor_proposes() -> None:
|
||
events = [
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000001",
|
||
"domain": "education",
|
||
"summary": "入学",
|
||
"event_kind": "education_start",
|
||
"precision": "month",
|
||
"date_start": "2016-09-01",
|
||
"date_end": "2016-09-30",
|
||
},
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000002",
|
||
"domain": "education",
|
||
"summary": "毕业",
|
||
"event_kind": "education_completion",
|
||
"precision": "month",
|
||
"date_start": "2020-06-01",
|
||
"date_end": "2020-06-30",
|
||
},
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000003",
|
||
"domain": "relationship",
|
||
"summary": "相识",
|
||
"event_kind": "relationship_start",
|
||
"precision": "day",
|
||
"date_start": "2024-05-01",
|
||
"date_end": "2024-05-01",
|
||
},
|
||
{
|
||
"id": "00000000-0000-4000-8000-000000000004",
|
||
"domain": "relationship",
|
||
"summary": "分手",
|
||
"event_kind": "relationship_end",
|
||
"precision": "day",
|
||
"date_start": "2024-08-08",
|
||
"date_end": "2024-08-08",
|
||
},
|
||
]
|
||
decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(
|
||
{"events": events},
|
||
decisions,
|
||
{"missing_layers": ["KP_cusps"], "matrix": {}},
|
||
{
|
||
"leave_one_event_out_retention_rate": 1,
|
||
"leave_one_domain_out_retention_rate": 0.5,
|
||
"date_sensitivity_retention_rate": 1,
|
||
"primary_secondary_margin_percent": 10.27,
|
||
},
|
||
)
|
||
# 原值: selection_allowed=True（区间门槛 3 件 2 域，4 件 2 域可选不可提议）
|
||
# 新值: selection_allowed=False，domain_diversity 不通过，原因 insufficient_domain_diversity
|
||
# 原因: R1 生时校正交付门槛 4 件 3 域（BUG-1193），2 个领域不再给区间
|
||
assert receipt["selection_allowed"] is False
|
||
assert receipt["gates"]["event_quality"]["passed"] is True
|
||
assert receipt["gates"]["domain_diversity"]["passed"] is False
|
||
assert receipt["gates"]["domain_diversity"]["minimum"] == 3
|
||
assert "insufficient_domain_diversity" in receipt["reasons"]
|
||
assert receipt["propose_allowed"] is False
|
||
assert receipt["gates"]["exact_confirmation"]["engine_granted"] is False
|
||
assert "insufficient_confirmation_domains" in receipt["confirmation_reasons"]
|
||
assert "insufficient_diagnostic_stability" in receipt["confirmation_reasons"]
|
||
|
||
|
||
def test_d9_missing_still_blocks_required_layers() -> None:
|
||
extra = [{
|
||
"id": "00000000-0000-4000-8000-000000000004",
|
||
"domain": "family",
|
||
"summary": "家人变化",
|
||
"event_kind": "family_event",
|
||
"precision": "day",
|
||
"date_start": "2020-01-01",
|
||
"date_end": "2020-01-01",
|
||
}]
|
||
decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(
|
||
_request(extra_events=extra),
|
||
decisions,
|
||
{"missing_layers": ["KP_cusps", "D9"], "matrix": {}},
|
||
_diagnostics(),
|
||
)
|
||
assert receipt["gates"]["required_layers"]["passed"] is False
|
||
assert receipt["gates"]["required_layers"]["missing_layers"] == ["D9"]
|
||
assert receipt["propose_allowed"] is False
|
||
assert "missing_mandatory_layers" in receipt["confirmation_reasons"]
|
||
|
||
|
||
def test_timeout_status_stays_not_evaluated_not_fail() -> None:
|
||
decisions = build_candidate_decisions(_rows(), result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(_request(), decisions, {"missing_layers": [], "matrix": {}}, _diagnostics())
|
||
apply_vedastro_minute_sensitive_to_receipt(
|
||
receipt,
|
||
"not_evaluated",
|
||
summary={"reason": "vedastro_minute_snapshot_timeout"},
|
||
)
|
||
exact = receipt["gates"]["exact_confirmation"]
|
||
assert exact["external_validation_status"] == "not_evaluated"
|
||
assert exact["external_validation_status"] != "failed"
|
||
assert receipt["confirmation_allowed"] is False
|
||
note = next(row["note"] for row in receipt["technique_audit_table"] if row["technique"] == "VedAstro 分钟级校验")
|
||
assert "不等于失败" in note
|
||
|
||
|
||
def test_indistinguishable_width_matches_public_span() -> None:
|
||
width = indistinguishable_width_minutes([
|
||
{"time": "04:45", "rank": 1, "tied_minute_count": 25},
|
||
{"time": "04:46", "rank": 2, "tied_minute_count": 25},
|
||
{"time": "04:47", "rank": 3, "tied_minute_count": 25},
|
||
])
|
||
assert width >= 25
|
||
|
||
|
||
def test_indistinguishable_width_uses_cluster_span_not_representatives() -> None:
|
||
width = indistinguishable_width_minutes([
|
||
{
|
||
"time": "04:53",
|
||
"rank": 1,
|
||
"tied_minute_count": 1,
|
||
"cluster_start": "04:50",
|
||
"cluster_end": "04:55",
|
||
},
|
||
{
|
||
"time": "04:59",
|
||
"rank": 2,
|
||
"tied_minute_count": 1,
|
||
"cluster_start": "04:56",
|
||
"cluster_end": "05:02",
|
||
},
|
||
])
|
||
assert width == 13
|
||
|
||
|
||
def test_fourteen_minute_tied_cluster_proposes_but_does_not_confirm() -> None:
|
||
extra = [{
|
||
"id": "00000000-0000-4000-8000-000000000004",
|
||
"domain": "family",
|
||
"summary": "家人变化",
|
||
"event_kind": "family_event",
|
||
"precision": "day",
|
||
"date_start": "2020-01-01",
|
||
"date_end": "2020-01-01",
|
||
}]
|
||
rows = [
|
||
{
|
||
"time": f"{15 + ((50 + offset) // 60):02d}:{(50 + offset) % 60:02d}",
|
||
"score": 20,
|
||
"evidence": [],
|
||
"missing_layers": [],
|
||
}
|
||
for offset in range(14)
|
||
]
|
||
decisions = build_candidate_decisions(rows, result_id="00000000-0000-4000-8000-000000000099")
|
||
receipt = build_decision_receipt(
|
||
_request(extra_events=extra),
|
||
decisions,
|
||
{"missing_layers": ["KP_cusps"], "matrix": {}},
|
||
_diagnostics(),
|
||
)
|
||
exact = receipt["gates"]["exact_confirmation"]
|
||
assert receipt["selection_allowed"] is True
|
||
assert receipt["propose_allowed"] is True
|
||
assert receipt["gates"]["unique_top"]["passed"] is False
|
||
assert exact["indistinguishable_width_minutes"] >= 14
|
||
assert exact["adjacent_passed"] is False
|
||
assert exact["engine_granted"] is False
|
||
assert receipt["confirmation_allowed"] is False
|
||
assert exact["unique_minute_path"] == "closed_at_representative"
|
||
assert "adjacent_minutes_indistinguishable" in receipt["confirmation_reasons"]
|
||
note = next(
|
||
row["note"]
|
||
for row in receipt["technique_audit_table"]
|
||
if row["technique"] == "唯一分钟确认"
|
||
)
|
||
assert "本会话以代表性时间收口" in note
|