jesse-uxandClaude Code
932f2fffba
Preserve closed confirmation gates and previously-exposed dataset boundaries. Add auditable 900-trial sensitivity results, current scorer freshness checks, and the v5 collection protocol. Co-Authored-By: Claude Code <noreply@anthropic.com>
80 lines
3.6 KiB
JSON
80 lines
3.6 KiB
JSON
{
|
|
"sealed_benchmark_id": "minute_rectification_fact_ranker_v4_holdout_v3",
|
|
"status": "not_ready",
|
|
"valid_public_aa_cases": 20,
|
|
"required_cases": 20,
|
|
"top_1_rate": 0.15,
|
|
"confirmation_coverage_rate": 0.0,
|
|
"previous_pilot_id": "minute_rectification_holdout_v2",
|
|
"dataset_path": "references/real_case_calibration/minute_rectification_holdout_v3.json",
|
|
"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json",
|
|
"source_audit_status": "corrected_known_date_errors",
|
|
"evaluated_on": "2026-09-20",
|
|
"evaluated_on_scope": "latest_fixed_protocol_rerun_not_historical_runtime_metrics",
|
|
"historical_source_audit_status": "invalidated_after_replay",
|
|
"report_status": "invalidated_after_source_audit",
|
|
"trial_count": 20,
|
|
"top_3_rate": 0.25,
|
|
"mean_absolute_minute_error": 6.95,
|
|
"metrics_produced_by": {
|
|
"implementation_sha256": "f41c298dd6cdcebe7a93e632f7191954be7987f6012ca2a34cecb6e447fbf196",
|
|
"algorithm_version": "birth-time-event-fact-ranker-v4-shadow",
|
|
"implementation_hash_matches_at_replay": true,
|
|
"source_report": "references/real_case_calibration/minute_rectification_holdout_v3_report.json",
|
|
"evaluated_on": "2026-07-21"
|
|
},
|
|
"current_tree_scorer": {
|
|
"implementation_sha256": "b15d9ea15227cd58b3097555537fa7c37fb5a993905ada4f5aa04f52de1f8f18",
|
|
"matches_metrics_scorer": false,
|
|
"official_eval_implementation_hash_matches": false,
|
|
"official_eval_trial_count": 0,
|
|
"source_report": "docs/research/sealed_holdout_rerun_2026_09_20.json",
|
|
"fixed_protocol_rerun_trial_count": 20,
|
|
"fixed_protocol_rerun_hash_matches": true,
|
|
"metrics": {
|
|
"top_1_rate": 0.45,
|
|
"top_3_rate": 0.5,
|
|
"mean_absolute_minute_error": 6.45,
|
|
"false_confirmation_rate": 0.0,
|
|
"correct_insufficient_evidence_rejection_rate": 1.0,
|
|
"confirmation_coverage_rate": 0.0
|
|
}
|
|
},
|
|
"current_tree_fixed_protocol_rerun": {
|
|
"evaluated_on": "2026-09-20",
|
|
"report_path": "docs/research/sealed_holdout_rerun_2026_09_20.json",
|
|
"freeze_record_path": "docs/research/sealed_holdout_rerun_2026_09_20.freeze.json",
|
|
"implementation_sha256": "b15d9ea15227cd58b3097555537fa7c37fb5a993905ada4f5aa04f52de1f8f18",
|
|
"scorer_frozen_before_rerun": true,
|
|
"source_audit_status": "corrected_known_date_errors",
|
|
"trial_count": 20,
|
|
"official_valid_independent_blind": false,
|
|
"is_blind_evaluation": false,
|
|
"truth_hidden_from_ranker": true,
|
|
"results_previously_seen": true,
|
|
"must_not_claim_as_release_metrics": true,
|
|
"must_not_use_for_tuning": true,
|
|
"verified_minute_claim_allowed": false,
|
|
"metric_gates_passed": false,
|
|
"events_per_case": 3,
|
|
"boundary": "First current-hash fixed-protocol rerun in this task, not a first independent official blind evaluation. Three-event low-information protocol is not representative of real sessions and is not a mathematical accuracy lower bound. Historical v3/v4 score exposure requires a fresh sealed set."
|
|
},
|
|
"current_tree_unfrozen_diagnostic": {
|
|
"is_blind_evaluation": false,
|
|
"results_already_seen": true,
|
|
"scorer_frozen": false,
|
|
"source_audit_passed": false,
|
|
"must_not_claim_as_release_metrics": true,
|
|
"must_not_use_for_tuning": true,
|
|
"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
|
|
"implementation_sha256": "99730c84c6434f52669a05e4d9a4a87df0a218e3237b5436b933252f2028384e",
|
|
"trial_count": 20,
|
|
"top_1_rate": 0.45,
|
|
"top_3_rate": 0.5,
|
|
"mean_absolute_minute_error": 5.8,
|
|
"confirmation_coverage_rate": 0.0,
|
|
"metric_gates_passed": false,
|
|
"verified_minute_claim_allowed": false
|
|
}
|
|
}
|