docs(rectification): mark v4 intake cases as already seen, not sealed
Keep the four migrated cases in the queue. Record prior score exposures, distinguish migrated review from a fresh biography audit, and reject exposed or auto-attested cases from blind holdout use. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -57,6 +57,7 @@ def _add_v4_review_safeguards(manifest: dict) -> None:
|
||||
for case in manifest["cases"]:
|
||||
case["adjudicator"] = "independent-reviewer"
|
||||
case["independent_human_reviewed"] = True
|
||||
case["human_review_kind"] = "fresh_biography_audit"
|
||||
case["frozen_before_scoring"] = True
|
||||
case["false_minute_commitments"] = [
|
||||
{
|
||||
@@ -96,6 +97,26 @@ def test_v4_validator_rejects_unreviewed_or_uncommitted_cases(tmp_path: Path) ->
|
||||
assert "false_minute_commitments_do_not_match_offsets" in errors
|
||||
|
||||
|
||||
def test_v4_safeguards_reject_exposed_or_migrated_review(tmp_path: Path) -> None:
|
||||
manifest = deepcopy(_manifest())
|
||||
_add_v4_review_safeguards(manifest)
|
||||
exposed = manifest["cases"][0]
|
||||
exposed["prior_score_exposures"] = [
|
||||
{"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json"},
|
||||
]
|
||||
path = tmp_path / "exposed-v4.json"
|
||||
path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
assert "results_already_seen_cannot_be_blind_holdout" in validate(path)["invalid_case_details"][0]["errors"]
|
||||
|
||||
migrated = deepcopy(_manifest())
|
||||
_add_v4_review_safeguards(migrated)
|
||||
migrated["cases"][0]["human_review_kind"] = "migrated_v3_record"
|
||||
path = tmp_path / "migrated-v4.json"
|
||||
path.write_text(json.dumps(migrated), encoding="utf-8")
|
||||
errors = validate(path)["invalid_case_details"][0]["errors"]
|
||||
assert "migrated_review_does_not_satisfy_safeguard" in errors
|
||||
assert "fresh_biography_audit_not_attested" in errors
|
||||
|
||||
def test_validator_rejects_tuning_case_and_non_independent_event_source(tmp_path: Path) -> None:
|
||||
manifest = deepcopy(_manifest())
|
||||
case = manifest["cases"][0]
|
||||
|
||||
Reference in New Issue
Block a user