root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

docs(rectification): mark v4 intake cases as already seen, not sealed

Browse Source 
Keep the four migrated cases in the queue. Record prior score exposures, distinguish migrated review from a fresh biography audit, and reject exposed or auto-attested cases from blind holdout use.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-29 01:33:20 +08:00
parent dc2879020a
commit a3f4e7128b
7 changed files with 214 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_minute_rectification_holdout_intake.py
+79 -3
View File
@@ -4,7 +4,12 @@ from copy import deepcopy
from pathlib import Path
from scripts.minute_rectification_holdout_intake import DEFAULT_INTAKE, append_case
from scripts.minute_rectification_holdout_validator import DEFAULT_MANIFEST, case_errors
from scripts.minute_rectification_holdout_validator import (
DEFAULT_MANIFEST,
FRESH_HUMAN_REVIEW_KIND,
MIGRATED_HUMAN_REVIEW_KIND,
case_errors,
)
V3_HOLDOUT = Path(__file__).resolve().parents[1] / "references" / "real_case_calibration" / "minute_rectification_holdout_v3.json"
EXPECTED_V4_INTAKE_CASE_IDS = {
@@ -20,6 +25,7 @@ def _reviewed_case() -> dict:
case["case_id"] = "new-reviewed-case"
case["adjudicator"] = "independent-reviewer"
case["independent_human_reviewed"] = True
case["human_review_kind"] = FRESH_HUMAN_REVIEW_KIND
case["frozen_before_scoring"] = True
case["false_minute_commitments"] = [
{
@@ -59,6 +65,19 @@ def test_intake_appends_reviewed_case_but_keeps_release_blocked(tmp_path: Path)
assert data["cases"][0]["ingested_at"].endswith("Z")
def test_intake_rejects_boolean_review_without_fresh_biography_audit(tmp_path: Path) -> None:
path = tmp_path / "intake.json"
_queue(path)
case = _reviewed_case()
case["human_review_kind"] = MIGRATED_HUMAN_REVIEW_KIND
report = append_case(path, case)
assert report["appended"] is False
assert "independent_review_requires_fresh_biography_audit" in report["errors"]
assert json.loads(path.read_text(encoding="utf-8"))["cases"] == []
def test_intake_rejects_missing_review_and_commitments(tmp_path: Path) -> None:
path = tmp_path / "intake.json"
_queue(path)
@@ -75,11 +94,65 @@ def test_intake_rejects_missing_review_and_commitments(tmp_path: Path) -> None:
def test_default_intake_is_non_production_and_contains_day_precision_v3_cases() -> None:
data = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))
gate = data["minimum_gate"]
assert {case["case_id"] for case in data["cases"]} == EXPECTED_V4_INTAKE_CASE_IDS
assert len(data["cases"]) == 4
assert data["production_tuning_allowed"] is False
assert data["verified_minute_claim_allowed"] is False
assert data["status"] == "exposed_awaiting_human_rereview"
assert data["blind_holdout_eligible_case_count"] == 0
assert data["exposed_awaiting_human_rereview_case_count"] == 4
for case in data["cases"]:
assert case["prior_score_exposures"]
assert case["human_review_kind"] == MIGRATED_HUMAN_REVIEW_KIND
assert case["independent_human_reviewed"] is False
assert case["frozen_before_scoring"] is False
assert case["holdout_partition"] == "exposed_awaiting_human_rereview"
errors = case_errors(case, gate, require_review_safeguards=True)
assert "results_already_seen_cannot_be_blind_holdout" in errors
assert "migrated_review_does_not_satisfy_safeguard" in errors
assert "fresh_biography_audit_not_attested" in errors
eligible = [
case["case_id"]
for case in data["cases"]
if not case_errors(case, gate, require_review_safeguards=True)
]
assert eligible == []
def test_exposed_case_cannot_pass_blind_holdout_safeguards() -> None:
case = _reviewed_case()
case["prior_score_exposures"] = [
{
"evaluation_kind": "v3_blind_replay",
"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json",
},
{
"evaluation_kind": "post_audit_sidecar_diagnostic",
"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
},
]
gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"]
errors = case_errors(case, gate, require_review_safeguards=True)
assert "results_already_seen_cannot_be_blind_holdout" in errors
def test_migrated_human_review_does_not_satisfy_safeguards() -> None:
case = _reviewed_case()
case["human_review_kind"] = MIGRATED_HUMAN_REVIEW_KIND
case["adjudicator"] = "v3_sealed_set_post_audit_migration"
gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"]
errors = case_errors(case, gate, require_review_safeguards=True)
assert "migrated_review_does_not_satisfy_safeguard" in errors
assert "fresh_biography_audit_not_attested" in errors
def test_fresh_unseen_reviewed_case_still_passes_safeguards() -> None:
case = _reviewed_case()
assert "prior_score_exposures" not in case
gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"]
assert case_errors(case, gate, require_review_safeguards=True) == []
def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() -> None:
@@ -90,6 +163,7 @@ def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() -
payload = deepcopy(case)
payload.update({
"adjudicator": "v3_sealed_set_post_audit_migration",
"human_review_kind": MIGRATED_HUMAN_REVIEW_KIND,
"independent_human_reviewed": True,
"frozen_before_scoring": True,
"false_minute_commitments": [
@@ -108,5 +182,7 @@ def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() -
assert "insufficient_day_precision_events" in rejected["iwao_takamoto_1925_aa_v4_holdout"]
assert "insufficient_day_precision_events" in rejected["angelina_jolie_1975_aa_v4_holdout"]
for case_id in EXPECTED_V4_INTAKE_CASE_IDS:
assert case_id not in rejected
assert all("insufficient_day_precision_events" in errors for errors in rejected.values())
assert "migrated_review_does_not_satisfy_safeguard" in rejected[case_id]
assert "fresh_biography_audit_not_attested" in rejected[case_id]
assert "insufficient_day_precision_events" not in rejected[case_id]
assert all("insufficient_day_precision_events" in errors for case_id, errors in rejected.items() if case_id not in EXPECTED_V4_INTAKE_CASE_IDS)
tests/test_minute_rectification_holdout_validator.py
+21
View File
@@ -57,6 +57,7 @@ def _add_v4_review_safeguards(manifest: dict) -> None:
for case in manifest["cases"]:
case["adjudicator"] = "independent-reviewer"
case["independent_human_reviewed"] = True
case["human_review_kind"] = "fresh_biography_audit"
case["frozen_before_scoring"] = True
case["false_minute_commitments"] = [
{
@@ -96,6 +97,26 @@ def test_v4_validator_rejects_unreviewed_or_uncommitted_cases(tmp_path: Path) ->
assert "false_minute_commitments_do_not_match_offsets" in errors
def test_v4_safeguards_reject_exposed_or_migrated_review(tmp_path: Path) -> None:
manifest = deepcopy(_manifest())
_add_v4_review_safeguards(manifest)
exposed = manifest["cases"][0]
exposed["prior_score_exposures"] = [
{"report_path": "references/real_case_calibration/minute_rectification_holdout_v3_report.json"},
]
path = tmp_path / "exposed-v4.json"
path.write_text(json.dumps(manifest), encoding="utf-8")
assert "results_already_seen_cannot_be_blind_holdout" in validate(path)["invalid_case_details"][0]["errors"]
migrated = deepcopy(_manifest())
_add_v4_review_safeguards(migrated)
migrated["cases"][0]["human_review_kind"] = "migrated_v3_record"
path = tmp_path / "migrated-v4.json"
path.write_text(json.dumps(migrated), encoding="utf-8")
errors = validate(path)["invalid_case_details"][0]["errors"]
assert "migrated_review_does_not_satisfy_safeguard" in errors
assert "fresh_biography_audit_not_attested" in errors
def test_validator_rejects_tuning_case_and_non_independent_event_source(tmp_path: Path) -> None:
manifest = deepcopy(_manifest())
case = manifest["cases"][0]