docs(rectification): mark v4 intake cases as already seen, not sealed

Keep the four migrated cases in the queue. Record prior score exposures, distinguish migrated review from a fresh biography audit, and reject exposed or auto-attested cases from blind holdout use.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-29 01:33:20 +08:00
parent dc2879020a
commit a3f4e7128b
7 changed files with 214 additions and 18 deletions
+15 -2
View File
@@ -10,9 +10,15 @@ from pathlib import Path
from typing import Any
try:
from scripts.minute_rectification_holdout_validator import case_errors
from scripts.minute_rectification_holdout_validator import (
FRESH_HUMAN_REVIEW_KIND,
case_errors,
)
except ModuleNotFoundError: # pragma: no cover - direct script execution
from minute_rectification_holdout_validator import case_errors
from minute_rectification_holdout_validator import (
FRESH_HUMAN_REVIEW_KIND,
case_errors,
)
INTAKE_SCHEMA_VERSION = "minute-rectification-holdout-v4-intake"
DEFAULT_INTAKE = (
@@ -34,6 +40,13 @@ def append_case(path: Path, case: dict[str, Any]) -> dict[str, Any]:
return {"appended": False, "errors": ["missing_case_id"]}
if any(isinstance(existing, dict) and existing.get("case_id") == case_id for existing in cases):
return {"appended": False, "errors": ["duplicate_case_id"]}
# Never auto-attest independent review. A true flag is accepted only when
# the caller already supplied a fresh biography-level human-review kind.
if (
case.get("independent_human_reviewed") is True
and case.get("human_review_kind") != FRESH_HUMAN_REVIEW_KIND
):
return {"appended": False, "errors": ["independent_review_requires_fresh_biography_audit"]}
errors = case_errors(case, gate, require_review_safeguards=True)
if errors:
return {"appended": False, "errors": errors}
@@ -19,6 +19,8 @@ ALLOWED_DOMAINS = {
"education", "relocation", "relationship", "career", "finance", "health_pressure",
}
ALLOWED_PRECISIONS = {"year", "month", "day"}
FRESH_HUMAN_REVIEW_KIND = "fresh_biography_audit"
MIGRATED_HUMAN_REVIEW_KIND = "migrated_v3_record"
def _is_public_url(value: Any) -> bool:
@@ -143,6 +145,13 @@ def _case_errors(case: Any, gate: dict[str, Any]) -> list[str]:
return errors
def _prior_score_exposures(case: Any) -> list[Any]:
raw = case.get("prior_score_exposures") if isinstance(case, dict) else None
if not isinstance(raw, list):
return []
return [item for item in raw if item]
def _review_safeguard_errors(case: Any, gate: dict[str, Any]) -> list[str]:
"""Validate safeguards required for newly admitted v4 holdout cases."""
if not isinstance(case, dict):
@@ -150,6 +159,13 @@ def _review_safeguard_errors(case: Any, gate: dict[str, Any]) -> list[str]:
errors: list[str] = []
if not isinstance(case.get("adjudicator"), str) or not case.get("adjudicator", "").strip():
errors.append("missing_independent_adjudicator")
if _prior_score_exposures(case):
errors.append("results_already_seen_cannot_be_blind_holdout")
review_kind = case.get("human_review_kind")
if review_kind == MIGRATED_HUMAN_REVIEW_KIND:
errors.append("migrated_review_does_not_satisfy_safeguard")
if review_kind != FRESH_HUMAN_REVIEW_KIND:
errors.append("fresh_biography_audit_not_attested")
if case.get("independent_human_reviewed") is not True:
errors.append("independent_review_not_attested")
if case.get("frozen_before_scoring") is not True: