docs(rectification): mark v4 intake cases as already seen, not sealed
Keep the four migrated cases in the queue. Record prior score exposures, distinguish migrated review from a fresh biography audit, and reject exposed or auto-attested cases from blind holdout use. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -10,9 +10,15 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from scripts.minute_rectification_holdout_validator import case_errors
|
||||
from scripts.minute_rectification_holdout_validator import (
|
||||
FRESH_HUMAN_REVIEW_KIND,
|
||||
case_errors,
|
||||
)
|
||||
except ModuleNotFoundError: # pragma: no cover - direct script execution
|
||||
from minute_rectification_holdout_validator import case_errors
|
||||
from minute_rectification_holdout_validator import (
|
||||
FRESH_HUMAN_REVIEW_KIND,
|
||||
case_errors,
|
||||
)
|
||||
|
||||
INTAKE_SCHEMA_VERSION = "minute-rectification-holdout-v4-intake"
|
||||
DEFAULT_INTAKE = (
|
||||
@@ -34,6 +40,13 @@ def append_case(path: Path, case: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"appended": False, "errors": ["missing_case_id"]}
|
||||
if any(isinstance(existing, dict) and existing.get("case_id") == case_id for existing in cases):
|
||||
return {"appended": False, "errors": ["duplicate_case_id"]}
|
||||
# Never auto-attest independent review. A true flag is accepted only when
|
||||
# the caller already supplied a fresh biography-level human-review kind.
|
||||
if (
|
||||
case.get("independent_human_reviewed") is True
|
||||
and case.get("human_review_kind") != FRESH_HUMAN_REVIEW_KIND
|
||||
):
|
||||
return {"appended": False, "errors": ["independent_review_requires_fresh_biography_audit"]}
|
||||
errors = case_errors(case, gate, require_review_safeguards=True)
|
||||
if errors:
|
||||
return {"appended": False, "errors": errors}
|
||||
|
||||
@@ -19,6 +19,8 @@ ALLOWED_DOMAINS = {
|
||||
"education", "relocation", "relationship", "career", "finance", "health_pressure",
|
||||
}
|
||||
ALLOWED_PRECISIONS = {"year", "month", "day"}
|
||||
FRESH_HUMAN_REVIEW_KIND = "fresh_biography_audit"
|
||||
MIGRATED_HUMAN_REVIEW_KIND = "migrated_v3_record"
|
||||
|
||||
|
||||
def _is_public_url(value: Any) -> bool:
|
||||
@@ -143,6 +145,13 @@ def _case_errors(case: Any, gate: dict[str, Any]) -> list[str]:
|
||||
return errors
|
||||
|
||||
|
||||
def _prior_score_exposures(case: Any) -> list[Any]:
|
||||
raw = case.get("prior_score_exposures") if isinstance(case, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [item for item in raw if item]
|
||||
|
||||
|
||||
def _review_safeguard_errors(case: Any, gate: dict[str, Any]) -> list[str]:
|
||||
"""Validate safeguards required for newly admitted v4 holdout cases."""
|
||||
if not isinstance(case, dict):
|
||||
@@ -150,6 +159,13 @@ def _review_safeguard_errors(case: Any, gate: dict[str, Any]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not isinstance(case.get("adjudicator"), str) or not case.get("adjudicator", "").strip():
|
||||
errors.append("missing_independent_adjudicator")
|
||||
if _prior_score_exposures(case):
|
||||
errors.append("results_already_seen_cannot_be_blind_holdout")
|
||||
review_kind = case.get("human_review_kind")
|
||||
if review_kind == MIGRATED_HUMAN_REVIEW_KIND:
|
||||
errors.append("migrated_review_does_not_satisfy_safeguard")
|
||||
if review_kind != FRESH_HUMAN_REVIEW_KIND:
|
||||
errors.append("fresh_biography_audit_not_attested")
|
||||
if case.get("independent_human_reviewed") is not True:
|
||||
errors.append("independent_review_not_attested")
|
||||
if case.get("frozen_before_scoring") is not True:
|
||||
|
||||
Reference in New Issue
Block a user