feat: sync skill truth audit safeguards
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
AUDIT = ROOT / "references" / "cross_project_contract" / "cross_repo_strategy_diff_audit_2026_07_19.json"
|
||||
|
||||
|
||||
def test_strategy_diff_audit_blocks_blind_overwrite() -> None:
|
||||
data = json.loads(AUDIT.read_text(encoding="utf-8"))
|
||||
|
||||
assert data["scope"] == "cross_repo_strategy_diff_audit"
|
||||
assert data["status"] == "do_not_blind_overwrite"
|
||||
assert data["summary"]["diff_count"] == 7
|
||||
|
||||
|
||||
def test_strategy_diff_audit_tracks_vedastro_and_holdout_policy_diffs() -> None:
|
||||
data = json.loads(AUDIT.read_text(encoding="utf-8"))
|
||||
items = {item["path"]: item for item in data["items"]}
|
||||
|
||||
assert "scripts/day_level_holdout_validator.py" in items
|
||||
assert "commercial_retains_observational_validation_mode_for_product_intake" in items[
|
||||
"scripts/day_level_holdout_validator.py"
|
||||
]["notes"]
|
||||
assert "scripts/vedastro_identity_archive.py" in items
|
||||
assert "research_has_stricter_self_host_truth_upgrade_gate" in items[
|
||||
"scripts/vedastro_identity_archive.py"
|
||||
]["notes"]
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
QUEUE = ROOT / "references" / "real_case_calibration" / "day_level_holdout_v3_pilot_source_queue_2026_07_19.json"
|
||||
|
||||
|
||||
def test_pilot_source_queue_has_three_public_subjects_but_no_truth_upgrade() -> None:
|
||||
data = json.loads(QUEUE.read_text(encoding="utf-8"))
|
||||
|
||||
assert data["status"] == "candidate_requires_adjudication"
|
||||
assert data["production_tuning_allowed"] is False
|
||||
assert data["candidate_count"] == 3
|
||||
assert "not holdout annotations" in data["truth_boundary"]
|
||||
assert "independent adjudicator" in data["required_next_step"]
|
||||
|
||||
subjects = {subject["subject_id"]: subject for subject in data["subjects"]}
|
||||
assert set(subjects) == {"steve_jobs", "barack_obama", "albert_einstein"}
|
||||
for subject in subjects.values():
|
||||
assert subject["candidate_positive_event"]["status"] == "candidate_requires_adjudication"
|
||||
assert len(subject["candidate_negative_windows"]) == 2
|
||||
assert all(window["status"] == "candidate_requires_adjudication" for window in subject["candidate_negative_windows"])
|
||||
assert all(url.startswith("https://") for url in subject["candidate_positive_event"]["source_urls"])
|
||||
|
||||
|
||||
def test_pilot_source_queue_is_not_misrepresented_as_day_level_holdout_manifest() -> None:
|
||||
data = json.loads(QUEUE.read_text(encoding="utf-8"))
|
||||
|
||||
assert "annotations" not in data
|
||||
assert data["scope"] == "day_level_holdout_v3_pilot_source_queue"
|
||||
assert data["status"] != "ready_for_blind_replay"
|
||||
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.day_level_holdout_pilot_source_queue_report import build_report
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
QUEUE = ROOT / "references/real_case_calibration/day_level_holdout_v3_pilot_source_queue_2026_07_19.json"
|
||||
|
||||
|
||||
def test_pilot_source_queue_expands_to_nine_unscored_windows() -> None:
|
||||
report = build_report(QUEUE)
|
||||
|
||||
assert report["scope"] == "day_level_holdout_pilot_source_queue_report"
|
||||
assert report["status"] == "awaiting_independent_human_labels"
|
||||
assert report["production_tuning_allowed"] is False
|
||||
assert report["blind_scoring_allowed"] is False
|
||||
assert report["summary"] == {
|
||||
"subject_count": 3,
|
||||
"window_count": 9,
|
||||
"positive_candidate_count": 3,
|
||||
"negative_candidate_count": 6,
|
||||
"ready_annotation_count": 0,
|
||||
"blocked_annotation_count": 9,
|
||||
}
|
||||
|
||||
|
||||
def test_pilot_source_queue_windows_keep_public_source_and_boundary() -> None:
|
||||
report = build_report(QUEUE)
|
||||
|
||||
for window in report["windows"]:
|
||||
assert window["claim_status"] == "candidate_not_label"
|
||||
assert window["required_next_step"] == "independent_human_adjudication"
|
||||
assert window["scoring_status"] == "blocked_not_frozen"
|
||||
if window["label_candidate"] == "target_event":
|
||||
assert window["source_urls"]
|
||||
assert all(url.startswith("https://") for url in window["source_urls"])
|
||||
else:
|
||||
assert window["event_absent_assertion"]
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL = ROOT / "SKILL.md"
|
||||
|
||||
|
||||
def test_skill_text_requires_effective_capability_view_before_claims() -> None:
|
||||
text = SKILL.read_text(encoding="utf-8")
|
||||
assert "references/oracle/effective_skill_capability_view_2026_07_19.json" in text
|
||||
assert "references/oracle/skill_truth_overlay_2026_07_19.json" in text
|
||||
assert "不得直接把 `references/technique_registry.json` 的旧 `covered` 当作完整闭环" in text
|
||||
|
||||
|
||||
def test_skill_text_does_not_overclaim_kp_muhurta_sahams() -> None:
|
||||
text = SKILL.read_text(encoding="utf-8")
|
||||
assert "KP/Muhurta/Gochara/Sahams/Sphuta/Tajika等高阶分支必须按 skill truth overlay 降级使用" in text
|
||||
assert "KP系统 | reference-only / partial" in text
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.technique_promotion_audit_vimsopaka_avastha import build_audit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_vimsopaka_avastha_audit_corrects_call_status() -> None:
|
||||
audit = build_audit(ROOT)
|
||||
assert audit["scope"] == "technique_promotion_audit_vimsopaka_avastha"
|
||||
assert audit["truth_policy"] == "runtime_presence_not_oracle_closure"
|
||||
assert audit["production_tuning_allowed"] is False
|
||||
statuses = {item["technique_id"]: item["current_call_status"] for item in audit["items"]}
|
||||
assert statuses["vimsopaka_bala"] == "formally_called_in_full_reading"
|
||||
assert statuses["avastha_states"] == "formally_called_via_deep_varga_avastha_endpoint"
|
||||
|
||||
|
||||
def test_vimsopaka_avastha_audit_records_entrypoints_and_boundaries() -> None:
|
||||
audit = build_audit(ROOT)
|
||||
for item in audit["items"]:
|
||||
assert item["main_artifacts"]
|
||||
assert item["entrypoints"]
|
||||
assert item["next_action"] in {
|
||||
"add_formula_source_and_oracle_packet",
|
||||
"add_display_contract_and_source_oracle_packet",
|
||||
}
|
||||
assert item["claim_boundary"]
|
||||
assert item["reuse_decision"] == "do_not_duplicate_runtime"
|
||||
|
||||
|
||||
def test_vimsopaka_avastha_audit_artifact_exists() -> None:
|
||||
artifact = ROOT / "references/oracle/technique_promotion_audit_vimsopaka_avastha_2026_07_19.json"
|
||||
data = json.loads(artifact.read_text(encoding="utf-8"))
|
||||
assert data["summary"] == {
|
||||
"items_checked": 2,
|
||||
"formally_called_count": 2,
|
||||
"duplicate_runtime_needed": 0,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user