feat: sync skill truth audit safeguards
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Expand pilot day-level holdout source candidates into unscored windows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def build_report(queue_path: Path) -> dict:
|
||||
queue = json.loads(queue_path.read_text(encoding="utf-8"))
|
||||
windows = []
|
||||
for subject in queue["subjects"]:
|
||||
positive = subject["candidate_positive_event"]
|
||||
windows.append(
|
||||
{
|
||||
"subject_id": subject["subject_id"],
|
||||
"name": subject["name"],
|
||||
"domain": subject["domain"],
|
||||
"label_candidate": positive["label"],
|
||||
"start": positive["start"],
|
||||
"end": positive["end"],
|
||||
"event_description": positive["event_description"],
|
||||
"event_absent_assertion": "",
|
||||
"source_urls": positive.get("source_urls", []),
|
||||
"claim_status": "candidate_not_label",
|
||||
"required_next_step": "independent_human_adjudication",
|
||||
"scoring_status": "blocked_not_frozen",
|
||||
}
|
||||
)
|
||||
for item in subject["candidate_negative_windows"]:
|
||||
windows.append(
|
||||
{
|
||||
"subject_id": subject["subject_id"],
|
||||
"name": subject["name"],
|
||||
"domain": subject["domain"],
|
||||
"label_candidate": "no_target_event",
|
||||
"start": item["start"],
|
||||
"end": item["end"],
|
||||
"event_description": "",
|
||||
"event_absent_assertion": item["event_absent_assertion"],
|
||||
"source_urls": positive.get("source_urls", []),
|
||||
"claim_status": "candidate_not_label",
|
||||
"required_next_step": "independent_human_adjudication",
|
||||
"scoring_status": "blocked_not_frozen",
|
||||
}
|
||||
)
|
||||
|
||||
positive_count = sum(1 for row in windows if row["label_candidate"] == "target_event")
|
||||
negative_count = sum(1 for row in windows if row["label_candidate"] == "no_target_event")
|
||||
return {
|
||||
"scope": "day_level_holdout_pilot_source_queue_report",
|
||||
"source_queue": str(queue_path),
|
||||
"status": "awaiting_independent_human_labels",
|
||||
"production_tuning_allowed": False,
|
||||
"blind_scoring_allowed": False,
|
||||
"truth_boundary": queue["truth_boundary"],
|
||||
"summary": {
|
||||
"subject_count": len(queue["subjects"]),
|
||||
"window_count": len(windows),
|
||||
"positive_candidate_count": positive_count,
|
||||
"negative_candidate_count": negative_count,
|
||||
"ready_annotation_count": 0,
|
||||
"blocked_annotation_count": len(windows),
|
||||
},
|
||||
"windows": windows,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--queue",
|
||||
type=Path,
|
||||
default=Path("references/real_case_calibration/day_level_holdout_v3_pilot_source_queue_2026_07_19.json"),
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
report = build_report(args.queue)
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text, encoding="utf-8")
|
||||
print(text, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit Vimsopaka/Avastha fragments against current runtime entrypoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _contains(path: Path, tokens: list[str]) -> bool:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
return all(token in text for token in tokens)
|
||||
|
||||
|
||||
def build_audit(root: Path) -> dict:
|
||||
full_reading = root / "scripts/full_reading.py"
|
||||
api_server = root / "scripts/jyotish_api_server.py"
|
||||
deep = root / "scripts/deep_varga_avastha.py"
|
||||
skill_map = root / "jyotish-app/skill-map.js"
|
||||
tests = root / "tests/test_cli_smoke.py"
|
||||
deep_tests = root / "tests/test_deep_varga_avastha.py"
|
||||
|
||||
vimsopaka_called = _contains(tests, ["vimsopaka_semantic_summary", "modules", "vimsopaka"])
|
||||
avastha_called = _contains(api_server, ["/api/deep_varga_avastha", "_compute_deep_varga_avastha"]) and _contains(
|
||||
deep, ["AvasthaCalculator", "DivisionalChartsCalculator"]
|
||||
)
|
||||
|
||||
items = [
|
||||
{
|
||||
"technique_id": "vimsopaka_bala",
|
||||
"current_call_status": "formally_called_in_full_reading" if vimsopaka_called else "partial",
|
||||
"main_artifacts": ["scripts/full_reading.py", "tests/test_cli_smoke.py"],
|
||||
"historical_artifacts": ["skills/jyotish-engine-modules/scripts/vimsopaka_calculator.py"],
|
||||
"entrypoints": ["full_reading prompt pack", "vimsopaka_semantic_summary"],
|
||||
"reuse_decision": "do_not_duplicate_runtime",
|
||||
"next_action": "add_formula_source_and_oracle_packet",
|
||||
"claim_boundary": "Runtime presence is real, but Vimsopaka weight table/source/oracle closure is still separate.",
|
||||
},
|
||||
{
|
||||
"technique_id": "avastha_states",
|
||||
"current_call_status": "formally_called_via_deep_varga_avastha_endpoint" if avastha_called else "partial",
|
||||
"main_artifacts": [
|
||||
"scripts/deep_varga_avastha.py",
|
||||
"scripts/jyotish_api_server.py",
|
||||
"jyotish-app/skill-map.js",
|
||||
"tests/test_deep_varga_avastha.py",
|
||||
],
|
||||
"historical_artifacts": ["skills/jyotish-engine-modules/scripts/avastha_calculator.py"],
|
||||
"entrypoints": ["/api/deep_varga_avastha", "jyotish-app skill-map deepVargaAvastha"],
|
||||
"reuse_decision": "do_not_duplicate_runtime",
|
||||
"next_action": "add_display_contract_and_source_oracle_packet",
|
||||
"claim_boundary": "Avastha is endpoint-visible, but formula variants and interpretive claim level still need source/oracle packet.",
|
||||
},
|
||||
]
|
||||
return {
|
||||
"scope": "technique_promotion_audit_vimsopaka_avastha",
|
||||
"created_at": "2026-07-19",
|
||||
"truth_policy": "runtime_presence_not_oracle_closure",
|
||||
"production_tuning_allowed": False,
|
||||
"summary": {
|
||||
"items_checked": len(items),
|
||||
"formally_called_count": sum("formally_called" in item["current_call_status"] for item in items),
|
||||
"duplicate_runtime_needed": 0,
|
||||
},
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path("."))
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
audit = build_audit(args.root)
|
||||
text = json.dumps(audit, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text, encoding="utf-8")
|
||||
print(text, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user