feat: sync remaining oracle evidence gaps
This commit is contained in:
732642856
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create human-review annotation packet from pilot source windows.
|
||||
|
||||
The packet is intentionally not a frozen holdout. Humans must fill final labels.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _stable_id(row: dict, index: int) -> str:
|
||||
base = f"{row['subject_id']}|{row['label_candidate']}|{row['start']}|{row['end']}|{index}"
|
||||
digest = hashlib.sha256(base.encode("utf-8")).hexdigest()[:10].upper()
|
||||
return f"DLH-PILOT-{index:03d}-{digest}"
|
||||
|
||||
|
||||
def build_packet(report_path: Path) -> dict:
|
||||
report = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
annotations = []
|
||||
for index, row in enumerate(report["windows"], start=1):
|
||||
annotations.append(
|
||||
{
|
||||
"annotation_id": _stable_id(row, index),
|
||||
"subject_id": row["subject_id"],
|
||||
"subject_name": row["name"],
|
||||
"domain": row["domain"],
|
||||
"start": row["start"],
|
||||
"end": row["end"],
|
||||
"candidate_label": row["label_candidate"],
|
||||
"final_label": None,
|
||||
"event_description": row["event_description"],
|
||||
"event_absent_assertion": row["event_absent_assertion"],
|
||||
"source_urls": row["source_urls"],
|
||||
"source_quote_or_summary": "",
|
||||
"adjudicator": "",
|
||||
"independent_human_reviewed": False,
|
||||
"frozen_before_scoring": False,
|
||||
"review_decision": "pending",
|
||||
"time_uncertainty_days": None,
|
||||
"notes": "",
|
||||
}
|
||||
)
|
||||
positive = sum(1 for row in annotations if row["candidate_label"] == "target_event")
|
||||
negative = sum(1 for row in annotations if row["candidate_label"] == "no_target_event")
|
||||
return {
|
||||
"scope": "day_level_holdout_human_annotation_packet",
|
||||
"created_at": "2026-07-19",
|
||||
"source_report": str(report_path),
|
||||
"status": "awaiting_independent_human_adjudication",
|
||||
"ready_for_blind_eval": False,
|
||||
"production_tuning_allowed": False,
|
||||
"truth_boundary": "This packet is for human labeling only. It must not be scored until final_label, adjudicator, independent_human_reviewed, and frozen_before_scoring are complete.",
|
||||
"instructions": [
|
||||
"Do not use candidate_label as final_label.",
|
||||
"For target_event, verify the event date from public sources.",
|
||||
"For no_target_event, verify absence in the interval from public sources.",
|
||||
"Freeze all labels before timing_ranker_blind_eval.py is run.",
|
||||
"Do not use existing observed control dates for tuning.",
|
||||
],
|
||||
"summary": {
|
||||
"annotation_count": len(annotations),
|
||||
"final_label_count": sum(1 for row in annotations if row["final_label"]),
|
||||
"frozen_count": sum(1 for row in annotations if row["frozen_before_scoring"]),
|
||||
"positive_candidate_count": positive,
|
||||
"negative_candidate_count": negative,
|
||||
},
|
||||
"annotations": annotations,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--source-report",
|
||||
type=Path,
|
||||
default=Path("references/real_case_calibration/day_level_holdout_v3_pilot_source_queue_report_2026_07_19.json"),
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
packet = build_packet(args.source_report)
|
||||
text = json.dumps(packet, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text, encoding="utf-8")
|
||||
print(text, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit VedAstro identity evidence without upgrading hosted truth."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _load(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _field(name: str, value, source: str, blocker: str = "") -> dict:
|
||||
ok = value not in (None, "", [], {})
|
||||
return {
|
||||
"field": name,
|
||||
"status": "complete" if ok else "blocked",
|
||||
"value": value if ok else None,
|
||||
"source": source,
|
||||
"blocker": "" if ok else blocker,
|
||||
}
|
||||
|
||||
|
||||
def build_audit(archive_path: Path, runtime_path: Path) -> dict:
|
||||
archive = _load(archive_path)
|
||||
runtime = _load(runtime_path)
|
||||
evidence = [
|
||||
_field("package_sha256", runtime.get("package_sha256") or archive.get("package_hash"), str(runtime_path)),
|
||||
_field("library_dll_sha256", runtime.get("library_dll_sha256"), str(runtime_path)),
|
||||
_field("assembly_version", runtime.get("assembly_version"), str(runtime_path)),
|
||||
_field("assembly_informational_version", runtime.get("assembly_informational_version"), str(runtime_path)),
|
||||
_field("public_method_contracts", runtime.get("public_method_contracts"), str(runtime_path)),
|
||||
_field("runtime_image_digest", runtime.get("runtime_image_digest"), str(runtime_path)),
|
||||
_field(
|
||||
"source_commit",
|
||||
archive.get("source_commit") or runtime.get("source_commit"),
|
||||
str(archive_path),
|
||||
"source commit not present in NuGet catalog/runtime contract; hosted API identity still needs upstream metadata or pinned source checkout.",
|
||||
),
|
||||
]
|
||||
complete = sum(row["status"] == "complete" for row in evidence)
|
||||
blocked = sum(row["status"] == "blocked" for row in evidence)
|
||||
return {
|
||||
"scope": "vedastro_identity_evidence_audit",
|
||||
"package": runtime["package"],
|
||||
"version": runtime["version"],
|
||||
"license": runtime["license"],
|
||||
"archive": str(archive_path),
|
||||
"runtime_contract": str(runtime_path),
|
||||
"runtime_candidate_status": "complete" if blocked <= 1 else "partial",
|
||||
"hosted_identity_status": archive["hosted_api_status"],
|
||||
"truth_upgrade_allowed": False,
|
||||
"production_tuning_allowed": False,
|
||||
"boundary": "NuGet/runtime identity can be pinned locally; hosted api.vedastro.org remains blocked without upstream build/method metadata.",
|
||||
"summary": {
|
||||
"required_field_count": len(evidence),
|
||||
"complete_count": complete,
|
||||
"blocked_count": blocked,
|
||||
"method_contract_count": len(runtime.get("public_method_contracts") or []),
|
||||
},
|
||||
"evidence": evidence,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--archive",
|
||||
type=Path,
|
||||
default=Path("references/oracle/vedastro_identity_archive_2026_07_19.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--runtime",
|
||||
type=Path,
|
||||
default=Path("references/oracle/artifacts/vedastro_nuget_1_2_0_runtime_contract.json"),
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
audit = build_audit(args.archive, args.runtime)
|
||||
text = json.dumps(audit, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text, encoding="utf-8")
|
||||
print(text, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Group Xalen Shadbala/AV deltas by component family.
|
||||
|
||||
Reads archived comparison inputs. Does not recompute astrology formulas.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.xalen_oracle_comparison import compare
|
||||
|
||||
|
||||
SECTION_CATEGORY = {
|
||||
"shadbala_components": "shadbala_formula_variant",
|
||||
"shadbala_total": "derived_total_from_component_variants",
|
||||
"ashtakavarga_bav": "ashtakavarga_table_or_contributor_variant",
|
||||
"ashtakavarga_sav": "ashtakavarga_table_or_contributor_variant",
|
||||
}
|
||||
|
||||
|
||||
def _registry_by_category(path: Path) -> dict[str, dict]:
|
||||
registry = json.loads(path.read_text(encoding="utf-8"))["registry"]
|
||||
return {row["category"]: row for row in registry}
|
||||
|
||||
|
||||
def _component_key(row: dict) -> str:
|
||||
if row["section"] == "shadbala_components":
|
||||
return row["field"].split(".", 1)[1]
|
||||
if row["section"] == "shadbala_total":
|
||||
return "total_rupa"
|
||||
return row["section"]
|
||||
|
||||
|
||||
def build_report(manifest_path: Path, xalen_path: Path, registry_path: Path) -> dict:
|
||||
comparison = compare(manifest_path, xalen_path)
|
||||
registry = _registry_by_category(registry_path)
|
||||
groups: dict[tuple[str, str], dict] = {}
|
||||
status_counts = Counter()
|
||||
section_counts = Counter()
|
||||
|
||||
for row in comparison["rows"]:
|
||||
category = SECTION_CATEGORY.get(row["section"])
|
||||
if not category:
|
||||
continue
|
||||
key = (category, _component_key(row))
|
||||
source = registry[category]
|
||||
group = groups.setdefault(
|
||||
key,
|
||||
{
|
||||
"category": category,
|
||||
"component": key[1],
|
||||
"allowed_claim": source["allowed_claim"],
|
||||
"unit_contract": source["unit_contract"],
|
||||
"required_evidence": source["next_evidence_required"],
|
||||
"closure_status": "open",
|
||||
"rows": [],
|
||||
"status_counts": defaultdict(int),
|
||||
},
|
||||
)
|
||||
group["rows"].append(row)
|
||||
group["status_counts"][row["status"]] += 1
|
||||
status_counts[(row["section"], row["status"])] += 1
|
||||
section_counts[row["section"]] += 1
|
||||
|
||||
component_groups = []
|
||||
for group in groups.values():
|
||||
group["status_counts"] = dict(sorted(group["status_counts"].items()))
|
||||
component_groups.append(group)
|
||||
component_groups.sort(key=lambda item: (item["category"], item["component"]))
|
||||
|
||||
return {
|
||||
"scope": "xalen_shadbala_av_component_delta_report",
|
||||
"source_commit": comparison["source_commit"],
|
||||
"license": comparison["license"],
|
||||
"truth_policy": "method_variant_not_majority_vote",
|
||||
"production_tuning_allowed": False,
|
||||
"boundary": "Xalen deltas identify formula/table/unit evidence needs; they are not majority-vote truth.",
|
||||
"source_artifacts": {
|
||||
"manifest": str(manifest_path),
|
||||
"xalen_raw": str(xalen_path),
|
||||
"provenance_registry": str(registry_path),
|
||||
},
|
||||
"summary": {
|
||||
"shadbala_component_rows": section_counts["shadbala_components"],
|
||||
"shadbala_component_mismatch_count": status_counts[("shadbala_components", "mismatch")],
|
||||
"shadbala_total_rows": section_counts["shadbala_total"],
|
||||
"shadbala_total_mismatch_count": status_counts[("shadbala_total", "mismatch")],
|
||||
"ashtakavarga_rows": section_counts["ashtakavarga_bav"] + section_counts["ashtakavarga_sav"],
|
||||
"ashtakavarga_mismatch_count": status_counts[("ashtakavarga_bav", "mismatch")]
|
||||
+ status_counts[("ashtakavarga_sav", "mismatch")],
|
||||
"component_group_count": len(component_groups),
|
||||
},
|
||||
"component_groups": component_groups,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--manifest",
|
||||
type=Path,
|
||||
default=Path("references/oracle/three_engine_parity_replay_manifest.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--xalen",
|
||||
type=Path,
|
||||
default=Path("references/oracle/artifacts/xalen_steve_jobs_high_rigor_raw.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--registry",
|
||||
type=Path,
|
||||
default=Path("references/oracle/shadbala_av_component_provenance_registry_2026_07_19.json"),
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
report = build_report(args.manifest, args.xalen, args.registry)
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text, encoding="utf-8")
|
||||
print(text, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user