Expose PyJHora oracle assets in benchmark dashboard
This commit is contained in:
@@ -1,13 +1,12 @@
|
||||
{
|
||||
"scope": "pyjhora_oracle_artifact_manifest",
|
||||
"generated_at": "2026-06-27T06:25:10.958318+00:00",
|
||||
"artifact_count": 9,
|
||||
"generated_at": "2026-06-27T06:34:04.627003+00:00",
|
||||
"artifact_count": 8,
|
||||
"packet_count": 8,
|
||||
"artifacts": [
|
||||
"pyjhora_extreme_latitude_kp_shadbala_stdout_20260627.txt",
|
||||
"pyjhora_redacted_place_shadbala_raman_stdout_20260627.txt",
|
||||
"pyjhora_historical_epoch_dasha_stdout_20260627.txt",
|
||||
"pyjhora_oracle_artifact_manifest.json",
|
||||
"pyjhora_steve_jobs_dasha_stdout_20260627.txt",
|
||||
"pyjhora_steve_jobs_shadbala_lahiri_stdout_20260627.txt",
|
||||
"pyjhora_steve_jobs_varshaphala_1984_lahiri_stdout_20260627.txt",
|
||||
@@ -26,11 +25,10 @@
|
||||
],
|
||||
"fronts": {
|
||||
"dasha": {
|
||||
"artifact_count": 4,
|
||||
"artifact_count": 3,
|
||||
"packet_count": 3,
|
||||
"artifacts": [
|
||||
"pyjhora_historical_epoch_dasha_stdout_20260627.txt",
|
||||
"pyjhora_oracle_artifact_manifest.json",
|
||||
"pyjhora_steve_jobs_dasha_stdout_20260627.txt",
|
||||
"pyjhora_user_REDACTED_YEAR_dasha_stdout_20260627.txt"
|
||||
],
|
||||
|
||||
@@ -32,7 +32,7 @@ def _relative(path: Path) -> str:
|
||||
def build_manifest() -> dict[str, Any]:
|
||||
artifacts = sorted(
|
||||
path for path in ARTIFACTS_DIR.glob("pyjhora_*")
|
||||
if path.is_file()
|
||||
if path.is_file() and path.name != OUTPUT_PATH.name
|
||||
)
|
||||
pending_packets = sorted(
|
||||
path for path in PENDING_DIR.glob("*pyjhora_20260627.json")
|
||||
|
||||
@@ -85,16 +85,23 @@ def _boundary_audit(oracle_file: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _pyjhora_blackbox_assets() -> dict[str, Any]:
|
||||
return _run_json([PYTHON, "scripts/generate_pyjhora_oracle_artifact_manifest.py"])
|
||||
|
||||
|
||||
def build_dashboard(oracle_file: str) -> dict[str, Any]:
|
||||
capability = _run_json([PYTHON, "scripts/audit_capabilities.py", "--mode", "validate"])
|
||||
oracle = _oracle_readiness(oracle_file)
|
||||
boundary = _boundary_audit(oracle_file)
|
||||
dasha = _dasha_readiness(oracle_file)
|
||||
shadbala = _shadbala_readiness(oracle_file)
|
||||
pyjhora_assets = _pyjhora_blackbox_assets()
|
||||
global_first_gap = (
|
||||
f"Dasha-only external oracle readiness is {dasha['valid_dasha_packets']}/"
|
||||
f"{dasha['total_dasha_packets']}; Shadbala external absolute-value readiness is "
|
||||
f"{shadbala['external_verified_shadbala_tasks']}/{shadbala['shadbala_task_count']}; "
|
||||
f"PyJHora black-box assets are {pyjhora_assets['artifact_count']} artifacts / "
|
||||
f"{pyjhora_assets['packet_count']} packets; "
|
||||
"public long-term benchmark history is not yet comparable to the strongest "
|
||||
"global open-source projects."
|
||||
)
|
||||
@@ -116,6 +123,7 @@ def build_dashboard(oracle_file: str) -> dict[str, Any]:
|
||||
"oracle_readiness": oracle,
|
||||
"dasha_oracle_readiness": dasha,
|
||||
"shadbala_oracle_readiness": shadbala,
|
||||
"pyjhora_blackbox_assets": pyjhora_assets,
|
||||
"boundary_audit": boundary,
|
||||
"public_claim": {
|
||||
"can_claim_global_first": can_claim_global_first,
|
||||
@@ -162,6 +170,14 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
f"- external_verified_shadbala_tasks: `{shadbala['external_verified_shadbala_tasks']}`",
|
||||
f"- shadbala_task_count: `{shadbala['shadbala_task_count']}`",
|
||||
"",
|
||||
"## PyJHora Black-Box Assets",
|
||||
"",
|
||||
f"- artifact_count: `{report['pyjhora_blackbox_assets']['artifact_count']}`",
|
||||
f"- packet_count: `{report['pyjhora_blackbox_assets']['packet_count']}`",
|
||||
f"- dasha_artifacts: `{report['pyjhora_blackbox_assets']['fronts']['dasha']['artifact_count']}`",
|
||||
f"- shadbala_artifacts: `{report['pyjhora_blackbox_assets']['fronts']['shadbala']['artifact_count']}`",
|
||||
f"- tajika_sahams_artifacts: `{report['pyjhora_blackbox_assets']['fronts']['tajika_sahams']['artifact_count']}`",
|
||||
"",
|
||||
"## Boundary Audit",
|
||||
"",
|
||||
f"- external_verified_template_cases: `{boundary['external_verified_template_cases']}`",
|
||||
|
||||
@@ -45,8 +45,12 @@ def test_public_benchmark_dashboard_outputs_stable_json_summary() -> None:
|
||||
assert report["dasha_oracle_readiness"]["valid_dasha_packets"] == 3
|
||||
assert report["dasha_oracle_readiness"]["total_dasha_packets"] == 3
|
||||
assert report["boundary_audit"]["production_tuning_recommended"] is False
|
||||
assert report["pyjhora_blackbox_assets"]["artifact_count"] >= 8
|
||||
assert report["pyjhora_blackbox_assets"]["packet_count"] >= 8
|
||||
assert report["pyjhora_blackbox_assets"]["fronts"]["tajika_sahams"]["artifact_count"] >= 1
|
||||
assert "Dasha-only external oracle readiness is 3/3" in report["global_first_gap"]
|
||||
assert "Shadbala external absolute-value readiness is 4/4" in report["global_first_gap"]
|
||||
assert "PyJHora black-box assets are 8 artifacts / 8 packets" in report["global_first_gap"]
|
||||
assert report["public_claim"]["can_claim_global_first"] is False
|
||||
assert report["public_claim"]["reason"]
|
||||
|
||||
@@ -60,6 +64,7 @@ def test_public_benchmark_dashboard_outputs_markdown_and_can_write_file(tmp_path
|
||||
markdown = output.read_text(encoding="utf-8")
|
||||
assert "# Public Jyotish Benchmark Dashboard" in markdown
|
||||
assert "Dasha/Shadbala Oracle Readiness" in markdown
|
||||
assert "PyJHora Black-Box Assets" in markdown
|
||||
assert "Global First Claim" in markdown
|
||||
assert "can_claim_global_first: `false`" in markdown
|
||||
assert "production_tuning_allowed: `false`" in markdown
|
||||
|
||||
@@ -35,3 +35,4 @@ def test_pyjhora_oracle_artifact_manifest_reports_current_blackbox_assets() -> N
|
||||
assert manifest_data["scope"] == "pyjhora_oracle_artifact_manifest"
|
||||
assert "pyjhora_steve_jobs_dasha_stdout_20260627.txt" in manifest_data["artifacts"]
|
||||
assert "external_template_steve_jobs_dasha_lahiri_pyjhora_20260627.json" in manifest_data["pending_packets"]
|
||||
assert "pyjhora_oracle_artifact_manifest.json" not in manifest_data["artifacts"]
|
||||
|
||||
Reference in New Issue
Block a user