feat(benchmark): publish oracle readiness dashboard
This commit is contained in:
732642856
@@ -203,6 +203,17 @@ python3 scripts/oracle_boundary_audit.py \
|
||||
|
||||
审计报告中的 `template_comparisons` 会列出 external-verified template 的 Dasha 起点差异、Shadbala 七曜分量/总分差异、每个分量的 Rupa 容差、单位说明和 `global_scaling_check.recommendation: reject_global_scaling`，并继续保持 `production_tuning_recommended: false`，防止用单个样本或全局倍率调生产常数。
|
||||
|
||||
公开 benchmark 看板用于长期展示能力状态、oracle readiness 和“是否可宣称全球第一”的诚实边界：
|
||||
|
||||
```bash
|
||||
python3 scripts/public_benchmark_dashboard.py \
|
||||
--oracle-file references/oracle/dasha_shadbala_oracle_cases.json \
|
||||
--format markdown \
|
||||
--output docs/benchmark/public_jyotish_benchmark_dashboard.md
|
||||
```
|
||||
|
||||
当前看板固定输出 `can_claim_global_first: false`，直到外部 oracle 样本、差异审计和长期公开 benchmark 都达到生产调参标准。
|
||||
|
||||
`full-reading` 也会输出 `ai_prompt_pack`：这是给网页/app、skill 或后端 AI 代理使用的结构化 Prompt/RAG 上下文包。它不会硬编码断语，而是携带 D1/D9/Dasha/Shadbala/Ashtakavarga 的证据快照、推荐检索文档和边界提示，要求大模型基于计算证据交叉验证，避免单一配置下结论。
|
||||
|
||||
### Prerequisites
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"scope": "public_jyotish_benchmark_dashboard",
|
||||
"schema_version": 1,
|
||||
"generated_at": "2026-06-26T09:20:53.904412+00:00",
|
||||
"summary": {
|
||||
"technique_count": 68,
|
||||
"capability_valid": true,
|
||||
"problem_count": 0,
|
||||
"status_counts": {
|
||||
"complete": 9,
|
||||
"covered": 59,
|
||||
"knowledge-only": 0,
|
||||
"missing": 0,
|
||||
"not-integrated": 0,
|
||||
"partial": 0,
|
||||
"workflow-only": 0
|
||||
}
|
||||
},
|
||||
"oracle_readiness": {
|
||||
"total_packets": 5,
|
||||
"valid_packets": 0,
|
||||
"ready_for_calibration": 0,
|
||||
"production_tuning_allowed": false,
|
||||
"all_packets_external_verified": false,
|
||||
"queue_status_counts": {
|
||||
"template_only": 5
|
||||
},
|
||||
"boundary": "Evidence packets can become review-ready only with external artifacts and filled target values. Local engine output remains rejected as an external oracle source."
|
||||
},
|
||||
"boundary_audit": {
|
||||
"template_cases": 5,
|
||||
"external_verified_template_cases": 0,
|
||||
"dasha_cases": 1,
|
||||
"longitude_cases": 1,
|
||||
"shadbala_cases": 1,
|
||||
"production_tuning_recommended": false,
|
||||
"open_items": [
|
||||
"Promote template cases to external_verified only after filling external target rows.",
|
||||
"Add multi-source Vimshottari rows with Moon longitude, ayanamsa and start-boundary settings.",
|
||||
"Add Shadbala component targets before claiming external absolute calibration."
|
||||
],
|
||||
"template_comparison_count": 0
|
||||
},
|
||||
"public_claim": {
|
||||
"can_claim_global_first": false,
|
||||
"reason": "Do not claim global first until Dasha/Shadbala external oracle packets are valid, production tuning is allowed, and public benchmark history is stable."
|
||||
},
|
||||
"global_first_gap": "Dasha/Shadbala external oracle readiness remains 0, Shadbala absolute values still need component-level external evidence, and public long-term benchmark history is not yet comparable to the strongest global open-source projects.",
|
||||
"next_actions": [
|
||||
"Fill the first external JHora/PyJHora packet under references/oracle/artifacts/pending_packets.",
|
||||
"Run oracle_evidence_validator.py until at least one packet is valid.",
|
||||
"Run oracle_boundary_audit.py to inspect Dasha/Shadbala deltas without tuning constants.",
|
||||
"Publish this dashboard after each validated sample batch."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
# Public Jyotish Benchmark Dashboard
|
||||
|
||||
Generated: `2026-06-26T09:20:54.438994+00:00`
|
||||
|
||||
## Capability Registry
|
||||
|
||||
- technique_count: `68`
|
||||
- capability_valid: `true`
|
||||
- problem_count: `0`
|
||||
|
||||
## Dasha/Shadbala Oracle Readiness
|
||||
|
||||
- total_packets: `5`
|
||||
- valid_packets: `0`
|
||||
- ready_for_calibration: `0`
|
||||
- production_tuning_allowed: `false`
|
||||
|
||||
## Boundary Audit
|
||||
|
||||
- external_verified_template_cases: `0`
|
||||
- template_comparison_count: `0`
|
||||
- production_tuning_recommended: `false`
|
||||
|
||||
## Global First Claim
|
||||
|
||||
- can_claim_global_first: `false`
|
||||
- reason: Do not claim global first until Dasha/Shadbala external oracle packets are valid, production tuning is allowed, and public benchmark history is stable.
|
||||
|
||||
## Remaining Gap
|
||||
|
||||
Dasha/Shadbala external oracle readiness remains 0, Shadbala absolute values still need component-level external evidence, and public long-term benchmark history is not yet comparable to the strongest global open-source projects.
|
||||
|
||||
## Next Actions
|
||||
|
||||
- Fill the first external JHora/PyJHora packet under references/oracle/artifacts/pending_packets.
|
||||
- Run oracle_evidence_validator.py until at least one packet is valid.
|
||||
- Run oracle_boundary_audit.py to inspect Dasha/Shadbala deltas without tuning constants.
|
||||
- Publish this dashboard after each validated sample batch.
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a public benchmark dashboard for Jyotish skill readiness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PYTHON = sys.executable
|
||||
|
||||
|
||||
def _run_json(command: list[str]) -> dict[str, Any]:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=90,
|
||||
check=False,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip())
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
|
||||
def _oracle_readiness(oracle_file: str) -> dict[str, Any]:
|
||||
queue = _run_json([PYTHON, "scripts/oracle_collection_queue.py", "--oracle-file", oracle_file, "--format", "json"])
|
||||
with tempfile.NamedTemporaryFile("w+", suffix=".json", encoding="utf-8", delete=True) as fh:
|
||||
json.dump(queue, fh, ensure_ascii=False)
|
||||
fh.flush()
|
||||
validation = _run_json([PYTHON, "scripts/oracle_evidence_validator.py", "--queue-file", fh.name])
|
||||
summary = validation["summary"]
|
||||
return {
|
||||
"total_packets": summary["total_packets"],
|
||||
"valid_packets": summary["valid_packets"],
|
||||
"ready_for_calibration": summary["ready_for_calibration"],
|
||||
"production_tuning_allowed": summary["production_tuning_allowed"],
|
||||
"all_packets_external_verified": summary["all_packets_external_verified"],
|
||||
"queue_status_counts": queue["summary"].get("by_status", {}),
|
||||
"boundary": validation["boundary"],
|
||||
}
|
||||
|
||||
|
||||
def _boundary_audit(oracle_file: str) -> dict[str, Any]:
|
||||
report = _run_json([PYTHON, "scripts/oracle_boundary_audit.py", "--oracle-file", oracle_file])
|
||||
summary = report["summary"]
|
||||
return {
|
||||
"template_cases": summary["template_cases"],
|
||||
"external_verified_template_cases": summary.get("external_verified_template_cases", 0),
|
||||
"dasha_cases": summary["dasha_cases"],
|
||||
"longitude_cases": summary["longitude_cases"],
|
||||
"shadbala_cases": summary["shadbala_cases"],
|
||||
"production_tuning_recommended": summary["production_tuning_recommended"],
|
||||
"open_items": summary["open_items"],
|
||||
"template_comparison_count": len(report.get("template_comparisons", [])),
|
||||
}
|
||||
|
||||
|
||||
def build_dashboard(oracle_file: str) -> dict[str, Any]:
|
||||
capability = _run_json([PYTHON, "scripts/audit_capabilities.py", "--mode", "validate"])
|
||||
oracle = _oracle_readiness(oracle_file)
|
||||
boundary = _boundary_audit(oracle_file)
|
||||
global_first_gap = (
|
||||
"Dasha/Shadbala external oracle readiness remains 0, Shadbala absolute values still need "
|
||||
"component-level external evidence, and public long-term benchmark history is not yet comparable "
|
||||
"to the strongest global open-source projects."
|
||||
)
|
||||
can_claim_global_first = bool(
|
||||
capability.get("valid")
|
||||
and oracle["production_tuning_allowed"]
|
||||
and boundary["production_tuning_recommended"]
|
||||
)
|
||||
return {
|
||||
"scope": "public_jyotish_benchmark_dashboard",
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"summary": {
|
||||
"technique_count": capability["technique_count"],
|
||||
"capability_valid": capability["valid"],
|
||||
"problem_count": capability["problem_count"],
|
||||
"status_counts": capability["status_counts"],
|
||||
},
|
||||
"oracle_readiness": oracle,
|
||||
"boundary_audit": boundary,
|
||||
"public_claim": {
|
||||
"can_claim_global_first": can_claim_global_first,
|
||||
"reason": (
|
||||
"Do not claim global first until Dasha/Shadbala external oracle packets are valid, "
|
||||
"production tuning is allowed, and public benchmark history is stable."
|
||||
),
|
||||
},
|
||||
"global_first_gap": global_first_gap,
|
||||
"next_actions": [
|
||||
"Fill the first external JHora/PyJHora packet under references/oracle/artifacts/pending_packets.",
|
||||
"Run oracle_evidence_validator.py until at least one packet is valid.",
|
||||
"Run oracle_boundary_audit.py to inspect Dasha/Shadbala deltas without tuning constants.",
|
||||
"Publish this dashboard after each validated sample batch.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
oracle = report["oracle_readiness"]
|
||||
boundary = report["boundary_audit"]
|
||||
claim = report["public_claim"]
|
||||
lines = [
|
||||
"# Public Jyotish Benchmark Dashboard",
|
||||
"",
|
||||
f"Generated: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Capability Registry",
|
||||
"",
|
||||
f"- technique_count: `{report['summary']['technique_count']}`",
|
||||
f"- capability_valid: `{str(report['summary']['capability_valid']).lower()}`",
|
||||
f"- problem_count: `{report['summary']['problem_count']}`",
|
||||
"",
|
||||
"## Dasha/Shadbala Oracle Readiness",
|
||||
"",
|
||||
f"- total_packets: `{oracle['total_packets']}`",
|
||||
f"- valid_packets: `{oracle['valid_packets']}`",
|
||||
f"- ready_for_calibration: `{oracle['ready_for_calibration']}`",
|
||||
f"- production_tuning_allowed: `{str(oracle['production_tuning_allowed']).lower()}`",
|
||||
"",
|
||||
"## Boundary Audit",
|
||||
"",
|
||||
f"- external_verified_template_cases: `{boundary['external_verified_template_cases']}`",
|
||||
f"- template_comparison_count: `{boundary['template_comparison_count']}`",
|
||||
f"- production_tuning_recommended: `{str(boundary['production_tuning_recommended']).lower()}`",
|
||||
"",
|
||||
"## Global First Claim",
|
||||
"",
|
||||
f"- can_claim_global_first: `{str(claim['can_claim_global_first']).lower()}`",
|
||||
f"- reason: {claim['reason']}",
|
||||
"",
|
||||
"## Remaining Gap",
|
||||
"",
|
||||
report["global_first_gap"],
|
||||
"",
|
||||
"## Next Actions",
|
||||
"",
|
||||
]
|
||||
lines.extend(f"- {item}" for item in report["next_actions"])
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate public Jyotish benchmark dashboard")
|
||||
parser.add_argument("--oracle-file", default="references/oracle/dasha_shadbala_oracle_cases.json")
|
||||
parser.add_argument("--format", choices=["json", "markdown"], default="json")
|
||||
parser.add_argument("--output", help="Optional output path")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
report = build_dashboard(args.oracle_file)
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2) if args.format == "json" else render_markdown(report)
|
||||
if args.output:
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(text, encoding="utf-8")
|
||||
print(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for the public Jyotish benchmark dashboard."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def run_dashboard(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/public_benchmark_dashboard.py",
|
||||
"--oracle-file",
|
||||
"references/oracle/dasha_shadbala_oracle_cases.json",
|
||||
*args,
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_public_benchmark_dashboard_outputs_stable_json_summary() -> None:
|
||||
completed = run_dashboard("--format", "json")
|
||||
|
||||
assert completed.returncode == 0, completed.stderr or completed.stdout
|
||||
report = json.loads(completed.stdout)
|
||||
assert report["scope"] == "public_jyotish_benchmark_dashboard"
|
||||
assert report["schema_version"] == 1
|
||||
assert report["summary"]["technique_count"] >= 60
|
||||
assert report["summary"]["capability_valid"] is True
|
||||
assert report["oracle_readiness"]["total_packets"] == 5
|
||||
assert report["oracle_readiness"]["valid_packets"] == 0
|
||||
assert report["oracle_readiness"]["ready_for_calibration"] == 0
|
||||
assert report["oracle_readiness"]["production_tuning_allowed"] is False
|
||||
assert report["boundary_audit"]["production_tuning_recommended"] is False
|
||||
assert "Dasha/Shadbala" in report["global_first_gap"]
|
||||
assert report["public_claim"]["can_claim_global_first"] is False
|
||||
assert report["public_claim"]["reason"]
|
||||
|
||||
|
||||
def test_public_benchmark_dashboard_outputs_markdown_and_can_write_file(tmp_path: Path) -> None:
|
||||
output = tmp_path / "benchmark.md"
|
||||
completed = run_dashboard("--format", "markdown", "--output", str(output))
|
||||
|
||||
assert completed.returncode == 0, completed.stderr or completed.stdout
|
||||
assert output.exists()
|
||||
markdown = output.read_text(encoding="utf-8")
|
||||
assert "# Public Jyotish Benchmark Dashboard" in markdown
|
||||
assert "Dasha/Shadbala Oracle Readiness" in markdown
|
||||
assert "Global First Claim" in markdown
|
||||
assert "can_claim_global_first: `false`" in markdown
|
||||
assert "production_tuning_allowed: `false`" in markdown
|
||||
Reference in New Issue
Block a user