diff --git a/README.md b/README.md index aaa0d7ac..39f113d9 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,17 @@ python3 scripts/oracle_boundary_audit.py \ 审计报告中的 `template_comparisons` 会列出 external-verified template 的 Dasha 起点差异、Shadbala 七曜分量/总分差异、每个分量的 Rupa 容差、单位说明和 `global_scaling_check.recommendation: reject_global_scaling`,并继续保持 `production_tuning_recommended: false`,防止用单个样本或全局倍率调生产常数。 +公开 benchmark 看板用于长期展示能力状态、oracle readiness 和“是否可宣称全球第一”的诚实边界: + +```bash +python3 scripts/public_benchmark_dashboard.py \ + --oracle-file references/oracle/dasha_shadbala_oracle_cases.json \ + --format markdown \ + --output docs/benchmark/public_jyotish_benchmark_dashboard.md +``` + +当前看板固定输出 `can_claim_global_first: false`,直到外部 oracle 样本、差异审计和长期公开 benchmark 都达到生产调参标准。 + `full-reading` 也会输出 `ai_prompt_pack`:这是给网页/app、skill 或后端 AI 代理使用的结构化 Prompt/RAG 上下文包。它不会硬编码断语,而是携带 D1/D9/Dasha/Shadbala/Ashtakavarga 的证据快照、推荐检索文档和边界提示,要求大模型基于计算证据交叉验证,避免单一配置下结论。 ### Prerequisites diff --git a/docs/benchmark/public_jyotish_benchmark_dashboard.json b/docs/benchmark/public_jyotish_benchmark_dashboard.json new file mode 100644 index 00000000..268444d1 --- /dev/null +++ b/docs/benchmark/public_jyotish_benchmark_dashboard.json @@ -0,0 +1,55 @@ +{ + "scope": "public_jyotish_benchmark_dashboard", + "schema_version": 1, + "generated_at": "2026-06-26T09:20:53.904412+00:00", + "summary": { + "technique_count": 68, + "capability_valid": true, + "problem_count": 0, + "status_counts": { + "complete": 9, + "covered": 59, + "knowledge-only": 0, + "missing": 0, + "not-integrated": 0, + "partial": 0, + "workflow-only": 0 + } + }, + "oracle_readiness": { + "total_packets": 5, + "valid_packets": 0, + "ready_for_calibration": 0, + "production_tuning_allowed": false, + "all_packets_external_verified": false, + "queue_status_counts": { + "template_only": 5 + }, + "boundary": "Evidence packets can become review-ready only with external artifacts and filled target values. Local engine output remains rejected as an external oracle source." + }, + "boundary_audit": { + "template_cases": 5, + "external_verified_template_cases": 0, + "dasha_cases": 1, + "longitude_cases": 1, + "shadbala_cases": 1, + "production_tuning_recommended": false, + "open_items": [ + "Promote template cases to external_verified only after filling external target rows.", + "Add multi-source Vimshottari rows with Moon longitude, ayanamsa and start-boundary settings.", + "Add Shadbala component targets before claiming external absolute calibration." + ], + "template_comparison_count": 0 + }, + "public_claim": { + "can_claim_global_first": false, + "reason": "Do not claim global first until Dasha/Shadbala external oracle packets are valid, production tuning is allowed, and public benchmark history is stable." + }, + "global_first_gap": "Dasha/Shadbala external oracle readiness remains 0, Shadbala absolute values still need component-level external evidence, and public long-term benchmark history is not yet comparable to the strongest global open-source projects.", + "next_actions": [ + "Fill the first external JHora/PyJHora packet under references/oracle/artifacts/pending_packets.", + "Run oracle_evidence_validator.py until at least one packet is valid.", + "Run oracle_boundary_audit.py to inspect Dasha/Shadbala deltas without tuning constants.", + "Publish this dashboard after each validated sample batch." + ] +} \ No newline at end of file diff --git a/docs/benchmark/public_jyotish_benchmark_dashboard.md b/docs/benchmark/public_jyotish_benchmark_dashboard.md new file mode 100644 index 00000000..c5584703 --- /dev/null +++ b/docs/benchmark/public_jyotish_benchmark_dashboard.md @@ -0,0 +1,38 @@ +# Public Jyotish Benchmark Dashboard + +Generated: `2026-06-26T09:20:54.438994+00:00` + +## Capability Registry + +- technique_count: `68` +- capability_valid: `true` +- problem_count: `0` + +## Dasha/Shadbala Oracle Readiness + +- total_packets: `5` +- valid_packets: `0` +- ready_for_calibration: `0` +- production_tuning_allowed: `false` + +## Boundary Audit + +- external_verified_template_cases: `0` +- template_comparison_count: `0` +- production_tuning_recommended: `false` + +## Global First Claim + +- can_claim_global_first: `false` +- reason: Do not claim global first until Dasha/Shadbala external oracle packets are valid, production tuning is allowed, and public benchmark history is stable. + +## Remaining Gap + +Dasha/Shadbala external oracle readiness remains 0, Shadbala absolute values still need component-level external evidence, and public long-term benchmark history is not yet comparable to the strongest global open-source projects. + +## Next Actions + +- Fill the first external JHora/PyJHora packet under references/oracle/artifacts/pending_packets. +- Run oracle_evidence_validator.py until at least one packet is valid. +- Run oracle_boundary_audit.py to inspect Dasha/Shadbala deltas without tuning constants. +- Publish this dashboard after each validated sample batch. diff --git a/scripts/public_benchmark_dashboard.py b/scripts/public_benchmark_dashboard.py new file mode 100644 index 00000000..7cc207c8 --- /dev/null +++ b/scripts/public_benchmark_dashboard.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Generate a public benchmark dashboard for Jyotish skill readiness.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +PYTHON = sys.executable + + +def _run_json(command: list[str]) -> dict[str, Any]: + completed = subprocess.run( + command, + cwd=ROOT, + text=True, + capture_output=True, + timeout=90, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError(completed.stderr.strip() or completed.stdout.strip()) + return json.loads(completed.stdout) + + +def _oracle_readiness(oracle_file: str) -> dict[str, Any]: + queue = _run_json([PYTHON, "scripts/oracle_collection_queue.py", "--oracle-file", oracle_file, "--format", "json"]) + with tempfile.NamedTemporaryFile("w+", suffix=".json", encoding="utf-8", delete=True) as fh: + json.dump(queue, fh, ensure_ascii=False) + fh.flush() + validation = _run_json([PYTHON, "scripts/oracle_evidence_validator.py", "--queue-file", fh.name]) + summary = validation["summary"] + return { + "total_packets": summary["total_packets"], + "valid_packets": summary["valid_packets"], + "ready_for_calibration": summary["ready_for_calibration"], + "production_tuning_allowed": summary["production_tuning_allowed"], + "all_packets_external_verified": summary["all_packets_external_verified"], + "queue_status_counts": queue["summary"].get("by_status", {}), + "boundary": validation["boundary"], + } + + +def _boundary_audit(oracle_file: str) -> dict[str, Any]: + report = _run_json([PYTHON, "scripts/oracle_boundary_audit.py", "--oracle-file", oracle_file]) + summary = report["summary"] + return { + "template_cases": summary["template_cases"], + "external_verified_template_cases": summary.get("external_verified_template_cases", 0), + "dasha_cases": summary["dasha_cases"], + "longitude_cases": summary["longitude_cases"], + "shadbala_cases": summary["shadbala_cases"], + "production_tuning_recommended": summary["production_tuning_recommended"], + "open_items": summary["open_items"], + "template_comparison_count": len(report.get("template_comparisons", [])), + } + + +def build_dashboard(oracle_file: str) -> dict[str, Any]: + capability = _run_json([PYTHON, "scripts/audit_capabilities.py", "--mode", "validate"]) + oracle = _oracle_readiness(oracle_file) + boundary = _boundary_audit(oracle_file) + global_first_gap = ( + "Dasha/Shadbala external oracle readiness remains 0, Shadbala absolute values still need " + "component-level external evidence, and public long-term benchmark history is not yet comparable " + "to the strongest global open-source projects." + ) + can_claim_global_first = bool( + capability.get("valid") + and oracle["production_tuning_allowed"] + and boundary["production_tuning_recommended"] + ) + return { + "scope": "public_jyotish_benchmark_dashboard", + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "summary": { + "technique_count": capability["technique_count"], + "capability_valid": capability["valid"], + "problem_count": capability["problem_count"], + "status_counts": capability["status_counts"], + }, + "oracle_readiness": oracle, + "boundary_audit": boundary, + "public_claim": { + "can_claim_global_first": can_claim_global_first, + "reason": ( + "Do not claim global first until Dasha/Shadbala external oracle packets are valid, " + "production tuning is allowed, and public benchmark history is stable." + ), + }, + "global_first_gap": global_first_gap, + "next_actions": [ + "Fill the first external JHora/PyJHora packet under references/oracle/artifacts/pending_packets.", + "Run oracle_evidence_validator.py until at least one packet is valid.", + "Run oracle_boundary_audit.py to inspect Dasha/Shadbala deltas without tuning constants.", + "Publish this dashboard after each validated sample batch.", + ], + } + + +def render_markdown(report: dict[str, Any]) -> str: + oracle = report["oracle_readiness"] + boundary = report["boundary_audit"] + claim = report["public_claim"] + lines = [ + "# Public Jyotish Benchmark Dashboard", + "", + f"Generated: `{report['generated_at']}`", + "", + "## Capability Registry", + "", + f"- technique_count: `{report['summary']['technique_count']}`", + f"- capability_valid: `{str(report['summary']['capability_valid']).lower()}`", + f"- problem_count: `{report['summary']['problem_count']}`", + "", + "## Dasha/Shadbala Oracle Readiness", + "", + f"- total_packets: `{oracle['total_packets']}`", + f"- valid_packets: `{oracle['valid_packets']}`", + f"- ready_for_calibration: `{oracle['ready_for_calibration']}`", + f"- production_tuning_allowed: `{str(oracle['production_tuning_allowed']).lower()}`", + "", + "## Boundary Audit", + "", + f"- external_verified_template_cases: `{boundary['external_verified_template_cases']}`", + f"- template_comparison_count: `{boundary['template_comparison_count']}`", + f"- production_tuning_recommended: `{str(boundary['production_tuning_recommended']).lower()}`", + "", + "## Global First Claim", + "", + f"- can_claim_global_first: `{str(claim['can_claim_global_first']).lower()}`", + f"- reason: {claim['reason']}", + "", + "## Remaining Gap", + "", + report["global_first_gap"], + "", + "## Next Actions", + "", + ] + lines.extend(f"- {item}" for item in report["next_actions"]) + lines.append("") + return "\n".join(lines) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate public Jyotish benchmark dashboard") + parser.add_argument("--oracle-file", default="references/oracle/dasha_shadbala_oracle_cases.json") + parser.add_argument("--format", choices=["json", "markdown"], default="json") + parser.add_argument("--output", help="Optional output path") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + report = build_dashboard(args.oracle_file) + text = json.dumps(report, ensure_ascii=False, indent=2) if args.format == "json" else render_markdown(report) + if args.output: + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(text, encoding="utf-8") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_public_benchmark_dashboard.py b/tests/test_public_benchmark_dashboard.py new file mode 100644 index 00000000..761d7b6a --- /dev/null +++ b/tests/test_public_benchmark_dashboard.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Tests for the public Jyotish benchmark dashboard.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def run_dashboard(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + "scripts/public_benchmark_dashboard.py", + "--oracle-file", + "references/oracle/dasha_shadbala_oracle_cases.json", + *args, + ], + cwd=ROOT, + text=True, + capture_output=True, + timeout=60, + check=False, + ) + + +def test_public_benchmark_dashboard_outputs_stable_json_summary() -> None: + completed = run_dashboard("--format", "json") + + assert completed.returncode == 0, completed.stderr or completed.stdout + report = json.loads(completed.stdout) + assert report["scope"] == "public_jyotish_benchmark_dashboard" + assert report["schema_version"] == 1 + assert report["summary"]["technique_count"] >= 60 + assert report["summary"]["capability_valid"] is True + assert report["oracle_readiness"]["total_packets"] == 5 + assert report["oracle_readiness"]["valid_packets"] == 0 + assert report["oracle_readiness"]["ready_for_calibration"] == 0 + assert report["oracle_readiness"]["production_tuning_allowed"] is False + assert report["boundary_audit"]["production_tuning_recommended"] is False + assert "Dasha/Shadbala" in report["global_first_gap"] + assert report["public_claim"]["can_claim_global_first"] is False + assert report["public_claim"]["reason"] + + +def test_public_benchmark_dashboard_outputs_markdown_and_can_write_file(tmp_path: Path) -> None: + output = tmp_path / "benchmark.md" + completed = run_dashboard("--format", "markdown", "--output", str(output)) + + assert completed.returncode == 0, completed.stderr or completed.stdout + assert output.exists() + markdown = output.read_text(encoding="utf-8") + assert "# Public Jyotish Benchmark Dashboard" in markdown + assert "Dasha/Shadbala Oracle Readiness" in markdown + assert "Global First Claim" in markdown + assert "can_claim_global_first: `false`" in markdown + assert "production_tuning_allowed: `false`" in markdown