root/Jyotisha
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add extraction review governance manifest

Browse Source
This commit is contained in:
732642856
2026-07-02 18:56:48 +08:00
parent 11440b539a
commit 6b331a356e
13 changed files with 1244 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/research/character_level_external_manifest_latest.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"scope": "external",
"status": "pass",
"generated_at": "2026-07-02T10:44:46.651144+00:00",
"generated_at": "2026-07-02T10:53:23.492131+00:00",
"mode": {
"heavy_ocr": false,
"whole_machine_scan": false,
docs/research/character_level_external_manifest_latest.md
+1 -1
View File
@@ -2,7 +2,7 @@
- status: pass
- scope: external
- generated_at: 2026-07-02T10:44:46.651144+00:00
- generated_at: 2026-07-02T10:53:23.492131+00:00
- total_files: 883
- unhashed_files: 0
- unclassified_files: 0
docs/research/character_level_extraction_queue_latest.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"scope": "extraction-queue",
"status": "pass",
"generated_at": "2026-07-02T10:45:03.271522+00:00",
"generated_at": "2026-07-02T10:53:47.897123+00:00",
"mode": {
"heavy_ocr": false,
"whole_machine_scan": false,
docs/research/character_level_extraction_queue_latest.md
+1 -1
View File
@@ -2,7 +2,7 @@
- status: pass
- scope: extraction-queue
- generated_at: 2026-07-02T10:45:03.271522+00:00
- generated_at: 2026-07-02T10:53:47.897123+00:00
- queued_files: 66
- unhashed_files: 0
- Heavy OCR: disabled
docs/research/character_level_extraction_results_latest.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"scope": "extraction-results",
"status": "pass",
"generated_at": "2026-07-02T10:45:41.296864+00:00",
"generated_at": "2026-07-02T10:55:01.179960+00:00",
"mode": {
"heavy_ocr": false,
"macos_vision_available": true,
docs/research/character_level_extraction_results_latest.md
+1 -1
View File
@@ -2,7 +2,7 @@
- status: pass
- scope: extraction-results
- generated_at: 2026-07-02T10:45:41.296864+00:00
- generated_at: 2026-07-02T10:55:01.179960+00:00
- total_files: 66
- unhashed_files: 0
- stored_text_payload_fields: 0
docs/research/character_level_extraction_review_latest.json
+1019
View File
File diff suppressed because it is too large Load Diff
docs/research/character_level_extraction_review_latest.md
+21
View File
@@ -0,0 +1,21 @@
# Extraction Review Manifest
- status: pass
- scope: extraction-review
- generated_at: 2026-07-02T10:56:07.354583+00:00
- reviewed_files: 66
- runtime_promotions: 0
- stored_text_payload_fields: 0
## Decision Counts
- `asset_or_empty_reference_only`: 17
- `private_reference_only`: 3
- `promote_review_candidate`: 38
- `reference_only`: 8
## Boundary
Review decisions are governance labels only. No extracted source is promoted to runtime truth here.
Promotion still requires human review, source grading, conflict arbitration, and explicit source-pack tests.
docs/research/character_level_inventory_manifest_latest.json
+48 -20
View File
@@ -1,7 +1,7 @@
{
"scope": "project",
"status": "pass",
"generated_at": "2026-07-02T10:44:17.758196+00:00",
"generated_at": "2026-07-02T10:52:27.980901+00:00",
"mode": {
"heavy_ocr": false,
"whole_machine_scan": false,
@@ -17,7 +17,7 @@
"AGENTS.md"
],
"summary": {
"total_files": 1075,
"total_files": 1077,
"unhashed_files": 0,
"unclassified_files": 0,
"unknown_extraction_status": 0,
@@ -28,12 +28,12 @@
"quarantined_draft": 93,
"real_case_calibration": 5,
"reference_candidate": 180,
"research_governance": 445
"research_governance": 447
},
"extraction_status_counts": {
"binary_indexed": 34,
"image_ocr_queued": 3,
"text_indexed": 1038
"text_indexed": 1040
}
},
"root_summary": {
@@ -46,8 +46,8 @@
"bytes": 43187
},
"docs/research": {
"files": 538,
"bytes": 2969221
"files": 540,
"bytes": 3019398
},
"references": {
"files": 236,
@@ -5173,7 +5173,7 @@
"path": "docs/research/character_level_external_manifest_latest.json",
"suffix": ".json",
"byte_count": 666472,
"sha256": "e7a98a44b5ad6c2582bddb9482bf7d980a93784d74a5a448b6aee51c358269fd",
"sha256": "e14c5b310be0a108b40857a463e2d7657bf911083d12a8639de845e60af9d415",
"extraction_status": "text_indexed",
"character_count": 661666,
"line_count": 12446,
@@ -5187,7 +5187,7 @@
"path": "docs/research/character_level_external_manifest_latest.md",
"suffix": ".md",
"byte_count": 1280,
"sha256": "dd2c84a86dc7e691d5bd9088829ba2088ff32ad305f8eb8e7d492f6eb40ab9f4",
"sha256": "416e1e27c89232f580e18eb75fdf89f90509a19f4ad02b3d0df351d4a693acbd",
"extraction_status": "text_indexed",
"character_count": 1272,
"line_count": 47,
@@ -5201,7 +5201,7 @@
"path": "docs/research/character_level_extraction_queue_latest.json",
"suffix": ".json",
"byte_count": 31649,
"sha256": "4946bd810c6b312bae1fc9bb7a78bc34f886a4a5c7532d74732399fe7f612cdb",
"sha256": "2564e16627c5326880827500e6c8cf4d7d671773bf942308b6e5e47d517a936f",
"extraction_status": "text_indexed",
"character_count": 30151,
"line_count": 758,
@@ -5215,7 +5215,7 @@
"path": "docs/research/character_level_extraction_queue_latest.md",
"suffix": ".md",
"byte_count": 633,
"sha256": "8baea40e157fca6adc9228988561253f0803eb719b7392bc74ef4e64145394b6",
"sha256": "0ee514b7288f7e54b076d2e791798053dba6d8c79b19fad80652f376d8edd40d",
"extraction_status": "text_indexed",
"character_count": 633,
"line_count": 27,
@@ -5228,11 +5228,11 @@
"docs/research/character_level_extraction_results_latest.json": {
"path": "docs/research/character_level_extraction_results_latest.json",
"suffix": ".json",
"byte_count": 65711,
"sha256": "a06874a9c54148891373069cd3933b3be51e3b045e51bd8b121c751f9dcf83ac",
"byte_count": 65824,
"sha256": "7460624396860dddc58e65c04487cbd5aa102d55c28c807fc2100399d2acd6e8",
"extraction_status": "text_indexed",
"character_count": 64213,
"line_count": 1650,
"character_count": 64326,
"line_count": 1652,
"decode_replacement_count": 0,
"classification": "research_governance",
"priority": "priority_3",
@@ -5242,11 +5242,39 @@
"docs/research/character_level_extraction_results_latest.md": {
"path": "docs/research/character_level_extraction_results_latest.md",
"suffix": ".md",
"byte_count": 749,
"sha256": "6f7d1e04b3bbb0a977e44b2cd6ddc6a71fe7b97b016a76b6e7d3dc0da37b32af",
"byte_count": 850,
"sha256": "d185c4239f0d3d66ad61bfb3f2f928581358c471883d07259772f445057f2cd5",
"extraction_status": "text_indexed",
"character_count": 749,
"line_count": 33,
"character_count": 850,
"line_count": 35,
"decode_replacement_count": 0,
"classification": "research_governance",
"priority": "priority_3",
"promotion_status": "governance_or_history",
"reason": "Research history or governance record; index separately from rule truth."
},
"docs/research/character_level_extraction_review_latest.json": {
"path": "docs/research/character_level_extraction_review_latest.json",
"suffix": ".json",
"byte_count": 49397,
"sha256": "510ff3d55c71ad376cf2b89a09c81394b6d2de02e2367168733ce00940d77537",
"extraction_status": "text_indexed",
"character_count": 47899,
"line_count": 1019,
"decode_replacement_count": 0,
"classification": "research_governance",
"priority": "priority_3",
"promotion_status": "governance_or_history",
"reason": "Research history or governance record; index separately from rule truth."
},
"docs/research/character_level_extraction_review_latest.md": {
"path": "docs/research/character_level_extraction_review_latest.md",
"suffix": ".md",
"byte_count": 566,
"sha256": "cabf37aea044f1ff82e5a98226d37e0f37ce2b4b293ab1b518ae8fec2fc9bb08",
"extraction_status": "text_indexed",
"character_count": 566,
"line_count": 22,
"decode_replacement_count": 0,
"classification": "research_governance",
"priority": "priority_3",
@@ -5257,7 +5285,7 @@
"path": "docs/research/character_level_inventory_manifest_latest.json",
"suffix": ".json",
"byte_count": 722955,
"sha256": "4fcf6ce8815022fbe308fca4d725c50a481ce2f1383397c3617f08bb138cc3fc",
"sha256": "414fc9d24a8d6662935442aedd37c0820da186227dbd9dc461d498c43044df48",
"extraction_status": "text_indexed",
"character_count": 722405,
"line_count": 15118,
@@ -5271,7 +5299,7 @@
"path": "docs/research/character_level_inventory_manifest_latest.md",
"suffix": ".md",
"byte_count": 1090,
"sha256": "547a1e4ec8187636e39dbb09551541a0d2296cb798ee1895804feaca2612e1c1",
"sha256": "1e1c58e9f1fa954fb179640f61f3de86f6ec3d3c9d9cf830ec9454444d9923b4",
"extraction_status": "text_indexed",
"character_count": 1090,
"line_count": 44,
docs/research/character_level_inventory_manifest_latest.md
+5 -5
View File
@@ -2,8 +2,8 @@
- status: pass
- scope: project
- generated_at: 2026-07-02T10:44:17.758196+00:00
- total_files: 1075
- generated_at: 2026-07-02T10:52:27.980901+00:00
- total_files: 1077
- unhashed_files: 0
- unclassified_files: 0
- unknown_extraction_status: 0
@@ -16,7 +16,7 @@
| --- | ---: | ---: |
| `AGENTS.md` | 1 | 3932 |
| `SKILL.md` | 1 | 43187 |
| `docs/research` | 538 | 2969221 |
| `docs/research` | 540 | 3019398 |
| `references` | 236 | 4008411 |
| `references/open_source_sources` | 299 | 9365015 |
@@ -24,7 +24,7 @@
- `binary_indexed`: 34
- `image_ocr_queued`: 3
- `text_indexed`: 1038
- `text_indexed`: 1040
## Classification Counts
@@ -34,7 +34,7 @@
- `quarantined_draft`: 93
- `real_case_calibration`: 5
- `reference_candidate`: 180
- `research_governance`: 445
- `research_governance`: 447
## Boundary
progress.md
+7
View File
@@ -843,3 +843,10 @@
- 采用更适合本机的 macOS Vision OCR：脚本自动编译并缓存 `~/.cache/jyotish-ocr/vision_ocr`，不依赖 Homebrew tesseract。
- `scripts/character_level_inventory_manifest.py --scope extraction-results` 新增 OCR cache；缓存只保存 OCR 文本 hash、字符数、行数、后端、状态，不保存 OCR 正文。
- 53 张图片已用 `macos_vision` 复跑完成；当前结果：`text_extracted=49`、`text_empty=17`、`ocr_blocked_missing_engine=0`、`extraction_failed=0`、`stored_text_payload_fields=0`。
## 2026-07-02T18:52:00+08:00 - 提取后分级 review manifest
- 新增 `scripts/character_level_inventory_manifest.py --scope extraction-review`，对 66 个提取结果做治理分级，不保存正文，不进入 runtime truth。
- 新增 `docs/research/character_level_extraction_review_latest.json` 与 `.md`，当前 reviewed_files=`66`、runtime_promotions=`0`、stored_text_payload_fields=`0`。
- 分级结果：`promote_review_candidate=38`、`private_reference_only=3`、`reference_only=8`、`asset_or_empty_reference_only=17`。
- 后续若要升格，只能从 `promote_review_candidate` 里逐批人工 source grading + conflict arbitration + source-pack 测试，不能直接把 OCR/外部文档当主链真源。
scripts/character_level_inventory_manifest.py
+106 -1
View File
@@ -30,6 +30,8 @@ EXTRACTION_QUEUE_JSON = ROOT / "docs" / "research" / "character_level_extraction
EXTRACTION_QUEUE_MD = ROOT / "docs" / "research" / "character_level_extraction_queue_latest.md"
EXTRACTION_RESULTS_JSON = ROOT / "docs" / "research" / "character_level_extraction_results_latest.json"
EXTRACTION_RESULTS_MD = ROOT / "docs" / "research" / "character_level_extraction_results_latest.md"
EXTRACTION_REVIEW_JSON = ROOT / "docs" / "research" / "character_level_extraction_review_latest.json"
EXTRACTION_REVIEW_MD = ROOT / "docs" / "research" / "character_level_extraction_review_latest.md"
VISION_OCR_SOURCE = r'''
import Foundation
import Vision
@@ -474,6 +476,8 @@ def build_manifest(*, scope: str = "project", write: bool = True) -> dict[str, A
return build_extraction_queue(write=write)
elif scope == "extraction-results":
return build_extraction_results(write=write)
elif scope == "extraction-review":
return build_extraction_review(write=write)
else:
raise ValueError(f"Unsupported scope: {scope}")
@@ -510,6 +514,75 @@ def build_manifest(*, scope: str = "project", write: bool = True) -> dict[str, A
return report
def _review_decision(item: dict[str, Any]) -> tuple[str, str]:
path = str(item["path"])
if item["extraction_result"] == "extraction_failed":
return "blocked_extraction_failure", "Extraction failed; keep blocked until repaired."
if item["post_extraction_classification"] == "extracted_private_reference_only":
return "private_reference_only", "Private or historical personal report; do not promote into runtime truth."
if item["extraction_result"] == "text_empty":
return "asset_or_empty_reference_only", "OCR/extraction produced empty text; keep as indexed asset only."
if "/文件仓库/印度占星文章/" in path or "/Downloads/" in path:
return "promote_review_candidate", "Extracted source-like material; requires human source grading before promotion."
if item["post_extraction_classification"] == "extracted_candidate_for_review":
return "promote_review_candidate", "Candidate text extracted; requires conflict arbitration and source grading."
return "reference_only", "Useful as context or asset evidence, not direct runtime truth."
def build_extraction_review(*, write: bool = True) -> dict[str, Any]:
results_report = build_extraction_results(write=False)
reviews: list[dict[str, Any]] = []
for item in results_report["results"]:
decision, reason = _review_decision(item)
reviews.append(
{
"path": item["path"],
"source_scope": item["source_scope"],
"suffix": item["suffix"],
"byte_count": item["byte_count"],
"sha256": item["sha256"],
"extraction_result": item["extraction_result"],
"extraction_method": item["extraction_method"],
"extracted_character_count": item["extracted_character_count"],
"text_sha256": item.get("text_sha256"),
"post_extraction_classification": item["post_extraction_classification"],
"decision": decision,
"decision_reason": reason,
"runtime_truth_allowed": False,
}
)
decision_counts: dict[str, int] = {}
for item in reviews:
decision_counts[item["decision"]] = decision_counts.get(item["decision"], 0) + 1
stored_text_payload_fields = sum(1 for item in reviews if "text" in item or "text_preview" in item)
report = {
"scope": "extraction-review",
"status": "pass",
"generated_at": datetime.now(timezone.utc).isoformat(),
"mode": {
"whole_machine_scan": False,
"semantic_promotion": False,
"runtime_truth_promotion": False,
"boundary": "Review decisions are governance labels only. No extracted source is promoted to runtime truth here.",
},
"summary": {
"reviewed_files": len(reviews),
"runtime_promotions": 0,
"stored_text_payload_fields": stored_text_payload_fields,
},
"decision_counts": dict(sorted(decision_counts.items())),
"reviews": reviews,
"artifacts": {
"json_report": _relative(EXTRACTION_REVIEW_JSON),
"markdown_report": _relative(EXTRACTION_REVIEW_MD),
},
"title": "Extraction Review Manifest",
}
if write:
_write_reports(report, json_report=EXTRACTION_REVIEW_JSON, markdown_report=EXTRACTION_REVIEW_MD)
return report
def _hash_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
@@ -889,6 +962,8 @@ def _render_markdown(report: dict[str, Any]) -> str:
return _render_extraction_queue_markdown(report)
if report["scope"] == "extraction-results":
return _render_extraction_results_markdown(report)
if report["scope"] == "extraction-review":
return _render_extraction_review_markdown(report)
lines = [
f"# {report.get('title', 'Character-Level Inventory Manifest')}",
"",
@@ -929,6 +1004,36 @@ def _render_markdown(report: dict[str, Any]) -> str:
return "\n".join(lines)
def _render_extraction_review_markdown(report: dict[str, Any]) -> str:
lines = [
f"# {report.get('title', 'Extraction Review Manifest')}",
"",
f"- status: {report['status']}",
f"- scope: {report['scope']}",
f"- generated_at: {report['generated_at']}",
f"- reviewed_files: {report['summary']['reviewed_files']}",
f"- runtime_promotions: {report['summary']['runtime_promotions']}",
f"- stored_text_payload_fields: {report['summary']['stored_text_payload_fields']}",
"",
"## Decision Counts",
"",
]
for name, count in report["decision_counts"].items():
lines.append(f"- `{name}`: {count}")
lines.extend(
[
"",
"## Boundary",
"",
report["mode"]["boundary"],
"",
"Promotion still requires human review, source grading, conflict arbitration, and explicit source-pack tests.",
"",
]
)
return "\n".join(lines)
def _render_extraction_results_markdown(report: dict[str, Any]) -> str:
lines = [
f"# {report.get('title', 'Extraction Results Manifest')}",
@@ -1015,7 +1120,7 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument(
"--scope",
default="project",
choices=["project", "external", "extraction-queue", "extraction-results"],
choices=["project", "external", "extraction-queue", "extraction-results", "extraction-review"],
)
parser.add_argument("--no-write", action="store_true", help="Print the manifest without writing report artifacts.")
parser.add_argument("--summary-only", action="store_true", help="Print only summary fields; still writes full artifacts unless --no-write is set.")
tests/test_character_level_inventory_manifest.py
+32
View File
@@ -223,3 +223,35 @@ def test_extraction_results_extract_pdf_and_docx_without_storing_text() -> None:
assert json_path.exists()
assert md_path.exists()
assert "Extraction Results Manifest" in md_path.read_text(encoding="utf-8")
def test_extraction_review_grades_extracted_sources_without_promoting_runtime_truth() -> None:
report = _run_manifest("--scope", "extraction-review")
assert report["scope"] == "extraction-review"
assert report["status"] == "pass"
assert report["summary"]["reviewed_files"] >= 60
assert report["summary"]["runtime_promotions"] == 0
assert report["summary"]["stored_text_payload_fields"] == 0
assert report["decision_counts"]["promote_review_candidate"] >= 5
assert report["decision_counts"]["private_reference_only"] >= 1
assert report["decision_counts"]["asset_or_empty_reference_only"] >= 1
for item in report["reviews"]:
assert item["sha256"]
assert item["decision"] in {
"promote_review_candidate",
"private_reference_only",
"reference_only",
"asset_or_empty_reference_only",
"blocked_extraction_failure",
}
assert item["runtime_truth_allowed"] is False
assert "text" not in item
assert "text_preview" not in item
json_path = ROOT / report["artifacts"]["json_report"]
md_path = ROOT / report["artifacts"]["markdown_report"]
assert json_path.exists()
assert md_path.exists()
assert "Extraction Review Manifest" in md_path.read_text(encoding="utf-8")