root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add extraction results manifest

Browse Source
This commit is contained in:
732642856
2026-07-02 17:26:19 +08:00
parent ac2cd9776f
commit d441a0f277
11 changed files with 1562 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/research/character_level_external_manifest_latest.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"scope": "external",
"status": "pass",
"generated_at": "2026-07-02T09:12:09.488794+00:00",
"generated_at": "2026-07-02T09:23:43.522488+00:00",
"mode": {
"heavy_ocr": false,
"whole_machine_scan": false,
docs/research/character_level_external_manifest_latest.md
+1 -1
View File
@@ -2,7 +2,7 @@
- status: pass
- scope: external
- generated_at: 2026-07-02T09:12:09.488794+00:00
- generated_at: 2026-07-02T09:23:43.522488+00:00
- total_files: 883
- unhashed_files: 0
- unclassified_files: 0
docs/research/character_level_extraction_queue_latest.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"scope": "extraction-queue",
"status": "pass",
"generated_at": "2026-07-02T09:12:17.049464+00:00",
"generated_at": "2026-07-02T09:24:04.124961+00:00",
"mode": {
"heavy_ocr": false,
"whole_machine_scan": false,
docs/research/character_level_extraction_queue_latest.md
+1 -1
View File
@@ -2,7 +2,7 @@
- status: pass
- scope: extraction-queue
- generated_at: 2026-07-02T09:12:17.049464+00:00
- generated_at: 2026-07-02T09:24:04.124961+00:00
- queued_files: 66
- unhashed_files: 0
- Heavy OCR: disabled
docs/research/character_level_extraction_results_latest.json
+1227
View File
File diff suppressed because it is too large Load Diff
docs/research/character_level_extraction_results_latest.md
+33
View File
@@ -0,0 +1,33 @@
# Extraction Results Manifest
- status: pass
- scope: extraction-results
- generated_at: 2026-07-02T09:25:00.613875+00:00
- total_files: 66
- unhashed_files: 0
- stored_text_payload_fields: 0
- Heavy OCR: disabled
## Result Counts
- `extraction_failed`: 1
- `ocr_blocked_missing_engine`: 53
- `text_extracted`: 12
## Method Counts
- `docx`: 11
- `pdfplumber`: 2
- `pytesseract`: 53
## Post-Extraction Classification Counts
- `extracted_candidate_for_review`: 10
- `extracted_private_reference_only`: 3
- `extracted_reference_only`: 53
## Boundary
Extraction results store text hashes and counts only. Extracted private text is not copied into the repo truth chain.
No extracted text or OCR text is stored in this report; only hashes, counts, methods, and statuses are persisted.
docs/research/character_level_inventory_manifest_latest.json
+40 -12
View File
@@ -1,7 +1,7 @@
{
"scope": "project",
"status": "pass",
"generated_at": "2026-07-02T09:11:51.839696+00:00",
"generated_at": "2026-07-02T09:23:01.280161+00:00",
"mode": {
"heavy_ocr": false,
"whole_machine_scan": false,
@@ -17,7 +17,7 @@
"AGENTS.md"
],
"summary": {
"total_files": 1073,
"total_files": 1075,
"unhashed_files": 0,
"unclassified_files": 0,
"unknown_extraction_status": 0,
@@ -28,12 +28,12 @@
"quarantined_draft": 93,
"real_case_calibration": 5,
"reference_candidate": 180,
"research_governance": 443
"research_governance": 445
},
"extraction_status_counts": {
"binary_indexed": 34,
"image_ocr_queued": 3,
"text_indexed": 1036
"text_indexed": 1038
}
},
"root_summary": {
@@ -46,8 +46,8 @@
"bytes": 43187
},
"docs/research": {
"files": 536,
"bytes": 2901473
"files": 538,
"bytes": 2955810
},
"references": {
"files": 236,
@@ -5173,7 +5173,7 @@
"path": "docs/research/character_level_external_manifest_latest.json",
"suffix": ".json",
"byte_count": 666472,
"sha256": "244465beREDACTED_TIMEe81a2c8b3abac4f41e28bd55ce9f42f3f203d6574c52617dcfd9",
"sha256": "eb5243158ce7a2fbfbdedddf4cdaea2fb5ad867042b6492f5ed5449a65abb6ca",
"extraction_status": "text_indexed",
"character_count": 661666,
"line_count": 12446,
@@ -5187,7 +5187,7 @@
"path": "docs/research/character_level_external_manifest_latest.md",
"suffix": ".md",
"byte_count": 1280,
"sha256": "71cc5b027ea056c23ce1d5d1cda8921eeade065a8cb003910ea44d2a1165e642",
"sha256": "3a371858f58ddf1f040e161bb63fd764ae079a1a46d931baab9fbb4478ca64a3",
"extraction_status": "text_indexed",
"character_count": 1272,
"line_count": 47,
@@ -5201,7 +5201,7 @@
"path": "docs/research/character_level_extraction_queue_latest.json",
"suffix": ".json",
"byte_count": 31649,
"sha256": "cbfc03eee31d16cc3c796728cf34504c6eaa0722c5e42dd2f7d2ec1da966c5ef",
"sha256": "41fd28ba9ef0df29fbeb017830e2e81fbe7d86e6e2e53090c0d873cbab51d823",
"extraction_status": "text_indexed",
"character_count": 30151,
"line_count": 758,
@@ -5215,7 +5215,7 @@
"path": "docs/research/character_level_extraction_queue_latest.md",
"suffix": ".md",
"byte_count": 633,
"sha256": "6a45bb22f9f44d248c6e8ff71d74949180eb3eba537cb42080d15fef88395725",
"sha256": "21c0bf36f66edda5f1a7b941ea12c508d0851ebf79a24cd882332e71d73ae750",
"extraction_status": "text_indexed",
"character_count": 633,
"line_count": 27,
@@ -5225,11 +5225,39 @@
"promotion_status": "governance_or_history",
"reason": "Research history or governance record; index separately from rule truth."
},
"docs/research/character_level_extraction_results_latest.json": {
"path": "docs/research/character_level_extraction_results_latest.json",
"suffix": ".json",
"byte_count": 53548,
"sha256": "bda3e585b7cb3e9ca0147b97e7186d56a8adf64ed4c46cc78d7a0d4b2b0f4acb",
"extraction_status": "text_indexed",
"character_count": 52050,
"line_count": 1227,
"decode_replacement_count": 0,
"classification": "research_governance",
"priority": "priority_3",
"promotion_status": "governance_or_history",
"reason": "Research history or governance record; index separately from rule truth."
},
"docs/research/character_level_extraction_results_latest.md": {
"path": "docs/research/character_level_extraction_results_latest.md",
"suffix": ".md",
"byte_count": 789,
"sha256": "d0169690fb8b8d5f4438d1628b93b5cf86f76f2af97987eb3a4649370093d1c4",
"extraction_status": "text_indexed",
"character_count": 789,
"line_count": 34,
"decode_replacement_count": 0,
"classification": "research_governance",
"priority": "priority_3",
"promotion_status": "governance_or_history",
"reason": "Research history or governance record; index separately from rule truth."
},
"docs/research/character_level_inventory_manifest_latest.json": {
"path": "docs/research/character_level_inventory_manifest_latest.json",
"suffix": ".json",
"byte_count": 721667,
"sha256": "aba3169d10fe86f02dfed264b150afb84473994225b2a62a36b720f485ad1a41",
"sha256": "de2aac78f33cb27575bac5e9f6f0a04e1655b8655b851fff397a8641ebc96c71",
"extraction_status": "text_indexed",
"character_count": 721117,
"line_count": 15090,
@@ -5243,7 +5271,7 @@
"path": "docs/research/character_level_inventory_manifest_latest.md",
"suffix": ".md",
"byte_count": 1090,
"sha256": "d088bb6312805a0f78506af37c2d32098e97af635eb3eed707a4943b75dae2ae",
"sha256": "e63d29e5ef20dab395ea86d4dddc4e41edab6e8e3019caef4b3fd31197e46a72",
"extraction_status": "text_indexed",
"character_count": 1090,
"line_count": 44,
docs/research/character_level_inventory_manifest_latest.md
+5 -5
View File
@@ -2,8 +2,8 @@
- status: pass
- scope: project
- generated_at: 2026-07-02T09:11:51.839696+00:00
- total_files: 1073
- generated_at: 2026-07-02T09:23:01.280161+00:00
- total_files: 1075
- unhashed_files: 0
- unclassified_files: 0
- unknown_extraction_status: 0
@@ -16,7 +16,7 @@
| --- | ---: | ---: |
| `AGENTS.md` | 1 | 3932 |
| `SKILL.md` | 1 | 43187 |
| `docs/research` | 536 | 2901473 |
| `docs/research` | 538 | 2955810 |
| `references` | 236 | 4008411 |
| `references/open_source_sources` | 299 | 9365015 |
@@ -24,7 +24,7 @@
- `binary_indexed`: 34
- `image_ocr_queued`: 3
- `text_indexed`: 1036
- `text_indexed`: 1038
## Classification Counts
@@ -34,7 +34,7 @@
- `quarantined_draft`: 93
- `real_case_calibration`: 5
- `reference_candidate`: 180
- `research_governance`: 443
- `research_governance`: 445
## Boundary
progress.md
+9
View File
@@ -820,3 +820,12 @@
- 新增 `docs/research/character_level_extraction_queue_latest.json` 与 `.md`，当前 queued_files=`66`、`unhashed_files=0`。
- 队列构成：`image_ocr_queued=53`、`document_text_extraction_queued=11`、`pdf_text_extraction_queued=2`；来源为 `external=63`、`project=3`。
- 边界：本轮只做提取队列，不跑重 OCR，不把外部私人资料正文写进仓库，也不升格为 runtime truth。
## 2026-07-02T17:22:00+08:00 - PDF/DOCX 提取结果与 OCR blocked 分级
- 新增 `scripts/character_level_inventory_manifest.py --scope extraction-results`，对 extraction queue 做实际提取，但只保存文本 hash、字符数、行数、方法、状态和分级，不保存正文。
- 新增 `docs/research/character_level_extraction_results_latest.json` 与 `.md`，当前 total_files=`66`、`unhashed_files=0`、`stored_text_payload_fields=0`。
- 提取结果：`text_extracted=12`、`ocr_blocked_missing_engine=53`、`extraction_failed=1`。
- 方法分布：`docx=11`、`pdfplumber=2`、`pytesseract=53`；系统当前没有 `tesseract` 可执行文件，因此图片 OCR 全部明确 blocked。
- 失败项：`/Users/wuyongnaren/文件仓库/印度占星文章/4印度占星.docx`，`python-docx` 返回 `KeyError: "There is no item named 'NULL' in the archive"`；保留为 extraction_failed，不伪装为已提取。
- 提取后分级：`extracted_candidate_for_review=10`、`extracted_private_reference_only=3`、`extracted_reference_only=53`；仍未进入 runtime truth chain。
scripts/character_level_inventory_manifest.py
+205 -1
View File
@@ -11,6 +11,7 @@ from __future__ import annotations
import argparse
import hashlib
import json
import shutil
import sys
from datetime import datetime, timezone
from pathlib import Path
@@ -24,6 +25,8 @@ EXTERNAL_REPORT_JSON = ROOT / "docs" / "research" / "character_level_external_ma
EXTERNAL_REPORT_MD = ROOT / "docs" / "research" / "character_level_external_manifest_latest.md"
EXTRACTION_QUEUE_JSON = ROOT / "docs" / "research" / "character_level_extraction_queue_latest.json"
EXTRACTION_QUEUE_MD = ROOT / "docs" / "research" / "character_level_extraction_queue_latest.md"
EXTRACTION_RESULTS_JSON = ROOT / "docs" / "research" / "character_level_extraction_results_latest.json"
EXTRACTION_RESULTS_MD = ROOT / "docs" / "research" / "character_level_extraction_results_latest.md"
PROJECT_SCAN_ROOTS = [
"references",
@@ -425,6 +428,8 @@ def build_manifest(*, scope: str = "project", write: bool = True) -> dict[str, A
}
elif scope == "extraction-queue":
return build_extraction_queue(write=write)
elif scope == "extraction-results":
return build_extraction_results(write=write)
else:
raise ValueError(f"Unsupported scope: {scope}")
@@ -461,6 +466,162 @@ def build_manifest(*, scope: str = "project", write: bool = True) -> dict[str, A
return report
def _hash_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
def _extract_docx_text(path: Path) -> tuple[str, str | None]:
try:
from docx import Document
document = Document(str(path))
paragraphs = [paragraph.text for paragraph in document.paragraphs]
table_text: list[str] = []
for table in document.tables:
for row in table.rows:
table_text.append("\t".join(cell.text for cell in row.cells))
return "\n".join(paragraphs + table_text), None
except Exception as exc: # pragma: no cover - exercised by real files.
return "", f"{type(exc).__name__}: {exc}"
def _extract_pdf_text(path: Path) -> tuple[str, str, str | None]:
try:
import pdfplumber
parts: list[str] = []
with pdfplumber.open(str(path)) as pdf:
for page in pdf.pages:
parts.append(page.extract_text() or "")
text = "\n".join(parts)
if text.strip():
return text, "pdfplumber", None
except Exception as exc:
pdfplumber_error = f"{type(exc).__name__}: {exc}"
else:
pdfplumber_error = None
try:
from pypdf import PdfReader
reader = PdfReader(str(path))
text = "\n".join(page.extract_text() or "" for page in reader.pages)
return text, "pypdf", None
except Exception as exc: # pragma: no cover - depends on local PDF shape.
errors = "; ".join(error for error in [pdfplumber_error, f"{type(exc).__name__}: {exc}"] if error)
return "", "pypdf", errors
def _extract_image_text(path: Path) -> tuple[str, str, str | None]:
if not shutil.which("tesseract"):
return "", "pytesseract", "tesseract executable not found"
try:
from PIL import Image
import pytesseract
with Image.open(path) as image:
return pytesseract.image_to_string(image, lang="chi_sim+eng"), "pytesseract", None
except Exception as exc: # pragma: no cover - depends on local OCR install/languages.
return "", "pytesseract", f"{type(exc).__name__}: {exc}"
def _post_extraction_classification(item: dict[str, Any]) -> str:
path = str(item["path"])
if "/Desktop/" in path or "/WorkBuddy/" in path:
return "extracted_private_reference_only"
if item["classification"] in {"external_book_or_document", "reference_candidate"}:
return "extracted_candidate_for_review"
return "extracted_reference_only"
def _extract_queued_item(item: dict[str, Any]) -> dict[str, Any]:
path = Path(item["path"])
status = item["extraction_status"]
if status == "document_text_extraction_queued" and path.suffix.lower() == ".docx":
text, error = _extract_docx_text(path)
method = "docx"
elif status == "pdf_text_extraction_queued":
text, method, error = _extract_pdf_text(path)
elif status == "image_ocr_queued":
text, method, error = _extract_image_text(path)
if error and "tesseract executable not found" in error:
return {
**item,
"extraction_result": "ocr_blocked_missing_engine",
"extraction_method": method,
"blocked_reason": error,
"extracted_character_count": 0,
"extracted_line_count": 0,
"text_sha256": None,
"post_extraction_classification": "extracted_reference_only",
}
else:
text = ""
method = "unsupported"
error = "unsupported extraction target"
normalized = text or ""
if error and not normalized.strip():
result = "extraction_failed"
elif normalized.strip():
result = "text_extracted"
else:
result = "text_empty"
return {
**item,
"extraction_result": result,
"extraction_method": method,
"blocked_reason": error,
"extracted_character_count": len(normalized),
"extracted_line_count": normalized.count("\n") + (1 if normalized else 0),
"text_sha256": _hash_text(normalized) if normalized else None,
"post_extraction_classification": _post_extraction_classification(item),
}
def build_extraction_results(*, write: bool = True) -> dict[str, Any]:
queue_report = build_extraction_queue(write=False)
results = [_extract_queued_item(item) for item in queue_report["queue"]]
result_counts: dict[str, int] = {}
method_counts: dict[str, int] = {}
classification_counts: dict[str, int] = {}
for item in results:
result_counts[item["extraction_result"]] = result_counts.get(item["extraction_result"], 0) + 1
method_counts[item["extraction_method"]] = method_counts.get(item["extraction_method"], 0) + 1
classification_counts[item["post_extraction_classification"]] = (
classification_counts.get(item["post_extraction_classification"], 0) + 1
)
stored_text_payload_fields = sum(1 for item in results if "text" in item or "text_preview" in item)
report = {
"scope": "extraction-results",
"status": "pass",
"generated_at": datetime.now(timezone.utc).isoformat(),
"mode": {
"heavy_ocr": shutil.which("tesseract") is not None,
"whole_machine_scan": False,
"external_high_relevance_scan": True,
"semantic_promotion": False,
"boundary": "Extraction results store text hashes and counts only. Extracted private text is not copied into the repo truth chain.",
},
"summary": {
"total_files": len(results),
"unhashed_files": sum(1 for item in results if not item.get("sha256")),
"stored_text_payload_fields": stored_text_payload_fields,
},
"result_counts": dict(sorted(result_counts.items())),
"method_counts": dict(sorted(method_counts.items())),
"post_extraction_classification_counts": dict(sorted(classification_counts.items())),
"results": results,
"artifacts": {
"json_report": _relative(EXTRACTION_RESULTS_JSON),
"markdown_report": _relative(EXTRACTION_RESULTS_MD),
},
"title": "Extraction Results Manifest",
}
if write:
_write_reports(report, json_report=EXTRACTION_RESULTS_JSON, markdown_report=EXTRACTION_RESULTS_MD)
return report
def build_extraction_queue(*, write: bool = True) -> dict[str, Any]:
project = build_manifest(scope="project", write=False)
external = build_manifest(scope="external", write=False)
@@ -527,6 +688,8 @@ def _render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
if report["scope"] == "extraction-queue":
return _render_extraction_queue_markdown(report)
if report["scope"] == "extraction-results":
return _render_extraction_results_markdown(report)
lines = [
f"# {report.get('title', 'Character-Level Inventory Manifest')}",
"",
@@ -567,6 +730,43 @@ def _render_markdown(report: dict[str, Any]) -> str:
return "\n".join(lines)
def _render_extraction_results_markdown(report: dict[str, Any]) -> str:
lines = [
f"# {report.get('title', 'Extraction Results Manifest')}",
"",
f"- status: {report['status']}",
f"- scope: {report['scope']}",
f"- generated_at: {report['generated_at']}",
f"- total_files: {report['summary']['total_files']}",
f"- unhashed_files: {report['summary']['unhashed_files']}",
f"- stored_text_payload_fields: {report['summary']['stored_text_payload_fields']}",
f"- Heavy OCR: {'enabled' if report['mode']['heavy_ocr'] else 'disabled'}",
"",
"## Result Counts",
"",
]
for name, count in report["result_counts"].items():
lines.append(f"- `{name}`: {count}")
lines.extend(["", "## Method Counts", ""])
for name, count in report["method_counts"].items():
lines.append(f"- `{name}`: {count}")
lines.extend(["", "## Post-Extraction Classification Counts", ""])
for name, count in report["post_extraction_classification_counts"].items():
lines.append(f"- `{name}`: {count}")
lines.extend(
[
"",
"## Boundary",
"",
report["mode"]["boundary"],
"",
"No extracted text or OCR text is stored in this report; only hashes, counts, methods, and statuses are persisted.",
"",
]
)
return "\n".join(lines)
def _render_extraction_queue_markdown(report: dict[str, Any]) -> str:
lines = [
f"# {report.get('title', 'Extraction Queue Manifest')}",
@@ -611,7 +811,11 @@ def _summary_view(report: dict[str, Any]) -> dict[str, Any]:
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--scope", default="project", choices=["project", "external", "extraction-queue"])
parser.add_argument(
"--scope",
default="project",
choices=["project", "external", "extraction-queue", "extraction-results"],
)
parser.add_argument("--no-write", action="store_true", help="Print the manifest without writing report artifacts.")
parser.add_argument("--summary-only", action="store_true", help="Print only summary fields; still writes full artifacts unless --no-write is set.")
args = parser.parse_args(argv)
tests/test_character_level_inventory_manifest.py
+39
View File
@@ -176,3 +176,42 @@ def test_extraction_queue_report_combines_project_and_external_binary_work() ->
assert json_path.exists()
assert md_path.exists()
assert "Extraction Queue Manifest" in md_path.read_text(encoding="utf-8")
def test_extraction_results_extract_pdf_and_docx_without_storing_text() -> None:
report = _run_manifest("--scope", "extraction-results")
assert report["scope"] == "extraction-results"
assert report["status"] == "pass"
assert report["summary"]["total_files"] >= 60
assert report["summary"]["unhashed_files"] == 0
assert report["summary"]["stored_text_payload_fields"] == 0
assert report["result_counts"]["text_extracted"] >= 10
assert report["method_counts"]["docx"] >= 10
assert report["method_counts"]["pdfplumber"] + report["method_counts"].get("pypdf", 0) >= 2
for item in report["results"]:
assert item["sha256"]
assert item["source_scope"] in {"project", "external"}
assert item["extraction_result"] in {
"text_extracted",
"text_empty",
"ocr_blocked_missing_engine",
"extraction_failed",
}
assert "text" not in item
assert "text_preview" not in item
if item["extraction_result"] == "text_extracted":
assert item["extracted_character_count"] > 0
assert item["text_sha256"]
assert item["post_extraction_classification"] in {
"extracted_reference_only",
"extracted_private_reference_only",
"extracted_candidate_for_review",
}
json_path = ROOT / report["artifacts"]["json_report"]
md_path = ROOT / report["artifacts"]["markdown_report"]
assert json_path.exists()
assert md_path.exists()
assert "Extraction Results Manifest" in md_path.read_text(encoding="utf-8")