diff --git a/docs/research/character_level_external_manifest_latest.json b/docs/research/character_level_external_manifest_latest.json index a8f490e5..bbfed507 100644 --- a/docs/research/character_level_external_manifest_latest.json +++ b/docs/research/character_level_external_manifest_latest.json @@ -1,7 +1,7 @@ { "scope": "external", "status": "pass", - "generated_at": "2026-07-02T09:23:43.522488+00:00", + "generated_at": "2026-07-02T09:43:31.236878+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, diff --git a/docs/research/character_level_external_manifest_latest.md b/docs/research/character_level_external_manifest_latest.md index 54bbee39..45afbc1e 100644 --- a/docs/research/character_level_external_manifest_latest.md +++ b/docs/research/character_level_external_manifest_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: external -- generated_at: 2026-07-02T09:23:43.522488+00:00 +- generated_at: 2026-07-02T09:43:31.236878+00:00 - total_files: 883 - unhashed_files: 0 - unclassified_files: 0 diff --git a/docs/research/character_level_extraction_queue_latest.json b/docs/research/character_level_extraction_queue_latest.json index a812a855..07a65157 100644 --- a/docs/research/character_level_extraction_queue_latest.json +++ b/docs/research/character_level_extraction_queue_latest.json @@ -1,7 +1,7 @@ { "scope": "extraction-queue", "status": "pass", - "generated_at": "2026-07-02T09:24:04.124961+00:00", + "generated_at": "2026-07-02T09:43:43.085623+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, diff --git a/docs/research/character_level_extraction_queue_latest.md b/docs/research/character_level_extraction_queue_latest.md index 2c928c58..0f310ccf 100644 --- a/docs/research/character_level_extraction_queue_latest.md +++ b/docs/research/character_level_extraction_queue_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: extraction-queue -- generated_at: 2026-07-02T09:24:04.124961+00:00 +- generated_at: 2026-07-02T09:43:43.085623+00:00 - queued_files: 66 - unhashed_files: 0 - Heavy OCR: disabled diff --git a/docs/research/character_level_extraction_results_latest.json b/docs/research/character_level_extraction_results_latest.json index 6f16458e..2be5c810 100644 --- a/docs/research/character_level_extraction_results_latest.json +++ b/docs/research/character_level_extraction_results_latest.json @@ -1,7 +1,7 @@ { "scope": "extraction-results", "status": "pass", - "generated_at": "2026-07-02T09:25:00.613875+00:00", + "generated_at": "2026-07-02T09:44:22.267617+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, @@ -15,9 +15,8 @@ "stored_text_payload_fields": 0 }, "result_counts": { - "extraction_failed": 1, "ocr_blocked_missing_engine": 53, - "text_extracted": 12 + "text_extracted": 13 }, "method_counts": { "docx": 11, @@ -166,12 +165,12 @@ "classification": "external_book_or_document", "priority": "priority_1", "promotion_status": "external_reference_only", - "extraction_result": "extraction_failed", + "extraction_result": "text_extracted", "extraction_method": "docx", - "blocked_reason": "KeyError: \"There is no item named 'NULL' in the archive\"", - "extracted_character_count": 0, - "extracted_line_count": 0, - "text_sha256": null, + "blocked_reason": "python-docx fallback used after KeyError: \"There is no item named 'NULL' in the archive\"", + "extracted_character_count": 19643, + "extracted_line_count": 437, + "text_sha256": "f496ed4e54897fed05b7a5e4905787c36baf3008214d90336b6a1d4f2406940d", "post_extraction_classification": "extracted_candidate_for_review" }, { @@ -244,7 +243,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -262,7 +267,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -280,7 +291,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -298,7 +315,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -316,7 +339,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -334,7 +363,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -352,7 +387,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -370,7 +411,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -388,7 +435,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -406,7 +459,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -424,7 +483,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -442,7 +507,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -460,7 +531,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -478,7 +555,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -496,7 +579,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -514,7 +603,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -532,7 +627,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -550,7 +651,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -568,7 +675,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -586,7 +699,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -604,7 +723,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -622,7 +747,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -640,7 +771,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -658,7 +795,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -676,7 +819,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -694,7 +843,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -712,7 +867,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -730,7 +891,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -748,7 +915,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -766,7 +939,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -784,7 +963,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -802,7 +987,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -820,7 +1011,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -838,7 +1035,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -856,7 +1059,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -874,7 +1083,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -892,7 +1107,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -910,7 +1131,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -928,7 +1155,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -946,7 +1179,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -964,7 +1203,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -982,7 +1227,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1000,7 +1251,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1018,7 +1275,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1036,7 +1299,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1054,7 +1323,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1072,7 +1347,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1090,7 +1371,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1108,7 +1395,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", @@ -1126,7 +1419,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "project", @@ -1144,7 +1443,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "project", @@ -1162,7 +1467,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "project", @@ -1180,7 +1491,13 @@ "extracted_character_count": 0, "extracted_line_count": 0, "text_sha256": null, - "post_extraction_classification": "extracted_reference_only" + "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": false, + "ocr_requested_languages": [ + "chi_sim", + "eng" + ], + "ocr_available_languages": [] }, { "source_scope": "external", diff --git a/docs/research/character_level_extraction_results_latest.md b/docs/research/character_level_extraction_results_latest.md index 2de6eddf..ed44e562 100644 --- a/docs/research/character_level_extraction_results_latest.md +++ b/docs/research/character_level_extraction_results_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: extraction-results -- generated_at: 2026-07-02T09:25:00.613875+00:00 +- generated_at: 2026-07-02T09:44:22.267617+00:00 - total_files: 66 - unhashed_files: 0 - stored_text_payload_fields: 0 @@ -10,9 +10,8 @@ ## Result Counts -- `extraction_failed`: 1 - `ocr_blocked_missing_engine`: 53 -- `text_extracted`: 12 +- `text_extracted`: 13 ## Method Counts diff --git a/docs/research/character_level_inventory_manifest_latest.json b/docs/research/character_level_inventory_manifest_latest.json index 3c73b161..3034167e 100644 --- a/docs/research/character_level_inventory_manifest_latest.json +++ b/docs/research/character_level_inventory_manifest_latest.json @@ -1,7 +1,7 @@ { "scope": "project", "status": "pass", - "generated_at": "2026-07-02T09:23:01.280161+00:00", + "generated_at": "2026-07-02T09:43:08.434092+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, @@ -47,7 +47,7 @@ }, "docs/research": { "files": 538, - "bytes": 2955810 + "bytes": 2965145 }, "references": { "files": 236, @@ -5173,7 +5173,7 @@ "path": "docs/research/character_level_external_manifest_latest.json", "suffix": ".json", "byte_count": 666472, - "sha256": "eb5243158ce7a2fbfbdedddf4cdaea2fb5ad867042b6492f5ed5449a65abb6ca", + "sha256": "9fceb19121870ae252b77645b4e49515b8c1c6b7bd465de227dca8de47437b35", "extraction_status": "text_indexed", "character_count": 661666, "line_count": 12446, @@ -5187,7 +5187,7 @@ "path": "docs/research/character_level_external_manifest_latest.md", "suffix": ".md", "byte_count": 1280, - "sha256": "3a371858f58ddf1f040e161bb63fd764ae079a1a46d931baab9fbb4478ca64a3", + "sha256": "c030624888b4bc361635468d268653358d51f359d57a597b30620a5ccf24182e", "extraction_status": "text_indexed", "character_count": 1272, "line_count": 47, @@ -5201,7 +5201,7 @@ "path": "docs/research/character_level_extraction_queue_latest.json", "suffix": ".json", "byte_count": 31649, - "sha256": "41fd28ba9ef0df29fbeb017830e2e81fbe7d86e6e2e53090c0d873cbab51d823", + "sha256": "7aba199609805e911f07d0443e7b759ca46cbffa0527af932581c712808358e6", "extraction_status": "text_indexed", "character_count": 30151, "line_count": 758, @@ -5215,7 +5215,7 @@ "path": "docs/research/character_level_extraction_queue_latest.md", "suffix": ".md", "byte_count": 633, - "sha256": "21c0bf36f66edda5f1a7b941ea12c508d0851ebf79a24cd882332e71d73ae750", + "sha256": "8e5277feb3e35c124f65869c39e9a6038a403f284ed7dea786fea50b76365ca2", "extraction_status": "text_indexed", "character_count": 633, "line_count": 27, @@ -5228,11 +5228,11 @@ "docs/research/character_level_extraction_results_latest.json": { "path": "docs/research/character_level_extraction_results_latest.json", "suffix": ".json", - "byte_count": 53548, - "sha256": "bda3e585b7cb3e9ca0147b97e7186d56a8adf64ed4c46cc78d7a0d4b2b0f4acb", + "byte_count": 61620, + "sha256": "02413ba1ec2877ef67794bffde2f4a0ed35ea60677ee94c9bcbe517fac5bd778", "extraction_status": "text_indexed", - "character_count": 52050, - "line_count": 1227, + "character_count": 60122, + "line_count": 1544, "decode_replacement_count": 0, "classification": "research_governance", "priority": "priority_3", @@ -5242,11 +5242,11 @@ "docs/research/character_level_extraction_results_latest.md": { "path": "docs/research/character_level_extraction_results_latest.md", "suffix": ".md", - "byte_count": 789, - "sha256": "d0169690fb8b8d5f4438d1628b93b5cf86f76f2af97987eb3a4649370093d1c4", + "byte_count": 764, + "sha256": "fd16ff882e9b290a4cb4aa6f613b41e54eb1e9065577e9af322f817455b0a383", "extraction_status": "text_indexed", - "character_count": 789, - "line_count": 34, + "character_count": 764, + "line_count": 33, "decode_replacement_count": 0, "classification": "research_governance", "priority": "priority_3", @@ -5256,11 +5256,11 @@ "docs/research/character_level_inventory_manifest_latest.json": { "path": "docs/research/character_level_inventory_manifest_latest.json", "suffix": ".json", - "byte_count": 721667, - "sha256": "de2aac78f33cb27575bac5e9f6f0a04e1655b8655b851fff397a8641ebc96c71", + "byte_count": 722955, + "sha256": "197892b1f3b259f7aa868593670cbb693dd8717bd5f304407b37bdaaf7a97463", "extraction_status": "text_indexed", - "character_count": 721117, - "line_count": 15090, + "character_count": 722405, + "line_count": 15118, "decode_replacement_count": 0, "classification": "research_governance", "priority": "priority_3", @@ -5271,7 +5271,7 @@ "path": "docs/research/character_level_inventory_manifest_latest.md", "suffix": ".md", "byte_count": 1090, - "sha256": "e63d29e5ef20dab395ea86d4dddc4e41edab6e8e3019caef4b3fd31197e46a72", + "sha256": "18ffd1520954f0a98b79a56f9d3919fdfb5a9e447c5802bc4a3c011204d2934e", "extraction_status": "text_indexed", "character_count": 1090, "line_count": 44, diff --git a/docs/research/character_level_inventory_manifest_latest.md b/docs/research/character_level_inventory_manifest_latest.md index f417ee63..37ed76c2 100644 --- a/docs/research/character_level_inventory_manifest_latest.md +++ b/docs/research/character_level_inventory_manifest_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: project -- generated_at: 2026-07-02T09:23:01.280161+00:00 +- generated_at: 2026-07-02T09:43:08.434092+00:00 - total_files: 1075 - unhashed_files: 0 - unclassified_files: 0 @@ -16,7 +16,7 @@ | --- | ---: | ---: | | `AGENTS.md` | 1 | 3932 | | `SKILL.md` | 1 | 43187 | -| `docs/research` | 538 | 2955810 | +| `docs/research` | 538 | 2965145 | | `references` | 236 | 4008411 | | `references/open_source_sources` | 299 | 9365015 | diff --git a/progress.md b/progress.md index e97e0248..5f3ce97b 100644 --- a/progress.md +++ b/progress.md @@ -829,3 +829,10 @@ - 方法分布：`docx=11`、`pdfplumber=2`、`pytesseract=53`；系统当前没有 `tesseract` 可执行文件，因此图片 OCR 全部明确 blocked。 - 失败项：`/Users/wuyongnaren/文件仓库/印度占星文章/4印度占星.docx`，`python-docx` 返回 `KeyError: "There is no item named 'NULL' in the archive"`；保留为 extraction_failed，不伪装为已提取。 - 提取后分级：`extracted_candidate_for_review=10`、`extracted_private_reference_only=3`、`extracted_reference_only=53`；仍未进入 runtime truth chain。 + +## 2026-07-02T17:32:00+08:00 - DOCX fallback 提取与 OCR 引擎审计 + +- 尝试用 Homebrew 安装 `tesseract tesseract-lang`，但 Homebrew 在 macOS 12 上需要拉取/构建 60+ 依赖；为避免把本轮变成长安装任务，已中断安装。 +- 增强 `scripts/character_level_inventory_manifest.py --scope extraction-results`：图片 OCR 记录 `ocr_engine_available`、`ocr_requested_languages`、`ocr_available_languages`，便于安装 OCR 后复跑。 +- 为异常 DOCX 增加 `word/document.xml` 备用提取路径；`/Users/wuyongnaren/文件仓库/印度占星文章/4印度占星.docx` 已从 `extraction_failed` 转为 `text_extracted`，字符数 `19643`。 +- 当前提取结果更新为：`text_extracted=13`、`ocr_blocked_missing_engine=53`、`extraction_failed=0`；仍未保存正文，`stored_text_payload_fields=0`。 diff --git a/scripts/character_level_inventory_manifest.py b/scripts/character_level_inventory_manifest.py index 331728f6..6b21fc6e 100644 --- a/scripts/character_level_inventory_manifest.py +++ b/scripts/character_level_inventory_manifest.py @@ -13,7 +13,9 @@ import hashlib import json import shutil import sys +import zipfile from datetime import datetime, timezone +from xml.etree import ElementTree from pathlib import Path from typing import Any @@ -482,7 +484,25 @@ def _extract_docx_text(path: Path) -> tuple[str, str | None]: table_text.append("\t".join(cell.text for cell in row.cells)) return "\n".join(paragraphs + table_text), None except Exception as exc: # pragma: no cover - exercised by real files. - return "", f"{type(exc).__name__}: {exc}" + fallback_text, fallback_error = _extract_docx_xml_text(path) + if fallback_text.strip(): + return fallback_text, f"python-docx fallback used after {type(exc).__name__}: {exc}" + return "", fallback_error or f"{type(exc).__name__}: {exc}" + + +def _extract_docx_xml_text(path: Path) -> tuple[str, str | None]: + try: + with zipfile.ZipFile(path) as archive: + xml_bytes = archive.read("word/document.xml") + root = ElementTree.fromstring(xml_bytes) + text_nodes = [] + namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + for node in root.iter(f"{namespace}t"): + if node.text: + text_nodes.append(node.text) + return "\n".join(text_nodes), None + except Exception as exc: + return "", f"docx_xml_fallback_failed {type(exc).__name__}: {exc}" def _extract_pdf_text(path: Path) -> tuple[str, str, str | None]: @@ -519,11 +539,29 @@ def _extract_image_text(path: Path) -> tuple[str, str, str | None]: import pytesseract with Image.open(path) as image: - return pytesseract.image_to_string(image, lang="chi_sim+eng"), "pytesseract", None + return pytesseract.image_to_string(image, lang="+".join(_ocr_languages())), "pytesseract", None except Exception as exc: # pragma: no cover - depends on local OCR install/languages. return "", "pytesseract", f"{type(exc).__name__}: {exc}" +def _ocr_available_languages() -> list[str]: + if not shutil.which("tesseract"): + return [] + try: + import pytesseract + + return sorted(str(lang) for lang in pytesseract.get_languages(config="")) + except Exception: + return [] + + +def _ocr_languages() -> list[str]: + available = set(_ocr_available_languages()) + preferred = ["chi_sim", "eng"] + selected = [lang for lang in preferred if lang in available] + return selected or (["eng"] if "eng" in available else preferred) + + def _post_extraction_classification(item: dict[str, Any]) -> str: path = str(item["path"]) if "/Desktop/" in path or "/WorkBuddy/" in path: @@ -553,6 +591,9 @@ def _extract_queued_item(item: dict[str, Any]) -> dict[str, Any]: "extracted_line_count": 0, "text_sha256": None, "post_extraction_classification": "extracted_reference_only", + "ocr_engine_available": False, + "ocr_requested_languages": ["chi_sim", "eng"], + "ocr_available_languages": [], } else: text = "" @@ -575,6 +616,15 @@ def _extract_queued_item(item: dict[str, Any]) -> dict[str, Any]: "extracted_line_count": normalized.count("\n") + (1 if normalized else 0), "text_sha256": _hash_text(normalized) if normalized else None, "post_extraction_classification": _post_extraction_classification(item), + **( + { + "ocr_engine_available": shutil.which("tesseract") is not None, + "ocr_requested_languages": ["chi_sim", "eng"], + "ocr_available_languages": _ocr_available_languages(), + } + if status == "image_ocr_queued" + else {} + ), } diff --git a/tests/test_character_level_inventory_manifest.py b/tests/test_character_level_inventory_manifest.py index 62e64877..02ebd045 100644 --- a/tests/test_character_level_inventory_manifest.py +++ b/tests/test_character_level_inventory_manifest.py @@ -186,7 +186,8 @@ def test_extraction_results_extract_pdf_and_docx_without_storing_text() -> None: assert report["summary"]["total_files"] >= 60 assert report["summary"]["unhashed_files"] == 0 assert report["summary"]["stored_text_payload_fields"] == 0 - assert report["result_counts"]["text_extracted"] >= 10 + assert report["result_counts"]["text_extracted"] >= 13 + assert report["result_counts"].get("extraction_failed", 0) == 0 assert report["method_counts"]["docx"] >= 10 assert report["method_counts"]["pdfplumber"] + report["method_counts"].get("pypdf", 0) >= 2 @@ -209,6 +210,10 @@ def test_extraction_results_extract_pdf_and_docx_without_storing_text() -> None: "extracted_private_reference_only", "extracted_candidate_for_review", } + if item["extraction_result"] == "ocr_blocked_missing_engine": + assert item["ocr_engine_available"] is False + assert item["ocr_requested_languages"] == ["chi_sim", "eng"] + assert item["ocr_available_languages"] == [] json_path = ROOT / report["artifacts"]["json_report"] md_path = ROOT / report["artifacts"]["markdown_report"]