From 5916a41e805ab26369f3bed43556df79eff5f074 Mon Sep 17 00:00:00 2001 From: 732642856 <732642856@qq.com> Date: Thu, 2 Jul 2026 19:19:05 +0800 Subject: [PATCH] Add source grading batch1 arbitration --- ...racter_level_external_manifest_latest.json | 2 +- ...haracter_level_external_manifest_latest.md | 2 +- ...aracter_level_extraction_queue_latest.json | 2 +- ...character_level_extraction_queue_latest.md | 2 +- ...acter_level_extraction_results_latest.json | 2 +- ...aracter_level_extraction_results_latest.md | 2 +- ...racter_level_extraction_review_latest.json | 2 +- ...haracter_level_extraction_review_latest.md | 2 +- ...acter_level_inventory_manifest_latest.json | 66 +++- ...aracter_level_inventory_manifest_latest.md | 10 +- ...er_level_source_grading_batch1_latest.json | 332 ++++++++++++++++++ ...cter_level_source_grading_batch1_latest.md | 32 ++ progress.md | 8 + scripts/character_level_inventory_manifest.py | 191 +++++++++- ...test_character_level_inventory_manifest.py | 49 +++ 15 files changed, 671 insertions(+), 33 deletions(-) create mode 100644 docs/research/character_level_source_grading_batch1_latest.json create mode 100644 docs/research/character_level_source_grading_batch1_latest.md diff --git a/docs/research/character_level_external_manifest_latest.json b/docs/research/character_level_external_manifest_latest.json index 966319a9..2b9a4d29 100644 --- a/docs/research/character_level_external_manifest_latest.json +++ b/docs/research/character_level_external_manifest_latest.json @@ -1,7 +1,7 @@ { "scope": "external", "status": "pass", - "generated_at": "2026-07-02T10:53:23.492131+00:00", + "generated_at": "2026-07-02T11:16:37.952323+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, diff --git a/docs/research/character_level_external_manifest_latest.md b/docs/research/character_level_external_manifest_latest.md index 9756f276..d5f39415 100644 --- a/docs/research/character_level_external_manifest_latest.md +++ b/docs/research/character_level_external_manifest_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: external -- generated_at: 2026-07-02T10:53:23.492131+00:00 +- generated_at: 2026-07-02T11:16:37.952323+00:00 - total_files: 883 - unhashed_files: 0 - unclassified_files: 0 diff --git a/docs/research/character_level_extraction_queue_latest.json b/docs/research/character_level_extraction_queue_latest.json index 5a9b31cd..f9cdbb40 100644 --- a/docs/research/character_level_extraction_queue_latest.json +++ b/docs/research/character_level_extraction_queue_latest.json @@ -1,7 +1,7 @@ { "scope": "extraction-queue", "status": "pass", - "generated_at": "2026-07-02T10:53:47.897123+00:00", + "generated_at": "2026-07-02T11:16:43.551753+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, diff --git a/docs/research/character_level_extraction_queue_latest.md b/docs/research/character_level_extraction_queue_latest.md index 497ae330..ebf15f97 100644 --- a/docs/research/character_level_extraction_queue_latest.md +++ b/docs/research/character_level_extraction_queue_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: extraction-queue -- generated_at: 2026-07-02T10:53:47.897123+00:00 +- generated_at: 2026-07-02T11:16:43.551753+00:00 - queued_files: 66 - unhashed_files: 0 - Heavy OCR: disabled diff --git a/docs/research/character_level_extraction_results_latest.json b/docs/research/character_level_extraction_results_latest.json index 5b7be31c..513b5e0f 100644 --- a/docs/research/character_level_extraction_results_latest.json +++ b/docs/research/character_level_extraction_results_latest.json @@ -1,7 +1,7 @@ { "scope": "extraction-results", "status": "pass", - "generated_at": "2026-07-02T10:55:01.179960+00:00", + "generated_at": "2026-07-02T11:16:59.606954+00:00", "mode": { "heavy_ocr": false, "macos_vision_available": true, diff --git a/docs/research/character_level_extraction_results_latest.md b/docs/research/character_level_extraction_results_latest.md index 3c0de20a..05e8b2f9 100644 --- a/docs/research/character_level_extraction_results_latest.md +++ b/docs/research/character_level_extraction_results_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: extraction-results -- generated_at: 2026-07-02T10:55:01.179960+00:00 +- generated_at: 2026-07-02T11:16:59.606954+00:00 - total_files: 66 - unhashed_files: 0 - stored_text_payload_fields: 0 diff --git a/docs/research/character_level_extraction_review_latest.json b/docs/research/character_level_extraction_review_latest.json index d1826929..848213ac 100644 --- a/docs/research/character_level_extraction_review_latest.json +++ b/docs/research/character_level_extraction_review_latest.json @@ -1,7 +1,7 @@ { "scope": "extraction-review", "status": "pass", - "generated_at": "2026-07-02T10:56:07.354583+00:00", + "generated_at": "2026-07-02T11:17:16.341748+00:00", "mode": { "whole_machine_scan": false, "semantic_promotion": false, diff --git a/docs/research/character_level_extraction_review_latest.md b/docs/research/character_level_extraction_review_latest.md index d5c8fc32..d980265f 100644 --- a/docs/research/character_level_extraction_review_latest.md +++ b/docs/research/character_level_extraction_review_latest.md @@ -2,7 +2,7 @@ - status: pass - scope: extraction-review -- generated_at: 2026-07-02T10:56:07.354583+00:00 +- generated_at: 2026-07-02T11:17:16.341748+00:00 - reviewed_files: 66 - runtime_promotions: 0 - stored_text_payload_fields: 0 diff --git a/docs/research/character_level_inventory_manifest_latest.json b/docs/research/character_level_inventory_manifest_latest.json index 9fc6d2ac..86fba488 100644 --- a/docs/research/character_level_inventory_manifest_latest.json +++ b/docs/research/character_level_inventory_manifest_latest.json @@ -1,7 +1,7 @@ { "scope": "project", "status": "pass", - "generated_at": "2026-07-02T10:52:27.980901+00:00", + "generated_at": "2026-07-02T11:16:27.729341+00:00", "mode": { "heavy_ocr": false, "whole_machine_scan": false, @@ -17,7 +17,7 @@ "AGENTS.md" ], "summary": { - "total_files": 1077, + "total_files": 1079, "unhashed_files": 0, "unclassified_files": 0, "unknown_extraction_status": 0, @@ -28,12 +28,12 @@ "quarantined_draft": 93, "real_case_calibration": 5, "reference_candidate": 180, - "research_governance": 447 + "research_governance": 449 }, "extraction_status_counts": { "binary_indexed": 34, "image_ocr_queued": 3, - "text_indexed": 1040 + "text_indexed": 1042 } }, "root_summary": { @@ -46,8 +46,8 @@ "bytes": 43187 }, "docs/research": { - "files": 540, - "bytes": 3019398 + "files": 542, + "bytes": 3039702 }, "references": { "files": 236, @@ -5173,7 +5173,7 @@ "path": "docs/research/character_level_external_manifest_latest.json", "suffix": ".json", "byte_count": 666472, - "sha256": "e14c5b310be0a108b40857a463e2d7657bf911083d12a8639de845e60af9d415", + "sha256": "831b4d08d4d0c3211a75990718b21cebf225bf68a6d05b848c2e1cd9ac85c64f", "extraction_status": "text_indexed", "character_count": 661666, "line_count": 12446, @@ -5187,7 +5187,7 @@ "path": "docs/research/character_level_external_manifest_latest.md", "suffix": ".md", "byte_count": 1280, - "sha256": "416e1e27c89232f580e18eb75fdf89f90509a19f4ad02b3d0df351d4a693acbd", + "sha256": "051a4141312a66d5fab490f1a8f696c5d2b78d803790b72ff05175936856cf03", "extraction_status": "text_indexed", "character_count": 1272, "line_count": 47, @@ -5201,7 +5201,7 @@ "path": "docs/research/character_level_extraction_queue_latest.json", "suffix": ".json", "byte_count": 31649, - "sha256": "2564e16627c5326880827500e6c8cf4d7d671773bf942308b6e5e47d517a936f", + "sha256": "7e6f7d5f2fa7623f5122652bcc1e8deecaad2666e90e39adac125d097b8b1c22", "extraction_status": "text_indexed", "character_count": 30151, "line_count": 758, @@ -5215,7 +5215,7 @@ "path": "docs/research/character_level_extraction_queue_latest.md", "suffix": ".md", "byte_count": 633, - "sha256": "0ee514b7288f7e54b076d2e791798053dba6d8c79b19fad80652f376d8edd40d", + "sha256": "ba1ea4ba8fff433d8afd79e2a94a86cf6e9c47d5a458dd9dd7030694e0fbf43b", "extraction_status": "text_indexed", "character_count": 633, "line_count": 27, @@ -5229,7 +5229,7 @@ "path": "docs/research/character_level_extraction_results_latest.json", "suffix": ".json", "byte_count": 65824, - "sha256": "7460624396860dddc58e65c04487cbd5aa102d55c28c807fc2100399d2acd6e8", + "sha256": "888cb4aa14ff072316b308186184a1931f7b3c397e78ed0d454173695672b54e", "extraction_status": "text_indexed", "character_count": 64326, "line_count": 1652, @@ -5243,7 +5243,7 @@ "path": "docs/research/character_level_extraction_results_latest.md", "suffix": ".md", "byte_count": 850, - "sha256": "d185c4239f0d3d66ad61bfb3f2f928581358c471883d07259772f445057f2cd5", + "sha256": "fe35e9edae8fb06021e3531ae4fed3de5771ba14b997aedd096259ffdbc27450", "extraction_status": "text_indexed", "character_count": 850, "line_count": 35, @@ -5257,7 +5257,7 @@ "path": "docs/research/character_level_extraction_review_latest.json", "suffix": ".json", "byte_count": 49397, - "sha256": "510ff3d55c71ad376cf2b89a09c81394b6d2de02e2367168733ce00940d77537", + "sha256": "9b3a9f4d2a31e25d8141bac99b96e354193b6f2529ab5011c1299a4b79e6c749", "extraction_status": "text_indexed", "character_count": 47899, "line_count": 1019, @@ -5271,7 +5271,7 @@ "path": "docs/research/character_level_extraction_review_latest.md", "suffix": ".md", "byte_count": 566, - "sha256": "cabf37aea044f1ff82e5a98226d37e0f37ce2b4b293ab1b518ae8fec2fc9bb08", + "sha256": "754057f8c846f108fdf6aafb4b6189eb74fa2a4ef2df5b2c1344306822ff489e", "extraction_status": "text_indexed", "character_count": 566, "line_count": 22, @@ -5284,11 +5284,11 @@ "docs/research/character_level_inventory_manifest_latest.json": { "path": "docs/research/character_level_inventory_manifest_latest.json", "suffix": ".json", - "byte_count": 722955, - "sha256": "414fc9d24a8d6662935442aedd37c0820da186227dbd9dc461d498c43044df48", + "byte_count": 725536, + "sha256": "05ba0bda2e2200c531dda5310822fd96b7d2e9ac761e44bdfbe2d664da453beb", "extraction_status": "text_indexed", - "character_count": 722405, - "line_count": 15118, + "character_count": 724986, + "line_count": 15174, "decode_replacement_count": 0, "classification": "research_governance", "priority": "priority_3", @@ -5299,7 +5299,7 @@ "path": "docs/research/character_level_inventory_manifest_latest.md", "suffix": ".md", "byte_count": 1090, - "sha256": "1e1c58e9f1fa954fb179640f61f3de86f6ec3d3c9d9cf830ec9454444d9923b4", + "sha256": "ba69a0e40457b0cac74b1cdf20849e1f9a4d54220a03c83af9b6e6b5e7244537", "extraction_status": "text_indexed", "character_count": 1090, "line_count": 44, @@ -5309,6 +5309,34 @@ "promotion_status": "governance_or_history", "reason": "Research history or governance record; index separately from rule truth." }, + "docs/research/character_level_source_grading_batch1_latest.json": { + "path": "docs/research/character_level_source_grading_batch1_latest.json", + "suffix": ".json", + "byte_count": 17054, + "sha256": "49cdc4825a6fce2bc5392b92967f1adee92039dd72b851969d7a41cbebfce131", + "extraction_status": "text_indexed", + "character_count": 16742, + "line_count": 330, + "decode_replacement_count": 0, + "classification": "research_governance", + "priority": "priority_3", + "promotion_status": "governance_or_history", + "reason": "Research history or governance record; index separately from rule truth." + }, + "docs/research/character_level_source_grading_batch1_latest.md": { + "path": "docs/research/character_level_source_grading_batch1_latest.md", + "suffix": ".md", + "byte_count": 669, + "sha256": "e0ccf97744870fbaf93e076c17b8fa76997ecd4493f29c9cf40dcfb0dd5e862a", + "extraction_status": "text_indexed", + "character_count": 669, + "line_count": 31, + "decode_replacement_count": 0, + "classification": "research_governance", + "priority": "priority_3", + "promotion_status": "governance_or_history", + "reason": "Research history or governance record; index separately from rule truth." + }, "docs/research/desktop_packaging_spike_2026_06_23.md": { "path": "docs/research/desktop_packaging_spike_2026_06_23.md", "suffix": ".md", diff --git a/docs/research/character_level_inventory_manifest_latest.md b/docs/research/character_level_inventory_manifest_latest.md index 731b8646..1fe1eeab 100644 --- a/docs/research/character_level_inventory_manifest_latest.md +++ b/docs/research/character_level_inventory_manifest_latest.md @@ -2,8 +2,8 @@ - status: pass - scope: project -- generated_at: 2026-07-02T10:52:27.980901+00:00 -- total_files: 1077 +- generated_at: 2026-07-02T11:16:27.729341+00:00 +- total_files: 1079 - unhashed_files: 0 - unclassified_files: 0 - unknown_extraction_status: 0 @@ -16,7 +16,7 @@ | --- | ---: | ---: | | `AGENTS.md` | 1 | 3932 | | `SKILL.md` | 1 | 43187 | -| `docs/research` | 540 | 3019398 | +| `docs/research` | 542 | 3039702 | | `references` | 236 | 4008411 | | `references/open_source_sources` | 299 | 9365015 | @@ -24,7 +24,7 @@ - `binary_indexed`: 34 - `image_ocr_queued`: 3 -- `text_indexed`: 1040 +- `text_indexed`: 1042 ## Classification Counts @@ -34,7 +34,7 @@ - `quarantined_draft`: 93 - `real_case_calibration`: 5 - `reference_candidate`: 180 -- `research_governance`: 447 +- `research_governance`: 449 ## Boundary diff --git a/docs/research/character_level_source_grading_batch1_latest.json b/docs/research/character_level_source_grading_batch1_latest.json new file mode 100644 index 00000000..62bacbd0 --- /dev/null +++ b/docs/research/character_level_source_grading_batch1_latest.json @@ -0,0 +1,332 @@ +{ + "scope": "source-grading-batch1", + "status": "pass", + "generated_at": "2026-07-02T11:18:48.369587+00:00", + "mode": { + "runtime_truth_promotion": false, + "semantic_promotion": false, + "boundary": "Batch 1 grades extracted candidates only; no source enters runtime truth without a later source-pack test." + }, + "summary": { + "total_review_candidates": 38, + "selected_count": 12, + "graded_files": 12, + "runtime_promotions": 0, + "stored_text_payload_fields": 0 + }, + "grade_counts": { + "promote_to_reference_pack_candidate": 9, + "reference_only": 3 + }, + "pack_decision_counts": { + "reference_only": 3, + "reference_pack_candidate": 9 + }, + "conflict_arbitration": { + "status": "batch1_arbitrated", + "scope": "source_pack_eligibility_only", + "runtime_truth_allowed": false, + "reference_pack_ready_count": 9, + "runtime_truth_ready_count": 0, + "reference_pack_next_steps": [ + "wire selected candidates into interpretation_source_inventory/source pack", + "add strict workflow visibility tests", + "keep content-level precedence below core rules until line-level rule review exists" + ] + }, + "graded_sources": [ + { + "path": "/Users/wuyongnaren/Downloads/jyotish_training.agent.final.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 186879, + "sha256": "a0d83aa8a70f9966cbe79945c4416906e1382c9093998c23d73bd395b02f23fe", + "extraction_method": "docx", + "extracted_character_count": 145078, + "text_sha256": "9607ceaafa8a38abd5943037b7a66113db4fee6abb6d77a246924c7770242049", + "review_decision": "promote_review_candidate", + "source_grade": "reference_only", + "grade_reason": "Training material can inform style/workflow but cannot outrank source rules.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_only", + "reason": "Kept as reference-only because the source is OCR-derived, training-oriented, private, or not first-order rule material.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_only", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/1印度占星.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 6142110, + "sha256": "210a09d764f8fe17ded6c908d5073648908892a99ae853f983e6f516ccc0164d", + "extraction_method": "docx", + "extracted_character_count": 30716, + "text_sha256": "9d02544ecab01e9dad012ec53336f0404ba316896e3ae5fd8efa7b2e7271330f", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/2印度占星.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 3031411, + "sha256": "16979422e7621475243e2739fda00767d1a9dc367f7f10153fe36d8af6e15c6d", + "extraction_method": "docx", + "extracted_character_count": 17967, + "text_sha256": "91cfc9364387007753bce62360ee7d546aa6357935775a32dc867f2d3bf09dde", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/3印度占星.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 2149772, + "sha256": "59ee274246547f4afb581ec26f5a4aad03ba2095309d66c40a60a51d1244df92", + "extraction_method": "docx", + "extracted_character_count": 29017, + "text_sha256": "47c4c9cfcfdc2bbd024d845c1c95eec9546d1d8c462fb25626e6b449d957d3cc", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/4印度占星.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 5572637, + "sha256": "e789430dc1df6ab7054a5b28f0665da325e7c5eb2e6daac39f1f17ac29ba7629", + "extraction_method": "docx", + "extracted_character_count": 19643, + "text_sha256": "f496ed4e54897fed05b7a5e4905787c36baf3008214d90336b6a1d4f2406940d", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/Tajika占星术是印度占星术体系之一.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 1338151, + "sha256": "254624594c26fc80ad5bfa9e9a880ac0cd81c9388b19721d5767dfa5c55e953c", + "extraction_method": "docx", + "extracted_character_count": 23606, + "text_sha256": "2f8b217e08b5c20afdfb328851bf5c719cd1ab98f2572d413ab359e6f16cef6f", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/印度占星Jaimini系统的Darakaraka的理解和说明如何解释.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 44926, + "sha256": "f2db03d1c0392e2cdad66da9a731117c0ddf290cad7712b9c3de0d8234a17a29", + "extraction_method": "docx", + "extracted_character_count": 20085, + "text_sha256": "72bba68ca08007492181c01936290315500e052a21fc03562c03605eb0204048", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/文件仓库/印度占星文章/高地位配偶与婚后命运积极转变的占星规则与组合.docx", + "source_scope": "external", + "suffix": ".docx", + "byte_count": 1120887, + "sha256": "9cdd6996670ad46063c9582957c6d98269e069f96801ff340ce0c84de325d412", + "extraction_method": "docx", + "extracted_character_count": 4507, + "text_sha256": "1e620fb59e95ae4fe59b1181bc9fe34238115e1f138d1433f2e1b66c10051ed6", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/Downloads/印度占星.pdf", + "source_scope": "external", + "suffix": ".pdf", + "byte_count": 135905, + "sha256": "121507cf1d0d66711bc30c18f589f36f2d3cce797ecb13cdf4427625e6ec8894", + "extraction_method": "pdfplumber", + "extracted_character_count": 31164, + "text_sha256": "e63b112988500a71704511d007c52cffdc71c300e518c438eb7f0cb71d732ede", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/Downloads/印度占星1.pdf", + "source_scope": "external", + "suffix": ".pdf", + "byte_count": 132486, + "sha256": "92c9288c5d0011a869c545e3459f037ff09cd20f7508543bc0eb11b03079d930", + "extraction_method": "pdfplumber", + "extracted_character_count": 31321, + "text_sha256": "d93283239c33fab588747a9354a162049b263d18bd06463bbb7b02634704a1be", + "review_decision": "promote_review_candidate", + "source_grade": "promote_to_reference_pack_candidate", + "grade_reason": "Source-like document; candidate for reference pack after conflict arbitration.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_pack_candidate", + "reason": "Document/PDF source lineage is clear enough for reference/source pack candidacy; still cannot enter runtime truth until source-pack visibility tests bind it explicitly.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_pack_candidate", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/Downloads/印度占星11页/IMG_3005.jpg", + "source_scope": "external", + "suffix": ".jpg", + "byte_count": 365592, + "sha256": "3743a89d6611dd8a3a344db2e9ca6877b6f1f610b48bd3cbb9486eb8a81f4195", + "extraction_method": "macos_vision", + "extracted_character_count": 2931, + "text_sha256": "468967a7cfdce5a6b0dc23e534ceae4776a6f9f415a9fcaa4a5bc34a4fb982cb", + "review_decision": "promote_review_candidate", + "source_grade": "reference_only", + "grade_reason": "OCR text is high-signal but still requires manual verification against image.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_only", + "reason": "Kept as reference-only because the source is OCR-derived, training-oriented, private, or not first-order rule material.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_only", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + }, + { + "path": "/Users/wuyongnaren/Downloads/印度占星11页/IMG_3006.jpg", + "source_scope": "external", + "suffix": ".jpg", + "byte_count": 387356, + "sha256": "03689b8775950267002cbbf9b96bf5038217c692c311188429ee99903beea285", + "extraction_method": "macos_vision", + "extracted_character_count": 3635, + "text_sha256": "ae6ded6ec4433f6a75e29c91d551fa3033b381be765d22d97915b7bfc4e0cf0c", + "review_decision": "promote_review_candidate", + "source_grade": "reference_only", + "grade_reason": "OCR text is high-signal but still requires manual verification against image.", + "conflict_arbitration": { + "status": "batch1_arbitrated", + "pack_decision": "reference_only", + "reason": "Kept as reference-only because the source is OCR-derived, training-oriented, private, or not first-order rule material.", + "runtime_truth_allowed": false, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth." + }, + "pack_decision": "reference_only", + "requires_conflict_arbitration": false, + "requires_source_pack_visibility_test": true, + "runtime_truth_allowed": false + } + ], + "artifacts": { + "json_report": "docs/research/character_level_source_grading_batch1_latest.json", + "markdown_report": "docs/research/character_level_source_grading_batch1_latest.md" + }, + "title": "Source Grading Batch 1 Manifest" +} \ No newline at end of file diff --git a/docs/research/character_level_source_grading_batch1_latest.md b/docs/research/character_level_source_grading_batch1_latest.md new file mode 100644 index 00000000..56c18462 --- /dev/null +++ b/docs/research/character_level_source_grading_batch1_latest.md @@ -0,0 +1,32 @@ +# Source Grading Batch 1 Manifest + +- status: pass +- scope: source-grading-batch1 +- generated_at: 2026-07-02T11:18:48.369587+00:00 +- total_review_candidates: 38 +- selected_count: 12 +- graded_files: 12 +- runtime_promotions: 0 +- stored_text_payload_fields: 0 + +## Grade Counts + +- `promote_to_reference_pack_candidate`: 9 +- `reference_only`: 3 + +## Pack Decisions + +- `reference_only`: 3 +- `reference_pack_candidate`: 9 + +## Conflict Arbitration + +- status: batch1_arbitrated +- scope: source_pack_eligibility_only +- reference_pack_ready_count: 9 +- runtime_truth_ready_count: 0 +- runtime_truth_allowed: False + +## Boundary + +Batch 1 grades extracted candidates only; no source enters runtime truth without a later source-pack test. diff --git a/progress.md b/progress.md index 6eb65047..660360b0 100644 --- a/progress.md +++ b/progress.md @@ -850,3 +850,11 @@ - 新增 `docs/research/character_level_extraction_review_latest.json` 与 `.md`，当前 reviewed_files=`66`、runtime_promotions=`0`、stored_text_payload_fields=`0`。 - 分级结果：`promote_review_candidate=38`、`private_reference_only=3`、`reference_only=8`、`asset_or_empty_reference_only=17`。 - 后续若要升格，只能从 `promote_review_candidate` 里逐批人工 source grading + conflict arbitration + source-pack 测试，不能直接把 OCR/外部文档当主链真源。 + +## 2026-07-02T19:06:00+08:00 - promote_review_candidate 第一批 source grading + +- 新增 `scripts/character_level_inventory_manifest.py --scope source-grading-batch1`，从 38 个 `promote_review_candidate` 中选取 12 个高信号候选做第一批分级。 +- 新增 `docs/research/character_level_source_grading_batch1_latest.json` 与 `.md`，当前 graded_files=`12`、runtime_promotions=`0`、stored_text_payload_fields=`0`。 +- 第一批结果：`promote_to_reference_pack_candidate=9`、`reference_only=3`；已完成 batch1 级别的 source-pack eligibility 仲裁。 +- 仲裁结论：`reference_pack_candidate=9`、`reference_only=3`、`runtime_truth_ready_count=0`；不允许直接进入 runtime truth。 +- 下一步必须做：把 9 个候选显式接入 interpretation source/source pack，并加 strict workflow visibility tests；内容级规则优先级仍不得压过核心主规则。 diff --git a/scripts/character_level_inventory_manifest.py b/scripts/character_level_inventory_manifest.py index 29d69c1c..ed48fc4d 100644 --- a/scripts/character_level_inventory_manifest.py +++ b/scripts/character_level_inventory_manifest.py @@ -32,6 +32,8 @@ EXTRACTION_RESULTS_JSON = ROOT / "docs" / "research" / "character_level_extracti EXTRACTION_RESULTS_MD = ROOT / "docs" / "research" / "character_level_extraction_results_latest.md" EXTRACTION_REVIEW_JSON = ROOT / "docs" / "research" / "character_level_extraction_review_latest.json" EXTRACTION_REVIEW_MD = ROOT / "docs" / "research" / "character_level_extraction_review_latest.md" +SOURCE_GRADING_BATCH1_JSON = ROOT / "docs" / "research" / "character_level_source_grading_batch1_latest.json" +SOURCE_GRADING_BATCH1_MD = ROOT / "docs" / "research" / "character_level_source_grading_batch1_latest.md" VISION_OCR_SOURCE = r''' import Foundation import Vision @@ -478,6 +480,8 @@ def build_manifest(*, scope: str = "project", write: bool = True) -> dict[str, A return build_extraction_results(write=write) elif scope == "extraction-review": return build_extraction_review(write=write) + elif scope == "source-grading-batch1": + return build_source_grading_batch1(write=write) else: raise ValueError(f"Unsupported scope: {scope}") @@ -514,6 +518,135 @@ def build_manifest(*, scope: str = "project", write: bool = True) -> dict[str, A return report +def _source_grading_batch1_items(reviews: list[dict[str, Any]]) -> list[dict[str, Any]]: + candidates = [item for item in reviews if item["decision"] == "promote_review_candidate"] + doc_pdf = [ + item + for item in candidates + if item["extraction_method"] in {"docx", "pdfplumber", "pypdf"} + ] + high_signal_ocr = [ + item + for item in candidates + if item["extraction_method"] == "macos_vision" and item["extracted_character_count"] >= 2500 + ] + selected = doc_pdf[:10] + high_signal_ocr[:2] + return selected[:12] + + +def _source_grade(item: dict[str, Any]) -> tuple[str, str]: + path = str(item["path"]) + if "/Desktop/" in path or "/WorkBuddy/" in path: + return "quarantine_or_private", "Private or historical personal report; keep out of reference pack." + if item["extraction_method"] == "macos_vision": + if item["extracted_character_count"] >= 2500: + return "reference_only", "OCR text is high-signal but still requires manual verification against image." + return "ocr_low_confidence_reference_only", "OCR text too short or noisy for promotion review." + if "jyotish_training" in path: + return "reference_only", "Training material can inform style/workflow but cannot outrank source rules." + if "/文件仓库/印度占星文章/" in path or path.endswith("印度占星.pdf") or path.endswith("印度占星1.pdf"): + return "promote_to_reference_pack_candidate", "Source-like document; candidate for reference pack after conflict arbitration." + return "reference_only", "Useful context, not first-order rule source." + + +def _arbitrate_batch1_source(item: dict[str, Any], grade: str) -> dict[str, Any]: + if grade == "promote_to_reference_pack_candidate": + pack_decision = "reference_pack_candidate" + status = "batch1_arbitrated" + reason = ( + "Document/PDF source lineage is clear enough for reference/source pack candidacy; " + "still cannot enter runtime truth until source-pack visibility tests bind it explicitly." + ) + else: + pack_decision = "reference_only" + status = "batch1_arbitrated" + reason = "Kept as reference-only because the source is OCR-derived, training-oriented, private, or not first-order rule material." + return { + "status": status, + "pack_decision": pack_decision, + "reason": reason, + "runtime_truth_allowed": False, + "runtime_truth_reason": "Batch 1 arbitrates source-pack eligibility only; it does not promote runtime truth.", + } + + +def build_source_grading_batch1(*, write: bool = True) -> dict[str, Any]: + review_report = build_extraction_review(write=False) + total_review_candidates = sum(1 for item in review_report["reviews"] if item["decision"] == "promote_review_candidate") + selected = _source_grading_batch1_items(review_report["reviews"]) + graded_sources: list[dict[str, Any]] = [] + for item in selected: + grade, reason = _source_grade(item) + arbitration = _arbitrate_batch1_source(item, grade) + graded_sources.append( + { + "path": item["path"], + "source_scope": item["source_scope"], + "suffix": item["suffix"], + "byte_count": item["byte_count"], + "sha256": item["sha256"], + "extraction_method": item["extraction_method"], + "extracted_character_count": item["extracted_character_count"], + "text_sha256": item.get("text_sha256"), + "review_decision": item["decision"], + "source_grade": grade, + "grade_reason": reason, + "conflict_arbitration": arbitration, + "pack_decision": arbitration["pack_decision"], + "requires_conflict_arbitration": False, + "requires_source_pack_visibility_test": True, + "runtime_truth_allowed": False, + } + ) + grade_counts: dict[str, int] = {} + pack_decision_counts: dict[str, int] = {} + for item in graded_sources: + grade_counts[item["source_grade"]] = grade_counts.get(item["source_grade"], 0) + 1 + pack_decision_counts[item["pack_decision"]] = pack_decision_counts.get(item["pack_decision"], 0) + 1 + stored_text_payload_fields = sum(1 for item in graded_sources if "text" in item or "text_preview" in item) + reference_pack_ready_count = pack_decision_counts.get("reference_pack_candidate", 0) + report = { + "scope": "source-grading-batch1", + "status": "pass", + "generated_at": datetime.now(timezone.utc).isoformat(), + "mode": { + "runtime_truth_promotion": False, + "semantic_promotion": False, + "boundary": "Batch 1 grades extracted candidates only; no source enters runtime truth without a later source-pack test.", + }, + "summary": { + "total_review_candidates": total_review_candidates, + "selected_count": len(graded_sources), + "graded_files": len(graded_sources), + "runtime_promotions": 0, + "stored_text_payload_fields": stored_text_payload_fields, + }, + "grade_counts": dict(sorted(grade_counts.items())), + "pack_decision_counts": dict(sorted(pack_decision_counts.items())), + "conflict_arbitration": { + "status": "batch1_arbitrated", + "scope": "source_pack_eligibility_only", + "runtime_truth_allowed": False, + "reference_pack_ready_count": reference_pack_ready_count, + "runtime_truth_ready_count": 0, + "reference_pack_next_steps": [ + "wire selected candidates into interpretation_source_inventory/source pack", + "add strict workflow visibility tests", + "keep content-level precedence below core rules until line-level rule review exists", + ], + }, + "graded_sources": graded_sources, + "artifacts": { + "json_report": _relative(SOURCE_GRADING_BATCH1_JSON), + "markdown_report": _relative(SOURCE_GRADING_BATCH1_MD), + }, + "title": "Source Grading Batch 1 Manifest", + } + if write: + _write_reports(report, json_report=SOURCE_GRADING_BATCH1_JSON, markdown_report=SOURCE_GRADING_BATCH1_MD) + return report + + def _review_decision(item: dict[str, Any]) -> tuple[str, str]: path = str(item["path"]) if item["extraction_result"] == "extraction_failed": @@ -964,6 +1097,8 @@ def _render_markdown(report: dict[str, Any]) -> str: return _render_extraction_results_markdown(report) if report["scope"] == "extraction-review": return _render_extraction_review_markdown(report) + if report["scope"] == "source-grading-batch1": + return _render_source_grading_batch1_markdown(report) lines = [ f"# {report.get('title', 'Character-Level Inventory Manifest')}", "", @@ -1004,6 +1139,53 @@ def _render_markdown(report: dict[str, Any]) -> str: return "\n".join(lines) +def _render_source_grading_batch1_markdown(report: dict[str, Any]) -> str: + lines = [ + f"# {report.get('title', 'Source Grading Batch 1 Manifest')}", + "", + f"- status: {report['status']}", + f"- scope: {report['scope']}", + f"- generated_at: {report['generated_at']}", + f"- total_review_candidates: {report['summary']['total_review_candidates']}", + f"- selected_count: {report['summary']['selected_count']}", + f"- graded_files: {report['summary']['graded_files']}", + f"- runtime_promotions: {report['summary']['runtime_promotions']}", + f"- stored_text_payload_fields: {report['summary']['stored_text_payload_fields']}", + "", + "## Grade Counts", + "", + ] + for name, count in report["grade_counts"].items(): + lines.append(f"- `{name}`: {count}") + lines.extend( + [ + "", + "## Pack Decisions", + "", + ] + ) + for name, count in report["pack_decision_counts"].items(): + lines.append(f"- `{name}`: {count}") + lines.extend( + [ + "", + "## Conflict Arbitration", + "", + f"- status: {report['conflict_arbitration']['status']}", + f"- scope: {report['conflict_arbitration']['scope']}", + f"- reference_pack_ready_count: {report['conflict_arbitration']['reference_pack_ready_count']}", + f"- runtime_truth_ready_count: {report['conflict_arbitration']['runtime_truth_ready_count']}", + f"- runtime_truth_allowed: {report['conflict_arbitration']['runtime_truth_allowed']}", + "", + "## Boundary", + "", + report["mode"]["boundary"], + "", + ] + ) + return "\n".join(lines) + + def _render_extraction_review_markdown(report: dict[str, Any]) -> str: lines = [ f"# {report.get('title', 'Extraction Review Manifest')}", @@ -1120,7 +1302,14 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--scope", default="project", - choices=["project", "external", "extraction-queue", "extraction-results", "extraction-review"], + choices=[ + "project", + "external", + "extraction-queue", + "extraction-results", + "extraction-review", + "source-grading-batch1", + ], ) parser.add_argument("--no-write", action="store_true", help="Print the manifest without writing report artifacts.") parser.add_argument("--summary-only", action="store_true", help="Print only summary fields; still writes full artifacts unless --no-write is set.") diff --git a/tests/test_character_level_inventory_manifest.py b/tests/test_character_level_inventory_manifest.py index c0f8751a..8a68125b 100644 --- a/tests/test_character_level_inventory_manifest.py +++ b/tests/test_character_level_inventory_manifest.py @@ -255,3 +255,52 @@ def test_extraction_review_grades_extracted_sources_without_promoting_runtime_tr assert json_path.exists() assert md_path.exists() assert "Extraction Review Manifest" in md_path.read_text(encoding="utf-8") + + +def test_source_grading_batch1_classifies_promote_review_candidates_without_runtime_promotion() -> None: + report = _run_manifest("--scope", "source-grading-batch1") + + assert report["scope"] == "source-grading-batch1" + assert report["status"] == "pass" + assert report["summary"]["total_review_candidates"] == 38 + assert report["summary"]["selected_count"] == 12 + assert report["summary"]["graded_files"] == 12 + assert report["summary"]["runtime_promotions"] == 0 + assert report["summary"]["stored_text_payload_fields"] == 0 + assert report["grade_counts"]["promote_to_reference_pack_candidate"] >= 4 + assert report["grade_counts"]["reference_only"] >= 1 + assert report["grade_counts"].get("quarantine_or_private", 0) == 0 + assert report["conflict_arbitration"]["runtime_truth_allowed"] is False + assert report["conflict_arbitration"]["status"] == "batch1_arbitrated" + assert report["conflict_arbitration"]["reference_pack_ready_count"] == report["pack_decision_counts"]["reference_pack_candidate"] + assert report["conflict_arbitration"]["runtime_truth_ready_count"] == 0 + assert report["pack_decision_counts"]["reference_pack_candidate"] == 9 + assert report["pack_decision_counts"]["reference_only"] == 3 + assert report["pack_decision_counts"].get("runtime_truth", 0) == 0 + + for item in report["graded_sources"]: + assert item["sha256"] + assert item["source_grade"] in { + "promote_to_reference_pack_candidate", + "reference_only", + "quarantine_or_private", + "ocr_low_confidence_reference_only", + } + assert item["runtime_truth_allowed"] is False + assert item["requires_conflict_arbitration"] is False + assert item["requires_source_pack_visibility_test"] is True + assert item["conflict_arbitration"]["status"] == "batch1_arbitrated" + assert item["conflict_arbitration"]["runtime_truth_allowed"] is False + assert item["pack_decision"] in {"reference_pack_candidate", "reference_only"} + if item["source_grade"] == "promote_to_reference_pack_candidate": + assert item["pack_decision"] == "reference_pack_candidate" + else: + assert item["pack_decision"] == "reference_only" + assert "text" not in item + assert "text_preview" not in item + + json_path = ROOT / report["artifacts"]["json_report"] + md_path = ROOT / report["artifacts"]["markdown_report"] + assert json_path.exists() + assert md_path.exists() + assert "Source Grading Batch 1 Manifest" in md_path.read_text(encoding="utf-8")