Owner order 2026-11-07 (astrologer plan item 2): metadata-only catalog and rule citations in git; full texts and passage index stay local (gitignored). POST /api/book_citations and a bounded consultation tool with chart condition check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DbBXoyn7Hj3LCW3BUs5bZ8
966 lines
52 KiB
Python
966 lines
52 KiB
Python
#!/usr/bin/env python3
|
||
"""Sentence-level book citation index and search (owner order 2026-11-07).
|
||
|
||
Every timing / event / technique answer must be able to cite a book line by
|
||
line: book -> page -> chapter / verse -> text -> applicability -> whether the
|
||
current chart meets it, and say which kind of source it is (classical rule,
|
||
modern commentary, software implementation, web experience, unverified).
|
||
|
||
Licence boundary (docs/tasks/TASK-upstream-sync6-20261007.md decision 2 and
|
||
the upstream corpus README): the book texts are third-party works that the
|
||
repository licence does not cover, and this repository has a public mirror.
|
||
So only metadata travels in git:
|
||
|
||
* ``references/book_corpus/catalog.json`` - one row per source text: title,
|
||
author, edition, language, rule type, SHA256 of the upstream text. No text.
|
||
* ``references/book_corpus/index/rule_citations.json`` - built from the
|
||
upstream ``book_rule_packets.json`` (non-verbatim summaries and locators).
|
||
* ``references/book_corpus/index/rule_conditions.json`` - hand-written,
|
||
machine-checkable preconditions for the packets that have them.
|
||
* ``references/book_corpus/index/passage_index_manifest.json`` - counts and
|
||
the content digest of the local passage index, no text.
|
||
|
||
The texts themselves are synced from a local upstream checkout into
|
||
``references/book_corpus/local/`` (gitignored) and indexed into
|
||
``references/book_corpus/index/book_passages.sqlite`` (gitignored). Search
|
||
returns short excerpts only. Where the passage index is absent (for example a
|
||
deploy image without the corpus) search still answers from the rule packets
|
||
and says the passage layer is unavailable.
|
||
|
||
CLI::
|
||
|
||
python -m scripts.book_citations.index catalog --upstream /path/to/yinduzhanxing
|
||
python -m scripts.book_citations.index sync --upstream /path/to/yinduzhanxing
|
||
python -m scripts.book_citations.index build
|
||
python -m scripts.book_citations.index rules
|
||
python -m scripts.book_citations.index search "upapada second house" --top-k 5
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import sqlite3
|
||
from collections.abc import Iterable
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
CORPUS_DIR = ROOT / "references" / "book_corpus"
|
||
CATALOG_PATH = CORPUS_DIR / "catalog.json"
|
||
LOCAL_TEXT_DIR = CORPUS_DIR / "local"
|
||
INDEX_DIR = CORPUS_DIR / "index"
|
||
PASSAGE_DB_PATH = INDEX_DIR / "book_passages.sqlite"
|
||
PASSAGE_MANIFEST_PATH = INDEX_DIR / "passage_index_manifest.json"
|
||
RULE_CITATIONS_PATH = INDEX_DIR / "rule_citations.json"
|
||
RULE_CONDITIONS_PATH = INDEX_DIR / "rule_conditions.json"
|
||
RULE_PACKETS_PATH = ROOT / "references" / "book_rule_packets.json"
|
||
RULE_LINEAGE_PATH = ROOT / "references" / "book_rule_technique_lineage.json"
|
||
ADMISSION_CARDS_PATH = ROOT / "references" / "book_admission_cards.json"
|
||
|
||
BUILDER_VERSION = "book-citation-index.v1"
|
||
MAX_RESPONSE_CHARS = 3900
|
||
MAX_TOP_K = 8
|
||
DEFAULT_TOP_K = 5
|
||
EXCERPT_STEPS = (280, 200, 140, 90)
|
||
MAX_SENTENCE_CHARS = 1000
|
||
|
||
RULE_TYPES = {
|
||
"classical_rule": "原典规则",
|
||
"modern_commentary": "现代注释",
|
||
"software_implementation": "软件实现",
|
||
"web_experience": "网友经验",
|
||
"unverified": "尚未验证",
|
||
}
|
||
CHART_CHECK_STATUSES = ("satisfied", "not_satisfied", "needs_reader_check")
|
||
|
||
COPYRIGHT_BOUNDARY = (
|
||
"Third-party works keep their own rights; the repository licence does not relicense them. "
|
||
"Git carries metadata, hashes, locators and non-verbatim rule summaries only. Full texts stay "
|
||
"local (references/book_corpus/local, gitignored); search returns short excerpts with page locators."
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Catalog: curated metadata for the upstream texts (matched by file name).
|
||
# rule_type is the book-level default. classical_with_commentary books give
|
||
# classical_rule to passages that carry a detected sloka / sutra id and
|
||
# modern_commentary to the rest.
|
||
# ---------------------------------------------------------------------------
|
||
_CURATED: tuple[tuple[str, dict[str, Any]], ...] = (
|
||
("(NEW)Science of Light_", dict(book_id="science_of_light_v1", title="The Science of Light, Vol. 1: An Introduction to Vedic Astrology", author="Freedom Tobias Cole", edition="ISBN 9780985012229", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
|
||
("A Manual of Hindu Astrology", dict(book_id="raman_manual_hindu_astrology", title="A Manual of Hindu Astrology", author="B. V. Raman", edition="unstated reprint", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("A course on Jaimini Maharṣi", dict(book_id="rath_jaimini_upadesa_sutra", title="A Course on Jaimini Maharṣi's Upadeśa Sūtra", author="Sanjay Rath", edition="New Delhi 2009, ISBN 9780976517726", language="en", rule_type="classical_with_commentary", school=["jaimini"])),
|
||
("Aarambha Jotisha", dict(book_id="raman_aarambha_jyotisha", title="Aarambha Jyotisha", author="B. V. Raman", edition="USB Publishers (Kannada text)", language="kn", rule_type="modern_commentary", school=["parashara"])),
|
||
("Ancient Hindu Astrology For Modern", dict(book_id="braha_ancient_hindu_astrology", title="Ancient Hindu Astrology for the Modern Western Astrologer", author="James Braha", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("Astrology For Beginners", dict(book_id="raman_astrology_for_beginners", title="Astrology for Beginners", author="B. V. Raman", edition="IBH Publication", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("BPHS读书笔记", dict(book_id="bphs_reading_notes", title="BPHS 读书笔记", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["parashara"])),
|
||
("Brihat Jataka 读书笔记", dict(book_id="brihat_jataka_reading_notes", title="Brihat Jataka 读书笔记", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["parashara"])),
|
||
("Crux of Vedic Astrology", dict(book_id="rath_crux_timing_of_events", title="Crux of Vedic Astrology: Timing of Events", author="Sanjay Rath", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
|
||
("Deva Keralam", dict(book_id="deva_keralam_v2", title="Deva Keralam (Chandra Kala Nadi), Vol. 2", author="tr. R. Santhanam", edition="unstated", language="en", rule_type="classical_with_commentary", school=["nadi"])),
|
||
("Hindu Predictive Astrology", dict(book_id="raman_hindu_predictive_astrology", title="Hindu Predictive Astrology", author="B. V. Raman", edition="IBH Prakashana 1981, ISBN 9788119394883", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("How to Judge a Horoscope (Vol 1)", dict(book_id="raman_how_to_judge_v1", title="How to Judge a Horoscope, Vol. 1", author="B. V. Raman", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("How to Judge a Horoscope (Vol 2)", dict(book_id="raman_how_to_judge_v2", title="How to Judge a Horoscope, Vol. 2", author="B. V. Raman", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("Jyotish Fundamentals of Rao", dict(book_id="rg_rao_nadi_fundamentals", title="Jyotish Fundamentals of Rao's System of Nadi", author="R. G. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["nadi"])),
|
||
("Learn Successful Predictive Techniques", dict(book_id="kn_rao_predictive_techniques", title="Learn Successful Predictive Techniques of Hindu Astrology", author="K. N. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("Light on Life", dict(book_id="light_on_life", title="Light on Life: An Introduction to the Astrology of India", author="Hart de Fouw, Robert Svoboda", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("Muhurtha or Electional Astrology", dict(book_id="raman_muhurtha_1969", title="Muhurtha or Electional Astrology", author="B. V. Raman", edition="1969", language="en", rule_type="modern_commentary", school=["muhurta"])),
|
||
("Muhurtha, Electional Astrology", dict(book_id="raman_muhurtha", title="Muhurtha (Electional Astrology)", author="B. V. Raman", edition="unstated reprint", language="en", rule_type="modern_commentary", school=["muhurta"])),
|
||
("Naadi Predictions", dict(book_id="oak_naadi_predictions", title="Naadi Predictions", author="Shashikant Oak", edition="Diamond Pocket Books 2014", language="en", rule_type="modern_commentary", school=["nadi"])),
|
||
("Nadi Astrology -- Chandulal", dict(book_id="patel_nadi_astrology", title="Nadi Astrology", author="Chandulal S. Patel", edition="Sagar Publications 2002, ISBN 9788170820161", language="en", rule_type="modern_commentary", school=["nadi"])),
|
||
("Nadi_Stellar Effects", dict(book_id="nadi_stellar_effects", title="Nadi: Stellar Effects", author="unknown", edition="2022", language="en", rule_type="modern_commentary", school=["nadi"])),
|
||
("Neol Tyl太阳弧读书摘要", dict(book_id="tyl_solar_arc_notes", title="Noel Tyl 太阳弧读书摘要", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["western"])),
|
||
("PVR_Integrated_Approach", dict(book_id="pvr_integrated_approach", title="Vedic Astrology: An Integrated Approach", author="P. V. R. Narasimha Rao", edition="2000, 2010 update (author's site edition)", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
|
||
("Prasna Marga (Pt_ 2)", dict(book_id="prasna_marga_v2", title="Prasna Marga, Part 2", author="tr. B. V. Raman", edition="Motilal Banarsidass 2010, ISBN 9788120809147", language="en", rule_type="classical_with_commentary", school=["prasna"])),
|
||
("Predict Effectively through Yogini Dasha", dict(book_id="goel_yogini_dasha", title="Predict Effectively through Yogini Dasha", author="V. P. Goel", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("Predicting Through Shasti Hayani Dasha", dict(book_id="goel_shasti_hayani_dasha", title="Predicting through Shashti Hayani Dasha", author="V. P. Goel", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
|
||
("Predicting through Jaimini Astrology", dict(book_id="goel_jaimini", title="Predicting through Jaimini Astrology", author="V. P. Goel", edition="unstated", language="en", rule_type="modern_commentary", school=["jaimini"])),
|
||
("Predicting through Jaiminis Chara Dasha", dict(book_id="kn_rao_chara_dasha", title="Predicting through Jaimini's Chara Dasha", author="K. N. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["jaimini"])),
|
||
("PredictingMajorLifeEvents", dict(book_id="boney_major_life_events", title="Predicting Major Life Events: A Composite Approach", author="Marc Boney", edition="Sarasvati Publications 2020", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
|
||
("Profession From The Position", dict(book_id="rg_rao_profession", title="Profession from the Position of Planets", author="R. G. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["nadi"])),
|
||
("Science of Light Volume 2", dict(book_id="science_of_light_v2", title="The Science of Light, Vol. 2", author="Freedom Tobias Cole", edition="ISBN 9780985012205", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
|
||
("The Horary Textbook", dict(book_id="horary_textbook_zh", title="The Horary Textbook(中文译本 2.0)", author="John Frawley (translation)", edition="translation 2.0", language="zh", rule_type="modern_commentary", school=["western"])),
|
||
("narayana-dasa-by-sanjay-rath", dict(book_id="rath_narayana_dasa", title="Narayana Dasa", author="Sanjay Rath", edition="unstated", language="en", rule_type="modern_commentary", school=["jaimini"])),
|
||
("《Beneath a Vedic Sky》节选", dict(book_id="beneath_a_vedic_sky_excerpt", title="Beneath a Vedic Sky(节选)", author="William R. Levacy (excerpt notes)", edition="excerpt", language="zh", rule_type="unverified", school=["parashara"])),
|
||
("印度占星读书摘要", dict(book_id="chinese_reading_summary", title="印度占星读书摘要", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["parashara"])),
|
||
("秦瑞生", dict(book_id="qin_indian_astrology", title="印度占星学:精准解读先天格局,论断命运走势", author="秦瑞生", edition="繁体横版扫描", language="zh-Hant", rule_type="modern_commentary", school=["parashara"])),
|
||
("辅助译本 - Brihat Parashara Hora Shastra", dict(book_id="bphs_portuguese_kerr", title="Brihat Parashara Hora Shastra (Portuguese translation)", author="tr. Andre Kerr", edition="2019", language="pt", rule_type="classical_with_commentary", school=["parashara"])),
|
||
("Installation Guide", dict(book_id="parasharas_light_install_guide", title="Parashara's Light Installation Guide", author="GeoVision Software", edition="unstated", language="en", rule_type="software_implementation", school=["software"])),
|
||
("晴空月明", dict(book_id="qingkong_vedic_astrology", title="印度占星学全接触", author="晴空月明", edition="255-page PDF", language="zh", rule_type="modern_commentary", school=["parashara"])),
|
||
("洪能平", dict(book_id="hong_nengping_intro", title="印度占星学入门", author="洪能平", edition="博扬文化, vertical print", language="zh-Hant", rule_type="modern_commentary", school=["parashara"])),
|
||
)
|
||
|
||
_UPSTREAM_MANIFESTS = (
|
||
("references/book_corpus/manifest.json", "references/book_corpus", "book"),
|
||
("references/book_corpus/supplement_manifest.json", "references/book_corpus", "book"),
|
||
("references/book_corpus/ocr_manifest.json", "references/book_corpus", "book_ocr"),
|
||
("references/book_corpus/vertical_ocr_manifest.json", "references/book_corpus", "book_ocr"),
|
||
("references/article_corpus/manifest.json", "references/article_corpus", "article"),
|
||
("references/article_corpus/ocr_manifest.json", "references/article_corpus", "article_ocr"),
|
||
)
|
||
|
||
# Admission card source_id -> catalog book_id (upstream book_admission_cards.json).
|
||
_ADMISSION_TO_BOOK = {
|
||
"light_on_life": "light_on_life", "shasti_hayani_dasha": "goel_shasti_hayani_dasha",
|
||
"raman_horoscope_v1": "raman_how_to_judge_v1", "raman_horoscope_v2": "raman_how_to_judge_v2",
|
||
"braha_ancient_hindu_astrology": "braha_ancient_hindu_astrology", "goel_jaimini": "goel_jaimini",
|
||
"rao_chara_dasha": "kn_rao_chara_dasha", "boney_major_life_events": "boney_major_life_events",
|
||
"goel_yogini_dasha": "goel_yogini_dasha", "rao_predictive_techniques": "kn_rao_predictive_techniques",
|
||
"chinese_vedic_summary": "chinese_reading_summary", "qin_indian_astrology": "qin_indian_astrology",
|
||
"horary_textbook_translation": "horary_textbook_zh", "brihat_jataka_notes": "brihat_jataka_reading_notes",
|
||
"bphs_notes": "bphs_reading_notes", "science_of_light_v1": "science_of_light_v1",
|
||
"science_of_light_v2": "science_of_light_v2",
|
||
}
|
||
# Rule packet source_id -> catalog book_id, for packets that cite an in-corpus book.
|
||
_PACKET_SOURCE_TO_BOOK = {
|
||
**_ADMISSION_TO_BOOK,
|
||
"local:raman_manual_hindu_astrology": "raman_manual_hindu_astrology",
|
||
"local:patel_nadi_astrology_2002": "patel_nadi_astrology",
|
||
"web:pvr_vedic_astro_textbook": "pvr_integrated_approach",
|
||
"web:rath_narayana_dasa": "rath_narayana_dasa",
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Technique tags (keyword map; tags are a search aid, not a doctrinal claim).
|
||
# ---------------------------------------------------------------------------
|
||
TECHNIQUE_PATTERNS: tuple[tuple[str, str], ...] = (
|
||
("vimshottari", r"vim[sś]h?ottari|maha\s?da[sś]h?a|antar\s?da[sś]h?a|bhukti|pratyantar|大运|小运"),
|
||
("narayana_dasha", r"n[aā]r[aā]ya[nṇ]a\s?da[sś]h?a|narayana"),
|
||
("chara_dasha", r"chara\s?da[sś]h?a|char\s?da[sś]h?a"),
|
||
("yogini_dasha", r"yogini"),
|
||
("other_dasha", r"ashtottari|shash?ti\s?hayani|kalachakra|sudar[sś]h?ana|moola\s?da[sś]a|niryana"),
|
||
("jaimini", r"jaimini|karaka?m[sś]h?a|atma\s?karaka|dara\s?karaka|arudha|upapada|argala|chara\s?karaka"),
|
||
("divisional", r"nava[mṁ][sś]h?a|da[sś]a[mṁ][sś]h?a|\bd-?(?:2|3|4|7|9|10|12|16|20|24|27|30|40|45|60)\b|varga|分盘"),
|
||
("ashtakavarga", r"ashtakavarga|a[sṣ][tṭ]akavarga|bindu"),
|
||
("shadbala", r"shad\s?bala|[sṣ]a[dḍ]bala|bhava\s?bala"),
|
||
("transit", r"transit|gochara?|行运|过运"),
|
||
("tajika_varshaphala", r"t[aā]jika|varsha\s?phala|annual\s+chart|muntha|sahams?|年盘"),
|
||
("kp_sub_lord", r"krishnamurti|\bkp\b|sub[\s-]?lord|significator|cuspal"),
|
||
("yoga", r"\byoga\b|yogas\b|格局"),
|
||
("functional_nature", r"functional(?:ly)?\s+(?:benefic|malefic)|yoga\s?karaka|功能性"),
|
||
("marriage", r"marriage|spouse|\bwife\b|husband|婚|配偶"),
|
||
("career", r"career|profession|occupation|10th\s+house|事业|职业"),
|
||
("wealth", r"wealth|dhana|riches|财富|财"),
|
||
("health_longevity", r"longevity|disease|illness|health|ayu[rs]?|寿|健康|疾病"),
|
||
("muhurta", r"muhurt?ha?|electional|择时|择日"),
|
||
("prasna", r"pra[sś]h?na|horary|问事|卜卦"),
|
||
("nadi", r"\bnaa?di\b"),
|
||
("rectification", r"rectification|birth\s?time|生时"),
|
||
)
|
||
_TECHNIQUE_RES = tuple((tag, re.compile(pattern, re.IGNORECASE)) for tag, pattern in TECHNIQUE_PATTERNS)
|
||
TECHNIQUE_TAGS = tuple(tag for tag, _ in TECHNIQUE_PATTERNS)
|
||
|
||
_PAGE_RE = re.compile(r"^##\s+Page\s+(\d+)\s*$")
|
||
_SEGMENT_RE = re.compile(r"^###\s+Image segment\s+(\d+)\b")
|
||
_CHAPTER_RE = re.compile(
|
||
r"^(?:(?:chapter|ch\.|lesson|part|section|adhyaya|adhyāya)\s*[\dIVXLC]+\b.{0,80}"
|
||
r"|\u7b2c[\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e\u96f6\u3007\d]+[\u7ae0\u8282\u7bc7\u8bb2].{0,40})$",
|
||
re.IGNORECASE,
|
||
)
|
||
_VERSE_RES = (
|
||
re.compile(r"\b(?:ch(?:apter)?\.?\s*(\d{1,3})\s*[,.;]?\s*(?:v|sl|verse|sloka|śloka)\.?\s*(\d{1,3}(?:[-–]\d{1,3})?))", re.IGNORECASE),
|
||
re.compile(r"\b(?:sloka|shloka|śloka|verse|sutra|sūtra|sl\.|st\.)\s*(?:no\.?\s*)?(\d{1,3}(?:[.\-–]\d{1,3}){0,2})\b", re.IGNORECASE),
|
||
re.compile(r"^\s*(\d{1,2}\.\d{1,2}\.\d{1,3})\b"),
|
||
)
|
||
_LEADING_NUMBER_RE = re.compile(r"^\s*\(?(\d{1,3})\)?[.)]\s+\S")
|
||
_CONDITION_RES = (
|
||
re.compile(r"^(?:if|when|whenever|should|in case|provided(?: that)?)\b[^.;:!?]{3,180}?(?=,|\bthen\b|$)", re.IGNORECASE),
|
||
re.compile(r"(?:\u5982\u679c|\u5047\u5982|\u82e5|\u5f53)[^\uff0c,\u3002\uff1b;]{2,60}"),
|
||
)
|
||
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
|
||
_INDIC_RE = re.compile(r"[\u0900-\u0dff]")
|
||
_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?;])\s+(?=[\"'(\[A-Z0-9]|[^\x00-\x7f])|(?<=[\u3002\uff01\uff1f\uff1b])")
|
||
_TERM_RE = re.compile(r"[0-9A-Za-z\u00c0-\u024f\u1e00-\u1eff']+|[\u3400-\u9fff]+|[\u0900-\u0dff]+")
|
||
# Words that alone must not pull a rule packet in (scored 1 instead of 2-3).
|
||
_GENERIC_TERMS = frozenset(["house", "houses", "lord", "lords", "sign", "signs", "first", "second", "third", "planet", "planets", "year", "years", "period", "periods", "chart", "dasa", "dasha", "rule", "rules", "from"])
|
||
_STOPWORDS = frozenset(["a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "the", "to", "with"])
|
||
|
||
SIGNS = ("Aries", "Taurus", "Gemini", "Cancer", "Leo", "Virgo", "Libra", "Scorpio",
|
||
"Sagittarius", "Capricorn", "Aquarius", "Pisces")
|
||
SIGN_LORDS = {"Aries": "Mars", "Taurus": "Venus", "Gemini": "Mercury", "Cancer": "Moon", "Leo": "Sun",
|
||
"Virgo": "Mercury", "Libra": "Venus", "Scorpio": "Mars", "Sagittarius": "Jupiter",
|
||
"Capricorn": "Saturn", "Aquarius": "Saturn", "Pisces": "Jupiter"}
|
||
|
||
|
||
def _sha256_bytes(raw: bytes) -> str:
|
||
return hashlib.sha256(raw).hexdigest()
|
||
|
||
|
||
def _load_json(path: Path) -> Any:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def _dump_json(path: Path, payload: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||
|
||
|
||
def technique_tags(text: str) -> list[str]:
|
||
return [tag for tag, pattern in _TECHNIQUE_RES if pattern.search(text)]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Catalog + sync
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _upstream_sources(upstream: Path) -> list[dict[str, Any]]:
|
||
out: list[dict[str, Any]] = []
|
||
for manifest_rel, base_rel, kind in _UPSTREAM_MANIFESTS:
|
||
manifest_path = upstream / manifest_rel
|
||
if not manifest_path.exists():
|
||
continue
|
||
manifest = _load_json(manifest_path)
|
||
for item in manifest.get("books") or manifest.get("artifacts") or []:
|
||
rel = Path(item["output_path"])
|
||
if rel.is_absolute() or ".." in rel.parts:
|
||
raise ValueError("upstream manifest path escapes its corpus")
|
||
path = upstream / base_rel / rel
|
||
raw = path.read_bytes()
|
||
digest = _sha256_bytes(raw)
|
||
if item.get("text_sha256") and item["text_sha256"] != digest:
|
||
raise ValueError(f"upstream text hash mismatch for a {kind} source")
|
||
out.append({"kind": kind, "name": rel.name, "path": path, "sha256": digest,
|
||
"quality": item.get("quality") or "source_transcription"})
|
||
return out
|
||
|
||
|
||
def _curated_for(name: str) -> dict[str, Any] | None:
|
||
for needle, meta in _CURATED:
|
||
if needle in name:
|
||
return dict(meta)
|
||
return None
|
||
|
||
|
||
def build_catalog(upstream: Path) -> dict[str, Any]:
|
||
"""Catalog rows for every upstream text. Metadata only; no text is copied."""
|
||
cards = {}
|
||
if ADMISSION_CARDS_PATH.exists():
|
||
for card in _load_json(ADMISSION_CARDS_PATH).get("cards", []):
|
||
book_id = _ADMISSION_TO_BOOK.get(card.get("source_id"))
|
||
if book_id:
|
||
cards[book_id] = card
|
||
rows: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for source in _upstream_sources(upstream):
|
||
meta = _curated_for(source["name"])
|
||
if meta is None:
|
||
if source["kind"] not in {"article", "article_ocr"}:
|
||
raise ValueError("uncurated upstream book; add it to _CURATED")
|
||
stem = Path(source["name"]).stem
|
||
title = stem if source["kind"] == "article" else f"Article screenshot {stem.split('_')[0]}"
|
||
meta = dict(book_id=f"article_{source['sha256'][:10]}", title=title,
|
||
author="collected web article", edition="web / social post",
|
||
language="zh" if source["kind"] == "article" else "mixed",
|
||
rule_type="web_experience", school=["web"])
|
||
if meta["book_id"] in seen:
|
||
raise ValueError("duplicate catalog book_id")
|
||
seen.add(meta["book_id"])
|
||
page_basis = {"book": "source_file_page", "book_ocr": "source_file_page",
|
||
"article": "none", "article_ocr": "image_segment"}[source["kind"]]
|
||
row = {
|
||
**meta,
|
||
"source_kind": source["kind"],
|
||
"text_sha256": source["sha256"],
|
||
"page_basis": page_basis,
|
||
"extraction_quality": source["quality"],
|
||
"license_boundary": "local_only_full_text",
|
||
}
|
||
card = cards.get(meta["book_id"])
|
||
if card:
|
||
row["admission_state"] = card.get("admission_state")
|
||
row["page_count"] = card.get("page_count")
|
||
rows.append(row)
|
||
rows.sort(key=lambda row: row["book_id"])
|
||
return {
|
||
"schema": "jyotish.book_catalog.v1",
|
||
"copyright_boundary": COPYRIGHT_BOUNDARY,
|
||
"rule_types": RULE_TYPES,
|
||
"rule_type_note": "classical_with_commentary: passages with a detected sloka/sutra id are classical_rule, the rest modern_commentary.",
|
||
"page_basis_note": "source_file_page is the PDF/file page index, not the printed page number.",
|
||
"books": rows,
|
||
}
|
||
|
||
|
||
def load_catalog(path: Path = CATALOG_PATH) -> dict[str, Any]:
|
||
return _load_json(path)
|
||
|
||
|
||
def sync_local_texts(upstream: Path, *, catalog_path: Path = CATALOG_PATH, dest: Path = LOCAL_TEXT_DIR) -> dict[str, int]:
|
||
"""Copy upstream texts whose SHA256 the catalog lists into the gitignored local dir."""
|
||
wanted = {row["text_sha256"]: row["book_id"] for row in load_catalog(catalog_path)["books"]}
|
||
dest.mkdir(parents=True, exist_ok=True)
|
||
copied = 0
|
||
for source in _upstream_sources(upstream):
|
||
book_id = wanted.get(source["sha256"])
|
||
if book_id:
|
||
shutil.copyfile(source["path"], dest / f"{book_id}.md")
|
||
copied += 1
|
||
return {"catalog_books": len(wanted), "copied": copied}
|
||
|
||
|
||
def local_text_dir() -> Path:
|
||
override = os.environ.get("JYOTISH_BOOK_CORPUS_DIR", "").strip()
|
||
return Path(override) if override else LOCAL_TEXT_DIR
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Passage extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _clean_join(lines: list[str]) -> str:
|
||
out = ""
|
||
for line in lines:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
if not out:
|
||
out = line
|
||
elif out.endswith("-") and not out.endswith(" -"):
|
||
out = out[:-1] + line
|
||
elif _CJK_RE.search(out[-1:]) or _CJK_RE.search(line[:1]):
|
||
out += line
|
||
else:
|
||
out += " " + line
|
||
return re.sub(r"\s+", " ", out).strip()
|
||
|
||
|
||
def _good_sentence(text: str) -> bool:
|
||
indic = len(_INDIC_RE.findall(text))
|
||
if indic >= 15:
|
||
return indic / max(1, len(text.replace(" ", ""))) > 0.5
|
||
cjk = len(_CJK_RE.findall(text))
|
||
if cjk >= 8:
|
||
return cjk / max(1, len(text.replace(" ", ""))) > 0.35
|
||
words = re.findall(r"[A-Za-zÀ-ɏḀ-ỿ]{2,}", text)
|
||
if len(words) < 5:
|
||
return False
|
||
letters = sum(ch.isalpha() for ch in text)
|
||
return letters / max(1, len(text)) > 0.6
|
||
|
||
|
||
def _detect_verse(paragraph: str, classical: bool) -> str | None:
|
||
for index, pattern in enumerate(_VERSE_RES):
|
||
match = pattern.search(paragraph)
|
||
if match:
|
||
if index == 0:
|
||
return f"{match.group(1)}.{match.group(2)}"
|
||
return match.group(1)
|
||
if classical:
|
||
match = _LEADING_NUMBER_RE.match(paragraph)
|
||
if match:
|
||
return match.group(1)
|
||
return None
|
||
|
||
|
||
_VERSE_PREFIX_RE = re.compile(r"^\s*(?:(?:sloka|shloka|verse|sutra|sl\.|st\.)\s*[\d.\-]+|\(?\d{1,3}(?:\.\d{1,3}){0,2}\)?[.)]?)\s*[:.-]?\s*", re.IGNORECASE)
|
||
_INNER_IF_RE = re.compile(r"\b(?:if|provided that)\b[^.;:!?]{3,180}?(?=,|\bthen\b|$)", re.IGNORECASE)
|
||
|
||
|
||
def _applicability(sentence: str) -> str | None:
|
||
sentence = _VERSE_PREFIX_RE.sub("", sentence, count=1)
|
||
for pattern in (*_CONDITION_RES, _INNER_IF_RE):
|
||
match = pattern.search(sentence)
|
||
if match:
|
||
return match.group(0).strip()[:200]
|
||
return None
|
||
|
||
|
||
def iter_passages(book: dict[str, Any], text: str) -> Iterable[dict[str, Any]]:
|
||
"""Yield sentence rows in source order. Deterministic for a given text."""
|
||
classical = book["rule_type"] == "classical_with_commentary"
|
||
page: int | None = None
|
||
chapter: str | None = None
|
||
started = book["page_basis"] == "none"
|
||
paragraph: list[str] = []
|
||
paragraph_line = 0
|
||
sentence_no = 0
|
||
|
||
def flush() -> Iterable[dict[str, Any]]:
|
||
nonlocal chapter, sentence_no
|
||
joined = _clean_join(paragraph)
|
||
if not joined:
|
||
return
|
||
if len(joined) <= 90 and _CHAPTER_RE.match(joined):
|
||
chapter = joined
|
||
return
|
||
verse = _detect_verse(joined, classical)
|
||
for raw in _SENTENCE_SPLIT_RE.split(joined):
|
||
sentence = raw.strip()
|
||
# A sentence that carries its own sloka / sutra marker starts that verse.
|
||
verse = _detect_verse(sentence, False) or verse
|
||
if not sentence or not _good_sentence(sentence):
|
||
continue
|
||
sentence_no += 1
|
||
rule_type = book["rule_type"]
|
||
if rule_type == "classical_with_commentary":
|
||
rule_type = "classical_rule" if verse else "modern_commentary"
|
||
yield {
|
||
"passage_id": f"{book['book_id']}:{page if page is not None else 0}:{sentence_no}",
|
||
"book_id": book["book_id"],
|
||
"page": page,
|
||
"line": paragraph_line,
|
||
"chapter": chapter,
|
||
"verse_id": verse,
|
||
"text": sentence[:MAX_SENTENCE_CHARS],
|
||
"applicability": _applicability(sentence),
|
||
"rule_type": rule_type,
|
||
"techniques": ",".join(technique_tags(sentence)),
|
||
}
|
||
|
||
for line_no, line in enumerate(text.splitlines(), start=1):
|
||
marker = _PAGE_RE.match(line) or _SEGMENT_RE.match(line)
|
||
if marker:
|
||
yield from flush()
|
||
paragraph = []
|
||
page = int(marker.group(1))
|
||
started = True
|
||
continue
|
||
if line.startswith("## Extracted Text"):
|
||
yield from flush()
|
||
paragraph = []
|
||
started = True
|
||
continue
|
||
if not started or line.startswith("# "):
|
||
continue
|
||
if not line.strip():
|
||
yield from flush()
|
||
paragraph = []
|
||
continue
|
||
if not paragraph:
|
||
paragraph_line = line_no
|
||
paragraph.append(line)
|
||
yield from flush()
|
||
|
||
|
||
_PASSAGE_COLUMNS = ("passage_id", "book_id", "page", "line", "chapter", "verse_id", "text",
|
||
"applicability", "rule_type", "techniques")
|
||
|
||
|
||
def build_passage_index(*, text_dir: Path | None = None, catalog_path: Path = CATALOG_PATH,
|
||
db_path: Path = PASSAGE_DB_PATH, manifest_path: Path | None = PASSAGE_MANIFEST_PATH) -> dict[str, Any]:
|
||
"""Build the SQLite passage index and its text-free manifest.
|
||
|
||
Determinism contract: same catalog + same texts -> same ``content_digest``
|
||
(SHA256 over the ordered rows) and byte-identical manifest. The SQLite file
|
||
itself is a cache and is not compared byte for byte.
|
||
"""
|
||
text_dir = text_dir or local_text_dir()
|
||
catalog = load_catalog(catalog_path)
|
||
if db_path.exists():
|
||
db_path.unlink()
|
||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||
connection = sqlite3.connect(db_path)
|
||
digest = hashlib.sha256()
|
||
books_manifest: list[dict[str, Any]] = []
|
||
missing: list[str] = []
|
||
total = 0
|
||
try:
|
||
connection.execute(
|
||
"CREATE TABLE passages (rowid INTEGER PRIMARY KEY, passage_id TEXT UNIQUE NOT NULL, book_id TEXT NOT NULL, "
|
||
"page INTEGER, line INTEGER, chapter TEXT, verse_id TEXT, text TEXT NOT NULL, applicability TEXT, "
|
||
"rule_type TEXT NOT NULL, techniques TEXT NOT NULL)")
|
||
connection.execute(
|
||
"CREATE VIRTUAL TABLE passages_fts USING fts5(text, content='passages', content_rowid='rowid', tokenize='trigram')")
|
||
for book in sorted(catalog["books"], key=lambda row: row["book_id"]):
|
||
path = text_dir / f"{book['book_id']}.md"
|
||
if not path.exists():
|
||
missing.append(book["book_id"])
|
||
continue
|
||
raw = path.read_bytes()
|
||
if _sha256_bytes(raw) != book["text_sha256"]:
|
||
raise ValueError(f"local text hash mismatch for {book['book_id']}; re-run sync")
|
||
stats = {"passages": 0, "with_verse": 0, "with_chapter": 0, "with_applicability": 0}
|
||
pages: set[int] = set()
|
||
for row in iter_passages(book, raw.decode("utf-8")):
|
||
values = tuple(row[column] for column in _PASSAGE_COLUMNS)
|
||
connection.execute(
|
||
f"INSERT INTO passages ({', '.join(_PASSAGE_COLUMNS)}) VALUES ({', '.join('?' * len(_PASSAGE_COLUMNS))})",
|
||
values)
|
||
digest.update(json.dumps(values, ensure_ascii=False).encode("utf-8") + b"\n")
|
||
stats["passages"] += 1
|
||
stats["with_verse"] += bool(row["verse_id"])
|
||
stats["with_chapter"] += bool(row["chapter"])
|
||
stats["with_applicability"] += bool(row["applicability"])
|
||
if row["page"] is not None:
|
||
pages.add(row["page"])
|
||
total += stats["passages"]
|
||
books_manifest.append({"book_id": book["book_id"], "title": book["title"], "rule_type": book["rule_type"],
|
||
"text_sha256": book["text_sha256"], "pages_with_passages": len(pages), **stats})
|
||
connection.execute("INSERT INTO passages_fts(passages_fts) VALUES ('rebuild')")
|
||
connection.commit()
|
||
finally:
|
||
connection.close()
|
||
manifest = {
|
||
"schema": "jyotish.book_passage_index_manifest.v1",
|
||
"builder_version": BUILDER_VERSION,
|
||
"copyright_boundary": COPYRIGHT_BOUNDARY,
|
||
"index_path": "references/book_corpus/index/book_passages.sqlite (gitignored, rebuilt locally)",
|
||
"content_digest": digest.hexdigest(),
|
||
"total_passages": total,
|
||
"books_indexed": len(books_manifest),
|
||
"books_missing_local_text": missing,
|
||
"books": books_manifest,
|
||
}
|
||
if manifest_path is not None:
|
||
_dump_json(manifest_path, manifest)
|
||
return manifest
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Rule packets -> citations
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _packet_rule_type(packet: dict[str, Any]) -> str:
|
||
source_id = str(packet.get("source_id") or "")
|
||
rule_id = str(packet.get("rule_id") or "")
|
||
if re.search(r"mhora|pyjhora|jhora", source_id + rule_id):
|
||
return "software_implementation"
|
||
if re.search(r"sripati|bphs", source_id):
|
||
return "classical_rule"
|
||
if packet.get("source_type") == "web_reference":
|
||
return "web_experience"
|
||
if source_id.startswith("web:"):
|
||
return "modern_commentary"
|
||
if source_id in _PACKET_SOURCE_TO_BOOK:
|
||
return "modern_commentary"
|
||
return "unverified"
|
||
|
||
|
||
def build_rule_citations(*, packets_path: Path = RULE_PACKETS_PATH, lineage_path: Path = RULE_LINEAGE_PATH,
|
||
conditions_path: Path = RULE_CONDITIONS_PATH, catalog_path: Path = CATALOG_PATH) -> dict[str, Any]:
|
||
packets = _load_json(packets_path)
|
||
lineage = {row["rule_id"]: row for row in _load_json(lineage_path).get("rows", [])} if lineage_path.exists() else {}
|
||
conditions = _load_json(conditions_path).get("rules", {}) if conditions_path.exists() else {}
|
||
books = {row["book_id"]: row for row in load_catalog(catalog_path)["books"]} if catalog_path.exists() else {}
|
||
rows: list[dict[str, Any]] = []
|
||
for packet in packets["packets"]:
|
||
rule_id = packet["rule_id"]
|
||
book_id = _PACKET_SOURCE_TO_BOOK.get(packet.get("source_id"))
|
||
book = books.get(book_id or "")
|
||
validation = packet.get("validation") or {}
|
||
lineage_row = lineage.get(rule_id, {})
|
||
summary = packet.get("non_verbatim_rule_summary") or ""
|
||
tags = sorted(set(lineage_row.get("techniques") or []) | set(technique_tags(f"{rule_id} {summary}")))
|
||
boundary = packet.get("assertion_boundary") or ""
|
||
not_validated = validation.get("not_validated_reason") or ""
|
||
rows.append({
|
||
"citation_id": f"rule:{rule_id}",
|
||
"rule_id": rule_id,
|
||
"source_id": packet.get("source_id"),
|
||
"book_id": book_id,
|
||
"book": book["title"] if book else (packet.get("source_attribution") or packet.get("source_id")),
|
||
"author": book["author"] if book else packet.get("source_attribution"),
|
||
"edition": book["edition"] if book else None,
|
||
"locator": packet.get("page_locator"),
|
||
"source_url": packet.get("source_url"),
|
||
"text_basis": "non_verbatim_summary",
|
||
"rule_summary": summary,
|
||
"rule_type": _packet_rule_type(packet),
|
||
"admission_state": packet.get("admission_state"),
|
||
"validation_status": validation.get("status"),
|
||
"applicability": " ".join(part for part in (boundary, not_validated) if part).strip() or None,
|
||
"conditions": conditions.get(rule_id),
|
||
"techniques": tags,
|
||
})
|
||
rows.sort(key=lambda row: row["citation_id"])
|
||
return {
|
||
"schema": "jyotish.book_rule_citations.v1",
|
||
"source": "references/book_rule_packets.json (upstream, non-verbatim summaries and locators)",
|
||
"copyright_boundary": packets.get("copyright_boundary"),
|
||
"rule_types": RULE_TYPES,
|
||
"citations": rows,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Chart condition check ("当前盘是否满足")
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _sign_name(value: Any) -> str | None:
|
||
if isinstance(value, dict):
|
||
value = value.get("sign")
|
||
if isinstance(value, str):
|
||
for sign in SIGNS:
|
||
if value.strip().casefold() == sign.casefold():
|
||
return sign
|
||
return None
|
||
|
||
|
||
def normalize_chart(chart: Any) -> dict[str, Any]:
|
||
"""Reduce an engine / evidence-card chart to signs, houses, lords and running dasha."""
|
||
if not isinstance(chart, dict):
|
||
return {}
|
||
houses: dict[int, str] = {}
|
||
raw_houses = chart.get("houses")
|
||
if isinstance(raw_houses, dict):
|
||
items = raw_houses.items()
|
||
elif isinstance(raw_houses, list):
|
||
items = ((row.get("number") if isinstance(row, dict) else None, row) for row in raw_houses)
|
||
else:
|
||
items = ()
|
||
for number, row in items:
|
||
try:
|
||
house = int(number)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
sign = _sign_name(row)
|
||
if 1 <= house <= 12 and sign:
|
||
houses[house] = sign
|
||
ascendant = chart.get("ascendant") if isinstance(chart.get("ascendant"), dict) else {}
|
||
lagna = _sign_name(ascendant) or houses.get(1)
|
||
if lagna and len(houses) < 12:
|
||
start = SIGNS.index(lagna)
|
||
houses = {house: SIGNS[(start + house - 1) % 12] for house in range(1, 13)}
|
||
planets: dict[str, dict[str, Any]] = {}
|
||
raw_planets = chart.get("planets")
|
||
planet_items = raw_planets.items() if isinstance(raw_planets, dict) else (
|
||
((row.get("name"), row) for row in raw_planets if isinstance(row, dict)) if isinstance(raw_planets, list) else ())
|
||
for name, row in planet_items:
|
||
if not isinstance(name, str) or not isinstance(row, dict):
|
||
continue
|
||
house = row.get("house")
|
||
sign = _sign_name(row)
|
||
if not isinstance(house, int) and sign and lagna:
|
||
house = (SIGNS.index(sign) - SIGNS.index(lagna)) % 12 + 1
|
||
planets[name.strip().title()] = {"sign": sign, "house": house if isinstance(house, int) else None}
|
||
dasha = chart.get("vimshottari") if isinstance(chart.get("vimshottari"), dict) else {}
|
||
|
||
def lord(value: Any) -> str | None:
|
||
value = value.get("lord") if isinstance(value, dict) else value
|
||
return value.strip().title() if isinstance(value, str) and value.strip() else None
|
||
|
||
return {
|
||
"lagna": lagna,
|
||
"lagna_nakshatra": ascendant.get("nakshatra") if isinstance(ascendant.get("nakshatra"), str) else None,
|
||
"houses": houses,
|
||
"planets": planets,
|
||
"mahadasha": lord(dasha.get("mahadasha") or dasha.get("md")),
|
||
"antardasha": lord(dasha.get("antardasha") or dasha.get("ad")),
|
||
}
|
||
|
||
|
||
def _check(condition: dict[str, Any], chart: dict[str, Any]) -> dict[str, Any]:
|
||
op = condition.get("op")
|
||
houses, planets = chart.get("houses") or {}, chart.get("planets") or {}
|
||
result: bool | None = None
|
||
observed: Any = None
|
||
if op == "lord_in_house":
|
||
sign = houses.get(int(condition["lord_of"]))
|
||
lord = SIGN_LORDS.get(sign or "")
|
||
house = (planets.get(lord or "") or {}).get("house")
|
||
observed = {"lord": lord, "house": house}
|
||
if lord and isinstance(house, int):
|
||
result = house == int(condition["house"])
|
||
elif op == "planet_in_house":
|
||
house = (planets.get(str(condition["planet"])) or {}).get("house")
|
||
observed = {"house": house}
|
||
if isinstance(house, int):
|
||
result = house in [int(value) for value in condition["houses"]]
|
||
elif op == "ascendant_nakshatra":
|
||
observed = {"nakshatra": chart.get("lagna_nakshatra")}
|
||
if chart.get("lagna_nakshatra"):
|
||
wanted = {str(value).casefold() for value in condition["nakshatras"]}
|
||
result = str(chart["lagna_nakshatra"]).casefold() in wanted
|
||
elif op == "vimshottari_pair":
|
||
observed = {"mahadasha": chart.get("mahadasha"), "antardasha": chart.get("antardasha")}
|
||
if chart.get("mahadasha") and chart.get("antardasha"):
|
||
result = [chart["mahadasha"], chart["antardasha"]] in [list(pair) for pair in condition["pairs"]]
|
||
return {"op": op, "expected": {k: v for k, v in condition.items() if k != "op"}, "observed": observed, "result": result}
|
||
|
||
|
||
def check_chart_conditions(conditions: dict[str, Any] | None, chart: Any) -> dict[str, Any]:
|
||
if not conditions or not conditions.get("all"):
|
||
return {"status": "needs_reader_check", "reason": "no_machine_checkable_conditions"}
|
||
normalized = normalize_chart(chart)
|
||
if not normalized.get("lagna") and not normalized.get("mahadasha"):
|
||
return {"status": "needs_reader_check", "reason": "no_chart_in_context"}
|
||
checks = [_check(condition, normalized) for condition in conditions["all"]]
|
||
residual = list(conditions.get("reader_checks") or [])
|
||
if any(check["result"] is False for check in checks):
|
||
status = "not_satisfied"
|
||
elif any(check["result"] is None for check in checks) or residual:
|
||
status = "needs_reader_check"
|
||
else:
|
||
status = "satisfied"
|
||
out: dict[str, Any] = {"status": status, "checks": checks}
|
||
if residual:
|
||
out["reader_checks"] = residual
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Search
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _terms(query: str) -> list[str]:
|
||
seen: list[str] = []
|
||
for term in _TERM_RE.findall(query or ""):
|
||
term = term.strip("'").casefold()
|
||
if term and term not in _STOPWORDS and term not in seen:
|
||
seen.append(term)
|
||
return seen[:8]
|
||
|
||
|
||
def _excerpt(text: str, terms: list[str], limit: int) -> str:
|
||
if len(text) <= limit:
|
||
return text
|
||
lower = text.casefold()
|
||
hit = min((lower.find(term) for term in terms if lower.find(term) >= 0), default=0)
|
||
start = max(0, min(hit - limit // 3, len(text) - limit))
|
||
snippet = text[start:start + limit].strip()
|
||
return ("…" if start > 0 else "") + snippet + ("…" if start + limit < len(text) else "")
|
||
|
||
|
||
def _search_passages(db_path: Path, terms: list[str], *, techniques: list[str], rule_types: list[str], limit: int) -> list[sqlite3.Row]:
|
||
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||
connection.row_factory = sqlite3.Row
|
||
try:
|
||
long_terms = [term for term in terms if len(term) >= 3]
|
||
short_terms = [term for term in terms if len(term) < 3]
|
||
filters, params = [], []
|
||
for term in short_terms:
|
||
filters.append("p.text LIKE ?")
|
||
params.append(f"%{term}%")
|
||
if techniques:
|
||
filters.append("(" + " OR ".join("(',' || p.techniques || ',') LIKE ?" for _ in techniques) + ")")
|
||
params.extend(f"%,{tag},%" for tag in techniques)
|
||
if rule_types:
|
||
filters.append(f"p.rule_type IN ({', '.join('?' * len(rule_types))})")
|
||
params.extend(rule_types)
|
||
where = (" AND " + " AND ".join(filters)) if filters else ""
|
||
if long_terms:
|
||
for joiner in (" AND ", " OR "):
|
||
match = joiner.join('"' + term.replace('"', '""') + '"' for term in long_terms)
|
||
rows = connection.execute(
|
||
"SELECT p.*, bm25(passages_fts) AS score FROM passages_fts JOIN passages p ON p.rowid = passages_fts.rowid "
|
||
f"WHERE passages_fts MATCH ?{where} ORDER BY score, p.rowid LIMIT ?",
|
||
(match, *params, limit)).fetchall()
|
||
if rows or len(long_terms) == 1:
|
||
return rows
|
||
return []
|
||
if not filters:
|
||
return []
|
||
return connection.execute(
|
||
f"SELECT p.*, 0.0 AS score FROM passages p WHERE 1=1{where} ORDER BY {_RULE_TYPE_ORDER}, p.rowid LIMIT ?", (*params, limit)).fetchall()
|
||
finally:
|
||
connection.close()
|
||
|
||
|
||
_RULE_TYPE_ORDER = ("CASE p.rule_type WHEN 'classical_rule' THEN 0 WHEN 'modern_commentary' THEN 1 "
|
||
"WHEN 'software_implementation' THEN 2 WHEN 'unverified' THEN 3 ELSE 4 END")
|
||
|
||
|
||
def _clip(value: Any, limit: int) -> Any:
|
||
if isinstance(value, str) and len(value) > limit:
|
||
return value[: limit - 1] + "…"
|
||
return value
|
||
|
||
|
||
def _rule_score(row: dict[str, Any], terms: list[str], techniques: list[str]) -> int:
|
||
hay = " ".join(str(row.get(key) or "") for key in ("rule_id", "rule_summary", "locator", "book")).casefold()
|
||
score = 0
|
||
for term in terms:
|
||
if term not in hay:
|
||
continue
|
||
if term in _GENERIC_TERMS:
|
||
score += 1
|
||
else:
|
||
score += 3 if term in row["rule_id"].casefold() else 2
|
||
if techniques and set(techniques) & set(row.get("techniques") or []):
|
||
score += 2
|
||
return score
|
||
|
||
|
||
def _bound(payload: dict[str, Any], max_chars: int) -> dict[str, Any]:
|
||
def size() -> int:
|
||
return len(json.dumps(payload, ensure_ascii=False))
|
||
|
||
for limit in EXCERPT_STEPS:
|
||
if size() <= max_chars:
|
||
break
|
||
for citation in payload["citations"]:
|
||
for key in ("original_text", "rule_summary", "applicability"):
|
||
if isinstance(citation.get(key), str) and len(citation[key]) > limit:
|
||
citation[key] = citation[key][: limit - 1] + "…"
|
||
while size() > max_chars and payload["citations"]:
|
||
payload["citations"].pop()
|
||
payload["truncated"] = True
|
||
return payload
|
||
|
||
|
||
def search_book_citations(
|
||
query: str,
|
||
*,
|
||
techniques: list[str] | None = None,
|
||
rule_types: list[str] | None = None,
|
||
top_k: int = DEFAULT_TOP_K,
|
||
chart: Any = None,
|
||
max_chars: int = MAX_RESPONSE_CHARS,
|
||
db_path: Path | None = None,
|
||
rule_citations_path: Path = RULE_CITATIONS_PATH,
|
||
catalog_path: Path = CATALOG_PATH,
|
||
) -> dict[str, Any]:
|
||
"""Top-k citations: rule packets first (with chart check), then book passages."""
|
||
db_path = db_path or PASSAGE_DB_PATH
|
||
terms = _terms(query)
|
||
techniques = [tag for tag in (techniques or []) if tag in TECHNIQUE_TAGS][:4]
|
||
rule_types = [kind for kind in (rule_types or []) if kind in RULE_TYPES]
|
||
top_k = max(1, min(MAX_TOP_K, int(top_k or DEFAULT_TOP_K)))
|
||
if not terms and not techniques:
|
||
raise ValueError("query or techniques required")
|
||
books = {row["book_id"]: row for row in load_catalog(catalog_path)["books"]}
|
||
citations: list[dict[str, Any]] = []
|
||
|
||
rules = _load_json(rule_citations_path)["citations"]
|
||
scored = sorted(((score, row) for row in rules
|
||
if (not rule_types or row["rule_type"] in rule_types)
|
||
and (score := _rule_score(row, terms, techniques)) >= min(4, 2 * len(terms))),
|
||
key=lambda pair: (-pair[0], pair[1]["citation_id"]))
|
||
rule_slots = min(2 if db_path.exists() else 3, top_k)
|
||
for _, row in scored[:rule_slots]:
|
||
citations.append({
|
||
"id": row["citation_id"], "kind": "rule_packet",
|
||
"book": row["book"], "author": row["author"], "edition": row["edition"],
|
||
# A packet has no verbatim text: rule_summary stands in for original_text,
|
||
# and its locator string carries page / chapter / verse together.
|
||
"page": _clip(row["locator"], 160), "page_basis": "locator_string",
|
||
"rule_summary": _clip(row["rule_summary"], 260),
|
||
"rule_type": row["rule_type"],
|
||
"admission_state": row["admission_state"],
|
||
"techniques": row["techniques"][:4],
|
||
"applicability": _clip(row["applicability"], 180),
|
||
"chart_check": check_chart_conditions(row.get("conditions"), chart),
|
||
**({"source_url": row["source_url"]} if row.get("source_url") else {}),
|
||
})
|
||
|
||
passages_status = "unavailable"
|
||
if db_path.exists():
|
||
passages_status = "available"
|
||
try:
|
||
hits = _search_passages(db_path, terms, techniques=techniques, rule_types=rule_types,
|
||
limit=top_k - len(citations))
|
||
except sqlite3.Error:
|
||
hits, passages_status = [], "error"
|
||
for hit in hits:
|
||
book = books.get(hit["book_id"], {})
|
||
citations.append({
|
||
"id": hit["passage_id"], "kind": "book_passage",
|
||
"book": book.get("title", hit["book_id"]), "author": book.get("author"), "edition": book.get("edition"),
|
||
"page": hit["page"], "page_basis": book.get("page_basis"), "chapter": hit["chapter"], "verse": hit["verse_id"],
|
||
"original_text": _excerpt(hit["text"], terms, EXCERPT_STEPS[0]),
|
||
"translation": None,
|
||
"rule_type": hit["rule_type"],
|
||
"techniques": [tag for tag in hit["techniques"].split(",") if tag][:4],
|
||
"applicability": hit["applicability"],
|
||
"chart_check": {"status": "needs_reader_check"},
|
||
**({"quality": book["extraction_quality"]}
|
||
if book.get("extraction_quality") not in (None, "source_transcription") else {}),
|
||
})
|
||
payload = {
|
||
"schema": "jyotish.book_citation_search.v1",
|
||
"query": query[:200],
|
||
"citations": citations,
|
||
"rule_type_labels": RULE_TYPES,
|
||
"coverage": {"rule_packets": len(rules), "passage_index": passages_status},
|
||
"boundary": "Citations locate sources, they do not validate a rule for this chart. Rule packets are "
|
||
"non-verbatim summaries; passage text is a short excerpt; source_file_page is the file page, not the printed page.",
|
||
}
|
||
return _bound(payload, max_chars)
|
||
|
||
|
||
def book_citation_api(body: dict[str, Any]) -> dict[str, Any]:
|
||
"""POST /api/book_citations body -> payload. Raises ValueError on bad input."""
|
||
if not isinstance(body, dict):
|
||
raise ValueError("body must be an object")
|
||
query = body.get("query") or ""
|
||
if not isinstance(query, str) or len(query) > 200:
|
||
raise ValueError("query must be a string of at most 200 characters")
|
||
techniques = body.get("techniques") or []
|
||
rule_types = body.get("rule_types") or []
|
||
if not isinstance(techniques, list) or not isinstance(rule_types, list):
|
||
raise ValueError("techniques and rule_types must be lists")
|
||
top_k = body.get("top_k", DEFAULT_TOP_K)
|
||
if not isinstance(top_k, int) or isinstance(top_k, bool):
|
||
raise ValueError("top_k must be an integer")
|
||
result = search_book_citations(query, techniques=[str(t) for t in techniques], rule_types=[str(r) for r in rule_types],
|
||
top_k=top_k, chart=body.get("chart"))
|
||
return {"success": True, "endpoint": "book_citations", **result}
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
sub = parser.add_subparsers(dest="command", required=True)
|
||
for name in ("catalog", "sync"):
|
||
cmd = sub.add_parser(name)
|
||
cmd.add_argument("--upstream", type=Path, required=True)
|
||
sub.add_parser("build")
|
||
sub.add_parser("rules")
|
||
search = sub.add_parser("search")
|
||
search.add_argument("query")
|
||
search.add_argument("--technique", action="append", default=[])
|
||
search.add_argument("--rule-type", action="append", default=[])
|
||
search.add_argument("--top-k", type=int, default=DEFAULT_TOP_K)
|
||
args = parser.parse_args(argv)
|
||
if args.command == "catalog":
|
||
_dump_json(CATALOG_PATH, build_catalog(args.upstream))
|
||
print(f"catalog: {len(load_catalog()['books'])} sources")
|
||
elif args.command == "sync":
|
||
print(json.dumps(sync_local_texts(args.upstream)))
|
||
elif args.command == "build":
|
||
manifest = build_passage_index()
|
||
print(json.dumps({k: manifest[k] for k in ("total_passages", "books_indexed", "content_digest")}))
|
||
elif args.command == "rules":
|
||
_dump_json(RULE_CITATIONS_PATH, build_rule_citations())
|
||
print(f"rule citations: {len(_load_json(RULE_CITATIONS_PATH)['citations'])}")
|
||
else:
|
||
print(json.dumps(search_book_citations(args.query, techniques=args.technique, rule_types=args.rule_type,
|
||
top_k=args.top_k), ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|