Files
Jyotisha/scripts/book_citations/index.py
T
ClaudeandClaude Opus 5.5 96be1174df feat(consult): sentence-level book citation index + search_jyotish_books tool
Owner order 2026-11-07 (astrologer plan item 2): metadata-only catalog and rule citations in git; full texts and passage index stay local (gitignored). POST /api/book_citations and a bounded consultation tool with chart condition check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Claude-Session: https://claude.ai/code/session_01DbBXoyn7Hj3LCW3BUs5bZ8
2026-11-07 14:43:43 +08:00

966 lines
52 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Sentence-level book citation index and search (owner order 2026-11-07).
Every timing / event / technique answer must be able to cite a book line by
line: book -> page -> chapter / verse -> text -> applicability -> whether the
current chart meets it, and say which kind of source it is (classical rule,
modern commentary, software implementation, web experience, unverified).
Licence boundary (docs/tasks/TASK-upstream-sync6-20261007.md decision 2 and
the upstream corpus README): the book texts are third-party works that the
repository licence does not cover, and this repository has a public mirror.
So only metadata travels in git:
* ``references/book_corpus/catalog.json`` - one row per source text: title,
author, edition, language, rule type, SHA256 of the upstream text. No text.
* ``references/book_corpus/index/rule_citations.json`` - built from the
upstream ``book_rule_packets.json`` (non-verbatim summaries and locators).
* ``references/book_corpus/index/rule_conditions.json`` - hand-written,
machine-checkable preconditions for the packets that have them.
* ``references/book_corpus/index/passage_index_manifest.json`` - counts and
the content digest of the local passage index, no text.
The texts themselves are synced from a local upstream checkout into
``references/book_corpus/local/`` (gitignored) and indexed into
``references/book_corpus/index/book_passages.sqlite`` (gitignored). Search
returns short excerpts only. Where the passage index is absent (for example a
deploy image without the corpus) search still answers from the rule packets
and says the passage layer is unavailable.
CLI::
python -m scripts.book_citations.index catalog --upstream /path/to/yinduzhanxing
python -m scripts.book_citations.index sync --upstream /path/to/yinduzhanxing
python -m scripts.book_citations.index build
python -m scripts.book_citations.index rules
python -m scripts.book_citations.index search "upapada second house" --top-k 5
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import shutil
import sqlite3
from collections.abc import Iterable
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[2]
CORPUS_DIR = ROOT / "references" / "book_corpus"
CATALOG_PATH = CORPUS_DIR / "catalog.json"
LOCAL_TEXT_DIR = CORPUS_DIR / "local"
INDEX_DIR = CORPUS_DIR / "index"
PASSAGE_DB_PATH = INDEX_DIR / "book_passages.sqlite"
PASSAGE_MANIFEST_PATH = INDEX_DIR / "passage_index_manifest.json"
RULE_CITATIONS_PATH = INDEX_DIR / "rule_citations.json"
RULE_CONDITIONS_PATH = INDEX_DIR / "rule_conditions.json"
RULE_PACKETS_PATH = ROOT / "references" / "book_rule_packets.json"
RULE_LINEAGE_PATH = ROOT / "references" / "book_rule_technique_lineage.json"
ADMISSION_CARDS_PATH = ROOT / "references" / "book_admission_cards.json"
BUILDER_VERSION = "book-citation-index.v1"
MAX_RESPONSE_CHARS = 3900
MAX_TOP_K = 8
DEFAULT_TOP_K = 5
EXCERPT_STEPS = (280, 200, 140, 90)
MAX_SENTENCE_CHARS = 1000
RULE_TYPES = {
"classical_rule": "原典规则",
"modern_commentary": "现代注释",
"software_implementation": "软件实现",
"web_experience": "网友经验",
"unverified": "尚未验证",
}
CHART_CHECK_STATUSES = ("satisfied", "not_satisfied", "needs_reader_check")
COPYRIGHT_BOUNDARY = (
"Third-party works keep their own rights; the repository licence does not relicense them. "
"Git carries metadata, hashes, locators and non-verbatim rule summaries only. Full texts stay "
"local (references/book_corpus/local, gitignored); search returns short excerpts with page locators."
)
# ---------------------------------------------------------------------------
# Catalog: curated metadata for the upstream texts (matched by file name).
# rule_type is the book-level default. classical_with_commentary books give
# classical_rule to passages that carry a detected sloka / sutra id and
# modern_commentary to the rest.
# ---------------------------------------------------------------------------
_CURATED: tuple[tuple[str, dict[str, Any]], ...] = (
("(NEW)Science of Light_", dict(book_id="science_of_light_v1", title="The Science of Light, Vol. 1: An Introduction to Vedic Astrology", author="Freedom Tobias Cole", edition="ISBN 9780985012229", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
("A Manual of Hindu Astrology", dict(book_id="raman_manual_hindu_astrology", title="A Manual of Hindu Astrology", author="B. V. Raman", edition="unstated reprint", language="en", rule_type="modern_commentary", school=["parashara"])),
("A course on Jaimini Maharṣi", dict(book_id="rath_jaimini_upadesa_sutra", title="A Course on Jaimini Maharṣi's Upadeśa Sūtra", author="Sanjay Rath", edition="New Delhi 2009, ISBN 9780976517726", language="en", rule_type="classical_with_commentary", school=["jaimini"])),
("Aarambha Jotisha", dict(book_id="raman_aarambha_jyotisha", title="Aarambha Jyotisha", author="B. V. Raman", edition="USB Publishers (Kannada text)", language="kn", rule_type="modern_commentary", school=["parashara"])),
("Ancient Hindu Astrology For Modern", dict(book_id="braha_ancient_hindu_astrology", title="Ancient Hindu Astrology for the Modern Western Astrologer", author="James Braha", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("Astrology For Beginners", dict(book_id="raman_astrology_for_beginners", title="Astrology for Beginners", author="B. V. Raman", edition="IBH Publication", language="en", rule_type="modern_commentary", school=["parashara"])),
("BPHS读书笔记", dict(book_id="bphs_reading_notes", title="BPHS 读书笔记", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["parashara"])),
("Brihat Jataka 读书笔记", dict(book_id="brihat_jataka_reading_notes", title="Brihat Jataka 读书笔记", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["parashara"])),
("Crux of Vedic Astrology", dict(book_id="rath_crux_timing_of_events", title="Crux of Vedic Astrology: Timing of Events", author="Sanjay Rath", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
("Deva Keralam", dict(book_id="deva_keralam_v2", title="Deva Keralam (Chandra Kala Nadi), Vol. 2", author="tr. R. Santhanam", edition="unstated", language="en", rule_type="classical_with_commentary", school=["nadi"])),
("Hindu Predictive Astrology", dict(book_id="raman_hindu_predictive_astrology", title="Hindu Predictive Astrology", author="B. V. Raman", edition="IBH Prakashana 1981, ISBN 9788119394883", language="en", rule_type="modern_commentary", school=["parashara"])),
("How to Judge a Horoscope (Vol 1)", dict(book_id="raman_how_to_judge_v1", title="How to Judge a Horoscope, Vol. 1", author="B. V. Raman", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("How to Judge a Horoscope (Vol 2)", dict(book_id="raman_how_to_judge_v2", title="How to Judge a Horoscope, Vol. 2", author="B. V. Raman", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("Jyotish Fundamentals of Rao", dict(book_id="rg_rao_nadi_fundamentals", title="Jyotish Fundamentals of Rao's System of Nadi", author="R. G. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["nadi"])),
("Learn Successful Predictive Techniques", dict(book_id="kn_rao_predictive_techniques", title="Learn Successful Predictive Techniques of Hindu Astrology", author="K. N. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("Light on Life", dict(book_id="light_on_life", title="Light on Life: An Introduction to the Astrology of India", author="Hart de Fouw, Robert Svoboda", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("Muhurtha or Electional Astrology", dict(book_id="raman_muhurtha_1969", title="Muhurtha or Electional Astrology", author="B. V. Raman", edition="1969", language="en", rule_type="modern_commentary", school=["muhurta"])),
("Muhurtha, Electional Astrology", dict(book_id="raman_muhurtha", title="Muhurtha (Electional Astrology)", author="B. V. Raman", edition="unstated reprint", language="en", rule_type="modern_commentary", school=["muhurta"])),
("Naadi Predictions", dict(book_id="oak_naadi_predictions", title="Naadi Predictions", author="Shashikant Oak", edition="Diamond Pocket Books 2014", language="en", rule_type="modern_commentary", school=["nadi"])),
("Nadi Astrology -- Chandulal", dict(book_id="patel_nadi_astrology", title="Nadi Astrology", author="Chandulal S. Patel", edition="Sagar Publications 2002, ISBN 9788170820161", language="en", rule_type="modern_commentary", school=["nadi"])),
("Nadi_Stellar Effects", dict(book_id="nadi_stellar_effects", title="Nadi: Stellar Effects", author="unknown", edition="2022", language="en", rule_type="modern_commentary", school=["nadi"])),
("Neol Tyl太阳弧读书摘要", dict(book_id="tyl_solar_arc_notes", title="Noel Tyl 太阳弧读书摘要", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["western"])),
("PVR_Integrated_Approach", dict(book_id="pvr_integrated_approach", title="Vedic Astrology: An Integrated Approach", author="P. V. R. Narasimha Rao", edition="2000, 2010 update (author's site edition)", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
("Prasna Marga (Pt_ 2)", dict(book_id="prasna_marga_v2", title="Prasna Marga, Part 2", author="tr. B. V. Raman", edition="Motilal Banarsidass 2010, ISBN 9788120809147", language="en", rule_type="classical_with_commentary", school=["prasna"])),
("Predict Effectively through Yogini Dasha", dict(book_id="goel_yogini_dasha", title="Predict Effectively through Yogini Dasha", author="V. P. Goel", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("Predicting Through Shasti Hayani Dasha", dict(book_id="goel_shasti_hayani_dasha", title="Predicting through Shashti Hayani Dasha", author="V. P. Goel", edition="unstated", language="en", rule_type="modern_commentary", school=["parashara"])),
("Predicting through Jaimini Astrology", dict(book_id="goel_jaimini", title="Predicting through Jaimini Astrology", author="V. P. Goel", edition="unstated", language="en", rule_type="modern_commentary", school=["jaimini"])),
("Predicting through Jaiminis Chara Dasha", dict(book_id="kn_rao_chara_dasha", title="Predicting through Jaimini's Chara Dasha", author="K. N. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["jaimini"])),
("PredictingMajorLifeEvents", dict(book_id="boney_major_life_events", title="Predicting Major Life Events: A Composite Approach", author="Marc Boney", edition="Sarasvati Publications 2020", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
("Profession From The Position", dict(book_id="rg_rao_profession", title="Profession from the Position of Planets", author="R. G. Rao", edition="unstated", language="en", rule_type="modern_commentary", school=["nadi"])),
("Science of Light Volume 2", dict(book_id="science_of_light_v2", title="The Science of Light, Vol. 2", author="Freedom Tobias Cole", edition="ISBN 9780985012205", language="en", rule_type="modern_commentary", school=["parashara", "jaimini"])),
("The Horary Textbook", dict(book_id="horary_textbook_zh", title="The Horary Textbook(中文译本 2.0)", author="John Frawley (translation)", edition="translation 2.0", language="zh", rule_type="modern_commentary", school=["western"])),
("narayana-dasa-by-sanjay-rath", dict(book_id="rath_narayana_dasa", title="Narayana Dasa", author="Sanjay Rath", edition="unstated", language="en", rule_type="modern_commentary", school=["jaimini"])),
("《Beneath a Vedic Sky》节选", dict(book_id="beneath_a_vedic_sky_excerpt", title="Beneath a Vedic Sky(节选)", author="William R. Levacy (excerpt notes)", edition="excerpt", language="zh", rule_type="unverified", school=["parashara"])),
("印度占星读书摘要", dict(book_id="chinese_reading_summary", title="印度占星读书摘要", author="unknown reader", edition="reading notes", language="zh", rule_type="unverified", school=["parashara"])),
("秦瑞生", dict(book_id="qin_indian_astrology", title="印度占星学:精准解读先天格局,论断命运走势", author="秦瑞生", edition="繁体横版扫描", language="zh-Hant", rule_type="modern_commentary", school=["parashara"])),
("辅助译本 - Brihat Parashara Hora Shastra", dict(book_id="bphs_portuguese_kerr", title="Brihat Parashara Hora Shastra (Portuguese translation)", author="tr. Andre Kerr", edition="2019", language="pt", rule_type="classical_with_commentary", school=["parashara"])),
("Installation Guide", dict(book_id="parasharas_light_install_guide", title="Parashara's Light Installation Guide", author="GeoVision Software", edition="unstated", language="en", rule_type="software_implementation", school=["software"])),
("晴空月明", dict(book_id="qingkong_vedic_astrology", title="印度占星学全接触", author="晴空月明", edition="255-page PDF", language="zh", rule_type="modern_commentary", school=["parashara"])),
("洪能平", dict(book_id="hong_nengping_intro", title="印度占星学入门", author="洪能平", edition="博扬文化, vertical print", language="zh-Hant", rule_type="modern_commentary", school=["parashara"])),
)
_UPSTREAM_MANIFESTS = (
("references/book_corpus/manifest.json", "references/book_corpus", "book"),
("references/book_corpus/supplement_manifest.json", "references/book_corpus", "book"),
("references/book_corpus/ocr_manifest.json", "references/book_corpus", "book_ocr"),
("references/book_corpus/vertical_ocr_manifest.json", "references/book_corpus", "book_ocr"),
("references/article_corpus/manifest.json", "references/article_corpus", "article"),
("references/article_corpus/ocr_manifest.json", "references/article_corpus", "article_ocr"),
)
# Admission card source_id -> catalog book_id (upstream book_admission_cards.json).
_ADMISSION_TO_BOOK = {
"light_on_life": "light_on_life", "shasti_hayani_dasha": "goel_shasti_hayani_dasha",
"raman_horoscope_v1": "raman_how_to_judge_v1", "raman_horoscope_v2": "raman_how_to_judge_v2",
"braha_ancient_hindu_astrology": "braha_ancient_hindu_astrology", "goel_jaimini": "goel_jaimini",
"rao_chara_dasha": "kn_rao_chara_dasha", "boney_major_life_events": "boney_major_life_events",
"goel_yogini_dasha": "goel_yogini_dasha", "rao_predictive_techniques": "kn_rao_predictive_techniques",
"chinese_vedic_summary": "chinese_reading_summary", "qin_indian_astrology": "qin_indian_astrology",
"horary_textbook_translation": "horary_textbook_zh", "brihat_jataka_notes": "brihat_jataka_reading_notes",
"bphs_notes": "bphs_reading_notes", "science_of_light_v1": "science_of_light_v1",
"science_of_light_v2": "science_of_light_v2",
}
# Rule packet source_id -> catalog book_id, for packets that cite an in-corpus book.
_PACKET_SOURCE_TO_BOOK = {
**_ADMISSION_TO_BOOK,
"local:raman_manual_hindu_astrology": "raman_manual_hindu_astrology",
"local:patel_nadi_astrology_2002": "patel_nadi_astrology",
"web:pvr_vedic_astro_textbook": "pvr_integrated_approach",
"web:rath_narayana_dasa": "rath_narayana_dasa",
}
# ---------------------------------------------------------------------------
# Technique tags (keyword map; tags are a search aid, not a doctrinal claim).
# ---------------------------------------------------------------------------
TECHNIQUE_PATTERNS: tuple[tuple[str, str], ...] = (
("vimshottari", r"vim[sś]h?ottari|maha\s?da[sś]h?a|antar\s?da[sś]h?a|bhukti|pratyantar|大运|小运"),
("narayana_dasha", r"n[aā]r[aā]ya[nṇ]a\s?da[sś]h?a|narayana"),
("chara_dasha", r"chara\s?da[sś]h?a|char\s?da[sś]h?a"),
("yogini_dasha", r"yogini"),
("other_dasha", r"ashtottari|shash?ti\s?hayani|kalachakra|sudar[sś]h?ana|moola\s?da[sś]a|niryana"),
("jaimini", r"jaimini|karaka?m[sś]h?a|atma\s?karaka|dara\s?karaka|arudha|upapada|argala|chara\s?karaka"),
("divisional", r"nava[mṁ][sś]h?a|da[sś]a[mṁ][sś]h?a|\bd-?(?:2|3|4|7|9|10|12|16|20|24|27|30|40|45|60)\b|varga|分盘"),
("ashtakavarga", r"ashtakavarga|a[sṣ][tṭ]akavarga|bindu"),
("shadbala", r"shad\s?bala|[sṣ]a[dḍ]bala|bhava\s?bala"),
("transit", r"transit|gochara?|行运|过运"),
("tajika_varshaphala", r"t[aā]jika|varsha\s?phala|annual\s+chart|muntha|sahams?|年盘"),
("kp_sub_lord", r"krishnamurti|\bkp\b|sub[\s-]?lord|significator|cuspal"),
("yoga", r"\byoga\b|yogas\b|格局"),
("functional_nature", r"functional(?:ly)?\s+(?:benefic|malefic)|yoga\s?karaka|功能性"),
("marriage", r"marriage|spouse|\bwife\b|husband|婚|配偶"),
("career", r"career|profession|occupation|10th\s+house|事业|职业"),
("wealth", r"wealth|dhana|riches|财富|财"),
("health_longevity", r"longevity|disease|illness|health|ayu[rs]?|寿|健康|疾病"),
("muhurta", r"muhurt?ha?|electional|择时|择日"),
("prasna", r"pra[sś]h?na|horary|问事|卜卦"),
("nadi", r"\bnaa?di\b"),
("rectification", r"rectification|birth\s?time|生时"),
)
_TECHNIQUE_RES = tuple((tag, re.compile(pattern, re.IGNORECASE)) for tag, pattern in TECHNIQUE_PATTERNS)
TECHNIQUE_TAGS = tuple(tag for tag, _ in TECHNIQUE_PATTERNS)
_PAGE_RE = re.compile(r"^##\s+Page\s+(\d+)\s*$")
_SEGMENT_RE = re.compile(r"^###\s+Image segment\s+(\d+)\b")
_CHAPTER_RE = re.compile(
r"^(?:(?:chapter|ch\.|lesson|part|section|adhyaya|adhyāya)\s*[\dIVXLC]+\b.{0,80}"
r"|\u7b2c[\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341\u767e\u96f6\u3007\d]+[\u7ae0\u8282\u7bc7\u8bb2].{0,40})$",
re.IGNORECASE,
)
_VERSE_RES = (
re.compile(r"\b(?:ch(?:apter)?\.?\s*(\d{1,3})\s*[,.;]?\s*(?:v|sl|verse|sloka|śloka)\.?\s*(\d{1,3}(?:[-–]\d{1,3})?))", re.IGNORECASE),
re.compile(r"\b(?:sloka|shloka|śloka|verse|sutra|sūtra|sl\.|st\.)\s*(?:no\.?\s*)?(\d{1,3}(?:[.\-–]\d{1,3}){0,2})\b", re.IGNORECASE),
re.compile(r"^\s*(\d{1,2}\.\d{1,2}\.\d{1,3})\b"),
)
_LEADING_NUMBER_RE = re.compile(r"^\s*\(?(\d{1,3})\)?[.)]\s+\S")
_CONDITION_RES = (
re.compile(r"^(?:if|when|whenever|should|in case|provided(?: that)?)\b[^.;:!?]{3,180}?(?=,|\bthen\b|$)", re.IGNORECASE),
re.compile(r"(?:\u5982\u679c|\u5047\u5982|\u82e5|\u5f53)[^\uff0c,\u3002\uff1b;]{2,60}"),
)
_CJK_RE = re.compile(r"[\u3400-\u9fff]")
_INDIC_RE = re.compile(r"[\u0900-\u0dff]")
_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?;])\s+(?=[\"'(\[A-Z0-9]|[^\x00-\x7f])|(?<=[\u3002\uff01\uff1f\uff1b])")
_TERM_RE = re.compile(r"[0-9A-Za-z\u00c0-\u024f\u1e00-\u1eff']+|[\u3400-\u9fff]+|[\u0900-\u0dff]+")
# Words that alone must not pull a rule packet in (scored 1 instead of 2-3).
_GENERIC_TERMS = frozenset(["house", "houses", "lord", "lords", "sign", "signs", "first", "second", "third", "planet", "planets", "year", "years", "period", "periods", "chart", "dasa", "dasha", "rule", "rules", "from"])
_STOPWORDS = frozenset(["a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "the", "to", "with"])
SIGNS = ("Aries", "Taurus", "Gemini", "Cancer", "Leo", "Virgo", "Libra", "Scorpio",
"Sagittarius", "Capricorn", "Aquarius", "Pisces")
SIGN_LORDS = {"Aries": "Mars", "Taurus": "Venus", "Gemini": "Mercury", "Cancer": "Moon", "Leo": "Sun",
"Virgo": "Mercury", "Libra": "Venus", "Scorpio": "Mars", "Sagittarius": "Jupiter",
"Capricorn": "Saturn", "Aquarius": "Saturn", "Pisces": "Jupiter"}
def _sha256_bytes(raw: bytes) -> str:
return hashlib.sha256(raw).hexdigest()
def _load_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def _dump_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def technique_tags(text: str) -> list[str]:
return [tag for tag, pattern in _TECHNIQUE_RES if pattern.search(text)]
# ---------------------------------------------------------------------------
# Catalog + sync
# ---------------------------------------------------------------------------
def _upstream_sources(upstream: Path) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for manifest_rel, base_rel, kind in _UPSTREAM_MANIFESTS:
manifest_path = upstream / manifest_rel
if not manifest_path.exists():
continue
manifest = _load_json(manifest_path)
for item in manifest.get("books") or manifest.get("artifacts") or []:
rel = Path(item["output_path"])
if rel.is_absolute() or ".." in rel.parts:
raise ValueError("upstream manifest path escapes its corpus")
path = upstream / base_rel / rel
raw = path.read_bytes()
digest = _sha256_bytes(raw)
if item.get("text_sha256") and item["text_sha256"] != digest:
raise ValueError(f"upstream text hash mismatch for a {kind} source")
out.append({"kind": kind, "name": rel.name, "path": path, "sha256": digest,
"quality": item.get("quality") or "source_transcription"})
return out
def _curated_for(name: str) -> dict[str, Any] | None:
for needle, meta in _CURATED:
if needle in name:
return dict(meta)
return None
def build_catalog(upstream: Path) -> dict[str, Any]:
"""Catalog rows for every upstream text. Metadata only; no text is copied."""
cards = {}
if ADMISSION_CARDS_PATH.exists():
for card in _load_json(ADMISSION_CARDS_PATH).get("cards", []):
book_id = _ADMISSION_TO_BOOK.get(card.get("source_id"))
if book_id:
cards[book_id] = card
rows: list[dict[str, Any]] = []
seen: set[str] = set()
for source in _upstream_sources(upstream):
meta = _curated_for(source["name"])
if meta is None:
if source["kind"] not in {"article", "article_ocr"}:
raise ValueError("uncurated upstream book; add it to _CURATED")
stem = Path(source["name"]).stem
title = stem if source["kind"] == "article" else f"Article screenshot {stem.split('_')[0]}"
meta = dict(book_id=f"article_{source['sha256'][:10]}", title=title,
author="collected web article", edition="web / social post",
language="zh" if source["kind"] == "article" else "mixed",
rule_type="web_experience", school=["web"])
if meta["book_id"] in seen:
raise ValueError("duplicate catalog book_id")
seen.add(meta["book_id"])
page_basis = {"book": "source_file_page", "book_ocr": "source_file_page",
"article": "none", "article_ocr": "image_segment"}[source["kind"]]
row = {
**meta,
"source_kind": source["kind"],
"text_sha256": source["sha256"],
"page_basis": page_basis,
"extraction_quality": source["quality"],
"license_boundary": "local_only_full_text",
}
card = cards.get(meta["book_id"])
if card:
row["admission_state"] = card.get("admission_state")
row["page_count"] = card.get("page_count")
rows.append(row)
rows.sort(key=lambda row: row["book_id"])
return {
"schema": "jyotish.book_catalog.v1",
"copyright_boundary": COPYRIGHT_BOUNDARY,
"rule_types": RULE_TYPES,
"rule_type_note": "classical_with_commentary: passages with a detected sloka/sutra id are classical_rule, the rest modern_commentary.",
"page_basis_note": "source_file_page is the PDF/file page index, not the printed page number.",
"books": rows,
}
def load_catalog(path: Path = CATALOG_PATH) -> dict[str, Any]:
return _load_json(path)
def sync_local_texts(upstream: Path, *, catalog_path: Path = CATALOG_PATH, dest: Path = LOCAL_TEXT_DIR) -> dict[str, int]:
"""Copy upstream texts whose SHA256 the catalog lists into the gitignored local dir."""
wanted = {row["text_sha256"]: row["book_id"] for row in load_catalog(catalog_path)["books"]}
dest.mkdir(parents=True, exist_ok=True)
copied = 0
for source in _upstream_sources(upstream):
book_id = wanted.get(source["sha256"])
if book_id:
shutil.copyfile(source["path"], dest / f"{book_id}.md")
copied += 1
return {"catalog_books": len(wanted), "copied": copied}
def local_text_dir() -> Path:
override = os.environ.get("JYOTISH_BOOK_CORPUS_DIR", "").strip()
return Path(override) if override else LOCAL_TEXT_DIR
# ---------------------------------------------------------------------------
# Passage extraction
# ---------------------------------------------------------------------------
def _clean_join(lines: list[str]) -> str:
out = ""
for line in lines:
line = line.strip()
if not line:
continue
if not out:
out = line
elif out.endswith("-") and not out.endswith(" -"):
out = out[:-1] + line
elif _CJK_RE.search(out[-1:]) or _CJK_RE.search(line[:1]):
out += line
else:
out += " " + line
return re.sub(r"\s+", " ", out).strip()
def _good_sentence(text: str) -> bool:
indic = len(_INDIC_RE.findall(text))
if indic >= 15:
return indic / max(1, len(text.replace(" ", ""))) > 0.5
cjk = len(_CJK_RE.findall(text))
if cjk >= 8:
return cjk / max(1, len(text.replace(" ", ""))) > 0.35
words = re.findall(r"[A-Za-zÀ-ɏḀ-ỿ]{2,}", text)
if len(words) < 5:
return False
letters = sum(ch.isalpha() for ch in text)
return letters / max(1, len(text)) > 0.6
def _detect_verse(paragraph: str, classical: bool) -> str | None:
for index, pattern in enumerate(_VERSE_RES):
match = pattern.search(paragraph)
if match:
if index == 0:
return f"{match.group(1)}.{match.group(2)}"
return match.group(1)
if classical:
match = _LEADING_NUMBER_RE.match(paragraph)
if match:
return match.group(1)
return None
_VERSE_PREFIX_RE = re.compile(r"^\s*(?:(?:sloka|shloka|verse|sutra|sl\.|st\.)\s*[\d.\-]+|\(?\d{1,3}(?:\.\d{1,3}){0,2}\)?[.)]?)\s*[:.-]?\s*", re.IGNORECASE)
_INNER_IF_RE = re.compile(r"\b(?:if|provided that)\b[^.;:!?]{3,180}?(?=,|\bthen\b|$)", re.IGNORECASE)
def _applicability(sentence: str) -> str | None:
sentence = _VERSE_PREFIX_RE.sub("", sentence, count=1)
for pattern in (*_CONDITION_RES, _INNER_IF_RE):
match = pattern.search(sentence)
if match:
return match.group(0).strip()[:200]
return None
def iter_passages(book: dict[str, Any], text: str) -> Iterable[dict[str, Any]]:
"""Yield sentence rows in source order. Deterministic for a given text."""
classical = book["rule_type"] == "classical_with_commentary"
page: int | None = None
chapter: str | None = None
started = book["page_basis"] == "none"
paragraph: list[str] = []
paragraph_line = 0
sentence_no = 0
def flush() -> Iterable[dict[str, Any]]:
nonlocal chapter, sentence_no
joined = _clean_join(paragraph)
if not joined:
return
if len(joined) <= 90 and _CHAPTER_RE.match(joined):
chapter = joined
return
verse = _detect_verse(joined, classical)
for raw in _SENTENCE_SPLIT_RE.split(joined):
sentence = raw.strip()
# A sentence that carries its own sloka / sutra marker starts that verse.
verse = _detect_verse(sentence, False) or verse
if not sentence or not _good_sentence(sentence):
continue
sentence_no += 1
rule_type = book["rule_type"]
if rule_type == "classical_with_commentary":
rule_type = "classical_rule" if verse else "modern_commentary"
yield {
"passage_id": f"{book['book_id']}:{page if page is not None else 0}:{sentence_no}",
"book_id": book["book_id"],
"page": page,
"line": paragraph_line,
"chapter": chapter,
"verse_id": verse,
"text": sentence[:MAX_SENTENCE_CHARS],
"applicability": _applicability(sentence),
"rule_type": rule_type,
"techniques": ",".join(technique_tags(sentence)),
}
for line_no, line in enumerate(text.splitlines(), start=1):
marker = _PAGE_RE.match(line) or _SEGMENT_RE.match(line)
if marker:
yield from flush()
paragraph = []
page = int(marker.group(1))
started = True
continue
if line.startswith("## Extracted Text"):
yield from flush()
paragraph = []
started = True
continue
if not started or line.startswith("# "):
continue
if not line.strip():
yield from flush()
paragraph = []
continue
if not paragraph:
paragraph_line = line_no
paragraph.append(line)
yield from flush()
_PASSAGE_COLUMNS = ("passage_id", "book_id", "page", "line", "chapter", "verse_id", "text",
"applicability", "rule_type", "techniques")
def build_passage_index(*, text_dir: Path | None = None, catalog_path: Path = CATALOG_PATH,
db_path: Path = PASSAGE_DB_PATH, manifest_path: Path | None = PASSAGE_MANIFEST_PATH) -> dict[str, Any]:
"""Build the SQLite passage index and its text-free manifest.
Determinism contract: same catalog + same texts -> same ``content_digest``
(SHA256 over the ordered rows) and byte-identical manifest. The SQLite file
itself is a cache and is not compared byte for byte.
"""
text_dir = text_dir or local_text_dir()
catalog = load_catalog(catalog_path)
if db_path.exists():
db_path.unlink()
db_path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(db_path)
digest = hashlib.sha256()
books_manifest: list[dict[str, Any]] = []
missing: list[str] = []
total = 0
try:
connection.execute(
"CREATE TABLE passages (rowid INTEGER PRIMARY KEY, passage_id TEXT UNIQUE NOT NULL, book_id TEXT NOT NULL, "
"page INTEGER, line INTEGER, chapter TEXT, verse_id TEXT, text TEXT NOT NULL, applicability TEXT, "
"rule_type TEXT NOT NULL, techniques TEXT NOT NULL)")
connection.execute(
"CREATE VIRTUAL TABLE passages_fts USING fts5(text, content='passages', content_rowid='rowid', tokenize='trigram')")
for book in sorted(catalog["books"], key=lambda row: row["book_id"]):
path = text_dir / f"{book['book_id']}.md"
if not path.exists():
missing.append(book["book_id"])
continue
raw = path.read_bytes()
if _sha256_bytes(raw) != book["text_sha256"]:
raise ValueError(f"local text hash mismatch for {book['book_id']}; re-run sync")
stats = {"passages": 0, "with_verse": 0, "with_chapter": 0, "with_applicability": 0}
pages: set[int] = set()
for row in iter_passages(book, raw.decode("utf-8")):
values = tuple(row[column] for column in _PASSAGE_COLUMNS)
connection.execute(
f"INSERT INTO passages ({', '.join(_PASSAGE_COLUMNS)}) VALUES ({', '.join('?' * len(_PASSAGE_COLUMNS))})",
values)
digest.update(json.dumps(values, ensure_ascii=False).encode("utf-8") + b"\n")
stats["passages"] += 1
stats["with_verse"] += bool(row["verse_id"])
stats["with_chapter"] += bool(row["chapter"])
stats["with_applicability"] += bool(row["applicability"])
if row["page"] is not None:
pages.add(row["page"])
total += stats["passages"]
books_manifest.append({"book_id": book["book_id"], "title": book["title"], "rule_type": book["rule_type"],
"text_sha256": book["text_sha256"], "pages_with_passages": len(pages), **stats})
connection.execute("INSERT INTO passages_fts(passages_fts) VALUES ('rebuild')")
connection.commit()
finally:
connection.close()
manifest = {
"schema": "jyotish.book_passage_index_manifest.v1",
"builder_version": BUILDER_VERSION,
"copyright_boundary": COPYRIGHT_BOUNDARY,
"index_path": "references/book_corpus/index/book_passages.sqlite (gitignored, rebuilt locally)",
"content_digest": digest.hexdigest(),
"total_passages": total,
"books_indexed": len(books_manifest),
"books_missing_local_text": missing,
"books": books_manifest,
}
if manifest_path is not None:
_dump_json(manifest_path, manifest)
return manifest
# ---------------------------------------------------------------------------
# Rule packets -> citations
# ---------------------------------------------------------------------------
def _packet_rule_type(packet: dict[str, Any]) -> str:
source_id = str(packet.get("source_id") or "")
rule_id = str(packet.get("rule_id") or "")
if re.search(r"mhora|pyjhora|jhora", source_id + rule_id):
return "software_implementation"
if re.search(r"sripati|bphs", source_id):
return "classical_rule"
if packet.get("source_type") == "web_reference":
return "web_experience"
if source_id.startswith("web:"):
return "modern_commentary"
if source_id in _PACKET_SOURCE_TO_BOOK:
return "modern_commentary"
return "unverified"
def build_rule_citations(*, packets_path: Path = RULE_PACKETS_PATH, lineage_path: Path = RULE_LINEAGE_PATH,
conditions_path: Path = RULE_CONDITIONS_PATH, catalog_path: Path = CATALOG_PATH) -> dict[str, Any]:
packets = _load_json(packets_path)
lineage = {row["rule_id"]: row for row in _load_json(lineage_path).get("rows", [])} if lineage_path.exists() else {}
conditions = _load_json(conditions_path).get("rules", {}) if conditions_path.exists() else {}
books = {row["book_id"]: row for row in load_catalog(catalog_path)["books"]} if catalog_path.exists() else {}
rows: list[dict[str, Any]] = []
for packet in packets["packets"]:
rule_id = packet["rule_id"]
book_id = _PACKET_SOURCE_TO_BOOK.get(packet.get("source_id"))
book = books.get(book_id or "")
validation = packet.get("validation") or {}
lineage_row = lineage.get(rule_id, {})
summary = packet.get("non_verbatim_rule_summary") or ""
tags = sorted(set(lineage_row.get("techniques") or []) | set(technique_tags(f"{rule_id} {summary}")))
boundary = packet.get("assertion_boundary") or ""
not_validated = validation.get("not_validated_reason") or ""
rows.append({
"citation_id": f"rule:{rule_id}",
"rule_id": rule_id,
"source_id": packet.get("source_id"),
"book_id": book_id,
"book": book["title"] if book else (packet.get("source_attribution") or packet.get("source_id")),
"author": book["author"] if book else packet.get("source_attribution"),
"edition": book["edition"] if book else None,
"locator": packet.get("page_locator"),
"source_url": packet.get("source_url"),
"text_basis": "non_verbatim_summary",
"rule_summary": summary,
"rule_type": _packet_rule_type(packet),
"admission_state": packet.get("admission_state"),
"validation_status": validation.get("status"),
"applicability": " ".join(part for part in (boundary, not_validated) if part).strip() or None,
"conditions": conditions.get(rule_id),
"techniques": tags,
})
rows.sort(key=lambda row: row["citation_id"])
return {
"schema": "jyotish.book_rule_citations.v1",
"source": "references/book_rule_packets.json (upstream, non-verbatim summaries and locators)",
"copyright_boundary": packets.get("copyright_boundary"),
"rule_types": RULE_TYPES,
"citations": rows,
}
# ---------------------------------------------------------------------------
# Chart condition check ("当前盘是否满足")
# ---------------------------------------------------------------------------
def _sign_name(value: Any) -> str | None:
if isinstance(value, dict):
value = value.get("sign")
if isinstance(value, str):
for sign in SIGNS:
if value.strip().casefold() == sign.casefold():
return sign
return None
def normalize_chart(chart: Any) -> dict[str, Any]:
"""Reduce an engine / evidence-card chart to signs, houses, lords and running dasha."""
if not isinstance(chart, dict):
return {}
houses: dict[int, str] = {}
raw_houses = chart.get("houses")
if isinstance(raw_houses, dict):
items = raw_houses.items()
elif isinstance(raw_houses, list):
items = ((row.get("number") if isinstance(row, dict) else None, row) for row in raw_houses)
else:
items = ()
for number, row in items:
try:
house = int(number)
except (TypeError, ValueError):
continue
sign = _sign_name(row)
if 1 <= house <= 12 and sign:
houses[house] = sign
ascendant = chart.get("ascendant") if isinstance(chart.get("ascendant"), dict) else {}
lagna = _sign_name(ascendant) or houses.get(1)
if lagna and len(houses) < 12:
start = SIGNS.index(lagna)
houses = {house: SIGNS[(start + house - 1) % 12] for house in range(1, 13)}
planets: dict[str, dict[str, Any]] = {}
raw_planets = chart.get("planets")
planet_items = raw_planets.items() if isinstance(raw_planets, dict) else (
((row.get("name"), row) for row in raw_planets if isinstance(row, dict)) if isinstance(raw_planets, list) else ())
for name, row in planet_items:
if not isinstance(name, str) or not isinstance(row, dict):
continue
house = row.get("house")
sign = _sign_name(row)
if not isinstance(house, int) and sign and lagna:
house = (SIGNS.index(sign) - SIGNS.index(lagna)) % 12 + 1
planets[name.strip().title()] = {"sign": sign, "house": house if isinstance(house, int) else None}
dasha = chart.get("vimshottari") if isinstance(chart.get("vimshottari"), dict) else {}
def lord(value: Any) -> str | None:
value = value.get("lord") if isinstance(value, dict) else value
return value.strip().title() if isinstance(value, str) and value.strip() else None
return {
"lagna": lagna,
"lagna_nakshatra": ascendant.get("nakshatra") if isinstance(ascendant.get("nakshatra"), str) else None,
"houses": houses,
"planets": planets,
"mahadasha": lord(dasha.get("mahadasha") or dasha.get("md")),
"antardasha": lord(dasha.get("antardasha") or dasha.get("ad")),
}
def _check(condition: dict[str, Any], chart: dict[str, Any]) -> dict[str, Any]:
op = condition.get("op")
houses, planets = chart.get("houses") or {}, chart.get("planets") or {}
result: bool | None = None
observed: Any = None
if op == "lord_in_house":
sign = houses.get(int(condition["lord_of"]))
lord = SIGN_LORDS.get(sign or "")
house = (planets.get(lord or "") or {}).get("house")
observed = {"lord": lord, "house": house}
if lord and isinstance(house, int):
result = house == int(condition["house"])
elif op == "planet_in_house":
house = (planets.get(str(condition["planet"])) or {}).get("house")
observed = {"house": house}
if isinstance(house, int):
result = house in [int(value) for value in condition["houses"]]
elif op == "ascendant_nakshatra":
observed = {"nakshatra": chart.get("lagna_nakshatra")}
if chart.get("lagna_nakshatra"):
wanted = {str(value).casefold() for value in condition["nakshatras"]}
result = str(chart["lagna_nakshatra"]).casefold() in wanted
elif op == "vimshottari_pair":
observed = {"mahadasha": chart.get("mahadasha"), "antardasha": chart.get("antardasha")}
if chart.get("mahadasha") and chart.get("antardasha"):
result = [chart["mahadasha"], chart["antardasha"]] in [list(pair) for pair in condition["pairs"]]
return {"op": op, "expected": {k: v for k, v in condition.items() if k != "op"}, "observed": observed, "result": result}
def check_chart_conditions(conditions: dict[str, Any] | None, chart: Any) -> dict[str, Any]:
if not conditions or not conditions.get("all"):
return {"status": "needs_reader_check", "reason": "no_machine_checkable_conditions"}
normalized = normalize_chart(chart)
if not normalized.get("lagna") and not normalized.get("mahadasha"):
return {"status": "needs_reader_check", "reason": "no_chart_in_context"}
checks = [_check(condition, normalized) for condition in conditions["all"]]
residual = list(conditions.get("reader_checks") or [])
if any(check["result"] is False for check in checks):
status = "not_satisfied"
elif any(check["result"] is None for check in checks) or residual:
status = "needs_reader_check"
else:
status = "satisfied"
out: dict[str, Any] = {"status": status, "checks": checks}
if residual:
out["reader_checks"] = residual
return out
# ---------------------------------------------------------------------------
# Search
# ---------------------------------------------------------------------------
def _terms(query: str) -> list[str]:
seen: list[str] = []
for term in _TERM_RE.findall(query or ""):
term = term.strip("'").casefold()
if term and term not in _STOPWORDS and term not in seen:
seen.append(term)
return seen[:8]
def _excerpt(text: str, terms: list[str], limit: int) -> str:
if len(text) <= limit:
return text
lower = text.casefold()
hit = min((lower.find(term) for term in terms if lower.find(term) >= 0), default=0)
start = max(0, min(hit - limit // 3, len(text) - limit))
snippet = text[start:start + limit].strip()
return ("…" if start > 0 else "") + snippet + ("…" if start + limit < len(text) else "")
def _search_passages(db_path: Path, terms: list[str], *, techniques: list[str], rule_types: list[str], limit: int) -> list[sqlite3.Row]:
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
connection.row_factory = sqlite3.Row
try:
long_terms = [term for term in terms if len(term) >= 3]
short_terms = [term for term in terms if len(term) < 3]
filters, params = [], []
for term in short_terms:
filters.append("p.text LIKE ?")
params.append(f"%{term}%")
if techniques:
filters.append("(" + " OR ".join("(',' || p.techniques || ',') LIKE ?" for _ in techniques) + ")")
params.extend(f"%,{tag},%" for tag in techniques)
if rule_types:
filters.append(f"p.rule_type IN ({', '.join('?' * len(rule_types))})")
params.extend(rule_types)
where = (" AND " + " AND ".join(filters)) if filters else ""
if long_terms:
for joiner in (" AND ", " OR "):
match = joiner.join('"' + term.replace('"', '""') + '"' for term in long_terms)
rows = connection.execute(
"SELECT p.*, bm25(passages_fts) AS score FROM passages_fts JOIN passages p ON p.rowid = passages_fts.rowid "
f"WHERE passages_fts MATCH ?{where} ORDER BY score, p.rowid LIMIT ?",
(match, *params, limit)).fetchall()
if rows or len(long_terms) == 1:
return rows
return []
if not filters:
return []
return connection.execute(
f"SELECT p.*, 0.0 AS score FROM passages p WHERE 1=1{where} ORDER BY {_RULE_TYPE_ORDER}, p.rowid LIMIT ?", (*params, limit)).fetchall()
finally:
connection.close()
_RULE_TYPE_ORDER = ("CASE p.rule_type WHEN 'classical_rule' THEN 0 WHEN 'modern_commentary' THEN 1 "
"WHEN 'software_implementation' THEN 2 WHEN 'unverified' THEN 3 ELSE 4 END")
def _clip(value: Any, limit: int) -> Any:
if isinstance(value, str) and len(value) > limit:
return value[: limit - 1] + "…"
return value
def _rule_score(row: dict[str, Any], terms: list[str], techniques: list[str]) -> int:
hay = " ".join(str(row.get(key) or "") for key in ("rule_id", "rule_summary", "locator", "book")).casefold()
score = 0
for term in terms:
if term not in hay:
continue
if term in _GENERIC_TERMS:
score += 1
else:
score += 3 if term in row["rule_id"].casefold() else 2
if techniques and set(techniques) & set(row.get("techniques") or []):
score += 2
return score
def _bound(payload: dict[str, Any], max_chars: int) -> dict[str, Any]:
def size() -> int:
return len(json.dumps(payload, ensure_ascii=False))
for limit in EXCERPT_STEPS:
if size() <= max_chars:
break
for citation in payload["citations"]:
for key in ("original_text", "rule_summary", "applicability"):
if isinstance(citation.get(key), str) and len(citation[key]) > limit:
citation[key] = citation[key][: limit - 1] + "…"
while size() > max_chars and payload["citations"]:
payload["citations"].pop()
payload["truncated"] = True
return payload
def search_book_citations(
query: str,
*,
techniques: list[str] | None = None,
rule_types: list[str] | None = None,
top_k: int = DEFAULT_TOP_K,
chart: Any = None,
max_chars: int = MAX_RESPONSE_CHARS,
db_path: Path | None = None,
rule_citations_path: Path = RULE_CITATIONS_PATH,
catalog_path: Path = CATALOG_PATH,
) -> dict[str, Any]:
"""Top-k citations: rule packets first (with chart check), then book passages."""
db_path = db_path or PASSAGE_DB_PATH
terms = _terms(query)
techniques = [tag for tag in (techniques or []) if tag in TECHNIQUE_TAGS][:4]
rule_types = [kind for kind in (rule_types or []) if kind in RULE_TYPES]
top_k = max(1, min(MAX_TOP_K, int(top_k or DEFAULT_TOP_K)))
if not terms and not techniques:
raise ValueError("query or techniques required")
books = {row["book_id"]: row for row in load_catalog(catalog_path)["books"]}
citations: list[dict[str, Any]] = []
rules = _load_json(rule_citations_path)["citations"]
scored = sorted(((score, row) for row in rules
if (not rule_types or row["rule_type"] in rule_types)
and (score := _rule_score(row, terms, techniques)) >= min(4, 2 * len(terms))),
key=lambda pair: (-pair[0], pair[1]["citation_id"]))
rule_slots = min(2 if db_path.exists() else 3, top_k)
for _, row in scored[:rule_slots]:
citations.append({
"id": row["citation_id"], "kind": "rule_packet",
"book": row["book"], "author": row["author"], "edition": row["edition"],
# A packet has no verbatim text: rule_summary stands in for original_text,
# and its locator string carries page / chapter / verse together.
"page": _clip(row["locator"], 160), "page_basis": "locator_string",
"rule_summary": _clip(row["rule_summary"], 260),
"rule_type": row["rule_type"],
"admission_state": row["admission_state"],
"techniques": row["techniques"][:4],
"applicability": _clip(row["applicability"], 180),
"chart_check": check_chart_conditions(row.get("conditions"), chart),
**({"source_url": row["source_url"]} if row.get("source_url") else {}),
})
passages_status = "unavailable"
if db_path.exists():
passages_status = "available"
try:
hits = _search_passages(db_path, terms, techniques=techniques, rule_types=rule_types,
limit=top_k - len(citations))
except sqlite3.Error:
hits, passages_status = [], "error"
for hit in hits:
book = books.get(hit["book_id"], {})
citations.append({
"id": hit["passage_id"], "kind": "book_passage",
"book": book.get("title", hit["book_id"]), "author": book.get("author"), "edition": book.get("edition"),
"page": hit["page"], "page_basis": book.get("page_basis"), "chapter": hit["chapter"], "verse": hit["verse_id"],
"original_text": _excerpt(hit["text"], terms, EXCERPT_STEPS[0]),
"translation": None,
"rule_type": hit["rule_type"],
"techniques": [tag for tag in hit["techniques"].split(",") if tag][:4],
"applicability": hit["applicability"],
"chart_check": {"status": "needs_reader_check"},
**({"quality": book["extraction_quality"]}
if book.get("extraction_quality") not in (None, "source_transcription") else {}),
})
payload = {
"schema": "jyotish.book_citation_search.v1",
"query": query[:200],
"citations": citations,
"rule_type_labels": RULE_TYPES,
"coverage": {"rule_packets": len(rules), "passage_index": passages_status},
"boundary": "Citations locate sources, they do not validate a rule for this chart. Rule packets are "
"non-verbatim summaries; passage text is a short excerpt; source_file_page is the file page, not the printed page.",
}
return _bound(payload, max_chars)
def book_citation_api(body: dict[str, Any]) -> dict[str, Any]:
"""POST /api/book_citations body -> payload. Raises ValueError on bad input."""
if not isinstance(body, dict):
raise ValueError("body must be an object")
query = body.get("query") or ""
if not isinstance(query, str) or len(query) > 200:
raise ValueError("query must be a string of at most 200 characters")
techniques = body.get("techniques") or []
rule_types = body.get("rule_types") or []
if not isinstance(techniques, list) or not isinstance(rule_types, list):
raise ValueError("techniques and rule_types must be lists")
top_k = body.get("top_k", DEFAULT_TOP_K)
if not isinstance(top_k, int) or isinstance(top_k, bool):
raise ValueError("top_k must be an integer")
result = search_book_citations(query, techniques=[str(t) for t in techniques], rule_types=[str(r) for r in rule_types],
top_k=top_k, chart=body.get("chart"))
return {"success": True, "endpoint": "book_citations", **result}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = parser.add_subparsers(dest="command", required=True)
for name in ("catalog", "sync"):
cmd = sub.add_parser(name)
cmd.add_argument("--upstream", type=Path, required=True)
sub.add_parser("build")
sub.add_parser("rules")
search = sub.add_parser("search")
search.add_argument("query")
search.add_argument("--technique", action="append", default=[])
search.add_argument("--rule-type", action="append", default=[])
search.add_argument("--top-k", type=int, default=DEFAULT_TOP_K)
args = parser.parse_args(argv)
if args.command == "catalog":
_dump_json(CATALOG_PATH, build_catalog(args.upstream))
print(f"catalog: {len(load_catalog()['books'])} sources")
elif args.command == "sync":
print(json.dumps(sync_local_texts(args.upstream)))
elif args.command == "build":
manifest = build_passage_index()
print(json.dumps({k: manifest[k] for k in ("total_passages", "books_indexed", "content_digest")}))
elif args.command == "rules":
_dump_json(RULE_CITATIONS_PATH, build_rule_citations())
print(f"rule citations: {len(_load_json(RULE_CITATIONS_PATH)['citations'])}")
else:
print(json.dumps(search_book_citations(args.query, techniques=args.technique, rule_types=args.rule_type,
top_k=args.top_k), ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())