From 18c6e13028f55c3201309b7a48971ee04cb96769 Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Wed, 28 Oct 2026 11:49:50 +0800 Subject: [PATCH] Consult card keeps parsed packets in memory, prewarms each turn, writes 0600 (issue #129, BUG-1476) - consult_packet_view keeps the last 3 parsed packets with their shared domain base (~30 MB each). A hit reads no file and copies nothing: attach for one domain 0.106 s -> 0.002 s. Memory holds the file's JSON round trip, i.e. exactly what a hit returned before; consult_card_facts sha256 equal old/new. - Cache files are created 0600, same bytes as before. - The consult route fires a warm for the turn's chart before reserving credit, so the day's first question does not build the packet on the answer path. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + docs/BUG_HISTORY.md | 15 +++ frontend/src/app/api/consult/route.ts | 4 +- frontend/src/lib/consult-packet-warm.ts | 51 ++++++++ .../consult-packet-warm-20261009.test.ts | 32 ++++- .../consultation-workflow-contract.test.ts | 4 +- scripts/consult_full_data_cache.py | 77 +++++++++++- ...test_consult_card_cache_memory_20261028.py | 119 ++++++++++++++++++ 8 files changed, 300 insertions(+), 8 deletions(-) create mode 100644 tests/test_consult_card_cache_memory_20261028.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b499b8..03c0b799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 印度占星 Skill 更新日志 +## 2026-10-28 — 咨询准备数据更快，含出生资料的缓存文件只有本机服务能读 + +- 同一张盘连着问时，每轮准备数据卡从约 0.1 秒降到几毫秒；每天第一问不再在回答路径上现算整包，问题一发出去就在后台提前算好。回答内容不变。 +- 含出生资料的缓存文件改为只有服务本身能读（0600）。 +- Skill 版本不变（BUG-1476，issue #129）。 + ## 2026-10-28 — 每个页面第一次打开再少下载约 25～30 KB - 首页、星盘页、档案页、报告页、星历页第一次打开再少下载约 25～30 KB（压缩后）：日期解析改用自带的小函数，「退出」和「分享」用的代码等到用时才下载。 diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index ac279dec..6776f99e 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -20141,3 +20141,18 @@ - 关联：BUG-1474（#124 首页懒加载校正界面）、BUG-1130。 - 复发自：无 - 修复版本：分支 `codex/home-first-screen-deps-20261028`。 + +## BUG-1476 | 咨询数据卡命中缓存也要每轮读 5.5 MB 文件再整份复制；每天首问现算；缓存文件谁都能读 + +- 状态：resolved（代码 + 回归测试；待部署） +- 首次发现 / 最近更新：2026-10-28 / 2026-10-28（issue #118 代码体检第 6 条 → #129） +- 影响面：咨询每一轮的数据卡（`scripts/consult_full_data_cache.py` `attach_consult_full_data` / `warm_consult_packet`）；同一缓存目录下报告参考、合盘缓存文件的权限；`frontend/src/app/api/consult/route.ts`、`frontend/src/lib/consult-packet-warm.ts`。 +- 现象：① 命中缓存时每个领域每轮仍要读 5.8 MB JSON、再深拷贝共享部分，本机实测每轮约 0.11 秒；② 缓存键带服务器日期，每天第一问要在回答路径上现算整包（本机实测约 5.8 秒）；③ 缓存文件权限 0644，里面有出生资料，不合 0600 规矩。 +- 触发条件：任何带数据卡的咨询轮。 +- 根因：缓存只有文件一层，命中也要整份解析；BUG-1471 已把每轮的深拷贝减到一次，但每轮仍有一次；预热只在保存星盘时做，跨天就失效；写文件用的是默认权限。 +- 修复：新增 `consult_packet_view`，进程内按缓存键留最近 3 份「已解析的包 + 它的共享底稿」（每份约 30 MB，合计约 90 MB），数据卡和预热都走它，命中时不读文件、不复制，只读共享对象；首次构建的这一轮仍用构建出的对象作答，内存里记的是文件读回的那一份（和以前「命中」读到的完全一样）。报告参考、合盘等其他用 `get_or_build` 的地方不走内存，仍各拿一份。缓存文件改为创建即 0600，内容字节不变（不排序键）。咨询开始时（扣点前、确定了这一轮用哪张盘之后）不等待地发一次预热，按同一缓存键在后台把今天的包建好；引擎里同键的构建只跑一次，数据卡来要时直接接上。 +- 验证：本机真包实测，命中时整段 `attach_consult_full_data`：1 个领域 0.106 → 0.0018 秒，2 个领域 0.105 → 0.0041 秒（每领域约 0.002 秒，单子要求 ≤0.05）。改前 / 改后代码对同一张虚构盘的全部领域，首次构建与之后命中两轮的 `consult_card_facts`（去掉耗时字段）sha256 逐个相同。新测试 `tests/test_consult_card_cache_memory_20261028.py`（命中逐字节相同且不读文件；读事实不改动共享包和底稿；只留最近 3 份；新文件 0600 且字节与以前一致；首次构建作答用构建对象、内存记文件读回的那份）；`frontend/tests/consult-packet-warm-20261009.test.ts` 新增一条（预热请求体与数据卡同一身份、不等待、引擎出错不影响这一轮）。既有 `tests/test_consult_card_full_source.py` 改前改后都全过。Python 快速门全部 Python 步骤通过（pytest 370 s）；前端全量测试与 staging 逐条比对无新增失败（`consultation-workflow-contract.test.ts` 一处断言锁着 `beforeReserve` 只解构 `consultationMode`，已改认新参数）；tsc 0、lint 0。 +- 防复发：上述测试。读多写少的大包留在内存只读共享，不要每轮重读重拷；含出生资料的文件一律创建即 0600。 +- 关联：BUG-1471（#122 每份报告只拷一次）、BUG-1305（保存星盘后的预热）。 +- 复发自：无 +- 修复版本：分支 `codex/consult-card-cache-memory-20261028`。 diff --git a/frontend/src/app/api/consult/route.ts b/frontend/src/app/api/consult/route.ts index 062f24f9..b51327c7 100644 --- a/frontend/src/app/api/consult/route.ts +++ b/frontend/src/app/api/consult/route.ts @@ -1,5 +1,6 @@ import { NextResponse } from "next/server"; import { refuseWhenAccountDeletionPending } from "@/lib/account-deletion-server"; +import { prewarmConsultPacketForTurn } from "@/lib/consult-packet-warm"; import { consultationInputSchema, consultationWorkflowReceipt, @@ -569,7 +570,8 @@ export async function POST(request: Request) { return data; }, loadCandidateRange: (profileUserId) => loadReportCandidateRange(accounting, { userId: profileUserId }), - beforeReserve: async ({ consultationMode }) => { + beforeReserve: async ({ consultationMode, serverChart }) => { + if (serverChart && consultationMode !== "declared_birth_window") prewarmConsultPacketForTurn(serverChart.toolInput); prepPhases.push({ phase: "prep.subject", durationMs: Math.max(0, Date.now() - prepareStarted), diff --git a/frontend/src/lib/consult-packet-warm.ts b/frontend/src/lib/consult-packet-warm.ts index d8c76c72..5c2e2231 100644 --- a/frontend/src/lib/consult-packet-warm.ts +++ b/frontend/src/lib/consult-packet-warm.ts @@ -70,3 +70,54 @@ export function scheduleConsultPacketWarm( } return true; } + +type ConsultTurnBirth = { + year: number; + month: number; + day: number; + hour: number; + minute: number; + lat: number; + lon: number; + tz: number; + ayanamsa?: string; +}; + +/** + * At the start of a consult turn, ask the engine to build (or load) the same + * packet the card will read. The cache key carries the server's date, so the + * day's first question otherwise built it on the critical path (issue #129). + * Fire and forget: the turn never waits on it, and the engine joins a running + * build when the card asks for the same key. + */ +export function prewarmConsultPacketForTurn( + birth: ConsultTurnBirth, + deps: Pick = { + fetch, + apiBase: process.env.JYOTISH_API_BASE ?? "http://127.0.0.1:5200", + }, +): void { + const body = { + year: birth.year, + month: birth.month, + day: birth.day, + hour: birth.hour, + minute: birth.minute, + second: 0, + lat: birth.lat, + lon: birth.lon, + tz: birth.tz, + ayanamsa: birth.ayanamsa || "lahiri", + node_mode: "mean", + consult_card_warm_only: true, + }; + try { + void deps.fetch(`${deps.apiBase ?? "http://127.0.0.1:5200"}/api/consultation_workflow`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }).catch(() => undefined); + } catch { + // A synchronous fetch failure must not touch the turn. + } +} diff --git a/frontend/tests/consult-packet-warm-20261009.test.ts b/frontend/tests/consult-packet-warm-20261009.test.ts index 0b03b710..3fd580c9 100644 --- a/frontend/tests/consult-packet-warm-20261009.test.ts +++ b/frontend/tests/consult-packet-warm-20261009.test.ts @@ -2,7 +2,9 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { consultWarmBodyFromColumns, scheduleConsultPacketWarm } from "../src/lib/consult-packet-warm.ts"; +import { readFileSync } from "node:fs"; + +import { consultWarmBodyFromColumns, prewarmConsultPacketForTurn, scheduleConsultPacketWarm } from "../src/lib/consult-packet-warm.ts"; const columns = { birth_date: "1901-01-01", @@ -49,3 +51,31 @@ test("a warm that cannot be scheduled does not break the save (BUG-1305)", () => assert.equal(scheduled, false); assert.equal(fetched, false); }); + +test("a consult turn prewarms the card packet with the same identity the card reads (issue #129)", async () => { + const calls: Array<{ url: string; body: Record }> = []; + prewarmConsultPacketForTurn( + { year: 1990, month: 3, day: 14, hour: 9, minute: 30, lat: 31.23, lon: 121.47, tz: 8, ayanamsa: "raman" }, + { + apiBase: "http://engine.test", + fetch: (async (url: string, init: RequestInit) => { + calls.push({ url, body: JSON.parse(String(init.body)) }); + return new Response("{}"); + }) as unknown as typeof fetch, + }, + ); + assert.equal(calls.length, 1, "sent without waiting"); + assert.equal(calls[0].url, "http://engine.test/api/consultation_workflow"); + assert.deepEqual(calls[0].body, { + year: 1990, month: 3, day: 14, hour: 9, minute: 30, second: 0, + lat: 31.23, lon: 121.47, tz: 8, ayanamsa: "raman", node_mode: "mean", + consult_card_warm_only: true, + }); + // A failing engine never reaches the turn. + prewarmConsultPacketForTurn( + { year: 1990, month: 3, day: 14, hour: 9, minute: 30, lat: 31.23, lon: 121.47, tz: 8 }, + { apiBase: "http://engine.test", fetch: (async () => { throw new Error("down"); }) as unknown as typeof fetch }, + ); + const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8"); + assert.match(route, /beforeReserve: async \(\{ consultationMode, serverChart \}\) => \{\n\s+if \(serverChart && consultationMode !== "declared_birth_window"\) prewarmConsultPacketForTurn\(serverChart\.toolInput\);/); +}); diff --git a/frontend/tests/consultation-workflow-contract.test.ts b/frontend/tests/consultation-workflow-contract.test.ts index 45c7a8fc..956399ed 100644 --- a/frontend/tests/consultation-workflow-contract.test.ts +++ b/frontend/tests/consultation-workflow-contract.test.ts @@ -165,7 +165,9 @@ test("consultation plans are server-owned and bounded", () => { assert.ok(beforeReserveAt >= 0 && reserveAt > beforeReserveAt); assert.match( prepareCall.slice(beforeReserveAt, reserveAt), - /beforeReserve:\s*async\s*\(\{\s*consultationMode\s*\}\)[\s\S]*createConsultationPlan\(\{/, + // 原值：参数只解构 consultationMode；新值：还解构 serverChart，用来在扣点前不等待地预热数据卡 + // 原因：issue #129 每天首问不在回答路径上现算整包；createConsultationPlan 仍在这个回调里、仍早于 reserve + /beforeReserve:\s*async\s*\(\{\s*consultationMode,\s*serverChart\s*\}\)[\s\S]*createConsultationPlan\(\{/, ); assert.match( prepareCall.slice(reserveAt), diff --git a/scripts/consult_full_data_cache.py b/scripts/consult_full_data_cache.py index 32ef94d9..5e128195 100644 --- a/scripts/consult_full_data_cache.py +++ b/scripts/consult_full_data_cache.py @@ -2,16 +2,19 @@ The key is chart identity + ayanamsa + node mode + engine version + the reference date used for current dasha, transits and the year chart. -The file name is a hash. Birth data stays inside the scratch file. -Files older than two days are removed. +The file name is a hash. Birth data stays inside the scratch file, mode 0600. +Files older than two days are removed. The consult card also keeps the last +few parsed packets in memory with their shared domain base (issue #129). """ from __future__ import annotations import hashlib import json +import os import threading import time +from collections import OrderedDict from datetime import date from pathlib import Path from types import SimpleNamespace @@ -24,6 +27,13 @@ KEEP_SECONDS = 2 * 24 * 60 * 60 PacketBuilder = Callable[[dict[str, Any]], dict[str, Any]] +# Parsed consult packets kept in this process, newest last. One entry is about +# 18 MB parsed plus its shared base copy (~30 MB together), so three stay well +# inside the 4 GB host. A hit skips the 5.5 MB file read and the base copy. +MEMORY_SLOTS = 3 +_memory: "OrderedDict[str, tuple[dict[str, Any], dict[str, Any]]]" = OrderedDict() +_memory_guard = threading.Lock() + # One build per cache key. A later caller waits and reads the stored packet. class _Flight: def __init__(self) -> None: @@ -137,6 +147,17 @@ def prune_cache(now: float | None = None, *, name: str = "consult_full_data_cach return removed +def _write_private_text(path: Path, text: str) -> None: + """Birth data: the file is created 0600, never readable by other users. + + Same bytes as before (no key sorting), so a packet read back is unchanged. + """ + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(text) + os.chmod(path, 0o600) + + def _read_cached(path: Path, started: float, now: Callable[[], float]) -> dict[str, Any]: packet = json.loads(path.read_text(encoding="utf-8")) return {"packet": packet, "hit": True, "elapsed_s": now() - started, "slow": False} @@ -186,7 +207,7 @@ def get_or_build( dump_kwargs: dict[str, Any] = {"ensure_ascii": False, "separators": (",", ":")} if json_default is not None: dump_kwargs["default"] = json_default - temporary.write_text(json.dumps(packet, **dump_kwargs), encoding="utf-8") + _write_private_text(temporary, json.dumps(packet, **dump_kwargs)) temporary.replace(path) elapsed = now() - started return { @@ -205,6 +226,51 @@ def get_or_build( _flights.pop(key, None) +def consult_packet_view(identity: dict[str, Any], builder: PacketBuilder | None = None) -> dict[str, Any]: + """The consult card's packet and its shared domain base, both read only. + + Kept in memory for the last ``MEMORY_SLOTS`` charts. Callers must not + mutate ``packet`` or ``base``: later turns read the same objects. Other + users of ``get_or_build`` (report reference, marriage match) still get + their own copy from the file. + """ + try: + from pl9_full_data_export import full_data_domain_base + except ImportError: # pragma: no cover - package import path + from scripts.pl9_full_data_export import full_data_domain_base + key = cache_key(identity) + started = time.perf_counter() + with _memory_guard: + entry = _memory.get(key) + if entry is not None: + _memory.move_to_end(key) + if entry is not None: + return {"packet": entry[0], "base": entry[1], "hit": True, "memory": True, + "elapsed_s": time.perf_counter() - started, "slow": False} + cached = get_or_build(identity, builder) + base = full_data_domain_base(cached["packet"]) + # Remember what a file hit returns: a fresh build is the builder's own + # object, and its JSON round trip (lists for tuples, string keys) is what + # every later turn used to read. This turn still answers from the build. + remembered = (cached["packet"], base) + if not cached["hit"]: + path = _cache_path(identity) + if path.is_file(): + parsed = json.loads(path.read_text(encoding="utf-8")) + remembered = (parsed, full_data_domain_base(parsed)) + with _memory_guard: + _memory[key] = remembered + _memory.move_to_end(key) + while len(_memory) > MEMORY_SLOTS: + _memory.popitem(last=False) + return {**cached, "base": base, "memory": False} + + +def clear_memory_cache() -> None: + with _memory_guard: + _memory.clear() + + def _attach_graha_drishti(packet: dict[str, Any], reading: dict[str, Any]) -> dict[str, Any]: """Copy the engine's whole-sign aspects onto the consult packet. Do not recompute.""" modules = reading.get("modules") if isinstance(reading, dict) else None @@ -274,7 +340,7 @@ def warm_consult_packet(body: dict[str, Any] | None) -> dict[str, Any]: } try: try: - cached = get_or_build(identity_from_body(dict(body or {}))) + cached = consult_packet_view(identity_from_body(dict(body or {}))) except Exception as exc: return { "success": False, @@ -317,12 +383,13 @@ def attach_consult_full_data(result: dict[str, Any], body: dict[str, Any] | None from scripts.consult_card_domain_facts import build_consult_card_facts try: identity = identity_from_body(payload) - cached = get_or_build(identity) + cached = consult_packet_view(identity) facts = build_consult_card_facts( cached["packet"], _requested_domains(payload), minute_confirmed=None, body=payload, + base=cached["base"], ) facts["timing"] = { "hit": bool(cached["hit"]), diff --git a/tests/test_consult_card_cache_memory_20261028.py b/tests/test_consult_card_cache_memory_20261028.py new file mode 100644 index 00000000..32dc5172 --- /dev/null +++ b/tests/test_consult_card_cache_memory_20261028.py @@ -0,0 +1,119 @@ +"""Issue #129: the consult card keeps parsed packets in memory, files are 0600.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import stat +import uuid +from pathlib import Path + +from consult_card_domain_facts import build_consult_card_facts +import consult_full_data_cache as cache + +ROOT = Path(__file__).resolve().parents[1] +DENSITY = ROOT / "frontend" / "tests" / "fixtures" / "report-density-fictional-engine.json" +DOMAINS = ["career", "wealth", "marriage", "health"] + + +def _identity(minute: int = 1) -> dict: + return { + "schema": cache.CACHE_SCHEMA, + "year": 1901, "month": 1, "day": 1, "hour": 0, "minute": minute, "second": 0, + "lat": 0, "lon": 0, "tz": 0, "ayanamsa": "raman", "node_mode": "mean", + "engine_version": f"test-{uuid.uuid4().hex}", + "reference_date": "2026-10-28", + } + + +def _builder(_identity: dict) -> dict: + return json.loads(DENSITY.read_text(encoding="utf-8")) + + +def _fingerprint(value) -> str: + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest() + + +def _facts_the_old_way(identity: dict) -> str: + # Before #129: every turn parsed the file and built facts without a shared base. + packet = json.loads(cache._cache_path(identity).read_text(encoding="utf-8")) + return json.dumps(build_consult_card_facts(packet, DOMAINS, minute_confirmed=True), ensure_ascii=False) + + +def test_memory_hits_give_byte_identical_facts_and_skip_the_file(): + cache.clear_memory_cache() + identity = _identity() + first = cache.consult_packet_view(identity, _builder) + assert first["hit"] is False and first["memory"] is False + expected = _facts_the_old_way(identity) + from_build = build_consult_card_facts(first["packet"], DOMAINS, minute_confirmed=True, base=first["base"]) + assert json.dumps(from_build, ensure_ascii=False) == expected + + # The file is no longer read on a memory hit. + path = cache._cache_path(identity) + saved = path.read_bytes() + path.unlink() + try: + second = cache.consult_packet_view(identity, _builder) + third = cache.consult_packet_view(identity, _builder) + finally: + path.write_bytes(saved) + assert second["hit"] is True and second["memory"] is True + assert second["packet"] is third["packet"] and second["base"] is third["base"] + for view in (second, third): + facts = build_consult_card_facts(view["packet"], DOMAINS, minute_confirmed=True, base=view["base"]) + assert json.dumps(facts, ensure_ascii=False) == expected + + +def test_reading_facts_never_changes_the_shared_packet_or_base(): + cache.clear_memory_cache() + identity = _identity() + cache.consult_packet_view(identity, _builder) + view = cache.consult_packet_view(identity, _builder) + before = (_fingerprint(view["packet"]), _fingerprint(view["base"])) + for _ in range(3): + facts = build_consult_card_facts(view["packet"], DOMAINS, minute_confirmed=False, base=view["base"]) + # A caller that edits its facts must not reach the shared objects. + for row in facts["domains"]: + row.clear() + assert (_fingerprint(view["packet"]), _fingerprint(view["base"])) == before + + +def test_memory_keeps_the_newest_charts_only(): + cache.clear_memory_cache() + identities = [_identity(minute) for minute in range(1, cache.MEMORY_SLOTS + 2)] + for identity in identities: + cache.consult_packet_view(identity, _builder) + assert len(cache._memory) == cache.MEMORY_SLOTS + assert cache.cache_key(identities[0]) not in cache._memory + assert cache.cache_key(identities[-1]) in cache._memory + + +def test_new_cache_files_are_private(): + identity = _identity() + cache.get_or_build(identity, _builder) + mode = stat.S_IMODE(cache._cache_path(identity).stat().st_mode) + assert mode == 0o600, oct(mode) + # Same bytes as before: compact, key order as built (not sorted). + expected = json.dumps(_builder(identity), ensure_ascii=False, separators=(",", ":")) + assert cache._cache_path(identity).read_text(encoding="utf-8") == expected + + +def test_a_miss_answers_from_the_build_and_remembers_the_file_copy(): + cache.clear_memory_cache() + built = {"value": None} + + def builder(identity: dict) -> dict: + packet = _builder(identity) + packet["worksheets"]["tuple_probe"] = ("a", "b") + built["value"] = packet + return packet + + identity = _identity() + first = cache.consult_packet_view(identity, builder) + assert first["packet"] is built["value"] + remembered = cache.consult_packet_view(identity, builder)["packet"] + assert remembered is not built["value"] + assert remembered["worksheets"]["tuple_probe"] == ["a", "b"], "what a file hit always returned" + assert copy.deepcopy(remembered) == json.loads(cache._cache_path(identity).read_text(encoding="utf-8"))