服务器缓存按期限清理，审计快照默认不再落地

排盘缓存、官方快照、负缓存、网关任务和分盘段过期后会删掉。同一张盘的数字写法合成一把缓存键。失败快照只用短期限。自测留存、异步任务、增长合同、安全缓存和隐私标记；一万个文件的清理慢于 2 秒，命中率提升不到 2.5 倍，都按实测写入记录。 Co-Authored-By: Grok (agent loop)
2026-10-14 04:53:30 +08:00
parent 66cd60081f
commit 81e2e1cfed
14 changed files with 1677 additions and 65 deletions
@@ -0,0 +1,314 @@
+"""Retention, chart-cache keys, and VedAstro audit switch."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import threading
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+import api_scratch_retention as retention
+import jyotish_api_server as api
+import vedastro_service_adapter as adapter
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _hex(label: str) -> str:
+    return hashlib.sha256(label.encode("utf-8")).hexdigest() + ".json"
+
+
+def _isolate(monkeypatch, tmp_path: Path) -> None:
+    chart = tmp_path / "chart"
+    jobs = tmp_path / "jobs"
+    chart.mkdir()
+    jobs.mkdir()
+    monkeypatch.setattr(api, "_api_chart_cache_dir", lambda: chart)
+    monkeypatch.setattr(api, "_async_job_dir", lambda scope: jobs)
+    monkeypatch.setattr(api, "_async_job_backend", lambda: "file")
+    monkeypatch.setattr(adapter, "ARTIFACT_DIR", tmp_path / "adapter")
+    monkeypatch.setenv("JYOTISH_VEDASTRO_SNAPSHOT_CACHE_DIR", str(tmp_path / "snap"))
+    monkeypatch.setenv("JYOTISH_VEDASTRO_SNAPSHOT_NEGATIVE_CACHE_DIR", str(tmp_path / "neg"))
+    monkeypatch.setenv("VEDASTRO_GATEWAY_QUEUE_DIR", str(tmp_path / "gw"))
+    monkeypatch.setenv("JYOTISH_CACHE_DIR", str(tmp_path / "cache"))
+    monkeypatch.setenv("JYOTISH_API_CHART_CACHE_TTL_SECONDS", "900")
+    monkeypatch.delenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", raising=False)
+
+
+def _write(path: Path, payload) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload), encoding="utf-8")
+
+
+def test_prune_all_drops_expired_and_old_bad_chart_cache_only(monkeypatch, tmp_path):
+    _isolate(monkeypatch, tmp_path)
+    chart = api._api_chart_cache_dir()
+    now = time.time()
+    fresh = chart / _hex("fresh")
+    expired = chart / _hex("expired")
+    bad_old = chart / _hex("bad-old")
+    bad_young = chart / _hex("bad-young")
+    stranger = chart / "notes.json"
+    _write(fresh, {"cache_key": "a", "created_at": now, "payload": {}})
+    _write(expired, {"cache_key": "b", "created_at": now - 5000, "payload": {}})
+    bad_old.write_text("{", encoding="utf-8")
+    bad_young.write_text("{", encoding="utf-8")
+    stranger.write_text("{}", encoding="utf-8")
+    old = now - (25 * 3600)
+    os.utime(bad_old, (old, old))
+    os.utime(stranger, (old, old))
+
+    result = retention.prune_all(now=now)
+
+    assert fresh.exists()
+    assert bad_young.exists()
+    assert stranger.exists()
+    assert not expired.exists()
+    assert not bad_old.exists()
+    assert result["directories"]["api_chart_cache"]["removed"] == 2
+    assert result["errors"] == 0
+
+
+def test_job_cleanup_still_removes_only_expired_records(monkeypatch, tmp_path):
+    _isolate(monkeypatch, tmp_path)
+    jobs = api._async_job_dir("any")
+    expired = jobs / "expired.json"
+    active = jobs / "active.json"
+    bad = jobs / "bad.json"
+    expired.write_text(json.dumps({"expires_at_unix": time.time() - 1}), encoding="utf-8")
+    active.write_text(json.dumps({"expires_at_unix": time.time() + 60}), encoding="utf-8")
+    bad.write_text("{", encoding="utf-8")
+
+    result = api.prune_expired_async_jobs()
+
+    assert result["removed"] == 1
+    assert not expired.exists()
+    assert active.exists()
+    assert bad.exists()
+
+
+def test_interval_zero_does_not_start_a_thread():
+    before = [item.name for item in threading.enumerate()]
+    assert retention.start_periodic_prune(0) is None
+    assert [item.name for item in threading.enumerate()] == before
+
+
+def test_prune_counts_unlink_errors_without_raising(monkeypatch, tmp_path):
+    _isolate(monkeypatch, tmp_path)
+    chart = api._api_chart_cache_dir()
+    target = chart / _hex("stuck")
+    _write(target, {"created_at": time.time() - 5000, "payload": {}})
+    real_unlink = Path.unlink
+
+    def _boom(self, *args, **kwargs):
+        if self == target:
+            raise OSError("denied")
+        return real_unlink(self, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "unlink", _boom)
+    result = retention.prune_all(now=time.time())
+    assert result["errors"] > 0
+    assert target.exists()
+
+
+def _chart_key(body):
+    return api._api_chart_cache_key(api._build_api_chart_cache_payload(body))
+
+
+def test_numeric_birth_spellings_share_one_chart_cache_key():
+    base = {
+        "year": 1990,
+        "month": 1,
+        "day": 1,
+        "minute": 0,
+        "lat": 28.6139,
+        "lon": 77.2090,
+        "tz": 5.5,
+        "today": "2026-10-14",
+    }
+    keys = {_chart_key(dict(base, hour=hour)) for hour in (12, 12.0, "12")}
+    assert len(keys) == 1
+    assert _chart_key(dict(base, hour=12, second=0)) in keys
+    assert _chart_key(dict(base, hour=12, lat="28.6139")) in keys
+    assert _chart_key(dict(base, hour=12, tz="5.5")) in keys
+    assert _chart_key(dict(base, hour=12, tz="Asia/Kolkata")) not in keys
+    skipped = _chart_key(dict(base, hour=12, skip_vedastro_main_entry_overview=True))
+    assert skipped not in keys
+    assert api._build_api_chart_cache_payload(base)["cache_schema_version"] == 6
+
+
+def test_audit_artifact_default_off_and_compact_when_on(monkeypatch, tmp_path):
+    monkeypatch.setattr(adapter, "ARTIFACT_DIR", tmp_path)
+    monkeypatch.delenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", raising=False)
+    manifest = {"requests": [{"section": "chart_core"}], "reference_date": "2026-10-14"}
+    quiet = adapter._normalize_official_full_snapshot_success(
+        "https://example.test/api",
+        manifest,
+        {"chart_core": {"Status": "Pass"}},
+        {"chart_core": "ok"},
+        1,
+        [],
+    )
+    assert quiet["source_metadata"]["artifact_path"] is None
+    assert list(tmp_path.rglob("*.json")) == []
+
+    monkeypatch.setenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", "1")
+    written = adapter._normalize_official_full_snapshot_success(
+        "https://example.test/api",
+        manifest,
+        {"chart_core": {"Status": "Pass"}},
+        {"chart_core": "ok"},
+        1,
+        [],
+    )
+    rel = written["source_metadata"]["artifact_path"]
+    path = Path(rel)
+    if not path.is_absolute():
+        path = adapter.ROOT / rel
+    assert path.exists()
+    # Linux reports the mode bit. Windows keeps the file writable and does not surface 0600.
+    if os.name != "nt":
+        assert path.stat().st_mode & 0o777 == 0o600
+    text = path.read_text(encoding="utf-8")
+    assert "\n" not in text
+    assert json.loads(text)["status"] == "ok"
+
+
+def test_failed_snapshot_expires_before_success(monkeypatch, tmp_path):
+    monkeypatch.setattr(adapter, "ARTIFACT_DIR", tmp_path)
+    monkeypatch.setenv("VEDASTRO_OFFICIAL_FULL_SNAPSHOT_CACHE_TTL_SECONDS", "604800")
+    monkeypatch.delenv("VEDASTRO_OFFICIAL_FULL_SNAPSHOT_NEGATIVE_TTL_SECONDS", raising=False)
+    calls = {"fail": 0, "ok": 0}
+
+    def fake_run(case, case_id="user_chart"):
+        if case.get("day") == 2:
+            calls["fail"] += 1
+            return {"available": False, "status": "timeout", "source_metadata": {}}
+        calls["ok"] += 1
+        return {"available": True, "status": "ok", "source_metadata": {}}
+
+    monkeypatch.setattr(adapter, "_run_official_full_snapshot_case", fake_run)
+    fail_case = {"year": 1990, "month": 1, "day": 2, "hour": 12, "lat": 1.0, "lon": 2.0, "tz": 5.5}
+    ok_case = {"year": 1990, "month": 1, "day": 3, "hour": 12, "lat": 1.0, "lon": 2.0, "tz": 5.5}
+    adapter.run_official_full_snapshot_for_case(fail_case, case_id="fail")
+    adapter.run_official_full_snapshot_for_case(ok_case, case_id="ok")
+    cache_dir = tmp_path / "official_full_snapshot_cache"
+    aged = time.time() - 301
+    for path in cache_dir.glob("*.json"):
+        record = json.loads(path.read_text(encoding="utf-8"))
+        record["created_at"] = aged
+        path.write_text(json.dumps(record), encoding="utf-8")
+        if record.get("negative") is True:
+            assert record["negative"] is True
+    adapter.run_official_full_snapshot_for_case(fail_case, case_id="fail")
+    adapter.run_official_full_snapshot_for_case(ok_case, case_id="ok")
+    assert calls["fail"] == 2
+    assert calls["ok"] == 1
+
+
+def test_gateway_status_reports_cache_ttl_fallback(monkeypatch):
+    from scripts import vedastro_gateway
+
+    monkeypatch.setenv("JYOTISH_SKIP_LOCAL_ENV", "1")
+    monkeypatch.delenv("VEDASTRO_OFFICIAL_FULL_SNAPSHOT_CACHE_TTL_SECONDS", raising=False)
+    monkeypatch.setenv("VEDASTRO_CACHE_TTL_SECONDS", "604800")
+    status = vedastro_gateway.gateway_status()
+    assert status["official_full_snapshot_cache_ttl_seconds"] == 604800
+    assert status["official_full_snapshot_negative_cache_ttl_seconds"] == 300
+
+
+def _day(offset: int) -> str:
+    return (datetime.now(timezone.utc).date() + timedelta(days=offset)).isoformat()
+
+
+def test_snapshot_negative_gateway_and_varga_prune_rules(monkeypatch, tmp_path):
+    _isolate(monkeypatch, tmp_path)
+    now = time.time()
+    today = datetime.fromtimestamp(now, timezone.utc).date()
+    snap = tmp_path / "snap"
+    neg = tmp_path / "neg"
+    gw = tmp_path / "gw"
+    varga = tmp_path / "cache" / "varga_segments"
+    keep_ref = (today - timedelta(days=7)).isoformat()
+    drop_ref = (today - timedelta(days=9)).isoformat()
+    _write(snap / _hex("snap-keep"), {"schema": "vedastro_snapshot_cache.v1", "reference_date": keep_ref})
+    _write(snap / _hex("snap-drop"), {"schema": "vedastro_snapshot_cache.v1", "reference_date": drop_ref})
+    _write(snap / "notes.json", {"reference_date": "1990-01-01"})
+    _write(neg / _hex("neg-today"), {"schema": "vedastro_snapshot_negative_cache.v1", "reference_date": today.isoformat()})
+    _write(neg / _hex("neg-yesterday"), {"schema": "vedastro_snapshot_negative_cache.v1", "reference_date": (today - timedelta(days=1)).isoformat()})
+    _write(neg / "stray.json", {"reference_date": "1990-01-01"})
+    job_id = "vgw_" + "ab" * 10
+    old_stamp = datetime.fromtimestamp(now - 90000, timezone.utc).isoformat()
+    fresh_stamp = datetime.fromtimestamp(now, timezone.utc).isoformat()
+    _write(gw / f"{job_id}.json", {"job_id": job_id, "status": "completed", "updated_at": old_stamp})
+    (gw / f"{job_id}.official_raw_response.json").write_text("{}", encoding="utf-8")
+    running = "vgw_" + "cd" * 10
+    _write(gw / f"{running}.json", {"job_id": running, "status": "running", "updated_at": old_stamp})
+    failed = "vgw_" + "ef" * 10
+    _write(gw / f"{failed}.json", {"job_id": failed, "status": "failed", "updated_at": fresh_stamp})
+    (gw / "notes.json").write_text("{}", encoding="utf-8")
+    old_varga = varga / _hex("old-segment")
+    young_varga = varga / _hex("young-segment")
+    _write(old_varga, {"window_key": "x"})
+    _write(young_varga, {"window_key": "y"})
+    _write(varga / "notes.json", {"window_key": "z"})
+    os.utime(old_varga, (now - 8 * 86400, now - 8 * 86400))
+
+    monkeypatch.setenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", "0")
+    artifact = tmp_path / "adapter" / "official_full_snapshot-0123456789ab-0123456789ab.json"
+    artifact.parent.mkdir(parents=True, exist_ok=True)
+    artifact.write_text("{}", encoding="utf-8")
+    os.utime(artifact, (now - 25 * 3600, now - 25 * 3600))
+    kept_artifact = tmp_path / "adapter" / "method_catalog_snapshot.json"
+    kept_artifact.write_text("{}", encoding="utf-8")
+    os.utime(kept_artifact, (now - 25 * 3600, now - 25 * 3600))
+
+    retention.prune_all(now=now)
+
+    assert (snap / _hex("snap-keep")).exists()
+    assert not (snap / _hex("snap-drop")).exists()
+    assert (snap / "notes.json").exists()
+    assert (neg / _hex("neg-today")).exists()
+    assert not (neg / _hex("neg-yesterday")).exists()
+    assert (neg / "stray.json").exists()
+    assert not (gw / f"{job_id}.json").exists()
+    assert not (gw / f"{job_id}.official_raw_response.json").exists()
+    assert (gw / f"{running}.json").exists()
+    assert (gw / f"{failed}.json").exists()
+    assert (gw / "notes.json").exists()
+    assert not old_varga.exists()
+    assert young_varga.exists()
+    assert (varga / "notes.json").exists()
+    assert not artifact.exists()
+    assert kept_artifact.exists()
+
+
+def _scratch_local_names() -> set[str]:
+    names: set[str] = set()
+    slash = re.compile(r"scratch/local/([A-Za-z0-9_.]+)")
+    joined = re.compile(
+        r"""['\"]scratch['\"]\s*/\s*['\"]local['\"]\s*/\s*(?:f)?['\"]([A-Za-z0-9_.]+)['\"]"""
+    )
+    after_local = re.compile(
+        r"""['\"]local['\"]\s*\)*\s*/\s*['\"]([A-Za-z0-9_.]+)['\"]"""
+    )
+    for path in (ROOT / "scripts").rglob("*.py"):
+        text = path.read_text(encoding="utf-8")
+        names.update(slash.findall(text))
+        names.update(joined.findall(text))
+        names.update(after_local.findall(text))
+    return names
+
+
+def test_new_scratch_local_dirs_are_registered_or_whitelisted():
+    allowed = set(retention.REGISTERED_DIR_NAMES) | set(retention.OFFLINE_SCRATCH_NAMES)
+    found = _scratch_local_names()
+    unexpected = sorted(found - allowed)
+    assert unexpected == [], unexpected
+    assert "api_chart_cache" in found
+    assert "consult_full_data_cache" in retention.OFFLINE_SCRATCH_NAMES
@@ -769,6 +769,10 @@ def test_vedastro_range_scan_records_hashes_and_artifact_path() -> None:
        env = os.environ.copy()
        env["VEDASTRO_API_ENDPOINT"] = f"http://127.0.0.1:{server.server_port}/vedastro"
        env["VEDASTRO_ENABLE_NETWORK"] = "1"
+        # 原值：沿用进程环境，没有 VEDASTRO_WRITE_AUDIT_ARTIFACTS。
+        # 新值：显式设为 1。
+        # 原因：审计落盘默认关闭后，这条测试仍要断言文件存在，所以打开开关。这是测试前置，不是放宽断言。
+        env["VEDASTRO_WRITE_AUDIT_ARTIFACTS"] = "1"
        completed = subprocess.run(
            [
                sys.executable,