Jyotisha/tests/test_api_scratch_retention.py

"""Retention, chart-cache keys, and VedAstro audit switch."""
from __future__ import annotations

import hashlib
import json
import os
import re
import threading
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path

import pytest

import api_scratch_retention as retention
import jyotish_api_server as api
import vedastro_service_adapter as adapter

ROOT = Path(__file__).resolve().parents[1]


def _hex(label: str) -> str:
    return hashlib.sha256(label.encode("utf-8")).hexdigest() + ".json"


def _isolate(monkeypatch, tmp_path: Path) -> None:
    chart = tmp_path / "chart"
    jobs = tmp_path / "jobs"
    chart.mkdir()
    jobs.mkdir()
    monkeypatch.setattr(api, "_api_chart_cache_dir", lambda: chart)
    monkeypatch.setattr(api, "_async_job_dir", lambda scope: jobs)
    monkeypatch.setattr(api, "_async_job_backend", lambda: "file")
    monkeypatch.setattr(adapter, "ARTIFACT_DIR", tmp_path / "adapter")
    monkeypatch.setenv("JYOTISH_VEDASTRO_SNAPSHOT_CACHE_DIR", str(tmp_path / "snap"))
    monkeypatch.setenv("JYOTISH_VEDASTRO_SNAPSHOT_NEGATIVE_CACHE_DIR", str(tmp_path / "neg"))
    monkeypatch.setenv("VEDASTRO_GATEWAY_QUEUE_DIR", str(tmp_path / "gw"))
    monkeypatch.setenv("JYOTISH_CACHE_DIR", str(tmp_path / "cache"))
    monkeypatch.setenv("JYOTISH_API_CHART_CACHE_TTL_SECONDS", "900")
    monkeypatch.delenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", raising=False)


def _write(path: Path, payload) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(payload), encoding="utf-8")


def test_prune_all_drops_expired_and_old_bad_chart_cache_only(monkeypatch, tmp_path):
    _isolate(monkeypatch, tmp_path)
    chart = api._api_chart_cache_dir()
    now = time.time()
    fresh = chart / _hex("fresh")
    expired = chart / _hex("expired")
    bad_old = chart / _hex("bad-old")
    bad_young = chart / _hex("bad-young")
    stranger = chart / "notes.json"
    _write(fresh, {"cache_key": "a", "created_at": now, "payload": {}})
    _write(expired, {"cache_key": "b", "created_at": now - 5000, "payload": {}})
    bad_old.write_text("{", encoding="utf-8")
    bad_young.write_text("{", encoding="utf-8")
    stranger.write_text("{}", encoding="utf-8")
    old = now - (25 * 3600)
    os.utime(bad_old, (old, old))
    os.utime(stranger, (old, old))

    result = retention.prune_all(now=now)

    assert fresh.exists()
    assert bad_young.exists()
    assert stranger.exists()
    assert not expired.exists()
    assert not bad_old.exists()
    assert result["directories"]["api_chart_cache"]["removed"] == 2
    assert result["errors"] == 0


def test_job_cleanup_still_removes_only_expired_records(monkeypatch, tmp_path):
    _isolate(monkeypatch, tmp_path)
    jobs = api._async_job_dir("any")
    expired = jobs / "expired.json"
    active = jobs / "active.json"
    bad = jobs / "bad.json"
    expired.write_text(json.dumps({"expires_at_unix": time.time() - 1}), encoding="utf-8")
    active.write_text(json.dumps({"expires_at_unix": time.time() + 60}), encoding="utf-8")
    bad.write_text("{", encoding="utf-8")

    result = api.prune_expired_async_jobs()

    assert result["removed"] == 1
    assert not expired.exists()
    assert active.exists()
    assert bad.exists()


def test_interval_zero_does_not_start_a_thread():
    before = [item.name for item in threading.enumerate()]
    assert retention.start_periodic_prune(0) is None
    assert [item.name for item in threading.enumerate()] == before


def test_prune_counts_unlink_errors_without_raising(monkeypatch, tmp_path):
    _isolate(monkeypatch, tmp_path)
    chart = api._api_chart_cache_dir()
    target = chart / _hex("stuck")
    _write(target, {"created_at": time.time() - 5000, "payload": {}})
    real_unlink = Path.unlink

    def _boom(self, *args, **kwargs):
        if self == target:
            raise OSError("denied")
        return real_unlink(self, *args, **kwargs)

    monkeypatch.setattr(Path, "unlink", _boom)
    result = retention.prune_all(now=time.time())
    assert result["errors"] > 0
    assert target.exists()


def _chart_key(body):
    return api._api_chart_cache_key(api._build_api_chart_cache_payload(body))


def test_numeric_birth_spellings_share_one_chart_cache_key():
    base = {
        "year": 1990,
        "month": 1,
        "day": 1,
        "minute": 0,
        "lat": 28.6139,
        "lon": 77.2090,
        "tz": 5.5,
        "today": "2026-10-14",
    }
    keys = {_chart_key(dict(base, hour=hour)) for hour in (12, 12.0, "12")}
    assert len(keys) == 1
    assert _chart_key(dict(base, hour=12, second=0)) in keys
    assert _chart_key(dict(base, hour=12, lat="28.6139")) in keys
    assert _chart_key(dict(base, hour=12, tz="5.5")) in keys
    assert _chart_key(dict(base, hour=12, tz="Asia/Kolkata")) not in keys
    skipped = _chart_key(dict(base, hour=12, skip_vedastro_main_entry_overview=True))
    assert skipped not in keys
    assert api._build_api_chart_cache_payload(base)["cache_schema_version"] == 6


def test_audit_artifact_default_off_and_compact_when_on(monkeypatch, tmp_path):
    monkeypatch.setattr(adapter, "ARTIFACT_DIR", tmp_path)
    monkeypatch.delenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", raising=False)
    manifest = {"requests": [{"section": "chart_core"}], "reference_date": "2026-10-14"}
    quiet = adapter._normalize_official_full_snapshot_success(
        "https://example.test/api",
        manifest,
        {"chart_core": {"Status": "Pass"}},
        {"chart_core": "ok"},
        1,
        [],
    )
    assert quiet["source_metadata"]["artifact_path"] is None
    assert list(tmp_path.rglob("*.json")) == []

    monkeypatch.setenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", "1")
    written = adapter._normalize_official_full_snapshot_success(
        "https://example.test/api",
        manifest,
        {"chart_core": {"Status": "Pass"}},
        {"chart_core": "ok"},
        1,
        [],
    )
    rel = written["source_metadata"]["artifact_path"]
    path = Path(rel)
    if not path.is_absolute():
        path = adapter.ROOT / rel
    assert path.exists()
    # Linux reports the mode bit. Windows keeps the file writable and does not surface 0600.
    if os.name != "nt":
        assert path.stat().st_mode & 0o777 == 0o600
    text = path.read_text(encoding="utf-8")
    assert "\n" not in text
    assert json.loads(text)["status"] == "ok"


def test_failed_snapshot_expires_before_success(monkeypatch, tmp_path):
    monkeypatch.setattr(adapter, "ARTIFACT_DIR", tmp_path)
    monkeypatch.setenv("VEDASTRO_OFFICIAL_FULL_SNAPSHOT_CACHE_TTL_SECONDS", "604800")
    monkeypatch.delenv("VEDASTRO_OFFICIAL_FULL_SNAPSHOT_NEGATIVE_TTL_SECONDS", raising=False)
    calls = {"fail": 0, "ok": 0}

    def fake_run(case, case_id="user_chart"):
        if case.get("day") == 2:
            calls["fail"] += 1
            return {"available": False, "status": "timeout", "source_metadata": {}}
        calls["ok"] += 1
        return {"available": True, "status": "ok", "source_metadata": {}}

    monkeypatch.setattr(adapter, "_run_official_full_snapshot_case", fake_run)
    fail_case = {"year": 1990, "month": 1, "day": 2, "hour": 12, "lat": 1.0, "lon": 2.0, "tz": 5.5}
    ok_case = {"year": 1990, "month": 1, "day": 3, "hour": 12, "lat": 1.0, "lon": 2.0, "tz": 5.5}
    adapter.run_official_full_snapshot_for_case(fail_case, case_id="fail")
    adapter.run_official_full_snapshot_for_case(ok_case, case_id="ok")
    cache_dir = tmp_path / "official_full_snapshot_cache"
    aged = time.time() - 301
    for path in cache_dir.glob("*.json"):
        record = json.loads(path.read_text(encoding="utf-8"))
        record["created_at"] = aged
        path.write_text(json.dumps(record), encoding="utf-8")
        if record.get("negative") is True:
            assert record["negative"] is True
    adapter.run_official_full_snapshot_for_case(fail_case, case_id="fail")
    adapter.run_official_full_snapshot_for_case(ok_case, case_id="ok")
    assert calls["fail"] == 2
    assert calls["ok"] == 1


def test_gateway_status_reports_cache_ttl_fallback(monkeypatch):
    from scripts import vedastro_gateway

    monkeypatch.setenv("JYOTISH_SKIP_LOCAL_ENV", "1")
    monkeypatch.delenv("VEDASTRO_OFFICIAL_FULL_SNAPSHOT_CACHE_TTL_SECONDS", raising=False)
    monkeypatch.setenv("VEDASTRO_CACHE_TTL_SECONDS", "604800")
    status = vedastro_gateway.gateway_status()
    assert status["official_full_snapshot_cache_ttl_seconds"] == 604800
    assert status["official_full_snapshot_negative_cache_ttl_seconds"] == 300


def _day(offset: int) -> str:
    return (datetime.now(timezone.utc).date() + timedelta(days=offset)).isoformat()


def test_snapshot_negative_gateway_and_varga_prune_rules(monkeypatch, tmp_path):
    _isolate(monkeypatch, tmp_path)
    now = time.time()
    today = datetime.fromtimestamp(now, timezone.utc).date()
    snap = tmp_path / "snap"
    neg = tmp_path / "neg"
    gw = tmp_path / "gw"
    varga = tmp_path / "cache" / "varga_segments"
    keep_ref = (today - timedelta(days=7)).isoformat()
    drop_ref = (today - timedelta(days=9)).isoformat()
    _write(snap / _hex("snap-keep"), {"schema": "vedastro_snapshot_cache.v1", "reference_date": keep_ref})
    _write(snap / _hex("snap-drop"), {"schema": "vedastro_snapshot_cache.v1", "reference_date": drop_ref})
    _write(snap / "notes.json", {"reference_date": "1990-01-01"})
    _write(neg / _hex("neg-today"), {"schema": "vedastro_snapshot_negative_cache.v1", "reference_date": today.isoformat()})
    _write(neg / _hex("neg-yesterday"), {"schema": "vedastro_snapshot_negative_cache.v1", "reference_date": (today - timedelta(days=1)).isoformat()})
    _write(neg / "stray.json", {"reference_date": "1990-01-01"})
    job_id = "vgw_" + "ab" * 10
    old_stamp = datetime.fromtimestamp(now - 90000, timezone.utc).isoformat()
    fresh_stamp = datetime.fromtimestamp(now, timezone.utc).isoformat()
    _write(gw / f"{job_id}.json", {"job_id": job_id, "status": "completed", "updated_at": old_stamp})
    (gw / f"{job_id}.official_raw_response.json").write_text("{}", encoding="utf-8")
    running = "vgw_" + "cd" * 10
    _write(gw / f"{running}.json", {"job_id": running, "status": "running", "updated_at": old_stamp})
    failed = "vgw_" + "ef" * 10
    _write(gw / f"{failed}.json", {"job_id": failed, "status": "failed", "updated_at": fresh_stamp})
    (gw / "notes.json").write_text("{}", encoding="utf-8")
    old_varga = varga / _hex("old-segment")
    young_varga = varga / _hex("young-segment")
    _write(old_varga, {"window_key": "x"})
    _write(young_varga, {"window_key": "y"})
    _write(varga / "notes.json", {"window_key": "z"})
    os.utime(old_varga, (now - 8 * 86400, now - 8 * 86400))

    monkeypatch.setenv("VEDASTRO_WRITE_AUDIT_ARTIFACTS", "0")
    artifact = tmp_path / "adapter" / "official_full_snapshot-0123456789ab-0123456789ab.json"
    artifact.parent.mkdir(parents=True, exist_ok=True)
    artifact.write_text("{}", encoding="utf-8")
    os.utime(artifact, (now - 25 * 3600, now - 25 * 3600))
    kept_artifact = tmp_path / "adapter" / "method_catalog_snapshot.json"
    kept_artifact.write_text("{}", encoding="utf-8")
    os.utime(kept_artifact, (now - 25 * 3600, now - 25 * 3600))

    retention.prune_all(now=now)

    assert (snap / _hex("snap-keep")).exists()
    assert not (snap / _hex("snap-drop")).exists()
    assert (snap / "notes.json").exists()
    assert (neg / _hex("neg-today")).exists()
    assert not (neg / _hex("neg-yesterday")).exists()
    assert (neg / "stray.json").exists()
    assert not (gw / f"{job_id}.json").exists()
    assert not (gw / f"{job_id}.official_raw_response.json").exists()
    assert (gw / f"{running}.json").exists()
    assert (gw / f"{failed}.json").exists()
    assert (gw / "notes.json").exists()
    assert not old_varga.exists()
    assert young_varga.exists()
    assert (varga / "notes.json").exists()
    assert not artifact.exists()
    assert kept_artifact.exists()


def _scratch_local_names() -> set[str]:
    names: set[str] = set()
    slash = re.compile(r"scratch/local/([A-Za-z0-9_.]+)")
    joined = re.compile(
        r"""['\"]scratch['\"]\s*/\s*['\"]local['\"]\s*/\s*(?:f)?['\"]([A-Za-z0-9_.]+)['\"]"""
    )
    after_local = re.compile(
        r"""['\"]local['\"]\s*\)*\s*/\s*['\"]([A-Za-z0-9_.]+)['\"]"""
    )
    for path in (ROOT / "scripts").rglob("*.py"):
        text = path.read_text(encoding="utf-8")
        names.update(slash.findall(text))
        names.update(joined.findall(text))
        names.update(after_local.findall(text))
    return names


def test_new_scratch_local_dirs_are_registered_or_whitelisted():
    allowed = set(retention.REGISTERED_DIR_NAMES) | set(retention.OFFLINE_SCRATCH_NAMES)
    found = _scratch_local_names()
    unexpected = sorted(found - allowed)
    assert unexpected == [], unexpected
    assert "api_chart_cache" in found
    assert "consult_full_data_cache" in retention.OFFLINE_SCRATCH_NAMES