jesse-uxandGrok (agent loop)
81e2e1cfed
排盘缓存、官方快照、负缓存、网关任务和分盘段过期后会删掉。同一张盘的数字写法合成一把缓存键。失败快照只用短期限。自测留存、异步任务、增长合同、安全缓存和隐私标记；一万个文件的清理慢于 2 秒，命中率提升不到 2.5 倍，都按实测写入记录。 Co-Authored-By: Grok (agent loop)
443 lines
16 KiB
Python
443 lines
16 KiB
Python
"""A/B for chart-cache keys, retention, and response bytes.
|
|
|
|
Parent functions come from git show of the branch base (origin/staging at
|
|
branch creation), not from the task text's older baseline. Scratch files
|
|
stay in a temporary directory.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import socket
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PARENT = "66cd60081f37a57680274633fcb3fb942d01bb2f"
|
|
os.environ.setdefault("JYOTISH_SKIP_LOCAL_ENV", "1")
|
|
os.environ.setdefault("VEDASTRO_ENABLE_NETWORK", "0")
|
|
|
|
_SDK_PRESENT = importlib.util.find_spec("vedastro") is not None
|
|
|
|
|
|
def _network_reachable() -> bool:
|
|
try:
|
|
sock = socket.create_connection(("api.vedastro.org", 443), 3)
|
|
except OSError:
|
|
return False
|
|
sock.close()
|
|
return True
|
|
|
|
|
|
_NETWORK = _network_reachable()
|
|
|
|
|
|
class _Block:
|
|
def find_spec(self, name, path=None, target=None):
|
|
if name.split(".")[0] in {"jhora", "vedastro"}:
|
|
raise ModuleNotFoundError(f"No module named '{name}'")
|
|
return None
|
|
|
|
|
|
if "--with-sdk" not in sys.argv:
|
|
sys.meta_path.insert(0, _Block())
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
import api_scratch_retention as retention # noqa: E402
|
|
import jyotish_api_server as current # noqa: E402
|
|
import vedastro_service_adapter as adapter # noqa: E402
|
|
|
|
|
|
def _load_parent():
|
|
raw = subprocess.check_output(["git", "show", f"{PARENT}:scripts/jyotish_api_server.py"], text=True)
|
|
raw = raw.replace(
|
|
"SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))",
|
|
f"SCRIPTS_DIR = {str(ROOT / 'scripts')!r}",
|
|
)
|
|
raw = raw.replace(
|
|
"REPO_ROOT = os.path.abspath(os.path.join(SCRIPTS_DIR, '..'))",
|
|
f"REPO_ROOT = {str(ROOT)!r}",
|
|
)
|
|
directory = Path(tempfile.mkdtemp(prefix="jyotish-cache-a-"))
|
|
path = directory / "jyotish_api_server_parent.py"
|
|
path.write_text(raw, encoding="utf-8")
|
|
spec = importlib.util.spec_from_file_location("jyotish_api_server_parent", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _handler(module):
|
|
cls = module.JyotishAPIHandler
|
|
return cls.__new__(cls)
|
|
|
|
|
|
def _strip(payload: dict) -> str:
|
|
cloned = json.loads(json.dumps(payload, default=str))
|
|
cloned.pop("runtime_cache", None)
|
|
return json.dumps(cloned, ensure_ascii=False, sort_keys=True, default=str)
|
|
|
|
|
|
def _diff_paths(left: str, right: str, limit: int = 12) -> list[str]:
|
|
a = json.loads(left)
|
|
b = json.loads(right)
|
|
|
|
found: list[str] = []
|
|
|
|
def walk(one, two, prefix: str) -> None:
|
|
if len(found) >= limit:
|
|
return
|
|
if type(one) is not type(two):
|
|
found.append(prefix or "$")
|
|
return
|
|
if isinstance(one, dict):
|
|
for key in sorted(set(one) | set(two)):
|
|
if key not in one or key not in two:
|
|
found.append(f"{prefix}.{key}")
|
|
continue
|
|
walk(one[key], two[key], f"{prefix}.{key}")
|
|
elif isinstance(one, list):
|
|
if len(one) != len(two):
|
|
found.append(f"{prefix}[]")
|
|
return
|
|
for index, (item, other) in enumerate(zip(one, two)):
|
|
walk(item, other, f"{prefix}[{index}]")
|
|
elif one != two:
|
|
found.append(prefix or "$")
|
|
|
|
walk(a, b, "")
|
|
return found
|
|
|
|
|
|
def _body(hour, lat, **extra) -> dict:
|
|
payload = {
|
|
"year": 1990,
|
|
"month": 1,
|
|
"day": 1,
|
|
"minute": 0,
|
|
"lon": 20.0,
|
|
"tz": 5.5,
|
|
"hour": hour,
|
|
"lat": lat,
|
|
"today": "2026-10-14",
|
|
}
|
|
payload.update(extra)
|
|
return payload
|
|
|
|
|
|
def run_byte_diff(parent=None) -> dict:
|
|
os.environ["JYOTISH_API_CHART_CACHE_TTL_SECONDS"] = "0"
|
|
os.environ["VEDASTRO_WRITE_AUDIT_ARTIFACTS"] = "0"
|
|
parent = parent or current
|
|
groups = {
|
|
"hour": [_body(12, 10.0), _body(12.0, 10.0), _body("12", 10.0)],
|
|
"lat": [_body(12, 10.0), _body(12, "10.0")],
|
|
"tz": [_body(12, 10.0), _body(12, 10.0, tz="5.5")],
|
|
"second": [_body(12, 10.0), _body(12, 10.0, second=0)],
|
|
}
|
|
report = {"groups": {}, "stable_repeat": None}
|
|
handler = _handler(current)
|
|
started = time.perf_counter()
|
|
try:
|
|
sample = handler._compute_chart_sync(_body(12, 10.0, skip_vedastro_main_entry_overview=True))
|
|
except Exception as exc:
|
|
return {"ok": False, "error": exc.__class__.__name__, "elapsed_s": round(time.perf_counter() - started, 3)}
|
|
sample_s = round(time.perf_counter() - started, 3)
|
|
repeat = handler._compute_chart_sync(_body(12, 10.0, skip_vedastro_main_entry_overview=True))
|
|
report["stable_repeat"] = _strip(sample) == _strip(repeat)
|
|
report["sample_seconds"] = sample_s
|
|
if not report["stable_repeat"]:
|
|
report["volatile_paths"] = _diff_paths(_strip(sample), _strip(repeat))
|
|
for name, bodies in groups.items():
|
|
for skip in (True, False):
|
|
rendered = []
|
|
for body in bodies:
|
|
item = dict(body)
|
|
if skip:
|
|
item["skip_vedastro_main_entry_overview"] = True
|
|
rendered.append(_strip(handler._compute_chart_sync(item)))
|
|
same = len(set(rendered)) == 1
|
|
entry = {"skip": skip, "same": same}
|
|
if not same:
|
|
entry["paths"] = _diff_paths(rendered[0], rendered[1])
|
|
clocks = ("cache_created_at", "cache_expires_at", "called_at")
|
|
entry["only_clock_fields"] = bool(entry["paths"]) and all(
|
|
path.endswith(clocks) for path in entry["paths"]
|
|
)
|
|
else:
|
|
entry["only_clock_fields"] = False
|
|
report["groups"][f"{name}:skip={skip}"] = entry
|
|
if parent is not current:
|
|
a_handler = _handler(parent)
|
|
b_handler = handler
|
|
body = _body(12, 10.0, skip_vedastro_main_entry_overview=True)
|
|
a_text = _strip(a_handler._compute_chart_sync(body))
|
|
b_text = _strip(b_handler._compute_chart_sync(body))
|
|
report["a_vs_b_same"] = a_text == b_text
|
|
if a_text != b_text:
|
|
report["a_vs_b_paths"] = _diff_paths(a_text, b_text)
|
|
report["ok"] = all(item["same"] or item.get("only_clock_fields") for item in report["groups"].values())
|
|
return report
|
|
|
|
|
|
def _requests():
|
|
charts = []
|
|
for index in range(20):
|
|
charts.append(
|
|
{
|
|
"year": 1990,
|
|
"month": 1,
|
|
"day": 1 + (index % 27),
|
|
"minute": index % 60,
|
|
"lon": float(index),
|
|
"tz": 5.5,
|
|
}
|
|
)
|
|
spellings = (
|
|
{"hour": 12, "lat": 10.0},
|
|
{"hour": 12.0, "lat": "10.0"},
|
|
{"hour": "12", "lat": 10.0},
|
|
)
|
|
days = [f"2026-10-{day:02d}" for day in range(8, 15)]
|
|
rows = []
|
|
for chart in charts:
|
|
for spelling in spellings:
|
|
for skip in (False, True):
|
|
for day in days:
|
|
body = dict(chart)
|
|
body.update(spelling)
|
|
body["today"] = day
|
|
if skip:
|
|
body["skip_vedastro_main_entry_overview"] = True
|
|
rows.append(body)
|
|
return rows
|
|
|
|
|
|
def _key(module, body) -> str:
|
|
return module._api_chart_cache_key(module._build_api_chart_cache_payload(body))
|
|
|
|
|
|
def _hit_rate(module, rows) -> dict:
|
|
seen = set()
|
|
hits = 0
|
|
for body in rows:
|
|
for _repeat in range(3):
|
|
key = _key(module, body)
|
|
if key in seen:
|
|
hits += 1
|
|
else:
|
|
seen.add(key)
|
|
total = len(rows) * 3
|
|
return {"requests": total, "unique_keys": len(seen), "hits": hits, "hit_rate": round(hits / total, 4)}
|
|
|
|
|
|
def _seed_and_prune(module, rows) -> dict:
|
|
now = time.time()
|
|
os.environ["JYOTISH_API_CHART_CACHE_TTL_SECONDS"] = "900"
|
|
scratch = Path(tempfile.mkdtemp(prefix="jyotish-cache-b-"))
|
|
chart = scratch / "api_chart_cache"
|
|
jobs = scratch / "jobs"
|
|
adapter_dir = scratch / "adapter"
|
|
chart.mkdir()
|
|
jobs.mkdir()
|
|
current._api_chart_cache_dir = lambda: chart
|
|
current._async_job_dir = lambda scope: jobs
|
|
current._async_job_backend = lambda: "file"
|
|
adapter.ARTIFACT_DIR = adapter_dir
|
|
os.environ["JYOTISH_VEDASTRO_SNAPSHOT_CACHE_DIR"] = str(scratch / "snap")
|
|
os.environ["JYOTISH_VEDASTRO_SNAPSHOT_NEGATIVE_CACHE_DIR"] = str(scratch / "neg")
|
|
os.environ["VEDASTRO_GATEWAY_QUEUE_DIR"] = str(scratch / "gw")
|
|
os.environ["JYOTISH_CACHE_DIR"] = str(scratch / "cache")
|
|
days = sorted({body["today"] for body in rows})
|
|
newest = days[-1]
|
|
written = {}
|
|
for body in rows:
|
|
key = _key(module, body)
|
|
age_days = days.index(body["today"])
|
|
age_days = len(days) - 1 - age_days
|
|
written[key] = now - age_days * 86400
|
|
for key, created in written.items():
|
|
path = chart / f"{key}.json"
|
|
path.write_text(json.dumps({"cache_key": key, "created_at": created, "payload": {}}), encoding="utf-8")
|
|
before = len(list(chart.glob("*.json")))
|
|
before_bytes = sum(path.stat().st_size for path in chart.glob("*.json"))
|
|
today_keys = len({_key(module, body) for body in rows if body["today"] == newest})
|
|
if module is current:
|
|
prune = retention.prune_all(now=now)
|
|
after = len(list(chart.glob("*.json")))
|
|
elapsed = prune["directories"]["api_chart_cache"]["elapsed_ms"]
|
|
else:
|
|
after = before
|
|
elapsed = None
|
|
return {
|
|
"files_before": before,
|
|
"bytes_before": before_bytes,
|
|
"files_after": after,
|
|
"today_active_keys": today_keys,
|
|
"chart_prune_ms": elapsed,
|
|
}
|
|
|
|
|
|
def _artifact_counts() -> dict:
|
|
scratch = Path(tempfile.mkdtemp(prefix="jyotish-artifact-ab-"))
|
|
adapter.ARTIFACT_DIR = scratch
|
|
os.environ.pop("VEDASTRO_WRITE_AUDIT_ARTIFACTS", None)
|
|
result = {
|
|
"operation": "official_full_snapshot",
|
|
"source_metadata": {
|
|
"operation": "official_full_snapshot",
|
|
"request_hash": "a" * 64,
|
|
"response_hash": "b" * 64,
|
|
},
|
|
}
|
|
adapter._write_artifact(result)
|
|
return {"b_artifact_files": len(list(scratch.glob("*.json")))}
|
|
|
|
|
|
def _negative_cache_after_prune() -> dict:
|
|
scratch = Path(tempfile.mkdtemp(prefix="jyotish-neg-ab-"))
|
|
os.environ["JYOTISH_VEDASTRO_SNAPSHOT_NEGATIVE_CACHE_DIR"] = str(scratch)
|
|
os.environ["JYOTISH_VEDASTRO_SNAPSHOT_CACHE_DIR"] = str(scratch / "unused-snap")
|
|
os.environ["VEDASTRO_GATEWAY_QUEUE_DIR"] = str(scratch / "unused-gw")
|
|
os.environ["JYOTISH_CACHE_DIR"] = str(scratch / "unused-cache")
|
|
current._api_chart_cache_dir = lambda: scratch / "unused-chart"
|
|
current._async_job_dir = lambda scope: scratch / "unused-jobs"
|
|
(scratch / "unused-chart").mkdir()
|
|
(scratch / "unused-jobs").mkdir()
|
|
today = time.strftime("%Y-%m-%d", time.gmtime())
|
|
yesterday = time.strftime("%Y-%m-%d", time.gmtime(time.time() - 86400))
|
|
for label, day in (("a" * 64, today), ("b" * 64, yesterday)):
|
|
path = scratch / f"{label}.json"
|
|
path.write_text(json.dumps({"reference_date": day}), encoding="utf-8")
|
|
retention.prune_all(now=time.time())
|
|
return {"files_after": len(list(scratch.glob("*.json")))}
|
|
|
|
|
|
def _prune_10k() -> dict:
|
|
scratch = Path(tempfile.mkdtemp(prefix="jyotish-prune-10k-"))
|
|
chart = scratch / "chart"
|
|
chart.mkdir()
|
|
current._api_chart_cache_dir = lambda: chart
|
|
current._async_job_dir = lambda scope: scratch / "jobs"
|
|
(scratch / "jobs").mkdir()
|
|
current._async_job_backend = lambda: "file"
|
|
adapter.ARTIFACT_DIR = scratch / "adapter"
|
|
os.environ["JYOTISH_API_CHART_CACHE_TTL_SECONDS"] = "900"
|
|
os.environ["JYOTISH_VEDASTRO_SNAPSHOT_CACHE_DIR"] = str(scratch / "snap")
|
|
os.environ["JYOTISH_VEDASTRO_SNAPSHOT_NEGATIVE_CACHE_DIR"] = str(scratch / "neg")
|
|
os.environ["VEDASTRO_GATEWAY_QUEUE_DIR"] = str(scratch / "gw")
|
|
os.environ["JYOTISH_CACHE_DIR"] = str(scratch / "cache")
|
|
for index in range(10000):
|
|
path = chart / f"{index:064x}.json"
|
|
path.write_text('{"created_at":1,"payload":{}}', encoding="utf-8")
|
|
started = time.perf_counter()
|
|
result = retention.prune_all(now=time.time())
|
|
elapsed = time.perf_counter() - started
|
|
return {
|
|
"prune_all_seconds": round(elapsed, 3),
|
|
"chart_removed": result["directories"]["api_chart_cache"]["removed"],
|
|
"chart_elapsed_ms": result["directories"]["api_chart_cache"]["elapsed_ms"],
|
|
"files_left": len(list(chart.glob("*.json"))),
|
|
}
|
|
|
|
|
|
def _one_chart_timing() -> dict:
|
|
os.environ["JYOTISH_API_CHART_CACHE_TTL_SECONDS"] = "0"
|
|
handler = _handler(current)
|
|
body = _body(12, 10.0, skip_vedastro_main_entry_overview=True)
|
|
misses = []
|
|
for _index in range(3):
|
|
started = time.perf_counter()
|
|
handler._compute_chart_sync(body)
|
|
misses.append((time.perf_counter() - started) * 1000)
|
|
os.environ["JYOTISH_API_CHART_CACHE_TTL_SECONDS"] = "900"
|
|
scratch = Path(tempfile.mkdtemp(prefix="jyotish-hit-time-"))
|
|
current._api_chart_cache_dir = lambda: scratch
|
|
cold = handler._compute_chart_sync(body)
|
|
hits = []
|
|
for _index in range(3):
|
|
started = time.perf_counter()
|
|
hit = handler._compute_chart_sync(body)
|
|
hits.append((time.perf_counter() - started) * 1000)
|
|
assert hit.get("runtime_cache", {}).get("cache_hit") is True
|
|
return {
|
|
"miss_median_ms": round(statistics.median(misses), 3),
|
|
"hit_median_ms": round(statistics.median(hits), 3),
|
|
"cold_has_runtime_cache": isinstance(cold, dict) and "runtime_cache" in cold,
|
|
}
|
|
|
|
|
|
def main() -> dict:
|
|
parent_error = None
|
|
try:
|
|
parent = _load_parent()
|
|
except Exception as exc:
|
|
parent = None
|
|
parent_error = exc.__class__.__name__
|
|
rows = _requests()
|
|
side_a = _hit_rate(parent, rows) if parent is not None else None
|
|
side_b = _hit_rate(current, rows)
|
|
report = {
|
|
"parent": PARENT,
|
|
"parent_import": parent_error or "ok",
|
|
"vedastro_sdk": _SDK_PRESENT,
|
|
"vedastro_network": _NETWORK,
|
|
"hit_rate_a": side_a,
|
|
"hit_rate_b": side_b,
|
|
"artifacts": _artifact_counts(),
|
|
"negative_cache_files_after_prune": _negative_cache_after_prune(),
|
|
"prune_10k": _prune_10k(),
|
|
}
|
|
if side_a and side_a["hit_rate"]:
|
|
report["hit_rate_ratio_b_over_a"] = round(side_b["hit_rate"] / side_a["hit_rate"], 3)
|
|
try:
|
|
report["files_b"] = _seed_and_prune(current, rows)
|
|
if parent is not None:
|
|
report["files_a"] = _seed_and_prune(parent, rows)
|
|
except Exception as exc:
|
|
report["files_error"] = exc.__class__.__name__
|
|
try:
|
|
report["timing"] = _one_chart_timing()
|
|
except Exception as exc:
|
|
report["timing_error"] = exc.__class__.__name__
|
|
try:
|
|
report["byte_diff"] = run_byte_diff(parent or current)
|
|
except Exception as exc:
|
|
report["byte_diff"] = {"ok": False, "error": exc.__class__.__name__}
|
|
return report
|
|
|
|
|
|
def _sdk_probe() -> dict:
|
|
os.environ["JYOTISH_API_CHART_CACHE_TTL_SECONDS"] = "0"
|
|
os.environ["VEDASTRO_ENABLE_NETWORK"] = "0"
|
|
os.environ["VEDASTRO_WRITE_AUDIT_ARTIFACTS"] = "0"
|
|
handler = _handler(current)
|
|
body = _body(12, 10.0)
|
|
started = time.perf_counter()
|
|
try:
|
|
handler._compute_chart_sync(body)
|
|
error = None
|
|
except Exception as exc:
|
|
error = type(exc).__name__
|
|
return {
|
|
"blocked_vedastro": False,
|
|
"vedastro_sdk": _SDK_PRESENT,
|
|
"vedastro_network": _NETWORK,
|
|
"network_enabled": False,
|
|
"elapsed_s": round(time.perf_counter() - started, 3),
|
|
"error": error,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
payload = _sdk_probe() if "--sdk-probe" in sys.argv else main()
|
|
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2)
|
|
sys.stdout.write("\n")
|