fix(capacity): limit each user and queue heavy compute briefly.

Consult, rectification turns, and report generation stop before a credit reservation when the product caps are hit. A full heavy-compute gate waits up to 8 seconds, and one consultation retries that 429 inside the same request.

Co-Authored-By: Grok (agent loop)

scripts/api_heavy_compute_gate.py

@@ -1,9 +1,11 @@
-"""Fail-fast bounded concurrency for heavy Jyotish API compute endpoints.
+"""Bounded concurrency for heavy Jyotish API compute endpoints.
 
-Request threads that run rectification scans or high-rigor workflows share one
-process-wide semaphore sized for the 2 vCPU production host. Saturated requests
-return immediately; they are not queued. Health checks and other light routes
-must not call this gate.
+Request threads that run rectification scans or consultation workflows share one
+process-wide semaphore sized for the 2 vCPU production host. A saturated request
+waits up to 8 seconds when fewer than 4 threads are already waiting. If that
+queue is full or the wait expires, the caller still gets HTTP 429 and
+Retry-After. A thread is never parked without a deadline. Health checks and
+other light routes must not call this gate.
 """
 
 from __future__ import annotations
@@ -15,6 +17,8 @@ import time
 
 DEFAULT_CONCURRENCY = 2
 DEFAULT_RETRY_AFTER_SECONDS = 2
+DEFAULT_QUEUE_WAIT_SECONDS = 8.0
+DEFAULT_MAX_WAITERS = 4
 ENV_CONCURRENCY = "JYOTISH_HEAVY_COMPUTE_CONCURRENCY"
 ENV_RETRY_AFTER = "JYOTISH_HEAVY_COMPUTE_RETRY_AFTER_SECONDS"
 ENV_GATE_CHART_OVERVIEW = "JYOTISH_GATE_CHART_OVERVIEW"
@@ -51,6 +55,9 @@ _state_lock = threading.Lock()
 _semaphore: threading.BoundedSemaphore | None = None
 _retry_after_seconds = DEFAULT_RETRY_AFTER_SECONDS
 _limit = DEFAULT_CONCURRENCY
+_wait_seconds = DEFAULT_QUEUE_WAIT_SECONDS
+_max_waiters = DEFAULT_MAX_WAITERS
+_waiters = 0
 _holders: list[dict] = []
 _holder_seq = 0
 
@@ -73,7 +80,7 @@ class _HeldSlot:
 
 
 class HeavyComputeBusy(RuntimeError):
-    """No heavy-compute slot is free; callers must fail fast with HTTP 429."""
+    """The short queue could not take this request; callers return HTTP 429."""
 
     error_code = "ERR_COMPUTE_BUSY"
 
@@ -102,9 +109,11 @@ def reset_heavy_compute_gate(
     *,
     limit: int | None = None,
     retry_after_seconds: int | None = None,
+    wait_seconds: float | None = None,
+    max_waiters: int | None = None,
 ) -> None:
     """Rebuild the process-wide semaphore. Tests must call this after env changes."""
-    global _semaphore, _retry_after_seconds, _limit, _holders
+    global _semaphore, _retry_after_seconds, _limit, _holders, _wait_seconds, _max_waiters, _waiters
     resolved_limit = (
         limit
         if limit is not None
@@ -115,9 +124,14 @@ def reset_heavy_compute_gate(
         if retry_after_seconds is not None
         else _parse_positive_int(os.environ.get(ENV_RETRY_AFTER), DEFAULT_RETRY_AFTER_SECONDS)
     )
+    resolved_wait = DEFAULT_QUEUE_WAIT_SECONDS if wait_seconds is None else max(0.0, float(wait_seconds))
+    resolved_waiters = DEFAULT_MAX_WAITERS if max_waiters is None else max(0, int(max_waiters))
     with _state_lock:
         _retry_after_seconds = resolved_retry
         _limit = resolved_limit
+        _wait_seconds = resolved_wait
+        _max_waiters = resolved_waiters
+        _waiters = 0
         _holders = []
         _semaphore = threading.BoundedSemaphore(resolved_limit)
 
@@ -143,35 +157,61 @@ def _holder_view(now: float) -> list[dict]:
     ]
 
 
-def _acquire_recorded(path: str) -> _HeldSlot:
-    """Take one permit and remember which path holds it. Fail fast when full."""
+def _remember_locked(path: str, semaphore: threading.BoundedSemaphore) -> _HeldSlot:
+    """Record a permit that this thread already holds. Caller owns ``_state_lock``."""
     global _holder_seq
+    _holder_seq += 1
+    holder_id = _holder_seq
+    _holders.append({"id": holder_id, "path": path, "started": time.monotonic()})
+    return _HeldSlot(semaphore, holder_id)
+
+
+def _reject_busy(path: str, retry_after: int) -> None:
+    with _state_lock:
+        snapshot = _holder_view(time.monotonic())
+    _LOG.warning(
+        "heavy_compute_busy rejected_path=%s holders=%s",
+        path,
+        snapshot,
+    )
+    raise HeavyComputeBusy(retry_after)
+
+
+def _acquire_recorded(path: str) -> _HeldSlot:
+    """Take one permit. Wait briefly when full; still raise once the queue cannot."""
+    global _waiters
     with _state_lock:
         semaphore, retry_after = _ensure_locked()
-        if not semaphore.acquire(blocking=False):
-            snapshot = _holder_view(time.monotonic())
-            failed = True
-            slot = None
+        if semaphore.acquire(blocking=False):
+            return _remember_locked(path, semaphore)
+        if _waiters >= _max_waiters:
+            queued = False
         else:
-            _holder_seq += 1
-            holder_id = _holder_seq
-            _holders.append({"id": holder_id, "path": path, "started": time.monotonic()})
-            failed = False
-            snapshot = []
-            slot = _HeldSlot(semaphore, holder_id)
-    if failed:
-        _LOG.warning(
-            "heavy_compute_busy rejected_path=%s holders=%s",
-            path,
-            snapshot,
-        )
-        raise HeavyComputeBusy(retry_after)
-    assert slot is not None
-    return slot
+            _waiters += 1
+            queued = True
+        wait_seconds = _wait_seconds
+    if not queued:
+        _reject_busy(path, retry_after)
+    acquired = False
+    try:
+        acquired = bool(semaphore.acquire(blocking=True, timeout=wait_seconds))
+    finally:
+        with _state_lock:
+            _waiters -= 1
+    if not acquired:
+        _reject_busy(path, retry_after)
+    with _state_lock:
+        return _remember_locked(path, semaphore)
+
+
+def heavy_compute_waiter_count() -> int:
+    """Threads parked inside the short queue. Tests use this to avoid races."""
+    with _state_lock:
+        return _waiters
 
 
 def acquire_heavy_compute_slot(path: str) -> _HeldSlot | None:
-    """Acquire a slot for a gated path. Light paths return None. Fail-fast on saturation."""
+    """Acquire a slot for a gated path. Light paths return None."""
     if not is_heavy_compute_path(path):
         return None
     return _acquire_recorded(path)