fix(capacity): limit each user and queue heavy compute briefly.

Consult, rectification turns, and report generation stop before a credit reservation when the product caps are hit. A full heavy-compute gate waits up to 8 seconds, and one consultation retries that 429 inside the same request.

Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
2026-10-15 13:33:06 +08:00
co-authored by Grok (agent loop)
parent ef49257b73
commit a54f75e5e3
28 changed files with 1035 additions and 70 deletions
+8 -2
View File
@@ -153,7 +153,10 @@ def test_chart_overview_gate_is_off_by_default(monkeypatch) -> None:
def test_chart_overview_gate_only_sync_unskipped_requests(monkeypatch) -> None:
monkeypatch.setenv(gate.ENV_GATE_CHART_OVERVIEW, "1")
gate.reset_heavy_compute_gate(limit=1)
# 原值:reset(limit=1) 后第二次立刻 HeavyComputeBusy
# 新值:wait_seconds=0
# 原因:默认改为短排队;本测试仍锁「名额被占时同步排盘被拒绝」,等待行为不在这里
gate.reset_heavy_compute_gate(limit=1, wait_seconds=0)
body = {"year": 1990, "lat": 28.6, "lon": 77.2}
slot = gate.acquire_heavy_compute_slot_for_request("/api/chart", body)
assert slot is not None
@@ -172,7 +175,10 @@ def test_chart_overview_gate_only_sync_unskipped_requests(monkeypatch) -> None:
def test_busy_log_names_holders_and_not_the_body(caplog) -> None:
gate.reset_heavy_compute_gate(limit=1)
# 原值:reset(limit=1) 后拒绝日志立刻出现
# 新值:wait_seconds=0
# 原因:默认会先等 8 秒;本测试仍锁日志只有路径、没有请求正文
gate.reset_heavy_compute_gate(limit=1, wait_seconds=0)
held = gate.acquire_heavy_compute_slot("/api/professional_report_reference")
assert held is not None
secret_body = {"birth_date": "1990-01-01", "name": "must-not-appear"}
+89 -2
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
import threading
import time
from io import BytesIO
import pytest
@@ -14,6 +15,7 @@ from scripts.api_heavy_compute_gate import (
HEAVY_COMPUTE_PATHS,
HeavyComputeBusy,
acquire_heavy_compute_slot,
heavy_compute_waiter_count,
is_heavy_compute_path,
release_heavy_compute_slot,
reset_heavy_compute_gate,
@@ -93,7 +95,10 @@ def limit_one_gate(monkeypatch):
monkeypatch.setenv("JYOTISH_HEAVY_COMPUTE_CONCURRENCY", "1")
monkeypatch.setenv("JYOTISH_HEAVY_COMPUTE_RETRY_AFTER_SECONDS", "2")
monkeypatch.setenv("JYOTISH_API_RATE_LIMIT_PER_MINUTE", "0")
reset_heavy_compute_gate()
# 原值:reset() 使用默认「满了立刻拒绝」
# 新值:wait_seconds=0,本夹具里的 429 断言仍是立刻拒绝
# 原因:产品改为最多等 8 秒;排队成功、等待者满、超时另有测试
reset_heavy_compute_gate(wait_seconds=0)
yield
monkeypatch.delenv("JYOTISH_HEAVY_COMPUTE_CONCURRENCY", raising=False)
monkeypatch.delenv("JYOTISH_HEAVY_COMPUTE_RETRY_AFTER_SECONDS", raising=False)
@@ -119,7 +124,10 @@ def test_health_and_light_paths_are_not_gated() -> None:
def test_acquire_fail_fast_then_succeeds_after_release() -> None:
reset_heavy_compute_gate(limit=1, retry_after_seconds=3)
# 原值:reset(limit=1, retry_after_seconds=3) 后第二路立刻 HeavyComputeBusy
# 新值:多传 wait_seconds=0
# 原因:默认改为短排队;本测试仍锁 Retry-After=3 与 ERR_COMPUTE_BUSY,不锁「禁止等待」
reset_heavy_compute_gate(limit=1, retry_after_seconds=3, wait_seconds=0)
first = acquire_heavy_compute_slot("/api/high_rigor_workflow")
assert first is not None
with pytest.raises(HeavyComputeBusy) as caught:
@@ -248,8 +256,87 @@ def test_warm_consultation_does_not_consume_the_user_slot(limit_one_gate, monkey
assert "packet" not in warm_box["payload"]
def _wait_until(predicate, timeout: float = 2.0) -> bool:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return True
time.sleep(0.01)
return False
def test_third_request_gets_the_slot_after_one_holder_releases() -> None:
reset_heavy_compute_gate(limit=2, retry_after_seconds=2, wait_seconds=2, max_waiters=4)
first = acquire_heavy_compute_slot("/api/consultation_workflow")
second = acquire_heavy_compute_slot("/api/rectification/v5/score")
assert first is not None and second is not None
found: dict = {}
def _third() -> None:
found["slot"] = acquire_heavy_compute_slot("/api/consultation_workflow")
worker = threading.Thread(target=_third)
worker.start()
assert _wait_until(lambda: heavy_compute_waiter_count() == 1)
release_heavy_compute_slot(first)
worker.join(timeout=3)
assert not worker.is_alive()
assert found["slot"] is not None
release_heavy_compute_slot(second)
release_heavy_compute_slot(found["slot"])
reset_heavy_compute_gate()
def test_waiter_overflow_rejects_immediately() -> None:
reset_heavy_compute_gate(limit=1, retry_after_seconds=2, wait_seconds=2, max_waiters=1)
held = acquire_heavy_compute_slot("/api/high_rigor_workflow")
assert held is not None
parked: dict = {}
def _wait() -> None:
try:
parked["slot"] = acquire_heavy_compute_slot("/api/consultation_workflow")
except HeavyComputeBusy as exc:
parked["error"] = exc
waiter = threading.Thread(target=_wait)
waiter.start()
assert _wait_until(lambda: heavy_compute_waiter_count() == 1)
started = time.monotonic()
with pytest.raises(HeavyComputeBusy) as caught:
acquire_heavy_compute_slot("/api/rectification/v5/score")
assert time.monotonic() - started < 0.3
assert caught.value.retry_after_seconds == 2
assert caught.value.error_code == "ERR_COMPUTE_BUSY"
release_heavy_compute_slot(held)
waiter.join(timeout=3)
assert not waiter.is_alive()
assert parked.get("slot") is not None
release_heavy_compute_slot(parked["slot"])
reset_heavy_compute_gate()
def test_queue_timeout_still_returns_busy() -> None:
reset_heavy_compute_gate(limit=1, retry_after_seconds=2, wait_seconds=0.2, max_waiters=4)
held = acquire_heavy_compute_slot("/api/professional_report_reference")
assert held is not None
started = time.monotonic()
with pytest.raises(HeavyComputeBusy) as caught:
acquire_heavy_compute_slot("/api/consultation_workflow")
elapsed = time.monotonic() - started
assert 0.1 <= elapsed < 2.0
assert caught.value.retry_after_seconds == 2
assert caught.value.error_code == "ERR_COMPUTE_BUSY"
light = acquire_heavy_compute_slot("/api/health")
assert light is None
release_heavy_compute_slot(held)
reset_heavy_compute_gate()
def test_quality_gate_runs_heavy_compute_gate() -> None:
from scripts.run_quality_gate import CORE_PYTEST_TARGETS
assert "tests/test_api_heavy_compute_gate.py" in CORE_PYTEST_TARGETS
assert gate.DEFAULT_CONCURRENCY == 2
assert gate.DEFAULT_QUEUE_WAIT_SECONDS == 8
assert gate.DEFAULT_MAX_WAITERS == 4