fix(capacity): limit each user and queue heavy compute briefly.
Consult, rectification turns, and report generation stop before a credit reservation when the product caps are hit. A full heavy-compute gate waits up to 8 seconds, and one consultation retries that 429 inside the same request. Co-Authored-By: Grok (agent loop)
This commit is contained in:
co-authored by
Grok (agent loop)
parent
ef49257b73
commit
a54f75e5e3
@@ -153,7 +153,10 @@ def test_chart_overview_gate_is_off_by_default(monkeypatch) -> None:
|
||||
|
||||
def test_chart_overview_gate_only_sync_unskipped_requests(monkeypatch) -> None:
|
||||
monkeypatch.setenv(gate.ENV_GATE_CHART_OVERVIEW, "1")
|
||||
gate.reset_heavy_compute_gate(limit=1)
|
||||
# 原值:reset(limit=1) 后第二次立刻 HeavyComputeBusy
|
||||
# 新值:wait_seconds=0
|
||||
# 原因:默认改为短排队;本测试仍锁「名额被占时同步排盘被拒绝」,等待行为不在这里
|
||||
gate.reset_heavy_compute_gate(limit=1, wait_seconds=0)
|
||||
body = {"year": 1990, "lat": 28.6, "lon": 77.2}
|
||||
slot = gate.acquire_heavy_compute_slot_for_request("/api/chart", body)
|
||||
assert slot is not None
|
||||
@@ -172,7 +175,10 @@ def test_chart_overview_gate_only_sync_unskipped_requests(monkeypatch) -> None:
|
||||
|
||||
|
||||
def test_busy_log_names_holders_and_not_the_body(caplog) -> None:
|
||||
gate.reset_heavy_compute_gate(limit=1)
|
||||
# 原值:reset(limit=1) 后拒绝日志立刻出现
|
||||
# 新值:wait_seconds=0
|
||||
# 原因:默认会先等 8 秒;本测试仍锁日志只有路径、没有请求正文
|
||||
gate.reset_heavy_compute_gate(limit=1, wait_seconds=0)
|
||||
held = gate.acquire_heavy_compute_slot("/api/professional_report_reference")
|
||||
assert held is not None
|
||||
secret_body = {"birth_date": "1990-01-01", "name": "must-not-appear"}
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
@@ -14,6 +15,7 @@ from scripts.api_heavy_compute_gate import (
|
||||
HEAVY_COMPUTE_PATHS,
|
||||
HeavyComputeBusy,
|
||||
acquire_heavy_compute_slot,
|
||||
heavy_compute_waiter_count,
|
||||
is_heavy_compute_path,
|
||||
release_heavy_compute_slot,
|
||||
reset_heavy_compute_gate,
|
||||
@@ -93,7 +95,10 @@ def limit_one_gate(monkeypatch):
|
||||
monkeypatch.setenv("JYOTISH_HEAVY_COMPUTE_CONCURRENCY", "1")
|
||||
monkeypatch.setenv("JYOTISH_HEAVY_COMPUTE_RETRY_AFTER_SECONDS", "2")
|
||||
monkeypatch.setenv("JYOTISH_API_RATE_LIMIT_PER_MINUTE", "0")
|
||||
reset_heavy_compute_gate()
|
||||
# 原值:reset() 使用默认「满了立刻拒绝」
|
||||
# 新值:wait_seconds=0,本夹具里的 429 断言仍是立刻拒绝
|
||||
# 原因:产品改为最多等 8 秒;排队成功、等待者满、超时另有测试
|
||||
reset_heavy_compute_gate(wait_seconds=0)
|
||||
yield
|
||||
monkeypatch.delenv("JYOTISH_HEAVY_COMPUTE_CONCURRENCY", raising=False)
|
||||
monkeypatch.delenv("JYOTISH_HEAVY_COMPUTE_RETRY_AFTER_SECONDS", raising=False)
|
||||
@@ -119,7 +124,10 @@ def test_health_and_light_paths_are_not_gated() -> None:
|
||||
|
||||
|
||||
def test_acquire_fail_fast_then_succeeds_after_release() -> None:
|
||||
reset_heavy_compute_gate(limit=1, retry_after_seconds=3)
|
||||
# 原值:reset(limit=1, retry_after_seconds=3) 后第二路立刻 HeavyComputeBusy
|
||||
# 新值:多传 wait_seconds=0
|
||||
# 原因:默认改为短排队;本测试仍锁 Retry-After=3 与 ERR_COMPUTE_BUSY,不锁「禁止等待」
|
||||
reset_heavy_compute_gate(limit=1, retry_after_seconds=3, wait_seconds=0)
|
||||
first = acquire_heavy_compute_slot("/api/high_rigor_workflow")
|
||||
assert first is not None
|
||||
with pytest.raises(HeavyComputeBusy) as caught:
|
||||
@@ -248,8 +256,87 @@ def test_warm_consultation_does_not_consume_the_user_slot(limit_one_gate, monkey
|
||||
assert "packet" not in warm_box["payload"]
|
||||
|
||||
|
||||
def _wait_until(predicate, timeout: float = 2.0) -> bool:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
def test_third_request_gets_the_slot_after_one_holder_releases() -> None:
|
||||
reset_heavy_compute_gate(limit=2, retry_after_seconds=2, wait_seconds=2, max_waiters=4)
|
||||
first = acquire_heavy_compute_slot("/api/consultation_workflow")
|
||||
second = acquire_heavy_compute_slot("/api/rectification/v5/score")
|
||||
assert first is not None and second is not None
|
||||
found: dict = {}
|
||||
|
||||
def _third() -> None:
|
||||
found["slot"] = acquire_heavy_compute_slot("/api/consultation_workflow")
|
||||
|
||||
worker = threading.Thread(target=_third)
|
||||
worker.start()
|
||||
assert _wait_until(lambda: heavy_compute_waiter_count() == 1)
|
||||
release_heavy_compute_slot(first)
|
||||
worker.join(timeout=3)
|
||||
assert not worker.is_alive()
|
||||
assert found["slot"] is not None
|
||||
release_heavy_compute_slot(second)
|
||||
release_heavy_compute_slot(found["slot"])
|
||||
reset_heavy_compute_gate()
|
||||
|
||||
|
||||
def test_waiter_overflow_rejects_immediately() -> None:
|
||||
reset_heavy_compute_gate(limit=1, retry_after_seconds=2, wait_seconds=2, max_waiters=1)
|
||||
held = acquire_heavy_compute_slot("/api/high_rigor_workflow")
|
||||
assert held is not None
|
||||
parked: dict = {}
|
||||
|
||||
def _wait() -> None:
|
||||
try:
|
||||
parked["slot"] = acquire_heavy_compute_slot("/api/consultation_workflow")
|
||||
except HeavyComputeBusy as exc:
|
||||
parked["error"] = exc
|
||||
|
||||
waiter = threading.Thread(target=_wait)
|
||||
waiter.start()
|
||||
assert _wait_until(lambda: heavy_compute_waiter_count() == 1)
|
||||
started = time.monotonic()
|
||||
with pytest.raises(HeavyComputeBusy) as caught:
|
||||
acquire_heavy_compute_slot("/api/rectification/v5/score")
|
||||
assert time.monotonic() - started < 0.3
|
||||
assert caught.value.retry_after_seconds == 2
|
||||
assert caught.value.error_code == "ERR_COMPUTE_BUSY"
|
||||
release_heavy_compute_slot(held)
|
||||
waiter.join(timeout=3)
|
||||
assert not waiter.is_alive()
|
||||
assert parked.get("slot") is not None
|
||||
release_heavy_compute_slot(parked["slot"])
|
||||
reset_heavy_compute_gate()
|
||||
|
||||
|
||||
def test_queue_timeout_still_returns_busy() -> None:
|
||||
reset_heavy_compute_gate(limit=1, retry_after_seconds=2, wait_seconds=0.2, max_waiters=4)
|
||||
held = acquire_heavy_compute_slot("/api/professional_report_reference")
|
||||
assert held is not None
|
||||
started = time.monotonic()
|
||||
with pytest.raises(HeavyComputeBusy) as caught:
|
||||
acquire_heavy_compute_slot("/api/consultation_workflow")
|
||||
elapsed = time.monotonic() - started
|
||||
assert 0.1 <= elapsed < 2.0
|
||||
assert caught.value.retry_after_seconds == 2
|
||||
assert caught.value.error_code == "ERR_COMPUTE_BUSY"
|
||||
light = acquire_heavy_compute_slot("/api/health")
|
||||
assert light is None
|
||||
release_heavy_compute_slot(held)
|
||||
reset_heavy_compute_gate()
|
||||
|
||||
|
||||
def test_quality_gate_runs_heavy_compute_gate() -> None:
|
||||
from scripts.run_quality_gate import CORE_PYTEST_TARGETS
|
||||
|
||||
assert "tests/test_api_heavy_compute_gate.py" in CORE_PYTEST_TARGETS
|
||||
assert gate.DEFAULT_CONCURRENCY == 2
|
||||
assert gate.DEFAULT_QUEUE_WAIT_SECONDS == 8
|
||||
assert gate.DEFAULT_MAX_WAITERS == 4
|
||||
|
||||
Reference in New Issue
Block a user