Each model call takes the enabled key with the fewest in-flight requests (ties round-robin) and gives it back when the stream ends, fails or is aborted. A key at its concurrency limit is skipped; when all are full the request waits in arrival order for at most 15 s and then goes ahead on the least-loaded key. HTTP 429 cools a key for 30 s (Retry-After honoured, 120 s cap), 401/403 for ten minutes. A call that fails with one of these before any output is retried once on another key. The catalog hands agents a pooled Mastra model in place of the single-key config, so every existing call site is covered; providers without key rows keep using the legacy column. State is in-process. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DbBXp2Vp62Ur3C7yRWQdSq
257 lines
11 KiB
TypeScript
257 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
import {
|
|
KEY_POOL_MAX_WAIT_MS,
|
|
KEY_POOL_RATE_LIMIT_COOLDOWN_MAX_MS,
|
|
KEY_POOL_RATE_LIMIT_COOLDOWN_MS,
|
|
KEY_POOL_REJECTED_COOLDOWN_MS,
|
|
ProviderKeyPool,
|
|
providerKeyPool,
|
|
providerKeyRuntimeStatus,
|
|
resetProviderKeyPoolsForTests,
|
|
type KeyLease,
|
|
type KeyPoolClock,
|
|
type PoolKey,
|
|
} from "../src/lib/model-key-pool.ts";
|
|
|
|
/** A clock the test moves by hand; timers fire in due order when time passes them. */
|
|
function fakeClock() {
|
|
let now = 1_000_000;
|
|
let nextId = 1;
|
|
const timers = new Map<number, { at: number; callback: () => void }>();
|
|
const clock: KeyPoolClock = {
|
|
now: () => now,
|
|
setTimer(callback, ms) { const id = nextId++; timers.set(id, { at: now + ms, callback }); return id; },
|
|
clearTimer(timer) { timers.delete(timer as number); },
|
|
};
|
|
return {
|
|
clock,
|
|
pending: () => timers.size,
|
|
advance(ms: number) {
|
|
const target = now + ms;
|
|
for (;;) {
|
|
const due = [...timers.entries()].filter(([, timer]) => timer.at <= target).sort((a, b) => a[1].at - b[1].at)[0];
|
|
if (!due) break;
|
|
timers.delete(due[0]);
|
|
now = due[1].at;
|
|
due[1].callback();
|
|
}
|
|
now = target;
|
|
},
|
|
};
|
|
}
|
|
const key = (id: string, maxConcurrency: number | null = null, fingerprint = "f1"): PoolKey => ({ id, label: id.toUpperCase(), maxConcurrency, fingerprint });
|
|
const settled = () => new Promise<void>((resolve) => setImmediate(resolve));
|
|
/** Resolves to the lease, or to "pending" when the acquire is still queued. */
|
|
async function peek(promise: Promise<KeyLease>): Promise<KeyLease | "pending"> {
|
|
return Promise.race([promise, settled().then(() => "pending" as const)]);
|
|
}
|
|
|
|
test("the key with the fewest in-flight requests is picked, ties go round-robin", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
|
|
pool.configure([key("a"), key("b"), key("c")]);
|
|
const first = [await pool.acquire(), await pool.acquire(), await pool.acquire()];
|
|
assert.deepEqual(first.map((lease) => lease.keyId), ["a", "b", "c"]);
|
|
first[1]!.release();
|
|
assert.equal((await pool.acquire()).keyId, "b");
|
|
for (const lease of first) lease.release();
|
|
// a and c are idle; a was picked longer ago than c.
|
|
assert.equal((await pool.acquire()).keyId, "a");
|
|
assert.equal((await pool.acquire()).keyId, "c");
|
|
assert.deepEqual(pool.snapshot().map((status) => [status.keyId, status.inFlight]), [["a", 1], ["b", 1], ["c", 1]]);
|
|
});
|
|
|
|
test("a key at its limit is skipped; when every key is full the request waits in arrival order", async () => {
|
|
const time = fakeClock();
|
|
const pool = new ProviderKeyPool({ clock: time.clock });
|
|
pool.configure([key("a", 1), key("b", 2)]);
|
|
const held = [await pool.acquire(), await pool.acquire(), await pool.acquire()];
|
|
assert.deepEqual(held.map((lease) => lease.keyId), ["a", "b", "b"]);
|
|
const order: string[] = [];
|
|
const second = pool.acquire().then((lease) => { order.push("second"); return lease; });
|
|
const third = pool.acquire().then((lease) => { order.push("third"); return lease; });
|
|
assert.equal(await peek(second), "pending");
|
|
time.advance(4_000);
|
|
held[0]!.release();
|
|
const granted = await second;
|
|
assert.equal(granted.keyId, "a");
|
|
assert.equal(granted.waitedMs, 4_000);
|
|
assert.equal(granted.overLimit, false);
|
|
assert.equal(await peek(third), "pending");
|
|
held[1]!.release();
|
|
assert.equal((await third).keyId, "b");
|
|
assert.deepEqual(order, ["second", "third"]);
|
|
assert.equal(time.pending(), 0, "no wait timer is left behind");
|
|
});
|
|
|
|
test("after the wait limit the request goes ahead on the least-loaded key instead of failing", async () => {
|
|
const time = fakeClock();
|
|
const pool = new ProviderKeyPool({ clock: time.clock });
|
|
pool.configure([key("a", 2), key("b", 1)]);
|
|
await pool.acquire(); await pool.acquire(); await pool.acquire();
|
|
const waiting = pool.acquire();
|
|
time.advance(KEY_POOL_MAX_WAIT_MS - 1);
|
|
assert.equal(await peek(waiting), "pending");
|
|
time.advance(1);
|
|
const lease = await waiting;
|
|
assert.equal(lease.keyId, "b");
|
|
assert.equal(lease.overLimit, true);
|
|
assert.equal(lease.waitedMs, KEY_POOL_MAX_WAIT_MS);
|
|
assert.deepEqual(pool.snapshot().map((status) => status.inFlight), [2, 2]);
|
|
});
|
|
|
|
test("a full queue does not grow: the next request goes ahead at once", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock, maxQueue: 1 });
|
|
pool.configure([key("a", 1)]);
|
|
await pool.acquire();
|
|
const queued = pool.acquire();
|
|
const overflow = await pool.acquire();
|
|
assert.equal(overflow.overLimit, true);
|
|
assert.equal(await peek(queued), "pending");
|
|
});
|
|
|
|
test("an abort while waiting leaves the queue and never takes a slot", async () => {
|
|
const time = fakeClock();
|
|
const pool = new ProviderKeyPool({ clock: time.clock });
|
|
pool.configure([key("a", 1)]);
|
|
const held = await pool.acquire();
|
|
const controller = new AbortController();
|
|
const waiting = pool.acquire({ signal: controller.signal });
|
|
const behind = pool.acquire();
|
|
controller.abort(new Error("client left"));
|
|
await assert.rejects(waiting, /client left/);
|
|
held.release();
|
|
assert.equal((await behind).keyId, "a");
|
|
assert.equal(pool.snapshot()[0]!.inFlight, 1);
|
|
assert.equal(time.pending(), 0);
|
|
await assert.rejects(pool.acquire({ signal: controller.signal }), /client left/);
|
|
});
|
|
|
|
test("release is idempotent and counts never go below zero", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
|
|
pool.configure([key("a", 1)]);
|
|
const lease = await pool.acquire();
|
|
lease.release();
|
|
lease.release();
|
|
assert.equal(pool.snapshot()[0]!.inFlight, 0);
|
|
const again = await pool.acquire();
|
|
assert.equal(again.overLimit, false);
|
|
});
|
|
|
|
test("429 cools a key for 30 s, honours Retry-After and caps it at 120 s", async () => {
|
|
const time = fakeClock();
|
|
const pool = new ProviderKeyPool({ clock: time.clock });
|
|
pool.configure([key("a"), key("b"), key("c")]);
|
|
const start = time.clock.now();
|
|
pool.report("a", { status: 429 });
|
|
pool.report("b", { status: 429, retryAfterMs: 5_000 });
|
|
pool.report("c", { status: 429, retryAfterMs: 900_000 });
|
|
assert.deepEqual(pool.snapshot().map((status) => status.cooldownUntil! - start), [
|
|
KEY_POOL_RATE_LIMIT_COOLDOWN_MS, 5_000, KEY_POOL_RATE_LIMIT_COOLDOWN_MAX_MS,
|
|
]);
|
|
assert.deepEqual(pool.snapshot().map((status) => status.lastRejectedAt), [null, null, null]);
|
|
time.advance(5_000);
|
|
// Only b is out of cooldown, so it takes every request.
|
|
assert.deepEqual([(await pool.acquire()).keyId, (await pool.acquire()).keyId], ["b", "b"]);
|
|
time.advance(25_000);
|
|
assert.equal((await pool.acquire()).keyId, "a");
|
|
assert.equal(pool.snapshot()[0]!.cooldownUntil, null);
|
|
});
|
|
|
|
test("401 and 403 cool a key for ten minutes and are shown until a later success", async () => {
|
|
const time = fakeClock();
|
|
const pool = new ProviderKeyPool({ clock: time.clock });
|
|
pool.configure([key("a"), key("b")]);
|
|
const at = time.clock.now();
|
|
pool.report("a", { status: 401 });
|
|
const [a] = pool.snapshot();
|
|
assert.equal(a!.lastRejectedAt, at);
|
|
assert.equal(a!.lastRejectedStatus, 401);
|
|
assert.equal(a!.cooldownUntil, at + KEY_POOL_REJECTED_COOLDOWN_MS);
|
|
assert.equal(pool.hasAlternative(new Set(["b"])), false);
|
|
assert.equal(pool.hasAlternative(new Set(["a"])), true);
|
|
for (let index = 0; index < 3; index += 1) assert.equal((await pool.acquire()).keyId, "b");
|
|
time.advance(KEY_POOL_REJECTED_COOLDOWN_MS);
|
|
assert.equal(pool.snapshot()[0]!.cooldownUntil, null);
|
|
assert.equal(pool.snapshot()[0]!.lastRejectedAt, at, "still shown after the cooldown");
|
|
pool.report("a", { status: 200 });
|
|
assert.equal(pool.snapshot()[0]!.lastRejectedAt, null);
|
|
pool.report("b", { status: 500 });
|
|
assert.equal(pool.snapshot()[1]!.cooldownUntil, null, "other errors do not cool a key");
|
|
});
|
|
|
|
test("when every key is cooling down requests still go out, least-loaded first", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
|
|
pool.configure([key("a"), key("b")]);
|
|
pool.report("a", { status: 429 });
|
|
pool.report("b", { status: 403 });
|
|
assert.deepEqual([(await pool.acquire()).keyId, (await pool.acquire()).keyId], ["a", "b"]);
|
|
});
|
|
|
|
test("a waiter is served when a cooldown ends, without waiting for a release", async () => {
|
|
const time = fakeClock();
|
|
const pool = new ProviderKeyPool({ clock: time.clock });
|
|
pool.configure([key("a", 1), key("b", 1)]);
|
|
await pool.acquire();
|
|
pool.report("b", { status: 429, retryAfterMs: 2_000 });
|
|
const waiting = pool.acquire();
|
|
assert.equal(await peek(waiting), "pending");
|
|
time.advance(2_000);
|
|
const lease = await waiting;
|
|
assert.equal(lease.keyId, "b");
|
|
assert.equal(lease.overLimit, false);
|
|
});
|
|
|
|
test("exclude skips a key that already failed this request", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
|
|
pool.configure([key("a"), key("b")]);
|
|
assert.equal((await pool.acquire({ exclude: new Set(["a"]) })).keyId, "b");
|
|
assert.equal((await pool.acquire({ exclude: new Set(["a"]) })).keyId, "b");
|
|
// Excluding everything still yields a key rather than an error.
|
|
assert.ok((await pool.acquire({ exclude: new Set(["a", "b"]) })).keyId);
|
|
});
|
|
|
|
test("reconfiguring keeps counters of keys that stay, clears a replaced key and drops removed ones", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
|
|
pool.configure([key("a", 1), key("b", 1)]);
|
|
const onA = await pool.acquire();
|
|
const onB = await pool.acquire();
|
|
pool.report("a", { status: 401 });
|
|
pool.configure([key("a", 1), key("c", 1)]);
|
|
assert.deepEqual(pool.snapshot().map((status) => [status.keyId, status.inFlight]), [["a", 1], ["c", 0]]);
|
|
assert.ok(pool.snapshot()[0]!.lastRejectedAt, "same stored key: the rejection stays");
|
|
onB.release();
|
|
assert.deepEqual(pool.snapshot().map((status) => status.inFlight), [1, 0], "a removed key's release touches nothing");
|
|
pool.configure([key("a", 3, "f2"), key("c", 1)]);
|
|
assert.equal(pool.snapshot()[0]!.lastRejectedAt, null, "replaced key starts clean");
|
|
assert.equal(pool.snapshot()[0]!.cooldownUntil, null);
|
|
onA.release();
|
|
assert.equal(pool.snapshot()[0]!.inFlight, 0);
|
|
});
|
|
|
|
test("raising a limit serves the queue; an empty pool rejects", async () => {
|
|
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
|
|
await assert.rejects(pool.acquire(), /model_provider_key_pool_empty/);
|
|
pool.configure([key("a", 1)]);
|
|
await pool.acquire();
|
|
const waiting = pool.acquire();
|
|
assert.equal(await peek(waiting), "pending");
|
|
pool.configure([key("a", 2)]);
|
|
assert.equal((await waiting).keyId, "a");
|
|
});
|
|
|
|
test("the registry keeps one pool per provider and reports status without key material", async () => {
|
|
resetProviderKeyPoolsForTests();
|
|
assert.deepEqual(providerKeyRuntimeStatus("p1"), []);
|
|
const pool = providerKeyPool("p1");
|
|
assert.equal(providerKeyPool("p1"), pool);
|
|
assert.notEqual(providerKeyPool("p2"), pool);
|
|
pool.configure([key("a")]);
|
|
await pool.acquire();
|
|
const status = providerKeyRuntimeStatus("p1");
|
|
assert.deepEqual(Object.keys(status[0]!).sort(), ["cooldownUntil", "inFlight", "keyId", "label", "lastRejectedAt", "lastRejectedStatus"]);
|
|
assert.equal(status[0]!.inFlight, 1);
|
|
resetProviderKeyPoolsForTests();
|
|
});
|