Files
Jyotisha/frontend/tests/model-key-pool.test.ts
ClaudeandClaude Fable 5.2 ceb964fda7 feat: spread model calls across a provider's API keys
Each model call takes the enabled key with the fewest in-flight
requests (ties round-robin) and gives it back when the stream ends,
fails or is aborted. A key at its concurrency limit is skipped; when all
are full the request waits in arrival order for at most 15 s and then
goes ahead on the least-loaded key. HTTP 429 cools a key for 30 s
(Retry-After honoured, 120 s cap), 401/403 for ten minutes. A call that
fails with one of these before any output is retried once on another
key. The catalog hands agents a pooled Mastra model in place of the
single-key config, so every existing call site is covered; providers
without key rows keep using the legacy column. State is in-process.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DbBXp2Vp62Ur3C7yRWQdSq
2026-11-16 10:19:03 +08:00

257 lines
11 KiB
TypeScript

import assert from "node:assert/strict";
import test from "node:test";
import {
KEY_POOL_MAX_WAIT_MS,
KEY_POOL_RATE_LIMIT_COOLDOWN_MAX_MS,
KEY_POOL_RATE_LIMIT_COOLDOWN_MS,
KEY_POOL_REJECTED_COOLDOWN_MS,
ProviderKeyPool,
providerKeyPool,
providerKeyRuntimeStatus,
resetProviderKeyPoolsForTests,
type KeyLease,
type KeyPoolClock,
type PoolKey,
} from "../src/lib/model-key-pool.ts";
/** A clock the test moves by hand; timers fire in due order when time passes them. */
function fakeClock() {
let now = 1_000_000;
let nextId = 1;
const timers = new Map<number, { at: number; callback: () => void }>();
const clock: KeyPoolClock = {
now: () => now,
setTimer(callback, ms) { const id = nextId++; timers.set(id, { at: now + ms, callback }); return id; },
clearTimer(timer) { timers.delete(timer as number); },
};
return {
clock,
pending: () => timers.size,
advance(ms: number) {
const target = now + ms;
for (;;) {
const due = [...timers.entries()].filter(([, timer]) => timer.at <= target).sort((a, b) => a[1].at - b[1].at)[0];
if (!due) break;
timers.delete(due[0]);
now = due[1].at;
due[1].callback();
}
now = target;
},
};
}
const key = (id: string, maxConcurrency: number | null = null, fingerprint = "f1"): PoolKey => ({ id, label: id.toUpperCase(), maxConcurrency, fingerprint });
const settled = () => new Promise<void>((resolve) => setImmediate(resolve));
/** Resolves to the lease, or to "pending" when the acquire is still queued. */
async function peek(promise: Promise<KeyLease>): Promise<KeyLease | "pending"> {
return Promise.race([promise, settled().then(() => "pending" as const)]);
}
test("the key with the fewest in-flight requests is picked, ties go round-robin", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
pool.configure([key("a"), key("b"), key("c")]);
const first = [await pool.acquire(), await pool.acquire(), await pool.acquire()];
assert.deepEqual(first.map((lease) => lease.keyId), ["a", "b", "c"]);
first[1]!.release();
assert.equal((await pool.acquire()).keyId, "b");
for (const lease of first) lease.release();
// a and c are idle; a was picked longer ago than c.
assert.equal((await pool.acquire()).keyId, "a");
assert.equal((await pool.acquire()).keyId, "c");
assert.deepEqual(pool.snapshot().map((status) => [status.keyId, status.inFlight]), [["a", 1], ["b", 1], ["c", 1]]);
});
test("a key at its limit is skipped; when every key is full the request waits in arrival order", async () => {
const time = fakeClock();
const pool = new ProviderKeyPool({ clock: time.clock });
pool.configure([key("a", 1), key("b", 2)]);
const held = [await pool.acquire(), await pool.acquire(), await pool.acquire()];
assert.deepEqual(held.map((lease) => lease.keyId), ["a", "b", "b"]);
const order: string[] = [];
const second = pool.acquire().then((lease) => { order.push("second"); return lease; });
const third = pool.acquire().then((lease) => { order.push("third"); return lease; });
assert.equal(await peek(second), "pending");
time.advance(4_000);
held[0]!.release();
const granted = await second;
assert.equal(granted.keyId, "a");
assert.equal(granted.waitedMs, 4_000);
assert.equal(granted.overLimit, false);
assert.equal(await peek(third), "pending");
held[1]!.release();
assert.equal((await third).keyId, "b");
assert.deepEqual(order, ["second", "third"]);
assert.equal(time.pending(), 0, "no wait timer is left behind");
});
test("after the wait limit the request goes ahead on the least-loaded key instead of failing", async () => {
const time = fakeClock();
const pool = new ProviderKeyPool({ clock: time.clock });
pool.configure([key("a", 2), key("b", 1)]);
await pool.acquire(); await pool.acquire(); await pool.acquire();
const waiting = pool.acquire();
time.advance(KEY_POOL_MAX_WAIT_MS - 1);
assert.equal(await peek(waiting), "pending");
time.advance(1);
const lease = await waiting;
assert.equal(lease.keyId, "b");
assert.equal(lease.overLimit, true);
assert.equal(lease.waitedMs, KEY_POOL_MAX_WAIT_MS);
assert.deepEqual(pool.snapshot().map((status) => status.inFlight), [2, 2]);
});
test("a full queue does not grow: the next request goes ahead at once", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock, maxQueue: 1 });
pool.configure([key("a", 1)]);
await pool.acquire();
const queued = pool.acquire();
const overflow = await pool.acquire();
assert.equal(overflow.overLimit, true);
assert.equal(await peek(queued), "pending");
});
test("an abort while waiting leaves the queue and never takes a slot", async () => {
const time = fakeClock();
const pool = new ProviderKeyPool({ clock: time.clock });
pool.configure([key("a", 1)]);
const held = await pool.acquire();
const controller = new AbortController();
const waiting = pool.acquire({ signal: controller.signal });
const behind = pool.acquire();
controller.abort(new Error("client left"));
await assert.rejects(waiting, /client left/);
held.release();
assert.equal((await behind).keyId, "a");
assert.equal(pool.snapshot()[0]!.inFlight, 1);
assert.equal(time.pending(), 0);
await assert.rejects(pool.acquire({ signal: controller.signal }), /client left/);
});
test("release is idempotent and counts never go below zero", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
pool.configure([key("a", 1)]);
const lease = await pool.acquire();
lease.release();
lease.release();
assert.equal(pool.snapshot()[0]!.inFlight, 0);
const again = await pool.acquire();
assert.equal(again.overLimit, false);
});
test("429 cools a key for 30 s, honours Retry-After and caps it at 120 s", async () => {
const time = fakeClock();
const pool = new ProviderKeyPool({ clock: time.clock });
pool.configure([key("a"), key("b"), key("c")]);
const start = time.clock.now();
pool.report("a", { status: 429 });
pool.report("b", { status: 429, retryAfterMs: 5_000 });
pool.report("c", { status: 429, retryAfterMs: 900_000 });
assert.deepEqual(pool.snapshot().map((status) => status.cooldownUntil! - start), [
KEY_POOL_RATE_LIMIT_COOLDOWN_MS, 5_000, KEY_POOL_RATE_LIMIT_COOLDOWN_MAX_MS,
]);
assert.deepEqual(pool.snapshot().map((status) => status.lastRejectedAt), [null, null, null]);
time.advance(5_000);
// Only b is out of cooldown, so it takes every request.
assert.deepEqual([(await pool.acquire()).keyId, (await pool.acquire()).keyId], ["b", "b"]);
time.advance(25_000);
assert.equal((await pool.acquire()).keyId, "a");
assert.equal(pool.snapshot()[0]!.cooldownUntil, null);
});
test("401 and 403 cool a key for ten minutes and are shown until a later success", async () => {
const time = fakeClock();
const pool = new ProviderKeyPool({ clock: time.clock });
pool.configure([key("a"), key("b")]);
const at = time.clock.now();
pool.report("a", { status: 401 });
const [a] = pool.snapshot();
assert.equal(a!.lastRejectedAt, at);
assert.equal(a!.lastRejectedStatus, 401);
assert.equal(a!.cooldownUntil, at + KEY_POOL_REJECTED_COOLDOWN_MS);
assert.equal(pool.hasAlternative(new Set(["b"])), false);
assert.equal(pool.hasAlternative(new Set(["a"])), true);
for (let index = 0; index < 3; index += 1) assert.equal((await pool.acquire()).keyId, "b");
time.advance(KEY_POOL_REJECTED_COOLDOWN_MS);
assert.equal(pool.snapshot()[0]!.cooldownUntil, null);
assert.equal(pool.snapshot()[0]!.lastRejectedAt, at, "still shown after the cooldown");
pool.report("a", { status: 200 });
assert.equal(pool.snapshot()[0]!.lastRejectedAt, null);
pool.report("b", { status: 500 });
assert.equal(pool.snapshot()[1]!.cooldownUntil, null, "other errors do not cool a key");
});
test("when every key is cooling down requests still go out, least-loaded first", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
pool.configure([key("a"), key("b")]);
pool.report("a", { status: 429 });
pool.report("b", { status: 403 });
assert.deepEqual([(await pool.acquire()).keyId, (await pool.acquire()).keyId], ["a", "b"]);
});
test("a waiter is served when a cooldown ends, without waiting for a release", async () => {
const time = fakeClock();
const pool = new ProviderKeyPool({ clock: time.clock });
pool.configure([key("a", 1), key("b", 1)]);
await pool.acquire();
pool.report("b", { status: 429, retryAfterMs: 2_000 });
const waiting = pool.acquire();
assert.equal(await peek(waiting), "pending");
time.advance(2_000);
const lease = await waiting;
assert.equal(lease.keyId, "b");
assert.equal(lease.overLimit, false);
});
test("exclude skips a key that already failed this request", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
pool.configure([key("a"), key("b")]);
assert.equal((await pool.acquire({ exclude: new Set(["a"]) })).keyId, "b");
assert.equal((await pool.acquire({ exclude: new Set(["a"]) })).keyId, "b");
// Excluding everything still yields a key rather than an error.
assert.ok((await pool.acquire({ exclude: new Set(["a", "b"]) })).keyId);
});
test("reconfiguring keeps counters of keys that stay, clears a replaced key and drops removed ones", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
pool.configure([key("a", 1), key("b", 1)]);
const onA = await pool.acquire();
const onB = await pool.acquire();
pool.report("a", { status: 401 });
pool.configure([key("a", 1), key("c", 1)]);
assert.deepEqual(pool.snapshot().map((status) => [status.keyId, status.inFlight]), [["a", 1], ["c", 0]]);
assert.ok(pool.snapshot()[0]!.lastRejectedAt, "same stored key: the rejection stays");
onB.release();
assert.deepEqual(pool.snapshot().map((status) => status.inFlight), [1, 0], "a removed key's release touches nothing");
pool.configure([key("a", 3, "f2"), key("c", 1)]);
assert.equal(pool.snapshot()[0]!.lastRejectedAt, null, "replaced key starts clean");
assert.equal(pool.snapshot()[0]!.cooldownUntil, null);
onA.release();
assert.equal(pool.snapshot()[0]!.inFlight, 0);
});
test("raising a limit serves the queue; an empty pool rejects", async () => {
const pool = new ProviderKeyPool({ clock: fakeClock().clock });
await assert.rejects(pool.acquire(), /model_provider_key_pool_empty/);
pool.configure([key("a", 1)]);
await pool.acquire();
const waiting = pool.acquire();
assert.equal(await peek(waiting), "pending");
pool.configure([key("a", 2)]);
assert.equal((await waiting).keyId, "a");
});
test("the registry keeps one pool per provider and reports status without key material", async () => {
resetProviderKeyPoolsForTests();
assert.deepEqual(providerKeyRuntimeStatus("p1"), []);
const pool = providerKeyPool("p1");
assert.equal(providerKeyPool("p1"), pool);
assert.notEqual(providerKeyPool("p2"), pool);
pool.configure([key("a")]);
await pool.acquire();
const status = providerKeyRuntimeStatus("p1");
assert.deepEqual(Object.keys(status[0]!).sort(), ["cooldownUntil", "inFlight", "keyId", "label", "lastRejectedAt", "lastRejectedStatus"]);
assert.equal(status[0]!.inFlight, 1);
resetProviderKeyPoolsForTests();
});