Files
Jyotisha/frontend/tests/pooled-language-model.test.ts
ClaudeandClaude Fable 5.2 ceb964fda7 feat: spread model calls across a provider's API keys
Each model call takes the enabled key with the fewest in-flight
requests (ties round-robin) and gives it back when the stream ends,
fails or is aborted. A key at its concurrency limit is skipped; when all
are full the request waits in arrival order for at most 15 s and then
goes ahead on the least-loaded key. HTTP 429 cools a key for 30 s
(Retry-After honoured, 120 s cap), 401/403 for ten minutes. A call that
fails with one of these before any output is retried once on another
key. The catalog hands agents a pooled Mastra model in place of the
single-key config, so every existing call site is covered; providers
without key rows keep using the legacy column. State is in-process.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DbBXp2Vp62Ur3C7yRWQdSq
2026-11-16 10:19:03 +08:00

290 lines
15 KiB
TypeScript

import assert from "node:assert/strict";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
import test from "node:test";
import { Agent } from "@mastra/core/agent";
import { ModelRouterLanguageModel } from "@mastra/core/llm";
import { agentGenerationSettings, cachedSystemMessage } from "../src/lib/agent-generation-settings.ts";
import { ProviderKeyPool } from "../src/lib/model-key-pool.ts";
import {
PooledProviderLanguageModel,
providerErrorStatus,
providerRetryAfterMs,
} from "../src/lib/pooled-language-model.ts";
type Options = Parameters<PooledProviderLanguageModel["doStream"]>[0];
type Result = Awaited<ReturnType<PooledProviderLanguageModel["doStream"]>>;
const options = (abortSignal?: AbortSignal) => ({ prompt: [], abortSignal }) as unknown as Options;
const config = { providerId: "deepseek", modelId: "deepseek-chat", url: "https://api.example.test/v1" };
const keys = [{ id: "k1", apiKey: "secret-one" }, { id: "k2", apiKey: "secret-two" }];
const poolKeys = (limit: number | null = null) => keys.map((key) => ({ id: key.id, label: key.id, maxConcurrency: limit }));
const apiError = (statusCode: number, responseHeaders: Record<string, string> = {}) =>
Object.assign(new Error(`provider answered ${statusCode}`), { statusCode, responseHeaders });
const inFlight = (pool: ProviderKeyPool) => pool.snapshot().map((status) => status.inFlight);
/** A controllable stream standing in for a provider response. */
function liveStream() {
let controller!: ReadableStreamDefaultController<unknown>;
let cancelled = false;
const stream = new ReadableStream<unknown>({ start(c) { controller = c; }, cancel() { cancelled = true; } });
return { stream, push: (part: unknown) => controller.enqueue(part), close: () => controller.close(), fail: (error: unknown) => controller.error(error), cancelled: () => cancelled };
}
/** Builds a pooled model whose per-key clients are scripted by the test. */
function scripted(handler: (apiKey: string, method: "doGenerate" | "doStream", options: Options) => Promise<Result>, limit: number | null = null) {
const pool = new ProviderKeyPool();
pool.configure(poolKeys(limit));
const calls: string[] = [];
const model = new PooledProviderLanguageModel(config, pool, keys, {
createInner: (inner) => ({
doGenerate: (callOptions) => { calls.push(inner.apiKey!); return handler(inner.apiKey!, "doGenerate", callOptions); },
doStream: (callOptions) => { calls.push(inner.apiKey!); return handler(inner.apiKey!, "doStream", callOptions); },
}),
});
return { pool, model, calls };
}
async function readAll(stream: ReadableStream<unknown>) {
const parts: unknown[] = [];
const reader = stream.getReader();
for (;;) { const next = await reader.read(); if (next.done) return parts; parts.push(next.value); }
}
test("the pooled model is accepted wherever the single-key config was read", () => {
const { model } = scripted(async () => ({ stream: liveStream().stream }) as unknown as Result);
assert.ok(model instanceof ModelRouterLanguageModel, "Mastra uses it as is");
assert.equal(model.providerId, "deepseek");
assert.equal(model.modelId, "deepseek-chat");
// Generation settings and prompt-cache markers branch on providerId.
assert.ok("deepseek" in (agentGenerationSettings(model, { thinking: "disabled" }).providerOptions as Record<string, unknown>));
assert.equal(cachedSystemMessage("x", model), null);
// Span serialization and JSON never carry a key.
assert.doesNotMatch(JSON.stringify(model.serializeForSpan()), /secret-/);
});
test("calls alternate between keys and a stream holds its key until it ends", async () => {
const streams = [liveStream(), liveStream()];
let index = 0;
const { pool, model, calls } = scripted(async () => ({ stream: streams[index++]!.stream }) as unknown as Result);
const first = await model.doStream(options());
const second = await model.doStream(options());
assert.deepEqual(calls, ["secret-one", "secret-two"]);
assert.deepEqual(inFlight(pool), [1, 1]);
streams[0]!.push({ type: "text-delta", delta: "a" });
streams[0]!.close();
assert.deepEqual(await readAll(first.stream as ReadableStream<unknown>), [{ type: "text-delta", delta: "a" }]);
assert.deepEqual(inFlight(pool), [0, 1]);
streams[1]!.fail(new Error("socket closed"));
await assert.rejects(readAll(second.stream as ReadableStream<unknown>), /socket closed/);
assert.deepEqual(inFlight(pool), [0, 0], "released on stream error");
});
test("cancelling the stream or aborting the request releases the key", async () => {
const streams = [liveStream(), liveStream()];
let index = 0;
const { pool, model } = scripted(async () => ({ stream: streams[index++]!.stream }) as unknown as Result);
const first = await model.doStream(options());
await (first.stream as ReadableStream<unknown>).cancel("client left");
assert.equal(streams[0]!.cancelled(), true, "the provider stream is cancelled too");
assert.deepEqual(inFlight(pool), [0, 0]);
const controller = new AbortController();
await model.doStream(options(controller.signal));
assert.deepEqual(inFlight(pool), [0, 1]);
controller.abort();
assert.deepEqual(inFlight(pool), [0, 0], "released on abort even if nobody reads the stream");
});
test("a stream that is never read gives its slot back after the hold limit", async () => {
const pool = new ProviderKeyPool();
pool.configure(poolKeys());
const model = new PooledProviderLanguageModel(config, pool, keys, {
maxHoldMs: 20,
createInner: () => ({
doGenerate: async () => ({ stream: liveStream().stream }) as unknown as Result,
doStream: async () => ({ stream: liveStream().stream }) as unknown as Result,
}),
});
await model.doStream(options());
assert.deepEqual(inFlight(pool), [1, 0]);
await new Promise((resolve) => setTimeout(resolve, 60));
assert.deepEqual(inFlight(pool), [0, 0]);
});
test("doGenerate releases as soon as it resolves and keeps fields attached to the result", async () => {
const marker = Symbol("transport");
const { pool, model } = scripted(async () => {
const result = { stream: liveStream().stream, response: { id: "r1" } };
Object.defineProperty(result, marker, { value: "handle", configurable: true });
return result as unknown as Result;
});
const generated = await model.doGenerate(options());
assert.deepEqual(inFlight(pool), [0, 0]);
assert.equal((generated as unknown as Record<symbol, unknown>)[marker], "handle");
const streamed = await model.doStream(options());
assert.equal((streamed as unknown as Record<symbol, unknown>)[marker], "handle", "non-enumerable fields survive the stream wrapper");
assert.deepEqual((streamed as unknown as { response: unknown }).response, { id: "r1" });
});
test("429 before any output: the key cools down and the call is retried once on another key", async () => {
const { pool, model, calls } = scripted(async (apiKey) => {
if (apiKey === "secret-one") throw apiError(429, { "retry-after": "7" });
return { stream: liveStream().stream } as unknown as Result;
});
const before = Date.now();
await model.doStream(options());
assert.deepEqual(calls, ["secret-one", "secret-two"]);
assert.deepEqual(inFlight(pool), [0, 1], "the failed key was released");
const cooled = pool.snapshot()[0]!;
assert.ok(cooled.cooldownUntil! >= before + 7_000 && cooled.cooldownUntil! <= Date.now() + 7_000, "Retry-After is honoured");
assert.equal(cooled.lastRejectedAt, null);
// While k1 cools down, every call goes to k2.
await model.doGenerate(options());
assert.equal(calls.at(-1), "secret-two");
});
test("401 marks the key as rejected and retries once; a second failure reaches the caller", async () => {
const { pool, model, calls } = scripted(async (apiKey) => { throw apiError(apiKey === "secret-one" ? 401 : 429); });
await assert.rejects(model.doGenerate(options()), /provider answered 429/);
assert.deepEqual(calls, ["secret-one", "secret-two"], "exactly one retry");
assert.deepEqual(inFlight(pool), [0, 0]);
const [first, second] = pool.snapshot();
assert.equal(first!.lastRejectedStatus, 401);
assert.ok(first!.lastRejectedAt);
assert.ok(second!.cooldownUntil);
});
test("other errors are not retried and do not cool the key", async () => {
const { pool, model, calls } = scripted(async () => { throw apiError(500); });
await assert.rejects(model.doStream(options()), /provider answered 500/);
assert.deepEqual(calls, ["secret-one"]);
assert.deepEqual(pool.snapshot().map((status) => status.cooldownUntil), [null, null]);
assert.deepEqual(inFlight(pool), [0, 0]);
});
test("no retry when the only other key is cooling down or the request was aborted", async () => {
const cooling = scripted(async () => { throw apiError(429); });
cooling.pool.report("k2", { status: 403 });
await assert.rejects(cooling.model.doStream(options()), /provider answered 429/);
assert.deepEqual(cooling.calls, ["secret-one"]);
const controller = new AbortController();
const aborted = scripted(async () => { controller.abort(); throw apiError(429); });
await assert.rejects(aborted.model.doStream(options(controller.signal)), /provider answered 429/);
assert.deepEqual(aborted.calls, ["secret-one"]);
assert.deepEqual(inFlight(aborted.pool), [0, 0]);
});
test("an error part inside the stream cools the key but the stream is passed through", async () => {
const live = liveStream();
const { pool, model } = scripted(async () => ({ stream: live.stream }) as unknown as Result);
const result = await model.doStream(options());
live.push({ type: "error", error: apiError(429) });
live.close();
const parts = await readAll(result.stream as ReadableStream<unknown>);
assert.equal((parts[0] as { type: string }).type, "error");
assert.ok(pool.snapshot()[0]!.cooldownUntil);
assert.deepEqual(inFlight(pool), [0, 0]);
});
test("a request waiting for a full pool is released from the queue by its abort", async () => {
const { pool, model, calls } = scripted(async () => ({ stream: liveStream().stream }) as unknown as Result, 1);
await model.doStream(options());
await model.doStream(options());
const controller = new AbortController();
const waiting = model.doStream(options(controller.signal));
controller.abort(new Error("stopped"));
await assert.rejects(waiting, /stopped/);
assert.equal(calls.length, 2, "the provider was never called for the aborted request");
assert.deepEqual(inFlight(pool), [1, 1]);
});
test("a model built before the keys were swapped still finishes its call", async () => {
const { pool, model, calls } = scripted(async () => ({ stream: liveStream().stream }) as unknown as Result);
pool.configure([{ id: "k9", label: "new", maxConcurrency: null }]);
await model.doGenerate(options());
assert.deepEqual(calls, ["secret-one"]);
assert.deepEqual(inFlight(pool), [0]);
});
test("provider error helpers read status and Retry-After through SDK wrappers", () => {
assert.equal(providerErrorStatus(apiError(429)), 429);
assert.equal(providerErrorStatus({ lastError: apiError(403) }), 403);
assert.equal(providerErrorStatus(new Error("x", { cause: apiError(401) })), 401);
assert.equal(providerErrorStatus(new Error("network")), null);
assert.equal(providerErrorStatus("429"), null);
assert.equal(providerRetryAfterMs(apiError(429, { "Retry-After": "12" })), 12_000);
assert.equal(providerRetryAfterMs(apiError(429, { "retry-after-ms": "250" })), 250);
assert.equal(providerRetryAfterMs(apiError(429, { "retry-after": new Date(60_000).toUTCString() }), 0), 60_000);
assert.equal(providerRetryAfterMs(apiError(429, { "retry-after": "soon" })), null);
assert.equal(providerRetryAfterMs(apiError(429)), null);
});
/** OpenAI-compatible stub: records the bearer token of each chat request. */
async function startProviderStub(answer: (token: string, response: ServerResponse, stream: boolean) => void) {
const tokens: string[] = [];
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
let body = "";
request.on("data", (chunk: Buffer) => { body += chunk.toString("utf8"); });
request.on("end", () => {
const token = (request.headers.authorization ?? "").replace(/^Bearer /, "");
tokens.push(token);
answer(token, response, JSON.parse(body).stream === true);
});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const { port } = server.address() as AddressInfo;
return { tokens, url: `http://127.0.0.1:${port}/v1`, close: () => new Promise<void>((resolve) => { server.closeAllConnections(); server.close(() => resolve()); }) };
}
function sse(response: ServerResponse, text: string, stream: boolean) {
if (!stream) {
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({ id: "c1", object: "chat.completion", created: 1, model: "stub", choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }));
return;
}
response.writeHead(200, { "content-type": "text/event-stream" });
const chunk = (delta: object, finish: string | null = null) => `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", created: 1, model: "stub", choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`;
response.write(chunk({ role: "assistant", content: text }));
response.write(chunk({}, "stop"));
response.end("data: [DONE]\n\n");
}
test("a real Mastra agent spreads requests over both keys and moves off a rate-limited key", async () => {
let limited = "";
const stub = await startProviderStub((token, response, stream) => {
if (token === limited) {
response.writeHead(429, { "content-type": "application/json", "retry-after": "60" });
response.end(JSON.stringify({ error: { message: "rate limited" } }));
return;
}
sse(response, `hello from ${token}`, stream);
});
try {
const pool = new ProviderKeyPool();
pool.configure(poolKeys());
const model = new PooledProviderLanguageModel({ providerId: "stub-vendor", modelId: "stub-model", url: stub.url }, pool, keys);
const agent = new Agent({ id: "pool-test", name: "Pool Test", model, instructions: "Answer briefly.", maxRetries: 0 });
const ask = async () => {
const result = await agent.stream([{ role: "user", content: "hi" }]);
let text = "";
for await (const chunk of result.textStream) text += chunk;
return text;
};
assert.equal(await ask(), "hello from secret-one");
assert.equal(await ask(), "hello from secret-two");
assert.deepEqual(inFlight(pool), [0, 0], "released after each stream");
limited = "secret-one";
assert.equal(await ask(), "hello from secret-two", "the 429 never reaches the user");
assert.deepEqual(stub.tokens.slice(2), ["secret-one", "secret-two"]);
assert.ok(pool.snapshot()[0]!.cooldownUntil! > Date.now() + 50_000);
assert.equal(await ask(), "hello from secret-two", "the cooling key is skipped");
assert.equal(stub.tokens.length, 5);
const generated = await agent.generate([{ role: "user", content: "hi" }]);
assert.equal(generated.text, "hello from secret-two");
assert.deepEqual(inFlight(pool), [0, 0]);
} finally {
await stub.close();
}
});