Each model call takes the enabled key with the fewest in-flight requests (ties round-robin) and gives it back when the stream ends, fails or is aborted. A key at its concurrency limit is skipped; when all are full the request waits in arrival order for at most 15 s and then goes ahead on the least-loaded key. HTTP 429 cools a key for 30 s (Retry-After honoured, 120 s cap), 401/403 for ten minutes. A call that fails with one of these before any output is retried once on another key. The catalog hands agents a pooled Mastra model in place of the single-key config, so every existing call site is covered; providers without key rows keep using the legacy column. State is in-process. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DbBXp2Vp62Ur3C7yRWQdSq
290 lines
15 KiB
TypeScript
290 lines
15 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
|
import type { AddressInfo } from "node:net";
|
|
import test from "node:test";
|
|
|
|
import { Agent } from "@mastra/core/agent";
|
|
import { ModelRouterLanguageModel } from "@mastra/core/llm";
|
|
|
|
import { agentGenerationSettings, cachedSystemMessage } from "../src/lib/agent-generation-settings.ts";
|
|
import { ProviderKeyPool } from "../src/lib/model-key-pool.ts";
|
|
import {
|
|
PooledProviderLanguageModel,
|
|
providerErrorStatus,
|
|
providerRetryAfterMs,
|
|
} from "../src/lib/pooled-language-model.ts";
|
|
|
|
type Options = Parameters<PooledProviderLanguageModel["doStream"]>[0];
|
|
type Result = Awaited<ReturnType<PooledProviderLanguageModel["doStream"]>>;
|
|
const options = (abortSignal?: AbortSignal) => ({ prompt: [], abortSignal }) as unknown as Options;
|
|
const config = { providerId: "deepseek", modelId: "deepseek-chat", url: "https://api.example.test/v1" };
|
|
const keys = [{ id: "k1", apiKey: "secret-one" }, { id: "k2", apiKey: "secret-two" }];
|
|
const poolKeys = (limit: number | null = null) => keys.map((key) => ({ id: key.id, label: key.id, maxConcurrency: limit }));
|
|
const apiError = (statusCode: number, responseHeaders: Record<string, string> = {}) =>
|
|
Object.assign(new Error(`provider answered ${statusCode}`), { statusCode, responseHeaders });
|
|
const inFlight = (pool: ProviderKeyPool) => pool.snapshot().map((status) => status.inFlight);
|
|
|
|
/** A controllable stream standing in for a provider response. */
|
|
function liveStream() {
|
|
let controller!: ReadableStreamDefaultController<unknown>;
|
|
let cancelled = false;
|
|
const stream = new ReadableStream<unknown>({ start(c) { controller = c; }, cancel() { cancelled = true; } });
|
|
return { stream, push: (part: unknown) => controller.enqueue(part), close: () => controller.close(), fail: (error: unknown) => controller.error(error), cancelled: () => cancelled };
|
|
}
|
|
/** Builds a pooled model whose per-key clients are scripted by the test. */
|
|
function scripted(handler: (apiKey: string, method: "doGenerate" | "doStream", options: Options) => Promise<Result>, limit: number | null = null) {
|
|
const pool = new ProviderKeyPool();
|
|
pool.configure(poolKeys(limit));
|
|
const calls: string[] = [];
|
|
const model = new PooledProviderLanguageModel(config, pool, keys, {
|
|
createInner: (inner) => ({
|
|
doGenerate: (callOptions) => { calls.push(inner.apiKey!); return handler(inner.apiKey!, "doGenerate", callOptions); },
|
|
doStream: (callOptions) => { calls.push(inner.apiKey!); return handler(inner.apiKey!, "doStream", callOptions); },
|
|
}),
|
|
});
|
|
return { pool, model, calls };
|
|
}
|
|
async function readAll(stream: ReadableStream<unknown>) {
|
|
const parts: unknown[] = [];
|
|
const reader = stream.getReader();
|
|
for (;;) { const next = await reader.read(); if (next.done) return parts; parts.push(next.value); }
|
|
}
|
|
|
|
test("the pooled model is accepted wherever the single-key config was read", () => {
|
|
const { model } = scripted(async () => ({ stream: liveStream().stream }) as unknown as Result);
|
|
assert.ok(model instanceof ModelRouterLanguageModel, "Mastra uses it as is");
|
|
assert.equal(model.providerId, "deepseek");
|
|
assert.equal(model.modelId, "deepseek-chat");
|
|
// Generation settings and prompt-cache markers branch on providerId.
|
|
assert.ok("deepseek" in (agentGenerationSettings(model, { thinking: "disabled" }).providerOptions as Record<string, unknown>));
|
|
assert.equal(cachedSystemMessage("x", model), null);
|
|
// Span serialization and JSON never carry a key.
|
|
assert.doesNotMatch(JSON.stringify(model.serializeForSpan()), /secret-/);
|
|
});
|
|
|
|
test("calls alternate between keys and a stream holds its key until it ends", async () => {
|
|
const streams = [liveStream(), liveStream()];
|
|
let index = 0;
|
|
const { pool, model, calls } = scripted(async () => ({ stream: streams[index++]!.stream }) as unknown as Result);
|
|
const first = await model.doStream(options());
|
|
const second = await model.doStream(options());
|
|
assert.deepEqual(calls, ["secret-one", "secret-two"]);
|
|
assert.deepEqual(inFlight(pool), [1, 1]);
|
|
streams[0]!.push({ type: "text-delta", delta: "a" });
|
|
streams[0]!.close();
|
|
assert.deepEqual(await readAll(first.stream as ReadableStream<unknown>), [{ type: "text-delta", delta: "a" }]);
|
|
assert.deepEqual(inFlight(pool), [0, 1]);
|
|
streams[1]!.fail(new Error("socket closed"));
|
|
await assert.rejects(readAll(second.stream as ReadableStream<unknown>), /socket closed/);
|
|
assert.deepEqual(inFlight(pool), [0, 0], "released on stream error");
|
|
});
|
|
|
|
test("cancelling the stream or aborting the request releases the key", async () => {
|
|
const streams = [liveStream(), liveStream()];
|
|
let index = 0;
|
|
const { pool, model } = scripted(async () => ({ stream: streams[index++]!.stream }) as unknown as Result);
|
|
const first = await model.doStream(options());
|
|
await (first.stream as ReadableStream<unknown>).cancel("client left");
|
|
assert.equal(streams[0]!.cancelled(), true, "the provider stream is cancelled too");
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
const controller = new AbortController();
|
|
await model.doStream(options(controller.signal));
|
|
assert.deepEqual(inFlight(pool), [0, 1]);
|
|
controller.abort();
|
|
assert.deepEqual(inFlight(pool), [0, 0], "released on abort even if nobody reads the stream");
|
|
});
|
|
|
|
test("a stream that is never read gives its slot back after the hold limit", async () => {
|
|
const pool = new ProviderKeyPool();
|
|
pool.configure(poolKeys());
|
|
const model = new PooledProviderLanguageModel(config, pool, keys, {
|
|
maxHoldMs: 20,
|
|
createInner: () => ({
|
|
doGenerate: async () => ({ stream: liveStream().stream }) as unknown as Result,
|
|
doStream: async () => ({ stream: liveStream().stream }) as unknown as Result,
|
|
}),
|
|
});
|
|
await model.doStream(options());
|
|
assert.deepEqual(inFlight(pool), [1, 0]);
|
|
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
});
|
|
|
|
test("doGenerate releases as soon as it resolves and keeps fields attached to the result", async () => {
|
|
const marker = Symbol("transport");
|
|
const { pool, model } = scripted(async () => {
|
|
const result = { stream: liveStream().stream, response: { id: "r1" } };
|
|
Object.defineProperty(result, marker, { value: "handle", configurable: true });
|
|
return result as unknown as Result;
|
|
});
|
|
const generated = await model.doGenerate(options());
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
assert.equal((generated as unknown as Record<symbol, unknown>)[marker], "handle");
|
|
const streamed = await model.doStream(options());
|
|
assert.equal((streamed as unknown as Record<symbol, unknown>)[marker], "handle", "non-enumerable fields survive the stream wrapper");
|
|
assert.deepEqual((streamed as unknown as { response: unknown }).response, { id: "r1" });
|
|
});
|
|
|
|
test("429 before any output: the key cools down and the call is retried once on another key", async () => {
|
|
const { pool, model, calls } = scripted(async (apiKey) => {
|
|
if (apiKey === "secret-one") throw apiError(429, { "retry-after": "7" });
|
|
return { stream: liveStream().stream } as unknown as Result;
|
|
});
|
|
const before = Date.now();
|
|
await model.doStream(options());
|
|
assert.deepEqual(calls, ["secret-one", "secret-two"]);
|
|
assert.deepEqual(inFlight(pool), [0, 1], "the failed key was released");
|
|
const cooled = pool.snapshot()[0]!;
|
|
assert.ok(cooled.cooldownUntil! >= before + 7_000 && cooled.cooldownUntil! <= Date.now() + 7_000, "Retry-After is honoured");
|
|
assert.equal(cooled.lastRejectedAt, null);
|
|
// While k1 cools down, every call goes to k2.
|
|
await model.doGenerate(options());
|
|
assert.equal(calls.at(-1), "secret-two");
|
|
});
|
|
|
|
test("401 marks the key as rejected and retries once; a second failure reaches the caller", async () => {
|
|
const { pool, model, calls } = scripted(async (apiKey) => { throw apiError(apiKey === "secret-one" ? 401 : 429); });
|
|
await assert.rejects(model.doGenerate(options()), /provider answered 429/);
|
|
assert.deepEqual(calls, ["secret-one", "secret-two"], "exactly one retry");
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
const [first, second] = pool.snapshot();
|
|
assert.equal(first!.lastRejectedStatus, 401);
|
|
assert.ok(first!.lastRejectedAt);
|
|
assert.ok(second!.cooldownUntil);
|
|
});
|
|
|
|
test("other errors are not retried and do not cool the key", async () => {
|
|
const { pool, model, calls } = scripted(async () => { throw apiError(500); });
|
|
await assert.rejects(model.doStream(options()), /provider answered 500/);
|
|
assert.deepEqual(calls, ["secret-one"]);
|
|
assert.deepEqual(pool.snapshot().map((status) => status.cooldownUntil), [null, null]);
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
});
|
|
|
|
test("no retry when the only other key is cooling down or the request was aborted", async () => {
|
|
const cooling = scripted(async () => { throw apiError(429); });
|
|
cooling.pool.report("k2", { status: 403 });
|
|
await assert.rejects(cooling.model.doStream(options()), /provider answered 429/);
|
|
assert.deepEqual(cooling.calls, ["secret-one"]);
|
|
|
|
const controller = new AbortController();
|
|
const aborted = scripted(async () => { controller.abort(); throw apiError(429); });
|
|
await assert.rejects(aborted.model.doStream(options(controller.signal)), /provider answered 429/);
|
|
assert.deepEqual(aborted.calls, ["secret-one"]);
|
|
assert.deepEqual(inFlight(aborted.pool), [0, 0]);
|
|
});
|
|
|
|
test("an error part inside the stream cools the key but the stream is passed through", async () => {
|
|
const live = liveStream();
|
|
const { pool, model } = scripted(async () => ({ stream: live.stream }) as unknown as Result);
|
|
const result = await model.doStream(options());
|
|
live.push({ type: "error", error: apiError(429) });
|
|
live.close();
|
|
const parts = await readAll(result.stream as ReadableStream<unknown>);
|
|
assert.equal((parts[0] as { type: string }).type, "error");
|
|
assert.ok(pool.snapshot()[0]!.cooldownUntil);
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
});
|
|
|
|
test("a request waiting for a full pool is released from the queue by its abort", async () => {
|
|
const { pool, model, calls } = scripted(async () => ({ stream: liveStream().stream }) as unknown as Result, 1);
|
|
await model.doStream(options());
|
|
await model.doStream(options());
|
|
const controller = new AbortController();
|
|
const waiting = model.doStream(options(controller.signal));
|
|
controller.abort(new Error("stopped"));
|
|
await assert.rejects(waiting, /stopped/);
|
|
assert.equal(calls.length, 2, "the provider was never called for the aborted request");
|
|
assert.deepEqual(inFlight(pool), [1, 1]);
|
|
});
|
|
|
|
test("a model built before the keys were swapped still finishes its call", async () => {
|
|
const { pool, model, calls } = scripted(async () => ({ stream: liveStream().stream }) as unknown as Result);
|
|
pool.configure([{ id: "k9", label: "new", maxConcurrency: null }]);
|
|
await model.doGenerate(options());
|
|
assert.deepEqual(calls, ["secret-one"]);
|
|
assert.deepEqual(inFlight(pool), [0]);
|
|
});
|
|
|
|
test("provider error helpers read status and Retry-After through SDK wrappers", () => {
|
|
assert.equal(providerErrorStatus(apiError(429)), 429);
|
|
assert.equal(providerErrorStatus({ lastError: apiError(403) }), 403);
|
|
assert.equal(providerErrorStatus(new Error("x", { cause: apiError(401) })), 401);
|
|
assert.equal(providerErrorStatus(new Error("network")), null);
|
|
assert.equal(providerErrorStatus("429"), null);
|
|
assert.equal(providerRetryAfterMs(apiError(429, { "Retry-After": "12" })), 12_000);
|
|
assert.equal(providerRetryAfterMs(apiError(429, { "retry-after-ms": "250" })), 250);
|
|
assert.equal(providerRetryAfterMs(apiError(429, { "retry-after": new Date(60_000).toUTCString() }), 0), 60_000);
|
|
assert.equal(providerRetryAfterMs(apiError(429, { "retry-after": "soon" })), null);
|
|
assert.equal(providerRetryAfterMs(apiError(429)), null);
|
|
});
|
|
|
|
/** OpenAI-compatible stub: records the bearer token of each chat request. */
|
|
async function startProviderStub(answer: (token: string, response: ServerResponse, stream: boolean) => void) {
|
|
const tokens: string[] = [];
|
|
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
|
|
let body = "";
|
|
request.on("data", (chunk: Buffer) => { body += chunk.toString("utf8"); });
|
|
request.on("end", () => {
|
|
const token = (request.headers.authorization ?? "").replace(/^Bearer /, "");
|
|
tokens.push(token);
|
|
answer(token, response, JSON.parse(body).stream === true);
|
|
});
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
const { port } = server.address() as AddressInfo;
|
|
return { tokens, url: `http://127.0.0.1:${port}/v1`, close: () => new Promise<void>((resolve) => { server.closeAllConnections(); server.close(() => resolve()); }) };
|
|
}
|
|
function sse(response: ServerResponse, text: string, stream: boolean) {
|
|
if (!stream) {
|
|
response.writeHead(200, { "content-type": "application/json" });
|
|
response.end(JSON.stringify({ id: "c1", object: "chat.completion", created: 1, model: "stub", choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }));
|
|
return;
|
|
}
|
|
response.writeHead(200, { "content-type": "text/event-stream" });
|
|
const chunk = (delta: object, finish: string | null = null) => `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", created: 1, model: "stub", choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`;
|
|
response.write(chunk({ role: "assistant", content: text }));
|
|
response.write(chunk({}, "stop"));
|
|
response.end("data: [DONE]\n\n");
|
|
}
|
|
|
|
test("a real Mastra agent spreads requests over both keys and moves off a rate-limited key", async () => {
|
|
let limited = "";
|
|
const stub = await startProviderStub((token, response, stream) => {
|
|
if (token === limited) {
|
|
response.writeHead(429, { "content-type": "application/json", "retry-after": "60" });
|
|
response.end(JSON.stringify({ error: { message: "rate limited" } }));
|
|
return;
|
|
}
|
|
sse(response, `hello from ${token}`, stream);
|
|
});
|
|
try {
|
|
const pool = new ProviderKeyPool();
|
|
pool.configure(poolKeys());
|
|
const model = new PooledProviderLanguageModel({ providerId: "stub-vendor", modelId: "stub-model", url: stub.url }, pool, keys);
|
|
const agent = new Agent({ id: "pool-test", name: "Pool Test", model, instructions: "Answer briefly.", maxRetries: 0 });
|
|
const ask = async () => {
|
|
const result = await agent.stream([{ role: "user", content: "hi" }]);
|
|
let text = "";
|
|
for await (const chunk of result.textStream) text += chunk;
|
|
return text;
|
|
};
|
|
assert.equal(await ask(), "hello from secret-one");
|
|
assert.equal(await ask(), "hello from secret-two");
|
|
assert.deepEqual(inFlight(pool), [0, 0], "released after each stream");
|
|
|
|
limited = "secret-one";
|
|
assert.equal(await ask(), "hello from secret-two", "the 429 never reaches the user");
|
|
assert.deepEqual(stub.tokens.slice(2), ["secret-one", "secret-two"]);
|
|
assert.ok(pool.snapshot()[0]!.cooldownUntil! > Date.now() + 50_000);
|
|
assert.equal(await ask(), "hello from secret-two", "the cooling key is skipped");
|
|
assert.equal(stub.tokens.length, 5);
|
|
|
|
const generated = await agent.generate([{ role: "user", content: "hi" }]);
|
|
assert.equal(generated.text, "hello from secret-two");
|
|
assert.deepEqual(inFlight(pool), [0, 0]);
|
|
} finally {
|
|
await stub.close();
|
|
}
|
|
});
|