jesse-uxandGrok (agent loop)
a54f75e5e3
Consult, rectification turns, and report generation stop before a credit reservation when the product caps are hit. A full heavy-compute gate waits up to 8 seconds, and one consultation retries that 429 inside the same request. Co-Authored-By: Grok (agent loop)
278 lines
10 KiB
TypeScript
278 lines
10 KiB
TypeScript
/**
|
|
* Staging-only smoke for 10 or 20 concurrent consultations.
|
|
*
|
|
* Not imported by frontend/src. Reads STAGING_TEST_OTP from the environment
|
|
* and never prints it. Refuses a production host. Does not sample server
|
|
* CPU or memory; that needs a staging shell this process does not have.
|
|
*
|
|
* node node_modules/tsx/dist/cli.mjs scripts/research/beta-load-smoke.mts --users 10
|
|
*/
|
|
const usersArg = process.argv.find((arg) => arg.startsWith("--users="));
|
|
const usersFlag = process.argv.indexOf("--users");
|
|
const requestedUsers = Number(usersArg ? usersArg.slice("--users=".length) : process.argv[usersFlag + 1]);
|
|
const baseUrl = (process.env.STAGING_BASE_URL || "https://staging.jyotisha.chat").replace(/\/$/, "");
|
|
const otp = process.env.STAGING_TEST_OTP || "";
|
|
const emailDomain = process.env.STAGING_TEST_EMAIL_DOMAIN || "example.test";
|
|
|
|
function refuse(message: string): never {
|
|
console.error(message);
|
|
process.exit(2);
|
|
}
|
|
|
|
let host = "";
|
|
try {
|
|
host = new URL(baseUrl).host;
|
|
} catch {
|
|
refuse("STAGING_BASE_URL is not a URL");
|
|
}
|
|
if (host === "jyotisha.chat" || host === "www.jyotisha.chat" || host === "admin.jyotisha.chat") {
|
|
refuse("refusing production host");
|
|
}
|
|
if (!/^\d{6}$/.test(otp)) refuse("STAGING_TEST_OTP is not set");
|
|
if (requestedUsers !== 10 && requestedUsers !== 20) refuse("pass --users 10 or --users 20");
|
|
if (!/^[a-z0-9.-]+$/.test(emailDomain)) refuse("STAGING_TEST_EMAIL_DOMAIN is not a plain domain");
|
|
|
|
type Sample = {
|
|
kind: "consult" | "rectification";
|
|
status: number;
|
|
firstByteMs: number | null;
|
|
totalMs: number;
|
|
failed: boolean;
|
|
http429: boolean;
|
|
http5xx: boolean;
|
|
};
|
|
|
|
const samples: Sample[] = [];
|
|
const CONSULT_QUEUE_FAILED = "人有点多，这一轮没排上，点数已经退回。";
|
|
|
|
function percentile(values: number[], ratio: number): number | null {
|
|
if (values.length === 0) return null;
|
|
const sorted = [...values].sort((a, b) => a - b);
|
|
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1));
|
|
return sorted[index] ?? null;
|
|
}
|
|
|
|
async function request(jar: Map<string, string>, path: string, init: RequestInit = {}): Promise<Response> {
|
|
const headers = new Headers(init.headers);
|
|
if (jar.size > 0) headers.set("cookie", [...jar].map(([key, value]) => `${key}=${value}`).join("; "));
|
|
const response = await fetch(`${baseUrl}${path}`, { ...init, headers, redirect: "manual" });
|
|
const setCookies = typeof response.headers.getSetCookie === "function" ? response.headers.getSetCookie() : [];
|
|
for (const line of setCookies) {
|
|
const pair = line.split(";")[0] ?? "";
|
|
const eq = pair.indexOf("=");
|
|
if (eq > 0) jar.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
|
|
}
|
|
return response;
|
|
}
|
|
|
|
async function timed(jar: Map<string, string>, path: string, body: unknown, kind: Sample["kind"]): Promise<void> {
|
|
const started = Date.now();
|
|
let status = 0;
|
|
let firstByteMs: number | null = null;
|
|
let failed = false;
|
|
try {
|
|
const response = await request(jar, path, {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify(body),
|
|
});
|
|
status = response.status;
|
|
const reader = response.body?.getReader();
|
|
if (reader) {
|
|
let text = "";
|
|
for (;;) {
|
|
const chunk = await reader.read();
|
|
if (firstByteMs === null && (chunk.value || chunk.done)) firstByteMs = Date.now() - started;
|
|
if (chunk.value) text += new TextDecoder().decode(chunk.value);
|
|
if (chunk.done) break;
|
|
}
|
|
failed = text.includes('"type":"run.failed"') || text.includes(CONSULT_QUEUE_FAILED);
|
|
} else {
|
|
firstByteMs = Date.now() - started;
|
|
failed = status >= 400;
|
|
}
|
|
} catch {
|
|
status = 0;
|
|
failed = true;
|
|
}
|
|
samples.push({
|
|
kind,
|
|
status,
|
|
firstByteMs,
|
|
totalMs: Date.now() - started,
|
|
failed,
|
|
http429: status === 429,
|
|
http5xx: status >= 500 || status === 0,
|
|
});
|
|
}
|
|
|
|
async function login(index: number): Promise<Map<string, string> | null> {
|
|
const jar = new Map<string, string>();
|
|
const email = `capacity-smoke-${index}@${emailDomain}`;
|
|
const sent = await request(jar, "/api/auth/email-otp/send-verification-otp", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ email, type: "sign-in" }),
|
|
});
|
|
if (!sent.ok) return null;
|
|
const signed = await request(jar, "/api/auth/sign-in/email-otp", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ email, otp }),
|
|
});
|
|
if (!signed.ok || jar.size === 0) return null;
|
|
return jar;
|
|
}
|
|
|
|
async function prepareAccount(jar: Map<string, string>): Promise<boolean> {
|
|
const saved = await request(jar, "/api/account", {
|
|
method: "PATCH",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
name: "容量冒烟",
|
|
birth_date: "1990-06-15",
|
|
reported_birth_time: "12:00",
|
|
birth_time_source: "family_exact",
|
|
birth_time_period: null,
|
|
birth_time_clue: null,
|
|
uncertainty_before_minutes: 0,
|
|
uncertainty_after_minutes: 0,
|
|
country_code: "CN",
|
|
province_code: "110000",
|
|
city_code: "110000",
|
|
district_code: "110101",
|
|
latitude: 39.9042,
|
|
longitude: 116.4074,
|
|
timezone_offset: 8,
|
|
}),
|
|
});
|
|
return saved.ok;
|
|
}
|
|
|
|
async function oneUser(index: number): Promise<string[]> {
|
|
const deleted: string[] = [];
|
|
const jar = await login(index);
|
|
if (!jar) {
|
|
samples.push({ kind: "consult", status: 0, firstByteMs: null, totalMs: 0, failed: true, http429: false, http5xx: true });
|
|
return deleted;
|
|
}
|
|
if (!await prepareAccount(jar)) {
|
|
samples.push({ kind: "consult", status: 0, firstByteMs: null, totalMs: 0, failed: true, http429: false, http5xx: true });
|
|
return deleted;
|
|
}
|
|
const models = await request(jar, "/api/models");
|
|
const catalog = models.ok ? await models.json().catch(() => null) as { defaultModelId?: string } | null : null;
|
|
const modelId = catalog?.defaultModelId;
|
|
if (!modelId) {
|
|
samples.push({ kind: "consult", status: 0, firstByteMs: null, totalMs: 0, failed: true, http429: false, http5xx: true });
|
|
return deleted;
|
|
}
|
|
const sessionId = crypto.randomUUID();
|
|
const created = await request(jar, "/api/sessions", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
id: sessionId,
|
|
title: "容量冒烟",
|
|
theme: "general",
|
|
model_id: modelId,
|
|
messages: [],
|
|
session_type: "consultation",
|
|
rectification_case_id: null,
|
|
chart_profile_role: "self",
|
|
}),
|
|
});
|
|
if (!created.ok) {
|
|
samples.push({ kind: "consult", status: created.status, firstByteMs: null, totalMs: 0, failed: true, http429: created.status === 429, http5xx: created.status >= 500 });
|
|
return deleted;
|
|
}
|
|
const chart = {
|
|
year: 1990,
|
|
month: 6,
|
|
day: 15,
|
|
hour: 12,
|
|
minute: 0,
|
|
lat: 39.9042,
|
|
lon: 116.4074,
|
|
tz: 8,
|
|
city: "北京",
|
|
ayanamsa: "lahiri",
|
|
};
|
|
await timed(jar, "/api/consult", {
|
|
...chart,
|
|
requestId: crypto.randomUUID(),
|
|
sessionId,
|
|
modelId,
|
|
consultationMode: "verified_chart",
|
|
theme: "general",
|
|
question: "今年工作整体顺不顺？",
|
|
history: [],
|
|
}, "consult");
|
|
await timed(jar, "/api/consult", {
|
|
...chart,
|
|
requestId: crypto.randomUUID(),
|
|
sessionId,
|
|
modelId,
|
|
consultationMode: "verified_chart",
|
|
theme: "general",
|
|
question: "那接下来几个月呢？",
|
|
history: [],
|
|
}, "consult");
|
|
if (index < 3) {
|
|
const opened = await request(jar, "/api/rectification/cases/open", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ intent: "homepage", requestId: crypto.randomUUID() }),
|
|
});
|
|
const openedBody = opened.ok ? await opened.json().catch(() => null) as { caseId?: string; sessionId?: string } | null : null;
|
|
if (openedBody?.caseId && openedBody.sessionId) {
|
|
for (const message of ["我小时候搬家过一次。", "那次大概是上小学之前。"]) {
|
|
await timed(jar, "/api/rectification/agent", {
|
|
caseId: openedBody.caseId,
|
|
sessionId: openedBody.sessionId,
|
|
requestId: crypto.randomUUID(),
|
|
action: "message",
|
|
message,
|
|
}, "rectification");
|
|
}
|
|
deleted.push(openedBody.sessionId);
|
|
} else {
|
|
samples.push({
|
|
kind: "rectification",
|
|
status: opened.status,
|
|
firstByteMs: null,
|
|
totalMs: 0,
|
|
failed: true,
|
|
http429: opened.status === 429,
|
|
http5xx: opened.status >= 500 || opened.status === 0,
|
|
});
|
|
}
|
|
}
|
|
const removed = await request(jar, `/api/sessions/${sessionId}`, { method: "DELETE" });
|
|
if (removed.ok) deleted.push(sessionId);
|
|
return deleted;
|
|
}
|
|
|
|
const startedAt = Date.now();
|
|
const cleaned = (await Promise.all(Array.from({ length: requestedUsers }, (_, index) => oneUser(index)))).flat();
|
|
const consults = samples.filter((sample) => sample.kind === "consult");
|
|
const finalFailures = consults.filter((sample) => sample.failed || sample.http429 || sample.http5xx);
|
|
const firstBytes = consults.map((sample) => sample.firstByteMs).filter((value): value is number => value !== null);
|
|
const totals = consults.map((sample) => sample.totalMs);
|
|
const summary = {
|
|
users: requestedUsers,
|
|
host,
|
|
requests: samples.length,
|
|
consults: consults.length,
|
|
http429: samples.filter((sample) => sample.http429).length,
|
|
http5xx: samples.filter((sample) => sample.http5xx).length,
|
|
finalFailures: finalFailures.length,
|
|
finalFailureRatio: consults.length === 0 ? null : Number((finalFailures.length / consults.length).toFixed(4)),
|
|
firstByteP95Ms: percentile(firstBytes, 0.95),
|
|
totalP95Ms: percentile(totals, 0.95),
|
|
sessionsDeleted: cleaned.length,
|
|
serverSample: "not taken; this process has no staging docker stats",
|
|
elapsedMs: Date.now() - startedAt,
|
|
};
|
|
console.log(JSON.stringify(summary));
|