/** * Staging-only smoke for 10 or 20 concurrent consultations. * * Not imported by frontend/src. Reads STAGING_TEST_OTP from the environment * and never prints it. Refuses a production host. Does not sample server * CPU or memory; that needs a staging shell this process does not have. * * node node_modules/tsx/dist/cli.mjs scripts/research/beta-load-smoke.mts --users 10 */ const usersArg = process.argv.find((arg) => arg.startsWith("--users=")); const usersFlag = process.argv.indexOf("--users"); const requestedUsers = Number(usersArg ? usersArg.slice("--users=".length) : process.argv[usersFlag + 1]); const baseUrl = (process.env.STAGING_BASE_URL || "https://staging.jyotisha.chat").replace(/\/$/, ""); const otp = process.env.STAGING_TEST_OTP || ""; const emailDomain = process.env.STAGING_TEST_EMAIL_DOMAIN || "example.test"; function refuse(message: string): never { console.error(message); process.exit(2); } let host = ""; try { host = new URL(baseUrl).host; } catch { refuse("STAGING_BASE_URL is not a URL"); } if (host === "jyotisha.chat" || host === "www.jyotisha.chat" || host === "admin.jyotisha.chat") { refuse("refusing production host"); } if (!/^\d{6}$/.test(otp)) refuse("STAGING_TEST_OTP is not set"); if (requestedUsers !== 10 && requestedUsers !== 20) refuse("pass --users 10 or --users 20"); if (!/^[a-z0-9.-]+$/.test(emailDomain)) refuse("STAGING_TEST_EMAIL_DOMAIN is not a plain domain"); type Sample = { kind: "consult" | "rectification"; status: number; firstByteMs: number | null; totalMs: number; failed: boolean; http429: boolean; http5xx: boolean; }; const samples: Sample[] = []; const CONSULT_QUEUE_FAILED = "人有点多,这一轮没排上,点数已经退回。"; function percentile(values: number[], ratio: number): number | null { if (values.length === 0) return null; const sorted = [...values].sort((a, b) => a - b); const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1)); return sorted[index] ?? null; } async function request(jar: Map, path: string, init: RequestInit = {}): Promise { const headers = new Headers(init.headers); if (jar.size > 0) headers.set("cookie", [...jar].map(([key, value]) => `${key}=${value}`).join("; ")); const response = await fetch(`${baseUrl}${path}`, { ...init, headers, redirect: "manual" }); const setCookies = typeof response.headers.getSetCookie === "function" ? response.headers.getSetCookie() : []; for (const line of setCookies) { const pair = line.split(";")[0] ?? ""; const eq = pair.indexOf("="); if (eq > 0) jar.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim()); } return response; } async function timed(jar: Map, path: string, body: unknown, kind: Sample["kind"]): Promise { const started = Date.now(); let status = 0; let firstByteMs: number | null = null; let failed = false; try { const response = await request(jar, path, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), }); status = response.status; const reader = response.body?.getReader(); if (reader) { let text = ""; for (;;) { const chunk = await reader.read(); if (firstByteMs === null && (chunk.value || chunk.done)) firstByteMs = Date.now() - started; if (chunk.value) text += new TextDecoder().decode(chunk.value); if (chunk.done) break; } failed = text.includes('"type":"run.failed"') || text.includes(CONSULT_QUEUE_FAILED); } else { firstByteMs = Date.now() - started; failed = status >= 400; } } catch { status = 0; failed = true; } samples.push({ kind, status, firstByteMs, totalMs: Date.now() - started, failed, http429: status === 429, http5xx: status >= 500 || status === 0, }); } async function login(index: number): Promise | null> { const jar = new Map(); const email = `capacity-smoke-${index}@${emailDomain}`; const sent = await request(jar, "/api/auth/email-otp/send-verification-otp", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ email, type: "sign-in" }), }); if (!sent.ok) return null; const signed = await request(jar, "/api/auth/sign-in/email-otp", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ email, otp }), }); if (!signed.ok || jar.size === 0) return null; return jar; } async function prepareAccount(jar: Map): Promise { const saved = await request(jar, "/api/account", { method: "PATCH", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "容量冒烟", birth_date: "1990-06-15", reported_birth_time: "12:00", birth_time_source: "family_exact", birth_time_period: null, birth_time_clue: null, uncertainty_before_minutes: 0, uncertainty_after_minutes: 0, country_code: "CN", province_code: "110000", city_code: "110000", district_code: "110101", latitude: 39.9042, longitude: 116.4074, timezone_offset: 8, }), }); return saved.ok; } async function oneUser(index: number): Promise { const deleted: string[] = []; const jar = await login(index); if (!jar) { samples.push({ kind: "consult", status: 0, firstByteMs: null, totalMs: 0, failed: true, http429: false, http5xx: true }); return deleted; } if (!await prepareAccount(jar)) { samples.push({ kind: "consult", status: 0, firstByteMs: null, totalMs: 0, failed: true, http429: false, http5xx: true }); return deleted; } const models = await request(jar, "/api/models"); const catalog = models.ok ? await models.json().catch(() => null) as { defaultModelId?: string } | null : null; const modelId = catalog?.defaultModelId; if (!modelId) { samples.push({ kind: "consult", status: 0, firstByteMs: null, totalMs: 0, failed: true, http429: false, http5xx: true }); return deleted; } const sessionId = crypto.randomUUID(); const created = await request(jar, "/api/sessions", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ id: sessionId, title: "容量冒烟", theme: "general", model_id: modelId, messages: [], session_type: "consultation", rectification_case_id: null, chart_profile_role: "self", }), }); if (!created.ok) { samples.push({ kind: "consult", status: created.status, firstByteMs: null, totalMs: 0, failed: true, http429: created.status === 429, http5xx: created.status >= 500 }); return deleted; } const chart = { year: 1990, month: 6, day: 15, hour: 12, minute: 0, lat: 39.9042, lon: 116.4074, tz: 8, city: "北京", ayanamsa: "lahiri", }; await timed(jar, "/api/consult", { ...chart, requestId: crypto.randomUUID(), sessionId, modelId, consultationMode: "verified_chart", theme: "general", question: "今年工作整体顺不顺?", history: [], }, "consult"); await timed(jar, "/api/consult", { ...chart, requestId: crypto.randomUUID(), sessionId, modelId, consultationMode: "verified_chart", theme: "general", question: "那接下来几个月呢?", history: [], }, "consult"); if (index < 3) { const opened = await request(jar, "/api/rectification/cases/open", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ intent: "homepage", requestId: crypto.randomUUID() }), }); const openedBody = opened.ok ? await opened.json().catch(() => null) as { caseId?: string; sessionId?: string } | null : null; if (openedBody?.caseId && openedBody.sessionId) { for (const message of ["我小时候搬家过一次。", "那次大概是上小学之前。"]) { await timed(jar, "/api/rectification/agent", { caseId: openedBody.caseId, sessionId: openedBody.sessionId, requestId: crypto.randomUUID(), action: "message", message, }, "rectification"); } deleted.push(openedBody.sessionId); } else { samples.push({ kind: "rectification", status: opened.status, firstByteMs: null, totalMs: 0, failed: true, http429: opened.status === 429, http5xx: opened.status >= 500 || opened.status === 0, }); } } const removed = await request(jar, `/api/sessions/${sessionId}`, { method: "DELETE" }); if (removed.ok) deleted.push(sessionId); return deleted; } const startedAt = Date.now(); const cleaned = (await Promise.all(Array.from({ length: requestedUsers }, (_, index) => oneUser(index)))).flat(); const consults = samples.filter((sample) => sample.kind === "consult"); const finalFailures = consults.filter((sample) => sample.failed || sample.http429 || sample.http5xx); const firstBytes = consults.map((sample) => sample.firstByteMs).filter((value): value is number => value !== null); const totals = consults.map((sample) => sample.totalMs); const summary = { users: requestedUsers, host, requests: samples.length, consults: consults.length, http429: samples.filter((sample) => sample.http429).length, http5xx: samples.filter((sample) => sample.http5xx).length, finalFailures: finalFailures.length, finalFailureRatio: consults.length === 0 ? null : Number((finalFailures.length / consults.length).toFixed(4)), firstByteP95Ms: percentile(firstBytes, 0.95), totalP95Ms: percentile(totals, 0.95), sessionsDeleted: cleaned.length, serverSample: "not taken; this process has no staging docker stats", elapsedMs: Date.now() - startedAt, }; console.log(JSON.stringify(summary));