fix(consult): move the answer-effort probe out of the image build
The dry-run script lived under frontend/scripts and read a test fixture, so the web image build would type-check a file the image does not copy. Move it to frontend/tests with a non-test filename. The switch default is unchanged. Checked: guard plus answer-effort tests 5 pass, 0 fail, 0 cancelled; probe --dry-run exit 0; tsc exit 0; lint 0 errors and 123 warnings. Co-Authored-By: Grok (agent loop)
This commit is contained in:
jesse-ux
co-authored by
Grok (agent loop)
Grok (agent loop)
parent
0d73cd332b
commit
9324253408
@@ -1,203 +0,0 @@
|
||||
/**
|
||||
* Request-body probe for issue #65.
|
||||
*
|
||||
* --dry-run needs no model key. It sends one agent.stream per shape through
|
||||
* Mastra's OpenAI-compatible channel and prints the thinking / reasoning_effort
|
||||
* / max_tokens fields of each request. A fake fetch never leaves the machine.
|
||||
*
|
||||
* Shapes:
|
||||
* (a) consultationGenerationSettings (today's answer settings)
|
||||
* (b) consultationSliceGenerationSettings (nested reasoningEffort, no caller)
|
||||
* (c) providerReasoningEffort low / medium / high / max at the top level
|
||||
*
|
||||
* With DS_KEY or BACKTEST_MODEL_KEY, the same file can ask the provider which
|
||||
* efforts it accepts. This checkout does not run that half.
|
||||
*/
|
||||
import { writeFileSync } from "node:fs";
|
||||
|
||||
import { Agent } from "@mastra/core/agent";
|
||||
import { createTool } from "@mastra/core/tools";
|
||||
import { z } from "zod";
|
||||
|
||||
import { agentGenerationSettings } from "../../src/lib/agent-generation-settings.ts";
|
||||
import {
|
||||
consultationContinueGenerationSettings,
|
||||
consultationGenerationSettings,
|
||||
consultationNatalPrepareStep,
|
||||
consultationSliceGenerationSettings,
|
||||
} from "../../src/mastra/consultation-tools.ts";
|
||||
|
||||
const MODEL = {
|
||||
providerId: "deepseek",
|
||||
modelId: "deepseek-flash",
|
||||
url: "http://127.0.0.1:9/v1",
|
||||
apiKey: "dummy",
|
||||
};
|
||||
|
||||
const markTool = createTool({
|
||||
id: "run-jyotish-consultation",
|
||||
description: "Mark the calculation done.",
|
||||
inputSchema: z.object({}).strict(),
|
||||
execute: async () => ({ ok: true }),
|
||||
});
|
||||
|
||||
function sse(chunks: unknown[]) {
|
||||
const body = `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")}data: [DONE]\n\n`;
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}
|
||||
|
||||
function toolCallChunk() {
|
||||
return sse([
|
||||
{
|
||||
id: "chatcmpl-tool",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{
|
||||
index: 0,
|
||||
delta: {
|
||||
role: "assistant",
|
||||
tool_calls: [{
|
||||
index: 0,
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: { name: "run-jyotish-consultation", arguments: "" },
|
||||
}],
|
||||
},
|
||||
finish_reason: null,
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-tool",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{
|
||||
index: 0,
|
||||
delta: { tool_calls: [{ index: 0, function: { arguments: "{}" } }] },
|
||||
finish_reason: null,
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-tool",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
function textChunk() {
|
||||
return sse([
|
||||
{
|
||||
id: "chatcmpl-text",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: { role: "assistant", content: "ok" }, finish_reason: null }],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-text",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, completion_tokens_details: { reasoning_tokens: 0 } },
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
export function requestFields(body: unknown) {
|
||||
const record = body && typeof body === "object" ? body as Record<string, unknown> : {};
|
||||
return {
|
||||
thinking: record.thinking ?? null,
|
||||
reasoning_effort: record.reasoning_effort ?? null,
|
||||
max_tokens: record.max_tokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function captureStreamBodies(input: {
|
||||
settings: ReturnType<typeof consultationGenerationSettings>;
|
||||
prepareStep?: (step: { stepNumber: number }) => { toolChoice?: "auto"; activeTools?: string[]; providerOptions?: unknown };
|
||||
steps: 1 | 2;
|
||||
}): Promise<unknown[]> {
|
||||
const bodies: unknown[] = [];
|
||||
const previous = globalThis.fetch;
|
||||
globalThis.fetch = async (_url, init) => {
|
||||
const raw = init?.body;
|
||||
bodies.push(JSON.parse(typeof raw === "string" ? raw : "{}"));
|
||||
return bodies.length === 1 && input.steps === 2 ? toolCallChunk() : textChunk();
|
||||
};
|
||||
try {
|
||||
const agent = new Agent({
|
||||
id: `answer-effort-probe-${bodies.length}-${Date.now()}`,
|
||||
name: "answer-effort-probe",
|
||||
instructions: "Reply ok.",
|
||||
model: MODEL,
|
||||
tools: { "run-jyotish-consultation": markTool },
|
||||
});
|
||||
const result = await agent.stream([{ role: "user", content: "ping" }], {
|
||||
maxSteps: input.steps,
|
||||
...input.settings,
|
||||
...(input.prepareStep ? { prepareStep: input.prepareStep } : {}),
|
||||
} as never);
|
||||
const drained: unknown[] = [];
|
||||
for await (const chunk of result.fullStream) drained.push(chunk);
|
||||
if (drained.length === 0) throw new Error("probe stream produced no chunks");
|
||||
} finally {
|
||||
globalThis.fetch = previous;
|
||||
}
|
||||
return bodies;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!process.argv.includes("--dry-run")) {
|
||||
console.log("A/B 未跑：缺模型 key。只做请求体拦截时加上 --dry-run。");
|
||||
return;
|
||||
}
|
||||
const model = MODEL;
|
||||
const natal = await captureStreamBodies({
|
||||
settings: consultationGenerationSettings(model),
|
||||
prepareStep: (input) => consultationNatalPrepareStep({
|
||||
stepNumber: input.stepNumber,
|
||||
answerReady: input.stepNumber >= 1,
|
||||
}),
|
||||
steps: 2,
|
||||
});
|
||||
const slice = await captureStreamBodies({
|
||||
settings: consultationSliceGenerationSettings(model),
|
||||
steps: 1,
|
||||
});
|
||||
const topLevel = [];
|
||||
for (const effort of ["low", "medium", "high", "max"] as const) {
|
||||
const bodies = await captureStreamBodies({
|
||||
settings: agentGenerationSettings(model, { thinking: "enabled", providerReasoningEffort: effort }),
|
||||
steps: 1,
|
||||
});
|
||||
topLevel.push({ effort, fields: requestFields(bodies[0]) });
|
||||
}
|
||||
const continued = await captureStreamBodies({
|
||||
settings: consultationContinueGenerationSettings(model),
|
||||
steps: 1,
|
||||
});
|
||||
const report = {
|
||||
fixtureProvenance: "frontend/scripts/research/answer-effort-probe.mts --dry-run",
|
||||
a_consultationGenerationSettings: natal.map(requestFields),
|
||||
b_consultationSliceGenerationSettings: slice.map(requestFields),
|
||||
c_providerReasoningEffort: topLevel,
|
||||
continue: continued.map(requestFields),
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (process.argv.includes("--save-baseline")) {
|
||||
const file = new URL("../../tests/fixtures/answer-effort-switch-20261013.json", import.meta.url);
|
||||
writeFileSync(file, `${JSON.stringify({
|
||||
fixtureProvenance: "node node_modules/tsx/dist/cli.mjs scripts/research/answer-effort-probe.mts --dry-run --save-baseline（开关尚未读取环境变量时的整份请求体）",
|
||||
natal: natal,
|
||||
continue: continued,
|
||||
}, null, 2)}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const invoked = process.argv[1]
|
||||
? new URL(`file:///${process.argv[1].replace(/\\/g, "/")}`).href === import.meta.url
|
||||
: false;
|
||||
if (invoked) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user