feat(consult): add an answer-step reasoning effort switch
The switch is off unless CONSULT_ANSWER_REASONING_EFFORT is set, so today's request body stays the same. A set value is sent as top-level reasoning_effort only on the answer step. The real A/B did not run: there is no model key. Request-body probe, the named frontend tests (114 pass, 0 fail), tsc, and lint (0 errors) were run. Co-Authored-By: Grok (agent loop) Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jesse-ux
co-authored by
Grok (agent loop)
Cursor
Grok (agent loop)
Cursor
parent
9274491681
commit
ab7f66be14
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Request-body probe for issue #65.
|
||||
*
|
||||
* --dry-run needs no model key. It sends one agent.stream per shape through
|
||||
* Mastra's OpenAI-compatible channel and prints the thinking / reasoning_effort
|
||||
* / max_tokens fields of each request. A fake fetch never leaves the machine.
|
||||
*
|
||||
* Shapes:
|
||||
* (a) consultationGenerationSettings (today's answer settings)
|
||||
* (b) consultationSliceGenerationSettings (nested reasoningEffort, no caller)
|
||||
* (c) providerReasoningEffort low / medium / high / max at the top level
|
||||
*
|
||||
* With DS_KEY or BACKTEST_MODEL_KEY, the same file can ask the provider which
|
||||
* efforts it accepts. This checkout does not run that half.
|
||||
*/
|
||||
import { writeFileSync } from "node:fs";
|
||||
|
||||
import { Agent } from "@mastra/core/agent";
|
||||
import { createTool } from "@mastra/core/tools";
|
||||
import { z } from "zod";
|
||||
|
||||
import { agentGenerationSettings } from "../../src/lib/agent-generation-settings.ts";
|
||||
import {
|
||||
consultationContinueGenerationSettings,
|
||||
consultationGenerationSettings,
|
||||
consultationNatalPrepareStep,
|
||||
consultationSliceGenerationSettings,
|
||||
} from "../../src/mastra/consultation-tools.ts";
|
||||
|
||||
const MODEL = {
|
||||
providerId: "deepseek",
|
||||
modelId: "deepseek-flash",
|
||||
url: "http://127.0.0.1:9/v1",
|
||||
apiKey: "dummy",
|
||||
};
|
||||
|
||||
const markTool = createTool({
|
||||
id: "run-jyotish-consultation",
|
||||
description: "Mark the calculation done.",
|
||||
inputSchema: z.object({}).strict(),
|
||||
execute: async () => ({ ok: true }),
|
||||
});
|
||||
|
||||
function sse(chunks: unknown[]) {
|
||||
const body = `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("")}data: [DONE]\n\n`;
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
});
|
||||
}
|
||||
|
||||
function toolCallChunk() {
|
||||
return sse([
|
||||
{
|
||||
id: "chatcmpl-tool",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{
|
||||
index: 0,
|
||||
delta: {
|
||||
role: "assistant",
|
||||
tool_calls: [{
|
||||
index: 0,
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: { name: "run-jyotish-consultation", arguments: "" },
|
||||
}],
|
||||
},
|
||||
finish_reason: null,
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-tool",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{
|
||||
index: 0,
|
||||
delta: { tool_calls: [{ index: 0, function: { arguments: "{}" } }] },
|
||||
finish_reason: null,
|
||||
}],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-tool",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
function textChunk() {
|
||||
return sse([
|
||||
{
|
||||
id: "chatcmpl-text",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: { role: "assistant", content: "ok" }, finish_reason: null }],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-text",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, completion_tokens_details: { reasoning_tokens: 0 } },
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
export function requestFields(body: unknown) {
|
||||
const record = body && typeof body === "object" ? body as Record<string, unknown> : {};
|
||||
return {
|
||||
thinking: record.thinking ?? null,
|
||||
reasoning_effort: record.reasoning_effort ?? null,
|
||||
max_tokens: record.max_tokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function captureStreamBodies(input: {
|
||||
settings: ReturnType<typeof consultationGenerationSettings>;
|
||||
prepareStep?: (step: { stepNumber: number }) => { toolChoice?: "auto"; activeTools?: string[]; providerOptions?: unknown };
|
||||
steps: 1 | 2;
|
||||
}): Promise<unknown[]> {
|
||||
const bodies: unknown[] = [];
|
||||
const previous = globalThis.fetch;
|
||||
globalThis.fetch = async (_url, init) => {
|
||||
const raw = init?.body;
|
||||
bodies.push(JSON.parse(typeof raw === "string" ? raw : "{}"));
|
||||
return bodies.length === 1 && input.steps === 2 ? toolCallChunk() : textChunk();
|
||||
};
|
||||
try {
|
||||
const agent = new Agent({
|
||||
id: `answer-effort-probe-${bodies.length}-${Date.now()}`,
|
||||
name: "answer-effort-probe",
|
||||
instructions: "Reply ok.",
|
||||
model: MODEL,
|
||||
tools: { "run-jyotish-consultation": markTool },
|
||||
});
|
||||
const result = await agent.stream([{ role: "user", content: "ping" }], {
|
||||
maxSteps: input.steps,
|
||||
...input.settings,
|
||||
...(input.prepareStep ? { prepareStep: input.prepareStep } : {}),
|
||||
} as never);
|
||||
const drained: unknown[] = [];
|
||||
for await (const chunk of result.fullStream) drained.push(chunk);
|
||||
if (drained.length === 0) throw new Error("probe stream produced no chunks");
|
||||
} finally {
|
||||
globalThis.fetch = previous;
|
||||
}
|
||||
return bodies;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!process.argv.includes("--dry-run")) {
|
||||
console.log("A/B 未跑：缺模型 key。只做请求体拦截时加上 --dry-run。");
|
||||
return;
|
||||
}
|
||||
const model = MODEL;
|
||||
const natal = await captureStreamBodies({
|
||||
settings: consultationGenerationSettings(model),
|
||||
prepareStep: (input) => consultationNatalPrepareStep({
|
||||
stepNumber: input.stepNumber,
|
||||
answerReady: input.stepNumber >= 1,
|
||||
}),
|
||||
steps: 2,
|
||||
});
|
||||
const slice = await captureStreamBodies({
|
||||
settings: consultationSliceGenerationSettings(model),
|
||||
steps: 1,
|
||||
});
|
||||
const topLevel = [];
|
||||
for (const effort of ["low", "medium", "high", "max"] as const) {
|
||||
const bodies = await captureStreamBodies({
|
||||
settings: agentGenerationSettings(model, { thinking: "enabled", providerReasoningEffort: effort }),
|
||||
steps: 1,
|
||||
});
|
||||
topLevel.push({ effort, fields: requestFields(bodies[0]) });
|
||||
}
|
||||
const continued = await captureStreamBodies({
|
||||
settings: consultationContinueGenerationSettings(model),
|
||||
steps: 1,
|
||||
});
|
||||
const report = {
|
||||
fixtureProvenance: "frontend/scripts/research/answer-effort-probe.mts --dry-run",
|
||||
a_consultationGenerationSettings: natal.map(requestFields),
|
||||
b_consultationSliceGenerationSettings: slice.map(requestFields),
|
||||
c_providerReasoningEffort: topLevel,
|
||||
continue: continued.map(requestFields),
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (process.argv.includes("--save-baseline")) {
|
||||
const file = new URL("../../tests/fixtures/answer-effort-switch-20261013.json", import.meta.url);
|
||||
writeFileSync(file, `${JSON.stringify({
|
||||
fixtureProvenance: "node node_modules/tsx/dist/cli.mjs scripts/research/answer-effort-probe.mts --dry-run --save-baseline（开关尚未读取环境变量时的整份请求体）",
|
||||
natal: natal,
|
||||
continue: continued,
|
||||
}, null, 2)}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const invoked = process.argv[1]
|
||||
? new URL(`file:///${process.argv[1].replace(/\\/g, "/")}`).href === import.meta.url
|
||||
: false;
|
||||
if (invoked) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -27,6 +27,7 @@
|
||||
* Usage (from frontend/, Node 22):
|
||||
* DS_KEY=... ./node_modules/.bin/tsx scripts/research/consult-biography-backtest.mts \
|
||||
* [--out <dir>] [--repeats 2] [--figures a,b] [--domains parents,marriage,health,career] \
|
||||
* [--cases steve_jobs:career,barack_obama:parents] [--efforts unset,low,medium] \
|
||||
* [--model deepseek-flash] [--base-url https://api.deepseek.com] [--concurrency 4] [--dump-cards]
|
||||
* [--correction barack_obama,marilyn_monroe] [--correction-text 我爸其实没怎么管过我]
|
||||
* [--correction-domain career] [--correction-texts "zinedine_zidane=我是足球运动员|barack_obama=我是政治家"]
|
||||
@@ -142,6 +143,8 @@ function parseArgs(argv: readonly string[]) {
|
||||
const at = pair.indexOf("=");
|
||||
return [pair.slice(0, at), pair.slice(at + 1)];
|
||||
})) as Record<string, string>,
|
||||
cases: value("cases")?.split(",").map((item) => item.trim()).filter(Boolean) ?? null,
|
||||
efforts: value("efforts")?.split(",").map((item) => item.trim()).filter(Boolean) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -270,7 +273,15 @@ function model(args: ReturnType<typeof parseArgs>, apiKey: string): ResolvedLang
|
||||
|
||||
/** A follow-up turn in the same session: the earlier question and answer are the history. */
|
||||
type FollowUp = { priorQuestion: string; priorAnswer: string };
|
||||
type Job = { figure: GoldenFigure; domain: string; question: string; run: number; followUp?: FollowUp };
|
||||
type Job = {
|
||||
figure: GoldenFigure;
|
||||
domain: string;
|
||||
question: string;
|
||||
run: number;
|
||||
followUp?: FollowUp;
|
||||
/** When set, this job overrides CONSULT_ANSWER_REASONING_EFFORT for its own stream. */
|
||||
effort?: string;
|
||||
};
|
||||
|
||||
type Usage = { inputTokens?: number; outputTokens?: number; reasoningTokens?: number; cachedInputTokens?: number };
|
||||
|
||||
@@ -316,6 +327,22 @@ function userTurn(question: string, followUp?: FollowUp) {
|
||||
}
|
||||
|
||||
async function runJob(job: Job, resolved: ResolvedLanguageModel) {
|
||||
const previousEffort = process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
if (job.effort !== undefined) {
|
||||
if (job.effort === "unset" || job.effort === "") delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
else process.env.CONSULT_ANSWER_REASONING_EFFORT = job.effort;
|
||||
}
|
||||
try {
|
||||
return await runJobOnce(job, resolved);
|
||||
} finally {
|
||||
if (job.effort !== undefined) {
|
||||
if (previousEffort === undefined) delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
else process.env.CONSULT_ANSWER_REASONING_EFFORT = previousEffort;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runJobOnce(job: Job, resolved: ResolvedLanguageModel) {
|
||||
const state = createConsultationRuntimeState();
|
||||
const executed: string[] = [];
|
||||
const ctx = agentContext(job, state, executed);
|
||||
@@ -333,7 +360,14 @@ async function runJob(job: Job, resolved: ResolvedLanguageModel) {
|
||||
maxSteps: AGENT_MAX_STEPS,
|
||||
...consultationGenerationSettings(resolved.model),
|
||||
};
|
||||
const natalStreamOptions = { ...streamOptions, prepareStep: consultationNatalPrepareStep };
|
||||
const natalStreamOptions = {
|
||||
...streamOptions,
|
||||
prepareStep: (input: { stepNumber: number }) => consultationNatalPrepareStep({
|
||||
stepNumber: input.stepNumber,
|
||||
answerReady: state.consultationToolCompleted,
|
||||
model: resolved.model,
|
||||
}),
|
||||
};
|
||||
const usages: Promise<unknown>[] = [];
|
||||
const startedAt = Date.now();
|
||||
let output: string | null = null;
|
||||
@@ -417,6 +451,17 @@ async function runJob(job: Job, resolved: ResolvedLanguageModel) {
|
||||
methodologySections: state.methodologySectionCount,
|
||||
lookups: state.evidenceLookupCallCount,
|
||||
finish: state.composeFinishReason ?? state.modelFinishReason ?? null,
|
||||
effort: process.env.CONSULT_ANSWER_REASONING_EFFORT?.trim() || null,
|
||||
answerReasoningMs: state.answerReasoningMs ?? null,
|
||||
steps: [0, 1].map((index) => {
|
||||
const step = state.modelStepTimings?.find((item) => item.index === index);
|
||||
return {
|
||||
index,
|
||||
seconds: step?.durationMs == null ? null : step.durationMs / 1000,
|
||||
reasoningTokens: step?.reasoningTokens ?? null,
|
||||
outputTokens: step?.outputTokens ?? null,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -477,7 +522,18 @@ console.info = (...items: unknown[]) => {
|
||||
productInfo(...items);
|
||||
};
|
||||
|
||||
type Row = { figure: string; domain: string; run: number; chars: number; seconds: number; checks: ReturnType<typeof deterministicChecks> };
|
||||
type Row = {
|
||||
figure: string;
|
||||
domain: string;
|
||||
run: number;
|
||||
chars: number;
|
||||
seconds: number;
|
||||
checks: ReturnType<typeof deterministicChecks>;
|
||||
empty?: boolean;
|
||||
truncated?: boolean;
|
||||
finish?: string | null;
|
||||
steps?: { seconds: number | null; reasoningTokens: number | null; outputTokens: number | null }[];
|
||||
};
|
||||
|
||||
function writeSummary(out: string, summary: Json & { rows: Row[] }, modelId: string) {
|
||||
const rows = summary.rows;
|
||||
@@ -488,13 +544,13 @@ function writeSummary(out: string, summary: Json & { rows: Row[] }, modelId: str
|
||||
writeFileSync(join(out, "summary.md"), [
|
||||
`# 名人生平回测 · ${modelId}`,
|
||||
"",
|
||||
`开始 ${summary.started_at}，${rows.length} 份，墙钟 ${Number(summary.wall_seconds).toFixed(0)} s，失败 ${summary.failures} 份。`,
|
||||
`开始 ${summary.started_at}，${rows.length} 份，墙钟 ${Number(summary.wall_seconds).toFixed(0)} s，失败 ${summary.failures} 份。推理强度：${String(summary.reasoning_effort ?? "不设")}。`,
|
||||
`用量合计：${JSON.stringify(summary.usage)}`,
|
||||
`确定性检查：禁句 ${summary.banned_hits}，性别代词（婚恋/子女）${summary.pronoun_hits}，must_not 原文 ${summary.must_not_literal_hits}。判断类评分需逐份阅读。`,
|
||||
"",
|
||||
"| 名人 | 领域 | run | 字数 | 秒 | 禁句 | 代词 | must_not |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
...rows.map((row) => `| ${row.figure} | ${row.domain} | ${row.run} | ${row.chars} | ${row.seconds.toFixed(0)} | ${row.checks.banned.length} | ${row.checks.pronouns.length} | ${row.checks.literalHits.length} |`),
|
||||
"| 名人 | 领域 | run | 字数 | 秒 | 空答 | 截断 | 第0步秒 | 第1步秒 | 第1步推理token | 结束 |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
...rows.map((row) => `| ${row.figure} | ${row.domain} | ${row.run} | ${row.chars} | ${row.seconds.toFixed(0)} | ${row.empty ? "是" : ""} | ${row.truncated ? "是" : ""} | ${row.steps?.[0]?.seconds ?? ""} | ${row.steps?.[1]?.seconds ?? ""} | ${row.steps?.[1]?.reasoningTokens ?? ""} | ${row.finish ?? ""} |`),
|
||||
"",
|
||||
"## must_not 原文命中",
|
||||
"",
|
||||
@@ -529,11 +585,28 @@ async function main() {
|
||||
}
|
||||
const questions = new Map(golden.routes.map((route) => [route.domain, route.question]));
|
||||
const figures = golden.figures.filter((figure) => !args.figures || args.figures.includes(figure.id));
|
||||
const jobs: Job[] = figures.flatMap((figure) => args.domains.flatMap((domain) => {
|
||||
const question = questions.get(domain);
|
||||
if (!question) throw new Error(`no golden question for domain ${domain}`);
|
||||
return Array.from({ length: args.repeats }, (_, index) => ({ figure, domain, question, run: index + 1 }));
|
||||
}));
|
||||
const pairs = args.cases
|
||||
? args.cases.map((spec) => {
|
||||
const splitAt = spec.indexOf(":");
|
||||
const figureId = spec.slice(0, splitAt);
|
||||
const domain = spec.slice(splitAt + 1);
|
||||
const figure = golden.figures.find((item) => item.id === figureId);
|
||||
const question = questions.get(domain);
|
||||
if (!figure || !question) throw new Error(`unknown case ${spec}`);
|
||||
return { figure, domain, question };
|
||||
})
|
||||
: figures.flatMap((figure) => args.domains.flatMap((domain) => {
|
||||
const question = questions.get(domain);
|
||||
if (!question) throw new Error(`no golden question for domain ${domain}`);
|
||||
return [{ figure, domain, question }];
|
||||
}));
|
||||
const efforts = args.efforts;
|
||||
const jobs: Job[] = pairs.flatMap((pair) => Array.from({ length: args.repeats }, (_, index) => {
|
||||
const run = index + 1;
|
||||
if (!efforts) return [{ ...pair, run }];
|
||||
return efforts.map((effort) => ({ ...pair, run, effort }));
|
||||
}).flat());
|
||||
const concurrency = efforts && efforts.length > 1 ? 1 : args.concurrency;
|
||||
mkdirSync(args.out, { recursive: true });
|
||||
if (args.dumpCards) {
|
||||
await dumpCards(jobs, args.out);
|
||||
@@ -543,12 +616,12 @@ async function main() {
|
||||
if (!apiKey) throw new Error("Set DS_KEY (or BACKTEST_MODEL_KEY); without a key the backtest is an environment gap.");
|
||||
const resolved = model(args, apiKey);
|
||||
const started = Date.now();
|
||||
const rows = await pool(jobs, args.concurrency, async (job) => {
|
||||
const rows = await pool(jobs, concurrency, async (job) => {
|
||||
let result: Awaited<ReturnType<typeof runJob>>;
|
||||
try {
|
||||
result = await runJob(job, resolved);
|
||||
} catch (error) {
|
||||
result = { output: "", failure: error instanceof Error ? error.message : String(error), executed: [], seconds: 0, usage: {}, cardChars: null, visibleChars: null, methodologySections: 0, lookups: 0, finish: null };
|
||||
result = { output: "", failure: error instanceof Error ? error.message : String(error), executed: [], seconds: 0, usage: {}, cardChars: null, visibleChars: null, methodologySections: 0, lookups: 0, finish: null, effort: job.effort ?? null, answerReasoningMs: null, steps: [{ index: 0, seconds: null, reasoningTokens: null, outputTokens: null }, { index: 1, seconds: null, reasoningTokens: null, outputTokens: null }] };
|
||||
}
|
||||
const mustNot = rubric.figures.find((item) => item.id === job.figure.id)?.domains[job.domain]?.must_not ?? [];
|
||||
const checks = deterministicChecks(result.output, job.domain, mustNot);
|
||||
@@ -573,7 +646,18 @@ async function main() {
|
||||
"",
|
||||
].join("\n"));
|
||||
console.log(`${job.figure.id} ${job.domain} run${job.run}: ${result.output.length} chars, ${result.seconds.toFixed(0)} s${result.failure ? `, FAILED ${result.failure}` : ""}`);
|
||||
return { figure: job.figure.id, domain: job.domain, run: job.run, chars: result.output.length, ...result, output: undefined, checks };
|
||||
return {
|
||||
figure: job.figure.id,
|
||||
domain: job.domain,
|
||||
run: job.run,
|
||||
chars: result.output.length,
|
||||
empty: result.output.trim() === "",
|
||||
truncated: result.finish === "length" || (result.failure ?? "").includes("answer_truncated"),
|
||||
outputTokens: result.usage.outputTokens ?? null,
|
||||
...result,
|
||||
output: undefined,
|
||||
checks,
|
||||
};
|
||||
});
|
||||
const corrections = await pool(
|
||||
jobs.filter((job) => args.correctionDomains.includes(job.domain)
|
||||
@@ -589,7 +673,7 @@ async function main() {
|
||||
try {
|
||||
result = await runJob(job, resolved);
|
||||
} catch (error) {
|
||||
result = { output: "", failure: error instanceof Error ? error.message : String(error), executed: [], seconds: 0, usage: {}, cardChars: null, visibleChars: null, methodologySections: 0, lookups: 0, finish: null };
|
||||
result = { output: "", failure: error instanceof Error ? error.message : String(error), executed: [], seconds: 0, usage: {}, cardChars: null, visibleChars: null, methodologySections: 0, lookups: 0, finish: null, effort: job.effort ?? null, answerReasoningMs: null, steps: [{ index: 0, seconds: null, reasoningTokens: null, outputTokens: null }, { index: 1, seconds: null, reasoningTokens: null, outputTokens: null }] };
|
||||
}
|
||||
const file = join(args.out, job.figure.id, `${job.domain}-correction-run${job.run}.md`);
|
||||
writeFileSync(file, [
|
||||
@@ -619,6 +703,9 @@ async function main() {
|
||||
jobs: rows.length,
|
||||
failures: rows.filter((row) => row.failure || row.chars === 0).length,
|
||||
usage: total,
|
||||
reasoning_effort: process.env.CONSULT_ANSWER_REASONING_EFFORT?.trim() || null,
|
||||
efforts: args.efforts,
|
||||
concurrency,
|
||||
gaps: GAPS,
|
||||
rows,
|
||||
corrections,
|
||||
|
||||
Reference in New Issue
Block a user