Answer step defaults to low reasoning effort (issue #134)
Product chose A after the A/B: low cut answer-step reasoning tokens ~45% and answer time 34s -> 22s with no quality loss. Unset now means "low"; CONSULT_ANSWER_REASONING_EFFORT=off restores the pre-change request body, and the old "unset keeps the body" guarantee moves to "off". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
537beb2934
commit
435c8e19f9
@@ -177,22 +177,26 @@ export const MAX_EVIDENCE_LOOKUPS_PER_TURN = 3;
|
||||
export const CONSULTATION_WINDOW_CALC_TOOL_ID = "run-jyotish-window-consultation";
|
||||
|
||||
/**
|
||||
* Answer-step reasoning effort. Unset or blank keeps today's request body.
|
||||
* An illegal value warns once and is treated as unset.
|
||||
* Answer-step reasoning effort. Unset or blank means "low" (issue #134: the
|
||||
* A/B halved reasoning tokens and cut answer time ~37% with no quality loss).
|
||||
* "off" sends no reasoning_effort, i.e. the pre-#134 request body.
|
||||
* An illegal value warns once and is treated as "off".
|
||||
*/
|
||||
const CONSULT_ANSWER_REASONING_EFFORTS = ["low", "medium", "high", "max"] as const;
|
||||
export const CONSULT_ANSWER_REASONING_EFFORT_DEFAULT: ProviderReasoningEffort = "low";
|
||||
let consultAnswerReasoningEffortWarned = false;
|
||||
|
||||
export function consultAnswerReasoningEffort(): ProviderReasoningEffort | undefined {
|
||||
const raw = process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
if (raw === undefined || raw.trim() === "") return undefined;
|
||||
if (raw === undefined || raw.trim() === "") return CONSULT_ANSWER_REASONING_EFFORT_DEFAULT;
|
||||
const value = raw.trim();
|
||||
if (value === "off") return undefined;
|
||||
if ((CONSULT_ANSWER_REASONING_EFFORTS as readonly string[]).includes(value)) {
|
||||
return value as ProviderReasoningEffort;
|
||||
}
|
||||
if (!consultAnswerReasoningEffortWarned) {
|
||||
consultAnswerReasoningEffortWarned = true;
|
||||
console.warn(`CONSULT_ANSWER_REASONING_EFFORT=${value} is not low, medium, high, or max; leaving reasoning effort unset.`);
|
||||
console.warn(`CONSULT_ANSWER_REASONING_EFFORT=${value} is not off, low, medium, high, or max; leaving reasoning effort unset.`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -51,9 +51,12 @@ test("an illegal effort warns once and stays unset", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("unset effort keeps the pre-change request bodies", async () => {
|
||||
// 原值: 不设环境变量时请求体与改动前逐字节相同
|
||||
// 新值: 设成 "off" 时请求体与改动前逐字节相同;不设时默认 "low"(见下一条)
|
||||
// 原因: issue #134 产品选 A,答题步默认低思考量;"off" 是一行回退
|
||||
test("off keeps the pre-change request bodies", async () => {
|
||||
const previous = process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
process.env.CONSULT_ANSWER_REASONING_EFFORT = "off";
|
||||
try {
|
||||
const natal = await captureStreamBodies({
|
||||
settings: consultationGenerationSettings(MODEL),
|
||||
@@ -76,6 +79,43 @@ test("unset effort keeps the pre-change request bodies", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("unset or blank effort defaults to low", () => {
|
||||
const previous = process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
try {
|
||||
delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
assert.equal(consultAnswerReasoningEffort(), "low");
|
||||
process.env.CONSULT_ANSWER_REASONING_EFFORT = " ";
|
||||
assert.equal(consultAnswerReasoningEffort(), "low");
|
||||
process.env.CONSULT_ANSWER_REASONING_EFFORT = "off";
|
||||
assert.equal(consultAnswerReasoningEffort(), undefined);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
else process.env.CONSULT_ANSWER_REASONING_EFFORT = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test("unset effort puts low on the answer step only", async () => {
|
||||
const previous = process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
try {
|
||||
const natal = await captureStreamBodies({
|
||||
settings: consultationGenerationSettings(MODEL),
|
||||
prepareStep: (input) => consultationNatalPrepareStep({
|
||||
stepNumber: input.stepNumber,
|
||||
answerReady: input.stepNumber >= 1,
|
||||
model: MODEL,
|
||||
}),
|
||||
steps: 2,
|
||||
}) as Record<string, unknown>[];
|
||||
assert.deepEqual(natal[0], fixture.natal[0]);
|
||||
assert.equal(natal[1]?.reasoning_effort, "low");
|
||||
assert.deepEqual(withoutEffort(natal[1]), fixture.natal[1]);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
else process.env.CONSULT_ANSWER_REASONING_EFFORT = previous;
|
||||
}
|
||||
});
|
||||
|
||||
test("low effort is only on the answer step", async () => {
|
||||
const previous = process.env.CONSULT_ANSWER_REASONING_EFFORT;
|
||||
process.env.CONSULT_ANSWER_REASONING_EFFORT = "low";
|
||||
|
||||
Reference in New Issue
Block a user