From d6a3285fe8fa151af7a7e549bb3ec482b10f85a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 08:24:22 +0800 Subject: [PATCH] fix: the basis rewrite is a small standalone call (BUG-1948) The rewrite of flagged basis lines no longer resends the system prompt and the answer (about 73k characters). It sends one fixed instruction, the flagged lines and the meanings of the words found: under 1,000 characters. Usage is recorded with the run's; the user's charge is unchanged. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01DbBXoyn7Hj3LCW3BUs5bZ8 --- docs/BUG_HISTORY.md | 2 +- .../research/consult-biography-backtest.mts | 18 ++++---- frontend/src/app/api/consult/route.ts | 19 ++++----- .../lib/consultation-internal-word-rewrite.ts | 42 +++++++++++++++++++ .../src/lib/consultation-internal-words.ts | 32 +++++++++----- frontend/src/lib/stream-agent-response.ts | 2 +- .../tests/consult-internal-words-1948.test.ts | 19 ++++++++- 7 files changed, 101 insertions(+), 33 deletions(-) create mode 100644 frontend/src/lib/consultation-internal-word-rewrite.ts diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index fcc12a5a1..1658ff165 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -26212,7 +26212,7 @@ - 触发条件:任何带「判断依据」的聊天回答。 - 根因:取数卡的键名和状态值是英文标识符,其中一部分是占星师状态合同里的词,她的方法原文每轮也送给模型;回答写法里「不写内部字段」一句拦不住。改名牵涉系统提示、卡、事实复查和她的原文(见 `内部词清单` 盘点),所以改在回答写完之后处理。 - 修复:(一)新增 `frontend/src/lib/consultation-internal-words.ts`。回答写完、盘面事实复查之后,检查「判断依据」每一行;内部词按三条正面定义认定:小写加下划线的标识符、十二个单独出现在中文里的状态词、由这两类组成的「键 = 值」片段。技法名、分盘代号、星名、书名、英文引文不在其内。命中的行发回模型改写一次(沿用事实复查那一路的做法,只改这几行,要求数字、行首标题不变);改写不合格或没有改写时,按一张对照表换成产品里和她文本里已有的中文说法,没有对应说法、又不带信息的词连同它的括号或「键 =」一起去掉。正文里的内部词只记录、不改。任何出错都保留原回答,不因这项检查扣下或判失败。用户看到的顺序:回答照常流式写完;若命中,出现一行「正在把判断依据里的内部用词改成白话」,随后整段回答被替换一次(与事实复查、内容审核的替换方式相同);没命中就没有任何变化。(二)来源上三处:5 到 15 分钟精度的值给模型时写成「出生时间精度在 15 分钟以内」;Argala 补充段去掉英文结果词,只留中文裁决;系统提示和精简清单里四处直接写键名的句子改成「受冲依据那一句」的说法。 -- 验证与防复发:`frontend/tests/consult-internal-words-1948.test.ts` 七条;七处被钉住的断言按原值、新值、原因更新。离线对此前 20 个回答跑检测:12 个命中、21 行,只用对照表后剩 0 行;正文命中 0;没有把技法名、书名、英文引文认成内部词。实测 8 个回答(DeepSeek flash,两张公开盘各四题):改写前 5 个回答共 7 行带内部词,改写后 0 行,对照表用到 0 次;另外 3 个回答本来就没有。改写只动了命中的行,数字、书名、技法名都在;没有回答变差。代价:命中的回答多一次模型调用(8 个里 5 个),每次约七万字符的请求(系统提示和原回答重发一遍)。父母题在提示改口之后两答仍然两面都写。每格 1 个回答,不能排除随机波动。 +- 验证与防复发:`frontend/tests/consult-internal-words-1948.test.ts` 七条;七处被钉住的断言按原值、新值、原因更新。离线对此前 20 个回答跑检测:12 个命中、21 行,只用对照表后剩 0 行;正文命中 0;没有把技法名、书名、英文引文认成内部词。实测 8 个回答(DeepSeek flash,两张公开盘各四题):改写前 5 个回答共 7 行带内部词,改写后 0 行,对照表用到 0 次;另外 3 个回答本来就没有。改写只动了命中的行,数字、书名、技法名都在;没有回答变差。代价:命中的回答多一次模型调用(8 个里 5 个);第一版把系统提示和原回答重发一遍,每次约七万字符。同日改为独立的小调用(`consultation-internal-word-rewrite.ts`):只发一句固定指令、命中的几行和这几个词的中文意思,不带系统提示、取数卡和工具;实测 3 次调用的请求 678 到 958 字符,输入 360 到 494 token,输出 219 到 440 token(含推理)。4 个回答复测:3 个命中共 4 行,改写后剩 0 行,其中 1 行因内部词在行首加粗标题里、改写动了标题而改用对照表,结果正常。用量和事实复查的重写一样记进本次请求的用量;用户扣的点数是回答前按功能定价预留的固定值,不因这次调用增加。父母题在提示改口之后两答仍然两面都写。每格 1 个回答,不能排除随机波动。 - 关联记录:BUG-1945、BUG-1789、BUG-1763。 - 修复版本:`claude/basis-internal-words-1948`,未推送。 diff --git a/frontend/scripts/research/consult-biography-backtest.mts b/frontend/scripts/research/consult-biography-backtest.mts index b9af80628..54491b2d4 100644 --- a/frontend/scripts/research/consult-biography-backtest.mts +++ b/frontend/scripts/research/consult-biography-backtest.mts @@ -77,7 +77,8 @@ import { createConsultationPlan } from "../../src/lib/consultation-plan.ts"; import { consultationUserTurnContent } from "../../src/lib/consultation-session-history.ts"; import { consultationAnswerAcceptanceFailure, consultationContinueMessages, consultationMissingAuditTableMessages, natalUserTurnShape, type ConsultationAnswerMode } from "../../src/lib/consultation-thinking-plan.ts"; import { consultationFactRepairMessages, type FactGuardUnit } from "../../src/lib/consultation-answer-fact-guard.ts"; -import { internalWordRepairMessages, type InternalWordUnit } from "../../src/lib/consultation-internal-words.ts"; +import type { InternalWordUnit } from "../../src/lib/consultation-internal-words.ts"; +import { internalWordRewriteChunks, rewriteInternalWordLinesText } from "../../src/lib/consultation-internal-word-rewrite.ts"; import { streamAgentResponse } from "../../src/lib/stream-agent-response.ts"; import type { ConsultationDomain } from "../../src/lib/consultation-domain-registry.ts"; import type { ServerChartConsultation } from "../../src/lib/consultation-route-service.ts"; @@ -400,6 +401,7 @@ async function runJobOnce(job: Job, resolved: ResolvedLanguageModel) { // BUG-1789: the answer as written before the fact guard rewrote it (research record only). let preGuardOutput: string | null = null; let preInternalWordOutput: string | null = null; + let internalWordRewriteUsage: unknown = null; const first = await agent.stream(baseMessages, natalStreamOptions); usages.push(first.totalUsage); const response = streamAgentResponse({ @@ -455,15 +457,13 @@ async function runJobOnce(job: Job, resolved: ResolvedLanguageModel) { usages.push(repaired.totalUsage); return repaired.fullStream as never; }, - // route.ts rewriteInternalWordLines (BUG-1948). + // route.ts rewriteInternalWordLines (BUG-1948): the small standalone call. rewriteInternalWordLines: async (units: InternalWordUnit[], text: string) => { preInternalWordOutput = text; - const repaired = await agent.stream(internalWordRepairMessages(baseMessages, text, units), { - ...streamOptions, - ...consultationContinueGenerationSettings(resolved.model), - }); - usages.push(repaired.totalUsage); - return repaired.fullStream as never; + const rewritten = await rewriteInternalWordLinesText(resolved, units); + internalWordRewriteUsage = rewritten.usage; + usages.push(Promise.resolve(rewritten.usage as never)); + return internalWordRewriteChunks(rewritten.text) as never; }, // route.ts acceptAnswer (BUG-1763). acceptAnswer: (text: string) => consultationAnswerAcceptanceFailure({ output: text, mode: ANSWER_MODE, auditRows: state.techniqueAuditTable?.length ?? 0 }), @@ -518,7 +518,7 @@ async function runJobOnce(job: Job, resolved: ResolvedLanguageModel) { factGuard: state.factGuard ?? null, preGuardOutput: preGuardOutput as string | null, preInternalWordOutput: preInternalWordOutput as string | null, - internalWords: state.internalWords ?? null, + internalWords: state.internalWords ? { ...state.internalWords, rewriteUsage: internalWordRewriteUsage } : null, continuations: state.steps.filter((step) => step.name === "answer-continue" || step.name === "answer-missing-section").map((step) => step.name), effort: process.env.CONSULT_ANSWER_REASONING_EFFORT?.trim() || null, answerReasoningMs: state.answerReasoningMs ?? null, diff --git a/frontend/src/app/api/consult/route.ts b/frontend/src/app/api/consult/route.ts index ef0217ce7..d836ed4a6 100644 --- a/frontend/src/app/api/consult/route.ts +++ b/frontend/src/app/api/consult/route.ts @@ -66,7 +66,8 @@ import { streamSmalltalkResponse } from "@/lib/stream-smalltalk-response"; import { streamFirstResponse } from "@/lib/stream-first-response"; import { consultationPublicActivityEvent, streamAgentResponse } from "@/lib/stream-agent-response"; import { consultationFactRepairMessages, type FactGuardUnit } from "@/lib/consultation-answer-fact-guard"; -import { internalWordRepairMessages, type InternalWordUnit } from "@/lib/consultation-internal-words"; +import type { InternalWordUnit } from "@/lib/consultation-internal-words"; +import { internalWordRewriteChunks, rewriteInternalWordLinesText } from "@/lib/consultation-internal-word-rewrite"; import type { AgentExecutionReceipt, WorkflowReceipt } from "@/lib/consultation-agent-events"; import { consultationContinueMessages, @@ -1640,15 +1641,13 @@ export async function POST(request: Request) { return continued.fullStream; })(); }; - // BUG-1948: one rewrite of the 「判断依据」 lines that carry internal words. - const rewriteInternalWordLines = async (units: InternalWordUnit[], output: string) => { - const repaired = await agent.stream(internalWordRepairMessages(baseMessages, output, units), { - ...streamOptions, - abortSignal: answerPhaseSignal(), - ...consultationContinueGenerationSettings(selectedModel.model), - }); - usages.push(repaired.totalUsage); - return repaired.fullStream; + // BUG-1948: one rewrite of the 「判断依据」 lines that carry internal words: a small standalone + // call (no system prompt, card or tools). Its usage is recorded with the run's, like the fact + // guard's repair; the user's charge is the fixed credit cost reserved before the answer. + const rewriteInternalWordLines = async (units: InternalWordUnit[]) => { + const rewritten = await rewriteInternalWordLinesText(selectedModel, units, answerPhaseSignal()); + usages.push(Promise.resolve(rewritten.usage as Usage)); + return internalWordRewriteChunks(rewritten.text); }; // BUG-1789: one regeneration for the sentences the evidence card contradicts. const regenerateFactSentences = async (units: FactGuardUnit[], output: string) => { diff --git a/frontend/src/lib/consultation-internal-word-rewrite.ts b/frontend/src/lib/consultation-internal-word-rewrite.ts new file mode 100644 index 000000000..b5c940cff --- /dev/null +++ b/frontend/src/lib/consultation-internal-word-rewrite.ts @@ -0,0 +1,42 @@ +import { Agent } from "@mastra/core/agent"; + +import type { ResolvedLanguageModel } from "@/mastra/model"; + +import { internalWordRepairPrompt, type InternalWordUnit } from "./consultation-internal-words.ts"; + +/** + * BUG-1948: the one rewrite of 「判断依据」 lines that carry internal words, as a small + * standalone call. No system prompt of the consultation, no evidence card, no tools: a + * short fixed instruction, the flagged lines and the meanings of the words found in them. + * The first version resent the whole conversation (about 73k characters per call). + */ +export const INTERNAL_WORD_REWRITE_INSTRUCTIONS = + "你只做一件事:把给出的几行文字里的内部字段名和英文状态码改成平常的中文说法,其余一个字不改。只输出要求的 JSON。"; +export const INTERNAL_WORD_REWRITE_MAX_TOKENS = 1600; + +export async function rewriteInternalWordLinesText( + model: ResolvedLanguageModel, + units: readonly InternalWordUnit[], + signal?: AbortSignal, +): Promise<{ text: string; usage: Record }> { + const agent = new Agent({ + id: `internal-word-rewrite-${model.id}`, + name: "Internal Word Rewrite", + model: model.model, + instructions: INTERNAL_WORD_REWRITE_INSTRUCTIONS, + }); + const result = await agent.generate([{ role: "user", content: internalWordRepairPrompt(units) }], { + abortSignal: signal, + modelSettings: { maxOutputTokens: INTERNAL_WORD_REWRITE_MAX_TOKENS }, + }); + const usage = (result as { totalUsage?: unknown; usage?: unknown }).totalUsage ?? (result as { usage?: unknown }).usage; + return { + text: typeof result.text === "string" ? result.text : "", + usage: usage && typeof usage === "object" ? usage as Record : {}, + }; +} + +/** The rewrite's text as the one-chunk stream streamAgentResponse reads. */ +export async function* internalWordRewriteChunks(text: string) { + yield { type: "text-delta", payload: { text } }; +} diff --git a/frontend/src/lib/consultation-internal-words.ts b/frontend/src/lib/consultation-internal-words.ts index 44f9bf2ee..d0cf74a5b 100644 --- a/frontend/src/lib/consultation-internal-words.ts +++ b/frontend/src/lib/consultation-internal-words.ts @@ -114,26 +114,38 @@ export function scanInternalWords(answer: string): InternalWordScan { return { units, bodyTokens }; } +/** What each flagged word means, from the table; a word without a row carries nothing for the reader. */ +function wordMeanings(units: readonly InternalWordUnit[]): string[] { + const seen = new Set(); + const lines: string[] = []; + for (const token of units.flatMap((unit) => unit.tokens)) { + if (seen.has(token)) continue; + seen.add(token); + const word = INTERNAL_VALUE_WORDS[token] ?? INTERNAL_KEY_WORDS[token]; + lines.push(word ? `${token}:${word}` : `${token}:字段名,没有实际意思,连同它后面的等号或「为」一起去掉`); + } + return lines; +} + +/** + * The whole user turn of the standalone rewrite call: instruction, the meanings of the + * words found, and the flagged lines. Nothing else of the conversation is sent. + */ export function internalWordRepairPrompt(units: readonly InternalWordUnit[]): string { return [ - "「判断依据」里下面这几行带着内部字段名或英文状态码,读的人看不懂。把每一行改写一遍:", - "- 只把列出的那几个词换成平常的中文说法;这个词在句子里没有实际意思时,连同它所在的括号或半句一起去掉。", + "下面几行来自一份星盘解读的「判断依据」,里面带着内部字段名或英文状态码,读的人看不懂。把每一行改写一遍:", + "- 只把列出的那几个词换成给出的中文说法;标着没有实际意思的,连同它所在的空括号或半句一起去掉。", "- 其余内容一个字不改:星、星座、宫、度数、日期、分盘和技法的名称、书名、行尾的等级标注都照原样保留。", "- 每行仍是一行,行首的「- **……**」照原样保留;不新增判断,不新增依据。", "- 只输出 JSON 数组:[{\"id\": 1, \"text\": \"改写后的整行\"}],不要别的文字。", "", + "这些词的意思:", + ...wordMeanings(units), + "", ...units.map((unit) => `#${unit.id}\n要换掉的词:${unit.tokens.join("、")}\n这一行:${unit.text}`), ].join("\n"); } -export function internalWordRepairMessages(base: readonly M[], output: string, units: readonly InternalWordUnit[]) { - return [ - ...base, - { role: "assistant" as const, content: output }, - { role: "user" as const, content: internalWordRepairPrompt(units) }, - ]; -} - const numbers = (text: string) => text.match(/\d+(?:[.\-:]\d+)*/g) ?? []; /** Why a rewritten line is not taken, or null when it is. */ diff --git a/frontend/src/lib/stream-agent-response.ts b/frontend/src/lib/stream-agent-response.ts index da9410da9..3763aca96 100644 --- a/frontend/src/lib/stream-agent-response.ts +++ b/frontend/src/lib/stream-agent-response.ts @@ -380,7 +380,7 @@ type StreamAgentResponseOptions = EventOptions & { regenerateFactSentences?: (units: FactGuardUnit[], output: string) => Promise; /** * BUG-1948: the one rewrite of 「判断依据」 lines that carry internal field names or status - * identifiers (internalWordRepairMessages). Lines the rewrite does not clean are replaced + * identifiers: a small standalone call (rewriteInternalWordLinesText). Lines it does not clean are replaced * from the word table; absent, the table alone is used. Never fails the answer. */ rewriteInternalWordLines?: (units: InternalWordUnit[], output: string) => Promise; diff --git a/frontend/tests/consult-internal-words-1948.test.ts b/frontend/tests/consult-internal-words-1948.test.ts index 8a3d44492..4c635486d 100644 --- a/frontend/tests/consult-internal-words-1948.test.ts +++ b/frontend/tests/consult-internal-words-1948.test.ts @@ -13,6 +13,7 @@ import { scanInternalWords, scrubInternalWords, } from "../src/lib/consultation-internal-words.ts"; +import { INTERNAL_WORD_REWRITE_INSTRUCTIONS } from "../src/lib/consultation-internal-word-rewrite.ts"; import { FACT_STATUS_LABELS } from "../src/lib/report-fact-table-display.ts"; import { YOGA_STATUS_MAPPING } from "../src/lib/yoga-status-words.ts"; import { MODEL_PRECISION_ANNOTATED } from "../src/mastra/consultation-tools.ts"; @@ -98,12 +99,26 @@ test("the table uses existing wording, drops what carries nothing, and leaves re assert.equal(INTERNAL_VALUE_WORDS.parameter_sensitive, FACT_STATUS_LABELS.parameter_sensitive); }); -test("the rewrite prompt lists only the flagged lines and asks for nothing new", () => { +test("the rewrite is a small standalone call: instruction, meanings of the words found, the flagged lines", () => { const scan = scanInternalWords(ANSWER); const prompt = internalWordRepairPrompt(scan.units); assert.match(prompt, /要换掉的词:chat_use、single_system_candidate/); - assert.doesNotMatch(prompt, /能挣/); + assert.match(prompt, new RegExp(`parameter_sensitive:${FACT_STATUS_LABELS.parameter_sensitive}`)); + assert.match(prompt, /single_system_candidate:单体系候选/); + assert.match(prompt, /chat_use:字段名,没有实际意思/); + assert.doesNotMatch(prompt, /能挣/, "unflagged lines are not sent"); assert.match(prompt, /只输出 JSON 数组/); + // 原值: 改写请求 = 系统提示 + 用户回合 + 整份回答 + 改写要求(实测每次约 7.3 万字符) + // 新值: 只有一句固定指令和这一段提示;原因: 产品负责人 2026-11-17:改写的 token 花费太高 + assert.ok(prompt.length + INTERNAL_WORD_REWRITE_INSTRUCTIONS.length < 1500, String(prompt.length)); + const source = readFileSync(new URL("../src/lib/consultation-internal-word-rewrite.ts", import.meta.url), "utf8"); + assert.match(source, /instructions: INTERNAL_WORD_REWRITE_INSTRUCTIONS,/); + assert.doesNotMatch(source, /getJyotishAgent|baseMessages|createConsultationTools/); + const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8"); + const call = route.slice(route.indexOf("const rewriteInternalWordLines = async"), route.indexOf("// BUG-1789: one regeneration")); + assert.match(call, /rewriteInternalWordLinesText\(selectedModel, units, answerPhaseSignal\(\)\)/); + assert.match(call, /usages\.push\(/, "its usage is recorded with the run's, like the fact guard's repair"); + assert.doesNotMatch(call, /agent\.stream|baseMessages/); }); test("the check runs after the fact guard, replaces the shown answer once, and never fails the run", () => {