/** * Sanitize text that may appear on the public thinking channel. * * English-only process narration, tool ids and UUIDs stay off the client. * Chinese thinking fragments are allowed through a dedicated event type, * never through the spoken answer. * * Filtering is sentence-buffered: stream chunks are often one or two English * words, so a per-chunk "drop long Latin, keep the rest" pass turns a withheld * sentence into word salad. Wait for `。!?\\n` or `.!?`, then decide. */ const CJK_RE = /[\u4e00-\u9fff]/; const PUBLIC_THINKING_UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}/gi; const PUBLIC_THINKING_TOOL_RE = /run-jyotish(?:-consultation)?-?|rectification-[a-z0-9-]+/gi; const SENTENCE_TERMINATOR_RE = /[。!?\n]|[.!?]/; function stripIds(text: string): string { return text .replace(PUBLIC_THINKING_UUID_RE, "") .replace(PUBLIC_THINKING_TOOL_RE, ""); } function sanitizeCompletedSentence(sentence: string): string | null { const cleaned = stripIds(sentence); if (!cleaned.trim()) return null; if (!CJK_RE.test(cleaned)) return null; const withoutEnglishWords = cleaned .replace(/[A-Za-z]{4,}/g, "") .replace(/[ \t]{2,}/g, " "); const trimmed = withoutEnglishWords.trim(); if (!trimmed) return null; return trimmed.slice(0, 4_000); } function terminatorEnd(buffer: string): number { const match = SENTENCE_TERMINATOR_RE.exec(buffer); if (!match || match.index === undefined) return -1; return match.index + match[0].length; } export function createPublicThinkingSanitizer() { let buffer = ""; function releaseCompleted(flush: boolean): string { let released = ""; while (buffer) { const end = terminatorEnd(buffer); if (end >= 0) { const sentence = buffer.slice(0, end); buffer = buffer.slice(end); const cjk = sentence.search(CJK_RE); const cleaned = sanitizeCompletedSentence(cjk > 0 ? sentence.slice(cjk) : sentence); if (cleaned) released += cleaned; continue; } const cjk = buffer.search(CJK_RE); if (cjk > 0) { buffer = buffer.slice(cjk); continue; } break; } if (flush && buffer) { const leftover = buffer; buffer = ""; const cleaned = sanitizeCompletedSentence(leftover); if (cleaned) released += cleaned; } return released; } return { push(chunk: string): string | null { if (!chunk) return null; buffer += chunk; const released = releaseCompleted(false); return released || null; }, flush(): string | null { const released = releaseCompleted(true); return released || null; }, }; } export function sanitizePublicThinkingText(text: string): string | null { const sanitizer = createPublicThinkingSanitizer(); const pushed = sanitizer.push(text) ?? ""; const flushed = sanitizer.flush() ?? ""; const combined = `${pushed}${flushed}`; return combined || null; }