Jyotisha/frontend/scripts/home-streaming-render-benchmark.mts

import { createElement, createRef } from "react";
import { renderToString } from "react-dom/server";
import { writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";

import {
  LatestAssistantEntry,
  SettledMessageList,
  UnsplitChatTranscript,
  type ChatTranscriptActions,
  type ChatTranscriptProps,
} from "../src/components/chat-transcript.tsx";
import type { ChatMessage } from "../src/lib/chat-message-view.ts";
import { settledChatMessageViews, streamingChatMessageView } from "../src/lib/chat-message-view.ts";
import {
  disableHomeStreamingRenderProbe,
  enableHomeStreamingRenderProbe,
  homeStreamingRenderProbeSnapshot,
  resetHomeStreamingRenderProbe,
  type HomeStreamingRenderProbeSnapshot,
} from "../src/lib/home-streaming-render-probe.ts";

const STREAMING_TURN = "事业宫的火星落在第十宫，说明你在对外责任上会反复被推到台前。接下来三个月更适合把已经开过口的项目收束成可验收的结果，而不是再开一条全新战线。感情上第七宫的土星要求把边界说清楚：能一起承担日常安排，才值得把时间押上去。财务上第二宫与第十一宫同时被木星照面，收入机会来自既有圈子的转介，而不是短期投机。健康上注意睡眠被工作节奏切碎，先把就寝时间固定，再谈加码。以上判断来自 D1 与 D10 的宫主联系，D9 尚未用外部引擎闭环，所以婚恋时点只保留为条件性提示。";
const STREAMING_REPLY = STREAMING_TURN.repeat(4);

function chunkText(text: string, size: number): string[] {
  const chunks: string[] = [];
  for (let index = size; index <= text.length; index += size) {
    chunks.push(text.slice(0, index));
  }
  if (chunks.at(-1) !== text) chunks.push(text);
  return chunks;
}

function historyMessages(count: number): ChatMessage[] {
  const messages: ChatMessage[] = [];
  for (let index = 0; index < count; index += 1) {
    messages.push(
      index % 2 === 0
        ? { role: "user", text: `这是第 ${index + 1} 条用户问题，请根据星盘说明接下来三个月的事业安排。` }
        : { role: "assistant", text: `这是第 ${index + 1} 条已完成回答。第十宫的工作节奏需要先收束已有项目，再考虑新的对外合作。${"证据层包括 D1 与 D10。".repeat(8)}` },
    );
  }
  if (messages.at(-1)?.role !== "user") {
    messages.push({ role: "user", text: "请在刚才的基础上继续说明这个月可以推进的安排。" });
  }
  return messages;
}

function emptyActions(): ChatTranscriptActions {
  return {
    onFeedback() {},
    onCopy() {},
    onRegenerate() {},
    onFollowUp() {},
    onStartRectification() {},
  };
}

function baseProps(messages: readonly ChatMessage[]): Omit<ChatTranscriptProps, "loading" | "streamingText"> {
  const actionsRef = createRef<ChatTranscriptActions>() as ChatTranscriptProps["actionsRef"];
  actionsRef.current = emptyActions();
  return {
    messages,
    sessionId: "session-benchmark",
    sessionType: "consultation",
    theme: "career",
    messageFeedback: {},
    copiedMessageKey: null,
    cancellationPending: false,
    productEntrypointsDisabled: true,
    actionsRef,
  };
}

export type ScenarioResult = Readonly<{
  name: string;
  tokens: number;
  elapsedMs: number;
  probe: HomeStreamingRenderProbeSnapshot;
}>;

function runUnsplit(name: string, messages: readonly ChatMessage[], reply: string): ScenarioResult {
  const tokens = chunkText(reply, 20);
  const props = baseProps(messages);
  resetHomeStreamingRenderProbe();
  enableHomeStreamingRenderProbe();
  const started = performance.now();
  for (const streamingText of tokens) {
    renderToString(createElement(UnsplitChatTranscript, {
      ...props,
      loading: true,
      streamingText,
    }));
  }
  const elapsedMs = performance.now() - started;
  const probe = homeStreamingRenderProbeSnapshot();
  disableHomeStreamingRenderProbe();
  return { name, tokens: tokens.length, elapsedMs, probe };
}

function runSplitArchitecture(name: string, messages: readonly ChatMessage[], reply: string): ScenarioResult {
  const tokens = chunkText(reply, 20);
  const props = baseProps(messages);
  resetHomeStreamingRenderProbe();
  enableHomeStreamingRenderProbe();
  const started = performance.now();
  renderToString(createElement(SettledMessageList, {
    messages: props.messages,
    loading: true,
    sessionId: props.sessionId,
    sessionType: props.sessionType,
    theme: props.theme,
    messageFeedback: props.messageFeedback,
    copiedMessageKey: props.copiedMessageKey,
    cancellationPending: props.cancellationPending,
    productEntrypointsDisabled: props.productEntrypointsDisabled,
    actionsRef: props.actionsRef,
  }));
  for (const streamingText of tokens) {
    const streamingMessage = streamingChatMessageView(messages, true, streamingText);
    if (streamingMessage) {
      renderToString(createElement(LatestAssistantEntry, {
        ...props,
        loading: true,
        message: streamingMessage,
        views: [...settledChatMessageViews(messages), streamingMessage],
        index: messages.length,
      }));
    }
  }
  const elapsedMs = performance.now() - started;
  const probe = homeStreamingRenderProbeSnapshot();
  disableHomeStreamingRenderProbe();
  return { name, tokens: tokens.length, elapsedMs, probe };
}

function relativeDelta(first: number, second: number): number {
  if (first === 0) return 0;
  return Math.abs(first - second) / first;
}

export type BenchmarkReport = Readonly<{
  generatedAt: string;
  replyChars: number;
  empty: { run1: ScenarioResult; run2: ScenarioResult; unsplit: ScenarioResult; split: ScenarioResult };
  long: { run1: ScenarioResult; run2: ScenarioResult; unsplit: ScenarioResult; split: ScenarioResult };
  variance: { emptyElapsed: number; longElapsed: number; emptyUnsplitRenders: number; longUnsplitRenders: number };
  splitGain: { emptyElapsed: number; longElapsed: number; longSettledListRenders: number };
  usable: boolean;
  notes: string;
}>;

export function runHomeStreamingRenderBenchmark(): BenchmarkReport {
  const emptyHistory = [{ role: "user" as const, text: "请根据星盘说明接下来三个月的事业安排。" }];
  const longHistory = historyMessages(40);
  runUnsplit("empty-warmup", emptyHistory, STREAMING_REPLY);
  runUnsplit("long-warmup", longHistory, STREAMING_REPLY);
  const emptyUnsplit1 = runUnsplit("empty-unsplit-1", emptyHistory, STREAMING_REPLY);
  const emptyUnsplit2 = runUnsplit("empty-unsplit-2", emptyHistory, STREAMING_REPLY);
  const longUnsplit1 = runUnsplit("long-unsplit-1", longHistory, STREAMING_REPLY);
  const longUnsplit2 = runUnsplit("long-unsplit-2", longHistory, STREAMING_REPLY);
  const emptySplit = runSplitArchitecture("empty-split", emptyHistory, STREAMING_REPLY);
  const longSplit = runSplitArchitecture("long-split", longHistory, STREAMING_REPLY);

  const emptyElapsedVariance = relativeDelta(emptyUnsplit1.elapsedMs, emptyUnsplit2.elapsedMs);
  const longElapsedVariance = relativeDelta(longUnsplit1.elapsedMs, longUnsplit2.elapsedMs);
  const emptyRenderVariance = relativeDelta(emptyUnsplit1.probe.unsplitListRenders, emptyUnsplit2.probe.unsplitListRenders);
  const longRenderVariance = relativeDelta(longUnsplit1.probe.unsplitListRenders, longUnsplit2.probe.unsplitListRenders);
  const emptyGain = 1 - emptySplit.elapsedMs / ((emptyUnsplit1.elapsedMs + emptyUnsplit2.elapsedMs) / 2);
  const longGain = 1 - longSplit.elapsedMs / ((longUnsplit1.elapsedMs + longUnsplit2.elapsedMs) / 2);
  const usable = emptyRenderVariance === 0 && longRenderVariance === 0
    && emptyUnsplit1.probe.streamingRowRenders === emptyUnsplit1.tokens
    && longUnsplit1.probe.streamingRowRenders === longUnsplit1.tokens
    && longSplit.probe.settledListRenders === 1
    && longSplit.probe.streamingRowRenders === longSplit.tokens
    && (emptyElapsedVariance < 0.2 && longElapsedVariance < 0.2
      || longGain > 0.2 && longGain > 2 * Math.max(emptyElapsedVariance, longElapsedVariance));

  return {
    generatedAt: new Date().toISOString(),
    replyChars: STREAMING_REPLY.length,
    empty: { run1: emptyUnsplit1, run2: emptyUnsplit2, unsplit: emptyUnsplit1, split: emptySplit },
    long: { run1: longUnsplit1, run2: longUnsplit2, unsplit: longUnsplit1, split: longSplit },
    variance: {
      emptyElapsed: emptyElapsedVariance,
      longElapsed: longElapsedVariance,
      emptyUnsplitRenders: emptyRenderVariance,
      longUnsplitRenders: longRenderVariance,
    },
    splitGain: {
      emptyElapsed: emptyGain,
      longElapsed: longGain,
      longSettledListRenders: longSplit.probe.settledListRenders,
    },
    usable,
    notes: "renderToString always walks the tree, so this measures the split architecture (settled list once + streaming row per token) rather than React.memo itself. Reverse verification is runUnsplit vs runSplitArchitecture.",
  };
}

const isDirectRun = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
if (isDirectRun) {
  const report = runHomeStreamingRenderBenchmark();
  const outputPath = join(dirname(fileURLToPath(import.meta.url)), "home-streaming-render-baseline.json");
  writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`);
  console.log(JSON.stringify(report, null, 2));
  if (!report.usable) {
    console.error("benchmark is not stable enough to detect a 20% change");
    process.exitCode = 1;
  }
}