From 1f1fdb1d155e472118ad7d1d4c49b1b5cb95bcb2 Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Sun, 30 Aug 2026 20:01:19 +0800 Subject: [PATCH] feat(reports): record personal report generation telemetry --- frontend/src/mastra/personal-report.ts | 68 +++++++++++++------ .../personal-report-agent-telemetry.test.ts | 58 ++++++++++++++++ 2 files changed, 106 insertions(+), 20 deletions(-) create mode 100644 frontend/tests/personal-report-agent-telemetry.test.ts diff --git a/frontend/src/mastra/personal-report.ts b/frontend/src/mastra/personal-report.ts index 066ecf9c..ff7bc4a3 100644 --- a/frontend/src/mastra/personal-report.ts +++ b/frontend/src/mastra/personal-report.ts @@ -122,6 +122,7 @@ export type PersonalReportAgentTelemetry = Readonly<{ inputTokens: number | null; outputTokens: number | null; totalTokens: number | null; + finishReason: string | null; repairAttempted: boolean; }>; @@ -142,18 +143,32 @@ Structure rules: - Keep actions concrete and cautious; caveats must state limits honestly. - Write formal, readable Simplified Chinese for a printed report.`; -function readUsage(value: unknown) { - const record = value !== null && typeof value === "object" && !Array.isArray(value) - ? value as Record - : {}; - const numberOrNull = (key: string) => ( - typeof record[key] === "number" && Number.isFinite(record[key]) ? record[key] as number : null - ); - return { - inputTokens: numberOrNull("inputTokens"), - outputTokens: numberOrNull("outputTokens"), - totalTokens: numberOrNull("totalTokens"), - }; +async function readUsage(value: unknown) { + try { + const resolved = await Promise.resolve(value); + const record = resolved !== null && typeof resolved === "object" && !Array.isArray(resolved) + ? resolved as Record + : {}; + const numberOrNull = (key: string) => ( + typeof record[key] === "number" && Number.isFinite(record[key]) ? record[key] as number : null + ); + return { + inputTokens: numberOrNull("inputTokens"), + outputTokens: numberOrNull("outputTokens"), + totalTokens: numberOrNull("totalTokens"), + }; + } catch { + return { inputTokens: null, outputTokens: null, totalTokens: null }; + } +} + +async function readFinishReason(value: unknown): Promise { + try { + const resolved = await Promise.resolve(value); + return typeof resolved === "string" && resolved.length <= 80 ? resolved : null; + } catch { + return null; + } } export class PersonalReportAgentOutputError extends Error { @@ -221,12 +236,13 @@ export function createPersonalReportAgent(model: ResolvedLanguageModel): ReportA }; let repairAttempted = false; + type GenerationResult = { object?: unknown; usage?: unknown; finishReason?: unknown }; const runOnce = (content: string) => agent.generate( [{ role: "user", content }], { abortSignal: signal, structuredOutput }, ); - - const accept = (result: { object?: unknown; usage?: unknown }): + let attemptReturned = false; + const accept = (result: GenerationResult): | { ok: true; data: PersonalReportAgentOutput } | { ok: false; cause: "schema" | "bind"; error?: unknown } => { const parsed = personalReportAgentOutputSchema.safeParse(result.object); @@ -241,22 +257,27 @@ export function createPersonalReportAgent(model: ResolvedLanguageModel): ReportA }; try { + attemptReturned = false; const first = await runOnce(prompt); + attemptReturned = true; const firstAccepted = accept(first); if (firstAccepted.ok) { - logTelemetry(model.id, startedAt, false, "resolved", first.usage); + await logTelemetry(model.id, startedAt, false, "resolved", first.usage, first.finishReason); return firstAccepted.data; } + await logTelemetry(model.id, startedAt, false, "failed", first.usage, first.finishReason); // Exactly one repair retry is allowed. A second failure is terminal. repairAttempted = true; + attemptReturned = false; const repaired = await runOnce(`${prompt}${REPAIR_PROMPT_SUFFIX}`); + attemptReturned = true; const repairedAccepted = accept(repaired); if (repairedAccepted.ok) { - logTelemetry(model.id, startedAt, true, "resolved", repaired.usage); + await logTelemetry(model.id, startedAt, true, "resolved", repaired.usage, repaired.finishReason); return repairedAccepted.data; } - logTelemetry(model.id, startedAt, true, "failed", repaired.usage); + await logTelemetry(model.id, startedAt, true, "failed", repaired.usage, repaired.finishReason); if (repairedAccepted.cause === "bind" && repairedAccepted.error) { throw repairedAccepted.error; } @@ -265,21 +286,27 @@ export function createPersonalReportAgent(model: ResolvedLanguageModel): ReportA if (error instanceof PersonalReportAgentOutputError) throw error; if (isAbortError(error, signal)) throw error; if (error instanceof Error && error.message.startsWith("report_writer_")) throw error; - logTelemetry(model.id, startedAt, repairAttempted, "failed", null); + if (!attemptReturned) { + await logTelemetry(model.id, startedAt, repairAttempted, "failed", null, null); + } throw error; } }, }; } -function logTelemetry( +async function logTelemetry( modelId: string, startedAt: number, repairAttempted: boolean, outcome: "resolved" | "failed", usage: unknown, + finishReason: unknown, ) { - const tokens = readUsage(usage); + const [tokens, finishReasonValue] = await Promise.all([ + readUsage(usage), + readFinishReason(finishReason), + ]); const telemetry: PersonalReportAgentTelemetry = { modelId, outcome, @@ -287,6 +314,7 @@ function logTelemetry( inputTokens: tokens.inputTokens, outputTokens: tokens.outputTokens, totalTokens: tokens.totalTokens, + finishReason: finishReasonValue, repairAttempted, }; // Telemetry must never include the prompt, the packet, birth data or the diff --git a/frontend/tests/personal-report-agent-telemetry.test.ts b/frontend/tests/personal-report-agent-telemetry.test.ts new file mode 100644 index 00000000..8d87fee1 --- /dev/null +++ b/frontend/tests/personal-report-agent-telemetry.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Agent } from "@mastra/core/agent"; +import { createPersonalReportAgent } from "../src/mastra/personal-report.ts"; + +const model = { + specificationVersion: "v2", + provider: "telemetry-test", + modelId: "telemetry-test", +} as never; + +function testModel() { + return { + id: "telemetry-test", + label: "Telemetry test", + description: "Telemetry test", + creditCost: 0, + isDefault: false, + mode: "openai" as const, + model, + }; +} + +test("personal report telemetry records each truncated attempt without private payloads", async () => { + const originalGenerate = Agent.prototype.generate; + const originalInfo = console.info; + const logs: unknown[][] = []; + let calls = 0; + Agent.prototype.generate = (async function () { + calls += 1; + return { + object: {}, + usage: Promise.resolve({ inputTokens: 11, outputTokens: 3, totalTokens: 14 }), + finishReason: Promise.resolve("length"), + }; + }) as never; + console.info = (...args: unknown[]) => logs.push(args); + + try { + const agent = createPersonalReportAgent(testModel()); + await assert.rejects(agent.generate({} as never, {} as never)); + } finally { + Agent.prototype.generate = originalGenerate; + console.info = originalInfo; + } + + assert.equal(calls, 2); + const telemetry = logs + .filter(([label]) => label === "[personal-report-agent]") + .map(([, payload]) => JSON.parse(String(payload)) as Record); + assert.equal(telemetry.length, 2); + assert.deepEqual(telemetry.map((entry) => entry.finishReason), ["length", "length"]); + assert.deepEqual(telemetry.map((entry) => entry.repairAttempted), [false, true]); + assert.deepEqual(telemetry.map((entry) => entry.outputTokens), [3, 3]); + assert.equal("prompt" in telemetry[0], false); + assert.equal("bundle" in telemetry[0], false); + assert.equal("report" in telemetry[0], false); +});