root/Jyotisha
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(billing): record prompt cache usage

Browse Source
This commit is contained in:
Jesse_Chen
2026-08-31 09:59:53 +08:00
parent a499c3444e
commit 3a762e1fca
12 changed files with 180 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/app/api/consult/route.ts
+26 -7
View File
@@ -29,6 +29,7 @@ import {
shouldLoadGeneralDailyPanchanga,
} from "@/lib/consultation-entrypoint";
import { CreditRpcError } from "@/lib/consultation-billing";
import { cachedSystemMessage, mergePromptCacheUsage, promptCacheUsage } from "@/lib/agent-generation-settings";
import { resolveFeaturePricing } from "@/lib/feature-pricing";
import { reserveConsultationModel } from "@/lib/consultation-model-selection";
import { resolveSessionLanguageModel } from "@/lib/model-catalog";
@@ -203,13 +204,22 @@ function chinaCalendarDate(now: Date) {
return new Date(now.getTime() + 8 * 60 * 60 * 1000).toISOString().slice(0, 10);
}
type Usage = { inputTokens?: number; outputTokens?: number };
type Usage = {
inputTokens?: number;
outputTokens?: number;
cache?: ReturnType<typeof promptCacheUsage>;
};
function mergeUsage(usages: Promise<Usage>[]): Promise<Usage> {
return Promise.all(usages).then((items) => items.reduce((total, item) => ({
inputTokens: (total.inputTokens ?? 0) + (item.inputTokens ?? 0),
outputTokens: (total.outputTokens ?? 0) + (item.outputTokens ?? 0),
}), {} as Usage));
return Promise.all(usages).then((items) => items.reduce<Usage>((total, item) => {
const usage = item && typeof item === "object" ? item as Record<string, unknown> : {};
const cache = promptCacheUsage(usage);
return {
inputTokens: (total.inputTokens ?? 0) + (typeof usage.inputTokens === "number" ? usage.inputTokens : 0),
outputTokens: (total.outputTokens ?? 0) + (typeof usage.outputTokens === "number" ? usage.outputTokens : 0),
cache: mergePromptCacheUsage([total.cache, cache]),
};
}, {} as Usage));
}
function shouldUseAgenticRuntime(user: { id: string; app_metadata?: Record<string, unknown> }) {
@@ -484,8 +494,10 @@ export async function POST(request: Request) {
const usageStartedAt = Date.now();
async function usagePayload(usage: Promise<{ inputTokens?: number; outputTokens?: number }>) {
const resolved = await usage;
const inputTokens = Math.max(0, Math.trunc(resolved.inputTokens ?? 0));
const outputTokens = Math.max(0, Math.trunc(resolved.outputTokens ?? 0));
const usageRecord = resolved as Record<string, unknown>;
const cache = promptCacheUsage(usageRecord);
const inputTokens = Math.max(0, Math.trunc(typeof usageRecord.inputTokens === "number" ? usageRecord.inputTokens : 0));
const outputTokens = Math.max(0, Math.trunc(typeof usageRecord.outputTokens === "number" ? usageRecord.outputTokens : 0));
return {
eventKey: requestId,
actualModelId: selectedModel.id,
@@ -497,6 +509,7 @@ export async function POST(request: Request) {
+ outputTokens * (selectedModel.outputCostMicrousdPerMillion ?? 0)
) / 1_000_000),
durationMs: Date.now() - usageStartedAt,
...(cache ? { metadata: { cache: { ...cache, hit: cache.readTokens > 0 } } } : {}),
};
}
@@ -674,7 +687,9 @@ export async function POST(request: Request) {
// failure response must not depend on it succeeding.
}
};
const cacheBoundary = cachedSystemMessage("【上下文缓存边界】后续内容为本轮请求输入。", selectedModel.model);
const baseMessages = [
...(cacheBoundary ? [cacheBoundary] : []),
...history.map((message) => message.role === "user"
? { role: "user" as const, content: message.text }
: { role: "assistant" as const, content: message.text }),
@@ -1033,7 +1048,9 @@ export async function POST(request: Request) {
return await runAgenticConsultation(consultationMode, history, name, generalDailyContext);
}
if (!shouldRunBirthChartWorkflow(consultationMode)) {
const cacheBoundary = cachedSystemMessage("【上下文缓存边界】后续内容为本轮请求输入。", selectedModel.model);
const result = await getGeneralJyotishAgent(selectedModel).stream([
...(cacheBoundary ? [cacheBoundary] : []),
{
role: "user",
content: [
@@ -1107,7 +1124,9 @@ export async function POST(request: Request) {
);
const workflowReceipt = consultationWorkflowReceipt(workflowContext);
const cacheBoundary = cachedSystemMessage("【上下文缓存边界】后续内容为本轮请求输入。", selectedModel.model);
const result = await getLegacyJyotishAgent(selectedModel, workflowContext).stream([
...(cacheBoundary ? [cacheBoundary] : []),
...history.map((message) => message.role === "user"
? { role: "user" as const, content: message.text }
: { role: "assistant" as const, content: message.text }),
frontend/src/app/api/rectification/agent/route.ts
+1
View File
@@ -594,6 +594,7 @@ export async function POST(request: Request) {
modelConfigVersion: selectedModel.configVersion,
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
...(usage.cache ? { metadata: { cache: { ...usage.cache, hit: usage.cache.readTokens > 0 } } } : {}),
costMicrousd: Math.round((
usage.inputTokens * (selectedModel.inputCostMicrousdPerMillion ?? 0)
+ usage.outputTokens * (selectedModel.outputCostMicrousdPerMillion ?? 0)
frontend/src/app/api/reports/route.ts
+1
View File
@@ -215,6 +215,7 @@ export async function POST(request: Request) {
eventKey: "report.full", actualModelId: usage.actualModelId,
modelConfigVersion: usage.modelConfigVersion, inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens, costMicrousd, durationMs: usage.durationMs,
...(usage.cache ? { metadata: { cache: { ...usage.cache, hit: usage.cache.readTokens > 0 } } } : {}),
});
return settled.success;
},
frontend/src/lib/agent-generation-settings.ts
+56
View File
@@ -17,6 +17,62 @@ export const AGENT_MAX_OUTPUT_TOKENS = AGENT_ANSWER_OUTPUT_TOKENS;
export type ThinkingMode = "enabled" | "disabled";
export type ReasoningEffort = "low" | "medium" | "high";
export type PromptCacheUsage = Readonly<{
readTokens: number;
writeTokens: number;
noCacheTokens: number;
}>;
function finiteTokenCount(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
}
/** Normalize AI SDK v6, legacy Mastra, and OpenAI raw cache usage shapes. */
export function promptCacheUsage(value: unknown): PromptCacheUsage | null {
const record = value && typeof value === "object" && !Array.isArray(value)
? value as Record<string, unknown>
: {};
const details = record.inputTokenDetails && typeof record.inputTokenDetails === "object"
? record.inputTokenDetails as Record<string, unknown>
: {};
const promptDetails = record.prompt_tokens_details && typeof record.prompt_tokens_details === "object"
? record.prompt_tokens_details as Record<string, unknown>
: {};
const usage = {
readTokens: finiteTokenCount(details.cacheReadTokens) || finiteTokenCount(record.cachedInputTokens) || finiteTokenCount(promptDetails.cached_tokens),
writeTokens: finiteTokenCount(details.cacheWriteTokens) || finiteTokenCount(record.cacheCreationInputTokens),
noCacheTokens: finiteTokenCount(details.noCacheTokens),
};
return usage.readTokens || usage.writeTokens || usage.noCacheTokens ? usage : null;
}
export function mergePromptCacheUsage(usages: readonly (PromptCacheUsage | null | undefined)[]): PromptCacheUsage | null {
const total = usages.reduce<{ readTokens: number; writeTokens: number; noCacheTokens: number }>((sum, usage) => ({
readTokens: sum.readTokens + (usage?.readTokens ?? 0),
writeTokens: sum.writeTokens + (usage?.writeTokens ?? 0),
noCacheTokens: sum.noCacheTokens + (usage?.noCacheTokens ?? 0),
}), { readTokens: 0, writeTokens: 0, noCacheTokens: 0 });
return total.readTokens || total.writeTokens || total.noCacheTokens ? total : null;
}
function modelProviderId(model: unknown): string | undefined {
return typeof model === "string"
? model.split("/")[0]
: model && typeof model === "object" && "providerId" in model && typeof model.providerId === "string"
? model.providerId
: undefined;
}
/** Adds only Anthropic's message-level cache marker; other providers keep current behavior. */
export function cachedSystemMessage(content: string, model?: unknown) {
if (modelProviderId(model) !== "anthropic") return null;
return {
role: "system" as const,
content,
providerOptions: { anthropic: { cacheControl: { type: "ephemeral" as const } } },
};
}
export function agentOutputTokenBudget(
thinking: ThinkingMode,
options: { answerTokens?: number; thinkingTokens?: number } = {},
frontend/src/lib/consultation-billing.ts
+9
View File
@@ -43,6 +43,14 @@ export type ActualUsage = {
outputTokens: number;
costMicrousd: number;
durationMs: number;
metadata?: {
cache?: {
readTokens: number;
writeTokens: number;
noCacheTokens: number;
hit: boolean;
};
};
};
export class CreditRpcError extends Error {
@@ -128,6 +136,7 @@ export async function completeUsage(
outputTokens: Math.max(0, Math.trunc(usage.outputTokens)),
costMicrousd: Math.max(0, Math.trunc(usage.costMicrousd)),
durationMs: Math.max(0, Math.trunc(usage.durationMs)),
...(usage.metadata ? { metadata: usage.metadata } : {}),
},
}, settlementSchema);
}
frontend/src/lib/personal-report-billing.ts
+2
View File
@@ -1,4 +1,5 @@
import type { UsageAuthorization } from "./consultation-billing";
import type { PromptCacheUsage } from "./agent-generation-settings";
export type ReportBillingUsage = Readonly<{
actualModelId: string;
@@ -6,6 +7,7 @@ export type ReportBillingUsage = Readonly<{
inputTokens: number;
outputTokens: number;
durationMs: number;
cache?: PromptCacheUsage | null;
}>;
export type ReportBillingPort = Readonly<{
frontend/src/lib/personal-report-generation.ts
+1 -1
View File
@@ -2132,7 +2132,7 @@ export type GeneratePersonalReportDeps = GeneratePersonalReportBaseDeps & Readon
export type ReportSchemaInnerReason = string;
export type GeneratePersonalReportResult = Readonly<
| { status: "ready"; document: ReportDocumentV2; evidenceHash: string; usage?: Readonly<{ inputTokens: number; outputTokens: number; actualModelId?: string; modelConfigVersion?: number }> }
| { status: "ready"; document: ReportDocumentV2; evidenceHash: string; usage?: Readonly<{ inputTokens: number; outputTokens: number; cache?: import("./agent-generation-settings").PromptCacheUsage | null; actualModelId?: string; modelConfigVersion?: number }> }
| { status: "failed"; failureCode: "report_schema_invalid"; innerReason: ReportSchemaInnerReason }
| { status: "failed"; failureCode: "report_guard_rejected" }
>;
frontend/src/lib/personal-report-route-core.ts
+1
View File
@@ -503,6 +503,7 @@ export async function resolveReportCreate(deps: ReportCreateCoreDeps): Promise<R
inputTokens: result.usage?.inputTokens ?? 0,
outputTokens: result.usage?.outputTokens ?? 0,
durationMs: 0,
cache: result.usage?.cache,
},
});
if (!settled) throw new Error("report billing settlement failed");
frontend/src/lib/personal-report-worker.ts
+2
View File
@@ -177,6 +177,7 @@ async function generateProductionReport(context: PersonalReportWorkerGenerationC
usage: {
inputTokens: result.usage?.inputTokens ?? 0,
outputTokens: result.usage?.outputTokens ?? 0,
cache: result.usage?.cache,
actualModelId: model.id,
modelConfigVersion: model.configVersion,
},
@@ -211,6 +212,7 @@ function createProductionWorker(workerId: string) {
eventKey: "report.full", actualModelId: usage.actualModelId,
modelConfigVersion: usage.modelConfigVersion, inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens, costMicrousd, durationMs: usage.durationMs,
...(usage.cache ? { metadata: { cache: { ...usage.cache, hit: usage.cache.readTokens > 0 } } } : {}),
});
return settled.success;
},
frontend/src/lib/rectification-agentic/v9/agent-run.ts
+24 -18
View File
@@ -25,7 +25,7 @@ import {
} from "./tool-service";
import { RECTIFICATION_SKILL_NAME, RECTIFICATION_SKILL_VERSION } from "./case-status";
import { RECTIFICATION_AGENT_TOOLS } from "./public-receipt";
import { agentGenerationSettings } from "../../agent-generation-settings.ts";
import { agentGenerationSettings, cachedSystemMessage, promptCacheUsage } from "../../agent-generation-settings.ts";
import { toAgentModelFinishReason } from "../../agent-observability.ts";
import { decideFromDossier } from "./decision-from-dossier";
import { parseAgentChoiceCopy, isPersistedFocusId } from "./choice-card";
@@ -68,7 +68,12 @@ import {
export type V9RunBilling = Readonly<{
reserve(): Promise<{ success: boolean; reason?: string; status: number }>;
complete(input: { inputTokens: number; outputTokens: number; durationMs: number }): Promise<boolean>;
complete(input: {
inputTokens: number;
outputTokens: number;
durationMs: number;
cache?: ReturnType<typeof promptCacheUsage>;
}): Promise<boolean>;
release(): Promise<boolean>;
}>;
@@ -115,7 +120,7 @@ export type V9AgentRunResult = Readonly<{
}>;
type AttemptStatus = "completed" | "failed" | "retryable";
type Usage = Readonly<{ inputTokens: number; outputTokens: number }>;
type Usage = Readonly<{ inputTokens: number; outputTokens: number; cache?: ReturnType<typeof promptCacheUsage> }>;
type AttemptOutcome = Readonly<{
ok: boolean;
status: AttemptStatus;
@@ -618,7 +623,7 @@ export async function runV9AgentTurn(options: V9AgentRunOptions): Promise<V9Agen
maxSteps: number;
abortSignal: AbortSignal;
modelSettings?: { maxOutputTokens?: number };
providerOptions?: Record<string, { thinking: { type: "disabled" | "enabled" } }>;
providerOptions?: Record<string, unknown>;
prepareStep: (input: { stepNumber: number }) => {
activeTools: string[];
toolChoice: "auto";
@@ -637,7 +642,7 @@ export async function runV9AgentTurn(options: V9AgentRunOptions): Promise<V9Agen
};
object?: unknown;
}>;
totalUsage?: Promise<{ inputTokens?: number; outputTokens?: number }>;
totalUsage?: Promise<Record<string, unknown>>;
}>;
}).stream(messages, {
maxSteps,
@@ -889,10 +894,12 @@ export async function runV9AgentTurn(options: V9AgentRunOptions): Promise<V9Agen
const completeAttempt = async (): Promise<AttemptOutcome> => {
let inputTokens = 0;
let outputTokens = 0;
let cache: ReturnType<typeof promptCacheUsage> = null;
try {
const raw = await (result.totalUsage ?? Promise.resolve({ inputTokens: 0, outputTokens: 0 }));
inputTokens = Math.max(0, Math.trunc(raw.inputTokens ?? 0));
outputTokens = Math.max(0, Math.trunc(raw.outputTokens ?? 0));
inputTokens = Math.max(0, Math.trunc(typeof raw.inputTokens === "number" ? raw.inputTokens : 0));
outputTokens = Math.max(0, Math.trunc(typeof raw.outputTokens === "number" ? raw.outputTokens : 0));
cache = promptCacheUsage(raw);
} catch {
// Timeout/abort can leave provider usage unread.
}
@@ -902,7 +909,7 @@ export async function runV9AgentTurn(options: V9AgentRunOptions): Promise<V9Agen
ok: true,
status: "completed",
errorCode: null,
usage: { inputTokens, outputTokens },
usage: { inputTokens, outputTokens, ...(cache ? { cache } : {}) },
answerText,
answerDeltas,
phases,
@@ -1061,16 +1068,15 @@ function buildAgentMessages(
const timeContext = options.timeContext
?? `服务端当前时间（权威）：${new Date().toISOString()}。涉及“现在、今天、今年、未来几个月”等相对时间时，以此为准。`;
const caseContext = `【服务端 Case ID】${options.caseId}。所有 rectification 工具调用的 caseId 必须原样使用此值。`;
const bootstrap = {
role: "system",
content: [
"【服务器已绑定当前 Case 的精确 Skill】运行器已在本 attempt 内加载并核验下列指令；不要重复调用 skill。第一步必须调用 rectification-read-case。",
skillInstructions,
...(attempt > 1
? ["【重试约束】不得复用上一次 attempt 的文本或工具状态；从 rectification-read-case 重新读取服务器事实。"]
: []),
].join("\n\n"),
};
const bootstrapContent = [
"【服务器已绑定当前 Case 的精确 Skill】运行器已在本 attempt 内加载并核验下列指令；不要重复调用 skill。第一步必须调用 rectification-read-case。",
skillInstructions,
...(attempt > 1
? ["【重试约束】不得复用上一次 attempt 的文本或工具状态；从 rectification-read-case 重新读取服务器事实。"]
: []),
].join("\n\n");
const bootstrap = cachedSystemMessage(bootstrapContent, options.generationModel)
?? { role: "system" as const, content: bootstrapContent };
if (options.action === "opening") {
return [bootstrap, {
role: "user",
frontend/src/mastra/personal-report.ts
+12 -4
View File
@@ -2,7 +2,7 @@ import { Agent } from "@mastra/core/agent";
import { z } from "zod";
import type { ResolvedLanguageModel } from "./model";
import type { PersonalReportSectionPlan, ReportSectionPlanEntry } from "@/lib/personal-report-plan";
import { agentGenerationSettings } from "@/lib/agent-generation-settings";
import { cachedSystemMessage, mergePromptCacheUsage, promptCacheUsage, agentGenerationSettings } from "@/lib/agent-generation-settings";
import type {
ReportChartHouse,
ReportDashaPeriod,
@@ -175,13 +175,15 @@ async function readUsage(value: unknown) {
const numberOrNull = (key: string) => (
typeof record[key] === "number" && Number.isFinite(record[key]) ? record[key] as number : null
);
const cache = promptCacheUsage(record);
return {
inputTokens: numberOrNull("inputTokens"),
outputTokens: numberOrNull("outputTokens"),
totalTokens: numberOrNull("totalTokens"),
cache,
};
} catch {
return { inputTokens: null, outputTokens: null, totalTokens: null };
return { inputTokens: null, outputTokens: null, totalTokens: null, cache: null };
}
}
@@ -234,7 +236,7 @@ export type ReportAgentSummaryOptions = Readonly<{
maxOutputTokens?: number;
}>;
export type ReportAgentUsage = Readonly<{ inputTokens: number; outputTokens: number }>;
export type ReportAgentUsage = Readonly<{ inputTokens: number; outputTokens: number; cache?: ReturnType<typeof promptCacheUsage> }>;
export type ReportAgentPort = Readonly<{
modelId: string;
@@ -290,9 +292,11 @@ export function createPersonalReportAgent(model: ResolvedLanguageModel): ReportA
let usageTotals: ReportAgentUsage = { inputTokens: 0, outputTokens: 0 };
const recordUsage = async (usage: unknown) => {
const tokens = await readUsage(usage);
const cache = mergePromptCacheUsage([usageTotals.cache, tokens.cache]);
usageTotals = {
inputTokens: usageTotals.inputTokens + Math.max(0, tokens.inputTokens ?? 0),
outputTokens: usageTotals.outputTokens + Math.max(0, tokens.outputTokens ?? 0),
...(cache ? { cache } : {}),
};
};
@@ -308,8 +312,12 @@ export function createPersonalReportAgent(model: ResolvedLanguageModel): ReportA
const prompt = input.prompt;
let repairAttempted = false;
let attemptReturned = false;
const cacheBoundary = cachedSystemMessage("【上下文缓存边界】后续内容为本次报告输入。", model.model);
const runOnce = (content: string) => agent.generate(
[{ role: "user", content }],
[
...(cacheBoundary ? [cacheBoundary] : []),
{ role: "user", content },
],
{
abortSignal: signal,
structuredOutput: { schema: input.schema, jsonPromptInjection: "inline" as const },
frontend/tests/agent-generation-settings.test.ts
+45
View File
@@ -0,0 +1,45 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
cachedSystemMessage,
mergePromptCacheUsage,
promptCacheUsage,
} from "../src/lib/agent-generation-settings.ts";
test("normalizes AI SDK cache token details", () => {
assert.deepEqual(promptCacheUsage({
inputTokenDetails: { cacheReadTokens: 120, cacheWriteTokens: 30, noCacheTokens: 10 },
}), { readTokens: 120, writeTokens: 30, noCacheTokens: 10 });
});
test("normalizes OpenAI and legacy cache usage", () => {
assert.deepEqual(promptCacheUsage({ prompt_tokens_details: { cached_tokens: 45 } }), {
readTokens: 45,
writeTokens: 0,
noCacheTokens: 0,
});
assert.deepEqual(promptCacheUsage({ cachedInputTokens: 12, cacheCreationInputTokens: 8 }), {
readTokens: 12,
writeTokens: 8,
noCacheTokens: 0,
});
});
test("merges cache usage and only reports a hit when reads exist", () => {
assert.deepEqual(mergePromptCacheUsage([
{ readTokens: 4, writeTokens: 5, noCacheTokens: 6 },
{ readTokens: 7, writeTokens: 0, noCacheTokens: 1 },
]), { readTokens: 11, writeTokens: 5, noCacheTokens: 7 });
assert.equal(promptCacheUsage({ inputTokenDetails: { cacheWriteTokens: 5 } })?.readTokens, 0);
assert.equal(promptCacheUsage({ inputTokenDetails: { cacheReadTokens: 5 } })?.readTokens, 5);
});
test("adds an Anthropic cache boundary and safely degrades for other providers", () => {
assert.deepEqual(cachedSystemMessage("stable", "anthropic/claude-sonnet"), {
role: "system",
content: "stable",
providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
});
assert.equal(cachedSystemMessage("stable", "openai/gpt-5"), null);
assert.equal(cachedSystemMessage("stable", { providerId: "deepseek" }), null);
});