import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; import { ZodError } from "zod"; import { agentObservabilityEventSchema, createAgentObservabilityLogger, settlementTelemetryOutcome, toAgentObservabilityErrorCode, } from "../src/lib/agent-observability.ts"; const baseEvent = { runId: "8d14b4f7-9d0b-4b4e-a48d-365e5d2a1d4b", requestId: "f2ff3466-913b-4a79-8527-b90f28a6c95d", sessionId: "2cf930ab-d59d-4f36-b92f-aa4ea035305a", caseId: "case-42", agentVersion: "consultation-agentic-v1", skillVersion: "6.9.14", modelVersion: "3", policyVersion: "consultation-runtime-contract-v1", toolCalls: [ { name: "run-jyotish-consultation", durationMs: 321, status: "completed" }, ], contractPhases: [ { phase: "skill.loaded", durationMs: 12, status: "completed" }, { phase: "answer.first_output", durationMs: 456, status: "completed" }, { phase: "billing.settled", status: "completed" }, ], retryCount: 1, errorCode: "runtime_contract_incomplete", inputTokens: 1200, outputTokens: 345, evidenceCount: 4, claimCount: 7, sectionCount: 5, themeCoverage: ["career", "wealth"], reportJobDurationMs: 2500, reportJobPeakMemoryBytes: 64 * 1024 * 1024, billingSettlementResult: "completed", } as const; test("strict schema accepts only bounded non-PII Agent run metrics", () => { assert.deepEqual(agentObservabilityEventSchema.parse(baseEvent), baseEvent); }); test("logger validates before emitting and returns the closed payload", () => { const emitted: unknown[] = []; const logger = createAgentObservabilityLogger((event) => emitted.push(event)); const parsed = logger(baseEvent); assert.deepEqual(parsed, baseEvent); assert.deepEqual(emitted, [baseEvent]); }); test("unknown or PII-bearing fields fail closed instead of being stripped", () => { const logger = createAgentObservabilityLogger(() => { assert.fail("invalid observability payload must not reach the sink"); }); const forbiddenFields = [ "text", "content", "input", "output", "question", "answer", "prompt", "messages", "birthDate", "birthTime", "birthPlace", "birthData", "name", "email", "secret", "apiKey", "providerPayload", "stack", "absolutePath", "metadata", ]; for (const field of forbiddenFields) { assert.throws( () => logger({ runId: baseEvent.runId, [field]: "sensitive-value" }), ZodError, `${field} must fail closed`, ); } assert.throws( () => logger({ runId: baseEvent.runId, toolCalls: [{ name: "run-jyotish-consultation", durationMs: 1, status: "completed", prompt: "raw prompt", }], }), ZodError, ); }); test("free-form prose, email-like values and internal paths are rejected", () => { assert.throws( () => agentObservabilityEventSchema.parse({ runId: baseEvent.runId, errorCode: "provider returned user@example.com", }), ZodError, ); assert.throws( () => agentObservabilityEventSchema.parse({ runId: baseEvent.runId, modelVersion: "/Users/jesse/private/model.json", }), ZodError, ); assert.throws( () => agentObservabilityEventSchema.parse({ inputTokens: 1 }), ZodError, ); }); test("sink failures are isolated after strict validation", () => { const logger = createAgentObservabilityLogger(() => { throw new Error("transport unavailable"); }); assert.deepEqual(logger({ runId: baseEvent.runId, inputTokens: 1 }), { runId: baseEvent.runId, inputTokens: 1, }); }); test("error normalization never records arbitrary exception messages", () => { assert.equal( toAgentObservabilityErrorCode(new Error("runtime_contract_incomplete")), "runtime_contract_incomplete", ); assert.equal( toAgentObservabilityErrorCode(new Error("/opt/internal/users/alice.json")), "calculation_failed", ); assert.equal( toAgentObservabilityErrorCode(new Error("provider said user@example.com")), "calculation_failed", ); }); test("settlement telemetry reports successful cancellation as cancelled", () => { assert.deepEqual( settlementTelemetryOutcome("cancelled", "cancelled"), { billingSettlementResult: "cancelled", errorCode: "cancelled", }, ); }); test("settlement telemetry fail-closes a final cancellation failure", () => { const rawFailure = "provider rejected user@example.com at /Users/alice/private.json"; const outcome = settlementTelemetryOutcome("failed", rawFailure); const parsed = agentObservabilityEventSchema.parse({ runId: baseEvent.runId, ...outcome, }); assert.deepEqual(outcome, { billingSettlementResult: "failed", errorCode: "settlement_failed", }); assert.deepEqual(parsed, { runId: baseEvent.runId, billingSettlementResult: "failed", errorCode: "settlement_failed", }); assert.doesNotMatch(JSON.stringify(parsed), /provider|example\.com|\/Users\//); }); test("ordinary consultation logRun uses the strict logger and aggregated usage", () => { const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8"); assert.match(route, /logAgentObservability\(\{/); assert.match(route, /inputTokens/); assert.match(route, /outputTokens/); assert.match(route, /billingSettlementResult/); assert.doesNotMatch(route, /\[consult-agentic\]/); });