Jyotisha/frontend/tests/agent-observability.test.ts

import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";

import { ZodError } from "zod";

import {
  agentModelFinishReasons,
  agentObservabilityEventSchema,
  createAgentObservabilityLogger,
  settlementTelemetryOutcome,
  toAgentModelFinishReason,
  toAgentObservabilityErrorCode,
} from "../src/lib/agent-observability.ts";

const baseEvent = {
  runId: "8d14b4f7-9d0b-4b4e-a48d-365e5d2a1d4b",
  requestId: "f2ff3466-913b-4a79-8527-b90f28a6c95d",
  sessionId: "2cf930ab-d59d-4f36-b92f-aa4ea035305a",
  caseId: "case-42",
  agentVersion: "consultation-agentic-v1",
  skillVersion: "6.9.14",
  modelVersion: "3",
  policyVersion: "consultation-runtime-contract-v1",
  toolCalls: [
    { name: "run-jyotish-consultation", durationMs: 321, status: "completed" },
  ],
  contractPhases: [
    { phase: "skill.loaded", durationMs: 12, status: "completed" },
    { phase: "answer.first_output", durationMs: 456, status: "completed" },
    { phase: "billing.settled", status: "completed" },
  ],
  retryCount: 1,
  errorCode: "runtime_contract_incomplete",
  modelFinishReason: "tool-calls",
  modelStepCount: 8,
  inputTokens: 1200,
  outputTokens: 345,
  evidenceCount: 4,
  claimCount: 7,
  sectionCount: 5,
  themeCoverage: ["career", "wealth"],
  reportJobDurationMs: 2500,
  reportJobPeakMemoryBytes: 64 * 1024 * 1024,
  billingSettlementResult: "completed",
} as const;

test("strict schema accepts only bounded non-PII Agent run metrics", () => {
  assert.deepEqual(agentObservabilityEventSchema.parse(baseEvent), baseEvent);
});

test("logger validates before emitting and returns the closed payload", () => {
  const emitted: unknown[] = [];
  const logger = createAgentObservabilityLogger((event) => emitted.push(event));

  const parsed = logger(baseEvent);

  assert.deepEqual(parsed, baseEvent);
  assert.deepEqual(emitted, [baseEvent]);
});

test("unknown or PII-bearing fields fail closed instead of being stripped", () => {
  const logger = createAgentObservabilityLogger(() => {
    assert.fail("invalid observability payload must not reach the sink");
  });
  const forbiddenFields = [
    "text",
    "content",
    "input",
    "output",
    "question",
    "answer",
    "prompt",
    "messages",
    "birthDate",
    "birthTime",
    "birthPlace",
    "birthData",
    "name",
    "email",
    "secret",
    "apiKey",
    "providerPayload",
    "stack",
    "absolutePath",
    "metadata",
  ];

  for (const field of forbiddenFields) {
    assert.throws(
      () => logger({ runId: baseEvent.runId, [field]: "sensitive-value" }),
      ZodError,
      `${field} must fail closed`,
    );
  }

  assert.throws(
    () => logger({
      runId: baseEvent.runId,
      toolCalls: [{
        name: "run-jyotish-consultation",
        durationMs: 1,
        status: "completed",
        prompt: "raw prompt",
      }],
    }),
    ZodError,
  );
});

test("free-form prose, email-like values and internal paths are rejected", () => {
  assert.throws(
    () => agentObservabilityEventSchema.parse({
      runId: baseEvent.runId,
      errorCode: "provider returned user@example.com",
    }),
    ZodError,
  );
  assert.throws(
    () => agentObservabilityEventSchema.parse({
      runId: baseEvent.runId,
      modelVersion: "/Users/jesse/private/model.json",
    }),
    ZodError,
  );
  assert.throws(
    () => agentObservabilityEventSchema.parse({ inputTokens: 1 }),
    ZodError,
  );
});

test("sink failures are isolated after strict validation", () => {
  const logger = createAgentObservabilityLogger(() => {
    throw new Error("transport unavailable");
  });

  assert.deepEqual(logger({ runId: baseEvent.runId, inputTokens: 1 }), {
    runId: baseEvent.runId,
    inputTokens: 1,
  });
});

test("error normalization never records arbitrary exception messages", () => {
  assert.equal(
    toAgentObservabilityErrorCode(new Error("runtime_contract_incomplete")),
    "runtime_contract_incomplete",
  );
  assert.equal(
    toAgentObservabilityErrorCode(new Error("answer_truncated")),
    "answer_truncated",
  );
  assert.equal(
    toAgentObservabilityErrorCode(new Error("/opt/internal/users/alice.json")),
    "calculation_failed",
  );
  assert.equal(
    toAgentObservabilityErrorCode(new Error("provider said user@example.com")),
    "calculation_failed",
  );
});

test("the model finish reason is a closed vocabulary, never a provider string", () => {
  for (const reason of agentModelFinishReasons) {
    assert.equal(toAgentModelFinishReason(reason), reason);
  }
  assert.equal(toAgentModelFinishReason("max-steps-exceeded"), "unknown");
  assert.equal(toAgentModelFinishReason("provider said user@example.com"), "unknown");
  assert.equal(toAgentModelFinishReason(undefined), "unknown");
  assert.equal(toAgentModelFinishReason({ reason: "stop" }), "unknown");
  assert.throws(
    () => agentObservabilityEventSchema.parse({
      runId: baseEvent.runId,
      modelFinishReason: "stopped because the provider said so",
    }),
    ZodError,
  );
});

test("step exhaustion is recorded rather than inferred from the step list", () => {
  const emitted: unknown[] = [];
  const logger = createAgentObservabilityLogger((event) => emitted.push(event));

  logger({
    runId: baseEvent.runId,
    modelFinishReason: toAgentModelFinishReason("tool-calls"),
    modelStepCount: 8,
  });

  assert.deepEqual(emitted, [{
    runId: baseEvent.runId,
    modelFinishReason: "tool-calls",
    modelStepCount: 8,
  }]);
});

test("settlement telemetry reports successful cancellation as cancelled", () => {
  assert.deepEqual(
    settlementTelemetryOutcome("cancelled", "cancelled"),
    {
      billingSettlementResult: "cancelled",
      errorCode: "cancelled",
    },
  );
});

test("settlement telemetry fail-closes a final cancellation failure", () => {
  const rawFailure = "provider rejected user@example.com at /Users/alice/private.json";
  const outcome = settlementTelemetryOutcome("failed", rawFailure);
  const parsed = agentObservabilityEventSchema.parse({
    runId: baseEvent.runId,
    ...outcome,
  });

  assert.deepEqual(outcome, {
    billingSettlementResult: "failed",
    errorCode: "settlement_failed",
  });
  assert.deepEqual(parsed, {
    runId: baseEvent.runId,
    billingSettlementResult: "failed",
    errorCode: "settlement_failed",
  });
  assert.doesNotMatch(JSON.stringify(parsed), /provider|example\.com|\/Users\//);
});

test("ordinary consultation logRun uses the strict logger and aggregated usage", () => {
  const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8");

  assert.match(route, /logAgentObservability\(\{/);
  assert.match(route, /inputTokens/);
  assert.match(route, /outputTokens/);
  assert.match(route, /billingSettlementResult/);
  assert.match(route, /\.\.\.consultationModelStepTelemetry\(state\),/);
  assert.doesNotMatch(route, /\[consult-agentic\]/);
});

test("an agentic run that fails before streaming still emits the observability event", () => {
  const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8");

  // streamAgentResponse's onError is the only settle-and-log path for a run
  // that got far enough to stream. Everything earlier—plan assembly, chart
  // truth, tool construction—throws into the request-level catch, which must
  // reach the same entry point instead of a bare cancel().
  assert.match(route, /agenticFailure\.report = async \(error\) => \{/);
  assert.match(route, /settleRun\(cancel, toAgentObservabilityErrorCode\(error\)\)/);
  const requestCatch = route.slice(route.lastIndexOf("} catch (error) {"));
  assert.match(requestCatch, /if \(agenticFailure\.report\) await agenticFailure\.report\(error\);\s*\n\s*else await cancel\(\);/);
});