325 lines
9.9 KiB
TypeScript
325 lines
9.9 KiB
TypeScript
import { z } from "zod";
|
|
import type { VersionedJourneyResponse } from "./birth-time-journey-service.ts";
|
|
|
|
export const conversationalRectificationTelemetryProtocols = [
|
|
"conversational-evidence-v3",
|
|
] as const;
|
|
export const conversationalRectificationTelemetryPhases = [
|
|
"entry",
|
|
"collecting_evidence",
|
|
"paused",
|
|
"confirming",
|
|
"completed",
|
|
"abandoned",
|
|
] as const;
|
|
export const conversationalRectificationTelemetryActionKinds = [
|
|
"start",
|
|
"resume",
|
|
"answer",
|
|
"regenerate",
|
|
"pause",
|
|
"abandon",
|
|
"confirm",
|
|
"unknown",
|
|
] as const;
|
|
export const conversationalRectificationTelemetryResultCategories = [
|
|
"success",
|
|
"rejected",
|
|
"conflict",
|
|
"failed",
|
|
] as const;
|
|
export const conversationalRectificationTelemetryLatencyBuckets = [
|
|
"lt_100ms",
|
|
"100_499ms",
|
|
"500_1999ms",
|
|
"2s_plus",
|
|
] as const;
|
|
export const conversationalRectificationTelemetryBillingStates = [
|
|
"not_applicable",
|
|
"charged",
|
|
"released",
|
|
"migration_waived",
|
|
"unchanged",
|
|
"unknown",
|
|
] as const;
|
|
export const conversationalRectificationTelemetryErrorCategories = [
|
|
"none",
|
|
"authentication",
|
|
"validation",
|
|
"conflict",
|
|
"billing",
|
|
"dependency",
|
|
"unknown",
|
|
] as const;
|
|
|
|
const deploymentShaSchema = z.union([
|
|
z.string().regex(/^[0-9a-f]{7,64}$/),
|
|
z.literal("unknown"),
|
|
]);
|
|
|
|
export const conversationalRectificationTelemetryPayloadSchema = z.object({
|
|
protocol: z.enum(conversationalRectificationTelemetryProtocols),
|
|
phase: z.enum(conversationalRectificationTelemetryPhases),
|
|
actionKind: z.enum(conversationalRectificationTelemetryActionKinds),
|
|
resultCategory: z.enum(conversationalRectificationTelemetryResultCategories),
|
|
latencyBucket: z.enum(conversationalRectificationTelemetryLatencyBuckets),
|
|
billingState: z.enum(conversationalRectificationTelemetryBillingStates),
|
|
errorCategory: z.enum(conversationalRectificationTelemetryErrorCategories),
|
|
deploymentSha: deploymentShaSchema,
|
|
}).strict().readonly();
|
|
|
|
export type ConversationalRectificationTelemetryPayload = z.infer<
|
|
typeof conversationalRectificationTelemetryPayloadSchema
|
|
>;
|
|
export type ConversationalRectificationTelemetrySink = (
|
|
payload: ConversationalRectificationTelemetryPayload,
|
|
) => void;
|
|
|
|
function conversationalRectificationConsoleSink(
|
|
payload: ConversationalRectificationTelemetryPayload,
|
|
): void {
|
|
console.info("[conversational-rectification]", JSON.stringify(payload));
|
|
}
|
|
|
|
export function createConversationalRectificationTelemetry(
|
|
sink: ConversationalRectificationTelemetrySink = conversationalRectificationConsoleSink,
|
|
): ConversationalRectificationTelemetrySink {
|
|
return (input) => {
|
|
try {
|
|
const payload = conversationalRectificationTelemetryPayloadSchema.parse(input);
|
|
sink(payload);
|
|
} catch { // no-excuse-ok: observability cannot break the product request
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
export const recordConversationalRectificationTelemetry =
|
|
createConversationalRectificationTelemetry();
|
|
|
|
export function conversationalRectificationLatencyBucket(
|
|
latencyMs: number,
|
|
): ConversationalRectificationTelemetryPayload["latencyBucket"] {
|
|
if (!Number.isFinite(latencyMs) || latencyMs < 0) return "2s_plus";
|
|
if (latencyMs < 100) return "lt_100ms";
|
|
if (latencyMs < 500) return "100_499ms";
|
|
if (latencyMs < 2_000) return "500_1999ms";
|
|
return "2s_plus";
|
|
}
|
|
|
|
export function safeConversationalRectificationDeploymentSha(
|
|
value: string | undefined,
|
|
): ConversationalRectificationTelemetryPayload["deploymentSha"] {
|
|
const normalized = value?.trim().toLowerCase() ?? "";
|
|
return /^[0-9a-f]{7,64}$/.test(normalized) ? normalized : "unknown";
|
|
}
|
|
|
|
export const journeyMetricNames = [
|
|
"turn_advanced",
|
|
"draft_corrected",
|
|
"journey_paused",
|
|
"scoring_failed",
|
|
"scoring_recovered",
|
|
"illegal_snapshot",
|
|
] as const;
|
|
|
|
export type JourneyMetricName = (typeof journeyMetricNames)[number];
|
|
|
|
export const journeyMetricPhases = ["baseline", "adaptive", "result"] as const;
|
|
export type JourneyMetricPhase = (typeof journeyMetricPhases)[number];
|
|
|
|
export const journeyMetricConfidences = ["low", "medium", "high"] as const;
|
|
export type JourneyMetricConfidence = (typeof journeyMetricConfidences)[number];
|
|
|
|
export type JourneyMetricLabels = Readonly<{
|
|
readonly phase: JourneyMetricPhase;
|
|
readonly confidence?: JourneyMetricConfidence;
|
|
}>;
|
|
|
|
export type JourneyMetricPayload = Readonly<{
|
|
readonly name: JourneyMetricName;
|
|
readonly phase: JourneyMetricPhase;
|
|
readonly confidence?: JourneyMetricConfidence;
|
|
}>;
|
|
|
|
export type JourneyMetricSink = (payload: JourneyMetricPayload) => void;
|
|
export type JourneyMetricRecorder = (
|
|
name: JourneyMetricName,
|
|
labels: JourneyMetricLabels,
|
|
) => void;
|
|
|
|
export type JourneyMetricEvent =
|
|
| Readonly<{
|
|
readonly kind: "transition";
|
|
readonly name: Extract<JourneyMetricName, "turn_advanced" | "draft_corrected" | "journey_paused">;
|
|
readonly phase: JourneyMetricPhase;
|
|
readonly confidence?: JourneyMetricConfidence;
|
|
}>
|
|
| Readonly<{
|
|
readonly kind: "scoring";
|
|
readonly outcome: "failed";
|
|
readonly phase: JourneyMetricPhase;
|
|
}>
|
|
| Readonly<{
|
|
readonly kind: "scoring";
|
|
readonly outcome: "succeeded";
|
|
readonly priorFailure: boolean;
|
|
readonly phase: JourneyMetricPhase;
|
|
readonly confidence: JourneyMetricConfidence;
|
|
}>
|
|
| Readonly<{
|
|
readonly kind: "error";
|
|
readonly reason: "illegal_state" | "scoring_failure";
|
|
readonly phase: JourneyMetricPhase;
|
|
}>;
|
|
|
|
const metricPayloadSchema = z.object({
|
|
name: z.enum(journeyMetricNames),
|
|
phase: z.enum(journeyMetricPhases),
|
|
confidence: z.enum(journeyMetricConfidences).optional(),
|
|
}).strict().readonly();
|
|
|
|
export const journeyMetricPayloadSchema = metricPayloadSchema;
|
|
|
|
function consoleSink(payload: JourneyMetricPayload): void {
|
|
console.info("[birth-time-journey]", JSON.stringify(payload));
|
|
}
|
|
|
|
export function createJourneyTelemetry(
|
|
sink: JourneyMetricSink = consoleSink,
|
|
): JourneyMetricRecorder {
|
|
return (name, labels) => {
|
|
const labelsValue = z.object({
|
|
phase: z.enum(journeyMetricPhases),
|
|
confidence: z.enum(journeyMetricConfidences).optional(),
|
|
}).strict().parse(labels);
|
|
const payload = metricPayloadSchema.parse({ name, ...labelsValue });
|
|
try {
|
|
sink(payload);
|
|
} catch { // no-excuse-ok: catch
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
export const journeyMetric = createJourneyTelemetry();
|
|
|
|
function metricPayload(
|
|
name: JourneyMetricName,
|
|
phase: JourneyMetricPhase,
|
|
confidence: JourneyMetricConfidence | undefined,
|
|
): JourneyMetricPayload {
|
|
return confidence === undefined ? { name, phase } : { name, phase, confidence };
|
|
}
|
|
|
|
export function decideJourneyMetric(event: JourneyMetricEvent): JourneyMetricPayload | null {
|
|
switch (event.kind) {
|
|
case "transition":
|
|
return metricPayload(event.name, event.phase, event.confidence);
|
|
case "scoring":
|
|
if (event.outcome === "failed") return metricPayload("scoring_failed", event.phase, undefined);
|
|
if (event.outcome === "succeeded") return metricPayload(
|
|
event.priorFailure ? "scoring_recovered" : "turn_advanced",
|
|
event.phase,
|
|
event.confidence,
|
|
);
|
|
return null;
|
|
case "error":
|
|
if (event.reason === "illegal_state") return metricPayload("illegal_snapshot", event.phase, undefined);
|
|
if (event.reason === "scoring_failure") return metricPayload("scoring_failed", event.phase, undefined);
|
|
return null;
|
|
default: {
|
|
const exhaustive: never = event;
|
|
return exhaustive;
|
|
}
|
|
}
|
|
}
|
|
|
|
export function recordJourneyMetricEvent(
|
|
event: JourneyMetricEvent,
|
|
recorder: JourneyMetricRecorder = journeyMetric,
|
|
): void {
|
|
const payload = decideJourneyMetric(event);
|
|
if (payload === null) return;
|
|
if (payload.confidence === undefined) {
|
|
recorder(payload.name, { phase: payload.phase });
|
|
return;
|
|
}
|
|
recorder(payload.name, { phase: payload.phase, confidence: payload.confidence });
|
|
}
|
|
|
|
export function journeyResponseMetricPhase(
|
|
response: VersionedJourneyResponse,
|
|
): JourneyMetricPhase {
|
|
switch (response.nextAction.kind) {
|
|
case "ask_baseline_evidence":
|
|
return "baseline";
|
|
case "ask_adaptive_evidence":
|
|
return "adaptive";
|
|
case "review_evidence_draft":
|
|
case "score_pending":
|
|
case "retry_scoring":
|
|
return response.progress.adaptiveRound > 0 ? "adaptive" : "baseline";
|
|
case "paused":
|
|
return response.candidateResult === null && response.progress.adaptiveRound === 0
|
|
? "baseline"
|
|
: response.candidateResult === null
|
|
? "adaptive"
|
|
: "result";
|
|
case "present_low_result":
|
|
case "present_medium_result":
|
|
case "candidate_saved":
|
|
case "request_candidate_confirmation":
|
|
case "ready":
|
|
return "result";
|
|
default: {
|
|
const exhaustive: never = response.nextAction;
|
|
return exhaustive;
|
|
}
|
|
}
|
|
}
|
|
|
|
export function recordJourneyTransitionMetric(
|
|
response: VersionedJourneyResponse,
|
|
name: Extract<JourneyMetricName, "turn_advanced" | "draft_corrected" | "journey_paused">,
|
|
recorder: JourneyMetricRecorder = journeyMetric,
|
|
): void {
|
|
recordJourneyMetricEvent({
|
|
kind: "transition",
|
|
name,
|
|
phase: journeyResponseMetricPhase(response),
|
|
...(response.candidateResult === null
|
|
? {}
|
|
: { confidence: response.candidateResult.confidence }),
|
|
}, recorder);
|
|
}
|
|
|
|
export function recordScoringJourneyMetric(
|
|
before: VersionedJourneyResponse,
|
|
after: VersionedJourneyResponse,
|
|
recorder: JourneyMetricRecorder = journeyMetric,
|
|
): void {
|
|
if (before.turnVersion === after.turnVersion) return;
|
|
if (after.nextAction.kind === "retry_scoring") {
|
|
recordJourneyMetricEvent({
|
|
kind: "scoring",
|
|
outcome: "failed",
|
|
phase: journeyResponseMetricPhase(before),
|
|
}, recorder);
|
|
return;
|
|
}
|
|
if (
|
|
before.nextAction.kind !== "score_pending"
|
|
&& before.nextAction.kind !== "retry_scoring"
|
|
) return;
|
|
const confidence = after.candidateResult?.confidence;
|
|
if (confidence === undefined) return;
|
|
recordJourneyMetricEvent({
|
|
kind: "scoring",
|
|
outcome: "succeeded",
|
|
priorFailure: before.nextAction.kind === "retry_scoring",
|
|
phase: "result",
|
|
confidence,
|
|
}, recorder);
|
|
}
|