test: add agent evals and safe observability
This commit is contained in:
Jesse_Chen
@@ -10,6 +10,12 @@ import {
|
||||
import { blocksPromptExtraction } from "@/lib/consult-safety";
|
||||
import { consultationDomainSchema } from "@/lib/consultation-domain-registry";
|
||||
import { parseAgentReply } from "@/lib/agent-reply";
|
||||
import {
|
||||
logAgentObservability,
|
||||
settlementTelemetryOutcome,
|
||||
toAgentObservabilityErrorCode,
|
||||
type AgentSettlementResult,
|
||||
} from "@/lib/agent-observability";
|
||||
import {
|
||||
consultationEntrypointSchema,
|
||||
resolveConsultationQuestion,
|
||||
@@ -355,7 +361,7 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
|
||||
async function cancel() {
|
||||
async function cancel(): Promise<AgentSettlementResult> {
|
||||
try {
|
||||
const result = await retryDetachedSettlement(async () => {
|
||||
const { data, error } = await accounting.rpc("cancel_consultation_credit", {
|
||||
@@ -371,11 +377,13 @@ export async function POST(request: Request) {
|
||||
if (!result.success && result.error_code !== "request_completed") {
|
||||
throw new CreditRpcError(result.error_code || "cancellation_rejected");
|
||||
}
|
||||
return "cancelled";
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.name : "UnknownError";
|
||||
console.error(
|
||||
`[billing] cancellation failed request=${requestId} reason=${reason}`,
|
||||
);
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,7 +412,7 @@ export async function POST(request: Request) {
|
||||
techniqueTruth: string,
|
||||
workflowReceipt: WorkflowReceipt,
|
||||
agentExecutionReceipt?: AgentExecutionReceipt,
|
||||
) {
|
||||
): Promise<AgentSettlementResult> {
|
||||
try {
|
||||
const reply = parseAgentReply(rawTransformedText, consultationTheme);
|
||||
if (!reply.text) throw new Error("empty_agent_reply");
|
||||
@@ -434,17 +442,21 @@ export async function POST(request: Request) {
|
||||
if (!completion.success && completion.error_code !== "request_cancelled") {
|
||||
throw new CreditRpcError(completion.error_code || "completion_rejected");
|
||||
}
|
||||
return "completed";
|
||||
} catch (error) {
|
||||
await cancel();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
let settlement: Promise<void> | null = null;
|
||||
function settle(action: () => Promise<void>) {
|
||||
let settlement: Promise<AgentSettlementResult> | null = null;
|
||||
function settleResult(action: () => Promise<AgentSettlementResult>) {
|
||||
settlement ??= action();
|
||||
return settlement;
|
||||
}
|
||||
async function settle(action: () => Promise<AgentSettlementResult>): Promise<void> {
|
||||
await settleResult(action);
|
||||
}
|
||||
|
||||
async function runAgenticConsultation(
|
||||
consultationMode: ConsultationBirthTimeMode,
|
||||
@@ -460,32 +472,82 @@ export async function POST(request: Request) {
|
||||
let logged = false;
|
||||
const markFirstActivity = () => { if (firstActivityMs < 0) firstActivityMs = Date.now() - agentStartedAt; };
|
||||
const markFirstText = () => { if (firstTextMs < 0) firstTextMs = Date.now() - agentStartedAt; };
|
||||
const logRun = (finishReason: string, settlementResult: string) => {
|
||||
const logRun = async (
|
||||
errorCode: string | undefined,
|
||||
settlementResult: AgentSettlementResult,
|
||||
) => {
|
||||
if (logged) return;
|
||||
logged = true;
|
||||
console.info([
|
||||
"[consult-agentic]",
|
||||
`request_id=${requestId}`,
|
||||
`run_id=${requestId}`,
|
||||
`session_id=${sessionId}`,
|
||||
`model_id=${selectedModel.id}`,
|
||||
`skill_loaded=${state.jyotishSkillLoaded}`,
|
||||
`skill_reference_read_count=${state.skillReferenceReadCount}`,
|
||||
`consultation_tool_call_count=${state.consultationToolCallCount}`,
|
||||
`consultation_tool_duration_ms=${state.consultationToolDurationMs ?? -1}`,
|
||||
`time_to_first_activity_ms=${firstActivityMs}`,
|
||||
`time_to_first_text_ms=${firstTextMs}`,
|
||||
`total_duration_ms=${Date.now() - agentStartedAt}`,
|
||||
`finish_reason=${finishReason}`,
|
||||
`settlement_result=${settlementResult}`,
|
||||
].join(" "));
|
||||
const resolvedUsage: Usage = await mergeUsage(usages).catch(() => ({}));
|
||||
const inputTokens = Math.max(0, Math.trunc(resolvedUsage.inputTokens ?? 0));
|
||||
const outputTokens = Math.max(0, Math.trunc(resolvedUsage.outputTokens ?? 0));
|
||||
const skillStep = state.steps.find((step) => step.kind === "skill");
|
||||
const runStatus = errorCode === undefined
|
||||
? "completed"
|
||||
: errorCode === "cancelled"
|
||||
? "cancelled"
|
||||
: "failed";
|
||||
logAgentObservability({
|
||||
runId: requestId,
|
||||
requestId,
|
||||
sessionId,
|
||||
agentVersion: "consultation-agentic-v1",
|
||||
modelVersion: String(selectedModel.configVersion),
|
||||
policyVersion: "consultation-runtime-contract-v1",
|
||||
toolCalls: state.steps
|
||||
.filter((step) => step.kind === "tool")
|
||||
.map((step) => ({
|
||||
name: step.name,
|
||||
durationMs: Math.max(0, Math.trunc(step.durationMs ?? 0)),
|
||||
status: step.status,
|
||||
})),
|
||||
contractPhases: [
|
||||
{
|
||||
phase: "skill.load",
|
||||
...(skillStep?.durationMs === undefined
|
||||
? {}
|
||||
: { durationMs: Math.max(0, Math.trunc(skillStep.durationMs)) }),
|
||||
status: state.jyotishSkillLoaded ? "completed" : "failed",
|
||||
},
|
||||
{
|
||||
phase: "answer.first_activity",
|
||||
...(firstActivityMs < 0 ? {} : { durationMs: firstActivityMs }),
|
||||
status: firstActivityMs < 0 ? "skipped" : "completed",
|
||||
},
|
||||
{
|
||||
phase: "answer.first_output",
|
||||
...(firstTextMs < 0 ? {} : { durationMs: firstTextMs }),
|
||||
status: firstTextMs < 0 ? "skipped" : "completed",
|
||||
},
|
||||
{
|
||||
phase: "run.total",
|
||||
durationMs: Date.now() - agentStartedAt,
|
||||
status: runStatus,
|
||||
},
|
||||
{
|
||||
phase: "billing.settled",
|
||||
status: settlementResult,
|
||||
},
|
||||
],
|
||||
retryCount: Math.max(0, usages.length - 1),
|
||||
...(errorCode === undefined ? {} : { errorCode }),
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
themeCoverage: state.workflowReceipt?.domains ?? [consultationTheme],
|
||||
billingSettlementResult: settlementResult,
|
||||
});
|
||||
};
|
||||
const settleRun = async (action: () => Promise<void>, finishReason: string, settlementResult: string) => {
|
||||
const settleRun = async (
|
||||
action: () => Promise<AgentSettlementResult>,
|
||||
errorCode: string | undefined,
|
||||
) => {
|
||||
try {
|
||||
await settle(action);
|
||||
logRun(finishReason, settlementResult);
|
||||
const actualSettlementResult = await settleResult(action);
|
||||
const outcome = settlementTelemetryOutcome(actualSettlementResult, errorCode);
|
||||
await logRun(outcome.errorCode, outcome.billingSettlementResult);
|
||||
} catch (error) {
|
||||
logRun("settlement_failed", "failed");
|
||||
const outcome = settlementTelemetryOutcome("failed", errorCode);
|
||||
await logRun(outcome.errorCode, outcome.billingSettlementResult);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -557,13 +619,12 @@ export async function POST(request: Request) {
|
||||
"not-applicable",
|
||||
workflowReceipt,
|
||||
agentExecutionReceipt,
|
||||
), "completed", "completed"),
|
||||
), undefined),
|
||||
onError: (error) => settleRun(
|
||||
cancel,
|
||||
error instanceof Error ? error.message : "failed",
|
||||
"cancelled",
|
||||
toAgentObservabilityErrorCode(error),
|
||||
),
|
||||
onCancel: () => settleRun(cancel, "cancelled", "cancelled"),
|
||||
onCancel: () => settleRun(cancel, "cancelled"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -622,13 +683,12 @@ export async function POST(request: Request) {
|
||||
state.techniqueTruth ?? "unknown",
|
||||
state.workflowReceipt ?? workflowReceipt,
|
||||
agentExecutionReceipt,
|
||||
), "completed", "completed"),
|
||||
), undefined),
|
||||
onError: (error) => settleRun(
|
||||
cancel,
|
||||
error instanceof Error ? error.message : "failed",
|
||||
"cancelled",
|
||||
toAgentObservabilityErrorCode(error),
|
||||
),
|
||||
onCancel: () => settleRun(cancel, "cancelled", "cancelled"),
|
||||
onCancel: () => settleRun(cancel, "cancelled"),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user