797 lines
25 KiB
TypeScript
797 lines
25 KiB
TypeScript
import { z } from "zod";
|
|
|
|
export const AGENT_GOLDEN_DATASET_VERSION = "agent_golden_dataset.v1" as const;
|
|
|
|
export const agentEvalGroups = [
|
|
"ordinary_consultation",
|
|
"birth_time_rectification",
|
|
"report",
|
|
"safety",
|
|
] as const;
|
|
|
|
export type AgentEvalGroup = typeof agentEvalGroups[number];
|
|
|
|
export const timingPrecisions = [
|
|
"none",
|
|
"broad_window",
|
|
"year",
|
|
"month",
|
|
"day",
|
|
"minute",
|
|
] as const;
|
|
|
|
export type TimingPrecision = typeof timingPrecisions[number];
|
|
|
|
const identifierSchema = z.string().regex(
|
|
/^[a-z][a-z0-9._-]{1,95}$/,
|
|
"must be a stable lower-case identifier",
|
|
);
|
|
|
|
const conversationTurnSchema = z.object({
|
|
turnId: identifierSchema,
|
|
speaker: z.enum(["user", "assistant"]),
|
|
intentCode: identifierSchema,
|
|
contextTags: z.array(identifierSchema).min(1).max(12),
|
|
syntheticSummaryOnly: z.literal(true),
|
|
}).strict();
|
|
|
|
const toolRequirementSchema = z.object({
|
|
tool: identifierSchema,
|
|
minCalls: z.number().int().positive().max(12),
|
|
}).strict();
|
|
|
|
const expectedOutcomeSchema = z.object({
|
|
requiredSkillIds: z.array(identifierSchema).max(4),
|
|
toolContract: z.object({
|
|
required: z.array(toolRequirementSchema).max(12),
|
|
allowed: z.array(identifierSchema).max(20),
|
|
maxCalls: z.number().int().nonnegative().max(30),
|
|
}).strict(),
|
|
requestedThemes: z.array(identifierSchema).max(12),
|
|
evidenceCatalog: z.array(identifierSchema).max(40),
|
|
minEvidenceBackedClaims: z.number().int().nonnegative().max(40),
|
|
timingPolicy: z.object({
|
|
maxPrecision: z.enum(timingPrecisions),
|
|
allowConfirmedExactMinute: z.boolean(),
|
|
allowGuaranteedTiming: z.boolean(),
|
|
}).strict(),
|
|
rectificationFocus: z.object({
|
|
expectedFocusId: identifierSchema,
|
|
expectedDomain: identifierSchema,
|
|
}).strict().nullable(),
|
|
performanceBudget: z.object({
|
|
maxLatencyMs: z.number().int().positive(),
|
|
maxCostUsd: z.number().nonnegative(),
|
|
}).strict(),
|
|
pendingModelReviews: z.tuple([
|
|
z.literal("naturalness_repetition"),
|
|
z.literal("follow_up_relevance"),
|
|
z.literal("unsupported_fact_model_review"),
|
|
]),
|
|
}).strict().superRefine((expected, context) => {
|
|
const allowed = new Set(expected.toolContract.allowed);
|
|
const requiredCalls = expected.toolContract.required.reduce(
|
|
(sum, requirement) => sum + requirement.minCalls,
|
|
0,
|
|
);
|
|
for (const requirement of expected.toolContract.required) {
|
|
if (!allowed.has(requirement.tool)) {
|
|
context.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
path: ["toolContract", "allowed"],
|
|
message: `${requirement.tool} must be allowlisted`,
|
|
});
|
|
}
|
|
}
|
|
if (requiredCalls > expected.toolContract.maxCalls) {
|
|
context.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
path: ["toolContract", "maxCalls"],
|
|
message: "maxCalls cannot be lower than the required call floor",
|
|
});
|
|
}
|
|
});
|
|
|
|
const goldenCaseSchema = z.object({
|
|
id: identifierSchema,
|
|
group: z.enum(agentEvalGroups),
|
|
subscenario: identifierSchema,
|
|
turns: z.array(conversationTurnSchema).min(2).max(12),
|
|
expected: expectedOutcomeSchema,
|
|
}).strict().superRefine((goldenCase, context) => {
|
|
if (goldenCase.group === "birth_time_rectification" && !goldenCase.expected.rectificationFocus) {
|
|
context.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
path: ["expected", "rectificationFocus"],
|
|
message: "rectification cases require an expected focus",
|
|
});
|
|
}
|
|
if (goldenCase.group !== "birth_time_rectification" && goldenCase.expected.rectificationFocus) {
|
|
context.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
path: ["expected", "rectificationFocus"],
|
|
message: "only rectification cases may define an expected focus",
|
|
});
|
|
}
|
|
});
|
|
|
|
export const agentGoldenDatasetSchema = z.object({
|
|
schemaVersion: z.literal(AGENT_GOLDEN_DATASET_VERSION),
|
|
deidentification: z.object({
|
|
mode: z.literal("synthetic_intent_codes_only"),
|
|
rawUserTextIncluded: z.literal(false),
|
|
}).strict(),
|
|
cases: z.array(goldenCaseSchema).min(4),
|
|
}).strict().superRefine((dataset, context) => {
|
|
const ids = new Set<string>();
|
|
for (const [index, goldenCase] of dataset.cases.entries()) {
|
|
if (ids.has(goldenCase.id)) {
|
|
context.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
path: ["cases", index, "id"],
|
|
message: `duplicate case id: ${goldenCase.id}`,
|
|
});
|
|
}
|
|
ids.add(goldenCase.id);
|
|
}
|
|
|
|
const coveredGroups = new Set(dataset.cases.map((goldenCase) => goldenCase.group));
|
|
for (const group of agentEvalGroups) {
|
|
if (!coveredGroups.has(group)) {
|
|
context.addIssue({
|
|
code: z.ZodIssueCode.custom,
|
|
path: ["cases"],
|
|
message: `missing eval group: ${group}`,
|
|
});
|
|
}
|
|
}
|
|
});
|
|
|
|
export type AgentGoldenDataset = z.infer<typeof agentGoldenDatasetSchema>;
|
|
export type AgentGoldenCase = AgentGoldenDataset["cases"][number];
|
|
|
|
export function parseAgentGoldenDataset(value: unknown): AgentGoldenDataset {
|
|
const dataset = agentGoldenDatasetSchema.parse(value);
|
|
const privacyViolations = findDatasetPrivacyViolations(dataset);
|
|
if (privacyViolations.length > 0) {
|
|
throw new Error(
|
|
`agent golden dataset privacy violation: ${privacyViolations[0]?.rule} at ${privacyViolations[0]?.path}`,
|
|
);
|
|
}
|
|
return dataset;
|
|
}
|
|
|
|
export type AgentSkillExecution = Readonly<{
|
|
skillId: string;
|
|
status: "completed" | "failed" | "pending";
|
|
}>;
|
|
|
|
export type AgentToolCall = Readonly<{
|
|
tool: string;
|
|
status: "completed" | "failed" | "pending";
|
|
inputDigest?: string;
|
|
latencyMs?: number;
|
|
costUsd?: number;
|
|
}>;
|
|
|
|
export type AgentClaim = Readonly<{
|
|
claimId: string;
|
|
kind: "fact" | "interpretation" | "recommendation" | "timing";
|
|
requiresEvidence: boolean;
|
|
evidenceIds: readonly string[];
|
|
themeIds: readonly string[];
|
|
timingPrecision?: TimingPrecision;
|
|
timingModality?: "candidate" | "accepted" | "confirmed" | "guaranteed";
|
|
}>;
|
|
|
|
export type AgentEvalRun = Readonly<{
|
|
caseId: string;
|
|
candidateResponse: string;
|
|
skillExecutions: readonly AgentSkillExecution[];
|
|
toolCalls: readonly AgentToolCall[];
|
|
availableEvidenceIds: readonly string[];
|
|
producedEvidenceIds: readonly string[];
|
|
claims: readonly AgentClaim[];
|
|
coveredThemes: readonly string[];
|
|
rectificationFocus?: Readonly<{
|
|
focusId: string | null;
|
|
domain: string | null;
|
|
}>;
|
|
observability: Readonly<{
|
|
latencyMs: number;
|
|
costUsd: number;
|
|
inputTokens?: number;
|
|
outputTokens?: number;
|
|
}>;
|
|
}>;
|
|
|
|
export type DeterministicMetric<T> = Readonly<{
|
|
evaluationMode: "deterministic";
|
|
status: "passed" | "failed" | "not_applicable";
|
|
score: number | null;
|
|
details: T;
|
|
}>;
|
|
|
|
function round(value: number, digits = 6) {
|
|
const factor = 10 ** digits;
|
|
return Math.round((value + Number.EPSILON) * factor) / factor;
|
|
}
|
|
|
|
function ratio(numerator: number, denominator: number) {
|
|
return denominator === 0 ? 1 : round(numerator / denominator);
|
|
}
|
|
|
|
function completedToolCounts(run: AgentEvalRun) {
|
|
const counts = new Map<string, number>();
|
|
for (const call of run.toolCalls) {
|
|
if (call.status !== "completed") continue;
|
|
counts.set(call.tool, (counts.get(call.tool) ?? 0) + 1);
|
|
}
|
|
return counts;
|
|
}
|
|
|
|
function missingToolRequirements(goldenCase: AgentGoldenCase, run: AgentEvalRun) {
|
|
const completed = completedToolCounts(run);
|
|
return goldenCase.expected.toolContract.required.flatMap((requirement) => {
|
|
const actual = completed.get(requirement.tool) ?? 0;
|
|
return actual >= requirement.minCalls
|
|
? []
|
|
: [{ tool: requirement.tool, expected: requirement.minCalls, actual }];
|
|
});
|
|
}
|
|
|
|
export function scoreSkillToolContractCompletion(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
missingSkills: readonly string[];
|
|
missingTools: readonly Readonly<{ tool: string; expected: number; actual: number }>[];
|
|
completedRequirements: number;
|
|
totalRequirements: number;
|
|
}> {
|
|
const completedSkills = new Set(
|
|
run.skillExecutions
|
|
.filter((execution) => execution.status === "completed")
|
|
.map((execution) => execution.skillId),
|
|
);
|
|
const missingSkills = goldenCase.expected.requiredSkillIds.filter(
|
|
(skillId) => !completedSkills.has(skillId),
|
|
);
|
|
const missingTools = missingToolRequirements(goldenCase, run);
|
|
const totalRequirements = goldenCase.expected.requiredSkillIds.length
|
|
+ goldenCase.expected.toolContract.required.length;
|
|
const completedRequirements = totalRequirements - missingSkills.length - missingTools.length;
|
|
|
|
if (totalRequirements === 0) {
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: "not_applicable",
|
|
score: null,
|
|
details: { missingSkills, missingTools, completedRequirements, totalRequirements },
|
|
};
|
|
}
|
|
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: missingSkills.length === 0 && missingTools.length === 0 ? "passed" : "failed",
|
|
score: ratio(completedRequirements, totalRequirements),
|
|
details: { missingSkills, missingTools, completedRequirements, totalRequirements },
|
|
};
|
|
}
|
|
|
|
function claimRequiresEvidence(claim: AgentClaim) {
|
|
return claim.kind === "fact" || claim.kind === "timing" || claim.requiresEvidence;
|
|
}
|
|
|
|
function runEvidenceIds(run: AgentEvalRun) {
|
|
return new Set([...run.availableEvidenceIds, ...run.producedEvidenceIds]);
|
|
}
|
|
|
|
function validEvidenceIds(goldenCase: AgentGoldenCase, run: AgentEvalRun, claim: AgentClaim) {
|
|
const catalog = new Set(goldenCase.expected.evidenceCatalog);
|
|
const available = runEvidenceIds(run);
|
|
return claim.evidenceIds.filter(
|
|
(evidenceId) => catalog.has(evidenceId) && available.has(evidenceId),
|
|
);
|
|
}
|
|
|
|
export function scoreEvidenceCitationClosure(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
requiredClaimCount: number;
|
|
closedClaimIds: readonly string[];
|
|
unclosedClaimIds: readonly string[];
|
|
danglingEvidenceIds: readonly string[];
|
|
uncatalogedEvidenceIds: readonly string[];
|
|
unavailableEvidenceIds: readonly string[];
|
|
missingExpectedClaims: number;
|
|
}> {
|
|
const evidenceRequiredClaims = run.claims.filter(claimRequiresEvidence);
|
|
const catalog = new Set(goldenCase.expected.evidenceCatalog);
|
|
const available = runEvidenceIds(run);
|
|
const citedEvidenceIds = [...new Set(run.claims.flatMap((claim) => claim.evidenceIds))];
|
|
const uncatalogedEvidenceIds = citedEvidenceIds.filter((evidenceId) => !catalog.has(evidenceId));
|
|
const unavailableEvidenceIds = citedEvidenceIds.filter((evidenceId) => !available.has(evidenceId));
|
|
const danglingEvidenceIds = citedEvidenceIds.filter(
|
|
(evidenceId) => !catalog.has(evidenceId) || !available.has(evidenceId),
|
|
);
|
|
const closedClaimIds = evidenceRequiredClaims
|
|
.filter((claim) => (
|
|
claim.evidenceIds.length > 0
|
|
&& claim.evidenceIds.every((id) => catalog.has(id) && available.has(id))
|
|
))
|
|
.map((claim) => claim.claimId);
|
|
const unclosedClaimIds = evidenceRequiredClaims
|
|
.filter((claim) => !closedClaimIds.includes(claim.claimId))
|
|
.map((claim) => claim.claimId);
|
|
const missingExpectedClaims = Math.max(
|
|
0,
|
|
goldenCase.expected.minEvidenceBackedClaims - evidenceRequiredClaims.length,
|
|
);
|
|
const denominator = Math.max(
|
|
evidenceRequiredClaims.length,
|
|
goldenCase.expected.minEvidenceBackedClaims,
|
|
);
|
|
const passed = unclosedClaimIds.length === 0
|
|
&& danglingEvidenceIds.length === 0
|
|
&& missingExpectedClaims === 0;
|
|
|
|
if (denominator === 0) {
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: danglingEvidenceIds.length === 0 ? "not_applicable" : "failed",
|
|
score: danglingEvidenceIds.length === 0 ? null : 0,
|
|
details: {
|
|
requiredClaimCount: 0,
|
|
closedClaimIds,
|
|
unclosedClaimIds,
|
|
danglingEvidenceIds,
|
|
uncatalogedEvidenceIds,
|
|
unavailableEvidenceIds,
|
|
missingExpectedClaims,
|
|
},
|
|
};
|
|
}
|
|
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: passed ? "passed" : "failed",
|
|
score: passed ? 1 : ratio(closedClaimIds.length, denominator),
|
|
details: {
|
|
requiredClaimCount: evidenceRequiredClaims.length,
|
|
closedClaimIds,
|
|
unclosedClaimIds,
|
|
danglingEvidenceIds,
|
|
uncatalogedEvidenceIds,
|
|
unavailableEvidenceIds,
|
|
missingExpectedClaims,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function scoreRequestedThemeCoverage(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
requestedThemes: readonly string[];
|
|
coveredThemes: readonly string[];
|
|
missingThemes: readonly string[];
|
|
}> {
|
|
const covered = new Set([
|
|
...run.coveredThemes,
|
|
...run.claims.flatMap((claim) => claim.themeIds),
|
|
]);
|
|
const requestedThemes = goldenCase.expected.requestedThemes;
|
|
const missingThemes = requestedThemes.filter((theme) => !covered.has(theme));
|
|
|
|
if (requestedThemes.length === 0) {
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: "not_applicable",
|
|
score: null,
|
|
details: { requestedThemes, coveredThemes: [...covered], missingThemes },
|
|
};
|
|
}
|
|
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: missingThemes.length === 0 ? "passed" : "failed",
|
|
score: ratio(requestedThemes.length - missingThemes.length, requestedThemes.length),
|
|
details: { requestedThemes, coveredThemes: [...covered], missingThemes },
|
|
};
|
|
}
|
|
|
|
export function countUnsupportedFactsByRule(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
count: number;
|
|
claimIds: readonly string[];
|
|
rule: "evidence_required_claim_without_valid_run_reference";
|
|
}> {
|
|
const claimIds = run.claims
|
|
.filter((claim) => claimRequiresEvidence(claim) && validEvidenceIds(goldenCase, run, claim).length === 0)
|
|
.map((claim) => claim.claimId);
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: claimIds.length === 0 ? "passed" : "failed",
|
|
score: claimIds.length === 0 ? 1 : 0,
|
|
details: {
|
|
count: claimIds.length,
|
|
claimIds,
|
|
rule: "evidence_required_claim_without_valid_run_reference",
|
|
},
|
|
};
|
|
}
|
|
|
|
const precisionRank: Record<TimingPrecision, number> = {
|
|
none: 0,
|
|
broad_window: 1,
|
|
year: 2,
|
|
month: 3,
|
|
day: 4,
|
|
minute: 5,
|
|
};
|
|
|
|
export function scorePreciseTimingViolations(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
count: number;
|
|
violations: readonly Readonly<{ claimId: string; rules: readonly string[] }>[];
|
|
}> {
|
|
const policy = goldenCase.expected.timingPolicy;
|
|
const violations = run.claims.flatMap((claim) => {
|
|
if (claim.kind !== "timing") return [];
|
|
const precision = claim.timingPrecision ?? "none";
|
|
const modality = claim.timingModality ?? "candidate";
|
|
const rules: string[] = [];
|
|
if (precisionRank[precision] > precisionRank[policy.maxPrecision]) {
|
|
rules.push("precision_exceeds_case_boundary");
|
|
}
|
|
if (
|
|
precision === "minute"
|
|
&& (modality === "confirmed" || modality === "guaranteed")
|
|
&& !policy.allowConfirmedExactMinute
|
|
) {
|
|
rules.push("exact_minute_confirmation_forbidden");
|
|
}
|
|
if (modality === "guaranteed" && !policy.allowGuaranteedTiming) {
|
|
rules.push("guaranteed_timing_forbidden");
|
|
}
|
|
return rules.length > 0 ? [{ claimId: claim.claimId, rules }] : [];
|
|
});
|
|
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: violations.length === 0 ? "passed" : "failed",
|
|
score: violations.length === 0 ? 1 : 0,
|
|
details: { count: violations.length, violations },
|
|
};
|
|
}
|
|
|
|
export function scoreRectificationFocusAccuracy(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
expectedFocusId: string | null;
|
|
actualFocusId: string | null;
|
|
expectedDomain: string | null;
|
|
actualDomain: string | null;
|
|
}> {
|
|
const expected = goldenCase.expected.rectificationFocus;
|
|
if (!expected) {
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: "not_applicable",
|
|
score: null,
|
|
details: {
|
|
expectedFocusId: null,
|
|
actualFocusId: run.rectificationFocus?.focusId ?? null,
|
|
expectedDomain: null,
|
|
actualDomain: run.rectificationFocus?.domain ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
const actualFocusId = run.rectificationFocus?.focusId ?? null;
|
|
const actualDomain = run.rectificationFocus?.domain ?? null;
|
|
const focusMatches = actualFocusId === expected.expectedFocusId;
|
|
const domainMatches = actualDomain === expected.expectedDomain;
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: focusMatches && domainMatches ? "passed" : "failed",
|
|
score: focusMatches && domainMatches ? 1 : focusMatches || domainMatches ? 0.5 : 0,
|
|
details: {
|
|
expectedFocusId: expected.expectedFocusId,
|
|
actualFocusId,
|
|
expectedDomain: expected.expectedDomain,
|
|
actualDomain,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function scoreToolCallEconomy(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): DeterministicMetric<{
|
|
totalCalls: number;
|
|
maxCalls: number;
|
|
failedCalls: number;
|
|
pendingCalls: number;
|
|
unallowedCalls: readonly string[];
|
|
duplicateInputCalls: readonly string[];
|
|
overBudgetCalls: number;
|
|
missingRequiredTools: readonly Readonly<{ tool: string; expected: number; actual: number }>[];
|
|
}> {
|
|
const allowed = new Set(goldenCase.expected.toolContract.allowed);
|
|
const failedCalls = run.toolCalls.filter((call) => call.status === "failed").length;
|
|
const pendingCalls = run.toolCalls.filter((call) => call.status === "pending").length;
|
|
const unallowedCalls = run.toolCalls
|
|
.filter((call) => !allowed.has(call.tool))
|
|
.map((call) => call.tool);
|
|
const seenDigests = new Set<string>();
|
|
const duplicateInputCalls: string[] = [];
|
|
for (const call of run.toolCalls) {
|
|
if (!call.inputDigest) continue;
|
|
const key = `${call.tool}:${call.inputDigest}`;
|
|
if (seenDigests.has(key)) duplicateInputCalls.push(key);
|
|
seenDigests.add(key);
|
|
}
|
|
const overBudgetCalls = Math.max(
|
|
0,
|
|
run.toolCalls.length - goldenCase.expected.toolContract.maxCalls,
|
|
);
|
|
const missingRequiredTools = missingToolRequirements(goldenCase, run);
|
|
const issueCount = failedCalls
|
|
+ pendingCalls
|
|
+ unallowedCalls.length
|
|
+ duplicateInputCalls.length
|
|
+ overBudgetCalls
|
|
+ missingRequiredTools.length;
|
|
const applicable = goldenCase.expected.toolContract.maxCalls > 0
|
|
|| goldenCase.expected.toolContract.allowed.length > 0
|
|
|| run.toolCalls.length > 0;
|
|
|
|
if (!applicable) {
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: "not_applicable",
|
|
score: null,
|
|
details: {
|
|
totalCalls: 0,
|
|
maxCalls: 0,
|
|
failedCalls,
|
|
pendingCalls,
|
|
unallowedCalls,
|
|
duplicateInputCalls,
|
|
overBudgetCalls,
|
|
missingRequiredTools,
|
|
},
|
|
};
|
|
}
|
|
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: issueCount === 0 ? "passed" : "failed",
|
|
score: issueCount === 0
|
|
? 1
|
|
: Math.max(0, round(1 - issueCount / Math.max(run.toolCalls.length + 1, 1))),
|
|
details: {
|
|
totalCalls: run.toolCalls.length,
|
|
maxCalls: goldenCase.expected.toolContract.maxCalls,
|
|
failedCalls,
|
|
pendingCalls,
|
|
unallowedCalls,
|
|
duplicateInputCalls,
|
|
overBudgetCalls,
|
|
missingRequiredTools,
|
|
},
|
|
};
|
|
}
|
|
|
|
type NumericStatistics = Readonly<{
|
|
min: number;
|
|
max: number;
|
|
mean: number;
|
|
p50: number;
|
|
p95: number;
|
|
total: number;
|
|
}>;
|
|
|
|
function numericStatistics(values: readonly number[]): NumericStatistics {
|
|
if (values.length === 0) {
|
|
return { min: 0, max: 0, mean: 0, p50: 0, p95: 0, total: 0 };
|
|
}
|
|
const sorted = [...values].sort((left, right) => left - right);
|
|
const total = sorted.reduce((sum, value) => sum + value, 0);
|
|
const nearestRank = (percentile: number) => {
|
|
const index = Math.max(0, Math.ceil(percentile * sorted.length) - 1);
|
|
return sorted[index] ?? 0;
|
|
};
|
|
return {
|
|
min: round(sorted[0] ?? 0),
|
|
max: round(sorted[sorted.length - 1] ?? 0),
|
|
mean: round(total / sorted.length),
|
|
p50: round(nearestRank(0.5)),
|
|
p95: round(nearestRank(0.95)),
|
|
total: round(total),
|
|
};
|
|
}
|
|
|
|
export function summarizeLatencyAndCost(
|
|
cases: readonly AgentGoldenCase[],
|
|
runs: readonly AgentEvalRun[],
|
|
): DeterministicMetric<{
|
|
runCount: number;
|
|
latencyMs: NumericStatistics;
|
|
costUsd: NumericStatistics;
|
|
inputTokens: NumericStatistics;
|
|
outputTokens: NumericStatistics;
|
|
latencyBudgetBreaches: readonly string[];
|
|
costBudgetBreaches: readonly string[];
|
|
}> {
|
|
const casesById = new Map(cases.map((goldenCase) => [goldenCase.id, goldenCase]));
|
|
const latencyBudgetBreaches: string[] = [];
|
|
const costBudgetBreaches: string[] = [];
|
|
for (const run of runs) {
|
|
const goldenCase = casesById.get(run.caseId);
|
|
if (!goldenCase) throw new Error(`missing golden case for run: ${run.caseId}`);
|
|
if (run.observability.latencyMs > goldenCase.expected.performanceBudget.maxLatencyMs) {
|
|
latencyBudgetBreaches.push(run.caseId);
|
|
}
|
|
if (run.observability.costUsd > goldenCase.expected.performanceBudget.maxCostUsd) {
|
|
costBudgetBreaches.push(run.caseId);
|
|
}
|
|
}
|
|
const breachCount = latencyBudgetBreaches.length + costBudgetBreaches.length;
|
|
const budgetChecks = runs.length * 2;
|
|
return {
|
|
evaluationMode: "deterministic",
|
|
status: breachCount === 0 ? "passed" : "failed",
|
|
score: ratio(budgetChecks - breachCount, budgetChecks),
|
|
details: {
|
|
runCount: runs.length,
|
|
latencyMs: numericStatistics(runs.map((run) => run.observability.latencyMs)),
|
|
costUsd: numericStatistics(runs.map((run) => run.observability.costUsd)),
|
|
inputTokens: numericStatistics(runs.map((run) => run.observability.inputTokens ?? 0)),
|
|
outputTokens: numericStatistics(runs.map((run) => run.observability.outputTokens ?? 0)),
|
|
latencyBudgetBreaches,
|
|
costBudgetBreaches,
|
|
},
|
|
};
|
|
}
|
|
|
|
export const pendingModelReviewCriteria = [
|
|
"naturalness_repetition",
|
|
"follow_up_relevance",
|
|
"unsupported_fact_model_review",
|
|
] as const;
|
|
|
|
export type PendingModelReview = Readonly<{
|
|
evaluationMode: "model_review";
|
|
status: "pending";
|
|
criterion: typeof pendingModelReviewCriteria[number];
|
|
caseId: string;
|
|
input: Readonly<{
|
|
candidateResponse: string;
|
|
conversationIntentCodes: readonly string[];
|
|
structuredClaimIds: readonly string[];
|
|
}>;
|
|
}>;
|
|
|
|
export function createPendingModelReviewInputs(
|
|
goldenCase: AgentGoldenCase,
|
|
run: AgentEvalRun,
|
|
): readonly PendingModelReview[] {
|
|
const input = {
|
|
candidateResponse: run.candidateResponse,
|
|
conversationIntentCodes: goldenCase.turns.map((turn) => turn.intentCode),
|
|
structuredClaimIds: run.claims.map((claim) => claim.claimId),
|
|
};
|
|
return goldenCase.expected.pendingModelReviews.map((criterion) => ({
|
|
evaluationMode: "model_review" as const,
|
|
status: "pending" as const,
|
|
criterion,
|
|
caseId: goldenCase.id,
|
|
input,
|
|
}));
|
|
}
|
|
|
|
export function evaluateAgentRun(goldenCase: AgentGoldenCase, run: AgentEvalRun) {
|
|
if (goldenCase.id !== run.caseId) {
|
|
throw new Error(`run caseId ${run.caseId} does not match golden case ${goldenCase.id}`);
|
|
}
|
|
return {
|
|
caseId: goldenCase.id,
|
|
deterministic: {
|
|
skillToolContractCompletion: scoreSkillToolContractCompletion(goldenCase, run),
|
|
evidenceCitationClosure: scoreEvidenceCitationClosure(goldenCase, run),
|
|
requestedThemeCoverage: scoreRequestedThemeCoverage(goldenCase, run),
|
|
unsupportedFactRuleCount: countUnsupportedFactsByRule(goldenCase, run),
|
|
preciseTimingViolation: scorePreciseTimingViolations(goldenCase, run),
|
|
rectificationFocusAccuracy: scoreRectificationFocusAccuracy(goldenCase, run),
|
|
toolCallEconomy: scoreToolCallEconomy(goldenCase, run),
|
|
latencyCostStatistics: summarizeLatencyAndCost([goldenCase], [run]),
|
|
},
|
|
modelReview: createPendingModelReviewInputs(goldenCase, run),
|
|
} as const;
|
|
}
|
|
|
|
export type DatasetPrivacyViolation = Readonly<{
|
|
path: string;
|
|
rule:
|
|
| "forbidden_identity_field"
|
|
| "raw_user_text_field"
|
|
| "email"
|
|
| "birth_date"
|
|
| "clock_time"
|
|
| "api_credential"
|
|
| "internal_absolute_path";
|
|
}>;
|
|
|
|
const forbiddenIdentityKeys = new Set([
|
|
"name",
|
|
"fullname",
|
|
"displayname",
|
|
"email",
|
|
"birthdate",
|
|
"birthtime",
|
|
"birthplace",
|
|
"location",
|
|
"latitude",
|
|
"longitude",
|
|
]);
|
|
const rawUserTextKeys = new Set([
|
|
"content",
|
|
"text",
|
|
"message",
|
|
"prompt",
|
|
"rawusertext",
|
|
"usertext",
|
|
"quote",
|
|
]);
|
|
|
|
export function findDatasetPrivacyViolations(value: unknown): readonly DatasetPrivacyViolation[] {
|
|
const violations: DatasetPrivacyViolation[] = [];
|
|
const visit = (current: unknown, path: string) => {
|
|
if (typeof current === "string") {
|
|
if (/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i.test(current)) {
|
|
violations.push({ path, rule: "email" });
|
|
}
|
|
if (/\b(?:19|20)\d{2}[-/.年](?:0?[1-9]|1[0-2])[-/.月](?:0?[1-9]|[12]\d|3[01])日?\b/.test(current)) {
|
|
violations.push({ path, rule: "birth_date" });
|
|
}
|
|
if (/(?:^|\D)(?:[01]?\d|2[0-3]):[0-5]\d(?:\D|$)/.test(current)) {
|
|
violations.push({ path, rule: "clock_time" });
|
|
}
|
|
if (/(?:^|[^A-Za-z0-9])(?:sk-[A-Za-z0-9_-]{12,}|api[_ -]?key\s*[:=]|bearer\s+[A-Za-z0-9._-]{12,})/i.test(current)) {
|
|
violations.push({ path, rule: "api_credential" });
|
|
}
|
|
if (/(?:\/Users\/|\/home\/|\/opt\/|\/private\/|[A-Za-z]:\\Users\\)/.test(current)) {
|
|
violations.push({ path, rule: "internal_absolute_path" });
|
|
}
|
|
return;
|
|
}
|
|
if (Array.isArray(current)) {
|
|
current.forEach((item, index) => visit(item, `${path}[${index}]`));
|
|
return;
|
|
}
|
|
if (!current || typeof current !== "object") return;
|
|
for (const [key, child] of Object.entries(current)) {
|
|
const normalizedKey = key.replace(/[^a-z]/gi, "").toLowerCase();
|
|
const childPath = path ? `${path}.${key}` : key;
|
|
if (forbiddenIdentityKeys.has(normalizedKey)) {
|
|
violations.push({ path: childPath, rule: "forbidden_identity_field" });
|
|
}
|
|
if (rawUserTextKeys.has(normalizedKey)) {
|
|
violations.push({ path: childPath, rule: "raw_user_text_field" });
|
|
}
|
|
visit(child, childPath);
|
|
}
|
|
};
|
|
visit(value, "$dataset");
|
|
return violations;
|
|
}
|