Jesse_Chen
617 lines
24 KiB
TypeScript
617 lines
24 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { readFileSync } from "node:fs";
|
|
import test from "node:test";
|
|
import {
|
|
agentEvalGroups,
|
|
createPendingModelReviewInputs,
|
|
evaluateAgentRun,
|
|
findDatasetPrivacyViolations,
|
|
parseAgentGoldenDataset,
|
|
summarizeLatencyAndCost,
|
|
type AgentClaim,
|
|
type AgentEvalRun,
|
|
type AgentGoldenCase,
|
|
} from "../src/lib/agent-evals.ts";
|
|
|
|
const fixtureText = readFileSync(
|
|
new URL("./fixtures/agent-golden-dataset-v1.json", import.meta.url),
|
|
"utf8",
|
|
);
|
|
const dataset = parseAgentGoldenDataset(JSON.parse(fixtureText));
|
|
|
|
function getCase(id: string): AgentGoldenCase {
|
|
const goldenCase = dataset.cases.find((candidate) => candidate.id === id);
|
|
assert.ok(goldenCase, `missing fixture case ${id}`);
|
|
return goldenCase;
|
|
}
|
|
|
|
function passingRun(goldenCase: AgentGoldenCase): AgentEvalRun {
|
|
const claims: AgentClaim[] = Array.from(
|
|
{ length: goldenCase.expected.minEvidenceBackedClaims },
|
|
(_, index) => ({
|
|
claimId: `claim.${index + 1}`,
|
|
kind: "interpretation",
|
|
requiresEvidence: true,
|
|
evidenceIds: [
|
|
goldenCase.expected.evidenceCatalog[index % goldenCase.expected.evidenceCatalog.length]!,
|
|
],
|
|
themeIds: goldenCase.expected.requestedThemes.length > 0
|
|
? [goldenCase.expected.requestedThemes[index % goldenCase.expected.requestedThemes.length]!]
|
|
: [],
|
|
}),
|
|
);
|
|
const toolCalls = goldenCase.expected.toolContract.required.flatMap((requirement) =>
|
|
Array.from({ length: requirement.minCalls }, (_, index) => ({
|
|
tool: requirement.tool,
|
|
status: "completed" as const,
|
|
inputDigest: `${requirement.tool}.${index}`,
|
|
latencyMs: 10,
|
|
costUsd: 0,
|
|
})),
|
|
);
|
|
const citedEvidenceIds = [...new Set(claims.flatMap((claim) => claim.evidenceIds))];
|
|
return {
|
|
caseId: goldenCase.id,
|
|
candidateResponse: "synthetic candidate response for pending model review",
|
|
skillExecutions: goldenCase.expected.requiredSkillIds.map((skillId) => ({
|
|
skillId,
|
|
status: "completed" as const,
|
|
})),
|
|
toolCalls,
|
|
availableEvidenceIds: citedEvidenceIds,
|
|
producedEvidenceIds: [],
|
|
claims,
|
|
coveredThemes: goldenCase.expected.requestedThemes,
|
|
rectificationFocus: goldenCase.expected.rectificationFocus
|
|
? {
|
|
focusId: goldenCase.expected.rectificationFocus.expectedFocusId,
|
|
domain: goldenCase.expected.rectificationFocus.expectedDomain,
|
|
}
|
|
: undefined,
|
|
observability: {
|
|
latencyMs: Math.floor(goldenCase.expected.performanceBudget.maxLatencyMs / 2),
|
|
costUsd: goldenCase.expected.performanceBudget.maxCostUsd / 2,
|
|
inputTokens: 100,
|
|
outputTokens: 50,
|
|
},
|
|
};
|
|
}
|
|
|
|
test("golden fixture is deidentified and exactly covers every PR-8 10.1 subscenario and tag", () => {
|
|
assert.equal(dataset.schemaVersion, "agent_golden_dataset.v1");
|
|
assert.deepEqual(dataset.deidentification, {
|
|
mode: "synthetic_intent_codes_only",
|
|
rawUserTextIncluded: false,
|
|
});
|
|
assert.deepEqual(
|
|
[...new Set(dataset.cases.map((goldenCase) => goldenCase.group))].sort(),
|
|
[...agentEvalGroups].sort(),
|
|
);
|
|
|
|
const expectedCoverageByCase: Record<string, {
|
|
group: AgentGoldenCase["group"];
|
|
subscenario: string;
|
|
requiredTags: readonly string[];
|
|
}> = {
|
|
"ordinary.career-direction": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "core-theme-career",
|
|
requiredTags: ["core-theme", "career"],
|
|
},
|
|
"ordinary.marriage-direction": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "core-theme-marriage",
|
|
requiredTags: ["core-theme", "marriage"],
|
|
},
|
|
"ordinary.wealth-direction": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "core-theme-wealth",
|
|
requiredTags: ["core-theme", "wealth"],
|
|
},
|
|
"ordinary.health-direction": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "core-theme-health",
|
|
requiredTags: ["core-theme", "health"],
|
|
},
|
|
"ordinary.education-direction": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "core-theme-education",
|
|
requiredTags: ["core-theme", "education"],
|
|
},
|
|
"ordinary.career-wealth-tradeoff": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "multi-theme-request",
|
|
requiredTags: ["multi-theme", "career", "wealth"],
|
|
},
|
|
"ordinary.evidence-follow-up": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "user-changes-question",
|
|
requiredTags: ["user-change-question", "initial-theme", "changed-theme", "current-request"],
|
|
},
|
|
"ordinary.missing-birth-minute": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "missing-birth-minute",
|
|
requiredTags: ["birth-minute-missing", "partial-birth-context", "no-invented-birth-data"],
|
|
},
|
|
"ordinary.timing-boundary": {
|
|
group: "ordinary_consultation",
|
|
subscenario: "accepted-confirmed-boundary",
|
|
requiredTags: ["accepted-time", "confirmed-time-boundary", "no-exact-confirmation"],
|
|
},
|
|
"rectification.multiple-events-same-turn": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "multiple-events-same-turn",
|
|
requiredTags: ["multiple-events", "same-turn", "focus-selection"],
|
|
},
|
|
"rectification.event-precision-year": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "event-precision-year",
|
|
requiredTags: ["event-precision", "year-precision"],
|
|
},
|
|
"rectification.event-precision-month": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "event-precision-month",
|
|
requiredTags: ["event-precision", "month-precision"],
|
|
},
|
|
"rectification.event-precision-day": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "event-precision-day",
|
|
requiredTags: ["event-precision", "day-precision"],
|
|
},
|
|
"rectification.confirm-clear-evidence": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "clear-event-same-turn-confirmation",
|
|
requiredTags: ["clear-event", "same-turn", "confirmed-evidence"],
|
|
},
|
|
"rectification.revise-evidence": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "correct-existing-event",
|
|
requiredTags: ["correction", "old-event", "revision"],
|
|
},
|
|
"rectification.refusal": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "refusal-to-answer",
|
|
requiredTags: ["refusal", "no-new-event", "no-write"],
|
|
},
|
|
"rectification.skip": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "skip-follow-up",
|
|
requiredTags: ["skip", "no-new-event", "no-write"],
|
|
},
|
|
"rectification.continuation-word": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "continuation-word-resolution",
|
|
requiredTags: ["continuation-word", "short-acknowledgement", "active-followup", "recent-turns"],
|
|
},
|
|
"rectification.long-conversation-recovery": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "long-conversation-recovery",
|
|
requiredTags: ["long-conversation", "session-resume", "recovery", "server-context", "no-raw-snapshot"],
|
|
},
|
|
"rectification.offer-candidates": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "candidate-acceptance",
|
|
requiredTags: ["candidate", "accepted", "not-confirmed", "truth-boundary"],
|
|
},
|
|
"rectification.confirm-exact-minute": {
|
|
group: "birth_time_rectification",
|
|
subscenario: "exact-minute-confirmation",
|
|
requiredTags: ["confirmation-allowed", "user-consent", "confirmed", "exact-minute"],
|
|
},
|
|
"report.complete-evidence": {
|
|
group: "report",
|
|
subscenario: "complete-evidence-report",
|
|
requiredTags: ["evidence-complete", "citation-closure", "audit-trace"],
|
|
},
|
|
"report.partial-evidence": {
|
|
group: "report",
|
|
subscenario: "partial-evidence-report",
|
|
requiredTags: ["evidence-partial", "coverage-disclosure", "no-fabrication"],
|
|
},
|
|
"report.conflicting-evidence": {
|
|
group: "report",
|
|
subscenario: "conflicting-evidence-report",
|
|
requiredTags: ["evidence-conflict", "conflict-disclosure", "reduced-confidence"],
|
|
},
|
|
"report.accepted-time": {
|
|
group: "report",
|
|
subscenario: "accepted-time-report",
|
|
requiredTags: ["accepted-time", "not-confirmed", "truth-boundary"],
|
|
},
|
|
"report.multi-theme": {
|
|
group: "report",
|
|
subscenario: "multi-theme-report",
|
|
requiredTags: ["multi-theme", "career", "marriage", "wealth", "health", "education"],
|
|
},
|
|
"safety.prompt-extraction": {
|
|
group: "safety",
|
|
subscenario: "prompt-extraction-refusal",
|
|
requiredTags: ["prompt-extraction", "security-boundary", "refusal"],
|
|
},
|
|
"safety.api-key-inducement": {
|
|
group: "safety",
|
|
subscenario: "api-key-inducement-refusal",
|
|
requiredTags: ["api-key-inducement", "credential-request", "credential-refusal"],
|
|
},
|
|
"safety.internal-path-inducement": {
|
|
group: "safety",
|
|
subscenario: "internal-path-inducement-refusal",
|
|
requiredTags: ["internal-path-inducement", "environment-request", "path-refusal"],
|
|
},
|
|
"safety.medical-certainty": {
|
|
group: "safety",
|
|
subscenario: "medical-certainty-refusal",
|
|
requiredTags: ["medical-certainty", "deterministic-request", "medical-boundary"],
|
|
},
|
|
"safety.legal-certainty": {
|
|
group: "safety",
|
|
subscenario: "legal-certainty-refusal",
|
|
requiredTags: ["legal-certainty", "deterministic-request", "legal-boundary"],
|
|
},
|
|
"safety.investment-certainty": {
|
|
group: "safety",
|
|
subscenario: "investment-certainty-refusal",
|
|
requiredTags: ["investment-certainty", "deterministic-request", "investment-boundary"],
|
|
},
|
|
"safety.fabricated-birth-data": {
|
|
group: "safety",
|
|
subscenario: "fabricated-birth-data-rejection",
|
|
requiredTags: ["fabricated-birth-data", "missing-source-data", "no-invented-birth-data"],
|
|
},
|
|
"safety.fabricated-candidate-id": {
|
|
group: "safety",
|
|
subscenario: "fabricated-candidate-id-rejection",
|
|
requiredTags: ["fabricated-candidate-id", "missing-server-state", "no-invented-candidate-id"],
|
|
},
|
|
};
|
|
|
|
assert.equal(dataset.cases.length, 34);
|
|
assert.deepEqual(
|
|
dataset.cases.map((goldenCase) => goldenCase.id).sort(),
|
|
Object.keys(expectedCoverageByCase).sort(),
|
|
);
|
|
for (const [caseId, coverage] of Object.entries(expectedCoverageByCase)) {
|
|
const goldenCase = getCase(caseId);
|
|
assert.equal(goldenCase.group, coverage.group, `${caseId} group`);
|
|
assert.equal(goldenCase.subscenario, coverage.subscenario, `${caseId} subscenario`);
|
|
const tags = new Set(goldenCase.turns.flatMap((turn) => turn.contextTags));
|
|
for (const requiredTag of coverage.requiredTags) {
|
|
assert.ok(tags.has(requiredTag), `${caseId} missing context tag ${requiredTag}`);
|
|
}
|
|
}
|
|
|
|
assert.deepEqual(getCase("ordinary.career-direction").expected.requestedThemes, ["career"]);
|
|
assert.deepEqual(getCase("ordinary.marriage-direction").expected.requestedThemes, ["marriage"]);
|
|
assert.deepEqual(getCase("ordinary.wealth-direction").expected.requestedThemes, ["wealth"]);
|
|
assert.deepEqual(getCase("ordinary.health-direction").expected.requestedThemes, ["health"]);
|
|
assert.deepEqual(getCase("ordinary.education-direction").expected.requestedThemes, ["education"]);
|
|
assert.deepEqual(getCase("ordinary.career-wealth-tradeoff").expected.requestedThemes, ["career", "wealth"]);
|
|
assert.deepEqual(getCase("ordinary.evidence-follow-up").expected.requestedThemes, ["marriage"]);
|
|
|
|
assert.equal(getCase("rectification.event-precision-year").expected.timingPolicy.maxPrecision, "year");
|
|
assert.equal(getCase("rectification.event-precision-month").expected.timingPolicy.maxPrecision, "month");
|
|
assert.equal(getCase("rectification.event-precision-day").expected.timingPolicy.maxPrecision, "day");
|
|
assert.equal(getCase("rectification.long-conversation-recovery").turns.length, 12);
|
|
assert.equal(getCase("ordinary.timing-boundary").expected.timingPolicy.allowConfirmedExactMinute, false);
|
|
assert.equal(getCase("rectification.offer-candidates").expected.timingPolicy.allowConfirmedExactMinute, false);
|
|
assert.equal(getCase("rectification.confirm-exact-minute").expected.timingPolicy.allowConfirmedExactMinute, true);
|
|
assert.equal(getCase("report.accepted-time").expected.timingPolicy.allowConfirmedExactMinute, false);
|
|
|
|
assert.ok(dataset.cases.every((goldenCase) => goldenCase.turns.length >= 2));
|
|
assert.deepEqual(findDatasetPrivacyViolations(dataset), []);
|
|
|
|
const allowedTurnKeys = ["contextTags", "intentCode", "speaker", "syntheticSummaryOnly", "turnId"];
|
|
for (const goldenCase of dataset.cases) {
|
|
for (const conversationTurn of goldenCase.turns) {
|
|
assert.equal(conversationTurn.syntheticSummaryOnly, true);
|
|
assert.deepEqual(Object.keys(conversationTurn).sort(), allowedTurnKeys);
|
|
assert.match(conversationTurn.intentCode, /^[a-z][a-z0-9._-]+$/);
|
|
for (const contextTag of conversationTurn.contextTags) {
|
|
assert.match(contextTag, /^[a-z][a-z0-9._-]+$/);
|
|
}
|
|
}
|
|
}
|
|
|
|
assert.doesNotMatch(fixtureText, /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i);
|
|
assert.doesNotMatch(fixtureText, /\b(?:19|20)\d{2}[-/.年]\d{1,2}[-/.月]\d{1,2}日?\b/);
|
|
assert.doesNotMatch(fixtureText, /(?:^|\D)(?:[01]?\d|2[0-3]):[0-5]\d(?:\D|$)/);
|
|
assert.doesNotMatch(fixtureText, /(?:^|[^A-Za-z0-9])(?:sk-[A-Za-z0-9_-]{12,}|api[_ -]?key\s*[:=])/i);
|
|
assert.doesNotMatch(fixtureText, /(?:\/Users\/|\/home\/|\/opt\/|\/private\/|[A-Za-z]:\\Users\\)/);
|
|
});
|
|
|
|
test("a conforming rectification run passes every applicable deterministic scorer", () => {
|
|
const goldenCase = getCase("rectification.confirm-clear-evidence");
|
|
const result = evaluateAgentRun(goldenCase, passingRun(goldenCase));
|
|
|
|
for (const [name, metric] of Object.entries(result.deterministic)) {
|
|
assert.notEqual(metric.status, "failed", `${name} should pass or be not applicable`);
|
|
assert.equal(metric.evaluationMode, "deterministic");
|
|
}
|
|
assert.equal(result.deterministic.skillToolContractCompletion.score, 1);
|
|
assert.equal(result.deterministic.evidenceCitationClosure.score, 1);
|
|
assert.equal(result.deterministic.rectificationFocusAccuracy.score, 1);
|
|
assert.equal(result.deterministic.toolCallEconomy.score, 1);
|
|
});
|
|
|
|
test("deterministic scorers expose contract, evidence, theme, timing, focus, economy, and budget failures", () => {
|
|
const goldenCase = getCase("rectification.offer-candidates");
|
|
const run: AgentEvalRun = {
|
|
caseId: goldenCase.id,
|
|
candidateResponse: "candidate response awaiting model review",
|
|
skillExecutions: [{ skillId: "jyotish-birth-time-rectification", status: "failed" }],
|
|
toolCalls: [
|
|
{ tool: "rectification-read-case", status: "completed", inputDigest: "same" },
|
|
{ tool: "rectification-read-case", status: "completed", inputDigest: "same" },
|
|
{ tool: "unallowlisted-internal-tool", status: "failed", inputDigest: "bad" },
|
|
{ tool: "rectification-offer-candidates", status: "pending", inputDigest: "pending" },
|
|
{ tool: "rectification-offer-candidates", status: "completed", inputDigest: "offer" },
|
|
],
|
|
availableEvidenceIds: ["ev-candidate-comparison"],
|
|
producedEvidenceIds: [],
|
|
claims: [
|
|
{
|
|
claimId: "claim.unsupported",
|
|
kind: "fact",
|
|
requiresEvidence: true,
|
|
evidenceIds: ["ev-not-in-catalog"],
|
|
themeIds: [],
|
|
},
|
|
{
|
|
claimId: "claim.exact-minute",
|
|
kind: "timing",
|
|
requiresEvidence: true,
|
|
evidenceIds: ["ev-candidate-comparison"],
|
|
themeIds: [],
|
|
timingPrecision: "minute",
|
|
timingModality: "confirmed",
|
|
},
|
|
],
|
|
coveredThemes: [],
|
|
rectificationFocus: { focusId: "focus.family-event", domain: "family" },
|
|
observability: {
|
|
latencyMs: goldenCase.expected.performanceBudget.maxLatencyMs + 1,
|
|
costUsd: goldenCase.expected.performanceBudget.maxCostUsd + 0.01,
|
|
inputTokens: 400,
|
|
outputTokens: 200,
|
|
},
|
|
};
|
|
|
|
const result = evaluateAgentRun(goldenCase, run);
|
|
assert.equal(result.deterministic.skillToolContractCompletion.status, "failed");
|
|
assert.ok(result.deterministic.skillToolContractCompletion.details.missingSkills.length > 0);
|
|
assert.ok(result.deterministic.skillToolContractCompletion.details.missingTools.length > 0);
|
|
assert.equal(result.deterministic.evidenceCitationClosure.status, "failed");
|
|
assert.deepEqual(result.deterministic.evidenceCitationClosure.details.danglingEvidenceIds, ["ev-not-in-catalog"]);
|
|
assert.equal(result.deterministic.unsupportedFactRuleCount.details.count, 1);
|
|
assert.equal(result.deterministic.preciseTimingViolation.details.count, 1);
|
|
assert.equal(result.deterministic.rectificationFocusAccuracy.score, 0);
|
|
assert.equal(result.deterministic.toolCallEconomy.status, "failed");
|
|
assert.ok(result.deterministic.toolCallEconomy.details.duplicateInputCalls.length > 0);
|
|
assert.ok(result.deterministic.toolCallEconomy.details.unallowedCalls.length > 0);
|
|
assert.equal(result.deterministic.latencyCostStatistics.status, "failed");
|
|
assert.deepEqual(result.deterministic.latencyCostStatistics.details.latencyBudgetBreaches, [goldenCase.id]);
|
|
assert.deepEqual(result.deterministic.latencyCostStatistics.details.costBudgetBreaches, [goldenCase.id]);
|
|
});
|
|
|
|
test("catalog-only citations fail until the run actually makes the evidence available", () => {
|
|
const goldenCase = getCase("ordinary.career-direction");
|
|
const catalogEvidenceId = goldenCase.expected.evidenceCatalog[0]!;
|
|
const base = passingRun(goldenCase);
|
|
const claim: AgentClaim = {
|
|
claimId: "claim.catalog-only",
|
|
kind: "fact",
|
|
requiresEvidence: false,
|
|
evidenceIds: [catalogEvidenceId],
|
|
themeIds: ["career"],
|
|
};
|
|
|
|
const unavailable = evaluateAgentRun(goldenCase, {
|
|
...base,
|
|
availableEvidenceIds: [],
|
|
producedEvidenceIds: [],
|
|
claims: [claim],
|
|
});
|
|
assert.equal(unavailable.deterministic.evidenceCitationClosure.status, "failed");
|
|
assert.deepEqual(
|
|
unavailable.deterministic.evidenceCitationClosure.details.unavailableEvidenceIds,
|
|
[catalogEvidenceId],
|
|
);
|
|
assert.deepEqual(
|
|
unavailable.deterministic.evidenceCitationClosure.details.uncatalogedEvidenceIds,
|
|
[],
|
|
);
|
|
assert.deepEqual(
|
|
unavailable.deterministic.evidenceCitationClosure.details.unclosedClaimIds,
|
|
[claim.claimId],
|
|
);
|
|
assert.deepEqual(
|
|
unavailable.deterministic.unsupportedFactRuleCount.details.claimIds,
|
|
[claim.claimId],
|
|
);
|
|
|
|
const produced = evaluateAgentRun(goldenCase, {
|
|
...base,
|
|
availableEvidenceIds: [],
|
|
producedEvidenceIds: [catalogEvidenceId],
|
|
claims: [claim],
|
|
});
|
|
assert.equal(produced.deterministic.evidenceCitationClosure.status, "passed");
|
|
assert.equal(produced.deterministic.unsupportedFactRuleCount.status, "passed");
|
|
});
|
|
|
|
test("fact and timing claims cannot disable deterministic evidence requirements", () => {
|
|
const goldenCase = getCase("ordinary.career-direction");
|
|
const base = passingRun(goldenCase);
|
|
const result = evaluateAgentRun(goldenCase, {
|
|
...base,
|
|
availableEvidenceIds: [],
|
|
producedEvidenceIds: [],
|
|
claims: [
|
|
{
|
|
claimId: "claim.fact-opt-out",
|
|
kind: "fact",
|
|
requiresEvidence: false,
|
|
evidenceIds: [],
|
|
themeIds: ["career"],
|
|
},
|
|
{
|
|
claimId: "claim.timing-opt-out",
|
|
kind: "timing",
|
|
requiresEvidence: false,
|
|
evidenceIds: [],
|
|
themeIds: ["career"],
|
|
timingPrecision: "broad_window",
|
|
timingModality: "candidate",
|
|
},
|
|
],
|
|
});
|
|
|
|
assert.equal(result.deterministic.evidenceCitationClosure.status, "failed");
|
|
assert.deepEqual(
|
|
result.deterministic.evidenceCitationClosure.details.unclosedClaimIds,
|
|
["claim.fact-opt-out", "claim.timing-opt-out"],
|
|
);
|
|
assert.equal(result.deterministic.unsupportedFactRuleCount.status, "failed");
|
|
assert.deepEqual(
|
|
result.deterministic.unsupportedFactRuleCount.details.claimIds,
|
|
["claim.fact-opt-out", "claim.timing-opt-out"],
|
|
);
|
|
});
|
|
|
|
test("minute candidate and accepted states pass while unconsented confirmation fails", () => {
|
|
const goldenCase = getCase("rectification.offer-candidates");
|
|
const base = passingRun(goldenCase);
|
|
const candidateClaim: AgentClaim = {
|
|
claimId: "claim.minute-candidate",
|
|
kind: "timing",
|
|
requiresEvidence: true,
|
|
evidenceIds: ["ev-candidate-comparison"],
|
|
themeIds: [],
|
|
timingPrecision: "minute",
|
|
timingModality: "candidate",
|
|
};
|
|
const candidate = evaluateAgentRun(goldenCase, { ...base, claims: [candidateClaim] });
|
|
assert.equal(candidate.deterministic.preciseTimingViolation.status, "passed");
|
|
|
|
const accepted = evaluateAgentRun(goldenCase, {
|
|
...base,
|
|
claims: [{ ...candidateClaim, timingModality: "accepted" }],
|
|
});
|
|
assert.equal(accepted.deterministic.preciseTimingViolation.status, "passed");
|
|
assert.equal(accepted.deterministic.evidenceCitationClosure.status, "passed");
|
|
|
|
const confirmed = evaluateAgentRun(goldenCase, {
|
|
...base,
|
|
claims: [{ ...candidateClaim, timingModality: "confirmed" }],
|
|
});
|
|
assert.equal(confirmed.deterministic.preciseTimingViolation.status, "failed");
|
|
assert.deepEqual(
|
|
confirmed.deterministic.preciseTimingViolation.details.violations[0]?.rules,
|
|
["exact_minute_confirmation_forbidden"],
|
|
);
|
|
});
|
|
|
|
test("consented exact-minute confirmation passes only in the explicit confirmation case", () => {
|
|
const goldenCase = getCase("rectification.confirm-exact-minute");
|
|
const base = passingRun(goldenCase);
|
|
const result = evaluateAgentRun(goldenCase, {
|
|
...base,
|
|
claims: [{
|
|
claimId: "claim.confirmed-minute",
|
|
kind: "timing",
|
|
requiresEvidence: true,
|
|
evidenceIds: ["ev-confirmation-gate"],
|
|
themeIds: [],
|
|
timingPrecision: "minute",
|
|
timingModality: "confirmed",
|
|
}],
|
|
});
|
|
assert.equal(result.deterministic.preciseTimingViolation.status, "passed");
|
|
});
|
|
|
|
test("requested theme coverage is independent from prose and uses structured theme ids", () => {
|
|
const goldenCase = getCase("ordinary.career-wealth-tradeoff");
|
|
const run = passingRun(goldenCase);
|
|
const result = evaluateAgentRun(goldenCase, {
|
|
...run,
|
|
candidateResponse: "prose may mention anything; deterministic coverage reads structured ids only",
|
|
coveredThemes: ["career"],
|
|
claims: run.claims.map((claim) => ({ ...claim, themeIds: ["career"] })),
|
|
});
|
|
assert.equal(result.deterministic.requestedThemeCoverage.status, "failed");
|
|
assert.deepEqual(result.deterministic.requestedThemeCoverage.details.missingThemes, ["wealth"]);
|
|
});
|
|
|
|
test("naturalness, repetition, follow-up relevance, and model fact review remain explicit pending inputs", () => {
|
|
const goldenCase = getCase("ordinary.evidence-follow-up");
|
|
const reviews = createPendingModelReviewInputs(goldenCase, passingRun(goldenCase));
|
|
assert.deepEqual(
|
|
reviews.map((review) => review.criterion),
|
|
["naturalness_repetition", "follow_up_relevance", "unsupported_fact_model_review"],
|
|
);
|
|
for (const review of reviews) {
|
|
assert.equal(review.evaluationMode, "model_review");
|
|
assert.equal(review.status, "pending");
|
|
assert.ok(review.input.candidateResponse.length > 0);
|
|
assert.ok(review.input.conversationIntentCodes.length >= 2);
|
|
assert.equal("score" in review, false);
|
|
assert.equal("verdict" in review, false);
|
|
assert.equal("passed" in review, false);
|
|
}
|
|
});
|
|
|
|
test("latency and cost statistics use deterministic nearest-rank percentiles and report budget breaches", () => {
|
|
const cases = [
|
|
getCase("ordinary.career-direction"),
|
|
getCase("ordinary.evidence-follow-up"),
|
|
getCase("ordinary.timing-boundary"),
|
|
];
|
|
const runs = cases.map((goldenCase, index) => ({
|
|
...passingRun(goldenCase),
|
|
observability: {
|
|
latencyMs: [100, 200, 400][index]!,
|
|
costUsd: [0.01, 0.02, 0.09][index]!,
|
|
inputTokens: [10, 20, 40][index]!,
|
|
outputTokens: [5, 10, 20][index]!,
|
|
},
|
|
}));
|
|
const result = summarizeLatencyAndCost(cases, runs);
|
|
assert.equal(result.status, "passed");
|
|
assert.deepEqual(result.details.latencyMs, {
|
|
min: 100,
|
|
max: 400,
|
|
mean: 233.333333,
|
|
p50: 200,
|
|
p95: 400,
|
|
total: 700,
|
|
});
|
|
assert.equal(result.details.costUsd.total, 0.12);
|
|
assert.deepEqual(result.details.latencyBudgetBreaches, []);
|
|
assert.deepEqual(result.details.costBudgetBreaches, []);
|
|
});
|
|
|
|
test("privacy scanner rejects identity fields, raw user bodies, credentials, dates, times, and internal paths", () => {
|
|
const poisoned = {
|
|
cases: [{
|
|
name: "synthetic-person",
|
|
email: "person@example.test",
|
|
birthDate: "2000-01-02",
|
|
birthTime: "08:30",
|
|
location: "synthetic-place",
|
|
content: "complete user body",
|
|
credential: "api_key=not-a-real-secret",
|
|
path: "/Users/example/private.txt",
|
|
}],
|
|
};
|
|
const rules = new Set(findDatasetPrivacyViolations(poisoned).map((violation) => violation.rule));
|
|
assert.deepEqual(rules, new Set([
|
|
"forbidden_identity_field",
|
|
"raw_user_text_field",
|
|
"email",
|
|
"birth_date",
|
|
"clock_time",
|
|
"api_credential",
|
|
"internal_absolute_path",
|
|
]));
|
|
});
|