import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; import { agentEvalGroups, createPendingModelReviewInputs, evaluateAgentRun, findDatasetPrivacyViolations, parseAgentGoldenDataset, summarizeLatencyAndCost, type AgentClaim, type AgentEvalRun, type AgentGoldenCase, } from "../src/lib/agent-evals.ts"; const fixtureText = readFileSync( new URL("./fixtures/agent-golden-dataset-v1.json", import.meta.url), "utf8", ); const dataset = parseAgentGoldenDataset(JSON.parse(fixtureText)); function getCase(id: string): AgentGoldenCase { const goldenCase = dataset.cases.find((candidate) => candidate.id === id); assert.ok(goldenCase, `missing fixture case ${id}`); return goldenCase; } function passingRun(goldenCase: AgentGoldenCase): AgentEvalRun { const claims: AgentClaim[] = Array.from( { length: goldenCase.expected.minEvidenceBackedClaims }, (_, index) => ({ claimId: `claim.${index + 1}`, kind: "interpretation", requiresEvidence: true, evidenceIds: [ goldenCase.expected.evidenceCatalog[index % goldenCase.expected.evidenceCatalog.length]!, ], themeIds: goldenCase.expected.requestedThemes.length > 0 ? [goldenCase.expected.requestedThemes[index % goldenCase.expected.requestedThemes.length]!] : [], }), ); const toolCalls = goldenCase.expected.toolContract.required.flatMap((requirement) => Array.from({ length: requirement.minCalls }, (_, index) => ({ tool: requirement.tool, status: "completed" as const, inputDigest: `${requirement.tool}.${index}`, latencyMs: 10, costUsd: 0, })), ); const citedEvidenceIds = [...new Set(claims.flatMap((claim) => claim.evidenceIds))]; return { caseId: goldenCase.id, candidateResponse: "synthetic candidate response for pending model review", skillExecutions: goldenCase.expected.requiredSkillIds.map((skillId) => ({ skillId, status: "completed" as const, })), toolCalls, availableEvidenceIds: citedEvidenceIds, producedEvidenceIds: [], claims, coveredThemes: goldenCase.expected.requestedThemes, rectificationFocus: goldenCase.expected.rectificationFocus ? { focusId: goldenCase.expected.rectificationFocus.expectedFocusId, domain: goldenCase.expected.rectificationFocus.expectedDomain, } : undefined, observability: { latencyMs: Math.floor(goldenCase.expected.performanceBudget.maxLatencyMs / 2), costUsd: goldenCase.expected.performanceBudget.maxCostUsd / 2, inputTokens: 100, outputTokens: 50, }, }; } test("golden fixture is deidentified and exactly covers every PR-8 10.1 subscenario and tag", () => { assert.equal(dataset.schemaVersion, "agent_golden_dataset.v1"); assert.deepEqual(dataset.deidentification, { mode: "synthetic_intent_codes_only", rawUserTextIncluded: false, }); assert.deepEqual( [...new Set(dataset.cases.map((goldenCase) => goldenCase.group))].sort(), [...agentEvalGroups].sort(), ); const expectedCoverageByCase: Record = { "ordinary.career-direction": { group: "ordinary_consultation", subscenario: "core-theme-career", requiredTags: ["core-theme", "career"], }, "ordinary.marriage-direction": { group: "ordinary_consultation", subscenario: "core-theme-marriage", requiredTags: ["core-theme", "marriage"], }, "ordinary.wealth-direction": { group: "ordinary_consultation", subscenario: "core-theme-wealth", requiredTags: ["core-theme", "wealth"], }, "ordinary.health-direction": { group: "ordinary_consultation", subscenario: "core-theme-health", requiredTags: ["core-theme", "health"], }, "ordinary.education-direction": { group: "ordinary_consultation", subscenario: "core-theme-education", requiredTags: ["core-theme", "education"], }, "ordinary.career-wealth-tradeoff": { group: "ordinary_consultation", subscenario: "multi-theme-request", requiredTags: ["multi-theme", "career", "wealth"], }, "ordinary.evidence-follow-up": { group: "ordinary_consultation", subscenario: "user-changes-question", requiredTags: ["user-change-question", "initial-theme", "changed-theme", "current-request"], }, "ordinary.missing-birth-minute": { group: "ordinary_consultation", subscenario: "missing-birth-minute", requiredTags: ["birth-minute-missing", "partial-birth-context", "no-invented-birth-data"], }, "ordinary.timing-boundary": { group: "ordinary_consultation", subscenario: "accepted-confirmed-boundary", requiredTags: ["accepted-time", "confirmed-time-boundary", "no-exact-confirmation"], }, "rectification.multiple-events-same-turn": { group: "birth_time_rectification", subscenario: "multiple-events-same-turn", requiredTags: ["multiple-events", "same-turn", "focus-selection"], }, "rectification.event-precision-year": { group: "birth_time_rectification", subscenario: "event-precision-year", requiredTags: ["event-precision", "year-precision"], }, "rectification.event-precision-month": { group: "birth_time_rectification", subscenario: "event-precision-month", requiredTags: ["event-precision", "month-precision"], }, "rectification.event-precision-day": { group: "birth_time_rectification", subscenario: "event-precision-day", requiredTags: ["event-precision", "day-precision"], }, "rectification.confirm-clear-evidence": { group: "birth_time_rectification", subscenario: "clear-event-same-turn-confirmation", requiredTags: ["clear-event", "same-turn", "confirmed-evidence"], }, "rectification.revise-evidence": { group: "birth_time_rectification", subscenario: "correct-existing-event", requiredTags: ["correction", "old-event", "revision"], }, "rectification.refusal": { group: "birth_time_rectification", subscenario: "refusal-to-answer", requiredTags: ["refusal", "no-new-event", "no-write"], }, "rectification.skip": { group: "birth_time_rectification", subscenario: "skip-follow-up", requiredTags: ["skip", "no-new-event", "no-write"], }, "rectification.continuation-word": { group: "birth_time_rectification", subscenario: "continuation-word-resolution", requiredTags: ["continuation-word", "short-acknowledgement", "active-followup", "recent-turns"], }, "rectification.long-conversation-recovery": { group: "birth_time_rectification", subscenario: "long-conversation-recovery", requiredTags: ["long-conversation", "session-resume", "recovery", "server-context", "no-raw-snapshot"], }, "rectification.offer-candidates": { group: "birth_time_rectification", subscenario: "candidate-acceptance", requiredTags: ["candidate", "accepted", "not-confirmed", "truth-boundary"], }, "rectification.confirm-exact-minute": { group: "birth_time_rectification", subscenario: "exact-minute-confirmation", requiredTags: ["confirmation-allowed", "user-consent", "confirmed", "exact-minute"], }, "report.complete-evidence": { group: "report", subscenario: "complete-evidence-report", requiredTags: ["evidence-complete", "citation-closure", "audit-trace"], }, "report.partial-evidence": { group: "report", subscenario: "partial-evidence-report", requiredTags: ["evidence-partial", "coverage-disclosure", "no-fabrication"], }, "report.conflicting-evidence": { group: "report", subscenario: "conflicting-evidence-report", requiredTags: ["evidence-conflict", "conflict-disclosure", "reduced-confidence"], }, "report.accepted-time": { group: "report", subscenario: "accepted-time-report", requiredTags: ["accepted-time", "not-confirmed", "truth-boundary"], }, "report.multi-theme": { group: "report", subscenario: "multi-theme-report", requiredTags: ["multi-theme", "career", "marriage", "wealth", "health", "education"], }, "safety.prompt-extraction": { group: "safety", subscenario: "prompt-extraction-refusal", requiredTags: ["prompt-extraction", "security-boundary", "refusal"], }, "safety.api-key-inducement": { group: "safety", subscenario: "api-key-inducement-refusal", requiredTags: ["api-key-inducement", "credential-request", "credential-refusal"], }, "safety.internal-path-inducement": { group: "safety", subscenario: "internal-path-inducement-refusal", requiredTags: ["internal-path-inducement", "environment-request", "path-refusal"], }, "safety.medical-certainty": { group: "safety", subscenario: "medical-certainty-refusal", requiredTags: ["medical-certainty", "deterministic-request", "medical-boundary"], }, "safety.legal-certainty": { group: "safety", subscenario: "legal-certainty-refusal", requiredTags: ["legal-certainty", "deterministic-request", "legal-boundary"], }, "safety.investment-certainty": { group: "safety", subscenario: "investment-certainty-refusal", requiredTags: ["investment-certainty", "deterministic-request", "investment-boundary"], }, "safety.fabricated-birth-data": { group: "safety", subscenario: "fabricated-birth-data-rejection", requiredTags: ["fabricated-birth-data", "missing-source-data", "no-invented-birth-data"], }, "safety.fabricated-candidate-id": { group: "safety", subscenario: "fabricated-candidate-id-rejection", requiredTags: ["fabricated-candidate-id", "missing-server-state", "no-invented-candidate-id"], }, }; assert.equal(dataset.cases.length, 34); assert.deepEqual( dataset.cases.map((goldenCase) => goldenCase.id).sort(), Object.keys(expectedCoverageByCase).sort(), ); for (const [caseId, coverage] of Object.entries(expectedCoverageByCase)) { const goldenCase = getCase(caseId); assert.equal(goldenCase.group, coverage.group, `${caseId} group`); assert.equal(goldenCase.subscenario, coverage.subscenario, `${caseId} subscenario`); const tags = new Set(goldenCase.turns.flatMap((turn) => turn.contextTags)); for (const requiredTag of coverage.requiredTags) { assert.ok(tags.has(requiredTag), `${caseId} missing context tag ${requiredTag}`); } } assert.deepEqual(getCase("ordinary.career-direction").expected.requestedThemes, ["career"]); assert.deepEqual(getCase("ordinary.marriage-direction").expected.requestedThemes, ["marriage"]); assert.deepEqual(getCase("ordinary.wealth-direction").expected.requestedThemes, ["wealth"]); assert.deepEqual(getCase("ordinary.health-direction").expected.requestedThemes, ["health"]); assert.deepEqual(getCase("ordinary.education-direction").expected.requestedThemes, ["education"]); assert.deepEqual(getCase("ordinary.career-wealth-tradeoff").expected.requestedThemes, ["career", "wealth"]); assert.deepEqual(getCase("ordinary.evidence-follow-up").expected.requestedThemes, ["marriage"]); assert.equal(getCase("rectification.event-precision-year").expected.timingPolicy.maxPrecision, "year"); assert.equal(getCase("rectification.event-precision-month").expected.timingPolicy.maxPrecision, "month"); assert.equal(getCase("rectification.event-precision-day").expected.timingPolicy.maxPrecision, "day"); assert.equal(getCase("rectification.long-conversation-recovery").turns.length, 12); assert.equal(getCase("ordinary.timing-boundary").expected.timingPolicy.allowConfirmedExactMinute, false); assert.equal(getCase("rectification.offer-candidates").expected.timingPolicy.allowConfirmedExactMinute, false); assert.equal(getCase("rectification.confirm-exact-minute").expected.timingPolicy.allowConfirmedExactMinute, true); assert.equal(getCase("report.accepted-time").expected.timingPolicy.allowConfirmedExactMinute, false); assert.ok(dataset.cases.every((goldenCase) => goldenCase.turns.length >= 2)); assert.deepEqual(findDatasetPrivacyViolations(dataset), []); const allowedTurnKeys = ["contextTags", "intentCode", "speaker", "syntheticSummaryOnly", "turnId"]; for (const goldenCase of dataset.cases) { for (const conversationTurn of goldenCase.turns) { assert.equal(conversationTurn.syntheticSummaryOnly, true); assert.deepEqual(Object.keys(conversationTurn).sort(), allowedTurnKeys); assert.match(conversationTurn.intentCode, /^[a-z][a-z0-9._-]+$/); for (const contextTag of conversationTurn.contextTags) { assert.match(contextTag, /^[a-z][a-z0-9._-]+$/); } } } assert.doesNotMatch(fixtureText, /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i); assert.doesNotMatch(fixtureText, /\b(?:19|20)\d{2}[-/.年]\d{1,2}[-/.月]\d{1,2}日?\b/); assert.doesNotMatch(fixtureText, /(?:^|\D)(?:[01]?\d|2[0-3]):[0-5]\d(?:\D|$)/); assert.doesNotMatch(fixtureText, /(?:^|[^A-Za-z0-9])(?:sk-[A-Za-z0-9_-]{12,}|api[_ -]?key\s*[:=])/i); assert.doesNotMatch(fixtureText, /(?:\/Users\/|\/home\/|\/opt\/|\/private\/|[A-Za-z]:\\Users\\)/); }); test("a conforming rectification run passes every applicable deterministic scorer", () => { const goldenCase = getCase("rectification.confirm-clear-evidence"); const result = evaluateAgentRun(goldenCase, passingRun(goldenCase)); for (const [name, metric] of Object.entries(result.deterministic)) { assert.notEqual(metric.status, "failed", `${name} should pass or be not applicable`); assert.equal(metric.evaluationMode, "deterministic"); } assert.equal(result.deterministic.skillToolContractCompletion.score, 1); assert.equal(result.deterministic.evidenceCitationClosure.score, 1); assert.equal(result.deterministic.rectificationFocusAccuracy.score, 1); assert.equal(result.deterministic.toolCallEconomy.score, 1); }); test("deterministic scorers expose contract, evidence, theme, timing, focus, economy, and budget failures", () => { const goldenCase = getCase("rectification.offer-candidates"); const run: AgentEvalRun = { caseId: goldenCase.id, candidateResponse: "candidate response awaiting model review", skillExecutions: [{ skillId: "jyotish-birth-time-rectification", status: "failed" }], toolCalls: [ { tool: "rectification-read-case", status: "completed", inputDigest: "same" }, { tool: "rectification-read-case", status: "completed", inputDigest: "same" }, { tool: "unallowlisted-internal-tool", status: "failed", inputDigest: "bad" }, { tool: "rectification-offer-candidates", status: "pending", inputDigest: "pending" }, { tool: "rectification-offer-candidates", status: "completed", inputDigest: "offer" }, ], availableEvidenceIds: ["ev-candidate-comparison"], producedEvidenceIds: [], claims: [ { claimId: "claim.unsupported", kind: "fact", requiresEvidence: true, evidenceIds: ["ev-not-in-catalog"], themeIds: [], }, { claimId: "claim.exact-minute", kind: "timing", requiresEvidence: true, evidenceIds: ["ev-candidate-comparison"], themeIds: [], timingPrecision: "minute", timingModality: "confirmed", }, ], coveredThemes: [], rectificationFocus: { focusId: "focus.family-event", domain: "family" }, observability: { latencyMs: goldenCase.expected.performanceBudget.maxLatencyMs + 1, costUsd: goldenCase.expected.performanceBudget.maxCostUsd + 0.01, inputTokens: 400, outputTokens: 200, }, }; const result = evaluateAgentRun(goldenCase, run); assert.equal(result.deterministic.skillToolContractCompletion.status, "failed"); assert.ok(result.deterministic.skillToolContractCompletion.details.missingSkills.length > 0); assert.ok(result.deterministic.skillToolContractCompletion.details.missingTools.length > 0); assert.equal(result.deterministic.evidenceCitationClosure.status, "failed"); assert.deepEqual(result.deterministic.evidenceCitationClosure.details.danglingEvidenceIds, ["ev-not-in-catalog"]); assert.equal(result.deterministic.unsupportedFactRuleCount.details.count, 1); assert.equal(result.deterministic.preciseTimingViolation.details.count, 1); assert.equal(result.deterministic.rectificationFocusAccuracy.score, 0); assert.equal(result.deterministic.toolCallEconomy.status, "failed"); assert.ok(result.deterministic.toolCallEconomy.details.duplicateInputCalls.length > 0); assert.ok(result.deterministic.toolCallEconomy.details.unallowedCalls.length > 0); assert.equal(result.deterministic.latencyCostStatistics.status, "failed"); assert.deepEqual(result.deterministic.latencyCostStatistics.details.latencyBudgetBreaches, [goldenCase.id]); assert.deepEqual(result.deterministic.latencyCostStatistics.details.costBudgetBreaches, [goldenCase.id]); }); test("catalog-only citations fail until the run actually makes the evidence available", () => { const goldenCase = getCase("ordinary.career-direction"); const catalogEvidenceId = goldenCase.expected.evidenceCatalog[0]!; const base = passingRun(goldenCase); const claim: AgentClaim = { claimId: "claim.catalog-only", kind: "fact", requiresEvidence: false, evidenceIds: [catalogEvidenceId], themeIds: ["career"], }; const unavailable = evaluateAgentRun(goldenCase, { ...base, availableEvidenceIds: [], producedEvidenceIds: [], claims: [claim], }); assert.equal(unavailable.deterministic.evidenceCitationClosure.status, "failed"); assert.deepEqual( unavailable.deterministic.evidenceCitationClosure.details.unavailableEvidenceIds, [catalogEvidenceId], ); assert.deepEqual( unavailable.deterministic.evidenceCitationClosure.details.uncatalogedEvidenceIds, [], ); assert.deepEqual( unavailable.deterministic.evidenceCitationClosure.details.unclosedClaimIds, [claim.claimId], ); assert.deepEqual( unavailable.deterministic.unsupportedFactRuleCount.details.claimIds, [claim.claimId], ); const produced = evaluateAgentRun(goldenCase, { ...base, availableEvidenceIds: [], producedEvidenceIds: [catalogEvidenceId], claims: [claim], }); assert.equal(produced.deterministic.evidenceCitationClosure.status, "passed"); assert.equal(produced.deterministic.unsupportedFactRuleCount.status, "passed"); }); test("fact and timing claims cannot disable deterministic evidence requirements", () => { const goldenCase = getCase("ordinary.career-direction"); const base = passingRun(goldenCase); const result = evaluateAgentRun(goldenCase, { ...base, availableEvidenceIds: [], producedEvidenceIds: [], claims: [ { claimId: "claim.fact-opt-out", kind: "fact", requiresEvidence: false, evidenceIds: [], themeIds: ["career"], }, { claimId: "claim.timing-opt-out", kind: "timing", requiresEvidence: false, evidenceIds: [], themeIds: ["career"], timingPrecision: "broad_window", timingModality: "candidate", }, ], }); assert.equal(result.deterministic.evidenceCitationClosure.status, "failed"); assert.deepEqual( result.deterministic.evidenceCitationClosure.details.unclosedClaimIds, ["claim.fact-opt-out", "claim.timing-opt-out"], ); assert.equal(result.deterministic.unsupportedFactRuleCount.status, "failed"); assert.deepEqual( result.deterministic.unsupportedFactRuleCount.details.claimIds, ["claim.fact-opt-out", "claim.timing-opt-out"], ); }); test("minute candidate and accepted states pass while unconsented confirmation fails", () => { const goldenCase = getCase("rectification.offer-candidates"); const base = passingRun(goldenCase); const candidateClaim: AgentClaim = { claimId: "claim.minute-candidate", kind: "timing", requiresEvidence: true, evidenceIds: ["ev-candidate-comparison"], themeIds: [], timingPrecision: "minute", timingModality: "candidate", }; const candidate = evaluateAgentRun(goldenCase, { ...base, claims: [candidateClaim] }); assert.equal(candidate.deterministic.preciseTimingViolation.status, "passed"); const accepted = evaluateAgentRun(goldenCase, { ...base, claims: [{ ...candidateClaim, timingModality: "accepted" }], }); assert.equal(accepted.deterministic.preciseTimingViolation.status, "passed"); assert.equal(accepted.deterministic.evidenceCitationClosure.status, "passed"); const confirmed = evaluateAgentRun(goldenCase, { ...base, claims: [{ ...candidateClaim, timingModality: "confirmed" }], }); assert.equal(confirmed.deterministic.preciseTimingViolation.status, "failed"); assert.deepEqual( confirmed.deterministic.preciseTimingViolation.details.violations[0]?.rules, ["exact_minute_confirmation_forbidden"], ); }); test("consented exact-minute confirmation passes only in the explicit confirmation case", () => { const goldenCase = getCase("rectification.confirm-exact-minute"); const base = passingRun(goldenCase); const result = evaluateAgentRun(goldenCase, { ...base, claims: [{ claimId: "claim.confirmed-minute", kind: "timing", requiresEvidence: true, evidenceIds: ["ev-confirmation-gate"], themeIds: [], timingPrecision: "minute", timingModality: "confirmed", }], }); assert.equal(result.deterministic.preciseTimingViolation.status, "passed"); }); test("requested theme coverage is independent from prose and uses structured theme ids", () => { const goldenCase = getCase("ordinary.career-wealth-tradeoff"); const run = passingRun(goldenCase); const result = evaluateAgentRun(goldenCase, { ...run, candidateResponse: "prose may mention anything; deterministic coverage reads structured ids only", coveredThemes: ["career"], claims: run.claims.map((claim) => ({ ...claim, themeIds: ["career"] })), }); assert.equal(result.deterministic.requestedThemeCoverage.status, "failed"); assert.deepEqual(result.deterministic.requestedThemeCoverage.details.missingThemes, ["wealth"]); }); test("naturalness, repetition, follow-up relevance, and model fact review remain explicit pending inputs", () => { const goldenCase = getCase("ordinary.evidence-follow-up"); const reviews = createPendingModelReviewInputs(goldenCase, passingRun(goldenCase)); assert.deepEqual( reviews.map((review) => review.criterion), ["naturalness_repetition", "follow_up_relevance", "unsupported_fact_model_review"], ); for (const review of reviews) { assert.equal(review.evaluationMode, "model_review"); assert.equal(review.status, "pending"); assert.ok(review.input.candidateResponse.length > 0); assert.ok(review.input.conversationIntentCodes.length >= 2); assert.equal("score" in review, false); assert.equal("verdict" in review, false); assert.equal("passed" in review, false); } }); test("latency and cost statistics use deterministic nearest-rank percentiles and report budget breaches", () => { const cases = [ getCase("ordinary.career-direction"), getCase("ordinary.evidence-follow-up"), getCase("ordinary.timing-boundary"), ]; const runs = cases.map((goldenCase, index) => ({ ...passingRun(goldenCase), observability: { latencyMs: [100, 200, 400][index]!, costUsd: [0.01, 0.02, 0.09][index]!, inputTokens: [10, 20, 40][index]!, outputTokens: [5, 10, 20][index]!, }, })); const result = summarizeLatencyAndCost(cases, runs); assert.equal(result.status, "passed"); assert.deepEqual(result.details.latencyMs, { min: 100, max: 400, mean: 233.333333, p50: 200, p95: 400, total: 700, }); assert.equal(result.details.costUsd.total, 0.12); assert.deepEqual(result.details.latencyBudgetBreaches, []); assert.deepEqual(result.details.costBudgetBreaches, []); }); test("privacy scanner rejects identity fields, raw user bodies, credentials, dates, times, and internal paths", () => { const poisoned = { cases: [{ name: "synthetic-person", email: "person@example.test", birthDate: "2000-01-02", birthTime: "08:30", location: "synthetic-place", content: "complete user body", credential: "api_key=not-a-real-secret", path: "/Users/example/private.txt", }], }; const rules = new Set(findDatasetPrivacyViolations(poisoned).map((violation) => violation.rule)); assert.deepEqual(rules, new Set([ "forbidden_identity_field", "raw_user_text_field", "email", "birth_date", "clock_time", "api_credential", "internal_absolute_path", ])); });