feat(consult): six-field evidence-card record in agent observability (log only)
After each natal turn the route logs a separate [agent-observability] event
{runId, agentVersion, evidenceCard: {domains, cardVersion, cardChars,
cardTokenEstimate, citedFieldIds, feedback}}. Cited field ids follow the
research R5 rule (ISO date, degree, planet-in-sign phrase of 8+ chars
appearing verbatim in a finished answer); matched text is discarded. Thumbs
are client state only today, so feedback is "none"; storing thumbs per turn
needs a table and is left to a follow-up. The strict schema has no user,
session, question or answer field.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017eEAG8HD3mm8gsKXgk8uU8
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
cb3ee55837
commit
48fb160fc9
@@ -62,6 +62,7 @@ import {
|
||||
createWindowConsultationAgentContext,
|
||||
precomputeWindowConsultation,
|
||||
windowPrecomputedPacketMessage,
|
||||
consultationEvidenceCardTelemetry,
|
||||
consultationModelStepTelemetry,
|
||||
consultationStepBudgetReceipt,
|
||||
createConsultationRuntimeHooks,
|
||||
@@ -879,6 +880,10 @@ export async function POST(request: Request) {
|
||||
let firstActivityMs = -1;
|
||||
let firstTextMs = -1;
|
||||
let logged = false;
|
||||
// The finished natal answer, read once by the evidence-card record to find
|
||||
// which card fields it quoted; never logged. A failed or cut run leaves it
|
||||
// empty, so only finished answers count citations.
|
||||
let answerForTelemetry = "";
|
||||
const markFirstActivity = () => { if (firstActivityMs < 0) firstActivityMs = Date.now() - agentStartedAt; };
|
||||
const markFirstText = () => { if (firstTextMs < 0) firstTextMs = Date.now() - agentStartedAt; };
|
||||
const consultEntrypoint = parsed.success ? parsed.data.entrypoint : undefined;
|
||||
@@ -948,6 +953,15 @@ export async function POST(request: Request) {
|
||||
themeCoverage: state.workflowReceipt?.domains ?? [consultationTheme],
|
||||
billingSettlementResult: settlementResult,
|
||||
});
|
||||
// D7: a separate six-field record, no session or user id in it.
|
||||
const evidenceCard = consultationEvidenceCardTelemetry(state, answerForTelemetry);
|
||||
if (evidenceCard) {
|
||||
logAgentObservability({
|
||||
runId: requestId,
|
||||
agentVersion: "consultation-evidence-card-v1",
|
||||
evidenceCard,
|
||||
});
|
||||
}
|
||||
};
|
||||
const settleRun = async (
|
||||
action: () => Promise<AgentSettlementResult>,
|
||||
@@ -1385,15 +1399,18 @@ export async function POST(request: Request) {
|
||||
headers: { "x-jyotish-birth-time-mode": consultationMode },
|
||||
onFirstActivity: markFirstActivity,
|
||||
onFirstOutput: markFirstText,
|
||||
onComplete: (output, agentExecutionReceipt, thinkingText, thinkingSections) => settleRun(() => completeResponse(
|
||||
output,
|
||||
mergeUsage(usages),
|
||||
state.techniqueTruth ?? "unknown",
|
||||
state.workflowReceipt ?? workflowReceipt,
|
||||
agentExecutionReceipt,
|
||||
thinkingText,
|
||||
thinkingSections,
|
||||
), undefined),
|
||||
onComplete: (output, agentExecutionReceipt, thinkingText, thinkingSections) => {
|
||||
answerForTelemetry = output;
|
||||
return settleRun(() => completeResponse(
|
||||
output,
|
||||
mergeUsage(usages),
|
||||
state.techniqueTruth ?? "unknown",
|
||||
state.workflowReceipt ?? workflowReceipt,
|
||||
agentExecutionReceipt,
|
||||
thinkingText,
|
||||
thinkingSections,
|
||||
), undefined);
|
||||
},
|
||||
onError: (error) => settleRun(
|
||||
cancel,
|
||||
toAgentObservabilityErrorCode(error),
|
||||
|
||||
@@ -113,6 +113,33 @@ export const agentObservabilityContractPhaseSchema = z.object({
|
||||
status: z.enum(agentObservabilityStepStatuses),
|
||||
}).strict().readonly();
|
||||
|
||||
/**
|
||||
* The evidence-card record (TASK-consult-evidence-card-20260927 D7): exactly
|
||||
* six fields, numbers / enums / card field names only. No question, answer,
|
||||
* birth data, name, email, user or session id, and never the matched text.
|
||||
*/
|
||||
export const EVIDENCE_CARD_TELEMETRY_FIELDS = [
|
||||
"domains",
|
||||
"cardVersion",
|
||||
"cardChars",
|
||||
"cardTokenEstimate",
|
||||
"citedFieldIds",
|
||||
"feedback",
|
||||
] as const;
|
||||
|
||||
export const evidenceCardTelemetrySchema = z.object({
|
||||
domains: z.array(consultationDomainSchema).min(1).max(2),
|
||||
cardVersion: z.literal("evidence-card-v1"),
|
||||
cardChars: countSchema,
|
||||
cardTokenEstimate: countSchema,
|
||||
citedFieldIds: z.array(z.string().min(1).max(120).regex(/^[A-Za-z][A-Za-z0-9._-]*$/, "invalid card field id")).max(64),
|
||||
// Thumbs are client state only today (not stored), so a turn is logged
|
||||
// with `none`; storing thumbs per turn needs a table and is a separate task.
|
||||
feedback: z.enum(["up", "down", "none"]),
|
||||
}).strict().readonly();
|
||||
|
||||
export type EvidenceCardTelemetry = z.infer<typeof evidenceCardTelemetrySchema>;
|
||||
|
||||
export const agentObservabilityEventSchema = z.object({
|
||||
runId: opaqueIdSchema.optional(),
|
||||
requestId: opaqueIdSchema.optional(),
|
||||
@@ -155,6 +182,7 @@ export const agentObservabilityEventSchema = z.object({
|
||||
reportJobDurationMs: durationMsSchema.optional(),
|
||||
reportJobPeakMemoryBytes: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).optional(),
|
||||
billingSettlementResult: z.enum(billingSettlementResults).optional(),
|
||||
evidenceCard: evidenceCardTelemetrySchema.optional(),
|
||||
}).strict().refine(
|
||||
(event) => Boolean(event.runId || event.requestId || event.sessionId || event.caseId),
|
||||
{ message: "at least one controlled identifier is required" },
|
||||
|
||||
@@ -14,12 +14,14 @@ import { runV9RangeReading } from "../lib/rectification-agentic/v9/engine-client
|
||||
import { createConsultationPlan, type ConsultationPlan } from "../lib/consultation-plan.ts";
|
||||
import type { TechniqueAuditRow, WorkflowReceipt } from "../lib/consultation-agent-events.ts";
|
||||
import { normalizeTechniqueAuditRows } from "../lib/consultation-technique-audit.ts";
|
||||
import type { AgentModelFinishReason } from "../lib/agent-observability.ts";
|
||||
import type { AgentModelFinishReason, EvidenceCardTelemetry } from "../lib/agent-observability.ts";
|
||||
import { agentGenerationSettings, AGENT_SLICE_ANSWER_OUTPUT_TOKENS, AGENT_SLICE_THINKING_OUTPUT_TOKENS } from "../lib/agent-generation-settings.ts";
|
||||
import { chartCalculationProgressLabel, evidenceLookupActivityLabel } from "../lib/consultation-activity-labels.ts";
|
||||
import {
|
||||
buildEvidenceCard,
|
||||
citedEvidenceCardFields,
|
||||
EVIDENCE_LOOKUP_SECTIONS,
|
||||
evidenceCardTokenEstimate,
|
||||
type EvidenceCard,
|
||||
type EvidenceLookupSection,
|
||||
} from "../lib/consultation-evidence-card.ts";
|
||||
@@ -338,6 +340,27 @@ export function consultationModelStepTelemetry(state: ConsultationRuntimeState)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The six-field evidence-card record for the observability log (D7). The
|
||||
* answer is read only to find which card fields it quoted (ISO date, degree,
|
||||
* planet-in-sign phrase); neither the answer nor the matched text is kept.
|
||||
*/
|
||||
export function consultationEvidenceCardTelemetry(
|
||||
state: ConsultationRuntimeState,
|
||||
answer: string,
|
||||
): EvidenceCardTelemetry | undefined {
|
||||
const card = state.evidenceCard;
|
||||
if (!card || state.evidenceCardChars === undefined) return undefined;
|
||||
return {
|
||||
domains: card.domains.slice(0, 2),
|
||||
cardVersion: card.card_version,
|
||||
cardChars: state.evidenceCardChars,
|
||||
cardTokenEstimate: evidenceCardTokenEstimate(state.evidenceCardChars),
|
||||
citedFieldIds: citedEvidenceCardFields(card, answer),
|
||||
feedback: "none",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Public receipts carry only the fields the client contract allows. Building
|
||||
* the list from an explicit allowlist keeps internal diagnostics, such as the
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
// TASK-consult-evidence-card-20260927 T6 (D7): per-turn evidence-card record
|
||||
// in [agent-observability], six fields only, no user text.
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
createAgentObservabilityLogger,
|
||||
EVIDENCE_CARD_TELEMETRY_FIELDS,
|
||||
evidenceCardTelemetrySchema,
|
||||
type AgentObservabilityEvent,
|
||||
} from "../src/lib/agent-observability.ts";
|
||||
import {
|
||||
consultationEvidenceCardTelemetry,
|
||||
createConsultationRuntimeState,
|
||||
createConsultationTools,
|
||||
} from "../src/mastra/consultation-tools.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
const golden = JSON.parse(readFileSync(
|
||||
new URL("./fixtures/consult-evidence-card-golden.json", import.meta.url),
|
||||
"utf8",
|
||||
)) as { charts: Array<{ id: string; workflow: Json }> };
|
||||
const workflow = golden.charts[0]!.workflow;
|
||||
const route = readFileSync(new URL("../src/app/api/consult/route.ts", import.meta.url), "utf8");
|
||||
|
||||
async function stateAfterCalculation() {
|
||||
const state = createConsultationRuntimeState();
|
||||
const tools = createConsultationTools({
|
||||
userId: "user-telemetry", sessionId: "session-telemetry", requestId: "req-telemetry", consultationMode: "verified_chart",
|
||||
serverChart: { name: "public", toolInput: { year: 2000, month: 1, day: 1, hour: 0, minute: 0, city: "fixture", lat: 0, lon: 0, tz: 0, ayanamsa: "raman" }, truth: { birthTimeSource: "reported" } } as never,
|
||||
state,
|
||||
runWorkflow: async () => structuredClone(workflow) as never,
|
||||
});
|
||||
await tools["run-jyotish-consultation"].execute!({ question: "q", domains: ["parents"] } as never, { writer: { custom: async () => {} } } as never);
|
||||
return state;
|
||||
}
|
||||
|
||||
test("the record has exactly the six approved fields", async () => {
|
||||
assert.deepEqual([...EVIDENCE_CARD_TELEMETRY_FIELDS].sort(), Object.keys(evidenceCardTelemetrySchema.unwrap().shape).sort());
|
||||
assert.deepEqual([...EVIDENCE_CARD_TELEMETRY_FIELDS], ["domains", "cardVersion", "cardChars", "cardTokenEstimate", "citedFieldIds", "feedback"]);
|
||||
const state = await stateAfterCalculation();
|
||||
const record = consultationEvidenceCardTelemetry(state, "");
|
||||
assert.ok(record);
|
||||
assert.deepEqual(Object.keys(record).sort(), [...EVIDENCE_CARD_TELEMETRY_FIELDS].sort());
|
||||
assert.deepEqual(record.domains, ["parents"]);
|
||||
assert.equal(record.cardVersion, "evidence-card-v1");
|
||||
assert.equal(record.cardChars, state.evidenceCardChars);
|
||||
assert.equal(record.cardTokenEstimate, Math.round(record.cardChars / 3.5));
|
||||
assert.deepEqual(record.citedFieldIds, []);
|
||||
assert.equal(record.feedback, "none");
|
||||
});
|
||||
|
||||
test("the schema is strict: no extra field, no free text in field ids", () => {
|
||||
const valid = { domains: ["parents"], cardVersion: "evidence-card-v1", cardChars: 10, cardTokenEstimate: 3, citedFieldIds: ["base.natal.ascendant.sign"], feedback: "up" };
|
||||
assert.equal(evidenceCardTelemetrySchema.safeParse(valid).success, true);
|
||||
for (const extra of ["question", "answer", "sessionId", "userId", "birthDate", "matched"]) {
|
||||
assert.equal(evidenceCardTelemetrySchema.safeParse({ ...valid, [extra]: "x" }).success, false, extra);
|
||||
}
|
||||
assert.equal(evidenceCardTelemetrySchema.safeParse({ ...valid, citedFieldIds: ["我和父母关系如何"] }).success, false);
|
||||
assert.equal(evidenceCardTelemetrySchema.safeParse({ ...valid, feedback: "great" }).success, false);
|
||||
});
|
||||
|
||||
test("the logged line keeps field ids and drops every word of the user's text", async () => {
|
||||
const state = await stateAfterCalculation();
|
||||
const card = state.evidenceCard!;
|
||||
const pdStart = String((card.base.timing.vimshottari.pratyantardasha as Json).start);
|
||||
// Fictional sensitive values, not a real person.
|
||||
const secrets = ["虚构名字王小明", "fictional.person@example.invalid", "1990-01-01", "我和父母关系如何"];
|
||||
const answer = `${secrets[0]}(${secrets[1]},生日 ${secrets[2]})问:${secrets[3]}。子运从 ${pdStart} 开始。`;
|
||||
const lines: AgentObservabilityEvent[] = [];
|
||||
const log = createAgentObservabilityLogger((event) => lines.push(event));
|
||||
log({ runId: "req-telemetry", agentVersion: "consultation-evidence-card-v1", evidenceCard: consultationEvidenceCardTelemetry(state, answer) });
|
||||
const text = JSON.stringify(lines);
|
||||
for (const secret of secrets) assert.equal(text.includes(secret), false, secret);
|
||||
assert.equal(text.includes(pdStart), false, "the matched date is not kept");
|
||||
assert.equal(text.includes("session-telemetry"), false);
|
||||
assert.equal(text.includes("user-telemetry"), false);
|
||||
assert.ok(lines[0]?.evidenceCard?.citedFieldIds.includes("base.timing.vimshottari.pratyantardasha.start"));
|
||||
});
|
||||
|
||||
test("the consult route logs the record once per natal turn, without session or user id", () => {
|
||||
assert.match(route, /const evidenceCard = consultationEvidenceCardTelemetry\(state, answerForTelemetry\);/);
|
||||
const block = route.slice(route.indexOf("const evidenceCard = consultationEvidenceCardTelemetry"), route.indexOf("const settleRun = async"));
|
||||
assert.match(block, /agentVersion: "consultation-evidence-card-v1"/);
|
||||
assert.doesNotMatch(block, /sessionId|userId/);
|
||||
const natal = route.slice(route.indexOf("const agent = getJyotishAgent(selectedModel, agentContext);"));
|
||||
assert.match(natal, /answerForTelemetry = output;[\s\S]*return settleRun\(\(\) => completeResponse\(/);
|
||||
// Only a finished answer counts citations; a failed run logs the card size with no citations.
|
||||
assert.doesNotMatch(natal, /onError: \(error, _emitted, output\)/);
|
||||
});
|
||||
Reference in New Issue
Block a user