fix(rectification): validate public agent grounding
This commit is contained in:
Jesse_Chen
@@ -13,7 +13,7 @@ import { assertRectificationSkillLoaded } from "./skill-runtime.ts";
|
||||
const skillPath = process.env.RECTIFICATION_SKILL_PATH?.trim() || path.resolve(process.cwd(), "..", "skills", "birth-time-rectification");
|
||||
const domains: EvidenceDomain[] = ["education", "relocation", "relationship", "career", "finance", "health_pressure", "family", "other"];
|
||||
const kinds: EventKind[] = ["education_milestone", "relocation", "relationship_start", "relationship_end", "relationship_change", "career_change", "finance_change", "self_health_event", "family_health_event", "family_bereavement", "family_event", "other"];
|
||||
const privatePattern = /(?:[0-9a-f]{8}-[0-9a-f-]{27,}|opportunity(?:id)?|snapshot(?:id)?|event(?:id)?|targetEventId|(?:raw|原始)?\s*(?:score|评分|得分)|权重|rule[_ -]?id|贡献矩阵|tool[_ -]?call|cluster[_ -]?id|工具原始(?:输出|轨迹)|内部推理链)/iu;
|
||||
const privatePattern = /(?:[0-9a-f]{8}-[0-9a-f-]{27,}|opportunity(?:id)?|snapshot(?:id)?|event(?:id)?|targetEventId|(?:raw|原始)?\s*(?:score|评分|得分)|权重|rule[_ -]?id|贡献矩阵|tool[_ -]?call|cluster[_ -]?id|\b(?:case_read|candidate_scan|evidence_gap|diagnostic_read)\b|工具原始(?:输出|轨迹)|内部推理链)/iu;
|
||||
const quantifiedStructurePattern = /(?:(?:D\d{1,2}|KP|Vimshottari|Narayana|Shadbala|Ashtakavarga|Chaturvimshamsha|上升星座|宫位|分盘)[^。！？\n]{0,60}(?:切换|变化|变动|遍历)[^。！？\n]{0,16}(?:\d+|[一二三四五六七八九十百]+)\s*次|(?:\d+|[一二三四五六七八九十百]+)\s*次[^。！？\n]{0,60}(?:切换|变化|变动|遍历))/iu;
|
||||
const exactMinutePattern = /(?:\b(?:[01]?\d|2[0-3]):[0-5]\d\b|(?:凌晨|清晨|上午|中午|下午|傍晚|晚上)?\s*[零〇一二两三四五六七八九十百\d]{1,4}\s*[点时]\s*[零〇一二两三四五六七八九十百\d]{1,4}\s*分)/u;
|
||||
const questionClausePattern = /(?:请|你(?:还)?(?:记得|能否|是否|有没有)|再(?:说|补充|回忆)|哪(?:一|个|年|月|天)?|什么|多少|几(?:年|月|号|日)?|吗|呢)/u;
|
||||
@@ -30,6 +30,15 @@ function asksMultipleQuestions(value: string): boolean {
|
||||
.filter((part) => questionClausePattern.test(part))
|
||||
.length > 1;
|
||||
}
|
||||
|
||||
function normalizedQuestion(value: string): string {
|
||||
return value.normalize("NFKC").toLocaleLowerCase().replace(/[\s?？。！!，,、；;：:“”‘’'"（）()]/gu, "");
|
||||
}
|
||||
|
||||
function mentionsTechnique(value: string, technique: string): boolean {
|
||||
const escaped = technique.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
return new RegExp(`(?<![A-Za-z0-9])${escaped}(?![A-Za-z0-9])`, "iu").test(value);
|
||||
}
|
||||
const declinedPattern = /(?:不想说|不方便说|不想回答|跳过|这个不说|换个方向|不聊这个)/u;
|
||||
type Generated = Readonly<{ object: unknown; totalUsage?: { inputTokens?: number; outputTokens?: number } | Promise<{ inputTokens?: number; outputTokens?: number }> }>;
|
||||
const regeneratedQuestionSchema = z.object({ question: z.string().trim().min(8).max(500) }).strict();
|
||||
@@ -152,22 +161,52 @@ export function validateRectificationTurnPlan(input: Readonly<{ plan: unknown; d
|
||||
const publicText = [plan.publicReply.acknowledgement, plan.publicReply.evidenceExplanation, plan.publicReply.candidateCommentary, plan.publicReply.limitation, plan.action.type === "ask_question" ? plan.action.question : null].filter(Boolean).join(" ");
|
||||
if (privatePattern.test(publicText)) issues.push("private_detail_exposed");
|
||||
if (containsExactMinute(publicText)) issues.push("exact_minute_claimed");
|
||||
const groundedEvent = latestGroundedEvent(input.dossier, input.latestAnswer);
|
||||
const capabilityFacts = new Map(input.dossier.capabilities.publicTechniqueCapabilities.map((item) => [`domain:${item.domain}`, item]));
|
||||
const observationFacts = new Map<"window_sensitivity" | "candidate_scan" | "diagnostic", Set<string>>([
|
||||
["window_sensitivity", new Set()], ["candidate_scan", new Set()], ["diagnostic", new Set()],
|
||||
]);
|
||||
input.dossier.runtime.observations.forEach((observation) => {
|
||||
if (observation.outcome !== "succeeded") return;
|
||||
const direct = Object.keys(observation.result);
|
||||
const window = observation.result.windowSensitivity;
|
||||
if (window && typeof window === "object" && !Array.isArray(window)) {
|
||||
Object.keys(window as Record<string, unknown>).forEach((key) => observationFacts.get("window_sensitivity")?.add(key));
|
||||
}
|
||||
if (observation.tool === "candidate_scan") direct.forEach((key) => observationFacts.get("candidate_scan")?.add(key));
|
||||
if (observation.tool === "diagnostic_read") direct.forEach((key) => observationFacts.get("diagnostic")?.add(key));
|
||||
});
|
||||
plan.publicExplanationGrounding.forEach((grounding) => {
|
||||
if (grounding.source === "capability_matrix") {
|
||||
const capability = capabilityFacts.get(grounding.factKey);
|
||||
if (!capability || (groundedEvent && capability.domain !== groundedEvent.domain)) issues.push("public_grounding_invalid");
|
||||
return;
|
||||
}
|
||||
if (!observationFacts.get(grounding.source)?.has(grounding.factKey)) issues.push("public_grounding_invalid");
|
||||
});
|
||||
if (groundedEvent && plan.publicReply.evidenceExplanation) {
|
||||
const citedCapabilities = plan.publicExplanationGrounding.flatMap((grounding) => {
|
||||
const capability = grounding.source === "capability_matrix" ? capabilityFacts.get(grounding.factKey) : null;
|
||||
return capability && capability.domain === groundedEvent.domain ? [capability] : [];
|
||||
});
|
||||
const allowedTechniques = new Set(citedCapabilities.flatMap((item) => item.techniqueLayers.map((layer) => layer.toLocaleLowerCase())));
|
||||
const knownTechniques = [...new Set([
|
||||
...input.dossier.capabilities.publicTechniqueCapabilities.flatMap((item) => item.techniqueLayers),
|
||||
"KP", "Shadbala", "Ashtakavarga", "Chaturvimshamsha",
|
||||
])];
|
||||
if (knownTechniques.some((technique) => mentionsTechnique(plan.publicReply.evidenceExplanation ?? "", technique)
|
||||
&& !allowedTechniques.has(technique.toLocaleLowerCase()))) issues.push("public_technique_not_grounded");
|
||||
}
|
||||
if (quantifiedStructurePattern.test(publicText)) {
|
||||
const windowFactKeys = new Set(plan.publicExplanationGrounding.filter((item) => item.source === "window_sensitivity").map((item) => item.factKey));
|
||||
const observedFactKeys = new Set(input.dossier.runtime.observations.flatMap((observation) => {
|
||||
if (observation.outcome !== "succeeded") return [];
|
||||
const result = observation.result;
|
||||
const direct = Object.keys(result);
|
||||
const nested = result.windowSensitivity && typeof result.windowSensitivity === "object" && !Array.isArray(result.windowSensitivity)
|
||||
? Object.keys(result.windowSensitivity as Record<string, unknown>)
|
||||
: [];
|
||||
return [...direct, ...nested];
|
||||
}));
|
||||
const observedFactKeys = observationFacts.get("window_sensitivity") ?? new Set<string>();
|
||||
if (![...windowFactKeys].some((factKey) => observedFactKeys.has(factKey))) issues.push("ungrounded_numeric_structure_claim");
|
||||
}
|
||||
if (plan.action.type === "ask_question") {
|
||||
if (asksMultipleQuestions(plan.action.question)) issues.push("multiple_questions");
|
||||
if (input.phase === "final" && latestGroundedEvent(input.dossier, input.latestAnswer)) {
|
||||
const nextQuestion = normalizedQuestion(plan.action.question);
|
||||
if (nextQuestion && input.dossier.interviewState.askedTopics.some((question) => normalizedQuestion(question).endsWith(nextQuestion))) issues.push("question_repeated");
|
||||
if (input.phase === "final" && groundedEvent) {
|
||||
if (genericAcknowledgementPattern.test(plan.publicReply.acknowledgement.trim())) issues.push("event_acknowledgement_generic");
|
||||
if (!plan.publicReply.evidenceExplanation || plan.publicReply.evidenceExplanation.trim().length < 12) issues.push("event_explanation_missing");
|
||||
if (!plan.publicReply.candidateCommentary || plan.publicReply.candidateCommentary.trim().length < 12) issues.push("event_value_commentary_missing");
|
||||
@@ -250,6 +289,7 @@ export async function runRectificationDirector(input: Readonly<{ caseValue: Rect
|
||||
};
|
||||
const toolKey = (request: Readonly<{ tool: RectificationAgentTool; diagnostic: RectificationDiagnostic | null }>) => `${request.tool}:${request.diagnostic ?? ""}`;
|
||||
const promptDossier = () => input.phase === "evidence" ? dossier : {
|
||||
interviewState: { askedTopics: dossier.interviewState.askedTopics },
|
||||
runtime: { revision: dossier.runtime.revision, observations: dossier.runtime.observations },
|
||||
capabilities: dossier.capabilities,
|
||||
availableTools: {
|
||||
|
||||
@@ -415,6 +415,7 @@ test("declined targets cannot be reopened and the Director adapts through server
|
||||
assert.equal(firstPrompt.dossier.eventLedger, undefined);
|
||||
assert.equal(firstPrompt.dossier.candidateState, undefined);
|
||||
assert.equal(firstPrompt.dossier.runtime.hypotheses, undefined);
|
||||
assert.deepEqual(firstPrompt.dossier.interviewState.askedTopics, []);
|
||||
assert.deepEqual(firstPrompt.dossier.availableTools.readOnly, ["case_read", "candidate_scan", "evidence_gap"]);
|
||||
const caseObservationPrompt = JSON.parse(prompts[1]!);
|
||||
assert.ok(Array.isArray(caseObservationPrompt.latestObservation.result.eventLedger));
|
||||
@@ -496,7 +497,7 @@ test("final Director repairs a generic acknowledgement and missing evidence-valu
|
||||
prompts.push(prompt);
|
||||
return {
|
||||
object: phase === "repair"
|
||||
? plan({ publicReply: { acknowledgement: "你提到2020年4月去石油化工研究院实习做研究员。", evidenceExplanation: "这是一条职业状态变化线索，按能力矩阵可参考 D10 与 Vimshottari；目前只是方法映射，不是候选结论。", candidateCommentary: "这段经历的时间和工作状态变化都很明确，可以和其他独立事件交叉比较候选范围。", limitation: null } })
|
||||
? plan({ publicReply: { acknowledgement: "你提到2020年4月去石油化工研究院实习做研究员。", evidenceExplanation: "这是一条职业状态变化线索，按能力矩阵可参考 D10 与 Vimshottari；目前只是方法映射，不是候选结论。", candidateCommentary: "这段经历的时间和工作状态变化都很明确，可以和其他独立事件交叉比较候选范围。", limitation: null }, publicExplanationGrounding: [{ source: "capability_matrix", factKey: "domain:career" }] })
|
||||
: plan(),
|
||||
};
|
||||
},
|
||||
@@ -542,7 +543,7 @@ test("public reply allows method names but rejects private internals and ungroun
|
||||
}).issues;
|
||||
assert.ok(!issues.includes("private_detail_exposed"), technique);
|
||||
}
|
||||
for (const privateDetail of ["原始评分 8.7", "权重 0.4", "贡献矩阵如下", "tool_call 原始输出"]) {
|
||||
for (const privateDetail of ["原始评分 8.7", "权重 0.4", "贡献矩阵如下", "tool_call 原始输出", "我先调用 case_read，再调用 candidate_scan 和 diagnostic_read"]) {
|
||||
const issues = validateRectificationTurnPlan({
|
||||
plan: plan({ publicReply: { acknowledgement: "这条经历已经保留。", evidenceExplanation: privateDetail, candidateCommentary: null, limitation: null } }),
|
||||
dossier: dossier(), latestAnswer: "", phase: "final",
|
||||
@@ -558,6 +559,61 @@ test("public reply allows method names but rejects private internals and ungroun
|
||||
assert.deepEqual(validateRectificationTurnPlan({ plan: single, dossier: dossier(), latestAnswer: "", phase: "final" }).issues, []);
|
||||
});
|
||||
|
||||
test("public explanations require real capability grounding for the current event", () => {
|
||||
const internship = event({
|
||||
domain: "career",
|
||||
eventKind: "career_change",
|
||||
summary: "2020年4月去研究院实习做研究员",
|
||||
rawText: "2020年4月去研究院实习做研究员",
|
||||
dateRange: { start: "2020-04-01", end: "2020-04-30", precision: "month", label: "2020年4月" },
|
||||
});
|
||||
const base = {
|
||||
acknowledgement: "你提到的是2020年4月去研究院实习做研究员。",
|
||||
candidateCommentary: "这条职业变化有明确月份，可以和其他独立事件交叉比较候选范围。",
|
||||
limitation: null,
|
||||
} as const;
|
||||
const madeUp = plan({
|
||||
publicReply: { ...base, evidenceExplanation: "这条职业变化可参考 D10；目前只是方法映射。" },
|
||||
publicExplanationGrounding: [{ source: "capability_matrix", factKey: "made-up-key" }],
|
||||
});
|
||||
assert.ok(validateRectificationTurnPlan({ plan: madeUp, dossier: dossier([internship]), latestAnswer: internship.rawText, phase: "final" }).issues.includes("public_grounding_invalid"));
|
||||
|
||||
const wrongDomain = plan({
|
||||
publicReply: { ...base, evidenceExplanation: "这条职业变化可参考 D24；目前只是方法映射。" },
|
||||
publicExplanationGrounding: [{ source: "capability_matrix", factKey: "domain:education" }],
|
||||
});
|
||||
const wrongDomainIssues = validateRectificationTurnPlan({ plan: wrongDomain, dossier: dossier([internship]), latestAnswer: internship.rawText, phase: "final" }).issues;
|
||||
assert.ok(wrongDomainIssues.includes("public_grounding_invalid"));
|
||||
assert.ok(wrongDomainIssues.includes("public_technique_not_grounded"));
|
||||
|
||||
const grounded = plan({
|
||||
publicReply: { ...base, evidenceExplanation: "这条职业变化可参考 D10 与 Vimshottari；目前只是方法映射。" },
|
||||
publicExplanationGrounding: [{ source: "capability_matrix", factKey: "domain:career" }],
|
||||
});
|
||||
assert.deepEqual(validateRectificationTurnPlan({ plan: grounded, dossier: dossier([internship]), latestAnswer: internship.rawText, phase: "final" }).issues, []);
|
||||
});
|
||||
|
||||
test("the final plan cannot repeat a previously asked question", () => {
|
||||
const previous = "你还能想到一件时间大致确定的重要经历吗？";
|
||||
const repeated = plan({
|
||||
action: {
|
||||
type: "ask_question",
|
||||
focus: { mode: "collect_independent_event", targetEventId: null, domain: null, requestedFacts: ["independent_event"], rationaleCodes: ["test"] },
|
||||
question: "你还能想到一件时间大致确定的重要经历吗",
|
||||
optionalQuickReplies: [],
|
||||
},
|
||||
});
|
||||
assert.ok(validateRectificationTurnPlan({ plan: repeated, dossier: dossier([], [turn(0, { question: previous })]), latestAnswer: "", phase: "final" }).issues.includes("question_repeated"));
|
||||
const storedPublicReply = composeRectificationPublicTurn({
|
||||
acknowledgement: "你提到的是2020年4月去研究院实习。",
|
||||
evidenceExplanation: "这条经历有明确月份，可以和其他经历交叉比较。",
|
||||
candidateUpdate: null,
|
||||
limitation: null,
|
||||
question: previous,
|
||||
});
|
||||
assert.ok(validateRectificationTurnPlan({ plan: repeated, dossier: dossier([], [turn(0, { question: storedPublicReply })]), latestAnswer: "", phase: "final" }).issues.includes("question_repeated"));
|
||||
});
|
||||
|
||||
test("candidate range requires the current approved snapshot id", () => {
|
||||
const snapshot: CandidateSnapshot = {
|
||||
id: diagnostics.snapshotId, caseId, caseVersion: 3, evidenceSetHash: "e".repeat(64), calculationSpecHash: "c".repeat(64), algorithmVersion: "rectification-v5-matrix-scoring-1",
|
||||
|
||||
Reference in New Issue
Block a user