Jesse_Chen
b1173f7245
Three collected events with a reserved holdout were stalling because the discriminator door counted holdout. Public selection_allowed still had snapshot fallbacks, and health only proved the image SHA. Co-authored-by: Cursor <cursoragent@cursor.com>
400 lines
15 KiB
TypeScript
400 lines
15 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
import { applyChoiceWithoutEvidence } from "../src/lib/rectification-agentic/v9/inference-adapter.ts";
|
|
import { applyHoldoutAnswer, buildInferenceState } from "../src/lib/rectification-agentic/core/build-state.ts";
|
|
import { unionStillValidRange } from "../src/lib/rectification-agentic/core/credible-range.ts";
|
|
import { decideRectification } from "../src/lib/rectification-agentic/core/rectification-decision.ts";
|
|
import { composeChoiceNarration } from "../src/lib/rectification-agentic/v9/choice-action.ts";
|
|
import { isPersistedFocusId } from "../src/lib/rectification-agentic/v9/choice-card.ts";
|
|
import { posteriorMap } from "../src/lib/rectification-agentic/core/decision-fingerprint.ts";
|
|
import { holdoutEventIds } from "../src/lib/rectification-agentic/core/split-holdout.ts";
|
|
import { selectDiscriminatorProbe, buildCandidateContrastPacket } from "../src/lib/rectification-agentic/core/candidate-contrast-packet.ts";
|
|
import type { ConflictProbe } from "../src/lib/rectification-agentic/core/types.ts";
|
|
|
|
function probe(input: {
|
|
id: string;
|
|
domain: string;
|
|
year: number;
|
|
yesSupports: readonly string[];
|
|
noSupports: readonly string[];
|
|
}): ConflictProbe {
|
|
return {
|
|
id: input.id,
|
|
semantic_key: `${input.domain}.${input.year}`,
|
|
candidate_split_hash: `${input.yesSupports.join(",")}|${input.noSupports.join(",")}`,
|
|
domain: input.domain,
|
|
year: input.year,
|
|
question: `${input.year} 年前后是否发生过相关前事？`,
|
|
candidate_ids: [...input.yesSupports, ...input.noSupports],
|
|
expected_outcomes: [
|
|
{ answer_class: "yes", supports: input.yesSupports, conflicts: [] },
|
|
{ answer_class: "no", supports: input.noSupports, conflicts: [] },
|
|
{ answer_class: "unsure", supports: [], conflicts: [] },
|
|
],
|
|
information_gain: 0.4,
|
|
source: "dasha_boundary",
|
|
};
|
|
}
|
|
|
|
test("credible range unions every still-valid cluster, not only rank=1", () => {
|
|
const range = unionStillValidRange([
|
|
{
|
|
id: "05:00",
|
|
time: "05:00",
|
|
cluster_range: ["05:00", "05:02"],
|
|
prior_score: 12,
|
|
posterior_score: 34,
|
|
probability: 0.34,
|
|
status: "active",
|
|
rank: 1,
|
|
strong_conflict_count: 0,
|
|
},
|
|
{
|
|
id: "05:20",
|
|
time: "05:20",
|
|
cluster_range: ["05:20", "05:22"],
|
|
prior_score: 12,
|
|
posterior_score: 33,
|
|
probability: 0.33,
|
|
status: "active",
|
|
rank: 2,
|
|
strong_conflict_count: 0,
|
|
},
|
|
{
|
|
id: "05:40",
|
|
time: "05:40",
|
|
cluster_range: ["05:40", "05:40"],
|
|
prior_score: 4,
|
|
posterior_score: 4,
|
|
probability: 0.04,
|
|
status: "active",
|
|
rank: 3,
|
|
strong_conflict_count: 0,
|
|
},
|
|
]);
|
|
assert.deepEqual(range, ["05:00", "05:22"]);
|
|
});
|
|
|
|
test("hidden case walks collection through holdout to a range or representative close", () => {
|
|
const first = probe({
|
|
id: "p-career",
|
|
domain: "career",
|
|
year: 2015,
|
|
yesSupports: ["05:00"],
|
|
noSupports: ["05:20"],
|
|
});
|
|
const collected = buildInferenceState({
|
|
range_start: "04:50",
|
|
range_end: "05:30",
|
|
candidates: [
|
|
{ id: "05:00", time: "05:00", relative_support: 10 },
|
|
{ id: "05:20", time: "05:20", relative_support: 4 },
|
|
],
|
|
events: [
|
|
{ id: "e1", domain: "education", year: 2016, precision: "month" },
|
|
{ id: "e2", domain: "career", year: 2018, precision: "year" },
|
|
{ id: "e3", domain: "relationship", year: 2021, precision: "year" },
|
|
{ id: "e4", domain: "family", year: 2023, precision: "year" },
|
|
],
|
|
probes: [first],
|
|
});
|
|
const holdoutIds = holdoutEventIds(collected.events);
|
|
assert.equal(holdoutIds.size, 1);
|
|
assert.ok(collected.events.some((item) => item.usage === "training"));
|
|
assert.ok(!collected.probes.some((item) => {
|
|
const holdout = collected.events.find((row) => row.usage === "holdout");
|
|
return holdout !== undefined && item.domain === holdout.domain && item.year === holdout.year;
|
|
}));
|
|
|
|
const packet = buildCandidateContrastPacket({
|
|
candidateSetVersion: collected.candidate_set_id,
|
|
calculationResultId: "11111111-1111-4111-8111-111111111111",
|
|
engineProbes: collected.probes.map((item) => ({
|
|
semantic_key: item.semantic_key,
|
|
candidate_split_hash: item.candidate_split_hash,
|
|
domain: item.domain,
|
|
year: item.year,
|
|
user_meaning: item.question,
|
|
information_gain: item.information_gain,
|
|
expected_outcomes: item.expected_outcomes,
|
|
candidate_ids: item.candidate_ids,
|
|
})),
|
|
});
|
|
const discriminator = selectDiscriminatorProbe(packet);
|
|
assert.ok(discriminator);
|
|
assert.ok(discriminator.informationGain > 0);
|
|
assert.ok(discriminator.expectedOutcomes.length >= 2);
|
|
const mapped = new Set(discriminator.expectedOutcomes.flatMap((row) => [
|
|
...row.supportsCandidateIds,
|
|
...row.conflictsCandidateIds,
|
|
]));
|
|
assert.ok(mapped.size >= 2);
|
|
|
|
const before = posteriorMap(collected.candidates);
|
|
const answered = applyChoiceWithoutEvidence(collected, {
|
|
choiceKey: "A",
|
|
schema: { probe_id: first.id, semantic_key: first.semantic_key },
|
|
});
|
|
assert.equal(answered.applied, true);
|
|
assert.notDeepEqual(posteriorMap(answered.state.candidates), before);
|
|
|
|
const scores = answered.state.candidates.map((item) => ({
|
|
time: item.time,
|
|
score: item.posterior_score,
|
|
}));
|
|
const afterAnswers = decideRectification({
|
|
methodCoverageAll: true,
|
|
candidateScores: scores,
|
|
holdoutValidation: "not_started",
|
|
});
|
|
assert.equal(afterAnswers.nextAction, "ask_holdout_validation");
|
|
assert.equal(afterAnswers.canConfirmExactMinute, false);
|
|
|
|
const holdoutPassed = applyHoldoutAnswer(answered.state, "yes");
|
|
assert.equal(holdoutPassed.holdout_passed, true);
|
|
assert.deepEqual(posteriorMap(holdoutPassed.candidates), posteriorMap(answered.state.candidates));
|
|
const closed = decideRectification({
|
|
methodCoverageAll: true,
|
|
candidateScores: holdoutPassed.candidates.map((item) => ({
|
|
time: item.time,
|
|
score: item.posterior_score,
|
|
})),
|
|
holdoutValidation: "passed",
|
|
inferenceCredibleRange: holdoutPassed.credible_range,
|
|
});
|
|
assert.ok(closed.nextAction === "ready_to_adopt" || closed.nextAction === "complete_with_range");
|
|
assert.equal(closed.canAdopt, true);
|
|
assert.equal(closed.canConfirmExactMinute, false);
|
|
assert.equal(closed.validated, true);
|
|
assert.equal(closed.completionStatus, "validated_range");
|
|
assert.ok(closed.credibleRange);
|
|
assert.notEqual(closed.sessionOutcome, "discriminate_candidates");
|
|
assert.notEqual(closed.sessionOutcome, "provisional_range_user_stopped");
|
|
});
|
|
|
|
test("holdout failure returns to candidate discrimination", () => {
|
|
const retry = probe({
|
|
id: "p-retry",
|
|
domain: "career",
|
|
year: 2018,
|
|
yesSupports: ["05:00"],
|
|
noSupports: ["05:20"],
|
|
});
|
|
const state = buildInferenceState({
|
|
range_start: "04:50",
|
|
range_end: "05:30",
|
|
candidates: [
|
|
{ id: "05:00", time: "05:00", relative_support: 40 },
|
|
{ id: "05:20", time: "05:20", relative_support: 12 },
|
|
],
|
|
events: [
|
|
{ id: "e1", domain: "education", year: 2016, precision: "month" },
|
|
{ id: "e2", domain: "career", year: 2018, precision: "year" },
|
|
{ id: "e3", domain: "relationship", year: 2021, precision: "year" },
|
|
{ id: "e4", domain: "family", year: 2023, precision: "year" },
|
|
],
|
|
probes: [retry],
|
|
});
|
|
const failed = applyHoldoutAnswer(state, "no");
|
|
assert.equal(failed.holdout_passed, false);
|
|
const retryPacket = buildCandidateContrastPacket({
|
|
candidateSetVersion: failed.candidate_set_id,
|
|
calculationResultId: "11111111-1111-4111-8111-111111111111",
|
|
engineProbes: [{
|
|
semantic_key: retry.semantic_key,
|
|
candidate_split_hash: retry.candidate_split_hash,
|
|
domain: retry.domain,
|
|
year: retry.year,
|
|
user_meaning: retry.question,
|
|
information_gain: retry.information_gain,
|
|
expected_outcomes: retry.expected_outcomes,
|
|
}],
|
|
});
|
|
const next = decideRectification({
|
|
methodCoverageAll: true,
|
|
candidateScores: failed.candidates.map((item) => ({ time: item.time, score: item.posterior_score })),
|
|
discriminatorProbe: selectDiscriminatorProbe(retryPacket),
|
|
holdoutValidation: "failed",
|
|
});
|
|
assert.equal(next.nextAction, "ask_candidate_discriminator");
|
|
});
|
|
|
|
test("mutated years and domains do not keep asking exam-quality copy", () => {
|
|
const shifted = probe({
|
|
id: "p-health",
|
|
domain: "health",
|
|
year: 2023,
|
|
yesSupports: ["05:00"],
|
|
noSupports: ["05:20"],
|
|
});
|
|
const state = buildInferenceState({
|
|
range_start: "04:50",
|
|
range_end: "05:30",
|
|
candidates: [
|
|
{ id: "05:00", time: "05:00", relative_support: 10 },
|
|
{ id: "05:20", time: "05:20", relative_support: 4 },
|
|
],
|
|
events: [
|
|
{ id: "e1", domain: "health", year: 2023, precision: "month" },
|
|
{ id: "e2", domain: "relocation", year: 2027, precision: "year" },
|
|
{ id: "e3", domain: "family", year: 2028, precision: "year" },
|
|
{ id: "e4", domain: "career", year: 2033, precision: "year" },
|
|
],
|
|
probes: [shifted],
|
|
});
|
|
const packet = buildCandidateContrastPacket({
|
|
candidateSetVersion: state.candidate_set_id,
|
|
calculationResultId: "11111111-1111-4111-8111-111111111111",
|
|
engineProbes: [{
|
|
semantic_key: shifted.semantic_key,
|
|
candidate_split_hash: shifted.candidate_split_hash,
|
|
domain: shifted.domain,
|
|
year: shifted.year,
|
|
user_meaning: shifted.question,
|
|
information_gain: shifted.information_gain,
|
|
expected_outcomes: shifted.expected_outcomes,
|
|
}],
|
|
});
|
|
const discriminator = selectDiscriminatorProbe(packet);
|
|
assert.ok(discriminator);
|
|
assert.equal(discriminator.domain, "health");
|
|
assert.equal(discriminator.year, 2023);
|
|
assert.doesNotMatch(discriminator.question, /高考|发挥失常|搬家/);
|
|
assert.doesNotMatch(JSON.stringify(state.probes), /高考|发挥失常/);
|
|
});
|
|
|
|
test("user stop is an unvalidated range, not a holdout pass", () => {
|
|
const stopped = decideRectification({
|
|
methodCoverageAll: true,
|
|
userStopped: true,
|
|
candidateScores: [
|
|
{ time: "05:00", score: 34 },
|
|
{ time: "05:06", score: 33 },
|
|
{ time: "05:07", score: 33 },
|
|
],
|
|
holdoutValidation: "not_started",
|
|
});
|
|
assert.equal(stopped.sessionOutcome, "provisional_range_user_stopped");
|
|
assert.equal(stopped.validated, false);
|
|
assert.equal(stopped.completionStatus, "provisional_range_user_stopped");
|
|
});
|
|
|
|
test("tied candidates without a high-information probe offer a range instead of inventing left/right", () => {
|
|
const next = decideRectification({
|
|
methodCoverageAll: true,
|
|
trainingGateOpen: true,
|
|
candidateScores: [
|
|
{ time: "05:00", score: 34 },
|
|
{ time: "05:06", score: 33 },
|
|
{ time: "05:07", score: 33 },
|
|
],
|
|
discriminatorProbe: null,
|
|
holdoutValidation: "unavailable",
|
|
});
|
|
assert.equal(next.nextAction, "offer_provisional_range");
|
|
assert.equal(next.sessionOutcome, "provisional_range");
|
|
assert.equal(next.validated, false);
|
|
assert.equal(next.probe, null);
|
|
});
|
|
|
|
test("narrator failure still leaves the choice applied and does not ask the user to repeat it", () => {
|
|
const applied = composeChoiceNarration({
|
|
optionId: "A",
|
|
scoring: true,
|
|
appliedInference: true,
|
|
});
|
|
assert.match(applied, /已记录你的选择/);
|
|
assert.doesNotMatch(applied, /请再选一次|重新回答/);
|
|
const stopped = composeChoiceNarration({
|
|
optionId: "stop",
|
|
scoring: true,
|
|
appliedInference: false,
|
|
});
|
|
assert.match(stopped, /独立核对尚未完成/);
|
|
assert.match(stopped, /不是最终校正结果/);
|
|
assert.doesNotMatch(stopped, /已完成验证/);
|
|
});
|
|
|
|
test("legacy derived question ids are not treated as persisted focus identity", () => {
|
|
assert.equal(isPersistedFocusId("d10_career:career_style:score"), false);
|
|
assert.equal(isPersistedFocusId("question-1"), false);
|
|
assert.equal(isPersistedFocusId("career-month-question"), false);
|
|
assert.equal(isPersistedFocusId("11111111-1111-4111-8111-111111111111"), true);
|
|
});
|
|
|
|
test("case A: two then three events keep collecting; four training events may discriminate", () => {
|
|
const events = [
|
|
{ id: "e1", domain: "education", year: 2016, precision: "month" as const },
|
|
{ id: "e2", domain: "career", year: 2020, precision: "month" as const },
|
|
{ id: "e3", domain: "career", year: 2020, precision: "month" as const },
|
|
{ id: "e4", domain: "relationship", year: 2024, precision: "month" as const },
|
|
{ id: "e5", domain: "career", year: 2026, precision: "day" as const },
|
|
];
|
|
const discriminator = probe({
|
|
id: "p-career",
|
|
domain: "career",
|
|
year: 2020,
|
|
yesSupports: ["05:00"],
|
|
noSupports: ["05:20"],
|
|
});
|
|
function nextFor(count: number) {
|
|
const state = buildInferenceState({
|
|
range_start: "04:45",
|
|
range_end: "05:15",
|
|
candidates: [
|
|
{ id: "05:00", time: "05:00", relative_support: 10 },
|
|
{ id: "05:20", time: "05:20", relative_support: 4 },
|
|
],
|
|
events: events.slice(0, count),
|
|
probes: [discriminator],
|
|
});
|
|
const packet = buildCandidateContrastPacket({
|
|
candidateSetVersion: state.candidate_set_id,
|
|
calculationResultId: "11111111-1111-4111-8111-111111111111",
|
|
engineProbes: state.probes.map((item) => ({
|
|
semantic_key: item.semantic_key,
|
|
candidate_split_hash: item.candidate_split_hash,
|
|
domain: item.domain,
|
|
year: item.year,
|
|
user_meaning: item.question,
|
|
information_gain: item.information_gain,
|
|
expected_outcomes: item.expected_outcomes,
|
|
candidate_ids: item.candidate_ids,
|
|
})),
|
|
});
|
|
return decideRectification({
|
|
methodCoverageAll: true,
|
|
trainingGateOpen: state.events.filter((item) => item.usage === "training").length >= 3
|
|
&& new Set(state.events.filter((item) => item.usage === "training").map((item) => item.domain)).size >= 2,
|
|
candidateScores: state.candidates.map((item) => ({ time: item.time, score: item.posterior_score })),
|
|
discriminatorProbe: selectDiscriminatorProbe(packet),
|
|
holdoutValidation: state.events.some((item) => item.usage === "holdout") ? "not_started" : "unavailable",
|
|
});
|
|
}
|
|
assert.equal(nextFor(2).nextAction, "ask_fact_collection");
|
|
assert.equal(nextFor(3).nextAction, "ask_fact_collection");
|
|
const ready = nextFor(4);
|
|
assert.equal(ready.nextAction, "ask_candidate_discriminator");
|
|
assert.ok(ready.probe);
|
|
assert.ok(ready.probe.informationGain > 0);
|
|
assert.ok(ready.probe.expectedOutcomes.length >= 2);
|
|
const mapped = new Set(ready.probe.expectedOutcomes.flatMap((row) => [
|
|
...row.supportsCandidateIds,
|
|
...row.conflictsCandidateIds,
|
|
]));
|
|
assert.ok(mapped.size >= 2);
|
|
const five = nextFor(5);
|
|
assert.equal(five.nextAction, "ask_candidate_discriminator");
|
|
assert.ok(holdoutEventIds(buildInferenceState({
|
|
range_start: "04:45",
|
|
range_end: "05:15",
|
|
candidates: [
|
|
{ id: "05:00", time: "05:00", relative_support: 10 },
|
|
{ id: "05:20", time: "05:20", relative_support: 4 },
|
|
],
|
|
events,
|
|
probes: [discriminator],
|
|
}).events).size >= 1);
|
|
});
|