fix(rectification): score varga-style groups at equal weight

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jesse_Chen
2026-08-29 00:15:21 +08:00
parent adfc75af8c
commit e41eaace8d
8 changed files with 222 additions and 5 deletions
@@ -19,9 +19,14 @@ export function outcomeForAnswer(probe: ConflictProbe, answer: AnswerClass): Pro
return probe.expected_outcomes.find((item) => item.answer_class === answer) ?? null;
}
export function directionFor(candidateId: string, outcome: ProbeOutcome): ScoreDirection {
if (outcome.supports.includes(candidateId)) return outcome.answer_class === "weak_yes" ? "weak_support" : "support";
if (outcome.conflicts.includes(candidateId)) return outcome.answer_class === "weak_yes" ? "weak_conflict" : "conflict";
export function directionFor(
candidateId: string,
outcome: ProbeOutcome,
choiceKind?: ConflictProbe["choice_kind"],
): ScoreDirection {
const halfWeight = outcome.answer_class === "weak_yes" && choiceKind !== "varga_style";
if (outcome.supports.includes(candidateId)) return halfWeight ? "weak_support" : "support";
if (outcome.conflicts.includes(candidateId)) return halfWeight ? "weak_conflict" : "conflict";
return "neutral";
}
@@ -60,7 +65,7 @@ export function applyProbeOutcome(
};
}
for (const [id, score] of Object.entries(scores)) {
const direction = directionFor(id, outcome);
const direction = directionFor(id, outcome, probe.choice_kind);
if (direction === "conflict") conflictCounts[id] = (conflictCounts[id] ?? 0) + 1;
if (eliminated.has(id)) {
next[id] = score;
@@ -602,6 +602,11 @@ export function conflictProbesFromContrast(
expected_outcomes: outcomes,
information_gain: probe.informationGain,
source: "varga_contrast",
...(probe.choiceKind === "varga_style"
|| probe.choiceKind === "event_quality"
|| probe.choiceKind === "existence"
? { choice_kind: probe.choiceKind }
: {}),
}];
});
}
@@ -32,6 +32,7 @@ const CANDIDATE_STATUSES = new Set(["active", "eliminated", "winner", "equivalen
const EVENT_PRECISIONS = new Set(["day", "month", "year", "unknown"]);
const EVENT_USAGES = new Set(["training", "holdout", "unused"]);
const ANSWER_CLASSES = new Set(["yes", "weak_yes", "no", "unsure"]);
const PROBE_CHOICE_KINDS = new Set(["existence", "varga_style", "event_quality"]);
const ANSWER_SOURCES = new Set(["choice", "evidence", "declined"]);
const ROUND_KINDS = new Set(["informative", "low_information"]);
@@ -88,7 +89,9 @@ function isConflictProbe(value: unknown): boolean {
&& isStringArray(value.candidate_ids)
&& Array.isArray(value.expected_outcomes) && value.expected_outcomes.every(isProbeOutcome)
&& isFiniteNumber(value.information_gain)
&& typeof value.source === "string" && value.source.length > 0;
&& typeof value.source === "string" && value.source.length > 0
&& (value.choice_kind === undefined
|| (typeof value.choice_kind === "string" && PROBE_CHOICE_KINDS.has(value.choice_kind)));
}
function isProbeAnswer(value: unknown): boolean {
@@ -31,6 +31,11 @@ export function probeFromEngine(probe: EngineProbeFields): ConflictProbe | null
expected_outcomes: outcomes,
information_gain: probe.information_gain ?? 0,
source: probe.source,
...(probe.choice_kind === "varga_style"
|| probe.choice_kind === "event_quality"
|| probe.choice_kind === "existence"
? { choice_kind: probe.choice_kind }
: {}),
};
}
@@ -36,6 +36,7 @@ export type ResultStatus =
export type CandidateStatus = "active" | "eliminated" | "winner" | "equivalent";
export type EventUsage = "training" | "holdout" | "unused";
export type AnswerClass = "yes" | "weak_yes" | "no" | "unsure";
export type ProbeChoiceKind = "existence" | "varga_style" | "event_quality";
export type ScoreDirection = "support" | "weak_support" | "neutral" | "weak_conflict" | "conflict";
export const SCORE_DELTA: Readonly<Record<ScoreDirection, number>> = {
@@ -83,6 +84,7 @@ export type ConflictProbe = Readonly<{
expected_outcomes: readonly ProbeOutcome[];
information_gain: number;
source: string;
choice_kind?: ProbeChoiceKind;
}>;
export type ProbeAnswer = Readonly<{
@@ -151,6 +151,11 @@ export function contrastPacketFromLatestResult(
information_gain: probe.information_gain,
expected_outcomes: probe.expected_outcomes,
candidate_ids: probe.candidate_ids,
...(probe.choice_kind === "varga_style"
|| probe.choice_kind === "event_quality"
|| probe.choice_kind === "existence"
? { choice_kind: probe.choice_kind }
: {}),
}];
});
const merged = mergeEngineProbes(
@@ -248,6 +253,11 @@ function contrastPacketFromState(state: InferenceState): CandidateContrastPacket
information_gain: item.information_gain,
expected_outcomes: item.expected_outcomes,
candidate_ids: item.candidate_ids,
...(item.choice_kind === "varga_style"
|| item.choice_kind === "event_quality"
|| item.choice_kind === "existence"
? { choice_kind: item.choice_kind }
: {}),
})),
candidateTimes: state.candidates
.filter((item) => item.status !== "eliminated")