fix(rectification): score varga-style groups at equal weight
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -19,9 +19,14 @@ export function outcomeForAnswer(probe: ConflictProbe, answer: AnswerClass): Pro
|
||||
return probe.expected_outcomes.find((item) => item.answer_class === answer) ?? null;
|
||||
}
|
||||
|
||||
export function directionFor(candidateId: string, outcome: ProbeOutcome): ScoreDirection {
|
||||
if (outcome.supports.includes(candidateId)) return outcome.answer_class === "weak_yes" ? "weak_support" : "support";
|
||||
if (outcome.conflicts.includes(candidateId)) return outcome.answer_class === "weak_yes" ? "weak_conflict" : "conflict";
|
||||
export function directionFor(
|
||||
candidateId: string,
|
||||
outcome: ProbeOutcome,
|
||||
choiceKind?: ConflictProbe["choice_kind"],
|
||||
): ScoreDirection {
|
||||
const halfWeight = outcome.answer_class === "weak_yes" && choiceKind !== "varga_style";
|
||||
if (outcome.supports.includes(candidateId)) return halfWeight ? "weak_support" : "support";
|
||||
if (outcome.conflicts.includes(candidateId)) return halfWeight ? "weak_conflict" : "conflict";
|
||||
return "neutral";
|
||||
}
|
||||
|
||||
@@ -60,7 +65,7 @@ export function applyProbeOutcome(
|
||||
};
|
||||
}
|
||||
for (const [id, score] of Object.entries(scores)) {
|
||||
const direction = directionFor(id, outcome);
|
||||
const direction = directionFor(id, outcome, probe.choice_kind);
|
||||
if (direction === "conflict") conflictCounts[id] = (conflictCounts[id] ?? 0) + 1;
|
||||
if (eliminated.has(id)) {
|
||||
next[id] = score;
|
||||
|
||||
@@ -602,6 +602,11 @@ export function conflictProbesFromContrast(
|
||||
expected_outcomes: outcomes,
|
||||
information_gain: probe.informationGain,
|
||||
source: "varga_contrast",
|
||||
...(probe.choiceKind === "varga_style"
|
||||
|| probe.choiceKind === "event_quality"
|
||||
|| probe.choiceKind === "existence"
|
||||
? { choice_kind: probe.choiceKind }
|
||||
: {}),
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ const CANDIDATE_STATUSES = new Set(["active", "eliminated", "winner", "equivalen
|
||||
const EVENT_PRECISIONS = new Set(["day", "month", "year", "unknown"]);
|
||||
const EVENT_USAGES = new Set(["training", "holdout", "unused"]);
|
||||
const ANSWER_CLASSES = new Set(["yes", "weak_yes", "no", "unsure"]);
|
||||
const PROBE_CHOICE_KINDS = new Set(["existence", "varga_style", "event_quality"]);
|
||||
const ANSWER_SOURCES = new Set(["choice", "evidence", "declined"]);
|
||||
const ROUND_KINDS = new Set(["informative", "low_information"]);
|
||||
|
||||
@@ -88,7 +89,9 @@ function isConflictProbe(value: unknown): boolean {
|
||||
&& isStringArray(value.candidate_ids)
|
||||
&& Array.isArray(value.expected_outcomes) && value.expected_outcomes.every(isProbeOutcome)
|
||||
&& isFiniteNumber(value.information_gain)
|
||||
&& typeof value.source === "string" && value.source.length > 0;
|
||||
&& typeof value.source === "string" && value.source.length > 0
|
||||
&& (value.choice_kind === undefined
|
||||
|| (typeof value.choice_kind === "string" && PROBE_CHOICE_KINDS.has(value.choice_kind)));
|
||||
}
|
||||
|
||||
function isProbeAnswer(value: unknown): boolean {
|
||||
|
||||
@@ -31,6 +31,11 @@ export function probeFromEngine(probe: EngineProbeFields): ConflictProbe | null
|
||||
expected_outcomes: outcomes,
|
||||
information_gain: probe.information_gain ?? 0,
|
||||
source: probe.source,
|
||||
...(probe.choice_kind === "varga_style"
|
||||
|| probe.choice_kind === "event_quality"
|
||||
|| probe.choice_kind === "existence"
|
||||
? { choice_kind: probe.choice_kind }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ export type ResultStatus =
|
||||
export type CandidateStatus = "active" | "eliminated" | "winner" | "equivalent";
|
||||
export type EventUsage = "training" | "holdout" | "unused";
|
||||
export type AnswerClass = "yes" | "weak_yes" | "no" | "unsure";
|
||||
export type ProbeChoiceKind = "existence" | "varga_style" | "event_quality";
|
||||
export type ScoreDirection = "support" | "weak_support" | "neutral" | "weak_conflict" | "conflict";
|
||||
|
||||
export const SCORE_DELTA: Readonly<Record<ScoreDirection, number>> = {
|
||||
@@ -83,6 +84,7 @@ export type ConflictProbe = Readonly<{
|
||||
expected_outcomes: readonly ProbeOutcome[];
|
||||
information_gain: number;
|
||||
source: string;
|
||||
choice_kind?: ProbeChoiceKind;
|
||||
}>;
|
||||
|
||||
export type ProbeAnswer = Readonly<{
|
||||
|
||||
@@ -151,6 +151,11 @@ export function contrastPacketFromLatestResult(
|
||||
information_gain: probe.information_gain,
|
||||
expected_outcomes: probe.expected_outcomes,
|
||||
candidate_ids: probe.candidate_ids,
|
||||
...(probe.choice_kind === "varga_style"
|
||||
|| probe.choice_kind === "event_quality"
|
||||
|| probe.choice_kind === "existence"
|
||||
? { choice_kind: probe.choice_kind }
|
||||
: {}),
|
||||
}];
|
||||
});
|
||||
const merged = mergeEngineProbes(
|
||||
@@ -248,6 +253,11 @@ function contrastPacketFromState(state: InferenceState): CandidateContrastPacket
|
||||
information_gain: item.information_gain,
|
||||
expected_outcomes: item.expected_outcomes,
|
||||
candidate_ids: item.candidate_ids,
|
||||
...(item.choice_kind === "varga_style"
|
||||
|| item.choice_kind === "event_quality"
|
||||
|| item.choice_kind === "existence"
|
||||
? { choice_kind: item.choice_kind }
|
||||
: {}),
|
||||
})),
|
||||
candidateTimes: state.candidates
|
||||
.filter((item) => item.status !== "eliminated")
|
||||
|
||||
Reference in New Issue
Block a user