401 lines
15 KiB
TypeScript
401 lines
15 KiB
TypeScript
import { applyProbeOutcome } from "./apply-probe-outcome.ts";
|
|
import { clusterRangeFor, clusterEquivalentCandidates } from "./cluster-candidates.ts";
|
|
import { evaluateConvergence, holdoutStillRanksFirst, rankActive } from "./convergence-evaluator.ts";
|
|
import { unionStillValidRange } from "./credible-range.ts";
|
|
import { entropyFromScores, normalizeScores } from "./entropy.ts";
|
|
import { selectHighestGainProbe } from "./select-probe.ts";
|
|
import { holdoutDomainYears, holdoutEventIds, stickyHoldoutEvents } from "./split-holdout.ts";
|
|
import {
|
|
INFERENCE_ALGORITHM_VERSION,
|
|
type AnswerClass,
|
|
type ConflictProbe,
|
|
type InferenceCandidate,
|
|
type InferenceEvent,
|
|
type InferenceState,
|
|
type ProbeAnswer,
|
|
type RectificationPhase,
|
|
type ResultStatus,
|
|
} from "./types.ts";
|
|
|
|
export type EngineCandidateInput = Readonly<{
|
|
id: string;
|
|
time: string;
|
|
relative_support: number;
|
|
}>;
|
|
|
|
export type EngineEventInput = Readonly<{
|
|
id: string;
|
|
domain: string;
|
|
year: number | null;
|
|
precision: InferenceEvent["precision"];
|
|
}>;
|
|
|
|
export function candidateSetId(rangeStart: string, rangeEnd: string, times: readonly string[]): string {
|
|
return `${rangeStart}-${rangeEnd}:${[...times].sort().join(",")}`;
|
|
}
|
|
|
|
export function buildInferenceState(input: {
|
|
range_start: string;
|
|
range_end: string;
|
|
candidates: readonly EngineCandidateInput[];
|
|
events: readonly EngineEventInput[];
|
|
probes: readonly ConflictProbe[];
|
|
previous?: InferenceState | null;
|
|
answered_probes?: readonly ProbeAnswer[];
|
|
transition_times?: readonly string[];
|
|
event_ledger?: Readonly<Record<string, Readonly<Record<string, number>>>>;
|
|
phase?: RectificationPhase;
|
|
}): InferenceState {
|
|
const setId = candidateSetId(
|
|
input.range_start,
|
|
input.range_end,
|
|
input.candidates.map((item) => item.time),
|
|
);
|
|
const sameSet = input.previous?.candidate_set_id === setId;
|
|
const previous = sameSet ? input.previous : null;
|
|
const events = stickyHoldoutEvents(input.events, previous?.events);
|
|
const holdoutKeys = holdoutDomainYears(events);
|
|
const probes = input.probes.filter((probe) => !holdoutKeys.has(`${probe.domain}:${probe.year}`));
|
|
const prior = Object.fromEntries(input.candidates.map((item) => [item.id, item.relative_support]));
|
|
const trainingPrior = subtractHoldout(prior, input.candidates, events, input.event_ledger);
|
|
const answers = mergeAnswers(input.previous?.answered_probes ?? [], input.answered_probes ?? []);
|
|
const seenRoundIds = new Set((previous?.rounds ?? []).map((item) => item.probe_id));
|
|
// Rebuild elimination from the current answer ledger. Superseded or removed
|
|
// answers must be able to revive a candidate that no longer reaches the threshold.
|
|
const eliminated = new Set<string>();
|
|
let scores = { ...trainingPrior };
|
|
let conflictCounts: Record<string, number> = Object.fromEntries(
|
|
input.candidates.map((item) => [item.id, 0]),
|
|
);
|
|
const rounds = [...(previous?.rounds ?? [])];
|
|
|
|
for (const answer of answers) {
|
|
// Engine event scores already include dated evidence. A structured A/yes
|
|
// choice is not an evidence row, so it must still move the posterior.
|
|
if (answer.answer_class === "yes" && answer.classified_from === "evidence") continue;
|
|
const probe = input.probes.find((item) => item.id === answer.probe_id)
|
|
?? input.probes.find((item) => item.semantic_key === answer.semantic_key)
|
|
?? input.previous?.probes.find((item) => item.id === answer.probe_id)
|
|
?? input.previous?.probes.find((item) => item.semantic_key === answer.semantic_key);
|
|
if (!probe) continue;
|
|
if (holdoutKeys.has(`${probe.domain}:${probe.year}`)) continue;
|
|
const before = { ...scores };
|
|
const applied = applyProbeOutcome(scores, probe, answer.answer_class, {
|
|
eliminatedIds: eliminated,
|
|
strongConflictCounts: conflictCounts,
|
|
});
|
|
scores = { ...applied.scores };
|
|
conflictCounts = { ...applied.strong_conflict_counts };
|
|
for (const id of applied.eliminated_ids) eliminated.add(id);
|
|
if (seenRoundIds.has(probe.id)) continue;
|
|
seenRoundIds.add(probe.id);
|
|
rounds.push({
|
|
round: rounds.length + 1,
|
|
phase: "discrimination",
|
|
probe_id: probe.id,
|
|
scores_before: before,
|
|
scores_after: scores,
|
|
score_deltas: applied.deltas,
|
|
entropy_before: entropyFromScores(before),
|
|
entropy_after: entropyFromScores(scores),
|
|
eliminated_ids: applied.eliminated_ids,
|
|
winner_id: rankScoreIds(scores, eliminated)[0] ?? null,
|
|
kind: applied.kind,
|
|
});
|
|
}
|
|
|
|
const probabilities = normalizeScores(omitEliminated(scores, eliminated));
|
|
const clusters = clusterEquivalentCandidates(
|
|
input.candidates.map((item) => ({ id: item.id, time: item.time, score: scores[item.id] ?? 0 })),
|
|
input.transition_times ?? [],
|
|
);
|
|
const rankedIds = rankScoreIds(scores, eliminated);
|
|
const candidates: InferenceCandidate[] = input.candidates.map((item) => {
|
|
const clustered = clusters.find((row) => row.member_ids.includes(item.id));
|
|
const rank = rankedIds.indexOf(item.id);
|
|
return {
|
|
id: item.id,
|
|
time: item.time,
|
|
cluster_range: clusterRangeFor(clusters, item.id, item.time),
|
|
prior_score: trainingPrior[item.id] ?? 0,
|
|
posterior_score: scores[item.id] ?? 0,
|
|
probability: eliminated.has(item.id) ? 0 : probabilities[item.id] ?? 0,
|
|
status: eliminated.has(item.id)
|
|
? "eliminated"
|
|
: (clustered?.member_ids.length ?? 1) > 1
|
|
? "equivalent"
|
|
: "active",
|
|
rank: rank >= 0 ? rank + 1 : input.candidates.length,
|
|
strong_conflict_count: conflictCounts[item.id] ?? 0,
|
|
};
|
|
});
|
|
|
|
const holdoutPassed = previous?.holdout_passed === true || previous?.holdout_passed === false
|
|
? previous.holdout_passed
|
|
: holdoutStillRanksFirst(
|
|
candidates,
|
|
holdoutOnlyScores(input.candidates, events, input.event_ledger),
|
|
);
|
|
const active = rankActive(candidates);
|
|
const top = active[0] ?? null;
|
|
const alreadyAnswered = new Set((input.previous?.answered_probes ?? []).map((item) => item.probe_id));
|
|
const newAnswerCount = answers.filter((item) => !alreadyAnswered.has(item.probe_id)).length;
|
|
const draft: InferenceState = {
|
|
algorithm_version: INFERENCE_ALGORITHM_VERSION,
|
|
candidate_set_id: setId,
|
|
revision: Math.max(1, (input.previous?.revision ?? 0) + (newAnswerCount > 0 ? 1 : 0)),
|
|
phase: input.phase ?? input.previous?.phase ?? "discrimination",
|
|
result_status: "discriminating",
|
|
range_start: input.range_start,
|
|
range_end: input.range_end,
|
|
candidates,
|
|
events,
|
|
probes,
|
|
answered_probes: answers,
|
|
rounds,
|
|
last_inference_round: rounds.at(-1) ?? null,
|
|
entropy: entropyFromScores(omitEliminated(scores, eliminated)),
|
|
representative_time: top?.time ?? null,
|
|
credible_range: unionStillValidRange(candidates),
|
|
holdout_passed: holdoutPassed,
|
|
};
|
|
const decision = evaluateConvergence({ ...draft, holdout_passed: holdoutPassed });
|
|
return {
|
|
...draft,
|
|
phase: phaseFor(decision.result_status, draft.phase),
|
|
result_status: decision.result_status,
|
|
representative_time: decision.representative_time,
|
|
credible_range: decision.credible_range ?? draft.credible_range,
|
|
candidates: candidates.map((item) => (
|
|
decision.result_status === "converged" && item.id === decision.winner_id
|
|
? { ...item, status: "winner" }
|
|
: item
|
|
)),
|
|
};
|
|
}
|
|
|
|
export function applyAnswerToState(
|
|
state: InferenceState,
|
|
probeId: string,
|
|
answer: AnswerClass,
|
|
): InferenceState {
|
|
const probe = state.probes.find((item) => item.id === probeId);
|
|
if (!probe) return state;
|
|
return rebuildWithAnswers(state, [{
|
|
probe_id: probe.id,
|
|
semantic_key: probe.semantic_key,
|
|
candidate_split_hash: probe.candidate_split_hash,
|
|
answer_class: answer,
|
|
classified_from: "choice",
|
|
}]);
|
|
}
|
|
|
|
export function applySupersedeAnswer(
|
|
state: InferenceState,
|
|
probeId: string,
|
|
answer: AnswerClass,
|
|
): InferenceState {
|
|
const live = state.probes.find((item) => item.id === probeId);
|
|
const previousAnswer = state.answered_probes.find((item) => item.probe_id === probeId);
|
|
const semanticKey = live?.semantic_key ?? previousAnswer?.semantic_key;
|
|
const splitHash = live?.candidate_split_hash ?? previousAnswer?.candidate_split_hash;
|
|
if (!semanticKey || !splitHash) return state;
|
|
const remaining = state.answered_probes.filter((item) => (
|
|
item.probe_id !== probeId && item.semantic_key !== semanticKey
|
|
));
|
|
const rounds = state.rounds.filter((item) => item.probe_id !== probeId);
|
|
return rebuildWithAnswers(
|
|
{ ...state, answered_probes: remaining, rounds },
|
|
[{
|
|
probe_id: probeId,
|
|
semantic_key: semanticKey,
|
|
candidate_split_hash: splitHash,
|
|
answer_class: answer,
|
|
classified_from: "choice",
|
|
}],
|
|
);
|
|
}
|
|
|
|
export function replayInferenceState(
|
|
state: InferenceState,
|
|
answers: readonly ProbeAnswer[],
|
|
): InferenceState {
|
|
return buildInferenceState({
|
|
range_start: state.range_start,
|
|
range_end: state.range_end,
|
|
candidates: state.candidates.map((item) => ({
|
|
id: item.id,
|
|
time: item.time,
|
|
relative_support: item.prior_score,
|
|
})),
|
|
events: state.events,
|
|
probes: state.probes,
|
|
previous: { ...state, answered_probes: [], rounds: [] },
|
|
answered_probes: answers,
|
|
});
|
|
}
|
|
|
|
export function nextProbe(state: InferenceState): ConflictProbe | null {
|
|
const holdoutKeys = holdoutDomainYears(state.events);
|
|
const probes = state.probes.filter((probe) => !holdoutKeys.has(`${probe.domain}:${probe.year}`));
|
|
return selectHighestGainProbe(probes, state.answered_probes);
|
|
}
|
|
|
|
export function answersFromEvidence(
|
|
probes: readonly ConflictProbe[],
|
|
events: readonly EngineEventInput[],
|
|
): ProbeAnswer[] {
|
|
const split = stickyHoldoutEvents(events);
|
|
const holdoutKeys = holdoutDomainYears(split);
|
|
const training = split.filter((item) => item.usage === "training");
|
|
return probes.flatMap((probe) => {
|
|
if (probe.source === "known_event_quality" || probe.source === "varga_contrast") return [];
|
|
if (holdoutKeys.has(`${probe.domain}:${probe.year}`)) return [];
|
|
if (!training.some((item) => item.domain === probe.domain && item.year === probe.year)) return [];
|
|
return [{
|
|
probe_id: probe.id,
|
|
semantic_key: probe.semantic_key,
|
|
candidate_split_hash: probe.candidate_split_hash,
|
|
answer_class: "yes" as const,
|
|
classified_from: "evidence" as const,
|
|
}];
|
|
});
|
|
}
|
|
|
|
export function applyHoldoutAnswer(
|
|
state: InferenceState,
|
|
answer: AnswerClass,
|
|
): InferenceState {
|
|
const passed = answer === "yes" || answer === "weak_yes";
|
|
const failed = answer === "no";
|
|
if (!passed && !failed) {
|
|
return {
|
|
...state,
|
|
phase: "holdout_validation",
|
|
holdout_passed: null,
|
|
};
|
|
}
|
|
if (failed) {
|
|
return {
|
|
...state,
|
|
revision: state.revision + 1,
|
|
phase: "discrimination",
|
|
result_status: "validation_failed",
|
|
holdout_passed: false,
|
|
};
|
|
}
|
|
return {
|
|
...state,
|
|
revision: state.revision + 1,
|
|
phase: "completed",
|
|
result_status: state.result_status === "converged" ? "converged" : "completed_with_range",
|
|
holdout_passed: true,
|
|
};
|
|
}
|
|
|
|
export function classifyChoiceAnswer(key: string, schema?: unknown): AnswerClass | null {
|
|
if (!schema || typeof schema !== "object" || Array.isArray(schema)) return null;
|
|
const row = schema as Record<string, unknown>;
|
|
const choice = row.choice && typeof row.choice === "object" && !Array.isArray(row.choice)
|
|
? row.choice as Record<string, unknown>
|
|
: row;
|
|
if (!Array.isArray(choice.options)) return null;
|
|
const option = choice.options.find((item) => (
|
|
item && typeof item === "object" && !Array.isArray(item) && (item as { key?: unknown }).key === key
|
|
));
|
|
if (!option || typeof option !== "object") return null;
|
|
const answerClass = (option as { answer_class?: unknown }).answer_class;
|
|
return answerClass === "yes" || answerClass === "weak_yes" || answerClass === "no" || answerClass === "unsure"
|
|
? answerClass
|
|
: null;
|
|
}
|
|
|
|
function rebuildWithAnswers(state: InferenceState, incoming: readonly ProbeAnswer[]): InferenceState {
|
|
return buildInferenceState({
|
|
range_start: state.range_start,
|
|
range_end: state.range_end,
|
|
candidates: state.candidates.map((item) => ({
|
|
id: item.id,
|
|
time: item.time,
|
|
relative_support: item.prior_score,
|
|
})),
|
|
events: state.events,
|
|
probes: state.probes,
|
|
previous: state,
|
|
answered_probes: incoming,
|
|
});
|
|
}
|
|
|
|
function mergeAnswers(previous: readonly ProbeAnswer[], incoming: readonly ProbeAnswer[]): ProbeAnswer[] {
|
|
const rows = [...previous];
|
|
for (const item of incoming) {
|
|
if (rows.some((row) => row.probe_id === item.probe_id || row.semantic_key === item.semantic_key)) continue;
|
|
rows.push(item);
|
|
}
|
|
return rows;
|
|
}
|
|
|
|
function omitEliminated(
|
|
scores: Readonly<Record<string, number>>,
|
|
eliminated: ReadonlySet<string>,
|
|
): Record<string, number> {
|
|
return Object.fromEntries(Object.entries(scores).filter(([id]) => !eliminated.has(id)));
|
|
}
|
|
|
|
function rankScoreIds(
|
|
scores: Readonly<Record<string, number>>,
|
|
eliminated: ReadonlySet<string>,
|
|
): string[] {
|
|
return Object.entries(omitEliminated(scores, eliminated))
|
|
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
|
|
.map(([id]) => id);
|
|
}
|
|
|
|
function subtractHoldout(
|
|
prior: Readonly<Record<string, number>>,
|
|
candidates: readonly EngineCandidateInput[],
|
|
events: readonly InferenceEvent[],
|
|
ledger: Readonly<Record<string, Readonly<Record<string, number>>>> | undefined,
|
|
): Record<string, number> {
|
|
const holdout = holdoutEventIds(events);
|
|
if (holdout.size === 0 || !ledger) return { ...prior };
|
|
const next = { ...prior };
|
|
for (const eventId of holdout) {
|
|
const byTime = ledger[eventId];
|
|
if (!byTime) continue;
|
|
for (const candidate of candidates) {
|
|
next[candidate.id] = (next[candidate.id] ?? 0) - (byTime[candidate.time] ?? 0);
|
|
}
|
|
}
|
|
return next;
|
|
}
|
|
|
|
function holdoutOnlyScores(
|
|
candidates: readonly EngineCandidateInput[],
|
|
events: readonly InferenceEvent[],
|
|
ledger: Readonly<Record<string, Readonly<Record<string, number>>>> | undefined,
|
|
): Record<string, number> {
|
|
const holdout = holdoutEventIds(events);
|
|
if (holdout.size === 0 || !ledger) return {};
|
|
const scores: Record<string, number> = Object.fromEntries(candidates.map((item) => [item.id, 0]));
|
|
for (const eventId of holdout) {
|
|
const byTime = ledger[eventId];
|
|
if (!byTime) continue;
|
|
for (const candidate of candidates) {
|
|
scores[candidate.id] = (scores[candidate.id] ?? 0) + (byTime[candidate.time] ?? 0);
|
|
}
|
|
}
|
|
return scores;
|
|
}
|
|
|
|
function phaseFor(status: ResultStatus, fallback: RectificationPhase): RectificationPhase {
|
|
if (status === "converged" || status === "credible_range" || status === "completed_with_range") {
|
|
return "completed";
|
|
}
|
|
if (status === "validation_failed") return "discrimination";
|
|
if (status === "max_rounds_reached") return "completed";
|
|
if (status === "insufficient_evidence") return "event_collection";
|
|
if (status === "discriminating") return "discrimination";
|
|
return fallback;
|
|
}
|