Add date-isolated caches and regression coverage, align scoring identity, and freeze full research reruns while preserving historical artifacts. Record unresolved cache/receipt identity and end-to-end acceptance gaps for branch review only. Co-Authored-By: Claude Code <noreply@anthropic.com>
196 lines
11 KiB
TypeScript
196 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { readFileSync } from "node:fs";
|
|
import test from "node:test";
|
|
|
|
import {
|
|
cachedEngineScoreIsReusable,
|
|
liveEngineScoringIdentityFromEnv,
|
|
readV9EngineScoringIdentity,
|
|
v9EngineVersion,
|
|
} from "../src/lib/rectification-agentic/v9/engine-client.ts";
|
|
import { openRectificationCase } from "../src/lib/rectification-agentic/v9/case-service.ts";
|
|
import { loadV9TurnReceipt } from "../src/lib/rectification-agentic/v9/tool-service.ts";
|
|
import { resolveSkillPackageVersion } from "../src/lib/skill-package-registry.ts";
|
|
import { createRectificationV9Tools } from "../src/mastra/rectification-v9-tools.ts";
|
|
import {
|
|
CASE_ID, SESSION_ID, TURN_ID, USER_ID, FOCUS_ID,
|
|
computeFixture, dossierFixture, fakeAccounting, receiptHandlers,
|
|
} from "./rectification-v9-test-support.ts";
|
|
|
|
// Real native-engine response, using explicitly fictional input, not a hand-built contract.
|
|
const golden = JSON.parse(readFileSync(new URL(
|
|
"./fixtures/rectification-engine-version-cross-midnight-golden.json", import.meta.url,
|
|
), "utf8"));
|
|
const CURRENT = "rectification-v5-matrix-scoring-8";
|
|
const PREVIOUS = "rectification-v5-matrix-scoring-7";
|
|
const envKeys = [
|
|
"RECTIFICATION_ENGINE_VERSION", "RECTIFICATION_ALGORITHM_VERSION",
|
|
"RECTIFICATION_DECISION_POLICY_VERSION",
|
|
] as const;
|
|
|
|
function isolateIdentityEnv(t: test.TestContext) {
|
|
const saved = envKeys.map((key) => [key, process.env[key]] as const);
|
|
for (const key of envKeys) delete process.env[key];
|
|
t.after(() => {
|
|
for (const [key, value] of saved) {
|
|
if (value === undefined) delete process.env[key];
|
|
else process.env[key] = value;
|
|
}
|
|
});
|
|
}
|
|
|
|
test("receipt default matches the cross-midnight native scoring version without changing Skill identity", (t) => {
|
|
isolateIdentityEnv(t);
|
|
assert.equal(v9EngineVersion(), CURRENT);
|
|
for (const blank of ["", " "]) {
|
|
process.env.RECTIFICATION_ENGINE_VERSION = blank;
|
|
assert.equal(v9EngineVersion(), CURRENT);
|
|
}
|
|
assert.equal(golden.score.algorithm_version, CURRENT);
|
|
assert.equal(golden.versions.algorithm_version, CURRENT);
|
|
const python = readFileSync(new URL("../../scripts/rectification/scoring_service.py", import.meta.url), "utf8");
|
|
assert.match(python, /ALGORITHM_VERSION = "rectification-v5-matrix-scoring-8"/);
|
|
// A receipt default must not become an environment override for cache identity.
|
|
assert.deepEqual(liveEngineScoringIdentityFromEnv({}), { algorithmVersion: null, policyVersion: null });
|
|
});
|
|
|
|
test("explicit engine override still wins and remains a deployment prerequisite, not silently rewritten", (t) => {
|
|
isolateIdentityEnv(t);
|
|
process.env.RECTIFICATION_ENGINE_VERSION = " rectification-v5 ";
|
|
assert.equal(v9EngineVersion(), "rectification-v5");
|
|
assert.equal(liveEngineScoringIdentityFromEnv().algorithmVersion, null);
|
|
process.env.RECTIFICATION_ENGINE_VERSION = ` ${PREVIOUS} `;
|
|
assert.equal(v9EngineVersion(), PREVIOUS);
|
|
// This existing cache-identity fallback predates this change; no new open gate is introduced.
|
|
assert.equal(liveEngineScoringIdentityFromEnv().algorithmVersion, PREVIOUS);
|
|
});
|
|
|
|
test("native versions invalidate matching-input old scores while same-version scores remain reusable", async (t) => {
|
|
isolateIdentityEnv(t);
|
|
t.mock.method(globalThis, "fetch", async (url: unknown) => {
|
|
assert.ok(String(url).endsWith("/api/rectification/v5/versions"));
|
|
return Response.json(golden.versions);
|
|
});
|
|
const live = await readV9EngineScoringIdentity();
|
|
assert.equal(live.algorithmVersion, CURRENT);
|
|
const fingerprints = { evidenceLedgerFingerprint: "e".repeat(64), candidateRangeFingerprint: "c".repeat(64) };
|
|
const previous = { ...fingerprints, algorithmVersion: PREVIOUS, policyVersion: golden.versions.decision_policy_version };
|
|
assert.equal(cachedEngineScoreIsReusable(previous, fingerprints, live), false);
|
|
assert.equal(cachedEngineScoreIsReusable({ ...previous, algorithmVersion: CURRENT }, fingerprints, live), true);
|
|
assert.equal(previous.algorithmVersion, PREVIOUS);
|
|
});
|
|
|
|
test("policy-only or stale algorithm overrides bypass native versions under the existing cache policy", async (t) => {
|
|
isolateIdentityEnv(t);
|
|
let fetches = 0;
|
|
t.mock.method(globalThis, "fetch", async () => {
|
|
fetches += 1;
|
|
return Response.json(golden.versions);
|
|
});
|
|
const fingerprints = { evidenceLedgerFingerprint: "e".repeat(64), candidateRangeFingerprint: "c".repeat(64) };
|
|
const stored = { ...fingerprints, algorithmVersion: PREVIOUS, policyVersion: golden.versions.decision_policy_version };
|
|
process.env.RECTIFICATION_DECISION_POLICY_VERSION = golden.versions.decision_policy_version;
|
|
const policyOnly = await readV9EngineScoringIdentity();
|
|
assert.equal(policyOnly.algorithmVersion, null);
|
|
assert.equal(cachedEngineScoreIsReusable(stored, fingerprints, policyOnly), true);
|
|
process.env.RECTIFICATION_ALGORITHM_VERSION = PREVIOUS;
|
|
const staleOverride = await readV9EngineScoringIdentity();
|
|
assert.equal(staleOverride.algorithmVersion, PREVIOUS);
|
|
assert.equal(cachedEngineScoreIsReusable(stored, fingerprints, staleOverride), true);
|
|
assert.equal(fetches, 0);
|
|
});
|
|
|
|
test("unreachable versions retain the existing unknown-identity cache fallback, not proof of rescore", async (t) => {
|
|
isolateIdentityEnv(t);
|
|
t.mock.method(globalThis, "fetch", async () => { throw new Error("fixture versions unavailable"); });
|
|
const live = await readV9EngineScoringIdentity();
|
|
assert.deepEqual(live, { algorithmVersion: null, policyVersion: null });
|
|
const fingerprints = { evidenceLedgerFingerprint: "e".repeat(64), candidateRangeFingerprint: "c".repeat(64) };
|
|
assert.equal(cachedEngineScoreIsReusable({ ...fingerprints, algorithmVersion: PREVIOUS }, fingerprints, live), true);
|
|
});
|
|
|
|
test("new Case scoring writes the new version on both started and completed receipts", async (t) => {
|
|
isolateIdentityEnv(t);
|
|
const request = golden.request;
|
|
const candidateRange = { start_time: request.start_time, end_time: request.end_time };
|
|
const evidence = request.events.map((event: Record<string, string>) => ({
|
|
id: event.id, source_turn_id: TURN_ID, subject: "self", event_kind: event.event_kind,
|
|
domain: event.domain, occurred_from: event.date_start, occurred_to: event.date_end,
|
|
date_precision: event.precision, summary: event.summary, status: "confirmed",
|
|
}));
|
|
const calls: string[] = [];
|
|
t.mock.method(globalThis, "fetch", async (url: unknown) => {
|
|
const path = new URL(String(url)).pathname;
|
|
calls.push(path);
|
|
if (path.endsWith("/versions")) return Response.json(golden.versions);
|
|
if (path.endsWith("/score") || path.endsWith("/diagnostics")) return Response.json(golden.score);
|
|
throw new Error(`unexpected engine request ${path}`);
|
|
});
|
|
const accounting = fakeAccounting({
|
|
...receiptHandlers,
|
|
get_agentic_rectification_case_dossier: () => dossierFixture({ candidateRange, evidence, latestResult: null }),
|
|
get_agentic_rectification_case_compute: () => ({
|
|
...computeFixture({ baselineBirthSnapshot: {
|
|
birth_date: request.birth_date, latitude: request.lat, longitude: request.lon,
|
|
timezone_offset: request.tz, timezone_id: "Asia/Shanghai", birth_time_source: "approximate",
|
|
} }), candidate_range: candidateRange,
|
|
}),
|
|
persist_agentic_rectification_candidate_v2: (_fn, args) => ({
|
|
result_id: args.p_engine_result_id, cached: false, candidates: args.p_candidates,
|
|
algorithm_version: args.p_algorithm_version, event_contract_version: args.p_event_contract_version,
|
|
decision_policy_version: args.p_decision_policy_version, decision_receipt: args.p_decision_receipt,
|
|
execution_ledger: args.p_execution_ledger, overall_confidence: golden.score.decision_receipt.overall_confidence,
|
|
selection_allowed: golden.score.decision_receipt.selection_allowed, confirmation_allowed: false,
|
|
representative_time: golden.score.decision_receipt.representative_time,
|
|
}),
|
|
set_agentic_rectification_conversation_focus: (_fn, args) => ({
|
|
focus: { id: FOCUS_ID, case_id: CASE_ID, question_id: args.p_question_id,
|
|
intent: args.p_intent, target_evidence_id: args.p_target_evidence_id,
|
|
target_domain: args.p_target_domain, target_kind: args.p_target_kind,
|
|
expected_answer_schema: args.p_expected_answer_schema, status: "active" }, idempotent: false,
|
|
}),
|
|
});
|
|
const tools = createRectificationV9Tools({ userId: USER_ID, caseId: CASE_ID, turnId: TURN_ID, accounting: accounting.client as never });
|
|
for (const name of ["rectification-compare-candidates", "rectification-read-diagnostics"] as const) {
|
|
await (tools[name] as unknown as { execute(input: unknown): Promise<unknown> }).execute({ caseId: CASE_ID });
|
|
const receipts = accounting.calls.filter((call) => call.fn === "insert_agentic_rectification_tool_receipt" && call.args.p_tool_name === name);
|
|
assert.deepEqual(receipts.map((call) => [call.args.p_status, call.args.p_engine_version]), [
|
|
["started", CURRENT], ["completed", CURRENT],
|
|
]);
|
|
}
|
|
assert.ok(calls.includes("/api/rectification/v5/score"));
|
|
const persisted = accounting.calls.find((call) => call.fn === "persist_agentic_rectification_candidate_v2");
|
|
assert.equal(persisted?.args.p_algorithm_version, CURRENT);
|
|
});
|
|
|
|
test("history opens with its bound Skill and reads old engine receipt values unchanged", async (t) => {
|
|
isolateIdentityEnv(t);
|
|
const bound = resolveSkillPackageVersion("jyotish-birth-time-rectification", "10.0.17");
|
|
for (const oldVersion of ["rectification-v5", PREVIOUS]) {
|
|
const historical = { turn_id: TURN_ID, engine_version: oldVersion, skill_name: bound.name, skill_version: bound.version, status: "completed" };
|
|
const before = structuredClone(historical);
|
|
const open = { disposition: "readonly", case_id: CASE_ID, session_id: SESSION_ID,
|
|
status: "closed", should_start_opening: false, skill_version: bound.version };
|
|
const accounting = fakeAccounting({
|
|
open_agentic_rectification_case: () => open,
|
|
get_agentic_rectification_skill_identity: () => ({ skill_name: bound.name, skill_version: bound.version, skill_sha256: bound.sha256, skill_source_commit: bound.sourceCommit }),
|
|
open_agentic_rectification_case_v2: (_fn, args) => {
|
|
assert.equal(args.p_skill_version, bound.version);
|
|
assert.equal(args.p_skill_sha256, bound.sha256);
|
|
assert.equal("p_engine_version" in args, false);
|
|
return open;
|
|
},
|
|
get_agentic_rectification_turn_receipt: () => historical,
|
|
});
|
|
const opened = await openRectificationCase(accounting.client as never, USER_ID, {
|
|
intent: "session", requestId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", sessionId: SESSION_ID,
|
|
});
|
|
assert.equal(opened.disposition, "readonly");
|
|
const receipt = await loadV9TurnReceipt(accounting.client as never, USER_ID, CASE_ID, TURN_ID);
|
|
assert.equal(v9EngineVersion(), CURRENT);
|
|
assert.equal(receipt?.engineVersion, oldVersion);
|
|
assert.deepEqual(historical, before);
|
|
assert.equal(accounting.calls.some((call) => /insert|persist|upgrade/.test(call.fn)), false);
|
|
}
|
|
});
|