import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; import { PersonalReportJobServiceError, type PersonalReportJobRecord, type RecoverExpiredPersonalReportJobsResult, } from "../src/lib/personal-report-job-service-core.ts"; import type { GeneratePersonalReportResult } from "../src/lib/personal-report-generation.ts"; import type { PersonalReportFailureCode, PersonalReportRecord, } from "../src/lib/personal-report-service-core.ts"; import { PersonalReportWorkerError, createPersonalReportWorker, } from "../src/lib/personal-report-worker-core.ts"; const USER_ID = "11111111-1111-4111-8111-111111111111"; const REQUEST_ID = "22222222-2222-4222-8222-222222222222"; const REPORT_ID = "33333333-3333-4333-8333-333333333333"; const JOB_ID = "44444444-4444-4444-8444-444444444444"; const LEASE_A = "55555555-5555-4555-8555-555555555555"; const FINGERPRINT = "a".repeat(64); const NOW = new Date("2026-08-14T12:00:00.000Z"); type ReadyDocument = Extract["document"]; const READY_DOCUMENT = { schemaVersion: "report_document.v2", reportId: REPORT_ID } as ReadyDocument; function reportRecord(overrides: Partial = {}): PersonalReportRecord { return { id: REPORT_ID, userId: USER_ID, sessionId: null, chartProfileId: null, requestId: REQUEST_ID, requestFingerprint: FINGERPRINT, reportType: "personal_full", status: "generating", schemaVersion: "report_document.v2", presentationMode: "default", depth: "standard", requestedThemes: ["career"], reportDocument: null, calculationHash: null, evidenceHash: null, skillName: "jyotish-personal-report", skillVersion: "1.0.0", skillSourceCommit: null, skillSnapshotSha256: "b".repeat(64), failureCode: null, createdAt: "2026-08-14T11:59:00.000Z", updatedAt: "2026-08-14T11:59:00.000Z", completedAt: null, ...overrides, }; } function jobRecord(overrides: Partial = {}): PersonalReportJobRecord { return { id: JOB_ID, userId: USER_ID, requestId: REQUEST_ID, requestFingerprint: FINGERPRINT, status: "queued", attemptCount: 0, maxAttempts: 3, leaseToken: null, leaseOwner: null, leaseAcquiredAt: null, leaseExpiresAt: null, heartbeatAt: null, nextAttemptAt: null, progressPhase: "queued", progressPercent: 0, lastErrorCode: null, lastErrorAt: null, startedAt: null, finishedAt: null, createdAt: "2026-08-14T11:59:00.000Z", updatedAt: "2026-08-14T11:59:00.000Z", ...overrides, }; } function createHarness(input: Readonly<{ job?: PersonalReportJobRecord; report?: PersonalReportRecord | null; generate?: () => Promise; loseLeaseBeforeCompletion?: boolean; markReadyError?: PersonalReportJobServiceError; }>) { let job = input.job ?? jobRecord(); let report = input.report === undefined ? reportRecord() : input.report; let leaseSerial = 0; const events: string[] = []; let generateCount = 0; let completeCount = 0; let reportFailureCount = 0; function requireLease(jobId: string, leaseToken: string): PersonalReportJobRecord { assert.equal(jobId, job.id); assert.equal(job.status, "running"); assert.equal(leaseToken, job.leaseToken); return job; } const jobs = { async recoverExpiredLeases(): Promise { events.push("recover"); if (job.status !== "running" || !job.leaseExpiresAt || Date.parse(job.leaseExpiresAt) > NOW.getTime()) { return { retryingCount: 0, failedCount: 0 }; } if (report?.status === "ready") { job = { ...job, status: "ready", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "ready", progressPercent: 100, }; return { retryingCount: 0, failedCount: 0 }; } if (job.attemptCount < job.maxAttempts) { job = { ...job, status: "retrying", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "retry_wait", lastErrorCode: "lease_expired", }; return { retryingCount: 1, failedCount: 0 }; } if (report?.status === "generating") { reportFailureCount += 1; report = reportRecord({ ...report, status: "failed", failureCode: "calculation_unavailable", }); } job = { ...job, status: "failed", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "failed", lastErrorCode: "lease_expired_attempts_exhausted", }; return { retryingCount: 0, failedCount: 1 }; }, async claimLease({ workerId }: { workerId: string; leaseSeconds?: number; jobId?: string | null }) { events.push("claim"); if (job.status !== "queued" && job.status !== "retrying") return null; leaseSerial += 1; job = { ...job, status: "running", attemptCount: job.attemptCount + 1, leaseToken: LEASE_A.replace(/.$/, String(leaseSerial)), leaseOwner: workerId, leaseAcquiredAt: NOW.toISOString(), leaseExpiresAt: new Date(NOW.getTime() + 120_000).toISOString(), }; return job; }, async heartbeatLease({ jobId, leaseToken }: { jobId: string; leaseToken: string; leaseSeconds?: number }) { requireLease(jobId, leaseToken); events.push("heartbeat"); return job; }, async updateProgress({ jobId, leaseToken, phase, percent }: { jobId: string; leaseToken: string; phase: string; percent: number; }) { requireLease(jobId, leaseToken); events.push(`progress:${phase}:${percent}`); job = { ...job, progressPhase: phase, progressPercent: percent }; return job; }, async markReady({ jobId, leaseToken }: { jobId: string; leaseToken: string }) { requireLease(jobId, leaseToken); events.push("job:ready"); if (input.markReadyError) throw input.markReadyError; job = { ...job, status: "ready", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "ready", progressPercent: 100, }; return job; }, async completeReportReady({ jobId, leaseToken, report: reportInput, document }: { jobId: string; leaseToken: string; report: PersonalReportRecord; document: unknown; }) { if (input.loseLeaseBeforeCompletion) { job = { ...job, leaseToken: LEASE_A }; throw new PersonalReportJobServiceError("lease_lost"); } requireLease(jobId, leaseToken); events.push("report+job:ready"); assert.equal(reportInput, report); assert.equal(document, READY_DOCUMENT); assert.ok(report); completeCount += 1; report = reportRecord({ ...report, status: "ready", reportDocument: document as PersonalReportRecord["reportDocument"], completedAt: NOW.toISOString(), }); job = { ...job, status: "ready", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "ready", progressPercent: 100, }; return job; }, async completeReportFailed({ jobId, leaseToken, errorCode, report: reportInput, failureCode }: { jobId: string; leaseToken: string; errorCode: string; report: PersonalReportRecord; failureCode: PersonalReportFailureCode; }) { if (input.loseLeaseBeforeCompletion) { job = { ...job, leaseToken: LEASE_A }; throw new PersonalReportJobServiceError("lease_lost"); } requireLease(jobId, leaseToken); events.push(`report+job:failed:${failureCode}:${errorCode}`); assert.equal(reportInput, report); assert.ok(report); reportFailureCount += 1; report = reportRecord({ ...report, status: "failed", failureCode }); job = { ...job, status: "failed", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "failed", lastErrorCode: errorCode, }; return job; }, async markFailed({ jobId, leaseToken, errorCode }: { jobId: string; leaseToken: string; errorCode: string; }) { requireLease(jobId, leaseToken); events.push(`job:failed:${errorCode}`); job = { ...job, status: "failed", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, progressPhase: "failed", lastErrorCode: errorCode, }; return job; }, async scheduleRetry({ jobId, leaseToken, errorCode, nextAttemptAt }: { jobId: string; leaseToken: string; errorCode: string; nextAttemptAt?: Date; }) { requireLease(jobId, leaseToken); events.push(`job:retry:${errorCode}`); if (job.attemptCount >= job.maxAttempts) { return { kind: "exhausted" as const, job }; } job = { ...job, status: "retrying", leaseToken: null, leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: nextAttemptAt?.toISOString() ?? NOW.toISOString(), lastErrorCode: errorCode, progressPhase: "retry_wait", }; return { kind: "scheduled" as const, job }; }, }; const reports = { async getByUserAndRequestId(userId: string, requestId: string) { events.push("report:get"); assert.equal(userId, USER_ID); assert.equal(requestId, REQUEST_ID); return report; }, }; const worker = createPersonalReportWorker({ workerId: "test-worker", jobs, reports, loadProfile: async (userId) => { events.push("profile:get"); assert.equal(userId, USER_ID); return { birth_time_status: "confirmed" }; }, generate: async () => { events.push("generate"); generateCount += 1; return input.generate?.() ?? { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) }; }, now: () => NOW, retryDelayMs: () => 1_000, heartbeatIntervalMs: 60_000, }); return { worker, events, get job() { return job; }, get report() { return report; }, get generateCount() { return generateCount; }, get completeCount() { return completeCount; }, get reportFailureCount() { return reportFailureCount; }, }; } test("tick recovers first, heartbeats, persists the report, then closes the leased job", async () => { const harness = createHarness({}); const result = await harness.worker.tick(); assert.equal(result.outcome, "ready"); assert.equal(harness.job.status, "ready"); assert.equal(harness.report?.status, "ready"); assert.equal(harness.completeCount, 1); assert.equal(harness.events.filter((event) => event === "report+job:ready").length, 1); assert.ok(harness.events.indexOf("recover") < harness.events.indexOf("claim")); assert.ok(harness.events.includes("heartbeat")); assert.equal(harness.events.includes("report:ready"), false); assert.equal(harness.events.includes("job:ready"), false); assert.deepEqual( harness.events.filter((event) => event.startsWith("progress:")), [ "progress:loading_context:10", "progress:generating_report:30", "progress:persisting_report:90", ], ); }); test("a restarted process reclaims an expired lease and does not create a second report or job", async () => { const harness = createHarness({ job: jobRecord({ status: "running", attemptCount: 1, leaseToken: LEASE_A, leaseOwner: "dead-process", leaseExpiresAt: "2026-08-14T11:59:59.000Z", }), }); const first = await harness.worker.tick(); const second = await harness.worker.tick(); assert.equal(first.recovery.retryingCount, 1); assert.equal(first.outcome, "ready"); assert.equal(second.outcome, "idle"); assert.equal(harness.job.attemptCount, 2); assert.equal(harness.generateCount, 1); assert.equal(harness.completeCount, 1); assert.equal(harness.events.filter((event) => event === "report:get").length, 1); }); test("restart after a historical split commit reconciles an expired final-attempt job to ready", async () => { const harness = createHarness({ job: jobRecord({ status: "running", attemptCount: 3, maxAttempts: 3, leaseToken: LEASE_A, leaseOwner: "dead-after-report-commit", leaseExpiresAt: "2026-08-14T11:59:59.000Z", }), report: reportRecord({ status: "ready", reportDocument: READY_DOCUMENT as PersonalReportRecord["reportDocument"], completedAt: "2026-08-14T11:59:58.000Z", }), }); const result = await harness.worker.tick(); assert.equal(result.outcome, "idle"); assert.equal(harness.job.status, "ready"); assert.equal(harness.generateCount, 0); assert.equal(harness.completeCount, 0); assert.deepEqual(harness.events, ["recover", "claim"]); }); test("expired final-attempt recovery atomically fails a still-generating report and job", async () => { const harness = createHarness({ job: jobRecord({ status: "running", attemptCount: 3, maxAttempts: 3, leaseToken: LEASE_A, leaseOwner: "expired-worker", leaseExpiresAt: "2026-08-14T11:59:59.000Z", }), }); const result = await harness.worker.tick(); assert.equal(result.outcome, "idle"); assert.equal(result.recovery.failedCount, 1); assert.equal(harness.report?.status, "failed"); assert.equal(harness.report?.failureCode, "calculation_unavailable"); assert.equal(harness.job.status, "failed"); assert.equal(harness.job.lastErrorCode, "lease_expired_attempts_exhausted"); assert.equal(harness.reportFailureCount, 1); assert.deepEqual(harness.events, ["recover", "claim"]); }); test("ready reconciliation storage failure never downgrades the job to failed", async () => { const harness = createHarness({ job: jobRecord({ maxAttempts: 1 }), report: reportRecord({ status: "ready", reportDocument: READY_DOCUMENT as PersonalReportRecord["reportDocument"], completedAt: "2026-08-14T11:59:58.000Z", }), markReadyError: new PersonalReportJobServiceError("storage_failed"), }); const result = await harness.worker.tick(); assert.equal(result.outcome, "reconcile_deferred"); assert.equal(harness.report?.status, "ready"); assert.equal(harness.job.status, "running"); assert.equal(harness.job.attemptCount, 1); assert.equal(harness.reportFailureCount, 0); assert.equal(harness.events.includes("job:failed:report_generation_failed"), false); assert.equal(harness.events.some((event) => event.startsWith("report+job:failed:")), false); }); test("a worker that loses its lease cannot atomically commit either terminal state", async () => { const harness = createHarness({ loseLeaseBeforeCompletion: true }); const result = await harness.worker.tick(); assert.equal(result.outcome, "lease_lost"); assert.equal(harness.completeCount, 0); assert.equal(harness.report?.status, "generating"); assert.equal(harness.job.status, "running"); assert.equal(harness.events.includes("report+job:ready"), false); }); test("a worker that loses its lease cannot atomically fail either terminal state", async () => { const harness = createHarness({ loseLeaseBeforeCompletion: true, generate: async () => ({ status: "failed", failureCode: "report_schema_invalid" }), }); const result = await harness.worker.tick(); assert.equal(result.outcome, "lease_lost"); assert.equal(harness.reportFailureCount, 0); assert.equal(harness.report?.status, "generating"); assert.equal(harness.job.status, "running"); assert.equal(harness.events.some((event) => event.startsWith("report+job:failed:")), false); }); test("invalid writer output atomically fails the report and lease-bound job", async () => { const harness = createHarness({ generate: async () => ({ status: "failed", failureCode: "report_schema_invalid" }), }); const result = await harness.worker.tick(); assert.equal(result.outcome, "failed"); assert.equal(harness.report?.status, "failed"); assert.equal(harness.report?.failureCode, "report_schema_invalid"); assert.equal(harness.job.status, "failed"); assert.equal(harness.job.lastErrorCode, "report_schema_invalid"); assert.equal(harness.reportFailureCount, 1); assert.equal( harness.events.filter((event) => event === "report+job:failed:report_schema_invalid:report_schema_invalid").length, 1, ); assert.equal(harness.events.some((event) => event.startsWith("job:retry:")), false); }); test("retryable failures use the bounded job retry budget and fail the report on exhaustion", async () => { const harness = createHarness({ job: jobRecord({ maxAttempts: 2 }), generate: async () => { throw new PersonalReportWorkerError("calculation_unavailable", true); }, }); const first = await harness.worker.tick(); const second = await harness.worker.tick(); assert.equal(first.outcome, "retry_scheduled"); assert.equal(second.outcome, "failed"); assert.equal(harness.job.attemptCount, 2); assert.equal(harness.job.status, "failed"); assert.equal(harness.report?.status, "failed"); assert.equal(harness.report?.failureCode, "calculation_unavailable"); assert.equal(harness.reportFailureCount, 1); }); test("instrumentation retains the Skill guard and starts a singleton Node worker loop", () => { const instrumentation = readFileSync(new URL("../src/instrumentation.ts", import.meta.url), "utf8"); const productionAdapter = readFileSync(new URL("../src/lib/personal-report-worker.ts", import.meta.url), "utf8"); assert.match(instrumentation, /verifyAllActiveSkillPackages\(\)/); assert.match(instrumentation, /resolveLiveJyotishSkill\(\)/); assert.match(instrumentation, /startPersonalReportWorker\(\)/); assert.ok( instrumentation.indexOf("verifyAllActiveSkillPackages()") < instrumentation.indexOf("resolveLiveJyotishSkill()"), ); assert.ok( instrumentation.indexOf("resolveLiveJyotishSkill()") < instrumentation.indexOf("startPersonalReportWorker()"), ); assert.match(productionAdapter, /jyotishaPersonalReportWorker/); assert.match(productionAdapter, /if \(state\.jyotishaPersonalReportWorker\) return/); assert.doesNotMatch(productionAdapter, /after\s*\(/); });