fix(report): keep personal-report heartbeats ref'd during chapter writes
Staging lost the job lease mid-chapter because the unref'd interval never fired outside an HTTP request. Refresh the lease on section progress and leave the heartbeat timer ref'd so a long model await cannot starve it. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -165,11 +165,6 @@ function reportFailureCode(code: PersonalReportWorkerErrorCode): PersonalReportF
|
||||
}
|
||||
}
|
||||
|
||||
function timerUnref(timer: ReturnType<typeof globalThis.setInterval>): void {
|
||||
const candidate = timer as ReturnType<typeof globalThis.setInterval> & { unref?: () => void };
|
||||
candidate.unref?.();
|
||||
}
|
||||
|
||||
function defaultRetryDelayMs(job: PersonalReportJobRecord): number {
|
||||
const exponent = Math.max(0, Math.min(job.attemptCount - 1, 5));
|
||||
return 5_000 * (2 ** exponent);
|
||||
@@ -269,22 +264,26 @@ export function createPersonalReportWorker(deps: PersonalReportWorkerDeps) {
|
||||
const controller = new AbortController();
|
||||
let heartbeatError: unknown = null;
|
||||
let heartbeatChain = Promise.resolve();
|
||||
const refreshLease = async () => {
|
||||
if (heartbeatError !== null || controller.signal.aborted) return;
|
||||
await deps.jobs.heartbeatLease({
|
||||
jobId: job.id,
|
||||
leaseToken: job.leaseToken!,
|
||||
leaseSeconds,
|
||||
});
|
||||
};
|
||||
const heartbeatTimer = setIntervalFn(() => {
|
||||
heartbeatChain = heartbeatChain
|
||||
.then(async () => {
|
||||
if (heartbeatError !== null || controller.signal.aborted) return;
|
||||
await deps.jobs.heartbeatLease({
|
||||
jobId: job.id,
|
||||
leaseToken: job.leaseToken!,
|
||||
leaseSeconds,
|
||||
});
|
||||
})
|
||||
.then(refreshLease)
|
||||
.catch((error) => {
|
||||
heartbeatError = error;
|
||||
controller.abort(error);
|
||||
});
|
||||
}, heartbeatIntervalMs);
|
||||
timerUnref(heartbeatTimer);
|
||||
// Keep this timer ref'd. The worker runs from instrumentation, not an HTTP
|
||||
// request; unref'd intervals in Next.js standalone are skipped while the
|
||||
// only in-flight work is a long model await, which is exactly when the
|
||||
// lease must be extended.
|
||||
|
||||
let report: PersonalReportRecord | null = null;
|
||||
const generationStartedAt = now().getTime();
|
||||
@@ -339,6 +338,8 @@ export function createPersonalReportWorker(deps: PersonalReportWorkerDeps) {
|
||||
signal: controller.signal,
|
||||
sectionService: deps.sectionService,
|
||||
onProgress: async (progress) => {
|
||||
await refreshLease();
|
||||
if (heartbeatError !== null) throw heartbeatError;
|
||||
const percent = progress.total > 0
|
||||
? Math.min(89, 30 + Math.floor((progress.completed / progress.total) * 55))
|
||||
: 30;
|
||||
|
||||
@@ -422,6 +422,12 @@ test("production writer and worker keep the same signal on initial and repair mo
|
||||
assert.match(workerSource, /generatePersonalReport\(\{[\s\S]*signal: context\.signal,[\s\S]*\}\)/);
|
||||
assert.match(workerSource, /leaseSeconds: 600/);
|
||||
assert.match(workerSource, /heartbeatIntervalMs: 20_000/);
|
||||
const workerCore = readFileSync(
|
||||
new URL("../src/lib/personal-report-worker-core.ts", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(workerCore, /onProgress: async \(progress\) => \{\s*await refreshLease\(\)/);
|
||||
assert.doesNotMatch(workerCore, /timerUnref\(heartbeatTimer\)/);
|
||||
});
|
||||
|
||||
test("an all-blocked plan still produces a valid v2 document with honest disclosures and no invented thematic section", async () => {
|
||||
|
||||
@@ -87,7 +87,9 @@ function jobRecord(overrides: Partial<PersonalReportJobRecord> = {}): PersonalRe
|
||||
function createHarness(input: Readonly<{
|
||||
job?: PersonalReportJobRecord;
|
||||
report?: PersonalReportRecord | null;
|
||||
generate?: () => Promise<GeneratePersonalReportResult>;
|
||||
generate?: (context: {
|
||||
onProgress?: (progress: Readonly<{ phase: string; completed: number; total: number }>) => Promise<void> | void;
|
||||
}) => Promise<GeneratePersonalReportResult>;
|
||||
loseLeaseBeforeCompletion?: boolean;
|
||||
markReadyError?: PersonalReportJobServiceError;
|
||||
billing?: ReportBillingPort;
|
||||
@@ -326,10 +328,10 @@ function createHarness(input: Readonly<{
|
||||
assert.equal(userId, USER_ID);
|
||||
return { birth_time_status: "confirmed" };
|
||||
},
|
||||
generate: async () => {
|
||||
generate: async (context) => {
|
||||
events.push("generate");
|
||||
generateCount += 1;
|
||||
return input.generate?.() ?? { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) };
|
||||
return input.generate?.(context) ?? { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) };
|
||||
},
|
||||
billing: input.billing
|
||||
? {
|
||||
@@ -385,6 +387,22 @@ test("tick recovers first, heartbeats, persists the report, then closes the leas
|
||||
);
|
||||
});
|
||||
|
||||
test("section progress heartbeats the lease before recording percent", async () => {
|
||||
const harness = createHarness({
|
||||
generate: async (context) => {
|
||||
await context.onProgress?.({ phase: "section:theme-career", completed: 1, total: 4 });
|
||||
return { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) };
|
||||
},
|
||||
});
|
||||
const result = await harness.worker.tick();
|
||||
assert.equal(result.outcome, "ready");
|
||||
const generateAt = harness.events.indexOf("generate");
|
||||
const progressAt = harness.events.indexOf("progress:section:theme-career:43");
|
||||
assert.ok(generateAt >= 0);
|
||||
assert.ok(progressAt > generateAt);
|
||||
assert.ok(harness.events.slice(generateAt + 1, progressAt).includes("heartbeat"));
|
||||
});
|
||||
|
||||
test("a restarted process reclaims an expired lease and does not create a second report or job", async () => {
|
||||
const harness = createHarness({
|
||||
job: jobRecord({
|
||||
|
||||
Reference in New Issue
Block a user