From 5c0bec0c9419e12e2c1fcab8e75fd35d535cb50e Mon Sep 17 00:00:00 2001 From: Jesse_Chen Date: Thu, 3 Sep 2026 02:13:13 +0800 Subject: [PATCH] fix(report): keep personal-report heartbeats ref'd during chapter writes Staging lost the job lease mid-chapter because the unref'd interval never fired outside an HTTP request. Refresh the lease on section progress and leave the heartbeat timer ref'd so a long model await cannot starve it. Co-authored-by: Cursor --- PROGRESS-report-writer-failure-20260902.md | 6 ++-- docs/BUG_HISTORY.md | 2 +- .../src/lib/personal-report-worker-core.ts | 29 ++++++++++--------- .../personal-report-generation-v2.test.ts | 6 ++++ frontend/tests/personal-report-worker.test.ts | 24 +++++++++++++-- 5 files changed, 46 insertions(+), 21 deletions(-) diff --git a/PROGRESS-report-writer-failure-20260902.md b/PROGRESS-report-writer-failure-20260902.md index 7fe85bfd..7ff8a28d 100644 --- a/PROGRESS-report-writer-failure-20260902.md +++ b/PROGRESS-report-writer-failure-20260902.md @@ -124,7 +124,7 @@ standard `personal_full`，主题 career / marriage / wealth / timing。started Job 在写作过程中 `lease_expired` 两次（默认租约 120s，单章可超过 2 分钟；heartbeat 未能保住租约），第三次 attempt 写完 wealth 后约 39ms 整份 `report_schema_invalid`。进度停在 71%（timing 完成后的值），wealth 的 85% `onProgress` 未落库。四个 section 行仍是 ready。摘要/装配没有产出 `report_document`。reservation `released`。 -默认 `leaseSeconds=120` 不够串行四章写作。本轮补丁改为 **600s 租约、20s heartbeat**。摘要预算与章节同一 cap **3072**（原公式 3180）。 +默认 `leaseSeconds=120` 不够串行四章写作。心跳 `setInterval` 被 `unref` 后，Next.js instrumentation 在没有 HTTP 请求时会跳过定时器，单章 4 分钟写作期间租约无法续上。补丁：**600s 租约、20s 心跳、心跳定时器保持 ref、章节 `onProgress` 先 `heartbeatLease` 再写进度**。摘要预算与章节同一 cap **3072**（原公式 3180）。 ### 回溯抽查（只记是否对得上 ref，不摘正文） @@ -148,7 +148,7 @@ Job 在写作过程中 `lease_expired` 两次（默认租约 120s，单章可超 ### 任务 4 结论 -Writer 修正在章节层成立：四章有正文、career 一次通过、无截断码。整份 ready 仍被 120s 租约和最后一步 `report_schema_invalid` 挡住。租约与摘要 cap 的补丁随本轮提交；整份 ready 要等该补丁部署后再验一次。 +Writer 修正在章节层成立：四章有正文、career 一次通过、无截断码。整份 ready 仍被 120s 租约和最后一步 `report_schema_invalid` 挡住。租约、心跳续租与摘要 cap 已推 staging；公开 health 在跟进部署前仍指向 `eda37c15`。整份 ready 要等新 SHA 部署后再验一次。 --- @@ -164,4 +164,4 @@ Writer 修正在章节层成立：四章有正文、career 一次通过、无截 未跑全量 `tests/*.test.ts`，不声称全套绿。未改 `.gitea/workflows/**`，未提升 main。 -任务 4 补丁（租约 600s、摘要 cap 3072）：`tsc --noEmit` 通过；plan / generation-v2 / worker 43 pass。 +任务 4 补丁（租约 600s、摘要 cap 3072、心跳保持 ref + 进度续租）：`tsc --noEmit` 通过；plan / generation-v2 / worker 聚焦测试通过。 diff --git a/docs/BUG_HISTORY.md b/docs/BUG_HISTORY.md index e1991a14..9a903ec0 100644 --- a/docs/BUG_HISTORY.md +++ b/docs/BUG_HISTORY.md @@ -7643,7 +7643,7 @@ - 触发条件：章节 writer 输出未通过 `assertWriterOutput`（常见为 `evidenceRefs` 与 plan 集合不等），或输出在 token 预算内被截断；随后整份报告失败。 - 根因：三层叠加。旧错误分类把任何含 `evidence` 的消息打成 `section_evidence_insufficient`，`report_writer_evidence_refs_mismatch` 被误标；章节输出预算用英文「字符 ÷ 2」，standard 上限 1200 字中文只分到 1024 token；一章 blocked 或未捕获抛错可以中止整份，job 停在约 43%。失败详情只有聚合码，页面无法区分截断、引用不齐或未开始。staging 事故窗口的 writer telemetry 在容器 recreate 后丢失，本事故不能写成 `finishReason=length`。 - 修复：中文口径重算每章/摘要 `maxOutputTokens`，去掉 ÷2；length 修复要求压到字数下限且预算不低于首次。Prompt 要求逐字复制 `plan.evidenceRefs`；repair 只加失败类别。refs/identity 先于泛化 `evidence` 分类。单章失败后继续其余主题；`all_sections_blocked` 走统一失败日志。详情与列表聚合已有 section 行，展示可读摘要与错误码，不改表、不放宽 writer schema。 -- 验证：预算表锁定 concise 1218 / standard 2128 / deep 3072 / research 3072 / summary 3072。refs mismatch 分类为 `section_refs_mismatch` 且其余主题仍交付。repair 提示含类别词、不含内容。失败详情从 section 行汇总。`tsc --noEmit`、改动文件 ESLint、个人报告聚焦测试。staging `eda37c15` 上真实 standard 四章均 ready（career 1 次成功，无截断码），整份仍因 120s 租约与最后一步 `report_schema_invalid` 未落 `report_document`；租约改为 600s。 +- 验证：预算表锁定 concise 1218 / standard 2128 / deep 3072 / research 3072 / summary 3072。refs mismatch 分类为 `section_refs_mismatch` 且其余主题仍交付。repair 提示含类别词、不含内容。失败详情从 section 行汇总。`tsc --noEmit`、改动文件 ESLint、个人报告聚焦测试。staging `eda37c15` 上真实 standard 四章均 ready（career 1 次成功，无截断码），整份仍因 120s 租约与最后一步 `report_schema_invalid` 未落 `report_document`；租约改为 600s，心跳保持 ref，章节进度先续租。 - 防复发：输出预算不得再用字符 ÷ 2。`assertWriterOutput` 保持 id/theme 全等与 refs 集合相等。section 错误码不得把 refs mismatch 归进 evidence insufficient。一章失败不得中止其余 write 主题。用户可见失败必须有错误码级摘要，不得只展示 `report_schema_invalid`。日志与 PROGRESS 不得写入 prompt、bundle、模型原文或用户资料。 - 相关记录：BUG-352、BUG-451、BUG-486、BUG-489 - 复发自：BUG-451（分章后仍把写作失败压成整份 schema 码，截断预算与失败分类未按中文口径收口） diff --git a/frontend/src/lib/personal-report-worker-core.ts b/frontend/src/lib/personal-report-worker-core.ts index 3a4ff551..3a513d94 100644 --- a/frontend/src/lib/personal-report-worker-core.ts +++ b/frontend/src/lib/personal-report-worker-core.ts @@ -165,11 +165,6 @@ function reportFailureCode(code: PersonalReportWorkerErrorCode): PersonalReportF } } -function timerUnref(timer: ReturnType): void { - const candidate = timer as ReturnType & { unref?: () => void }; - candidate.unref?.(); -} - function defaultRetryDelayMs(job: PersonalReportJobRecord): number { const exponent = Math.max(0, Math.min(job.attemptCount - 1, 5)); return 5_000 * (2 ** exponent); @@ -269,22 +264,26 @@ export function createPersonalReportWorker(deps: PersonalReportWorkerDeps) { const controller = new AbortController(); let heartbeatError: unknown = null; let heartbeatChain = Promise.resolve(); + const refreshLease = async () => { + if (heartbeatError !== null || controller.signal.aborted) return; + await deps.jobs.heartbeatLease({ + jobId: job.id, + leaseToken: job.leaseToken!, + leaseSeconds, + }); + }; const heartbeatTimer = setIntervalFn(() => { heartbeatChain = heartbeatChain - .then(async () => { - if (heartbeatError !== null || controller.signal.aborted) return; - await deps.jobs.heartbeatLease({ - jobId: job.id, - leaseToken: job.leaseToken!, - leaseSeconds, - }); - }) + .then(refreshLease) .catch((error) => { heartbeatError = error; controller.abort(error); }); }, heartbeatIntervalMs); - timerUnref(heartbeatTimer); + // Keep this timer ref'd. The worker runs from instrumentation, not an HTTP + // request; unref'd intervals in Next.js standalone are skipped while the + // only in-flight work is a long model await, which is exactly when the + // lease must be extended. let report: PersonalReportRecord | null = null; const generationStartedAt = now().getTime(); @@ -339,6 +338,8 @@ export function createPersonalReportWorker(deps: PersonalReportWorkerDeps) { signal: controller.signal, sectionService: deps.sectionService, onProgress: async (progress) => { + await refreshLease(); + if (heartbeatError !== null) throw heartbeatError; const percent = progress.total > 0 ? Math.min(89, 30 + Math.floor((progress.completed / progress.total) * 55)) : 30; diff --git a/frontend/tests/personal-report-generation-v2.test.ts b/frontend/tests/personal-report-generation-v2.test.ts index 8938e2fb..fb2c33cf 100644 --- a/frontend/tests/personal-report-generation-v2.test.ts +++ b/frontend/tests/personal-report-generation-v2.test.ts @@ -422,6 +422,12 @@ test("production writer and worker keep the same signal on initial and repair mo assert.match(workerSource, /generatePersonalReport\(\{[\s\S]*signal: context\.signal,[\s\S]*\}\)/); assert.match(workerSource, /leaseSeconds: 600/); assert.match(workerSource, /heartbeatIntervalMs: 20_000/); + const workerCore = readFileSync( + new URL("../src/lib/personal-report-worker-core.ts", import.meta.url), + "utf8", + ); + assert.match(workerCore, /onProgress: async \(progress\) => \{\s*await refreshLease\(\)/); + assert.doesNotMatch(workerCore, /timerUnref\(heartbeatTimer\)/); }); test("an all-blocked plan still produces a valid v2 document with honest disclosures and no invented thematic section", async () => { diff --git a/frontend/tests/personal-report-worker.test.ts b/frontend/tests/personal-report-worker.test.ts index 8cab12ab..b8a67b6c 100644 --- a/frontend/tests/personal-report-worker.test.ts +++ b/frontend/tests/personal-report-worker.test.ts @@ -87,7 +87,9 @@ function jobRecord(overrides: Partial = {}): PersonalRe function createHarness(input: Readonly<{ job?: PersonalReportJobRecord; report?: PersonalReportRecord | null; - generate?: () => Promise; + generate?: (context: { + onProgress?: (progress: Readonly<{ phase: string; completed: number; total: number }>) => Promise | void; + }) => Promise; loseLeaseBeforeCompletion?: boolean; markReadyError?: PersonalReportJobServiceError; billing?: ReportBillingPort; @@ -326,10 +328,10 @@ function createHarness(input: Readonly<{ assert.equal(userId, USER_ID); return { birth_time_status: "confirmed" }; }, - generate: async () => { + generate: async (context) => { events.push("generate"); generateCount += 1; - return input.generate?.() ?? { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) }; + return input.generate?.(context) ?? { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) }; }, billing: input.billing ? { @@ -385,6 +387,22 @@ test("tick recovers first, heartbeats, persists the report, then closes the leas ); }); +test("section progress heartbeats the lease before recording percent", async () => { + const harness = createHarness({ + generate: async (context) => { + await context.onProgress?.({ phase: "section:theme-career", completed: 1, total: 4 }); + return { status: "ready", document: READY_DOCUMENT, evidenceHash: "c".repeat(64) }; + }, + }); + const result = await harness.worker.tick(); + assert.equal(result.outcome, "ready"); + const generateAt = harness.events.indexOf("generate"); + const progressAt = harness.events.indexOf("progress:section:theme-career:43"); + assert.ok(generateAt >= 0); + assert.ok(progressAt > generateAt); + assert.ok(harness.events.slice(generateAt + 1, progressAt).includes("heartbeat")); +}); + test("a restarted process reclaims an expired lease and does not create a second report or job", async () => { const harness = createHarness({ job: jobRecord({