From 37c915c3da6b2183b0befe1521a643bb36edcce6 Mon Sep 17 00:00:00 2001 From: SnapOtter Date: Tue, 21 Jul 2026 15:22:57 +0800 Subject: [PATCH] fix(jobs): make timeout messages tool-agnostic and CPU-aware (#596) Replace the job timeout message that hardcoded "background-removal" for every tool with a tool-agnostic one that sets the CPU-vs-GPU expectation, the usual reason heavy AI times out on modest hardware. The client-side SSE stall message gets the same treatment. Both stay under friendlyError's 280-char limit so the guidance reaches the user instead of collapsing to a generic "Processing failed". Refs #591 --- apps/api/src/jobs/worker.ts | 3 ++- apps/api/src/lib/timeout.ts | 19 +++++++++++++ apps/web/src/hooks/use-tool-processor.ts | 6 ++--- tests/unit/api/timeout.test.ts | 34 +++++++++++++++++++++++- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/apps/api/src/jobs/worker.ts b/apps/api/src/jobs/worker.ts index 9c5db595..ea6034a4 100644 --- a/apps/api/src/jobs/worker.ts +++ b/apps/api/src/jobs/worker.ts @@ -52,6 +52,7 @@ import { } from "../lib/object-storage.js"; import { OCR_MAX_ENCODED_INPUT_BYTES } from "../lib/ocr-limits.js"; import { SCRUB_PDF_PRODUCER_TOOLS, scrubPdfProducer } from "../lib/pdf-producer.js"; +import { timeoutMessage } from "../lib/timeout.js"; import { InputValidationError } from "../modality/contract.js"; import { publishEphemeral, @@ -508,7 +509,7 @@ async function processToolJob(job: Job): Promise { const finalError = isCanceled ? "Canceled" : isTimeout - ? `Timed out after ${Math.round(timeoutMs / 1000)}s. On the first run the background-removal model may still be downloading; the image may be too large for CPU inference; or the worker may be busy or unavailable. Retry once the model has downloaded, or try a smaller image.` + ? timeoutMessage(timeoutMs) : errorMessage; // Log genuine processing faults at error level (clients only ever see diff --git a/apps/api/src/lib/timeout.ts b/apps/api/src/lib/timeout.ts index 5806710b..ec79c208 100644 --- a/apps/api/src/lib/timeout.ts +++ b/apps/api/src/lib/timeout.ts @@ -24,3 +24,22 @@ export function computeExternalToolTimeout(megapixels: number): number { } return Math.max(60_000, megapixels * TIMEOUT_RATES.external * 1000); } + +/** + * User-facing message for a job that exceeded its worker timeout. Tool-agnostic + * on purpose: the previous copy hardcoded "background-removal", so an AI upscale + * that timed out on a modest CPU told the user a background-removal model was + * still downloading. Set the CPU-vs-GPU expectation instead, which is the usual + * reason heavy AI times out on self-hosted hardware (#591). + */ +export function timeoutMessage(timeoutMs: number): string { + const seconds = Math.round(timeoutMs / 1000); + // Must stay under friendlyError's 280-char / 3-line limit, or the whole + // message collapses to the generic "Processing failed" fallback + // (worker-timeout.test.ts guards this). + return ( + `Timed out after ${seconds}s. Heavy tools run much slower on CPU than a GPU, ` + + `so a large input can exceed the limit; on the first run the model may still ` + + `be downloading. Try a smaller input or retry.` + ); +} diff --git a/apps/web/src/hooks/use-tool-processor.ts b/apps/web/src/hooks/use-tool-processor.ts index 902c8488..0c4fca15 100644 --- a/apps/web/src/hooks/use-tool-processor.ts +++ b/apps/web/src/hooks/use-tool-processor.ts @@ -147,7 +147,7 @@ export function useToolProcessor(toolId: string) { if (elapsedRef.current) clearInterval(elapsedRef.current); clearActiveJob(); setError( - "Processing timed out with no progress for 5 minutes. Try again or use a smaller file.", + "Processing timed out after 5 minutes without an update from the server. Heavy tools run much slower on CPU than a GPU; try a smaller file, or retry if the connection dropped.", ); setProcessing(false); setProgress(IDLE_PROGRESS); @@ -167,7 +167,7 @@ export function useToolProcessor(toolId: string) { if (elapsedRef.current) clearInterval(elapsedRef.current); clearActiveJob(); setError( - "Processing timed out with no progress for 5 minutes. Try again or use a smaller file.", + "Processing timed out after 5 minutes without an update from the server. Heavy tools run much slower on CPU than a GPU; try a smaller file, or retry if the connection dropped.", ); setProcessing(false); setProgress(IDLE_PROGRESS); @@ -329,7 +329,7 @@ export function useToolProcessor(toolId: string) { error: "Processing timed out", }); setError( - "Processing timed out with no progress for 5 minutes. Try again or use a smaller file.", + "Processing timed out after 5 minutes without an update from the server. Heavy tools run much slower on CPU than a GPU; try a smaller file, or retry if the connection dropped.", ); setProcessing(false); setProgress(IDLE_PROGRESS); diff --git a/tests/unit/api/timeout.test.ts b/tests/unit/api/timeout.test.ts index c27e4a05..dc58a366 100644 --- a/tests/unit/api/timeout.test.ts +++ b/tests/unit/api/timeout.test.ts @@ -4,7 +4,11 @@ vi.mock("../../../apps/api/src/config.js", () => ({ env: { PROCESSING_TIMEOUT_S: 0 }, })); -import { computeExternalToolTimeout, computeTimeout } from "../../../apps/api/src/lib/timeout.js"; +import { + computeExternalToolTimeout, + computeTimeout, + timeoutMessage, +} from "../../../apps/api/src/lib/timeout.js"; describe("computeTimeout", () => { it("returns correct timeout for sharp category", () => { @@ -122,3 +126,31 @@ describe("computeExternalToolTimeout", () => { expect(result).toBe(60_000); }); }); + +describe("timeoutMessage", () => { + it("reports the elapsed timeout in seconds", () => { + expect(timeoutMessage(120_000)).toContain("120s"); + expect(timeoutMessage(5_000)).toContain("5s"); + }); + + it("is tool-agnostic (no hardcoded background-removal)", () => { + // The old copy told every timed-out job a background-removal model was + // downloading, which was wrong for AI upscale, video, etc. (#591). + expect(timeoutMessage(120_000)).not.toMatch(/background-removal/i); + }); + + it("sets the CPU-vs-GPU expectation for heavy tools", () => { + const msg = timeoutMessage(120_000); + expect(msg).toMatch(/CPU/); + expect(msg).toMatch(/GPU/); + }); + + it("survives friendlyError (single line, at most 280 chars)", () => { + // friendlyError collapses any message over 280 chars or more than 3 lines + // to a generic "Processing failed" sentence, which would hide this guidance. + // Use the longest realistic timeout (JOB_TIMEOUT_LONG_S default of 2h). + const msg = timeoutMessage(7_200_000); + expect(msg.length).toBeLessThanOrEqual(280); + expect(msg.split("\n")).toHaveLength(1); + }); +});