External/SnapOtter
1
0
Fork 0
mirror of https://github.com/snapotter-hq/SnapOtter.git synced 2026-08-03 07:46:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: honor unlimited processing timeout (#638)

Browse Source 
* fix(web): recover stalled job progress streams

* fix(ai): honor unlimited processing timeout

* fix(web): keep retrying stalled progress streams
This commit is contained in:
SnapOtter
2026-07-25 11:36:02 +08:00
committed by GitHub
parent 841f47f6ca
commit 025851beef
5 changed files with 399 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/bridge.ts
+44 -30
View File
@@ -9,12 +9,25 @@ import { acquireVenvRead, tryAcquireVenvRead } from "./venv-lock.js";
const __dirname = dirname(fileURLToPath(import.meta.url));
const PYTHON_DIR = resolve(__dirname, "../python");
const DEFAULT_PYTHON_TIMEOUT_MS = 600_000;
function appendEnvPath(base: string, suffix: string): string {
const normalizedBase = base.replace(/\/+$/, "");
return `${normalizedBase || "/"}${normalizedBase === "" ? "" : "/"}${suffix}`;
}
function resolvePythonTimeout(explicitTimeout?: number): number | undefined {
const configured = process.env.PROCESSING_TIMEOUT_S?.trim();
if (configured !== undefined && configured !== "") {
const seconds = Number(configured);
if (Number.isFinite(seconds)) {
if (seconds === 0) return undefined;
if (seconds > 0) return seconds * 1000;
}
}
return explicitTimeout ?? DEFAULT_PYTHON_TIMEOUT_MS;
}
/**
* Build a minimal environment for spawned Python processes.
* Only passes through variables needed for venv, CUDA, model cache,
@@ -476,32 +489,34 @@ export class PythonDispatcher {
if (!proc || !proc.stdin || !this.childReady) return null;
const id = randomUUID();
const timeout =
options.timeout ??
(process.env.PROCESSING_TIMEOUT_S && parseInt(process.env.PROCESSING_TIMEOUT_S, 10) > 0
? parseInt(process.env.PROCESSING_TIMEOUT_S, 10) * 1000
: 600000);
const timeout = resolvePythonTimeout(options.timeout);
return new Promise((resolvePromise, rejectPromise) => {
const timer = setTimeout(() => {
this.pending.delete(id);
// Kill the stuck dispatcher so it restarts on the next request instead of
// blocking all subsequent operations behind the timed-out script.
if (this.child && !this.child.killed) {
this.child.kill("SIGTERM");
}
rejectPromise(
new SafeError("Python script timed out", { kind: "operational", code: "timeout" }),
);
}, timeout);
const timer =
timeout === undefined
? undefined
: setTimeout(() => {
this.pending.delete(id);
// Kill the stuck dispatcher so it restarts on the next request instead of
// blocking all subsequent operations behind the timed-out script.
if (this.child && !this.child.killed) {
this.child.kill("SIGTERM");
}
rejectPromise(
new SafeError("Python script timed out", {
kind: "operational",
code: "timeout",
}),
);
}, timeout);
const wrappedResolve = (result: { stdout: string; stderr: string }) => {
clearTimeout(timer);
if (timer) clearTimeout(timer);
resolvePromise(result);
};
const wrappedReject = (err: Error) => {
clearTimeout(timer);
if (timer) clearTimeout(timer);
rejectPromise(err);
};
@@ -529,7 +544,7 @@ export class PythonDispatcher {
proc.stdin!.write(request + "\n");
} catch {
this.pending.delete(id);
clearTimeout(timer);
if (timer) clearTimeout(timer);
rejectPromise(
new SafeError("Python dispatcher stdin closed unexpectedly", {
kind: "operational",
@@ -564,11 +579,7 @@ export class PythonDispatcher {
);
}
const scriptPath = resolve(PYTHON_DIR, scriptName);
const timeout =
options.timeout ??
(process.env.PROCESSING_TIMEOUT_S && parseInt(process.env.PROCESSING_TIMEOUT_S, 10) > 0
? parseInt(process.env.PROCESSING_TIMEOUT_S, 10) * 1000
: 600000);
const timeout = resolvePythonTimeout(options.timeout);
return new Promise((resolvePromise, rejectPromise) => {
const trySpawn = (pythonBin: string, isFallback: boolean) => {
@@ -582,10 +593,13 @@ export class PythonDispatcher {
let stderrBuffer = "";
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
proc.kill("SIGTERM");
}, timeout);
const timer =
timeout === undefined
? undefined
: setTimeout(() => {
timedOut = true;
proc.kill("SIGTERM");
}, timeout);
proc.stdout.on("data", (chunk: Buffer) => {
stdout += chunk.toString();
@@ -614,7 +628,7 @@ export class PythonDispatcher {
});
proc.on("error", (err: NodeJS.ErrnoException) => {
clearTimeout(timer);
if (timer) clearTimeout(timer);
if (err.code === "ENOENT" && !isFallback) {
trySpawn("python3", true);
} else {
@@ -628,7 +642,7 @@ export class PythonDispatcher {
});
proc.on("close", (code, signal) => {
clearTimeout(timer);
if (timer) clearTimeout(timer);
if (stderrBuffer.trim()) {
stderrLines.push(stderrBuffer.trim());