mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: format preservation, dispatcher stability, and health reporting
Closes #17, #18, #19, #31, #32, #33, #34 Format preservation (#17, #18, #19): - Add resolveOutputFormat to rotate, resize, text-overlay, watermark-text, border, replace-color, blur-faces, upscale, erase-object, restore-photo - Alpha-aware fallback: border with corner radius/shadow and replace-color with makeTransparent fall back to PNG for non-alpha formats (JPEG) - Python sidecar tools (blur-faces, upscale, erase-object) now convert PNG output back to input format, matching restore-photo/colorize pattern - Upscale and erase-object default to "auto" format detection instead of PNG Dispatcher stability (#31, #32): - Add gc.collect() and torch.cuda.empty_cache() after each dispatcher request - Add configurable max_requests (default 50) for periodic dispatcher restart - Add exponential backoff to dispatcher crash recovery in bridge.ts - Circuit breaker: 5 crashes within 60s permanently disables dispatcher - Reset crash counter on successful dispatcher startup Health & security (#33, #34): - Export getDispatcherStatus() from @snapotter/ai with running/ready/failed/ gpu/pid/consecutiveCrashes fields - Admin health endpoint now includes full dispatcher status - Add pip-audit job to CI workflow for Python dependency scanning
This commit is contained in:
@@ -74,6 +74,41 @@ let dispatcherGpuAvailable = false;
|
||||
const pendingRequests = new Map<string, PendingRequest>();
|
||||
let stdoutBuffer = "";
|
||||
|
||||
// Crash recovery with exponential backoff
|
||||
// biome-ignore lint/style/useConst: reassigned on crash events
|
||||
let consecutiveCrashes = 0;
|
||||
// biome-ignore lint/style/useConst: reassigned on crash events
|
||||
let lastCrashTime = 0;
|
||||
// biome-ignore lint/style/useConst: reassigned on crash events
|
||||
let backoffUntil = 0;
|
||||
const CRASH_WINDOW_MS = 60_000;
|
||||
const MAX_CONSECUTIVE_CRASHES = 5;
|
||||
const BASE_BACKOFF_MS = 1_000;
|
||||
|
||||
function recordCrash(): void {
|
||||
const now = Date.now();
|
||||
if (now - lastCrashTime > CRASH_WINDOW_MS) {
|
||||
consecutiveCrashes = 1;
|
||||
} else {
|
||||
consecutiveCrashes++;
|
||||
}
|
||||
lastCrashTime = now;
|
||||
|
||||
if (consecutiveCrashes >= MAX_CONSECUTIVE_CRASHES) {
|
||||
console.error(
|
||||
`[bridge] Dispatcher crashed ${consecutiveCrashes} times in ${CRASH_WINDOW_MS / 1000}s, disabling permanently`,
|
||||
);
|
||||
dispatcherFailed = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const delay = BASE_BACKOFF_MS * 2 ** (consecutiveCrashes - 1);
|
||||
backoffUntil = now + delay;
|
||||
console.warn(
|
||||
`[bridge] Dispatcher crash #${consecutiveCrashes}, backing off ${delay}ms before restart`,
|
||||
);
|
||||
}
|
||||
|
||||
function startDispatcher(): ChildProcess | null {
|
||||
if (dispatcherFailed) return null;
|
||||
|
||||
@@ -100,6 +135,7 @@ function startDispatcher(): ChildProcess | null {
|
||||
if (parsed.ready === true) {
|
||||
dispatcherReady = true;
|
||||
dispatcherGpuAvailable = parsed.gpu === true;
|
||||
consecutiveCrashes = 0;
|
||||
console.log(`[bridge] Python dispatcher ready (GPU: ${parsed.gpu === true})`);
|
||||
continue;
|
||||
}
|
||||
@@ -168,10 +204,10 @@ function startDispatcher(): ChildProcess | null {
|
||||
child.on("error", (err: NodeJS.ErrnoException) => {
|
||||
console.error(`[bridge] Dispatcher error: ${err.message} (code: ${err.code})`);
|
||||
if (err.code === "ENOENT") {
|
||||
// Venv python not found - mark as failed, will fall back to per-request
|
||||
dispatcherFailed = true;
|
||||
} else {
|
||||
recordCrash();
|
||||
}
|
||||
// Reject all pending requests
|
||||
for (const [id, req] of pendingRequests.entries()) {
|
||||
req.reject(new Error(extractPythonError(err)));
|
||||
pendingRequests.delete(id);
|
||||
@@ -181,11 +217,11 @@ function startDispatcher(): ChildProcess | null {
|
||||
});
|
||||
|
||||
child.on("close", () => {
|
||||
// Reject all pending requests
|
||||
for (const [id, req] of pendingRequests.entries()) {
|
||||
req.reject(new Error("Python dispatcher exited unexpectedly"));
|
||||
pendingRequests.delete(id);
|
||||
}
|
||||
recordCrash();
|
||||
dispatcher = null;
|
||||
dispatcherReady = false;
|
||||
});
|
||||
@@ -200,6 +236,7 @@ function startDispatcher(): ChildProcess | null {
|
||||
function getDispatcher(): ChildProcess | null {
|
||||
if (dispatcherFailed) return null;
|
||||
if (!dispatcher || dispatcher.killed) {
|
||||
if (Date.now() < backoffUntil) return null;
|
||||
dispatcher = startDispatcher();
|
||||
}
|
||||
return dispatcher;
|
||||
@@ -259,6 +296,26 @@ export function isGpuAvailable(): boolean {
|
||||
return dispatcherGpuAvailable;
|
||||
}
|
||||
|
||||
export interface DispatcherStatus {
|
||||
running: boolean;
|
||||
ready: boolean;
|
||||
failed: boolean;
|
||||
gpu: boolean;
|
||||
pid: number | null;
|
||||
consecutiveCrashes: number;
|
||||
}
|
||||
|
||||
export function getDispatcherStatus(): DispatcherStatus {
|
||||
return {
|
||||
running: dispatcher !== null && !dispatcher.killed,
|
||||
ready: dispatcherReady,
|
||||
failed: dispatcherFailed,
|
||||
gpu: dispatcherGpuAvailable,
|
||||
pid: dispatcher?.pid ?? null,
|
||||
consecutiveCrashes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Shut down the persistent dispatcher process.
|
||||
*/
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export { removeBackground } from "./background-removal.js";
|
||||
export { isGpuAvailable, shutdownDispatcher } from "./bridge.js";
|
||||
export type { DispatcherStatus } from "./bridge.js";
|
||||
export { getDispatcherStatus, isGpuAvailable, shutdownDispatcher } from "./bridge.js";
|
||||
export { colorize } from "./colorization.js";
|
||||
export type { DetectFacesResult, FaceRegion } from "./face-detection.js";
|
||||
export { blurFaces, detectFaces } from "./face-detection.js";
|
||||
|
||||
Reference in New Issue
Block a user