mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: improve AI feature install error handling and resource limits
Handle OOM kills (exit code 137) with actionable memory guidance, filter ANSI/progress noise from error output, add --no-cache-dir to pip installs, reduce download concurrency to 2, and bump default container memory from 4g to 6g.
This commit is contained in:
@@ -225,13 +225,24 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
|
||||
}
|
||||
}
|
||||
if (!errorMsg) {
|
||||
const meaningful = lastStderrLines.filter(
|
||||
(l) => !l.startsWith("{") && !l.includes("pthread_setaffinity_np"),
|
||||
);
|
||||
errorMsg =
|
||||
meaningful.join("\n") ||
|
||||
stdoutBuffer.trim() ||
|
||||
`Install failed with exit code ${code}`;
|
||||
if (code === 137) {
|
||||
errorMsg =
|
||||
"Installation was killed due to insufficient memory. " +
|
||||
"Try increasing the container's memory limit (e.g. mem_limit: 6g in docker-compose.yml) and retry.";
|
||||
} else {
|
||||
const meaningful = lastStderrLines.filter(
|
||||
(l) =>
|
||||
!l.startsWith("{") &&
|
||||
!l.includes("pthread_setaffinity_np") &&
|
||||
!l.includes("\x1b[") &&
|
||||
!l.includes("━") &&
|
||||
!/^\s*\d+%\|/.test(l),
|
||||
);
|
||||
errorMsg =
|
||||
meaningful.join("\n") ||
|
||||
stdoutBuffer.trim() ||
|
||||
`Install failed with exit code ${code}`;
|
||||
}
|
||||
}
|
||||
setInstallProgress(bundleId, null, errorMsg);
|
||||
updateSingleFileProgress({ jobId, phase: "failed", percent: 0, error: errorMsg });
|
||||
|
||||
@@ -52,8 +52,8 @@ services:
|
||||
# - COOKIE_SECRET=
|
||||
restart: unless-stopped
|
||||
# --- Security hardening ---
|
||||
mem_limit: 4g
|
||||
memswap_limit: 4g
|
||||
mem_limit: 6g
|
||||
memswap_limit: 6g
|
||||
cpus: 4
|
||||
pids_limit: 512
|
||||
cap_drop:
|
||||
|
||||
@@ -125,7 +125,7 @@ def check_disk_space(path: str, min_bytes: int = 100 * 1024 * 1024) -> None:
|
||||
|
||||
def pip_install(package: str, extra_flags: list[str] | None = None) -> None:
|
||||
"""Run pip install for a single package spec. Raises on failure."""
|
||||
cmd = [sys.executable, "-m", "pip", "install"]
|
||||
cmd = [sys.executable, "-m", "pip", "install", "--no-cache-dir"]
|
||||
if extra_flags:
|
||||
cmd.extend(extra_flags)
|
||||
|
||||
@@ -471,7 +471,7 @@ def download_models(models: list[dict], models_dir: str) -> list[str]:
|
||||
except Exception as e:
|
||||
return (model_id, e)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = {
|
||||
pool.submit(_download, i, m): i
|
||||
for i, m in enumerate(models)
|
||||
|
||||
Reference in New Issue
Block a user