mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: improve AI feature install error handling and resource limits
Handle OOM kills (exit code 137) with actionable memory guidance, filter ANSI/progress noise from error output, add --no-cache-dir to pip installs, reduce download concurrency to 2, and bump default container memory from 4g to 6g.
This commit is contained in:
@@ -225,13 +225,24 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!errorMsg) {
|
if (!errorMsg) {
|
||||||
const meaningful = lastStderrLines.filter(
|
if (code === 137) {
|
||||||
(l) => !l.startsWith("{") && !l.includes("pthread_setaffinity_np"),
|
errorMsg =
|
||||||
);
|
"Installation was killed due to insufficient memory. " +
|
||||||
errorMsg =
|
"Try increasing the container's memory limit (e.g. mem_limit: 6g in docker-compose.yml) and retry.";
|
||||||
meaningful.join("\n") ||
|
} else {
|
||||||
stdoutBuffer.trim() ||
|
const meaningful = lastStderrLines.filter(
|
||||||
`Install failed with exit code ${code}`;
|
(l) =>
|
||||||
|
!l.startsWith("{") &&
|
||||||
|
!l.includes("pthread_setaffinity_np") &&
|
||||||
|
!l.includes("\x1b[") &&
|
||||||
|
!l.includes("━") &&
|
||||||
|
!/^\s*\d+%\|/.test(l),
|
||||||
|
);
|
||||||
|
errorMsg =
|
||||||
|
meaningful.join("\n") ||
|
||||||
|
stdoutBuffer.trim() ||
|
||||||
|
`Install failed with exit code ${code}`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
setInstallProgress(bundleId, null, errorMsg);
|
setInstallProgress(bundleId, null, errorMsg);
|
||||||
updateSingleFileProgress({ jobId, phase: "failed", percent: 0, error: errorMsg });
|
updateSingleFileProgress({ jobId, phase: "failed", percent: 0, error: errorMsg });
|
||||||
|
|||||||
@@ -52,8 +52,8 @@ services:
|
|||||||
# - COOKIE_SECRET=
|
# - COOKIE_SECRET=
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
# --- Security hardening ---
|
# --- Security hardening ---
|
||||||
mem_limit: 4g
|
mem_limit: 6g
|
||||||
memswap_limit: 4g
|
memswap_limit: 6g
|
||||||
cpus: 4
|
cpus: 4
|
||||||
pids_limit: 512
|
pids_limit: 512
|
||||||
cap_drop:
|
cap_drop:
|
||||||
|
|||||||
@@ -125,7 +125,7 @@ def check_disk_space(path: str, min_bytes: int = 100 * 1024 * 1024) -> None:
|
|||||||
|
|
||||||
def pip_install(package: str, extra_flags: list[str] | None = None) -> None:
|
def pip_install(package: str, extra_flags: list[str] | None = None) -> None:
|
||||||
"""Run pip install for a single package spec. Raises on failure."""
|
"""Run pip install for a single package spec. Raises on failure."""
|
||||||
cmd = [sys.executable, "-m", "pip", "install"]
|
cmd = [sys.executable, "-m", "pip", "install", "--no-cache-dir"]
|
||||||
if extra_flags:
|
if extra_flags:
|
||||||
cmd.extend(extra_flags)
|
cmd.extend(extra_flags)
|
||||||
|
|
||||||
@@ -471,7 +471,7 @@ def download_models(models: list[dict], models_dir: str) -> list[str]:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
return (model_id, e)
|
return (model_id, e)
|
||||||
|
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||||
futures = {
|
futures = {
|
||||||
pool.submit(_download, i, m): i
|
pool.submit(_download, i, m): i
|
||||||
for i, m in enumerate(models)
|
for i, m in enumerate(models)
|
||||||
|
|||||||
Reference in New Issue
Block a user