mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix(ai-bundles): lock the numpy-1.x ABI closure so the OCR bundle can't strand scipy (#437)
* fix(ai-bundles): lock the numpy-1.x ABI closure so the OCR bundle can't strand scipy The OCR bundle installs paddleocr[doc-parser] 3.4, whose dependency closure drags numpy 1.26.4 up to 2.5.1 and pulls scipy/scikit-learn/pandas wheels built against the numpy 2.x ABI. build-bundle.sh re-pinned only numpy (basePackages), so those numpy-2.x wheels stayed behind; the by-dir-name site-packages diff then shipped them, and once merged onto the numpy==1.26.4 base they raise "numpy.dtype size changed" on import. Because the dispatcher pre-imports every ML library at startup and disables all AI after 5 crashes in 60s, one stranded scipy takes down every AI tool, not just OCR (observed on a CPU host: remove-background worked before the OCR bundle and broke after). All-7 installs escaped it through last-writer-wins ordering; a subset install did not, which is why it surfaced only intermittently. Fix: add a manifest "constraints" list (numpy, scipy, scikit-learn, scikit-image, pandas pinned to numpy-1.x-ABI versions) and apply it via PIP_CONSTRAINT to every bundle pip install, so no bundle can pull a numpy-2.x wheel. paddleocr 3.4.1 still resolves cleanly under the lock and the pinned stack imports without ABI error on numpy 1.26.4 (validated on py3.12). Also import scipy/sklearn in the OCR path of verify-bundle.sh so CI catches this class in isolation, and add a manifest regression test. Note: the published bundles must be rebuilt and republished (ai-bundles.yml) for this to reach already-installed bases. Claude-Session: https://claude.ai/code/session_01UvVCMNUBrgpghk8gye5gav * chore(ai-bundles): sync OCR manifest sha256 to the rebuilt numpy-1.x bundles Rebuilt the OCR bundle for both arches with the numpy-1.x-ABI constraints from this PR and republished the tars to deepsafe/feature-bundles/v2.0.0, then updated the baked manifest sha256 and sizes so installs verify against the fixed archives: amd64-gpu 5.93 GB sha 2a00a3184f6a635f1fa9ae2a6517ad740a11f9e5ff58c098d2fd369a2bb1e16b arm64-cpu 1.98 GB sha 6868c264069dcb74c6675c0b1f58dc1c9f60d9aa4459725e3dbde07a99a6a09a Both tars ship scipy 1.12.0 / scikit-learn 1.4.2 / pandas 2.2.2 (numpy-1.x-ABI) and zero numpy-2.x wheels, verified by listing the archive contents. Stopgap note: these tars were built against the ghcr.io latest base (the 2.0.0 image is not published to GHCR), so they are not byte-identical to what the CI build will produce. When ai-bundles.yml rebuilds at the 2.0.0 release, it will mint fresh sha256 values and this manifest must be re-synced to them. Claude-Session: https://claude.ai/code/session_01UvVCMNUBrgpghk8gye5gav
This commit is contained in:
@@ -70,6 +70,42 @@ describe("Feature manifest structure", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Feature manifest: numpy-2.x-ABI closure lock (regression for OCR bundle strand)", () => {
|
||||
// paddleocr[doc-parser] (OCR bundle) drags numpy 1.26.4 -> 2.5.1 and pulls
|
||||
// scipy/scikit-learn/pandas wheels built against the numpy 2.x ABI. Re-pinning
|
||||
// numpy alone leaves those stranded on the numpy==1.26.4 base, which the bundle
|
||||
// diff ships, breaking rembg's scipy import (a dispatcher-wide AI outage) after
|
||||
// a last-writer-wins merge. build-bundle.sh applies `constraints` to every pip
|
||||
// install to keep the whole closure on numpy-1.x-ABI versions.
|
||||
const constraints = (manifest.constraints ?? []) as string[];
|
||||
|
||||
it("manifest declares a non-empty constraints array", () => {
|
||||
expect(manifest.constraints).toBeInstanceOf(Array);
|
||||
expect(constraints.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("constraints pin every numpy-2.x-closure package to an exact version", () => {
|
||||
const pinned = new Map<string, string>();
|
||||
for (const c of constraints) {
|
||||
const m = c.match(/^([A-Za-z0-9_.-]+)==(.+)$/);
|
||||
expect(m, `constraint "${c}" must be an exact ==pin`).not.toBeNull();
|
||||
if (m) pinned.set(m[1].toLowerCase(), m[2]);
|
||||
}
|
||||
for (const pkg of ["numpy", "scipy", "scikit-learn", "scikit-image", "pandas"]) {
|
||||
expect(pinned.has(pkg), `constraints must pin ${pkg} to a numpy-1.x-ABI version`).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("constraints numpy matches basePackages numpy (single source of truth)", () => {
|
||||
const cNumpy = constraints.find((c) => c.toLowerCase().startsWith("numpy=="));
|
||||
const bNumpy = (manifest.basePackages as string[]).find((c) =>
|
||||
c.toLowerCase().startsWith("numpy=="),
|
||||
);
|
||||
expect(cNumpy).toBeDefined();
|
||||
expect(cNumpy).toBe(bNumpy);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Feature manifest: mediapipe dependency (regression for #129)", () => {
|
||||
it("upscale-enhance bundle includes mediapipe for amd64", () => {
|
||||
expect(archPackagesInclude("upscale-enhance", "amd64", "mediapipe")).toBe(true);
|
||||
|
||||
Reference in New Issue
Block a user