External/SnapOtter
1
0
Fork 0
mirror of https://github.com/snapotter-hq/SnapOtter.git synced 2026-08-03 07:46:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(ai-bundles): repair bundle build + publish pipeline (deepsafe repo, CPU provider, manifest)

Browse Source 
Bundle build/publish fixes: CPUExecutionProvider in rembg build, pip/import/arm64 deps, hf-CLI publish to deepsafe/feature-bundles, real manifest sha256+sizes, installer fallback repo.
This commit is contained in:
SnapOtter
2026-06-19 18:34:33 +08:00
committed by GitHub
parent 7f8f6b5e6b
commit 3b50bcdc5c
4 changed files with 133 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-bundles.yml
+8 -5
View File
@@ -104,6 +104,7 @@ jobs:
mkdir -p /tmp/bundles mkdir -p /tmp/bundles
docker run --rm --entrypoint bash \ docker run --rm --entrypoint bash \
-v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \ -v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
-v "$PWD/docker/feature-manifest.json:/app/docker/feature-manifest.json:ro" \
-v "/tmp/bundles:/output" \ -v "/tmp/bundles:/output" \
"ghcr.io/snapotter-hq/snapotter:${VERSION}" \ "ghcr.io/snapotter-hq/snapotter:${VERSION}" \
/build-bundle.sh "${BUNDLE}" "${ARCH}" /output /build-bundle.sh "${BUNDLE}" "${ARCH}" /output
@@ -127,8 +128,8 @@ jobs:
with: with:
path: /tmp/artifacts path: /tmp/artifacts
- name: Install huggingface-hub - name: Install hf CLI
run: pip install huggingface-hub run: pip install -U "huggingface_hub[hf_xet]"
- name: Organize and upload to HuggingFace - name: Organize and upload to HuggingFace
env: env:
@@ -156,8 +157,10 @@ jobs:
done done
echo ']}' >> manifest.json echo ']}' >> manifest.json
huggingface-cli upload snapotter/feature-bundles \ # Interim: publish to the deepsafe account (the HF_TOKEN secret is deepsafe's).
# Switch back to snapotter/feature-bundles once that account is restored.
hf upload deepsafe/feature-bundles \
"/tmp/upload/v${VERSION}" "v${VERSION}" \ "/tmp/upload/v${VERSION}" "v${VERSION}" \
--repo-type model --token "${HF_TOKEN}" --repo-type model
echo "Published: https://huggingface.co/snapotter/feature-bundles/tree/main/v${VERSION}" echo "Published: https://huggingface.co/deepsafe/feature-bundles/tree/main/v${VERSION}"
docker/build-bundle.sh
+43 -8
View File
@@ -77,7 +77,7 @@ echo " Base entries: $(wc -l < /tmp/base-packages.txt)"
# ── Step 2: Install packages ───────────────────────────────────────────── # ── Step 2: Install packages ─────────────────────────────────────────────
echo "=== Installing packages ===" echo "=== Installing packages ==="
python3 << 'PYINSTALL' python3 << 'PYINSTALL'
import json, subprocess, sys, os import json, shlex, subprocess, sys, os
with open("/app/docker/feature-manifest.json") as f: with open("/app/docker/feature-manifest.json") as f:
manifest = json.load(f) manifest = json.load(f)
@@ -104,7 +104,7 @@ for pkg_string in packages:
# pkg_string may contain embedded flags (e.g. --index-url), so pass as-is # pkg_string may contain embedded flags (e.g. --index-url), so pass as-is
cmd = f"{sys.executable} -m pip install --no-cache-dir {extra_flags} {pkg_string}".strip() cmd = f"{sys.executable} -m pip install --no-cache-dir {extra_flags} {pkg_string}".strip()
print(f" > {cmd}", flush=True) print(f" > {cmd}", flush=True)
result = subprocess.run(cmd, shell=True) result = subprocess.run(shlex.split(cmd))
if result.returncode != 0: if result.returncode != 0:
print(f"ERROR: pip install failed for: {pkg_string}", file=sys.stderr) print(f"ERROR: pip install failed for: {pkg_string}", file=sys.stderr)
sys.exit(1) sys.exit(1)
@@ -115,7 +115,7 @@ PYINSTALL
# ── Step 3: Post-install fixups ─────────────────────────────────────────── # ── Step 3: Post-install fixups ───────────────────────────────────────────
echo "=== Running post-install fixups ===" echo "=== Running post-install fixups ==="
python3 << 'PYPOST' python3 << 'PYPOST'
import json, subprocess, sys, os import json, shlex, subprocess, sys, os
with open("/app/docker/feature-manifest.json") as f: with open("/app/docker/feature-manifest.json") as f:
manifest = json.load(f) manifest = json.load(f)
@@ -130,7 +130,7 @@ if not post_install:
for pkg in post_install: for pkg in post_install:
cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkg}" cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkg}"
print(f" > {cmd}", flush=True) print(f" > {cmd}", flush=True)
result = subprocess.run(cmd, shell=True) result = subprocess.run(shlex.split(cmd))
if result.returncode != 0: if result.returncode != 0:
print(f"ERROR: post-install fixup failed for: {pkg}", file=sys.stderr) print(f"ERROR: post-install fixup failed for: {pkg}", file=sys.stderr)
sys.exit(1) sys.exit(1)
@@ -141,7 +141,7 @@ PYPOST
# ── Step 4: Re-pin base packages ───────────────────────────────────────── # ── Step 4: Re-pin base packages ─────────────────────────────────────────
echo "=== Re-pinning base packages ===" echo "=== Re-pinning base packages ==="
python3 << 'PYREPIN' python3 << 'PYREPIN'
import json, subprocess, sys import json, shlex, subprocess, sys
with open("/app/docker/feature-manifest.json") as f: with open("/app/docker/feature-manifest.json") as f:
manifest = json.load(f) manifest = json.load(f)
@@ -154,7 +154,7 @@ if not base_packages:
pkgs = " ".join(base_packages) pkgs = " ".join(base_packages)
cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkgs}" cmd = f"{sys.executable} -m pip install --no-cache-dir --force-reinstall {pkgs}"
print(f" > {cmd}", flush=True) print(f" > {cmd}", flush=True)
result = subprocess.run(cmd, shell=True) result = subprocess.run(shlex.split(cmd))
if result.returncode != 0: if result.returncode != 0:
print("ERROR: base package re-pin failed", file=sys.stderr) print("ERROR: base package re-pin failed", file=sys.stderr)
sys.exit(1) sys.exit(1)
@@ -162,6 +162,39 @@ if result.returncode != 0:
print(" Base packages re-pinned") print(" Base packages re-pinned")
PYREPIN PYREPIN
# ── Step 4.5: Apply source patches ───────────────────────────────────────
echo "=== Applying source patches ==="
python3 << 'PYPATCH'
import json, os, site, sys
with open("/app/docker/feature-manifest.json") as f:
manifest = json.load(f)
bundle = manifest["bundles"][os.environ["BUNDLE_ID"]]
patches = bundle.get("patches", [])
if not patches:
print(" No patches")
sys.exit(0)
site_packages = site.getsitepackages()[0]
for patch in patches:
target = os.path.join(site_packages, patch["file"])
if not os.path.exists(target):
print(f" WARNING: patch target not found: {patch['file']}", file=sys.stderr)
continue
with open(target, "r") as fh:
content = fh.read()
if patch["search"] not in content:
print(f" WARNING: search text not found in {patch['file']} (already patched?)", file=sys.stderr)
continue
with open(target, "w") as fh:
fh.write(content.replace(patch["search"], patch["replace"]))
print(f" Patched {patch['file']}")
print(" Source patches applied")
PYPATCH
# ── Step 5: Download models ────────────────────────────────────────────── # ── Step 5: Download models ──────────────────────────────────────────────
echo "=== Downloading models ===" echo "=== Downloading models ==="
python3 << 'PYMODELS' python3 << 'PYMODELS'
@@ -245,8 +278,10 @@ for model in models:
size = os.path.getsize(dest) size = os.path.getsize(dest)
print(f" [{model_id}] Done ({size:,} bytes)") print(f" [{model_id}] Done ({size:,} bytes)")
else: else:
from rembg.sessions import new_session from rembg import new_session
new_session(session_name) # Force CPU-only provider: onnxruntime-gpu segfaults trying to load
# TensorRT libs that aren't present in build containers (no GPU).
new_session(session_name, providers=["CPUExecutionProvider"])
print(f" [{model_id}] Done") print(f" [{model_id}] Done")
else: else:
docker/feature-manifest.json
+81 -48
View File
@@ -1,9 +1,9 @@
{ {
"manifestVersion": 2, "manifestVersion": 2,
"imageVersion": "2.0.0", "imageVersion": "2.0.0",
"pythonVersion": "3.12", "pythonVersion": "3.11",
"basePackages": ["numpy==1.26.4", "Pillow==12.2.0", "opencv-python-headless==4.10.0.84"], "basePackages": ["numpy==1.26.4", "Pillow==12.2.0", "opencv-python-headless==4.10.0.84"],
"bundleRepo": "snapotter/feature-bundles", "bundleRepo": "deepsafe/feature-bundles",
"bundles": { "bundles": {
"background-removal": { "background-removal": {
"name": "Background Removal", "name": "Background Removal",
@@ -12,15 +12,15 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/background-removal-amd64-gpu.tar.gz", "file": "v2.0.0/background-removal-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "a1dfbdd5862fd32c7719ca79b0c7e4b76a327330a5991ddad98a2761abca10cb",
"compressedSize": 1, "compressedSize": 4808242472,
"extractedSize": 1 "extractedSize": 0
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/background-removal-arm64-cpu.tar.gz", "file": "v2.0.0/background-removal-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "cbb570b81bfd740dd05e265dd8c93ade53a9f41fdd30cc8b3fbc47a8daa824af",
"compressedSize": 1, "compressedSize": 4631430295,
"extractedSize": 1 "extractedSize": 0
} }
}, },
"packages": { "packages": {
@@ -31,13 +31,21 @@
"pipFlags": {}, "pipFlags": {},
"postInstall": [], "postInstall": [],
"models": [ "models": [
{ "id": "rembg-u2net", "downloadFn": "rembg_session", "args": ["u2net"] }, {
"id": "rembg-u2net",
"downloadFn": "rembg_session",
"args": ["u2net"]
},
{ {
"id": "rembg-isnet-general-use", "id": "rembg-isnet-general-use",
"downloadFn": "rembg_session", "downloadFn": "rembg_session",
"args": ["isnet-general-use"] "args": ["isnet-general-use"]
}, },
{ "id": "rembg-bria-rmbg", "downloadFn": "rembg_session", "args": ["bria-rmbg"] }, {
"id": "rembg-bria-rmbg",
"downloadFn": "rembg_session",
"args": ["bria-rmbg"]
},
{ {
"id": "rembg-birefnet-general-lite", "id": "rembg-birefnet-general-lite",
"downloadFn": "rembg_session", "downloadFn": "rembg_session",
@@ -79,15 +87,15 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/face-detection-amd64-gpu.tar.gz", "file": "v2.0.0/face-detection-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "0ee28b86dac88c60e005604ba0595746f455ec97f08d0c8307ba9b57e131bc7f",
"compressedSize": 1, "compressedSize": 75358868,
"extractedSize": 1 "extractedSize": 209838080
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/face-detection-arm64-cpu.tar.gz", "file": "v2.0.0/face-detection-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "8ad922514aba1721de154d370f9335f78c5ce5aceade5b19515e0192d4367909",
"compressedSize": 1, "compressedSize": 222878753,
"extractedSize": 1 "extractedSize": 704276480
} }
}, },
"packages": { "packages": {
@@ -120,15 +128,15 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/object-eraser-colorize-amd64-gpu.tar.gz", "file": "v2.0.0/object-eraser-colorize-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "a3f6cd2eca9e5dd907497d0d58ca5f6532887bb748e4062de0f4fb9815782c31",
"compressedSize": 1, "compressedSize": 1545472964,
"extractedSize": 1 "extractedSize": 2249379840
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/object-eraser-colorize-arm64-cpu.tar.gz", "file": "v2.0.0/object-eraser-colorize-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "e043d91acca20b3259711fd3a54521890f624b9952e617159f21a97216830630",
"compressedSize": 1, "compressedSize": 1266922599,
"extractedSize": 1 "extractedSize": 0
} }
}, },
"packages": { "packages": {
@@ -186,15 +194,15 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/upscale-enhance-amd64-gpu.tar.gz", "file": "v2.0.0/upscale-enhance-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "37f0e7a62dcb83b2dc0760532fb41ac9d1c40d336abe1aea01d889ad039ebc65",
"compressedSize": 1, "compressedSize": 5314188723,
"extractedSize": 1 "extractedSize": 8412712960
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/upscale-enhance-arm64-cpu.tar.gz", "file": "v2.0.0/upscale-enhance-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "de65321f1cc5d5664c63e38e49cd39c36fe88e0569d666821e982ff6d6999bea",
"compressedSize": 1, "compressedSize": 2262502457,
"extractedSize": 1 "extractedSize": 0
} }
}, },
"packages": { "packages": {
@@ -206,7 +214,13 @@
"realesrgan==0.3.0", "realesrgan==0.3.0",
"mediapipe>=0.10.21" "mediapipe>=0.10.21"
], ],
"arm64": ["lpips", "basicsr==1.4.2", "realesrgan==0.3.0", "mediapipe>=0.10.18"] "arm64": [
"torch==2.7.0 torchvision==0.22.0 --index-url https://download.pytorch.org/whl/cpu",
"lpips",
"basicsr==1.4.2",
"realesrgan==0.3.0",
"mediapipe>=0.10.18"
]
}, },
"pipFlags": { "pipFlags": {
"codeformer-pip==0.0.4": "--no-deps", "codeformer-pip==0.0.4": "--no-deps",
@@ -267,6 +281,13 @@
"minSize": 60000000 "minSize": 60000000
} }
], ],
"patches": [
{
"file": "basicsr/data/degradations.py",
"search": "from torchvision.transforms.functional_tensor import rgb_to_grayscale",
"replace": "from torchvision.transforms.functional import rgb_to_grayscale"
}
],
"enablesTools": ["upscale", "enhance-faces", "noise-removal"] "enablesTools": ["upscale", "enhance-faces", "noise-removal"]
}, },
"photo-restoration": { "photo-restoration": {
@@ -276,15 +297,15 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/photo-restoration-amd64-gpu.tar.gz", "file": "v2.0.0/photo-restoration-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "2ccac73c862f04dafc8d7ecc6543bbe8481b9d796a53a9c4e7cf34e53f261a24",
"compressedSize": 1, "compressedSize": 4698091043,
"extractedSize": 1 "extractedSize": 8246353920
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/photo-restoration-arm64-cpu.tar.gz", "file": "v2.0.0/photo-restoration-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "13afbaae45d84f1148d00b76a20115283ab3f75d7b2a9e33b940c9bf800b40e1",
"compressedSize": 1, "compressedSize": 1367026285,
"extractedSize": 1 "extractedSize": 0
} }
}, },
"packages": { "packages": {
@@ -298,6 +319,7 @@
"realesrgan==0.3.0" "realesrgan==0.3.0"
], ],
"arm64": [ "arm64": [
"torch==2.7.0 torchvision==0.22.0 --index-url https://download.pytorch.org/whl/cpu",
"onnxruntime==1.20.1", "onnxruntime==1.20.1",
"mediapipe>=0.10.18", "mediapipe>=0.10.18",
"lpips", "lpips",
@@ -364,6 +386,13 @@
"minSize": 3000000 "minSize": 3000000
} }
], ],
"patches": [
{
"file": "basicsr/data/degradations.py",
"search": "from torchvision.transforms.functional_tensor import rgb_to_grayscale",
"replace": "from torchvision.transforms.functional import rgb_to_grayscale"
}
],
"enablesTools": ["restore-photo"] "enablesTools": ["restore-photo"]
}, },
"ocr": { "ocr": {
@@ -373,15 +402,15 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/ocr-amd64-gpu.tar.gz", "file": "v2.0.0/ocr-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "80db475eb442899cedad3590b18f080225deaf50985224b302475fbfdf5883f6",
"compressedSize": 1, "compressedSize": 5927587552,
"extractedSize": 1 "extractedSize": 9367838720
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/ocr-arm64-cpu.tar.gz", "file": "v2.0.0/ocr-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "5c40dfc5f3147dd61bf8f3b24604eefb5591e6412a2ca61f7e2fb7e1c7d11b9c",
"compressedSize": 1, "compressedSize": 1977460762,
"extractedSize": 1 "extractedSize": 0
} }
}, },
"packages": { "packages": {
@@ -445,18 +474,22 @@
"archives": { "archives": {
"amd64-gpu": { "amd64-gpu": {
"file": "v2.0.0/transcription-amd64-gpu.tar.gz", "file": "v2.0.0/transcription-amd64-gpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "3033d480c6183df66855d78c355233dbc86a754d07414e544e1df1038337c9e1",
"compressedSize": 1, "compressedSize": 559189061,
"extractedSize": 1 "extractedSize": 845987840
}, },
"arm64-cpu": { "arm64-cpu": {
"file": "v2.0.0/transcription-arm64-cpu.tar.gz", "file": "v2.0.0/transcription-arm64-cpu.tar.gz",
"sha256": "0000000000000000000000000000000000000000000000000000000000000000", "sha256": "6184434bc10986677d15e96e22b6998c3a780d6619b6457b8854c01dfd0f9a28",
"compressedSize": 1, "compressedSize": 531990919,
"extractedSize": 1 "extractedSize": 0
} }
}, },
"packages": { "common": ["faster-whisper>=1.0.0"], "amd64": [], "arm64": [] }, "packages": {
"common": ["faster-whisper>=1.0.0"],
"amd64": [],
"arm64": []
},
"pipFlags": {}, "pipFlags": {},
"postInstall": [], "postInstall": [],
"models": [ "models": [
packages/ai/python/install_feature.py
+1 -1
View File
@@ -335,7 +335,7 @@ def main() -> None:
) )
else: else:
# Remote mode: download from HuggingFace # Remote mode: download from HuggingFace
bundle_repo = manifest.get("bundleRepo", "snapotter/feature-bundles") bundle_repo = manifest.get("bundleRepo", "deepsafe/feature-bundles")
url = f"https://huggingface.co/{bundle_repo}/resolve/main/{archive_file}" url = f"https://huggingface.co/{bundle_repo}/resolve/main/{archive_file}"
# Disk space check # Disk space check