name: AI Bundles on: workflow_dispatch: inputs: version: description: "Version to build bundles for (e.g., 2.0.0)" required: true type: string workflow_call: inputs: version: type: string required: true concurrency: group: ai-bundles-${{ inputs.version }} cancel-in-progress: false permissions: {} jobs: build: name: ${{ matrix.bundle }} (${{ matrix.arch }}) strategy: fail-fast: false matrix: include: - bundle: background-removal arch: amd64-gpu runner: ubuntu-latest - bundle: background-removal arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: face-detection arch: amd64-gpu runner: ubuntu-latest - bundle: face-detection arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: object-eraser-colorize arch: amd64-gpu runner: ubuntu-latest - bundle: object-eraser-colorize arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: upscale-enhance arch: amd64-gpu runner: ubuntu-latest - bundle: upscale-enhance arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: photo-restoration arch: amd64-gpu runner: ubuntu-latest - bundle: photo-restoration arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: ocr arch: amd64-gpu runner: ubuntu-latest - bundle: ocr arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: transcription arch: amd64-gpu runner: ubuntu-latest - bundle: transcription arch: arm64-cpu runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} permissions: contents: read packages: read steps: - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ /usr/local/share/boost /opt/hostedtoolcache/CodeQL sudo docker system prune -af df -h / - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: v${{ inputs.version }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Log in to GHCR uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GHCR_TOKEN }} - name: Build bundle inside Docker image env: BUNDLE: ${{ matrix.bundle }} ARCH: ${{ matrix.arch }} VERSION: ${{ inputs.version }} run: | mkdir -p /tmp/bundles docker run --rm --entrypoint bash \ -v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \ -v "$PWD/docker/feature-manifest.json:/app/docker/feature-manifest.json:ro" \ -v "/tmp/bundles:/output" \ "ghcr.io/snapotter-hq/snapotter:${VERSION}" \ /build-bundle.sh "${BUNDLE}" "${ARCH}" /output - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.bundle }}-${{ matrix.arch }} path: /tmp/bundles/ retention-days: 1 verify: name: Verify ${{ matrix.bundle }} (${{ matrix.arch }}) needs: build timeout-minutes: 20 strategy: fail-fast: false matrix: include: - bundle: background-removal arch: amd64-gpu runner: ubuntu-latest - bundle: background-removal arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: face-detection arch: amd64-gpu runner: ubuntu-latest - bundle: face-detection arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: object-eraser-colorize arch: amd64-gpu runner: ubuntu-latest - bundle: object-eraser-colorize arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: upscale-enhance arch: amd64-gpu runner: ubuntu-latest - bundle: upscale-enhance arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: photo-restoration arch: amd64-gpu runner: ubuntu-latest - bundle: photo-restoration arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: ocr arch: amd64-gpu runner: ubuntu-latest - bundle: ocr arch: arm64-cpu runner: ubuntu-24.04-arm - bundle: transcription arch: amd64-gpu runner: ubuntu-latest - bundle: transcription arch: arm64-cpu runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} permissions: contents: read packages: read steps: - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ /usr/local/share/boost /opt/hostedtoolcache/CodeQL sudo docker system prune -af df -h / - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: v${{ inputs.version }} - name: Log in to GHCR uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GHCR_TOKEN }} - name: Download build artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ matrix.bundle }}-${{ matrix.arch }} path: /tmp/bundles - name: Verify bundle env: BUNDLE: ${{ matrix.bundle }} ARCH: ${{ matrix.arch }} VERSION: ${{ inputs.version }} run: | docker run --rm --entrypoint bash \ -v "$PWD/docker/verify-bundle.sh:/verify-bundle.sh:ro" \ -v "$PWD/packages/ai/python:/app/packages/ai/python:ro" \ -v "$PWD/tests/fixtures:/fixtures:ro" \ -v "/tmp/bundles:/bundles:ro" \ -e SNAPOTTER_GPU=0 \ -e CUDA_VISIBLE_DEVICES="" \ -e MODELS_PATH=/tmp/verify-models \ -e U2NET_HOME=/tmp/verify-models/rembg \ -e PYTHONPATH=/app/packages/ai/python \ "ghcr.io/snapotter-hq/snapotter:${VERSION}" \ /verify-bundle.sh "${BUNDLE}" "${ARCH}" publish: name: Publish to HuggingFace needs: verify runs-on: ubuntu-latest permissions: contents: read steps: - name: Download all artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: /tmp/artifacts - name: Install hf CLI run: pip install -U "huggingface_hub[hf_xet]" - name: Organize and upload to HuggingFace env: HF_TOKEN: ${{ secrets.HF_TOKEN }} VERSION: ${{ inputs.version }} run: | mkdir -p "/tmp/upload/v${VERSION}" find /tmp/artifacts -name '*.tar.gz' -exec cp {} "/tmp/upload/v${VERSION}/" \; find /tmp/artifacts -name '*.sha256' -exec cp {} "/tmp/upload/v${VERSION}/" \; cd "/tmp/upload/v${VERSION}" echo '{"version":"'"${VERSION}"'","archives":[' > manifest.json first=true for archive in *.tar.gz; do [ -f "$archive" ] || continue sha=$(sha256sum "$archive" | cut -d' ' -f1) size=$(stat --format=%s "$archive") if [ "$first" = true ]; then first=false else echo ',' >> manifest.json fi printf '{"file":"%s","sha256":"%s","size":%s}' "$archive" "$sha" "$size" >> manifest.json done echo ']}' >> manifest.json # Interim: publish to the deepsafe account (the HF_TOKEN secret is deepsafe's). # Switch back to snapotter/feature-bundles once that account is restored. hf upload deepsafe/feature-bundles \ "/tmp/upload/v${VERSION}" "v${VERSION}" \ --repo-type model echo "Published: https://huggingface.co/deepsafe/feature-bundles/tree/main/v${VERSION}"