SnapOtter/.github/workflows/ai-bundles.yml

name: AI Bundles

on:
  workflow_dispatch:
    inputs:
      version:
        description: "Version to build bundles for (e.g., 2.0.0)"
        required: true
        type: string
  workflow_call:
    inputs:
      version:
        type: string
        required: true

concurrency:
  group: ai-bundles-${{ inputs.version }}
  cancel-in-progress: false

permissions: {}

jobs:
  build:
    name: ${{ matrix.bundle }} (${{ matrix.arch }})
    strategy:
      fail-fast: false
      matrix:
        include:
          - bundle: background-removal
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: background-removal
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: face-detection
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: face-detection
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: object-eraser-colorize
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: object-eraser-colorize
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: upscale-enhance
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: upscale-enhance
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: photo-restoration
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: photo-restoration
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: ocr
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: ocr
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: transcription
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: transcription
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
      packages: read
    steps:
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/boost /opt/hostedtoolcache/CodeQL
          sudo docker system prune -af
          df -h /

      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: v${{ inputs.version }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0

      - name: Log in to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Build bundle inside Docker image
        env:
          BUNDLE: ${{ matrix.bundle }}
          ARCH: ${{ matrix.arch }}
          VERSION: ${{ inputs.version }}
        run: |
          mkdir -p /tmp/bundles
          docker run --rm --entrypoint bash \
            -v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
            -v "$PWD/docker/feature-manifest.json:/app/docker/feature-manifest.json:ro" \
            -v "/tmp/bundles:/output" \
            "ghcr.io/snapotter-hq/snapotter:${VERSION}" \
            /build-bundle.sh "${BUNDLE}" "${ARCH}" /output

      - name: Upload artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ matrix.bundle }}-${{ matrix.arch }}
          path: /tmp/bundles/
          retention-days: 1

  verify:
    name: Verify ${{ matrix.bundle }} (${{ matrix.arch }})
    needs: build
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        include:
          - bundle: background-removal
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: background-removal
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: face-detection
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: face-detection
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: object-eraser-colorize
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: object-eraser-colorize
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: upscale-enhance
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: upscale-enhance
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: photo-restoration
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: photo-restoration
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: ocr
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: ocr
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
          - bundle: transcription
            arch: amd64-gpu
            runner: ubuntu-latest
          - bundle: transcription
            arch: arm64-cpu
            runner: ubuntu-24.04-arm
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
      packages: read
    steps:
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/boost /opt/hostedtoolcache/CodeQL
          sudo docker system prune -af
          df -h /

      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: v${{ inputs.version }}

      - name: Log in to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Download build artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ matrix.bundle }}-${{ matrix.arch }}
          path: /tmp/bundles

      - name: Verify bundle
        env:
          BUNDLE: ${{ matrix.bundle }}
          ARCH: ${{ matrix.arch }}
          VERSION: ${{ inputs.version }}
        run: |
          docker run --rm --entrypoint bash \
            -v "$PWD/docker/verify-bundle.sh:/verify-bundle.sh:ro" \
            -v "$PWD/packages/ai/python:/app/packages/ai/python:ro" \
            -v "$PWD/tests/fixtures:/fixtures:ro" \
            -v "/tmp/bundles:/bundles:ro" \
            -e SNAPOTTER_GPU=0 \
            -e CUDA_VISIBLE_DEVICES="" \
            -e MODELS_PATH=/tmp/verify-models \
            -e U2NET_HOME=/tmp/verify-models/rembg \
            -e PYTHONPATH=/app/packages/ai/python \
            "ghcr.io/snapotter-hq/snapotter:${VERSION}" \
            /verify-bundle.sh "${BUNDLE}" "${ARCH}"

  publish:
    name: Publish to HuggingFace
    needs: verify
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Download all artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: /tmp/artifacts

      - name: Install hf CLI
        run: pip install -U "huggingface_hub[hf_xet]"

      - name: Organize and upload to HuggingFace
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          VERSION: ${{ inputs.version }}
        run: |
          mkdir -p "/tmp/upload/v${VERSION}"

          find /tmp/artifacts -name '*.tar.gz' -exec cp {} "/tmp/upload/v${VERSION}/" \;
          find /tmp/artifacts -name '*.sha256' -exec cp {} "/tmp/upload/v${VERSION}/" \;

          cd "/tmp/upload/v${VERSION}"
          echo '{"version":"'"${VERSION}"'","archives":[' > manifest.json
          first=true
          for archive in *.tar.gz; do
            [ -f "$archive" ] || continue
            sha=$(sha256sum "$archive" | cut -d' ' -f1)
            size=$(stat --format=%s "$archive")
            if [ "$first" = true ]; then
              first=false
            else
              echo ',' >> manifest.json
            fi
            printf '{"file":"%s","sha256":"%s","size":%s}' "$archive" "$sha" "$size" >> manifest.json
          done
          echo ']}' >> manifest.json

          # Interim: publish to the deepsafe account (the HF_TOKEN secret is deepsafe's).
          # Switch back to snapotter/feature-bundles once that account is restored.
          hf upload deepsafe/feature-bundles \
            "/tmp/upload/v${VERSION}" "v${VERSION}" \
            --repo-type model

          echo "Published: https://huggingface.co/deepsafe/feature-bundles/tree/main/v${VERSION}"