name: AI Bundles

on:
  workflow_call:
    inputs:
      version:
        type: string
        required: true
      release_commit:
        type: string
        required: true
    secrets:
      GHCR_TOKEN:
        required: true
      HF_TOKEN:
        required: true
      OCR_RUNTIME_INDEX_SIGNING_KEY_B64:
        required: true
  workflow_dispatch:
    inputs:
      source_run_id:
        description: Optional successful Release workflow run containing the scanned amd64 digest
        required: false
        type: string
  schedule:
    - cron: "17 3 * * 1"

concurrency:
  group: ai-bundles-${{ inputs.version }}
  cancel-in-progress: false

permissions: {}

jobs:
  validate-inputs:
    name: Validate release inputs
    timeout-minutes: 10
    if: inputs.release_commit != ''
    runs-on: ubuntu-latest
    permissions: {}
    steps:
      - name: Reject unsafe or non-semantic versions
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
        run: |
          [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9]+([.-][A-Za-z0-9]+)*)?$ ]] || {
            echo "::error::Bundle version must be a path-safe semantic version"
            exit 1
          }
          [[ ${#VERSION} -le 64 ]] || {
            echo "::error::Bundle version is too long"
            exit 1
          }
          [[ "${RELEASE_COMMIT}" =~ ^[a-f0-9]{40}$ ]] || {
            echo "::error::Release commit must be a full immutable Git commit"
            exit 1
          }

      - name: Validate OCR runtime trust configuration
        env:
          OCR_RUNTIME_INDEX_KEY_ID: ${{ vars.OCR_RUNTIME_INDEX_KEY_ID }}
          OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64: ${{ vars.OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64 }}
        run: |
          : "${OCR_RUNTIME_INDEX_KEY_ID:?Set repository variable OCR_RUNTIME_INDEX_KEY_ID}"
          : "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64:?Set repository variable OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}"
          [[ "${OCR_RUNTIME_INDEX_KEY_ID}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]] || {
            echo "::error::OCR runtime signing key ID is not a safe identifier"
            exit 1
          }
          umask 077
          trap 'rm -f /tmp/ocr-runtime-public.pem' EXIT
          printf '%s' "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}" \
            | base64 --decode > /tmp/ocr-runtime-public.pem
          [[ "$(base64 --wrap=0 < /tmp/ocr-runtime-public.pem)" == "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}" ]] || {
            echo "::error::OCR runtime public key must use canonical base64"
            exit 1
          }
          openssl pkey -pubin -in /tmp/ocr-runtime-public.pem -text -noout \
            | grep -q ED25519 || {
              echo "::error::Configured OCR runtime public key is not Ed25519"
              exit 1
            }

  preflight-gpu-runner:
    # The NVIDIA verification below is the only job that needs physical hardware,
    # and it cannot start until build-ocr has produced an image digest. That used
    # to mean an offline GPU box burned a full multi-arch build and then parked in
    # "Queued" for 24h at sign-ocr-index, with nothing in the run explaining why.
    #
    # This job claims the same labels with no dependencies, so it is the first
    # thing scheduled. If the box is offline the run visibly sits here at second
    # zero instead of failing much later and much less legibly. It also proves the
    # GPU is usable BEFORE the expensive build, rather than after it.
    #
    # Note: this cannot be an API check. Listing self-hosted runners needs
    # Administration:read, which is not an available GITHUB_TOKEN permission, so
    # an API preflight would 403 on every run and pass vacuously. Claiming the
    # label is the only signal available to the workflow itself. RELEASE.md
    # carries the maintainer-side `gh api` check to run before dispatching.
    name: Preflight GPU runner
    needs: validate-inputs
    timeout-minutes: 10
    runs-on: [self-hosted, linux, x64, snapotter-nvidia]
    permissions: {}
    steps:
      - name: Require a working NVIDIA container runtime
        run: |
          command -v nvidia-smi >/dev/null || {
            echo "::error::nvidia-smi is missing on the snapotter-nvidia runner"
            exit 1
          }
          nvidia-smi --query-gpu=name,driver_version --format=csv,noheader
          docker info --format '{{json .Runtimes}}' | grep -q 'nvidia' || {
            echo "::error::Docker on the snapotter-nvidia runner has no nvidia runtime"
            exit 1
          }

      - name: Require enough free disk for the release image and bundles
        run: |
          free_gb="$(df -BG --output=avail / | tail -1 | tr -dc '0-9')"
          echo "free on /: ${free_gb}G"
          [[ "${free_gb}" -ge 25 ]] || {
            echo "::error::Need at least 25G free on the runner, found ${free_gb}G"
            exit 1
          }

  build-ocr:
    name: Build OCR (${{ matrix.target }})
    # Gated on preflight-gpu-runner: nothing here can be signed or published
    # without the NVIDIA verification, so spending 90 minutes building before
    # confirming that box is up only makes the failure slower and less obvious.
    needs: [validate-inputs, preflight-gpu-runner]
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include:
          - target: linux-amd64-cpu-py312
            image_platform: linux-amd64
            runner: ubuntu-latest
          - target: linux-arm64-cpu-py311
            image_platform: linux-arm64
            runner: ubuntu-24.04-arm
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
      packages: read
    steps:
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/boost /opt/hostedtoolcache/CodeQL
          sudo docker system prune -af
          df -h /

      - name: Checkout release tag
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ inputs.release_commit }}
          fetch-depth: 0
          persist-credentials: false

      - name: Verify immutable release tag binding
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
        run: |
          tag_commit="$(git rev-parse "refs/tags/v${VERSION}^{commit}")"
          [[ "$(git rev-parse HEAD)" == "${RELEASE_COMMIT}" \
            && "${tag_commit}" == "${RELEASE_COMMIT}" ]] || {
              echo "::error::Release tag no longer resolves to the selected commit"
              exit 1
            }

      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: "3.11"

      - name: Install pinned dependency auditor
        run: pip install "pip-audit==2.10.0"

      - name: Audit exact OCR runtime dependency lock
        env:
          TARGET: ${{ matrix.target }}
        run: |
          case "${TARGET}" in
            linux-amd64-*) requirements="docker/ocr-runtime-requirements-amd64.txt" ;;
            linux-arm64-*) requirements="docker/ocr-runtime-requirements-arm64.txt" ;;
            *) echo "::error::Unknown OCR runtime target: ${TARGET}"; exit 1 ;;
          esac
          pip-audit -r "${requirements}" --no-deps --disable-pip --aliases

      - name: Download unpublished image digest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-${{ matrix.image_platform }}
          path: /tmp/image-digest

      - name: Resolve exact build image
        env:
          IMAGE_PLATFORM: ${{ matrix.image_platform }}
        run: |
          mapfile -t digest_files < <(find /tmp/image-digest -maxdepth 1 -type f -print)
          [[ ${#digest_files[@]} -eq 1 ]] || {
            echo "::error::Expected exactly one ${IMAGE_PLATFORM} image digest"
            exit 1
          }
          digest="$(basename "${digest_files[0]}")"
          [[ "${digest}" =~ ^[a-f0-9]{64}$ ]] || {
            echo "::error::Invalid image digest artifact"
            exit 1
          }
          echo "SNAPOTTER_BUNDLE_IMAGE=ghcr.io/snapotter-hq/snapotter@sha256:${digest}" >> "$GITHUB_ENV"
          echo "SNAPOTTER_BUNDLE_IMAGE_DIGEST=${digest}" >> "$GITHUB_ENV"

      - name: Log in to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Build immutable OCR runtime in its native app image
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          TARGET: ${{ matrix.target }}
          VERSION: ${{ inputs.version }}
          WORKFLOW_COMMIT: ${{ github.workflow_sha }}
        run: |
          rm -rf /tmp/bundles
          mkdir -p /tmp/bundles/primary /tmp/bundles/rebuild
          [[ "${WORKFLOW_COMMIT}" =~ ^[a-f0-9]{40}$ ]] || {
            echo "::error::Reusable workflow revision is not an immutable commit"
            exit 1
          }
          for build in primary rebuild; do
            docker run --rm --entrypoint bash \
              -v "$PWD/docker/build-bundle.sh:/build-bundle.sh:ro" \
              -v "$PWD/docker/build-ocr-runtime.sh:/build-ocr-runtime.sh:ro" \
              -v "$PWD/docker/ocr-runtime-models.json:/ocr-runtime-models.json:ro" \
              -v "$PWD/docker/ocr-runtime-requirements-amd64.txt:/ocr-runtime-requirements-amd64.txt:ro" \
              -v "$PWD/docker/ocr-runtime-requirements-arm64.txt:/ocr-runtime-requirements-arm64.txt:ro" \
              -v "$PWD/docker/ocr-best-v1-calibration.json:/app/docker/ocr-best-v1-calibration.json:ro" \
              -v "$PWD/docker/feature-manifest.json:/app/docker/feature-manifest.json:ro" \
              -v "/tmp/bundles/${build}:/output" \
              -e SNAPOTTER_VERSION="${VERSION}" \
              -e SNAPOTTER_OCR_SOURCE_IMAGE_DIGEST="${SNAPOTTER_BUNDLE_IMAGE_DIGEST}" \
              -e SNAPOTTER_OCR_SOURCE_COMMIT="${RELEASE_COMMIT}" \
              -e SNAPOTTER_OCR_BUILDER_ID="github-actions:.github/workflows/ai-bundles.yml@${WORKFLOW_COMMIT}" \
              "${SNAPOTTER_BUNDLE_IMAGE}" \
              /build-bundle.sh ocr "${TARGET}" /output
          done
          for suffix in tar.gz tar.gz.sha256 artifact.json index-input.json; do
            test -s "/tmp/bundles/primary/ocr-${TARGET}.${suffix}"
            test -s "/tmp/bundles/rebuild/ocr-${TARGET}.${suffix}"
            cmp --silent "/tmp/bundles/primary/ocr-${TARGET}.${suffix}" \
              "/tmp/bundles/rebuild/ocr-${TARGET}.${suffix}" || {
                echo "::error::OCR runtime rebuild is not byte-reproducible: ${suffix}"
                exit 1
              }
          done

      - name: Upload OCR build artifact for verification
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ocr-${{ matrix.target }}
          overwrite: true
          path: |
            /tmp/bundles/primary/ocr-${{ matrix.target }}.tar.gz
            /tmp/bundles/primary/ocr-${{ matrix.target }}.tar.gz.sha256
            /tmp/bundles/primary/ocr-${{ matrix.target }}.artifact.json
          if-no-files-found: error
          retention-days: 1

  verify-ocr:
    name: Verify OCR (${{ matrix.target }})
    needs: build-ocr
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include:
          - target: linux-amd64-cpu-py312
            image_platform: linux-amd64
            runner: ubuntu-latest
            gpu_visible: none
            cuda_visible: ""
          - target: linux-arm64-cpu-py311
            image_platform: linux-arm64
            runner: ubuntu-24.04-arm
            gpu_visible: none
            cuda_visible: ""
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
      packages: read
    steps:
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /usr/local/share/boost /opt/hostedtoolcache/CodeQL
          sudo docker system prune -af
          df -h /

      - name: Checkout release tag
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ inputs.release_commit }}
          fetch-depth: 0
          persist-credentials: false

      - name: Verify immutable release tag binding
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
        run: |
          tag_commit="$(git rev-parse "refs/tags/v${VERSION}^{commit}")"
          [[ "$(git rev-parse HEAD)" == "${RELEASE_COMMIT}" \
            && "${tag_commit}" == "${RELEASE_COMMIT}" ]] || {
              echo "::error::Release tag no longer resolves to the selected commit"
              exit 1
            }

      - name: Download unpublished image digest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-${{ matrix.image_platform }}
          path: /tmp/image-digest

      - name: Resolve exact verification image
        env:
          IMAGE_PLATFORM: ${{ matrix.image_platform }}
        run: |
          mapfile -t digest_files < <(find /tmp/image-digest -maxdepth 1 -type f -print)
          [[ ${#digest_files[@]} -eq 1 ]] || {
            echo "::error::Expected exactly one ${IMAGE_PLATFORM} image digest"
            exit 1
          }
          digest="$(basename "${digest_files[0]}")"
          [[ "${digest}" =~ ^[a-f0-9]{64}$ ]] || {
            echo "::error::Invalid image digest artifact"
            exit 1
          }
          echo "SNAPOTTER_BUNDLE_IMAGE=ghcr.io/snapotter-hq/snapotter@sha256:${digest}" >> "$GITHUB_ENV"

      - name: Log in to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Verify image has the release trust identity
        env:
          OCR_RUNTIME_INDEX_KEY_ID: ${{ vars.OCR_RUNTIME_INDEX_KEY_ID }}
          OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64: ${{ vars.OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64 }}
        run: |
          : "${OCR_RUNTIME_INDEX_KEY_ID:?Set repository variable OCR_RUNTIME_INDEX_KEY_ID}"
          : "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64:?Set repository variable OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}"
          # The official image bakes the trust store as a FILE
          # (/app/docker/ocr-runtime-trust.json, written by write-ocr-runtime-trust.mjs)
          # and deliberately leaves OCR_RUNTIME_INDEX_* unset in the image env: those
          # are the operator-override path, and loadOcrRuntimeTrustKeys
          # (packages/ai/src/runtime-index.ts:52-71) reads env only when set, else the
          # baked file. So verify the file the runtime actually trusts, not the env.
          trust_json="$(docker run --rm --entrypoint sh "${SNAPOTTER_BUNDLE_IMAGE}" \
            -c 'cat /app/docker/ocr-runtime-trust.json')"
          actual_key_id="$(printf '%s' "${trust_json}" | jq -r '.keys[0].keyId')"
          actual_public_key="$(printf '%s' "${trust_json}" | jq -r '.keys[0].publicKey')"
          expected_public_key="$(printf '%s' "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}" | base64 --decode)"
          actual_official_container="$(docker run --rm --entrypoint sh "${SNAPOTTER_BUNDLE_IMAGE}" \
            -c 'printf %s "$SNAPOTTER_OFFICIAL_CONTAINER"')"
          [[ "${actual_key_id}" == "${OCR_RUNTIME_INDEX_KEY_ID}" ]] || {
            echo "::error::Baked OCR trust key ID (${actual_key_id}) does not match the release identity"
            exit 1
          }
          [[ "${actual_public_key}" == "${expected_public_key}" ]] || {
            echo "::error::Baked OCR trust public key does not match the release identity"
            exit 1
          }
          [[ "${actual_official_container}" == "1" ]] || {
            echo "::error::Verification image is not marked as an official container"
            exit 1
          }

      - name: Download OCR build artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ocr-${{ matrix.target }}
          path: /tmp/bundles

      - name: Verify exact Fast OCR payload
        run: |
          RUNNER_UID="$(id -u)"
          RUNNER_GID="$(id -g)"
          docker run --rm --network none \
            -e EMBEDDED=0 \
            -e PUID="${RUNNER_UID}" \
            -e PGID="${RUNNER_GID}" \
            "${SNAPOTTER_BUNDLE_IMAGE}" \
            bash -ceu '
              expected=(chi_sim deu eng fra jpn spa)
              mapfile -t actual < <(tesseract --list-langs 2>/dev/null | tail -n +2 | LC_ALL=C sort)
              [[ "${actual[*]}" == "${expected[*]}" ]] || {
                echo "FAIL: Fast OCR language payload changed: ${actual[*]}" >&2
                exit 1
              }
              tessdata=/usr/share/tesseract-ocr/5/tessdata
              [[ ! -e "${tessdata}/osd.traineddata" && ! -e "${tessdata}/kor.traineddata" ]] || {
                echo "FAIL: Fast OCR unexpectedly contains orientation or Korean traineddata" >&2
                exit 1
              }
              traineddata_bytes="$(find "${tessdata}" -maxdepth 1 -type f -name "*.traineddata" -printf "%s\n" \
                | awk "{ total += \$1 } END { print total + 0 }")"
              [[ "${traineddata_bytes}" == "14003738" ]] || {
                echo "FAIL: Fast OCR traineddata size changed: ${traineddata_bytes}" >&2
                exit 1
              }
              echo "PASS: exact Fast OCR languages and 14003738 traineddata bytes; about 25 MiB remains a measured approximate image-layer size"
            '

      - name: Verify offline install and runtime smoke
        env:
          CUDA_VISIBLE_DEVICES: ${{ matrix.cuda_visible }}
          NVIDIA_VISIBLE_DEVICES: ${{ matrix.gpu_visible }}
          TARGET: ${{ matrix.target }}
        run: |
          RUNNER_UID="$(id -u)"
          RUNNER_GID="$(id -g)"
          rm -rf /tmp/ocr-quality
          install -d -m 1777 /tmp/ocr-quality
          docker run --rm --network none --memory 3g --memory-swap 3g \
            -v "$PWD/docker/verify-ocr-runtime.sh:/verify-ocr-runtime.sh:ro" \
            -v "$PWD/packages/ai/python/install_runtime.py:/app/packages/ai/python/install_runtime.py:ro" \
            -v "/tmp/bundles:/bundles:ro" \
            -e EMBEDDED=0 \
            -e PUID="${RUNNER_UID}" \
            -e PGID="${RUNNER_GID}" \
            -e CUDA_VISIBLE_DEVICES \
            -e NVIDIA_VISIBLE_DEVICES \
            "${SNAPOTTER_BUNDLE_IMAGE}" \
            /verify-ocr-runtime.sh "${TARGET}" memory-preflight
          docker run --rm --network none --memory 4g --memory-swap 4g \
            --user 20001:0 \
            -v "$PWD/docker/verify-ocr-runtime.sh:/verify-ocr-runtime.sh:ro" \
            -v "$PWD/packages/ai/python/install_runtime.py:/app/packages/ai/python/install_runtime.py:ro" \
            -v "/tmp/bundles:/bundles:ro" \
            -e EMBEDDED=0 \
            -e CUDA_VISIBLE_DEVICES \
            -e NVIDIA_VISIBLE_DEVICES \
            "${SNAPOTTER_BUNDLE_IMAGE}" \
            /verify-ocr-runtime.sh "${TARGET}" install-smoke
          docker run --rm --network none --memory 4g --memory-swap 4g \
            -v "$PWD/docker/verify-ocr-runtime.sh:/verify-ocr-runtime.sh:ro" \
            -v "$PWD/packages/ai/python/install_runtime.py:/app/packages/ai/python/install_runtime.py:ro" \
            -v "$PWD/tests/fixtures:/fixtures:ro" \
            -v "/tmp/bundles:/bundles:ro" \
            -v "/tmp/ocr-quality:/reports" \
            -e EMBEDDED=0 \
            -e PUID="${RUNNER_UID}" \
            -e PGID="${RUNNER_GID}" \
            -e CUDA_VISIBLE_DEVICES \
            -e NVIDIA_VISIBLE_DEVICES \
            -e OCR_VERIFY_ENVIRONMENT=native-cpu \
            -e OCR_VERIFY_REPORT_DIR=/reports \
            "${SNAPOTTER_BUNDLE_IMAGE}" \
            /verify-ocr-runtime.sh "${TARGET}" full
          chmod 0755 /tmp/ocr-quality
          test -r "/tmp/ocr-quality/ocr-${TARGET}-native-cpu.quality.json"

      - name: Upload native OCR quality and resource report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ocr-quality-${{ matrix.target }}
          overwrite: true
          path: /tmp/ocr-quality/*.quality.json
          if-no-files-found: error
          retention-days: 30

      - name: Extract verified OCR runtime for supply-chain inspection
        id: extract_runtime
        env:
          TARGET: ${{ matrix.target }}
        run: |
          rm -rf /tmp/ocr-runtime-scan /tmp/ocr-security
          mkdir -p /tmp/ocr-runtime-scan /tmp/ocr-security
          python3 - <<'PY'
          import hashlib
          import json
          import os
          import pathlib
          import stat
          import tarfile

          bundles = pathlib.Path("/tmp/bundles")
          scan_root = pathlib.Path("/tmp/ocr-runtime-scan")
          target = os.environ["TARGET"]
          artifact_path = bundles / f"ocr-{target}.artifact.json"
          artifact = json.loads(artifact_path.read_bytes())
          archive_metadata = artifact["archive"]
          archive_name = f"ocr-{target}.tar.gz"

          def file_sha256(path):
              digest = hashlib.sha256()
              with path.open("rb") as handle:
                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                      digest.update(chunk)
              return digest.hexdigest()

          if (
              artifact.get("family") != "ocr"
              or artifact.get("target") != target
              or archive_metadata.get("file") != archive_name
          ):
              raise SystemExit("OCR artifact identity changed after runtime verification")
          if artifact.get("resources", {}).get("minimumMemoryBytes") != 4 * 1024 * 1024 * 1024:
              raise SystemExit("OCR artifact memory policy changed after runtime verification")

          archive_path = bundles / archive_name
          if archive_path.stat().st_size != archive_metadata["size"]:
              raise SystemExit("OCR archive size changed after runtime verification")
          if file_sha256(archive_path) != artifact["archive"]["sha256"]:
              raise SystemExit("OCR archive digest changed after runtime verification")

          with tarfile.open(archive_path, "r:gz") as archive:
              for member in archive.getmembers():
                  path = pathlib.PurePosixPath(member.name)
                  if path.is_absolute() or ".." in path.parts:
                      raise SystemExit(f"Unsafe OCR archive path: {member.name}")
                  if not member.isfile() and not member.isdir():
                      raise SystemExit(f"Unsupported OCR archive entry: {member.name}")
              archive.extractall(scan_root, filter=tarfile.data_filter)

          expected_files = {record["path"]: record for record in artifact["files"]}
          if len(expected_files) != len(artifact["files"]):
              raise SystemExit("OCR artifact file manifest contains duplicate paths")
          actual_files = {
              path.relative_to(scan_root).as_posix(): path
              for path in scan_root.rglob("*")
              if path.is_file()
          }
          if set(actual_files) != set(expected_files):
              raise SystemExit("Extracted OCR runtime file manifest mismatch")
          for relative_path, expected in expected_files.items():
              path = actual_files[relative_path]
              info = path.stat()
              if (
                  info.st_size != expected["size"]
                  or stat.S_IMODE(info.st_mode) != expected["mode"]
                  or file_sha256(path) != expected["sha256"]
              ):
                  raise SystemExit(f"Extracted OCR runtime file mismatch: {relative_path}")
          PY
          echo "ready=true" >> "$GITHUB_OUTPUT"

      - name: Install pinned Syft 1.42.3 from verified release bytes
        env:
          SYFT_VERSION: "1.42.3"
        run: |
          # Published in Syft's v1.42.3 syft_1.42.3_checksums.txt release asset.
          case "$(uname -m)" in
            x86_64)
              syft_arch="amd64"
              expected_sha256="0d6be741479eddd2c8644a288990c04f3df0d609bbc1599a005532a9dff63509"
              ;;
            aarch64 | arm64)
              syft_arch="arm64"
              expected_sha256="dc630590c953347789d08f8ebf57c7d8094db89100785fcd94b1cddeac791804"
              ;;
            *)
              echo "::error::Unsupported Syft installer architecture: $(uname -m)"
              exit 1
              ;;
          esac
          archive="syft_${SYFT_VERSION}_linux_${syft_arch}.tar.gz"
          install_root="${RUNNER_TEMP}/syft-${SYFT_VERSION}"
          rm -rf "${install_root}"
          mkdir -p "${install_root}"
          curl --fail --location --silent --show-error \
            --proto '=https' --tlsv1.2 --retry 3 \
            --output "${install_root}/${archive}" \
            "https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}/${archive}"
          printf '%s  %s\n' "${expected_sha256}" "${install_root}/${archive}" \
            | sha256sum --check --strict -
          tar -xzf "${install_root}/${archive}" -C "${install_root}" syft
          chmod 0755 "${install_root}/syft"
          "${install_root}/syft" version -o json \
            | jq -e --arg version "${SYFT_VERSION}" '.version == $version' >/dev/null
          echo "${install_root}" >> "$GITHUB_PATH"

      - name: Generate per-target OCR runtime SBOMs
        env:
          TARGET: ${{ matrix.target }}
          VERSION: ${{ inputs.version }}
        run: |
          syft scan "dir:/tmp/ocr-runtime-scan" \
            --source-name "snapotter-ocr-${TARGET}" \
            --source-version "${VERSION}" \
            -o "cyclonedx-json=/tmp/ocr-security/ocr-${TARGET}-sbom.cdx.json" \
            -o "spdx-json=/tmp/ocr-security/ocr-${TARGET}-sbom.spdx.json"

      - name: Gate OCR runtime vulnerabilities
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          version: v0.70.0
          scan-type: sbom
          scan-ref: /tmp/ocr-security/ocr-${{ matrix.target }}-sbom.cdx.json
          scanners: vuln
          format: table
          exit-code: "1"
          severity: "CRITICAL,HIGH"
          trivyignores: ".trivyignore"
          timeout: 10m0s

      - name: Record per-target OCR runtime vulnerability report
        if: always() && steps.extract_runtime.outputs.ready == 'true'
        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
        with:
          version: v0.70.0
          scan-type: sbom
          scan-ref: /tmp/ocr-security/ocr-${{ matrix.target }}-sbom.cdx.json
          scanners: vuln
          format: json
          output: /tmp/ocr-security/ocr-${{ matrix.target }}-trivy.json
          exit-code: "0"
          severity: "CRITICAL,HIGH"
          trivyignores: ".trivyignore"
          timeout: 10m0s

      - name: Upload per-target OCR security reports
        if: always() && steps.extract_runtime.outputs.ready == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ocr-security-${{ matrix.target }}
          overwrite: true
          path: |
            /tmp/ocr-security/ocr-${{ matrix.target }}-sbom.cdx.json
            /tmp/ocr-security/ocr-${{ matrix.target }}-sbom.spdx.json
            /tmp/ocr-security/ocr-${{ matrix.target }}-trivy.json
          if-no-files-found: error
          retention-days: 30

  verify-ocr-nvidia:
    name: Verify amd64 CPU OCR with a real NVIDIA GPU exposed
    needs: build-ocr
    timeout-minutes: 60
    runs-on: [self-hosted, linux, x64, snapotter-nvidia]
    env:
      DOCKER_CONFIG: /tmp/snapotter-ocr-docker-config
    permissions:
      contents: read
      packages: read
    steps:
      - name: Clear persistent runner release state
        run: |
          rm -rf /tmp/image-digest /tmp/bundles /tmp/ocr-quality /tmp/snapotter-ocr-docker-config
          mkdir -p /tmp/image-digest /tmp/bundles /tmp/ocr-quality /tmp/snapotter-ocr-docker-config

      - name: Checkout release tag
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ inputs.release_commit }}
          fetch-depth: 0
          persist-credentials: false

      - name: Verify immutable release tag binding
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
        run: |
          tag_commit="$(git rev-parse "refs/tags/v${VERSION}^{commit}")"
          [[ "$(git rev-parse HEAD)" == "${RELEASE_COMMIT}" \
            && "${tag_commit}" == "${RELEASE_COMMIT}" ]] || {
              echo "::error::Release tag no longer resolves to the selected commit"
              exit 1
            }

      - name: Require a working NVIDIA container runtime
        run: |
          command -v nvidia-smi >/dev/null
          nvidia-smi --query-gpu=name,driver_version --format=csv,noheader
          docker info --format '{{json .Runtimes}}' | grep -q 'nvidia'

      - name: Download unpublished amd64 image digest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-linux-amd64
          path: /tmp/image-digest

      - name: Resolve exact amd64 verification image
        run: |
          mapfile -t digest_files < <(find /tmp/image-digest -maxdepth 1 -type f -print)
          [[ ${#digest_files[@]} -eq 1 ]] || {
            echo "::error::Expected exactly one linux-amd64 image digest"
            exit 1
          }
          digest="$(basename "${digest_files[0]}")"
          [[ "${digest}" =~ ^[a-f0-9]{64}$ ]] || {
            echo "::error::Invalid amd64 image digest artifact"
            exit 1
          }
          echo "SNAPOTTER_BUNDLE_IMAGE=ghcr.io/snapotter-hq/snapotter@sha256:${digest}" >> "$GITHUB_ENV"

      - name: Log in to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Download amd64 OCR build artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ocr-linux-amd64-cpu-py312
          path: /tmp/bundles

      - name: Prove the GPU is exposed inside the release image
        run: |
          docker run --rm --gpus all --entrypoint sh "${SNAPOTTER_BUNDLE_IMAGE}" \
            -c 'test -e /dev/nvidiactl && test -r /proc/driver/nvidia/version'

      - name: Verify the portable CPU runtime with NVIDIA exposed
        run: |
          RUNNER_UID="$(id -u)"
          RUNNER_GID="$(id -g)"
          rm -rf /tmp/ocr-quality
          install -d -m 1777 /tmp/ocr-quality
          docker run --rm --gpus all --network none --memory 4g --memory-swap 4g \
            -v "$PWD/docker/verify-ocr-runtime.sh:/verify-ocr-runtime.sh:ro" \
            -v "$PWD/packages/ai/python/install_runtime.py:/app/packages/ai/python/install_runtime.py:ro" \
            -v "$PWD/tests/fixtures:/fixtures:ro" \
            -v "/tmp/bundles:/bundles:ro" \
            -v "/tmp/ocr-quality:/reports" \
            -e EMBEDDED=0 \
            -e PUID="${RUNNER_UID}" \
            -e PGID="${RUNNER_GID}" \
            -e CUDA_VISIBLE_DEVICES=0 \
            -e NVIDIA_VISIBLE_DEVICES=all \
            -e OCR_VERIFY_ENVIRONMENT=nvidia-exposed-cpu \
            -e OCR_VERIFY_REPORT_DIR=/reports \
            "${SNAPOTTER_BUNDLE_IMAGE}" \
            /verify-ocr-runtime.sh linux-amd64-cpu-py312 full
          chmod 0755 /tmp/ocr-quality
          test -r /tmp/ocr-quality/ocr-linux-amd64-cpu-py312-nvidia-exposed-cpu.quality.json

      - name: Upload NVIDIA-exposed CPU OCR quality and resource report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ocr-quality-linux-amd64-cpu-py312-nvidia
          overwrite: true
          path: /tmp/ocr-quality/*.quality.json
          if-no-files-found: error
          retention-days: 30

      - name: Remove release-scoped state from persistent runner
        if: always()
        run: |
          if [[ -n "${SNAPOTTER_BUNDLE_IMAGE:-}" ]]; then
            docker image rm "${SNAPOTTER_BUNDLE_IMAGE}" || true
          fi
          rm -rf /tmp/image-digest /tmp/bundles /tmp/ocr-quality /tmp/snapotter-ocr-docker-config

  sign-ocr-index:
    name: Sign verified OCR runtime index
    timeout-minutes: 20
    needs: [build-ocr, verify-ocr, verify-ocr-nvidia]
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Checkout exact release tag for provenance verification
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ inputs.release_commit }}
          fetch-depth: 0
          persist-credentials: false

      - name: Bind signing to the checked-out release tag commit
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
          WORKFLOW_COMMIT: ${{ github.workflow_sha }}
        run: |
          tag_commit="$(git rev-parse "refs/tags/v${VERSION}^{commit}")"
          [[ "$(git rev-parse HEAD)" == "${RELEASE_COMMIT}" \
            && "${tag_commit}" == "${RELEASE_COMMIT}" ]] || {
            echo "::error::Checked-out commit does not match release tag v${VERSION}"
            exit 1
          }
          [[ "${WORKFLOW_COMMIT}" =~ ^[a-f0-9]{40}$ ]] || {
            echo "::error::Reusable workflow revision is not an immutable commit"
            exit 1
          }
          echo "SNAPOTTER_RELEASE_COMMIT=${RELEASE_COMMIT}" >> "$GITHUB_ENV"
          echo "SNAPOTTER_WORKFLOW_COMMIT=${WORKFLOW_COMMIT}" >> "$GITHUB_ENV"

      - name: Download exact amd64 release image digest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-linux-amd64
          path: /tmp/image-digests/linux-amd64

      - name: Download exact arm64 release image digest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-linux-arm64
          path: /tmp/image-digests/linux-arm64

      - name: Download both verified OCR artifact sets
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: ocr-linux-*-cpu-py*
          path: /tmp/ocr-artifacts
          merge-multiple: true

      - name: Download OCR security attestations
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: ocr-security-*
          path: /tmp/ocr-attestations
          merge-multiple: true

      - name: Download OCR quality and resource attestations
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: ocr-quality-*
          path: /tmp/ocr-attestations
          merge-multiple: true

      - name: Create canonical unsigned index and validate its archives
        env:
          VERSION: ${{ inputs.version }}
        run: |
          mkdir -p /tmp/ocr-index
          python3 - <<'PY'
          import hashlib
          import json
          import os
          from pathlib import Path

          source = Path("/tmp/ocr-artifacts")
          attestation_root = Path("/tmp/ocr-attestations")
          output = Path("/tmp/ocr-index")
          expected_targets = {"linux-amd64-cpu-py312", "linux-arm64-cpu-py311"}
          expected_image_platforms = {
              "linux-amd64-cpu-py312": "linux-amd64",
              "linux-arm64-cpu-py311": "linux-arm64",
          }
          artifacts = []
          seen_targets = set()
          source_commits = set()

          def canonical(value):
              return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()

          def file_sha256(path):
              digest = hashlib.sha256()
              with path.open("rb") as handle:
                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                      digest.update(chunk)
              return digest.hexdigest()

          # OCR_MEASURED_ESTIMATE_CONTRACT_BEGIN
          MEASURED_ESTIMATE_MINIMUM_TOLERANCE_BYTES = 1024 * 1024

          def validate_measured_estimate(target, manifest_target, archive):
              if not isinstance(manifest_target, dict):
                  raise SystemExit(f"missing measured-estimate metadata for {target}")
              if manifest_target.get("sizeKind") != "measured-estimate":
                  raise SystemExit(f"invalid measured-estimate sizeKind for {target}")
              if not isinstance(archive, dict):
                  raise SystemExit(f"missing archive size metadata for {target}")

              size_fields = (
                  ("compressed", "compressedSizeEstimate", archive.get("size")),
                  ("expanded", "extractedSizeEstimate", archive.get("expandedSize")),
              )
              for label, estimate_field, actual in size_fields:
                  estimate = manifest_target.get(estimate_field)
                  if type(estimate) is not int or estimate <= 0:
                      raise SystemExit(
                          f"invalid {label} measured-estimate field for {target}"
                      )
                  if type(actual) is not int or actual <= 0:
                      raise SystemExit(f"invalid actual {label} size for {target}")
                  difference = abs(actual - estimate)
                  if (
                      difference > MEASURED_ESTIMATE_MINIMUM_TOLERANCE_BYTES
                      and difference * 100 > estimate
                  ):
                      raise SystemExit(
                          f"OCR {label} size drift exceeds measured-estimate tolerance for "
                          f"{target}: estimate={estimate}, actual={actual}, "
                          f"difference={difference}, minimumTolerance="
                          f"{MEASURED_ESTIMATE_MINIMUM_TOLERANCE_BYTES}, "
                          "relativeTolerance=1%"
                      )
          # OCR_MEASURED_ESTIMATE_CONTRACT_END

          manifest_path = Path("docker/feature-manifest.json")
          try:
              manifest = json.loads(manifest_path.read_bytes())
          except (OSError, json.JSONDecodeError) as error:
              raise SystemExit(f"invalid checked-out OCR feature manifest: {error}") from error
          bundles = manifest.get("bundles") if isinstance(manifest, dict) else None
          ocr_manifest = bundles.get("ocr") if isinstance(bundles, dict) else None
          manifest_targets = (
              ocr_manifest.get("targets") if isinstance(ocr_manifest, dict) else None
          )
          if (
              not isinstance(manifest_targets, dict)
              or set(manifest_targets) != expected_targets
          ):
              raise SystemExit("checked-out OCR measured-estimate target set is invalid")

          workflow_commit = os.environ["SNAPOTTER_WORKFLOW_COMMIT"]
          if (
              len(workflow_commit) != 40
              or any(character not in "0123456789abcdef" for character in workflow_commit)
          ):
              raise SystemExit("invalid reusable-workflow provenance")
          expected_image_digests = {}
          for target, image_platform in expected_image_platforms.items():
              digest_root = Path("/tmp/image-digests") / image_platform
              digest_entries = list(digest_root.iterdir())
              if (
                  len(digest_entries) != 1
                  or not digest_entries[0].is_file()
                  or digest_entries[0].is_symlink()
                  or digest_entries[0].stat().st_size != 0
              ):
                  raise SystemExit(
                      f"expected exactly one release image digest for {image_platform}"
                  )
              digest = digest_entries[0].name
              if len(digest) != 64 or any(
                  character not in "0123456789abcdef" for character in digest
              ):
                  raise SystemExit(f"invalid release image digest for {image_platform}")
              expected_image_digests[target] = f"sha256:{digest}"

          for metadata_path in sorted(source.glob("*.artifact.json")):
              if metadata_path.is_symlink() or not metadata_path.is_file():
                  raise SystemExit(f"unsafe OCR artifact metadata: {metadata_path.name}")
              raw = metadata_path.read_bytes()
              artifact = json.loads(raw)
              if raw != canonical(artifact):
                  raise SystemExit(f"artifact metadata is not canonical: {metadata_path.name}")
              target = artifact.get("target")
              if artifact.get("family") != "ocr" or target not in expected_targets:
                  raise SystemExit(f"unexpected OCR artifact identity: {metadata_path.name}")
              if target in seen_targets:
                  raise SystemExit(f"duplicate OCR target: {target}")
              seen_targets.add(target)
              if metadata_path.name != f"ocr-{target}.artifact.json":
                  raise SystemExit(f"artifact filename does not bind target {target}")
              expected_arch = "amd64" if target == "linux-amd64-cpu-py312" else "arm64"
              if artifact.get("platform") != "linux" or artifact.get("arch") != expected_arch:
                  raise SystemExit(f"platform mismatch for {target}")
              if artifact.get("version") != os.environ["VERSION"]:
                  raise SystemExit(f"version mismatch for {target}")
              if artifact.get("capabilities", {}).get("providers") != ["CPUExecutionProvider"]:
                  raise SystemExit(f"non-portable provider set for {target}")
              if artifact.get("resources", {}).get("minimumMemoryBytes") != 4 * 1024 * 1024 * 1024:
                  raise SystemExit(f"invalid memory policy for {target}")
              provenance = artifact.get("provenance")
              if not isinstance(provenance, dict):
                  raise SystemExit(f"missing build provenance for {target}")
              source_commit = provenance.get("sourceCommit")
              if (
                  not isinstance(source_commit, str)
                  or len(source_commit) != 40
                  or any(character not in "0123456789abcdef" for character in source_commit)
                  or provenance.get("builderId")
                  != f"github-actions:.github/workflows/ai-bundles.yml@{workflow_commit}"
                  or source_commit != os.environ["SNAPOTTER_RELEASE_COMMIT"]
                  or provenance.get("sourceImageDigest") != expected_image_digests[target]
              ):
                  raise SystemExit(f"invalid build provenance for {target}")
              source_commits.add(source_commit)
              model_objects = artifact.get("modelObjects")
              models = artifact.get("models")
              if (
                  not isinstance(model_objects, list)
                  or not isinstance(models, dict)
                  or len(model_objects) != len(models)
              ):
                  raise SystemExit(f"missing model provenance for {target}")
              if {model.get("id") for model in model_objects if isinstance(model, dict)} != set(models):
                  raise SystemExit(f"model provenance IDs do not match the digest map for {target}")
              file_records = {
                  record.get("path"): record
                  for record in artifact.get("files", [])
                  if isinstance(record, dict)
              }
              if len(file_records) != len(artifact.get("files", [])):
                  raise SystemExit(f"duplicate or malformed runtime file records for {target}")
              for model in model_objects:
                  if not isinstance(model, dict):
                      raise SystemExit(f"malformed model provenance for {target}")
                  model_id = model.get("id")
                  path = model.get("path")
                  digest = model.get("sha256")
                  file_record = file_records.get(path)
                  if (
                      not isinstance(model_id, str)
                      or models.get(model_id) != digest
                      or not isinstance(file_record, dict)
                      or file_record.get("sha256") != digest
                      or file_record.get("size") != model.get("size")
                      or not isinstance(model.get("license"), str)
                      or not model["license"]
                  ):
                      raise SystemExit(f"model provenance does not match runtime bytes for {target}")
                  if "url" in model:
                      revision = model.get("revision")
                      if (
                          not isinstance(revision, str)
                          or len(revision) != 40
                          or f"/resolve/{revision}/" not in str(model.get("url"))
                      ):
                          raise SystemExit(
                              f"model provenance does not bind its immutable revision for {target}"
                          )
                  elif not isinstance(model.get("source"), str):
                      raise SystemExit(f"local model provenance is missing its source for {target}")
              legal_materials = artifact.get("legalMaterials")
              expected_legal_ids = {
                  "snapotter-agpl",
                  "apache-2.0",
                  "antlr-4.9.3-license",
              }
              if (
                  not isinstance(legal_materials, list)
                  or len(legal_materials) != len(expected_legal_ids)
                  or {
                      material.get("id")
                      for material in legal_materials
                      if isinstance(material, dict)
                  }
                  != expected_legal_ids
              ):
                  raise SystemExit(f"incomplete legal material set for {target}")
              for material in legal_materials:
                  if not isinstance(material, dict):
                      raise SystemExit(f"malformed legal material for {target}")
                  file_record = file_records.get(material.get("path"))
                  if (
                      not isinstance(file_record, dict)
                      or file_record.get("sha256") != material.get("sha256")
                      or file_record.get("size") != material.get("size")
                  ):
                      raise SystemExit(
                          f"legal material does not match runtime bytes for {target}"
                      )
              notices = file_records.get("THIRD_PARTY_NOTICES.json")
              if (
                  not isinstance(notices, dict)
                  or not isinstance(notices.get("sha256"), str)
                  or len(notices["sha256"]) != 64
                  or not isinstance(notices.get("size"), int)
                  or notices["size"] <= 0
              ):
                  raise SystemExit(f"runtime third-party notices are missing for {target}")
              archive = artifact.get("archive", {})
              archive_name = archive.get("file")
              if archive_name != f"ocr-{target}.tar.gz":
                  raise SystemExit(f"unexpected archive name for {target}")
              archive_path = source / archive_name
              if not archive_path.is_file() or archive_path.is_symlink():
                  raise SystemExit(f"missing archive for {target}")
              if archive_path.stat().st_size != archive.get("size"):
                  raise SystemExit(f"archive size mismatch for {target}")
              digest = file_sha256(archive_path)
              if digest != archive.get("sha256"):
                  raise SystemExit(f"archive digest mismatch for {target}")
              checksum_path = source / f"{archive_name}.sha256"
              if (
                  not checksum_path.is_file()
                  or checksum_path.is_symlink()
                  or checksum_path.read_bytes() != f"{digest}\n".encode()
              ):
                  raise SystemExit(f"archive checksum sidecar mismatch for {target}")
              validate_measured_estimate(target, manifest_targets[target], archive)
              artifacts.append(artifact)

          if seen_targets != expected_targets:
              raise SystemExit(f"expected both OCR targets, found {sorted(seen_targets)}")
          if len(source_commits) != 1:
              raise SystemExit(f"OCR targets were built from different commits: {source_commits}")
          expected_attestations = {
              *(f"ocr-{target}-sbom.cdx.json" for target in expected_targets),
              *(f"ocr-{target}-sbom.spdx.json" for target in expected_targets),
              *(f"ocr-{target}-trivy.json" for target in expected_targets),
              *(f"ocr-{target}-native-cpu.quality.json" for target in expected_targets),
              "ocr-linux-amd64-cpu-py312-nvidia-exposed-cpu.quality.json",
          }
          actual_attestations = {
              path.name for path in attestation_root.iterdir() if path.is_file()
          }
          if actual_attestations != expected_attestations:
              raise SystemExit(
                  "OCR attestation set mismatch: "
                  f"expected {sorted(expected_attestations)}, got {sorted(actual_attestations)}"
              )
          attestations = [
              {
                  "file": path.name,
                  "sha256": file_sha256(path),
                  "size": path.stat().st_size,
              }
              for path in sorted(attestation_root.iterdir())
          ]
          index = {
              "artifacts": sorted(artifacts, key=lambda artifact: artifact["target"]),
              "attestations": attestations,
              "schemaVersion": 1,
              "version": os.environ["VERSION"],
          }
          (output / "ocr-runtime-index.unsigned.json").write_bytes(canonical(index))
          PY

      - name: Sign with repository-trusted Ed25519 identity
        env:
          OCR_RUNTIME_INDEX_KEY_ID: ${{ vars.OCR_RUNTIME_INDEX_KEY_ID }}
          OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64: ${{ vars.OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64 }}
          OCR_RUNTIME_INDEX_SIGNING_KEY_B64: ${{ secrets.OCR_RUNTIME_INDEX_SIGNING_KEY_B64 }}
        run: |
          : "${OCR_RUNTIME_INDEX_KEY_ID:?Set repository variable OCR_RUNTIME_INDEX_KEY_ID}"
          : "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64:?Set repository variable OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}"
          : "${OCR_RUNTIME_INDEX_SIGNING_KEY_B64:?Set secret OCR_RUNTIME_INDEX_SIGNING_KEY_B64}"
          [[ "${OCR_RUNTIME_INDEX_KEY_ID}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]] || {
            echo "::error::OCR runtime signing key ID is not a safe identifier"
            exit 1
          }
          umask 077
          trap 'rm -f /tmp/ocr-index/private.pem /tmp/ocr-index/configured-public.pem /tmp/ocr-index/derived-public.pem' EXIT
          printf '%s' "${OCR_RUNTIME_INDEX_SIGNING_KEY_B64}" | base64 --decode > /tmp/ocr-index/private.pem
          printf '%s' "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}" | base64 --decode > /tmp/ocr-index/configured-public.pem
          [[ "$(base64 --wrap=0 < /tmp/ocr-index/private.pem)" == "${OCR_RUNTIME_INDEX_SIGNING_KEY_B64}" ]] || {
            echo "::error::OCR signing private key must use canonical base64"
            exit 1
          }
          [[ "$(base64 --wrap=0 < /tmp/ocr-index/configured-public.pem)" == "${OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64}" ]] || {
            echo "::error::OCR runtime public key must use canonical base64"
            exit 1
          }
          openssl pkey -in /tmp/ocr-index/private.pem -text -noout | grep -q ED25519 || {
            echo "::error::Configured OCR signing private key is not Ed25519"
            exit 1
          }
          openssl pkey -in /tmp/ocr-index/private.pem -pubout -out /tmp/ocr-index/derived-public.pem
          openssl pkey -pubin -in /tmp/ocr-index/configured-public.pem -pubout -out /tmp/ocr-index/trusted-public.pem
          openssl pkey -pubin -in /tmp/ocr-index/trusted-public.pem -text -noout | grep -q ED25519 || {
            echo "::error::Configured OCR runtime key is not Ed25519"
            exit 1
          }
          cmp \
            <(openssl pkey -pubin -in /tmp/ocr-index/derived-public.pem -outform DER) \
            <(openssl pkey -pubin -in /tmp/ocr-index/trusted-public.pem -outform DER) || {
              echo "::error::Signing key does not match the repository-trusted public key"
              exit 1
            }
          openssl pkeyutl -sign -rawin \
            -inkey /tmp/ocr-index/private.pem \
            -in /tmp/ocr-index/ocr-runtime-index.unsigned.json \
            -out /tmp/ocr-index/ocr-runtime-index.signature
          openssl pkeyutl -verify -rawin \
            -pubin -inkey /tmp/ocr-index/trusted-public.pem \
            -in /tmp/ocr-index/ocr-runtime-index.unsigned.json \
            -sigfile /tmp/ocr-index/ocr-runtime-index.signature

      - name: Seal canonical index and public trust artifact
        env:
          OCR_RUNTIME_INDEX_KEY_ID: ${{ vars.OCR_RUNTIME_INDEX_KEY_ID }}
        run: |
          python3 - <<'PY'
          import base64
          import json
          import os
          from pathlib import Path

          root = Path("/tmp/ocr-index")

          def canonical(value):
              return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()

          unsigned_path = root / "ocr-runtime-index.unsigned.json"
          unsigned_bytes = unsigned_path.read_bytes()
          unsigned = json.loads(unsigned_bytes)
          unsigned["signature"] = {
              "algorithm": "ed25519",
              "keyId": os.environ["OCR_RUNTIME_INDEX_KEY_ID"],
              "value": base64.b64encode((root / "ocr-runtime-index.signature").read_bytes()).decode(),
          }
          sealed_path = root / "ocr-runtime-index.json"
          sealed_path.write_bytes(canonical(unsigned))
          sealed = json.loads(sealed_path.read_bytes())
          sealed.pop("signature")
          if canonical(sealed) != unsigned_bytes:
              raise SystemExit("sealed OCR index does not reconstruct its signed payload")

          trust = {
              "keys": [{
                  "algorithm": "ed25519",
                  "keyId": os.environ["OCR_RUNTIME_INDEX_KEY_ID"],
                  "publicKey": (root / "trusted-public.pem").read_text(),
              }],
              "schemaVersion": 1,
          }
          (root / "ocr-runtime-trusted-keys.json").write_bytes(canonical(trust))
          PY
          rm -f \
            /tmp/ocr-index/private.pem \
            /tmp/ocr-index/configured-public.pem \
            /tmp/ocr-index/derived-public.pem \
            /tmp/ocr-index/ocr-runtime-index.signature

      - name: Upload verified signed OCR metadata
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ocr-runtime-metadata
          overwrite: true
          path: |
            /tmp/ocr-index/ocr-runtime-index.json
            /tmp/ocr-index/ocr-runtime-trusted-keys.json
          if-no-files-found: error
          retention-days: 1

  verify-signed-ocr-index:
    name: Verify signed OCR index (${{ matrix.target }})
    needs: sign-ocr-index
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        include:
          - target: linux-amd64-cpu-py312
            image_platform: linux-amd64
            runner: ubuntu-latest
          - target: linux-arm64-cpu-py311
            image_platform: linux-arm64
            runner: ubuntu-24.04-arm
    runs-on: ${{ matrix.runner }}
    permissions:
      packages: read
    steps:
      - name: Download final signed OCR index
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ocr-runtime-metadata
          path: /tmp/ocr-index

      - name: Download exact unpublished image digest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-${{ matrix.image_platform }}
          path: /tmp/image-digest

      - name: Resolve exact native image
        env:
          IMAGE_PLATFORM: ${{ matrix.image_platform }}
        run: |
          mapfile -t digest_files < <(find /tmp/image-digest -maxdepth 1 -type f -print)
          [[ ${#digest_files[@]} -eq 1 ]] || {
            echo "::error::Expected exactly one ${IMAGE_PLATFORM} image digest"
            exit 1
          }
          digest="$(basename "${digest_files[0]}")"
          [[ "${digest}" =~ ^[a-f0-9]{64}$ ]] || {
            echo "::error::Invalid ${IMAGE_PLATFORM} image digest artifact"
            exit 1
          }
          echo "SNAPOTTER_BUNDLE_IMAGE=ghcr.io/snapotter-hq/snapotter@sha256:${digest}" >> "$GITHUB_ENV"

      - name: Log in to GHCR
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Verify final index with image-baked application trust
        env:
          TARGET: ${{ matrix.target }}
          VERSION: ${{ inputs.version }}
        run: |
          docker run --rm --network none \
            -v /tmp/ocr-index/ocr-runtime-index.json:/tmp/ocr-runtime-index.json:ro \
            -e EMBEDDED=0 \
            -e OCR_VERIFY_TARGET="${TARGET}" \
            -e OCR_VERIFY_VERSION="${VERSION}" \
            "${SNAPOTTER_BUNDLE_IMAGE}" \
            /app/apps/api/node_modules/.bin/tsx -e '
              import { readFileSync } from "node:fs";
              import {
                loadOcrRuntimeTrustKeys,
                OCR_RUNTIME_INDEX_MAX_BYTES,
                verifyRuntimeIndex,
              } from "/app/packages/ai/src/runtime-index.ts";
              const target = process.env.OCR_VERIFY_TARGET;
              const version = process.env.OCR_VERIFY_VERSION;
              if (!target || !version) throw new Error("Signed-index verification environment is incomplete");
              const raw = readFileSync("/tmp/ocr-runtime-index.json");
              if (raw.length > OCR_RUNTIME_INDEX_MAX_BYTES) {
                throw new Error(`Final OCR runtime index exceeds ${OCR_RUNTIME_INDEX_MAX_BYTES} bytes`);
              }
              const parsed = JSON.parse(raw.toString("ascii"));
              const targets = parsed.artifacts
                ?.map((artifact) => artifact?.target)
                .sort();
              const expectedTargets = [
                "linux-amd64-cpu-py312",
                "linux-arm64-cpu-py311",
              ];
              if (JSON.stringify(targets) !== JSON.stringify(expectedTargets)) {
                throw new Error(`Final OCR runtime index target set is invalid: ${JSON.stringify(targets)}`);
              }
              const trustKeys = loadOcrRuntimeTrustKeys();
              const verified = verifyRuntimeIndex(raw, target, trustKeys, version);
              if (verified.artifact.target !== target || verified.artifact.version !== version) {
                throw new Error("Final OCR runtime index selected the wrong target or version");
              }
              process.stdout.write(`PASS: image-baked trust verified ${target} for ${version}\n`);
            '

  publish:
    name: Publish verified bundles to HuggingFace
    # Uploads the full v3 set to HuggingFace and then re-downloads and hashes
    # every published object. Untimed, a stalled transfer burns the 360 minute
    # default before anyone notices.
    timeout-minutes: 60
    needs: [verify-ocr, sign-ocr-index, verify-signed-ocr-index]
    # Hugging Face exposes one repository-wide branch head. Keep expensive
    # builds parallel across releases, then serialize only the short CAS write
    # boundary so two versions cannot race the shared `main` revision.
    concurrency:
      group: snapotter-hf-feature-bundles-publish
      cancel-in-progress: false
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - name: Check out hash-locked release tooling
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ inputs.release_commit }}
          fetch-depth: 0
          persist-credentials: false

      - name: Verify immutable release tag binding
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
        run: |
          tag_commit="$(git rev-parse "refs/tags/v${VERSION}^{commit}")"
          [[ "$(git rev-parse HEAD)" == "${RELEASE_COMMIT}" \
            && "${tag_commit}" == "${RELEASE_COMMIT}" ]] || {
              echo "::error::Release tag no longer resolves to the selected commit"
              exit 1
            }

      - name: Set up pinned release Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
          python-version: "3.11.14"

      - name: Download all verified build and metadata artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: /tmp/artifacts

      - name: Install hash-locked hf CLI
        run: |
          python -m venv /tmp/hf-release-venv
          /tmp/hf-release-venv/bin/python -m pip install \
            --disable-pip-version-check --require-hashes --no-deps --only-binary=:all: \
            --requirement docker/hf-release-requirements.txt
          echo "/tmp/hf-release-venv/bin" >> "$GITHUB_PATH"

      - name: Organize and verify the exact signed OCR release closure
        env:
          OCR_RUNTIME_INDEX_KEY_ID: ${{ vars.OCR_RUNTIME_INDEX_KEY_ID }}
          OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64: ${{ vars.OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64 }}
          VERSION: ${{ inputs.version }}
        run: |
          python3 - <<'PY'
          import base64
          import hashlib
          import json
          import os
          import shutil
          import subprocess
          import tempfile
          from pathlib import Path

          root = Path("/tmp/upload") / f"v{os.environ['VERSION']}"
          v3 = root / "v3"
          source = Path("/tmp/artifacts")
          if root.exists():
              shutil.rmtree(root)
          v3.mkdir(parents=True)

          expected_targets = {"linux-amd64-cpu-py312", "linux-arm64-cpu-py311"}
          expected_attestations = {
              *(f"ocr-{target}-sbom.cdx.json" for target in expected_targets),
              *(f"ocr-{target}-sbom.spdx.json" for target in expected_targets),
              *(f"ocr-{target}-trivy.json" for target in expected_targets),
              *(f"ocr-{target}-native-cpu.quality.json" for target in expected_targets),
              "ocr-linux-amd64-cpu-py312-nvidia-exposed-cpu.quality.json",
          }
          expected_artifact_objects = {
              *(f"ocr-{target}.tar.gz" for target in expected_targets),
              *(f"ocr-{target}.tar.gz.sha256" for target in expected_targets),
              *(f"ocr-{target}.artifact.json" for target in expected_targets),
          }
          fixed_objects = {"ocr-runtime-index.json", "ocr-runtime-trusted-keys.json"}
          expected_sources = expected_artifact_objects | expected_attestations | fixed_objects

          source_files = [path for path in source.rglob("*") if path.is_file()]
          for name in sorted(expected_sources):
              matches = [path for path in source_files if path.name == name]
              if len(matches) != 1:
                  raise SystemExit(
                      f"missing or colliding OCR release objects for {name}: {len(matches)}"
                  )
              if matches[0].is_symlink():
                  raise SystemExit(f"OCR release object is a symlink: {name}")
              shutil.copyfile(matches[0], v3 / name)

          unexpected_ocr_sources = {
              path.name for path in source_files
              if path.name.startswith("ocr-") and path.name not in expected_sources
          }
          if unexpected_ocr_sources:
              raise SystemExit(
                  f"unexpected OCR release objects: {sorted(unexpected_ocr_sources)}"
              )

          def file_sha256(path):
              digest = hashlib.sha256()
              with path.open("rb") as handle:
                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                      digest.update(chunk)
              return digest.hexdigest()

          def canonical(value):
              return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()

          index_path = v3 / "ocr-runtime-index.json"
          index_raw = index_path.read_bytes()
          signed_index = json.loads(index_raw)
          if index_raw != canonical(signed_index):
              raise SystemExit("Candidate OCR runtime index is not canonical JSON")
          signature = signed_index.get("signature")
          if (
              not isinstance(signature, dict)
              or set(signature) != {"algorithm", "keyId", "value"}
              or signature.get("algorithm") != "ed25519"
              or signature.get("keyId") != os.environ["OCR_RUNTIME_INDEX_KEY_ID"]
          ):
              raise SystemExit("Candidate OCR runtime index has an untrusted signature identity")
          unsigned_index = dict(signed_index)
          unsigned_index.pop("signature")
          try:
              signature_bytes = base64.b64decode(signature["value"], validate=True)
              public_key_bytes = base64.b64decode(
                  os.environ["OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64"], validate=True
              )
          except (KeyError, ValueError) as error:
              raise SystemExit("Candidate OCR runtime index signature is invalid") from error
          with tempfile.TemporaryDirectory() as temporary:
              temporary_root = Path(temporary)
              payload_path = temporary_root / "index.json"
              signature_path = temporary_root / "index.sig"
              public_key_path = temporary_root / "public.pem"
              payload_path.write_bytes(canonical(unsigned_index))
              signature_path.write_bytes(signature_bytes)
              public_key_path.write_bytes(public_key_bytes)
              try:
                  subprocess.run(
                      [
                          "openssl", "pkeyutl", "-verify", "-rawin", "-pubin",
                          "-inkey", str(public_key_path), "-in", str(payload_path),
                          "-sigfile", str(signature_path),
                      ],
                      check=True,
                      stdout=subprocess.DEVNULL,
                      stderr=subprocess.DEVNULL,
                  )
              except subprocess.CalledProcessError as error:
                  raise SystemExit(
                      "Candidate OCR runtime index signature is invalid"
                  ) from error

          if (
              signed_index.get("schemaVersion") != 1
              or signed_index.get("version") != os.environ["VERSION"]
              or not isinstance(signed_index.get("artifacts"), list)
              or not isinstance(signed_index.get("attestations"), list)
          ):
              raise SystemExit("Candidate OCR runtime index has an invalid identity")

          artifact_names = set()
          artifact_targets = set()
          for artifact in signed_index["artifacts"]:
              if not isinstance(artifact, dict):
                  raise SystemExit("Signed OCR artifact record is malformed")
              target = artifact.get("target")
              if artifact.get("family") != "ocr" or target not in expected_targets:
                  raise SystemExit("Signed OCR artifact has an unexpected identity")
              if target in artifact_targets:
                  raise SystemExit(f"Signed OCR artifact target collides: {target}")
              artifact_targets.add(target)
              metadata_name = f"ocr-{target}.artifact.json"
              metadata_path = v3 / metadata_name
              metadata_raw = metadata_path.read_bytes()
              metadata = json.loads(metadata_raw)
              if metadata_raw != canonical(metadata) or metadata != artifact:
                  raise SystemExit(f"Signed OCR artifact metadata mismatch: {target}")
              archive = artifact.get("archive")
              archive_name = f"ocr-{target}.tar.gz"
              if not isinstance(archive, dict) or archive.get("file") != archive_name:
                  raise SystemExit(f"Signed OCR archive identity mismatch: {target}")
              digest = archive.get("sha256")
              size = archive.get("size")
              if (
                  not isinstance(digest, str)
                  or len(digest) != 64
                  or any(character not in "0123456789abcdef" for character in digest)
                  or not isinstance(size, int)
                  or size <= 0
              ):
                  raise SystemExit(f"Signed OCR archive record is malformed: {target}")
              archive_path = v3 / archive_name
              if archive_path.stat().st_size != size or file_sha256(archive_path) != digest:
                  raise SystemExit(f"Signed OCR archive bytes mismatch: {target}")
              sidecar_name = f"{archive_name}.sha256"
              if (v3 / sidecar_name).read_bytes() != f"{digest}\n".encode():
                  raise SystemExit(f"Signed OCR archive sidecar mismatch: {target}")
              artifact_names.update({metadata_name, archive_name, sidecar_name})

          if artifact_targets != expected_targets or artifact_names != expected_artifact_objects:
              raise SystemExit("Signed OCR artifact closure mismatch")

          attestation_names = set()
          for attestation in signed_index["attestations"]:
              if not isinstance(attestation, dict) or set(attestation) != {"file", "sha256", "size"}:
                  raise SystemExit("Signed OCR attestation record is malformed")
              name = attestation["file"]
              if not isinstance(name, str) or Path(name).name != name or name in attestation_names:
                  raise SystemExit(f"Signed OCR attestation path collides: {name}")
              attestation_names.add(name)
              path = v3 / name
              if not path.is_file() or path.stat().st_size != attestation.get("size"):
                  raise SystemExit(f"Signed OCR attestation size mismatch: {name}")
              if file_sha256(path) != attestation.get("sha256"):
                  raise SystemExit(f"Signed OCR attestation digest mismatch: {name}")
          if attestation_names != expected_attestations:
              raise SystemExit("Signed OCR attestation closure mismatch")

          trust_path = v3 / "ocr-runtime-trusted-keys.json"
          trust_raw = trust_path.read_bytes()
          trust = json.loads(trust_raw)
          if trust_raw != canonical(trust) or trust.get("schemaVersion") != 1:
              raise SystemExit("Candidate OCR trust artifact is not canonical")
          keys = trust.get("keys")
          if not isinstance(keys, list) or len(keys) != 1 or not isinstance(keys[0], dict):
              raise SystemExit("Candidate OCR trust artifact has an invalid key set")
          if (
              keys[0].get("algorithm") != "ed25519"
              or keys[0].get("keyId") != os.environ["OCR_RUNTIME_INDEX_KEY_ID"]
              or not isinstance(keys[0].get("publicKey"), str)
          ):
              raise SystemExit("Candidate OCR trust artifact has an untrusted identity")
          with tempfile.TemporaryDirectory() as temporary:
              temporary_root = Path(temporary)
              configured_path = temporary_root / "configured.pem"
              trust_key_path = temporary_root / "trust.pem"
              configured_der = temporary_root / "configured.der"
              trust_der = temporary_root / "trust.der"
              configured_path.write_bytes(public_key_bytes)
              trust_key_path.write_text(keys[0]["publicKey"])
              for source_path, output_path in (
                  (configured_path, configured_der),
                  (trust_key_path, trust_der),
              ):
                  subprocess.run(
                      [
                          "openssl", "pkey", "-pubin", "-in", str(source_path),
                          "-outform", "DER", "-out", str(output_path),
                      ],
                      check=True,
                      stdout=subprocess.DEVNULL,
                      stderr=subprocess.DEVNULL,
                  )
              if configured_der.read_bytes() != trust_der.read_bytes():
                  raise SystemExit("Candidate OCR trust artifact contains the wrong public key")

          actual_objects = {path.name for path in v3.iterdir() if path.is_file()}
          expected_objects = artifact_names | attestation_names | fixed_objects
          if actual_objects != expected_objects or any(path.is_dir() for path in v3.iterdir()):
              raise SystemExit("Signed OCR artifact closure mismatch")

          files = []
          for file in sorted(path for path in root.rglob("*") if path.is_file()):
              if file.name == "manifest.json":
                  continue
              files.append({
                  "file": file.relative_to(root).as_posix(),
                  "sha256": file_sha256(file),
                  "size": file.stat().st_size,
              })
          manifest = {
              "files": files,
              "schemaVersion": 2,
              "version": os.environ["VERSION"],
          }
          (root / "manifest.json").write_bytes(canonical(manifest))
          PY

      - name: Refuse to mutate an existing version
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          OCR_RUNTIME_INDEX_KEY_ID: ${{ vars.OCR_RUNTIME_INDEX_KEY_ID }}
          OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64: ${{ vars.OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64 }}
          VERSION: ${{ inputs.version }}
        run: |
          /tmp/hf-release-venv/bin/python - <<'PY'
          import base64
          import hashlib
          import json
          import os
          import subprocess
          import tempfile
          from pathlib import Path, PurePosixPath
          from huggingface_hub import HfApi, hf_hub_download

          repo_id = "deepsafe/feature-bundles"
          version = os.environ["VERSION"]
          remote_path = f"v{version}/manifest.json"
          token = os.environ["HF_TOKEN"]
          api = HfApi(token=token)
          snapshot_revision = api.repo_info(
              repo_id, repo_type="model", revision="main", token=token
          ).sha
          if (
              not isinstance(snapshot_revision, str)
              or len(snapshot_revision) != 40
              or any(character not in "0123456789abcdef" for character in snapshot_revision)
          ):
              raise SystemExit("Hugging Face repository did not resolve to an immutable revision")
          with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as handle:
              handle.write(f"HF_PARENT_COMMIT={snapshot_revision}\n")
          if not api.file_exists(
              repo_id,
              remote_path,
              repo_type="model",
              revision=snapshot_revision,
              token=token,
          ):
              existing_version_files = [
                  path
                  for path in api.list_repo_files(
                      repo_id,
                      repo_type="model",
                      revision=snapshot_revision,
                      token=token,
                  )
                  if path.startswith(f"v{version}/")
              ]
              if existing_version_files:
                  raise SystemExit(
                      f"Refusing to publish into an existing feature-bundle version without a manifest: {existing_version_files[:5]}"
                  )
              raise SystemExit(0)

          def canonical(value):
              return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()

          def file_sha256(path):
              digest = hashlib.sha256()
              with path.open("rb") as handle:
                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                      digest.update(chunk)
              return digest.hexdigest()

          def manifest_map(manifest):
              if (
                  not isinstance(manifest, dict)
                  or manifest.get("schemaVersion") != 2
                  or manifest.get("version") != version
                  or not isinstance(manifest.get("files"), list)
              ):
                  raise SystemExit("Existing feature-bundle manifest has an invalid identity")
              mapped = {}
              for entry in manifest["files"]:
                  relative_path = entry.get("file") if isinstance(entry, dict) else None
                  parsed_path = PurePosixPath(relative_path) if isinstance(relative_path, str) else None
                  if (
                      not isinstance(entry, dict)
                      or not isinstance(relative_path, str)
                      or parsed_path is None
                      or parsed_path.is_absolute()
                      or ".." in parsed_path.parts
                      or not relative_path.startswith("v3/")
                      or not isinstance(entry.get("sha256"), str)
                      or len(entry["sha256"]) != 64
                      or any(
                          character not in "0123456789abcdef"
                          for character in entry["sha256"]
                      )
                      or not isinstance(entry.get("size"), int)
                      or entry["size"] <= 0
                      or relative_path in mapped
                  ):
                      raise SystemExit("Existing feature-bundle manifest has malformed files")
                  mapped[relative_path] = entry
              return mapped

          def is_variable_ocr_attestation(path):
              return path == "v3/ocr-runtime-index.json" or (
                  path.startswith("v3/ocr-")
                  and (
                      "-sbom." in path
                      or path.endswith("-trivy.json")
                      or path.endswith(".quality.json")
                  )
              )

          existing_path = Path(hf_hub_download(
              repo_id,
              remote_path,
              repo_type="model",
              revision=snapshot_revision,
              token=token,
          ))
          existing_raw = existing_path.read_bytes()
          existing_manifest = json.loads(existing_raw)
          if existing_raw != canonical(existing_manifest):
              raise SystemExit("Existing feature-bundle manifest is not canonical JSON")
          candidate_path = Path("/tmp/upload") / f"v{version}" / "manifest.json"
          candidate_raw = candidate_path.read_bytes()
          candidate_manifest = json.loads(candidate_raw)
          if candidate_raw != canonical(candidate_manifest):
              raise SystemExit("Candidate feature-bundle manifest is not canonical JSON")
          existing_files = manifest_map(existing_manifest)
          candidate_files = manifest_map(candidate_manifest)
          candidate_disk_files = {
              path.relative_to(candidate_path.parent).as_posix()
              for path in candidate_path.parent.rglob("*")
              if path.is_file() and path != candidate_path
          }
          if candidate_disk_files != set(candidate_files):
              raise SystemExit("Candidate feature-bundle manifest closure mismatch")
          remote_prefix_files = {
              path.removeprefix(f"v{version}/")
              for path in api.list_repo_files(
                  repo_id,
                  repo_type="model",
                  revision=snapshot_revision,
                  token=token,
              )
              if path.startswith(f"v{version}/")
          }
          if remote_prefix_files != {"manifest.json", *existing_files}:
              raise SystemExit("Existing feature-bundle version has uncommitted or missing objects")

          existing_deterministic = {
              path: entry for path, entry in existing_files.items()
              if not is_variable_ocr_attestation(path)
          }
          candidate_deterministic = {
              path: entry for path, entry in candidate_files.items()
              if not is_variable_ocr_attestation(path)
          }
          if existing_deterministic != candidate_deterministic:
              raise SystemExit(
                  "Refusing to replace an existing feature-bundle version with different runtime bytes"
              )

          # Verify every object committed by the remote manifest before treating
          # an interrupted/retried publication as a no-op. Raw performance and
          # SBOM metadata may differ between valid reruns, so their candidate
          # hashes are deliberately not used as the immutable identity.
          remote_files = {}
          for entry in existing_manifest["files"]:
              remote_file = Path(hf_hub_download(
                  repo_id,
                  f"v{version}/{entry['file']}",
                  repo_type="model",
                  revision=snapshot_revision,
                  token=token,
              ))
              if remote_file.stat().st_size != entry["size"]:
                  raise SystemExit(f"Existing bundle object has wrong size: {entry['file']}")
              if file_sha256(remote_file) != entry["sha256"]:
                  raise SystemExit(f"Existing bundle object has wrong digest: {entry['file']}")
              remote_files[entry["file"]] = remote_file

          remote_index_path = remote_files.get("v3/ocr-runtime-index.json")
          if remote_index_path is None:
              raise SystemExit("Existing release has no signed OCR runtime index")
          remote_index_raw = remote_index_path.read_bytes()
          remote_index = json.loads(remote_index_raw)
          if remote_index_raw != canonical(remote_index):
              raise SystemExit("Existing OCR runtime index is not canonical JSON")
          candidate_index_path = (
              Path("/tmp/upload") / f"v{version}" / "v3" / "ocr-runtime-index.json"
          )
          candidate_index_raw = candidate_index_path.read_bytes()
          candidate_index = json.loads(candidate_index_raw)
          if candidate_index_raw != canonical(candidate_index):
              raise SystemExit("Candidate OCR runtime index is not canonical JSON")
          remote_signature = remote_index.get("signature")
          if (
              not isinstance(remote_signature, dict)
              or remote_signature.get("algorithm") != "ed25519"
              or remote_signature.get("keyId") != os.environ["OCR_RUNTIME_INDEX_KEY_ID"]
          ):
              raise SystemExit("Existing OCR runtime index has an untrusted signature identity")
          signed_payload = dict(remote_index)
          signed_payload.pop("signature", None)
          with tempfile.TemporaryDirectory() as temporary:
              root = Path(temporary)
              payload_path = root / "index.json"
              signature_path = root / "index.sig"
              public_key_path = root / "public.pem"
              payload_path.write_bytes(canonical(signed_payload))
              signature_path.write_bytes(base64.b64decode(remote_signature.get("value", ""), validate=True))
              public_key_path.write_bytes(
                  base64.b64decode(os.environ["OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64"], validate=True)
              )
              subprocess.run(
                  [
                      "openssl", "pkeyutl", "-verify", "-rawin", "-pubin",
                      "-inkey", str(public_key_path), "-in", str(payload_path),
                      "-sigfile", str(signature_path),
                  ],
                  check=True,
                  stdout=subprocess.DEVNULL,
              )

          if remote_index.get("artifacts") != candidate_index.get("artifacts"):
              raise SystemExit("Existing release contains different OCR runtime artifacts")
          remote_attestations = remote_index.get("attestations")
          candidate_attestations = candidate_index.get("attestations")
          if not isinstance(remote_attestations, list) or not isinstance(candidate_attestations, list):
              raise SystemExit("OCR runtime index attestation set is malformed")
          if not all(isinstance(entry, dict) for entry in [*remote_attestations, *candidate_attestations]):
              raise SystemExit("OCR runtime index attestation record is malformed")
          remote_attestation_names = {entry.get("file") for entry in remote_attestations}
          candidate_attestation_names = {entry.get("file") for entry in candidate_attestations}
          if (
              len(remote_attestation_names) != len(remote_attestations)
              or len(candidate_attestation_names) != len(candidate_attestations)
              or remote_attestation_names != candidate_attestation_names
          ):
              raise SystemExit("Existing release has a different OCR attestation set")
          for attestation in remote_attestations:
              entry = existing_files.get(f"v3/{attestation.get('file')}")
              if (
                  entry is None
                  or entry.get("sha256") != attestation.get("sha256")
                  or entry.get("size") != attestation.get("size")
              ):
                  raise SystemExit(
                      f"Existing signed OCR attestation is not bound by the manifest: {attestation.get('file')}"
                  )

          current_revision = api.repo_info(
              repo_id, repo_type="model", revision="main", token=token
          ).sha
          if current_revision != snapshot_revision:
              raise SystemExit("Hugging Face repository changed during release audit")
          with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as handle:
              handle.write("BUNDLE_RELEASE_EXISTS=true\n")
          PY

      - name: Upload only after every verification and signing gate passed
        if: env.BUNDLE_RELEASE_EXISTS != 'true'
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          RELEASE_COMMIT: ${{ inputs.release_commit }}
          VERSION: ${{ inputs.version }}
        run: |
          /tmp/hf-release-venv/bin/python - <<'PY'
          import hashlib
          import json
          import os
          import subprocess
          from pathlib import Path, PurePosixPath

          from huggingface_hub import HfApi, hf_hub_download

          repo_id = "deepsafe/feature-bundles"
          token = os.environ["HF_TOKEN"]
          version = os.environ["VERSION"]
          parent_commit = os.environ["HF_PARENT_COMMIT"]
          if (
              len(parent_commit) != 40
              or any(character not in "0123456789abcdef" for character in parent_commit)
          ):
              raise SystemExit("Hugging Face parent revision is not an immutable commit")

          root = Path("/tmp/upload") / f"v{version}"
          manifest_path = root / "manifest.json"
          manifest_raw = manifest_path.read_bytes()
          manifest = json.loads(manifest_raw)

          def canonical(value):
              return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()

          def file_sha256(path):
              digest = hashlib.sha256()
              with path.open("rb") as handle:
                  for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                      digest.update(chunk)
              return digest.hexdigest()

          if (
              manifest_raw != canonical(manifest)
              or manifest.get("schemaVersion") != 2
              or manifest.get("version") != version
              or not isinstance(manifest.get("files"), list)
          ):
              raise SystemExit("Candidate feature-bundle manifest has an invalid identity")
          expected_files = {}
          for entry in manifest["files"]:
              relative_path = entry.get("file") if isinstance(entry, dict) else None
              parsed_path = PurePosixPath(relative_path) if isinstance(relative_path, str) else None
              if (
                  not isinstance(entry, dict)
                  or not isinstance(relative_path, str)
                  or parsed_path is None
                  or parsed_path.is_absolute()
                  or ".." in parsed_path.parts
                  or not relative_path.startswith("v3/")
                  or relative_path in expected_files
                  or not isinstance(entry.get("sha256"), str)
                  or len(entry["sha256"]) != 64
                  or any(
                      character not in "0123456789abcdef"
                      for character in entry["sha256"]
                  )
                  or not isinstance(entry.get("size"), int)
                  or entry["size"] <= 0
              ):
                  raise SystemExit("Candidate feature-bundle manifest has malformed files")
              expected_files[relative_path] = entry
          disk_files = {
              path.relative_to(root).as_posix()
              for path in root.rglob("*")
              if path.is_file() and path != manifest_path
          }
          if disk_files != set(expected_files):
              raise SystemExit("Candidate feature-bundle manifest closure mismatch")

          publish_ref = "refs/snapotter/hf-publish-boundary-tag"
          subprocess.run(
              [
                  "git", "fetch", "--force", "--no-tags", "origin",
                  f"refs/tags/v{version}:{publish_ref}",
              ],
              check=True,
              stdout=subprocess.DEVNULL,
          )
          published_tag_commit = subprocess.check_output(
              ["git", "rev-parse", f"{publish_ref}^{{commit}}"], text=True
          ).strip()
          checked_out_commit = subprocess.check_output(
              ["git", "rev-parse", "HEAD"], text=True
          ).strip()
          if (
              published_tag_commit != os.environ["RELEASE_COMMIT"]
              or checked_out_commit != os.environ["RELEASE_COMMIT"]
          ):
              raise SystemExit(
                  "Release tag changed before Hugging Face publication"
              )

          api = HfApi(token=token)
          commit_info = api.upload_folder(
              repo_id=repo_id,
              folder_path=root,
              path_in_repo=f"v{version}",
              commit_message=f"Publish immutable feature bundles v{version}",
              token=token,
              repo_type="model",
              revision="main",
              parent_commit=parent_commit,
          )
          published_revision = commit_info.oid
          if (
              not isinstance(published_revision, str)
              or len(published_revision) != 40
              or any(
                  character not in "0123456789abcdef"
                  for character in published_revision
              )
          ):
              raise SystemExit("Hugging Face upload returned an invalid commit")
          head_revision = api.repo_info(
              repo_id, repo_type="model", revision="main", token=token
          ).sha
          if head_revision != published_revision:
              raise SystemExit("Hugging Face repository changed during post-upload audit")

          remote_prefix = f"v{version}/"
          remote_files = {
              path.removeprefix(remote_prefix)
              for path in api.list_repo_files(
                  repo_id,
                  repo_type="model",
                  revision=published_revision,
                  token=token,
              )
              if path.startswith(remote_prefix)
          }
          if remote_files != {"manifest.json", *expected_files}:
              raise SystemExit("Published feature-bundle closure mismatch")
          published_manifest = Path(hf_hub_download(
              repo_id,
              f"v{version}/manifest.json",
              repo_type="model",
              revision=published_revision,
              token=token,
          ))
          if published_manifest.read_bytes() != manifest_raw:
              raise SystemExit("Published feature-bundle manifest bytes mismatch")
          for relative_path, entry in expected_files.items():
              published_file = Path(hf_hub_download(
                  repo_id,
                  f"v{version}/{relative_path}",
                  repo_type="model",
                  revision=published_revision,
                  token=token,
              ))
              if (
                  published_file.stat().st_size != entry["size"]
                  or file_sha256(published_file) != entry["sha256"]
              ):
                  raise SystemExit(
                      f"Published feature-bundle object mismatch: {relative_path}"
                  )
          final_revision = api.repo_info(
              repo_id, repo_type="model", revision="main", token=token
          ).sha
          if final_revision != published_revision:
              raise SystemExit("Hugging Face repository changed during post-upload audit")
          print(
              f"Published: https://huggingface.co/{repo_id}/tree/{published_revision}/v{version}"
          )
          PY

      - name: Report identical existing bundle release
        if: env.BUNDLE_RELEASE_EXISTS == 'true'
        env:
          VERSION: ${{ inputs.version }}
        run: echo "Verified immutable existing bundle release v${VERSION}; upload skipped"

  installed-ai-production:
    name: Verify installed AI in production image
    runs-on: ubuntu-latest
    timeout-minutes: 180
    permissions:
      actions: read
      contents: read
      packages: read
    steps:
      - name: Resolve a successful scanned release run
        id: source
        if: inputs.release_commit == ''
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SOURCE_RUN_ID: ${{ inputs.source_run_id }}
        run: |
          set -euo pipefail
          if [[ -n "${SOURCE_RUN_ID}" ]]; then
            [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]] || {
              echo "::error::source_run_id must be a positive workflow run ID"
              exit 1
            }
            candidate_run_ids=("${SOURCE_RUN_ID}")
          else
            gh api \
              "repos/${GITHUB_REPOSITORY}/actions/workflows/release.yml/runs?status=completed&per_page=50" \
              > "${RUNNER_TEMP}/release-runs.json"
            mapfile -t candidate_run_ids < <(
              jq -r '.workflow_runs[] | select(.conclusion == "success") | .id' \
                "${RUNNER_TEMP}/release-runs.json"
            )
          fi

          selected_run=""
          for candidate_run_id in "${candidate_run_ids[@]}"; do
            gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate_run_id}" \
              > "${RUNNER_TEMP}/candidate-run.json"
            if ! jq -e \
              '.path == ".github/workflows/release.yml"
                and .status == "completed"
                and .conclusion == "success"' \
              "${RUNNER_TEMP}/candidate-run.json" >/dev/null; then
              if [[ -n "${SOURCE_RUN_ID}" ]]; then
                echo "::error::Selected run is not a completed successful Release workflow"
                exit 1
              fi
              continue
            fi

            gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate_run_id}/artifacts?per_page=100" \
              > "${RUNNER_TEMP}/candidate-artifacts.json"
            artifact_count="$(
              jq '[.artifacts[] | select(.name == "digests-linux-amd64" and .expired == false)] | length' \
                "${RUNNER_TEMP}/candidate-artifacts.json"
            )"
            if [[ "${artifact_count}" != "1" ]]; then
              if [[ -n "${SOURCE_RUN_ID}" ]]; then
                echo "::error::Selected release run does not have one live amd64 digest artifact"
                exit 1
              fi
              continue
            fi

            gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate_run_id}/jobs?per_page=100" \
              > "${RUNNER_TEMP}/candidate-jobs.json"
            if ! jq -e \
              '([.jobs[] | select(.name == "Trivy Container Scan (linux-amd64)")] | length) == 1
                and ([.jobs[] | select(.name == "Trivy Container Scan (linux-amd64)")][0].conclusion == "success")' \
              "${RUNNER_TEMP}/candidate-jobs.json" >/dev/null; then
              if [[ -n "${SOURCE_RUN_ID}" ]]; then
                echo "::error::Selected release run has no successful amd64 container scan"
                exit 1
              fi
              continue
            fi

            selected_run="${candidate_run_id}"
            break
          done
          [[ -n "${selected_run}" ]] || {
            echo "::error::No successful Release run has a live, scanned amd64 digest artifact"
            exit 1
          }
          echo "run_id=${selected_run}" >> "${GITHUB_OUTPUT}"

      - name: Download current release amd64 digest
        if: inputs.release_commit != ''
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-linux-amd64
          path: ${{ runner.temp }}/installed-ai-digest

      - name: Download historical release amd64 digest
        if: inputs.release_commit == ''
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: digests-linux-amd64
          path: ${{ runner.temp }}/installed-ai-digest
          github-token: ${{ secrets.GITHUB_TOKEN }}
          repository: ${{ github.repository }}
          run-id: ${{ steps.source.outputs.run_id }}

      - name: Resolve the exact image digest
        run: |
          set -euo pipefail
          mapfile -t digest_files < <(
            find "${RUNNER_TEMP}/installed-ai-digest" -mindepth 1 -maxdepth 1 -type f -print
          )
          [[ ${#digest_files[@]} -eq 1 ]] || {
            echo "::error::Expected exactly one amd64 image digest"
            exit 1
          }
          IMAGE_DIGEST="$(basename "${digest_files[0]}")"
          [[ "${IMAGE_DIGEST}" =~ ^[a-f0-9]{64}$ ]] || {
            echo "::error::amd64 image artifact contains an invalid digest"
            exit 1
          }
          [[ ! -s "${digest_files[0]}" ]] || {
            echo "::error::amd64 digest marker unexpectedly contains mutable payload data"
            exit 1
          }
          echo "IMAGE_DIGEST=${IMAGE_DIGEST}" >> "${GITHUB_ENV}"

      - name: Reclaim unused hosted-runner SDK space
        run: |
          sudo rm -rf \
            /usr/share/dotnet \
            /usr/local/lib/android \
            /opt/ghc \
            /usr/local/share/boost \
            /opt/hostedtoolcache/CodeQL

      - name: Require enough free disk for the image and installed bundles
        run: |
          set -euo pipefail
          docker_root_dir="$(docker info --format '{{.DockerRootDir}}')"
          [[ "${docker_root_dir}" == /* ]] || {
            echo "::error::Docker reported an invalid storage root"
            exit 1
          }
          available_kib="$(sudo df -Pk "${docker_root_dir}" | awk 'NR == 2 { print $4 }')"
          [[ "${available_kib}" =~ ^[0-9]+$ ]] || {
            echo "::error::Could not measure runner disk availability"
            exit 1
          }
          # background-removal's 4.8 GB archive reserves roughly 18.4 GiB in
          # /data by itself; keep room for the image, venv, and transcription.
          minimum_kib=$((30 * 1024 * 1024))
          (( available_kib >= minimum_kib )) || {
            echo "::error::Installed-AI verification requires 30 GiB free; runner has $((available_kib / 1024 / 1024)) GiB"
            exit 1
          }
          sudo df -h "${docker_root_dir}"

      - name: Log in to GitHub Container Registry
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Pull and validate the exact amd64 release image
        id: image
        env:
          RELEASE_COMMIT: ${{ inputs.release_commit }}
        run: |
          set -euo pipefail
          image="ghcr.io/snapotter-hq/snapotter@sha256:${IMAGE_DIGEST}"
          docker pull "${image}"
          architecture="$(docker image inspect --format '{{.Architecture}}' "${image}")"
          source_repository="$(
            docker image inspect \
              --format '{{ index .Config.Labels "org.opencontainers.image.source" }}' \
              "${image}"
          )"
          source_commit="$(
            docker image inspect \
              --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' \
              "${image}"
          )"
          [[ "${architecture}" == "amd64" ]] || {
            echo "::error::Digest resolved to ${architecture}, not amd64"
            exit 1
          }
          [[ "${source_repository}" == "https://github.com/${GITHUB_REPOSITORY}" ]] || {
            echo "::error::Release image source label does not match this repository"
            exit 1
          }
          [[ "${source_commit}" =~ ^[a-f0-9]{40}$ ]] || {
            echo "::error::Release image has no immutable source revision"
            exit 1
          }
          if [[ -n "${RELEASE_COMMIT}" && "${source_commit}" != "${RELEASE_COMMIT}" ]]; then
            echo "::error::Release image revision differs from the requested release commit"
            exit 1
          fi
          echo "source_commit=${source_commit}" >> "${GITHUB_OUTPUT}"

      - name: Checkout the immutable QA harness revision
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          ref: ${{ github.workflow_sha }}
          fetch-depth: 1
          persist-credentials: false

      - name: Verify immutable QA harness checkout
        env:
          WORKFLOW_SHA: ${{ github.workflow_sha }}
        run: |
          [[ "${WORKFLOW_SHA}" =~ ^[a-f0-9]{40}$ \
            && "$(git rev-parse HEAD)" == "${WORKFLOW_SHA}" ]] || {
              echo "::error::QA harness checkout differs from the workflow revision"
              exit 1
            }

      - uses: ./.github/actions/setup

      - name: Create exact labeled production resources
        run: |
          set -euo pipefail
          RUN_SCOPE="snapotter-ai-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
          [[ "${RUN_SCOPE}" =~ ^snapotter-ai-[0-9]+-[0-9]+$ ]] || {
            echo "::error::Unsafe installed-AI run scope"
            exit 1
          }
          QA_PASSWORD="$(openssl rand -hex 24)"
          echo "::add-mask::${QA_PASSWORD}"
          echo "RUN_SCOPE=${RUN_SCOPE}" >> "${GITHUB_ENV}"
          echo "QA_PASSWORD=${QA_PASSWORD}" >> "${GITHUB_ENV}"

          docker network create \
            --label "io.snapotter.qa.scope=${RUN_SCOPE}" \
            "${RUN_SCOPE}-network"
          for suffix in data workspace pgdata redisdata; do
            docker volume create \
              --label "io.snapotter.qa.scope=${RUN_SCOPE}" \
              "${RUN_SCOPE}-${suffix}"
          done

          docker run --detach \
            --name "${RUN_SCOPE}-postgres" \
            --label "io.snapotter.qa.scope=${RUN_SCOPE}" \
            --network "${RUN_SCOPE}-network" \
            --network-alias postgres \
            --env POSTGRES_USER=snapotter \
            --env "POSTGRES_PASSWORD=${QA_PASSWORD}" \
            --env POSTGRES_DB=snapotter \
            --volume "${RUN_SCOPE}-pgdata:/var/lib/postgresql/data" \
            postgres:17-alpine@sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193

          docker run --detach \
            --name "${RUN_SCOPE}-redis" \
            --label "io.snapotter.qa.scope=${RUN_SCOPE}" \
            --network "${RUN_SCOPE}-network" \
            --network-alias redis \
            --volume "${RUN_SCOPE}-redisdata:/data" \
            redis:8-alpine@sha256:9d317178eceac8454a2284a9e6df2466b93c745529947f0cd42a0fa9609d7005 \
            redis-server --maxmemory-policy noeviction --maxmemory 512mb \
              --appendonly yes --requirepass "${QA_PASSWORD}"

          dependencies_ready=false
          for _attempt in $(seq 1 90); do
            if docker exec "${RUN_SCOPE}-postgres" \
              pg_isready -U snapotter -d snapotter >/dev/null 2>&1 \
              && docker exec "${RUN_SCOPE}-redis" \
                redis-cli -a "${QA_PASSWORD}" --no-auth-warning ping 2>/dev/null \
                | grep -qx PONG; then
              dependencies_ready=true
              break
            fi
            sleep 2
          done
          [[ "${dependencies_ready}" == "true" ]] || {
            echo "::error::Production dependencies did not become ready"
            exit 1
          }

          docker run --detach \
            --name "${RUN_SCOPE}-app" \
            --label "io.snapotter.qa.scope=${RUN_SCOPE}" \
            --network "${RUN_SCOPE}-network" \
            --publish 127.0.0.1::1349 \
            --env AUTH_ENABLED=true \
            --env DEFAULT_USERNAME=admin \
            --env "DEFAULT_PASSWORD=${QA_PASSWORD}" \
            --env SKIP_MUST_CHANGE_PASSWORD=true \
            --env SNAPOTTER_GPU=0 \
            --env "DATABASE_URL=postgres://snapotter:${QA_PASSWORD}@postgres:5432/snapotter" \
            --env "REDIS_URL=redis://:${QA_PASSWORD}@redis:6379" \
            --env PROCESSING_TIMEOUT_S=0 \
            --env RATE_LIMIT_PER_MIN=1000 \
            --volume "${RUN_SCOPE}-data:/data" \
            --volume "${RUN_SCOPE}-workspace:/tmp/workspace" \
            "ghcr.io/snapotter-hq/snapotter@sha256:${IMAGE_DIGEST}"

          mapfile -t port_bindings < <(docker port "${RUN_SCOPE}-app" 1349/tcp)
          [[ ${#port_bindings[@]} -eq 1 \
            && "${port_bindings[0]}" =~ ^127\.0\.0\.1:([1-9][0-9]{0,4})$ ]] || {
              echo "::error::Production app did not publish one loopback-only dynamic port"
              exit 1
            }
          QA_PORT="${BASH_REMATCH[1]}"
          (( QA_PORT <= 65535 )) || {
            echo "::error::Docker returned an invalid published port"
            exit 1
          }
          echo "QA_BASE_URL=http://127.0.0.1:${QA_PORT}" >> "${GITHUB_ENV}"

      - name: Wait for the production API
        run: |
          set -euo pipefail
          api_ready=false
          for _attempt in $(seq 1 120); do
            if curl --fail --silent --show-error --max-time 5 \
              "${QA_BASE_URL}/api/v1/health" \
              | jq -e '.status == "healthy"' >/dev/null; then
              api_ready=true
              break
            fi
            if ! docker inspect --format '{{.State.Running}}' "${RUN_SCOPE}-app" \
              | grep -qx true; then
              echo "::error::Production app container exited before becoming healthy"
              docker logs "${RUN_SCOPE}-app" || true
              exit 1
            fi
            sleep 3
          done
          [[ "${api_ready}" == "true" ]] || {
            echo "::error::Production API did not become healthy"
            docker logs "${RUN_SCOPE}-app" || true
            exit 1
          }

      - name: Install bundles and verify four public tool artifacts
        env:
          QA_USERNAME: admin
        run: ./apps/api/node_modules/.bin/tsx tests/qa/verify-installed-ai-production.mts

      - name: Show exact production-container diagnostics
        if: ${{ failure() }}
        run: |
          docker logs "${RUN_SCOPE}-app" || true
          docker logs "${RUN_SCOPE}-postgres" || true
          docker logs "${RUN_SCOPE}-redis" || true

      - name: Remove only exact labeled run resources
        if: ${{ always() }}
        run: |
          set -u
          cleanup_failed=0
          if [[ "${RUN_SCOPE:-}" =~ ^snapotter-ai-[0-9]+-[0-9]+$ ]]; then
            if docker container inspect "${RUN_SCOPE}-app" >/dev/null 2>&1; then
              app_scope="$(
                docker container inspect \
                  --format '{{ index .Config.Labels "io.snapotter.qa.scope" }}' \
                  "${RUN_SCOPE}-app"
              )"
              if [[ "${app_scope}" == "${RUN_SCOPE}" ]]; then
                docker rm --force "${RUN_SCOPE}-app" || cleanup_failed=1
              else
                echo "::error::Refusing to remove app container without the exact run label"
                cleanup_failed=1
              fi
            fi
            for suffix in postgres redis; do
              name="${RUN_SCOPE}-${suffix}"
              if docker container inspect "${name}" >/dev/null 2>&1; then
                scope="$(
                  docker container inspect \
                    --format '{{ index .Config.Labels "io.snapotter.qa.scope" }}' \
                    "${name}"
                )"
                if [[ "${scope}" == "${RUN_SCOPE}" ]]; then
                  docker rm --force "${name}" || cleanup_failed=1
                else
                  echo "::error::Refusing to remove ${name} without the exact run label"
                  cleanup_failed=1
                fi
              fi
            done

            if docker volume inspect "${RUN_SCOPE}-data" >/dev/null 2>&1; then
              data_scope="$(
                docker volume inspect \
                  --format '{{ index .Labels "io.snapotter.qa.scope" }}' \
                  "${RUN_SCOPE}-data"
              )"
              if [[ "${data_scope}" == "${RUN_SCOPE}" ]]; then
                docker volume rm "${RUN_SCOPE}-data" || cleanup_failed=1
              else
                echo "::error::Refusing to remove data volume without the exact run label"
                cleanup_failed=1
              fi
            fi
            for suffix in workspace pgdata redisdata; do
              name="${RUN_SCOPE}-${suffix}"
              if docker volume inspect "${name}" >/dev/null 2>&1; then
                scope="$(
                  docker volume inspect \
                    --format '{{ index .Labels "io.snapotter.qa.scope" }}' \
                    "${name}"
                )"
                if [[ "${scope}" == "${RUN_SCOPE}" ]]; then
                  docker volume rm "${name}" || cleanup_failed=1
                else
                  echo "::error::Refusing to remove ${name} without the exact run label"
                  cleanup_failed=1
                fi
              fi
            done

            if docker network inspect "${RUN_SCOPE}-network" >/dev/null 2>&1; then
              network_scope="$(
                docker network inspect \
                  --format '{{ index .Labels "io.snapotter.qa.scope" }}' \
                  "${RUN_SCOPE}-network"
              )"
              if [[ "${network_scope}" == "${RUN_SCOPE}" ]]; then
                docker network rm "${RUN_SCOPE}-network" || cleanup_failed=1
              else
                echo "::error::Refusing to remove network without the exact run label"
                cleanup_failed=1
              fi
            fi
          elif [[ -n "${RUN_SCOPE:-}" ]]; then
            echo "::error::Refusing cleanup for an unsafe run scope"
            cleanup_failed=1
          fi
          exit "${cleanup_failed}"