SnapOtter/.github/workflows/nightly.yml

name: Nightly

on:
  workflow_dispatch:
  schedule:
    - cron: "0 3 * * *"

permissions:
  contents: read

env:
  # Keep in sync with ci.yml's install step. tesseract-ocr + language packs are
  # required by the built-in Fast OCR tier: without them the OCR integration
  # tests throw "spawn tesseract ENOENT" and fail the coverage/matrix jobs.
  SYSTEM_DEPS: libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl libraw-bin imagemagick ghostscript libjxl-tools libopenjp2-tools ffmpeg qpdf libreoffice-calc libreoffice-impress libreoffice-writer tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra tesseract-ocr-spa tesseract-ocr-chi-sim tesseract-ocr-jpn
  # See ci.yml: ryuk's Docker Hub pull is a recurring flake source; disable the reaper
  # (tests/global-setup.ts stops its containers explicitly; runners are ephemeral).
  TESTCONTAINERS_RYUK_DISABLED: "true"

jobs:
  e2e-full:
    name: E2E Full (${{ matrix.shard }}/4)
    runs-on: ubuntu-latest
    timeout-minutes: 60
    services:
      postgres:
        image: postgres:17-alpine
        env:
          POSTGRES_USER: snapotter
          POSTGRES_PASSWORD: snapotter
          POSTGRES_DB: snapotter
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U snapotter"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
      redis:
        image: redis:8-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - uses: ./.github/actions/setup
      - name: Get Playwright version
        id: pw-version
        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
      - name: Cache Playwright browsers
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/ms-playwright
          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
      - name: Install Playwright Chromium
        run: pnpm playwright install --with-deps chromium
      - name: Run full e2e suite (shard)
        run: pnpm playwright test --project=chromium --shard=${{ matrix.shard }}/4
        env:
          PW_WORKERS: "2"
      - name: Upload report on failure
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: e2e-full-report-shard-${{ matrix.shard }}
          path: playwright-report/
          retention-days: 7

  e2e-serial:
    name: E2E Serial Bucket
    runs-on: ubuntu-latest
    timeout-minutes: 60
    services:
      postgres:
        image: postgres:17-alpine
        env:
          POSTGRES_USER: snapotter
          POSTGRES_PASSWORD: snapotter
          POSTGRES_DB: snapotter
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U snapotter"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
      redis:
        image: redis:8-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - uses: ./.github/actions/setup
      - name: Get Playwright version
        id: pw-version
        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
      - name: Cache Playwright browsers
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/ms-playwright
          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
      - name: Install Playwright Chromium
        run: pnpm playwright install --with-deps chromium
      - name: Run serial bucket (global-state specs)
        run: pnpm playwright test --project=chromium-serial --workers=1
      # chromium-widths owns the exact CSS boundary and wide-screen assertions.
      # It is five plain functional specs with no screenshot baselines, so it
      # runs here rather than needing the platform-suffixed visual machinery.
      - name: Run viewport-width bucket
        run: pnpm playwright test --project=chromium-widths --workers=1
      - name: Upload report on failure
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: e2e-serial-report
          path: playwright-report/
          retention-days: 7

  # The editor, the analytics opt-out flow and the AUTH_ENABLED=false mode each
  # have their own Playwright config and, until now, no workflow at all: 174 + 9
  # + 5 tests that collected and passed locally but gated nothing. All three need
  # Postgres and Redis and build the web app themselves, so they share a runner.
  e2e-surfaces:
    name: E2E Editor + Analytics Opt-Out + No-Auth
    runs-on: ubuntu-latest
    # The editor suite is 174 tests pinned to workers: 1 by its own config and
    # took 23.4 minutes on a loaded dev machine, so leave real headroom for it
    # plus the apt install, the browser download and the two smaller suites.
    timeout-minutes: 60
    services:
      postgres:
        image: postgres:17-alpine
        env:
          POSTGRES_USER: snapotter
          POSTGRES_PASSWORD: snapotter
          POSTGRES_DB: snapotter
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U snapotter"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
      redis:
        image: redis:8-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - uses: ./.github/actions/setup
      - name: Get Playwright version
        id: pw-version
        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
      - name: Cache Playwright browsers
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/ms-playwright
          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
      - name: Install Playwright Chromium
        run: pnpm playwright install --with-deps chromium
      - name: Run editor suite
        run: pnpm playwright test --config playwright.editor.config.ts
        env:
          PW_WORKERS: "2"
      - name: Run analytics opt-out suite
        # The in-repo docker/feature-manifest.json makes the API believe it is
        # inside a container and mkdir /data; this config does not pin DATA_DIR
        # itself, so pin it here the way playwright.config.ts does.
        run: pnpm test:e2e:analytics
        env:
          DATA_DIR: ${{ runner.temp }}/analytics-local-data
      - name: Run no-auth mode suite
        # AUTH_ENABLED=false injects a synthetic anonymous admin. Nothing else in
        # CI exercises that path, so a regression only surfaced on a user's box.
        run: pnpm test:e2e:noauth
      - name: Upload report on failure
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: e2e-surfaces-report
          path: playwright-report/
          retention-days: 7

  e2e-cross-browser:
    name: E2E Cross-Browser (Firefox + WebKit)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    services:
      postgres:
        image: postgres:17-alpine
        env:
          POSTGRES_USER: snapotter
          POSTGRES_PASSWORD: snapotter
          POSTGRES_DB: snapotter
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U snapotter"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
      redis:
        image: redis:8-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: ./.github/actions/setup
      - name: Get Playwright version
        id: pw-version
        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
      - name: Cache Playwright browsers
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/ms-playwright
          key: playwright-all-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
      - name: Install Playwright browsers
        run: pnpm playwright install --with-deps chromium firefox webkit
      - name: Run cross-browser spec
        run: pnpm playwright test --project=firefox --project=webkit
        env:
          PW_WORKERS: "1"
      - name: Upload report on failure
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: e2e-cross-browser-report
          path: playwright-report/
          retention-days: 7

  e2e-device-matrix:
    name: E2E Device Matrix (mobile + tablet)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    services:
      postgres:
        image: postgres:17-alpine
        env:
          POSTGRES_USER: snapotter
          POSTGRES_PASSWORD: snapotter
          POSTGRES_DB: snapotter
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U snapotter"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
      redis:
        image: redis:8-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - uses: ./.github/actions/setup
      - name: Get Playwright version
        id: pw-version
        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
      - name: Cache Playwright browsers
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/ms-playwright
          key: playwright-all-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
      - name: Install Playwright browsers (chromium + webkit)
        run: pnpm playwright install --with-deps chromium webkit
      - name: Run all device projects
        run: pnpm playwright test --project=mobile-chromium --project=mobile-webkit --project=tablet-webkit --project=tablet-chromium --grep-invert "@visual"
        env:
          PW_WORKERS: "2"
      - name: Upload report on failure
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: e2e-device-matrix-report
          path: playwright-report/
          retention-days: 7

  docker-e2e:
    name: Docker Container E2E
    runs-on: ubuntu-latest
    # A from-scratch image build (~14 min) plus the full in-container suite runs
    # ~65 min, so the old 60-minute budget cancelled the job before it finished.
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Free disk space
        # The from-scratch image is ~15 GB and the build layers add more on top
        # of three running containers, which crowds a standard runner and has
        # correlated with the VM being terminated mid-run (exit 137). Reclaim the
        # preinstalled toolchains this job never uses before building.
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /opt/hostedtoolcache/CodeQL /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true
          sudo docker image prune -af || true
          df -h /
      - name: Run container test suite
        # Attribute the job to test-unit: it runs the full integration suite
        # against the container's real binaries (ffmpeg/qpdf/tesseract/...), which
        # is the in-container validation this job exists for. --exit-code-from
        # test-e2e never worked: --abort-on-container-exit (which it implies) kills
        # test-e2e the moment test-unit exits, so test-e2e's SIGKILL (137) became
        # the result. Browser e2e is covered by the E2E Full/Serial/Cross-Browser
        # jobs; running test-e2e in-container too is a separate follow-up (it also
        # would not fit this job's 60-minute budget alongside test-unit).
        run: docker compose -f docker/docker-compose.test.yml up --build --exit-code-from test-unit

  extended-matrix:
    name: Extended Matrix + Fuzz (${{ matrix.shard }}/4)
    runs-on: ubuntu-latest
    # The full format x tool matrix plus property fuzz overran a single 90-min
    # job. Shard it four ways (mirroring E2E Full) so each runner takes a quarter
    # of the integration files; the generous per-shard ceiling absorbs the fact
    # that the heavy format-matrix-* files cluster onto one shard.
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
        run: |
          POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
          if [ -n "$POLICY_FILE" ]; then
            sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
            sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
          fi
      - uses: ./.github/actions/doc-binaries
      - uses: ./.github/actions/setup
      - name: Pre-pull testcontainer images (retry transient Docker Hub errors)
        run: |
          for img in postgres:17-alpine redis:8-alpine; do
            ok=
            for attempt in 1 2 3 4 5; do
              if docker pull "$img"; then ok=1; break; fi
              echo "::warning::docker pull $img failed (attempt $attempt/5); retrying in $((attempt * 5))s"
              sleep "$((attempt * 5))"
            done
            [ -n "$ok" ] || { echo "::error::could not pull $img after 5 attempts"; exit 1; }
          done
      - name: Run integration suite with full matrix and fuzz enabled
        run: pnpm vitest run tests/integration/ --reporter=verbose --shard=${{ matrix.shard }}/4
        env:
          FULL_MATRIX: "1"
          FUZZ: "1"
          FUZZ_RUNS: "50"
          # This lane installs no AI bundles, so it must not run the strict
          # required-AI contract (that 501s every installed-contract case). The
          # strict contract runs on the release-QA fleet, which has bundles.
          # Full-matrix tests iterate every format x tool; with 4 forks the heavy
          # media conversions starve and hit the 30s default. Fewer forks (more
          # CPU each) plus a generous timeout keeps them from flaking.
          VITEST_MAX_FORKS: "2"
          VITEST_TEST_TIMEOUT: "600000"
          VITEST_HOOK_TIMEOUT: "120000"

  api-fuzz:
    name: Schemathesis API Fuzz
    runs-on: ubuntu-latest
    timeout-minutes: 45
    services:
      postgres:
        image: postgres:17-alpine
        env:
          POSTGRES_USER: snapotter
          POSTGRES_PASSWORD: snapotter
          POSTGRES_DB: snapotter
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U snapotter"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
      redis:
        image: redis:8-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - uses: ./.github/actions/setup
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - name: Install Schemathesis
        run: pip install schemathesis
      - name: Start API server
        run: |
          mkdir -p /tmp/st-data
          AUTH_ENABLED=false ANALYTICS_ENABLED=false SYNC_WAIT_MS=0 \
            DATABASE_URL=postgres://snapotter:snapotter@localhost:5432/snapotter \
            REDIS_URL=redis://localhost:6379 \
            WORKSPACE_PATH=/tmp/st-data/workspace DATA_DIR=/tmp/st-data \
            pnpm --filter @snapotter/api dev &
          for i in $(seq 1 60); do
            if curl -fsS http://localhost:13490/api/v1/health > /dev/null 2>&1; then
              echo "API up after ${i}s"; exit 0
            fi
            sleep 1
          done
          echo "API failed to start"; exit 1
      - name: Fuzz tool endpoints from the OpenAPI spec
        run: |
          # This fuzz targets the async tool-factory endpoints: they validate the
          # request, enqueue a job, and return 202 immediately (the server is
          # started above with SYNC_WAIT_MS=0). Excluded are the feature-gated
          # AI/media tools and the handful of tools with bespoke handlers that
          # process synchronously in-request (meme-generator, collage, stitch,
          # compose, beautify, vectorize, ...). On adversarial input that
          # in-request work is unbounded and trips the request timeout, so those
          # tools are fuzzed by the integration and e2e suites instead.
          #
          # --suppress-health-check all: the tool endpoints need a real uploaded
          # file, so the fuzzer's random bytes get rejected (404) and Hypothesis's
          # filter_too_much check would fail the run. Those checks grade data
          # generation quality, not API correctness; not_a_server_error still runs
          # on every generated case (5000+ per run).
          schemathesis run http://localhost:13490/api/v1/openapi.yaml \
            --url http://localhost:13490 \
            --checks not_a_server_error \
            --suppress-health-check all \
            --request-timeout 120 \
            --include-path-regex "^/api/v1/(tools|health|info)" \
            --exclude-path-regex "/tools/(audio|video)/|/(remove-background|remove-gif-background|upscale|html-to-image|blur-faces|erase-object|ocr|ocr-pdf|colorize|enhance-faces|noise-removal|smart-crop|red-eye-removal|restore-photo|passport-photo|transparency-fixer|ai-canvas-expand|transcribe-audio|auto-subtitles|background-replace|blur-background|meme-generator|compare|stitch|collage|bulk-rename|watermark-image|vectorize|find-duplicates|split|sign-pdf|beautify|favicon|color-palette|compose)(/[a-z-]+)?$" \
            --max-examples 25 \
            --report junit \
            --report-dir st-report
      - name: Upload fuzz report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: schemathesis-report
          path: st-report/
          retention-days: 14

  coverage-report:
    name: Coverage Report
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install system dependencies
        run: |
          read -r -a system_deps <<< "$SYSTEM_DEPS"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends "${system_deps[@]}"
      - name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
        run: |
          POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
          if [ -n "$POLICY_FILE" ]; then
            sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
            sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
          fi
      - uses: ./.github/actions/doc-binaries
      - uses: ./.github/actions/setup
      - name: Run tests with coverage
        run: pnpm vitest run --coverage tests/unit/ tests/integration/
        # Coverage instrumentation slows every test; fewer forks + a longer
        # timeout keep slow media tests (e.g. qr-generate) from timing out.
        env:
          VITEST_MAX_FORKS: "2"
          VITEST_TEST_TIMEOUT: "300000"
          VITEST_HOOK_TIMEOUT: "120000"
      - name: Upload coverage artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-report
          path: coverage/
          retention-days: 14