test: testing overhaul -- CI e2e gates, parallel suites, generated matrices, mutation testing (#215)

Closes the "e2e never runs in CI" hole. Adds per-PR e2e smoke gate,
nightly full-suite workflows, parallel vitest forks (per-fork DBs),
Playwright parallel/serial/visual projects against production builds,
metadata-generated test suites (drift guards, hostile inputs, format
matrix, pairwise settings, property-based fuzz), Stryker mutation
testing, Schemathesis API fuzz, coverage ratchet, and fixes for three
session-poisoning bugs that caused 200+ serial-bucket failures.

Bug fix included: favicon/split/bulk-rename could hang clients forever
when ZIP streaming failed after reply.hijack().

.github/workflows/ci.yml

@@ -83,6 +83,35 @@ jobs:
       - uses: ./.github/actions/setup
       - run: pnpm vitest run tests/integration/ --reporter=verbose --shard=${{ matrix.shard }}/4
 
+  test-e2e-smoke:
+    name: E2E Smoke (Chromium)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        run: pnpm playwright install --with-deps chromium
+      - name: Run smoke specs
+        run: pnpm playwright test tests/e2e/smoke.spec.ts tests/e2e/tools-all.spec.ts tests/e2e/navigation.spec.ts tests/e2e/home-page.spec.ts --project=chromium
+        env:
+          PW_WORKERS: "1"
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-smoke-report
+          path: playwright-report/
+          retention-days: 7
+
   pip-audit:
     name: Python Dependency Audit
     runs-on: ubuntu-latest
@@ -122,7 +151,7 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
-    needs: [lint, typecheck, test-unit, test-integration]
+    needs: [lint, typecheck, test-unit, test-integration, test-e2e-smoke]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: ./.github/actions/setup

.github/workflows/cla.yml

@@ -16,7 +16,9 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       actions: write
-      contents: read
+      # The action writes signatures/cla.json to the cla-signatures branch;
+      # read-only contents made it fail with "Resource not accessible".
+      contents: write
       pull-requests: write
       statuses: write
     steps:

.github/workflows/mutation.yml

@@ -0,0 +1,34 @@
+name: Mutation Testing
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 4 * * 0"
+
+permissions:
+  contents: read
+
+jobs:
+  mutation-image-engine:
+    name: Stryker (image-engine)
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/setup
+      - name: Restore incremental mutation state
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: packages/image-engine/reports/stryker-incremental.json
+          key: stryker-incremental-${{ github.sha }}
+          restore-keys: |
+            stryker-incremental-
+      - name: Run mutation tests
+        run: pnpm --filter @snapotter/image-engine exec stryker run
+      - name: Upload mutation report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: mutation-report-image-engine
+          path: packages/image-engine/reports/mutation/
+          retention-days: 30

.github/workflows/nightly.yml

@@ -0,0 +1,207 @@
+name: Nightly
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 3 * * *"
+
+permissions:
+  contents: read
+
+env:
+  SYSTEM_DEPS: libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl imagemagick ghostscript libjxl-tools libopenjp2-tools
+
+jobs:
+  e2e-full:
+    name: E2E Full (${{ matrix.shard }}/4)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        run: pnpm playwright install --with-deps chromium
+      - name: Run full e2e suite (shard)
+        run: pnpm playwright test --project=chromium --shard=${{ matrix.shard }}/4
+        env:
+          PW_WORKERS: "2"
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-full-report-shard-${{ matrix.shard }}
+          path: playwright-report/
+          retention-days: 7
+
+  e2e-serial:
+    name: E2E Serial Bucket
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        run: pnpm playwright install --with-deps chromium
+      - name: Run serial bucket (global-state specs)
+        run: pnpm playwright test --project=chromium-serial --workers=1
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-serial-report
+          path: playwright-report/
+          retention-days: 7
+
+  e2e-cross-browser:
+    name: E2E Cross-Browser (Firefox + WebKit)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-all-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright browsers
+        run: pnpm playwright install --with-deps chromium firefox webkit
+      - name: Run cross-browser spec
+        run: pnpm playwright test --project=firefox --project=webkit
+        env:
+          PW_WORKERS: "1"
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-cross-browser-report
+          path: playwright-report/
+          retention-days: 7
+
+  docker-e2e:
+    name: Docker Container E2E
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Run container test suite
+        run: docker compose -f docker/docker-compose.test.yml up --build --exit-code-from test-e2e
+
+  extended-matrix:
+    name: Extended Matrix + Fuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
+        run: |
+          POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
+          if [ -n "$POLICY_FILE" ]; then
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
+          fi
+      - uses: ./.github/actions/setup
+      - name: Run integration suite with full matrix and fuzz enabled
+        run: pnpm vitest run tests/integration/ --reporter=verbose
+        env:
+          FULL_MATRIX: "1"
+          FUZZ: "1"
+          FUZZ_RUNS: "50"
+
+  api-fuzz:
+    name: Schemathesis API Fuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - uses: ./.github/actions/setup
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+      - name: Install Schemathesis
+        run: pip install schemathesis
+      - name: Start API server
+        run: |
+          mkdir -p /tmp/st-data
+          AUTH_ENABLED=false ANALYTICS_ENABLED=false DB_PATH=/tmp/st-data/st.db \
+            WORKSPACE_PATH=/tmp/st-data/workspace DATA_DIR=/tmp/st-data \
+            pnpm --filter @snapotter/api dev &
+          for i in $(seq 1 60); do
+            if curl -fsS http://localhost:13490/api/v1/health > /dev/null 2>&1; then
+              echo "API up after ${i}s"; exit 0
+            fi
+            sleep 1
+          done
+          echo "API failed to start"; exit 1
+      - name: Fuzz tool endpoints from the OpenAPI spec
+        run: |
+          schemathesis run http://localhost:13490/api/v1/openapi.yaml \
+            --url http://localhost:13490 \
+            --checks not_a_server_error \
+            --include-path-regex "^/api/v1/(tools|health|info)" \
+            --max-examples 25 \
+            --report junit \
+            --report-dir st-report
+      - name: Upload fuzz report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: schemathesis-report
+          path: st-report/
+          retention-days: 14
+
+  coverage-report:
+    name: Coverage Report
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
+        run: |
+          POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
+          if [ -n "$POLICY_FILE" ]; then
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
+          fi
+      - uses: ./.github/actions/setup
+      - name: Run tests with coverage
+        run: pnpm vitest run --coverage tests/unit/ tests/integration/
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: coverage-report
+          path: coverage/
+          retention-days: 14

.github/workflows/update-visual-baselines.yml

@@ -0,0 +1,52 @@
+name: Update Visual Baselines
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-baselines:
+    name: Regenerate linux visual baselines
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        run: pnpm playwright install --with-deps chromium
+      - name: Regenerate baselines
+        run: pnpm playwright test --project=chromium-visual --update-snapshots
+        env:
+          PW_WORKERS: "2"
+      - name: Open PR with refreshed baselines
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if git diff --quiet -- 'tests/e2e/__screenshots__'; then
+            untracked=$(git ls-files --others --exclude-standard -- 'tests/e2e/__screenshots__')
+            if [ -z "$untracked" ]; then
+              echo "No baseline changes."
+              exit 0
+            fi
+          fi
+          BRANCH="chore/visual-baselines-${GITHUB_RUN_ID}"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add tests/e2e/__screenshots__
+          git commit -m "test(e2e): refresh linux visual baselines"
+          git push origin "$BRANCH"
+          gh pr create --title "test(e2e): refresh linux visual baselines" \
+            --body "Automated baseline refresh from the update-visual-baselines workflow. Review the image diffs before merging." \
+            --base main --head "$BRANCH"