test: testing overhaul -- CI e2e gates, parallel suites, generated matrices, mutation testing (#215)

Closes the "e2e never runs in CI" hole. Adds per-PR e2e smoke gate,
nightly full-suite workflows, parallel vitest forks (per-fork DBs),
Playwright parallel/serial/visual projects against production builds,
metadata-generated test suites (drift guards, hostile inputs, format
matrix, pairwise settings, property-based fuzz), Stryker mutation
testing, Schemathesis API fuzz, coverage ratchet, and fixes for three
session-poisoning bugs that caused 200+ serial-bucket failures.

Bug fix included: favicon/split/bulk-rename could hang clients forever
when ZIP streaming failed after reply.hijack().

.github/workflows/nightly.yml

@@ -0,0 +1,207 @@
+name: Nightly
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 3 * * *"
+
+permissions:
+  contents: read
+
+env:
+  SYSTEM_DEPS: libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl imagemagick ghostscript libjxl-tools libopenjp2-tools
+
+jobs:
+  e2e-full:
+    name: E2E Full (${{ matrix.shard }}/4)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        run: pnpm playwright install --with-deps chromium
+      - name: Run full e2e suite (shard)
+        run: pnpm playwright test --project=chromium --shard=${{ matrix.shard }}/4
+        env:
+          PW_WORKERS: "2"
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-full-report-shard-${{ matrix.shard }}
+          path: playwright-report/
+          retention-days: 7
+
+  e2e-serial:
+    name: E2E Serial Bucket
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright Chromium
+        run: pnpm playwright install --with-deps chromium
+      - name: Run serial bucket (global-state specs)
+        run: pnpm playwright test --project=chromium-serial --workers=1
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-serial-report
+          path: playwright-report/
+          retention-days: 7
+
+  e2e-cross-browser:
+    name: E2E Cross-Browser (Firefox + WebKit)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ./.github/actions/setup
+      - name: Get Playwright version
+        id: pw-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
+      - name: Cache Playwright browsers
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-all-${{ runner.os }}-${{ steps.pw-version.outputs.version }}
+      - name: Install Playwright browsers
+        run: pnpm playwright install --with-deps chromium firefox webkit
+      - name: Run cross-browser spec
+        run: pnpm playwright test --project=firefox --project=webkit
+        env:
+          PW_WORKERS: "1"
+      - name: Upload report on failure
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: failure()
+        with:
+          name: e2e-cross-browser-report
+          path: playwright-report/
+          retention-days: 7
+
+  docker-e2e:
+    name: Docker Container E2E
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Run container test suite
+        run: docker compose -f docker/docker-compose.test.yml up --build --exit-code-from test-e2e
+
+  extended-matrix:
+    name: Extended Matrix + Fuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
+        run: |
+          POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
+          if [ -n "$POLICY_FILE" ]; then
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
+          fi
+      - uses: ./.github/actions/setup
+      - name: Run integration suite with full matrix and fuzz enabled
+        run: pnpm vitest run tests/integration/ --reporter=verbose
+        env:
+          FULL_MATRIX: "1"
+          FUZZ: "1"
+          FUZZ_RUNS: "50"
+
+  api-fuzz:
+    name: Schemathesis API Fuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - uses: ./.github/actions/setup
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+      - name: Install Schemathesis
+        run: pip install schemathesis
+      - name: Start API server
+        run: |
+          mkdir -p /tmp/st-data
+          AUTH_ENABLED=false ANALYTICS_ENABLED=false DB_PATH=/tmp/st-data/st.db \
+            WORKSPACE_PATH=/tmp/st-data/workspace DATA_DIR=/tmp/st-data \
+            pnpm --filter @snapotter/api dev &
+          for i in $(seq 1 60); do
+            if curl -fsS http://localhost:13490/api/v1/health > /dev/null 2>&1; then
+              echo "API up after ${i}s"; exit 0
+            fi
+            sleep 1
+          done
+          echo "API failed to start"; exit 1
+      - name: Fuzz tool endpoints from the OpenAPI spec
+        run: |
+          schemathesis run http://localhost:13490/api/v1/openapi.yaml \
+            --url http://localhost:13490 \
+            --checks not_a_server_error \
+            --include-path-regex "^/api/v1/(tools|health|info)" \
+            --max-examples 25 \
+            --report junit \
+            --report-dir st-report
+      - name: Upload fuzz report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: schemathesis-report
+          path: st-report/
+          retention-days: 14
+
+  coverage-report:
+    name: Coverage Report
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends $SYSTEM_DEPS
+      - name: Allow ImageMagick to read EPS/PS via Ghostscript delegate
+        run: |
+          POLICY_FILE=$(find /etc/ImageMagick* -name policy.xml 2>/dev/null | head -1)
+          if [ -n "$POLICY_FILE" ]; then
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="EPS"/<policy domain="coder" rights="read" pattern="EPS"/' "$POLICY_FILE"
+            sudo sed -i 's/<policy domain="coder" rights="none" pattern="PS"/<policy domain="coder" rights="read" pattern="PS"/' "$POLICY_FILE"
+          fi
+      - uses: ./.github/actions/setup
+      - name: Run tests with coverage
+        run: pnpm vitest run --coverage tests/unit/ tests/integration/
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: coverage-report
+          path: coverage/
+          retention-days: 14