From 33dfcecd6a62cd432a29297c08b237a47b142a86 Mon Sep 17 00:00:00 2001 From: SnapOtter Date: Wed, 24 Jun 2026 18:23:23 +0800 Subject: [PATCH] test(nightly): stabilize the exhaustive nightly suite (#345) Triaged the nightly failures (all pre-existing, unrelated to the analytics work) and fixed the ones with clear root causes: - video-speed: a 1s tiny.mp4 sped up 2x rounds to ~0.75s, flaking the +/-25% duration assertion under heavy CI load. Use the 8s hero.mp4 (still 44.1kHz) so rounding is negligible. Verified locally. - Extended Matrix + Coverage timeouts: full-matrix / coverage-instrumented runs starve the heavy media tests under 4 forks at the 30s default. Make maxForks env-overridable (VITEST_MAX_FORKS) and run those jobs with 2 forks + a 300s timeout so format-matrix conversions and qr-generate stop timing out. - Device Matrix visual baselines: the update-visual-baselines workflow could not start the app ('failed to create database') because it never provisioned Postgres/Redis. Add the same services block the e2e jobs use. - Docker E2E: a container pnpm install network blip exits 254. Add fetch retries + a longer network timeout (frozen-lockfile already passes locally). - Cross-browser: the home page is the tool catalog now (no dropzone), and the tool routes moved to /
/. Point the upload test at a real tool page and fix the stale single-segment routes (/resize -> /image/resize, etc.). The flaky/timeout and cross-browser fixes can only be confirmed by the nightly (they are load- and browser-specific); a fresh nightly run will verify. --- .github/workflows/nightly.yml | 12 +++++++ .github/workflows/update-visual-baselines.yml | 26 ++++++++++++++++ docker/Dockerfile.test | 8 ++++- tests/e2e/gui-cross-browser.spec.ts | 31 +++++++++++-------- .../tools/video/video-speed.test.ts | 5 ++- vitest.config.ts | 9 ++++-- 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1bbe4550..2a7da8b1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -273,6 +273,12 @@ jobs: FULL_MATRIX: "1" FUZZ: "1" FUZZ_RUNS: "50" + # Full-matrix tests iterate every format x tool; with 4 forks the heavy + # media conversions starve and hit the 30s default. Fewer forks (more + # CPU each) plus a generous timeout keeps them from flaking. + VITEST_MAX_FORKS: "2" + VITEST_TEST_TIMEOUT: "300000" + VITEST_HOOK_TIMEOUT: "120000" api-fuzz: name: Schemathesis API Fuzz @@ -361,6 +367,12 @@ jobs: - uses: ./.github/actions/setup - name: Run tests with coverage run: pnpm vitest run --coverage tests/unit/ tests/integration/ + # Coverage instrumentation slows every test; fewer forks + a longer + # timeout keep slow media tests (e.g. qr-generate) from timing out. + env: + VITEST_MAX_FORKS: "2" + VITEST_TEST_TIMEOUT: "300000" + VITEST_HOOK_TIMEOUT: "120000" - name: Upload coverage artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.github/workflows/update-visual-baselines.yml b/.github/workflows/update-visual-baselines.yml index 15efda40..547b5f50 100644 --- a/.github/workflows/update-visual-baselines.yml +++ b/.github/workflows/update-visual-baselines.yml @@ -12,6 +12,32 @@ jobs: name: Regenerate linux visual baselines runs-on: ubuntu-latest timeout-minutes: 60 + # The e2e webServer (playwright.config.ts) creates an isolated database via + # tests/e2e-pg-create-db.cjs, so it needs Postgres + Redis on localhost. + # Without these services the webServer fails with "failed to create database". + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: snapotter + POSTGRES_PASSWORD: snapotter + POSTGRES_DB: snapotter + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U snapotter" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + redis: + image: redis:8-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: ./.github/actions/setup diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 98f02bba..8243f536 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -81,7 +81,13 @@ COPY packages/ai/package.json packages/ai/tsconfig.json ./packages/ai/ # imports at module load; without it that suite fails to collect. COPY packages/enterprise/package.json packages/enterprise/tsconfig.json ./packages/enterprise/ -# Install ALL dependencies (including devDependencies for testing) +# Install ALL dependencies (including devDependencies for testing). +# Registry hiccups during the container build surface as a hard exit 254 +# (interrupted fetch / integrity check). Harden the install with more retries +# and a longer network timeout so a transient blip doesn't fail the whole job. +RUN pnpm config set fetch-retries 5 \ + && pnpm config set fetch-retry-maxtimeout 120000 \ + && pnpm config set network-timeout 600000 RUN pnpm install --frozen-lockfile # Copy source code diff --git a/tests/e2e/gui-cross-browser.spec.ts b/tests/e2e/gui-cross-browser.spec.ts index d7bee8bb..f931dbc2 100644 --- a/tests/e2e/gui-cross-browser.spec.ts +++ b/tests/e2e/gui-cross-browser.spec.ts @@ -37,13 +37,18 @@ test.describe("Cross-browser smoke tests", () => { expect(errors).toHaveLength(0); }); - test("home page file upload: upload image, verify preview", async ({ loggedInPage: page }) => { + test("tool page file upload: upload image, verify it is accepted", async ({ + loggedInPage: page, + }) => { const errors = collectConsoleErrors(page); + // The home page is the tool catalog now; uploads happen on a tool page. + await page.goto("/image/resize"); await page.waitForLoadState("networkidle"); await uploadImage(page); - await expect(page.locator("[class*='text-green']").first()).toBeVisible(); + // After a successful upload the tool's settings panel renders. + await expect(page.getByText("Settings").first()).toBeVisible(); expect(errors).toHaveLength(0); }); @@ -51,7 +56,7 @@ test.describe("Cross-browser smoke tests", () => { test("resize E2E: upload, set dimensions, process, download", async ({ loggedInPage: page }) => { const errors = collectConsoleErrors(page); - await page.goto("/resize"); + await page.goto("/image/resize"); await page.waitForLoadState("networkidle"); await uploadImage(page); @@ -81,7 +86,7 @@ test.describe("Cross-browser smoke tests", () => { }) => { const errors = collectConsoleErrors(page); - await page.goto("/compress"); + await page.goto("/image/compress"); await page.waitForLoadState("networkidle"); // Upload and wait for processing to produce the before-after view @@ -278,7 +283,7 @@ test.describe("Cross-browser smoke tests", () => { expect(bgColor).not.toBe(""); // Navigate to a tool page and verify CSS layout - await page.goto("/resize"); + await page.goto("/image/resize"); await page.waitForLoadState("networkidle"); await page.waitForTimeout(500); @@ -297,7 +302,7 @@ test.describe("Cross-browser smoke tests", () => { }) => { const errors = collectConsoleErrors(page); - await page.goto("/resize"); + await page.goto("/image/resize"); await page.waitForLoadState("networkidle"); await uploadImage(page); @@ -338,7 +343,7 @@ test.describe("Cross-browser smoke tests", () => { test("drag-and-drop: drop image file onto dropzone", async ({ loggedInPage: page }) => { const errors = collectConsoleErrors(page); - await page.goto("/resize"); + await page.goto("/image/resize"); await page.waitForLoadState("networkidle"); await page.waitForTimeout(500); @@ -363,7 +368,7 @@ test.describe("Cross-browser smoke tests", () => { test("canvas interactions: crop tool draw and adjust region", async ({ loggedInPage: page }) => { const errors = collectConsoleErrors(page); - await page.goto("/crop"); + await page.goto("/image/crop"); await page.waitForLoadState("networkidle"); await uploadImage(page); @@ -458,7 +463,7 @@ test.describe("Cross-browser smoke tests", () => { test("convert E2E: upload, select format, process", async ({ loggedInPage: page }) => { const errors = collectConsoleErrors(page); - await page.goto("/convert"); + await page.goto("/image/convert"); await page.waitForLoadState("networkidle"); await uploadImage(page); @@ -557,7 +562,7 @@ test.describe("Cross-browser smoke tests", () => { test("qr-generate: enter text and verify preview renders", async ({ loggedInPage: page }) => { const errors = collectConsoleErrors(page); - await page.goto("/qr-generate"); + await page.goto("/image/qr-generate"); await page.waitForLoadState("networkidle"); await page.waitForTimeout(500); @@ -593,7 +598,7 @@ test.describe("Cross-browser smoke tests", () => { test("collage tool renders templates across browsers", async ({ loggedInPage: page }) => { const errors = collectConsoleErrors(page); - await page.goto("/collage"); + await page.goto("/image/collage"); await page.waitForLoadState("networkidle"); await page.waitForTimeout(500); @@ -645,12 +650,12 @@ test.describe("Cross-browser smoke tests", () => { const errors = collectConsoleErrors(page); // Navigate to resize - await page.goto("/resize"); + await page.goto("/image/resize"); await page.waitForLoadState("networkidle"); await page.waitForTimeout(300); // Navigate to compress - await page.goto("/compress"); + await page.goto("/image/compress"); await page.waitForLoadState("networkidle"); await page.waitForTimeout(300); diff --git a/tests/integration/tools/video/video-speed.test.ts b/tests/integration/tools/video/video-speed.test.ts index b08efe7a..f12d9f0d 100644 --- a/tests/integration/tools/video/video-speed.test.ts +++ b/tests/integration/tools/video/video-speed.test.ts @@ -11,7 +11,10 @@ import { type TestApp, } from "../../test-server.js"; -const MP4 = readFixture(fixtures.video.tiny("mp4")); +// An 8s clip (44.1kHz audio): long enough that the 2x-speed duration check is +// robust to ffmpeg's frame/packet rounding. The 1s tiny.mp4 rounded to ~0.75s +// for a 2x speed-up, which flaked the +/-25% assertion under heavy CI load. +const MP4 = readFixture(fixtures.video.hero.mp4); let testApp: TestApp; let adminToken: string; diff --git a/vitest.config.ts b/vitest.config.ts index acbe337f..8cfa54ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -41,8 +41,13 @@ export default defineConfig({ poolOptions: { forks: { // Parallel forks; each fork gets an isolated DB + workspace via - // tests/setup/per-fork-env.ts. CI runners have 4 vCPUs. - maxForks: process.env.CI ? 4 : Math.max(2, Math.floor(os.availableParallelism() / 2)), + // tests/setup/per-fork-env.ts. CI runners have 4 vCPUs. Env-overridable + // (VITEST_MAX_FORKS) so the heavy nightly jobs (FULL_MATRIX, coverage) + // can drop to fewer forks and give each slow media test more CPU, which + // keeps format-matrix conversions from starving and timing out. + maxForks: + Number(process.env.VITEST_MAX_FORKS) || + (process.env.CI ? 4 : Math.max(2, Math.floor(os.availableParallelism() / 2))), }, }, globalSetup: ["tests/global-setup.ts"],