ci(desktop): surface flaky E2E tests instead of retry-masking them (#1838)

Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
Co-authored-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@sprout-oss.stage.blox.sqprod.co>

.github/workflows/ci.yml

@@ -234,15 +234,21 @@ jobs:
         run: just desktop-build
       - name: Desktop smoke e2e
         run: cd desktop && pnpm exec playwright test --project=smoke --shard=${{ matrix.shard }}/4
+      - name: Summarize flaky tests
+        if: ${{ !cancelled() }}
+        run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop Smoke E2E (${{ matrix.shard }})"
+        working-directory: desktop
       - name: Upload desktop smoke e2e artifacts
-        if: failure()
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: desktop-smoke-e2e-artifacts-${{ matrix.shard }}
           path: |
             desktop/playwright-report
+            desktop/playwright-report.json
             desktop/test-results
           if-no-files-found: ignore
+          retention-days: 7
 
   desktop:
     name: Desktop
@@ -443,16 +449,22 @@ jobs:
         run: bash scripts/setup-desktop-test-data.sh
       - name: Desktop relay-backed e2e
         run: cd desktop && pnpm exec playwright test --project=integration --shard=${{ matrix.shard }}/2
+      - name: Summarize flaky tests
+        if: ${{ !cancelled() }}
+        run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop E2E Integration (${{ matrix.shard }}/2)"
+        working-directory: desktop
       - name: Upload desktop integration artifacts
-        if: failure()
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: desktop-e2e-integration-artifacts-${{ matrix.shard }}
           path: |
             desktop/playwright-report
+            desktop/playwright-report.json
             desktop/test-results
             /tmp/buzz-relay.log
           if-no-files-found: ignore
+          retention-days: 7
       - name: Save pnpm store cache
         if: github.event_name == 'push'
         uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5

desktop/.gitignore

@@ -12,6 +12,7 @@ node_modules
 dist
 dist-ssr
 playwright-report
+playwright-report.json
 test-results
 *.local
 playwright-report

desktop/playwright.config.ts

@@ -8,6 +8,7 @@ export default defineConfig({
   reporter: [
     ["list"],
     ["html", { open: "never", outputFolder: "playwright-report" }],
+    ["json", { outputFile: "playwright-report.json" }],
   ],
   use: {
     baseURL: "http://127.0.0.1:4173",

desktop/scripts/summarize-flaky-tests.mjs

@@ -0,0 +1,71 @@
+import { appendFile, readFile } from "node:fs/promises";
+
+// Playwright's `retries: 2` (desktop/playwright.config.ts) lets a test fail
+// then pass on retry with no durable signal beyond a one-line "N flaky" in
+// the console log — the exact gap that hid the stream.spec.ts membership
+// race (#1798) for months. This walks the JSON reporter's suite tree
+// (recursive: `describe` blocks nest as child `suites`) and appends any
+// `status === "flaky"` test to the job's GitHub Actions summary so retried
+// failures stay visible even when the shard ultimately goes green.
+//
+// Usage: node scripts/summarize-flaky-tests.mjs <report.json> <run-label>
+
+function collectFlakyTests(suite, out) {
+  for (const spec of suite.specs ?? []) {
+    for (const test of spec.tests ?? []) {
+      if (test.status !== "flaky") continue;
+      out.push({
+        title: `${suite.file} › ${spec.title}`,
+        project: test.projectName,
+        attempts: test.results?.length ?? 0,
+      });
+    }
+  }
+  for (const child of suite.suites ?? []) {
+    collectFlakyTests(child, out);
+  }
+}
+
+const [reportPath, runLabel] = process.argv.slice(2);
+if (!reportPath || !runLabel) {
+  console.error(
+    "Usage: node scripts/summarize-flaky-tests.mjs <report.json> <run-label>",
+  );
+  process.exit(1);
+}
+
+// This step runs `if: !cancelled()` purely to surface flaky tests — it must
+// never fail the job on its own, so a malformed/unexpected report (from a
+// Playwright version bump or a crashed run) is swallowed, not thrown.
+try {
+  const report = JSON.parse(await readFile(reportPath, "utf8"));
+
+  const flaky = [];
+  for (const suite of report.suites ?? []) {
+    collectFlakyTests(suite, flaky);
+  }
+
+  if (flaky.length > 0) {
+    const escapeCell = (value) => String(value).replaceAll("|", "\\|");
+    const rows = flaky
+      .map(
+        (t) =>
+          `| ${escapeCell(t.title)} | ${escapeCell(t.project)} | ${t.attempts} |`,
+      )
+      .join("\n");
+    const summary =
+      `### Flaky tests — ${runLabel}\n\n` +
+      `${flaky.length} test(s) failed at least once before passing on retry:\n\n` +
+      "| Test | Project | Attempts |\n| --- | --- | --- |\n" +
+      `${rows}\n`;
+
+    console.log(summary);
+
+    const summaryFile = process.env.GITHUB_STEP_SUMMARY;
+    if (summaryFile) {
+      await appendFile(summaryFile, `${summary}\n`);
+    }
+  }
+} catch (error) {
+  console.log(`Skipping flaky-test summary: ${error.message}`);
+}