[codex] Optimize CI topology and Playwright suites (#4460)

* Consolidate PR validation into ci

* Tune absorbed CI runner tiers

* Fold visual validation into CI Playwright shards

* Run PR shards during manual CI dispatch

* Optimize CI topology and scopes

* Increase UI P0 Playwright workers

* Relax visual Playwright timeout

scripts/check-ui-p0-path-parity.ts

@@ -1,109 +0,0 @@
-import { readFile } from "node:fs/promises";
-import path from "node:path";
-
-const repoRoot = path.resolve(import.meta.dirname, "..");
-const ciWorkflowPath = ".github/workflows/ci.yml";
-const ciChangeScopesPath = "scripts/ci-change-scopes.ts";
-const uiP0ShardsPath = "e2e/scripts/ui-p0-shards.ts";
-
-async function main(): Promise<void> {
-  const [ciWorkflow, ciChangeScopes, uiP0Shards] = await Promise.all([
-    readFile(path.join(repoRoot, ciWorkflowPath), "utf8"),
-    readFile(path.join(repoRoot, ciChangeScopesPath), "utf8"),
-    readFile(path.join(repoRoot, uiP0ShardsPath), "utf8"),
-  ]);
-
-  const errors = [
-    ...checkCiScopeOutput(ciWorkflow),
-    ...checkUiP0Jobs(ciWorkflow),
-    ...checkShardMatrix(ciWorkflow, uiP0Shards),
-    ...checkScopeRules(ciChangeScopes),
-  ];
-
-  if (errors.length > 0) {
-    console.error("UI P0 CI wiring check failed.");
-    for (const error of errors) console.error(`- ${error}`);
-    process.exitCode = 1;
-    return;
-  }
-
-  console.log("UI P0 CI wiring check passed.");
-}
-
-function checkCiScopeOutput(source: string): string[] {
-  const required = [
-    "ui_p0_pr_required: ${{ steps.detect.outputs.ui_p0_pr_required }}",
-    "needs.change_scopes.outputs.ui_p0_pr_required == 'true'",
-  ];
-  return required
-    .filter((needle) => !source.includes(needle))
-    .map((needle) => `${ciWorkflowPath} is missing ${needle}`);
-}
-
-function checkUiP0Jobs(source: string): string[] {
-  const required = [
-    "ui_p0_smoke:",
-    "name: UI P0 smoke",
-    "ui_p0:",
-    "name: UI P0 (${{ matrix.name }})",
-    "pnpm -C e2e exec tsx scripts/ui-p0-shards.ts smoke",
-    "pnpm -C e2e exec tsx scripts/ui-p0-shards.ts ${{ matrix.shard }}",
-  ];
-  return required
-    .filter((needle) => !source.includes(needle))
-    .map((needle) => `${ciWorkflowPath} is missing ${needle}`);
-}
-
-function checkShardMatrix(ciWorkflow: string, uiP0Shards: string): string[] {
-  const matrixShards = extractCiMatrixShards(ciWorkflow);
-  const definedShards = extractUiP0ShardNames(uiP0Shards).filter((name) => !["smoke", "settings-smoke"].includes(name));
-  return [
-    ...difference(definedShards, matrixShards).map((name) => `${ciWorkflowPath} matrix is missing UI P0 shard ${name}`),
-    ...difference(matrixShards, definedShards).map((name) => `${ciWorkflowPath} matrix contains unknown UI P0 shard ${name}`),
-  ];
-}
-
-function checkScopeRules(source: string): string[] {
-  const required = [
-    "function isUiP0RelevantFile(file: string): boolean",
-    '"apps/web/"',
-    '"apps/daemon/"',
-    '"e2e/ui/"',
-    '"e2e/lib/"',
-    '"e2e/scripts/"',
-    '".github/actions/setup-playwright/"',
-    '".github/actions/setup-workspace/"',
-    '".github/workflows/ci.yml"',
-    '".github/workflows/ui-extended-main.yml"',
-  ];
-  return required
-    .filter((needle) => !source.includes(needle))
-    .map((needle) => `${ciChangeScopesPath} is missing ${needle}`);
-}
-
-function extractCiMatrixShards(source: string): string[] {
-  const jobMatch = source.match(/\n  ui_p0:\n(?<body>[\s\S]+?)\n\n  playwright_visual:/u);
-  const body = jobMatch?.groups?.body;
-  if (!body) throw new Error(`Unable to find ui_p0 job in ${ciWorkflowPath}.`);
-  return [...body.matchAll(/^\s+shard:\s+([a-z0-9-]+)\s*$/gmu)]
-    .map((match) => match[1])
-    .filter((name): name is string => Boolean(name))
-    .sort();
-}
-
-function extractUiP0ShardNames(source: string): string[] {
-  const objectMatch = source.match(/const shards: Record<string, Shard> = \{(?<body>[\s\S]+?)\n\};/u);
-  const body = objectMatch?.groups?.body;
-  if (!body) throw new Error(`Unable to find shards object in ${uiP0ShardsPath}.`);
-  return [...body.matchAll(/^\s{2}'?([a-z0-9-]+)'?:\s+\{/gmu)]
-    .map((match) => match[1])
-    .filter((name): name is string => Boolean(name))
-    .sort();
-}
-
-function difference(left: string[], right: string[]): string[] {
-  const rightSet = new Set(right);
-  return left.filter((value) => !rightSet.has(value));
-}
-
-await main();

scripts/guard.ts

@@ -9,6 +9,7 @@ import { checkDesignSystemPackageQuality } from "./check-design-system-package-q
 import { checkDesignSystemComponentFixtureReport } from "./check-components-fixtures.ts";
 import { checkDesignSystemFlagParity } from "./check-design-system-flag-parity.ts";
 import { checkComponentsManifestExtraction } from "./check-components-manifest-extraction.ts";
+import { validatePlaywrightSuiteTopology } from "../e2e/lib/playwright/suites.ts";
 import {
   checkDesignSystemA1RequiredTokens,
   checkDesignSystemA2DefaultsParity,
@@ -24,7 +25,6 @@ const repoRoot = path.resolve(import.meta.dirname, "..");
 const allowedE2eScripts = new Set([
   "e2e/scripts/playwright.ts",
   "e2e/scripts/release-smoke.ts",
-  "e2e/scripts/ui-p0-shards.ts",
   "e2e/scripts/visual-report.ts",
 ]);
 
@@ -1231,6 +1231,33 @@ async function checkStylePolicy(): Promise<boolean> {
   return true;
 }
 
+async function checkCiTopology(): Promise<boolean> {
+  const ciWorkflow = await readFile(path.join(repoRoot, ".github/workflows/ci.yml"), "utf8");
+  const errors = [
+    ...validatePlaywrightSuiteTopology(),
+    ...[
+      "run: node --experimental-strip-types scripts/scopes.ts github-output",
+      "ui_p0_matrix: ${{ steps.detect.outputs.ui_p0_matrix }}",
+      "visual_matrix: ${{ steps.detect.outputs.visual_matrix }}",
+      "include: ${{ fromJSON(needs.scopes.outputs.ui_p0_matrix) }}",
+      "include: ${{ fromJSON(needs.scopes.outputs.visual_matrix) }}",
+      "pnpm -C e2e exec tsx scripts/playwright.ts run-ui-group smoke",
+      "pnpm -C e2e exec tsx scripts/playwright.ts run-ui-group ${{ matrix.shard }}",
+    ]
+      .filter((needle) => !ciWorkflow.includes(needle))
+      .map((needle) => `.github/workflows/ci.yml is missing ${needle}`),
+  ];
+
+  if (errors.length > 0) {
+    console.error("CI topology check failed:");
+    for (const error of errors) console.error(`- ${error}`);
+    return false;
+  }
+
+  console.log("CI topology check passed: scopes, Playwright suites, and workflow matrices stay aligned.");
+  return true;
+}
+
 const checks: GuardCheck[] = [
   { name: "residual JavaScript", run: checkResidualJavaScript },
   { name: "package dependency specs", run: checkPackageDependencySpecs },
@@ -1242,6 +1269,7 @@ const checks: GuardCheck[] = [
   { name: "web import isolation", run: checkWebImportIsolation },
   { name: "tools layout", run: checkToolsLayout },
   { name: "style policy", run: checkStylePolicy },
+  { name: "CI topology", run: checkCiTopology },
   { name: "craft references", run: checkCraftReferences },
   { name: "design system manifests", run: checkDesignSystemManifests },
   { name: "design system package quality", run: checkDesignSystemPackageQuality },

scripts/scopes.ts

@@ -1,5 +1,7 @@
 import { execFileSync } from "node:child_process";
-import { readFileSync, appendFileSync } from "node:fs";
+import { appendFileSync, readFileSync } from "node:fs";
+
+import { uiP0CiMatrix, visualCiMatrix } from "../e2e/lib/playwright/suites.ts";
 
 type ScopeOutputs = {
   daemon_tests_required: boolean;
@@ -13,66 +15,91 @@ type ScopeOutputs = {
   workspace_validation_required: boolean;
 };
 
+type ScopePlan = ScopeOutputs & {
+  ui_p0_matrix: string;
+  visual_matrix: string;
+};
+
 type GitHubEvent = {
   pull_request?: {
     number?: number;
   };
 };
 
-const outputs: ScopeOutputs = {
-  daemon_tests_required: false,
-  web_tests_required: false,
-  tools_dev_tests_required: false,
-  tools_pack_tests_required: false,
-  nix_validation_required: false,
-  ui_p0_pr_required: false,
-  visual_validation_required: false,
-  docker_validation_required: false,
-  workspace_validation_required: false,
-};
+const commandName = process.argv[2] ?? "github-output";
 
-const eventName = requiredEnv("GITHUB_EVENT_NAME");
-
-if (eventName === "pull_request") {
-  for (const file of changedPullRequestFiles()) {
-    applyChangedFile(file, outputs);
-    if (allOutputsTrue(outputs)) break;
-  }
-
-  if (
-    outputs.daemon_tests_required ||
-    outputs.web_tests_required ||
-    outputs.tools_dev_tests_required ||
-    outputs.tools_pack_tests_required
-  ) {
-    outputs.workspace_validation_required = true;
-  }
-} else if (eventName === "push") {
-  outputs.daemon_tests_required = true;
-  outputs.web_tests_required = true;
-  outputs.tools_dev_tests_required = true;
-  outputs.tools_pack_tests_required = true;
-  // Main already runs .github/workflows/nix-check.yml, so keep this workflow's
-  // push path focused on the non-Nix workspace signal.
-  outputs.nix_validation_required = false;
-  // Main Docker publishing stays owned by .github/workflows/docker-image.yml.
-  outputs.docker_validation_required = false;
-  outputs.workspace_validation_required = true;
+if (commandName === "github-output") {
+  writeGithubOutputs(createScopePlan());
+} else if (commandName === "print") {
+  console.log(JSON.stringify(createScopePlan(), null, 2));
+} else if (commandName === "help") {
+  printUsage();
 } else {
-  outputs.daemon_tests_required = true;
-  outputs.web_tests_required = true;
-  outputs.tools_dev_tests_required = true;
-  outputs.tools_pack_tests_required = true;
-  outputs.nix_validation_required = true;
-  if (eventName === "workflow_dispatch") {
-    outputs.ui_p0_pr_required = true;
-  }
-  outputs.visual_validation_required = true;
-  outputs.docker_validation_required = true;
-  outputs.workspace_validation_required = true;
+  console.error(`Unknown scopes command: ${commandName}`);
+  printUsage();
+  process.exitCode = 1;
 }
 
-writeOutputs(outputs);
+function createScopePlan(): ScopePlan {
+  const outputs: ScopeOutputs = {
+    daemon_tests_required: false,
+    web_tests_required: false,
+    tools_dev_tests_required: false,
+    tools_pack_tests_required: false,
+    nix_validation_required: false,
+    ui_p0_pr_required: false,
+    visual_validation_required: false,
+    docker_validation_required: false,
+    workspace_validation_required: false,
+  };
+
+  const eventName = requiredEnv("GITHUB_EVENT_NAME");
+
+  if (eventName === "pull_request") {
+    for (const file of changedPullRequestFiles()) {
+      applyChangedFile(file, outputs);
+      if (allScopeOutputsTrue(outputs)) break;
+    }
+
+    if (
+      outputs.daemon_tests_required ||
+      outputs.web_tests_required ||
+      outputs.tools_dev_tests_required ||
+      outputs.tools_pack_tests_required
+    ) {
+      outputs.workspace_validation_required = true;
+    }
+  } else if (eventName === "push") {
+    outputs.daemon_tests_required = true;
+    outputs.web_tests_required = true;
+    outputs.tools_dev_tests_required = true;
+    outputs.tools_pack_tests_required = true;
+    // Main already runs .github/workflows/nix-check.yml, so keep this workflow's
+    // push path focused on the non-Nix workspace signal.
+    outputs.nix_validation_required = false;
+    // Main Docker publishing stays owned by .github/workflows/docker-image.yml.
+    outputs.docker_validation_required = false;
+    outputs.workspace_validation_required = true;
+  } else {
+    outputs.daemon_tests_required = true;
+    outputs.web_tests_required = true;
+    outputs.tools_dev_tests_required = true;
+    outputs.tools_pack_tests_required = true;
+    outputs.nix_validation_required = true;
+    if (eventName === "workflow_dispatch") {
+      outputs.ui_p0_pr_required = true;
+    }
+    outputs.visual_validation_required = true;
+    outputs.docker_validation_required = true;
+    outputs.workspace_validation_required = true;
+  }
+
+  return {
+    ...outputs,
+    ui_p0_matrix: JSON.stringify(uiP0CiMatrix),
+    visual_matrix: JSON.stringify(visualCiMatrix),
+  };
+}
 
 function changedPullRequestFiles(): string[] {
   const eventPath = requiredEnv("GITHUB_EVENT_PATH");
@@ -341,12 +368,12 @@ function startsWithAny(value: string, prefixes: string[]): boolean {
   return prefixes.some((prefix) => value.startsWith(prefix));
 }
 
-function allOutputsTrue(value: ScopeOutputs): boolean {
+function allScopeOutputsTrue(value: ScopeOutputs): boolean {
   return Object.values(value).every(Boolean);
 }
 
-function writeOutputs(value: ScopeOutputs): void {
-  const lines = Object.entries(value).map(([key, enabled]) => `${key}=${enabled ? "true" : "false"}`);
+function writeGithubOutputs(value: ScopePlan): void {
+  const lines = Object.entries(value).map(([key, output]) => `${key}=${formatOutput(output)}`);
   console.log(lines.join("\n"));
   const outputPath = process.env.GITHUB_OUTPUT;
   if (outputPath != null && outputPath.length > 0) {
@@ -354,6 +381,10 @@ function writeOutputs(value: ScopeOutputs): void {
   }
 }
 
+function formatOutput(value: boolean | string): string {
+  return typeof value === "boolean" ? (value ? "true" : "false") : value;
+}
+
 function requiredEnv(name: string): string {
   const value = process.env[name];
   if (value == null || value.length === 0) {
@@ -361,3 +392,13 @@ function requiredEnv(name: string): string {
   }
   return value;
 }
+
+function printUsage(): void {
+  console.log(`Usage: node --experimental-strip-types scripts/scopes.ts <command>
+
+Commands:
+  github-output  Write validation scope outputs for GitHub Actions
+  print          Print the validation scope plan as JSON
+  help           Show this help
+`);
+}