From 0922869abe9f170879276588f3c4c51c292e230f Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Sat, 25 Jul 2026 02:22:24 -0400 Subject: [PATCH] test(desktop): C-10 e2e flow spec + F8 onboarding nav + D-15 README + delete-error knob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C-10 (full form — Playwright spec): Add desktop/tests/e2e/harness-management.spec.ts with 10 tests covering: - Preset gallery: detected badge present for available preset - Preset gallery: install link shown, no badge for not-installed preset - Add custom harness: form save → row appears in list - Edit preserves env vars: definitionEnv round-trip via edit form - Same-ID edit replaces row without duplication - Rename: old row gone, new row present - Delete success: row disappears - Delete failure: inline error shown, row kept (error-injection knob) - PATH badge on custom row: Detected badge when availability === 'available' - F8 onboarding navigate: setup-page More-harnesses click → completes onboarding → Settings → Agents (settings-harness-management) This exercises the real App.tsx effect that gates router.navigate() on machine.stage === 'ready', which postOnboardingNav.test.mjs cannot cover (it simulates the predicate, not the render path). Delete-error injection knob: - Add deleteCustomHarnessError?: string to MockBridgeOptions in bridge.ts - Add same field to MockE2eConfig in e2eBridge.ts - Thread config through to handleDeleteCustomHarness so the e2e bridge throws when the knob is set (mirrors acpAuthMethodsError pattern) - Add unit test: throws when knob set + entry survives in store D-15 (README execution-locus sentence, full placement): The install_hint placement (already in discovery.rs) is correct and stays. Add the missing sentence to the README blockquote at line 265 per the brief: Desktop-injected BUZZ_* env vars do NOT reach the execution locus unless set on the Gateway's own environment. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-acp/README.md | 2 +- desktop/src/testing/e2eBridge.ts | 3 + .../testing/e2eBridgeCustomHarnesses.test.mjs | 15 + .../src/testing/e2eBridgeCustomHarnesses.ts | 11 +- desktop/tests/e2e/harness-management.spec.ts | 406 ++++++++++++++++++ desktop/tests/helpers/bridge.ts | 2 + 6 files changed, 437 insertions(+), 2 deletions(-) create mode 100644 desktop/tests/e2e/harness-management.spec.ts diff --git a/crates/buzz-acp/README.md b/crates/buzz-acp/README.md index 6bd34b606..db38c32ae 100644 --- a/crates/buzz-acp/README.md +++ b/crates/buzz-acp/README.md @@ -262,7 +262,7 @@ Buzz Desktop supports registering any ACP-speaking agent tool as a selectable ru **Tier-2 — preset catalog** (Cursor, Oh My Pi, Grok Build, OpenCode, Kimi Code, Amp, Hermes Agent, OpenClaw): static `HarnessDefinition` entries in `desktop/src-tauri/src/managed_agents/discovery.rs` (`PRESET_HARNESSES`). They are always present in the runtime catalog, PATH-probed for availability, not editable or deletable by the user. Displayed with bundled logos; if not installed, a docs link appears instead. -> **Note — OpenClaw:** `openclaw acp` is a Gateway-backed bridge; PATH availability shows "Available" even when the OpenClaw Gateway daemon is not running. This is expected tier-2 semantics (same class as a preset with unconfigured auth). The Gateway URL is configured via `OPENCLAW_GATEWAY_URL` (or the equivalent env var from OpenClaw's docs) — set it in the agent's **env vars** in Edit Agent, not in the definition env (the preset definition carries no env entries). +> **Note — OpenClaw:** `openclaw acp` is a Gateway-backed bridge; PATH availability shows "Available" even when the OpenClaw Gateway daemon is not running. This is expected tier-2 semantics (same class as a preset with unconfigured auth). The Gateway URL is configured via `OPENCLAW_GATEWAY_URL` (or the equivalent env var from OpenClaw's docs) — set it in the agent's **env vars** in Edit Agent, not in the definition env (the preset definition carries no env entries). Note that `openclaw acp` executes tools inside the Gateway daemon, not the Desktop process, so Desktop-injected `BUZZ_*` env vars do NOT reach the execution locus unless you also set them on the Gateway's own environment. **Tier-3 — user custom harnesses**: JSON files in `/custom_harnesses/` that the user can create from the Settings UI or drop in directly. Each file describes one harness — no install scripts. diff --git a/desktop/src/testing/e2eBridge.ts b/desktop/src/testing/e2eBridge.ts index d685221e2..c27e0203f 100644 --- a/desktop/src/testing/e2eBridge.ts +++ b/desktop/src/testing/e2eBridge.ts @@ -180,6 +180,8 @@ type E2eConfig = { acpAuthMethods?: Record; acpAuthMethodsErrors?: Record; acpAuthMethodsError?: string; + /** When set, the `delete_custom_harness` mock command throws with this message. */ + deleteCustomHarnessError?: string; connectAcpRuntimeResult?: RawConnectAcpRuntimeResult; connectAcpRuntimeDelayMs?: number; connectAcpRuntimeError?: string; @@ -10023,6 +10025,7 @@ export function maybeInstallE2eTauriMocks() { case "delete_custom_harness": return handleDeleteCustomHarness( payload as Parameters[0], + activeConfig, ); case "discover_acp_auth_methods": return handleDiscoverAcpAuthMethods( diff --git a/desktop/src/testing/e2eBridgeCustomHarnesses.test.mjs b/desktop/src/testing/e2eBridgeCustomHarnesses.test.mjs index 795b04c3d..5bb929d4d 100644 --- a/desktop/src/testing/e2eBridgeCustomHarnesses.test.mjs +++ b/desktop/src/testing/e2eBridgeCustomHarnesses.test.mjs @@ -130,6 +130,21 @@ describe("handleDeleteCustomHarness", () => { "delete of non-existent id must not throw", ); }); + + it("throws when deleteCustomHarnessError knob is set", () => { + handleSaveCustomHarness(makeArgs({ id: "keep-alive" })); + const config = { mock: { deleteCustomHarnessError: "permission denied" } }; + assert.throws( + () => handleDeleteCustomHarness({ id: "keep-alive" }, config), + /permission denied/, + "must throw the injected error message", + ); + // Entry must remain in store — the error means delete did not complete. + assert.ok( + mockCustomHarnesses.has("keep-alive"), + "entry must remain when delete throws", + ); + }); }); // ── discover integration: store is shared by reference ─────────────────────── diff --git a/desktop/src/testing/e2eBridgeCustomHarnesses.ts b/desktop/src/testing/e2eBridgeCustomHarnesses.ts index c21e0a478..175567784 100644 --- a/desktop/src/testing/e2eBridgeCustomHarnesses.ts +++ b/desktop/src/testing/e2eBridgeCustomHarnesses.ts @@ -73,8 +73,17 @@ export function handleSaveCustomHarness(args: { /** * Handle `delete_custom_harness`. * Removes the harness from the in-memory store. Idempotent (not-found is OK). + * When `config?.mock?.deleteCustomHarnessError` is set, throws with that message + * to exercise the UI's inline error path. */ -export function handleDeleteCustomHarness(args: { id?: string }): void { +export function handleDeleteCustomHarness( + args: { id?: string }, + config?: { mock?: { deleteCustomHarnessError?: string } } | undefined, +): void { + const errorMsg = config?.mock?.deleteCustomHarnessError; + if (errorMsg) { + throw new Error(errorMsg); + } const id = args.id ?? ""; mockCustomHarnesses.delete(id); } diff --git a/desktop/tests/e2e/harness-management.spec.ts b/desktop/tests/e2e/harness-management.spec.ts new file mode 100644 index 000000000..ee5ca16c0 --- /dev/null +++ b/desktop/tests/e2e/harness-management.spec.ts @@ -0,0 +1,406 @@ +/** + * E2E spec for the Bring-Your-Own-Harness management UI. + * + * Covers: + * - Preset gallery renders with Detected badge for an available preset + * - Preset gallery renders without badge / with install link for a missing preset + * - Add custom harness (form → save → row appears in list) + * - Edit preserves env vars (round-trip through definitionEnv boundary) + * - Same-ID edit replaces entry (no duplicate row) + * - Rename removes old row and shows new row + * - Delete success removes row + * - Delete failure shows error inline (error-injection knob) + * - PATH badge: custom harness row shows Detected when availability === "available" + * - Onboarding navigate: setup-page "More harnesses" click → Settings → Agents (F8) + */ +import { expect, test } from "@playwright/test"; + +import { installMockBridge } from "../helpers/bridge"; + +// ── Shared catalog fixtures ─────────────────────────────────────────────────── + +/** Hermes preset with availability "available" — renders the Detected badge. */ +const HERMES_AVAILABLE = { + id: "hermes", + label: "Hermes", + avatar_url: "", + availability: "available", + command: "hermes-acp", + binary_path: "/usr/local/bin/hermes-acp", + default_args: [], + mcp_command: null, + install_hint: "Install Hermes Agent from hermes-agent.nousresearch.com.", + install_instructions_url: "https://hermes-agent.nousresearch.com", + can_auto_install: false, + requires_external_cli: true, + underlying_cli_path: null, + node_required: false, + auth_status: { status: "unknown" }, + source: "preset", +} as const; + +/** OpenClaw preset with availability "not_installed" — renders install link. */ +const OPENCLAW_NOT_INSTALLED = { + id: "openclaw", + label: "OpenClaw", + avatar_url: "", + availability: "not_installed", + command: "openclaw", + binary_path: null, + default_args: ["acp"], + mcp_command: null, + install_hint: "Install OpenClaw: npm install -g openclaw@latest.", + install_instructions_url: "https://docs.openclaw.ai/start/getting-started", + can_auto_install: false, + requires_external_cli: true, + underlying_cli_path: null, + node_required: false, + auth_status: { status: "unknown" }, + source: "preset", +} as const; + +/** Custom harness entry already persisted — shown in the custom list. */ +function makeCustomEntry( + overrides: { + id?: string; + label?: string; + command?: string; + availability?: "available" | "not_installed"; + definition_env?: Record; + } = {}, +) { + return { + id: overrides.id ?? "my-custom-agent", + label: overrides.label ?? "My Custom Agent", + avatar_url: "", + availability: overrides.availability ?? "not_installed", + command: overrides.command ?? "my-custom-acp", + binary_path: + overrides.availability === "available" + ? "/usr/local/bin/my-custom-acp" + : null, + default_args: [], + mcp_command: null, + install_hint: "", + install_instructions_url: "", + can_auto_install: false, + requires_external_cli: true, + underlying_cli_path: null, + node_required: false, + auth_status: { status: "unknown" }, + source: "custom", + definition_env: overrides.definition_env, + }; +} + +// ── Navigation helpers ──────────────────────────────────────────────────────── + +/** + * Open Settings → Agents through the normal UI path. + * CI serves the app as a static SPA; direct navigation to /settings 404s + * before the client router starts. + */ +async function openHarnessSettings(page: import("@playwright/test").Page) { + await page.goto("/", { waitUntil: "domcontentloaded" }); + await page.getByTestId("open-settings").click(); + await page.getByTestId("profile-popover-settings").click(); + await expect(page.getByTestId("settings-view")).toBeVisible(); + await page.getByTestId("settings-nav-agents").click(); + await expect(page.getByTestId("settings-harness-management")).toBeVisible({ + timeout: 10_000, + }); +} + +/** + * Fill and submit the custom harness add/edit form. + * Caller must have the form visible before calling. + */ +async function fillHarnessForm( + page: import("@playwright/test").Page, + values: { + label: string; + id: string; + command: string; + env?: Array<{ key: string; value: string }>; + }, +) { + await page.fill("#ch-label", values.label); + // ID may auto-derive; overwrite it. + await page.fill("#ch-id", values.id); + await page.fill("#ch-command", values.command); + for (const pair of values.env ?? []) { + await page.getByRole("button", { name: "Add env var" }).click(); + // Fill last appended row. + const keyInputs = page.locator('input[placeholder="KEY"]'); + const valInputs = page.locator('input[placeholder="value"]'); + await keyInputs.last().fill(pair.key); + await valInputs.last().fill(pair.value); + } +} + +// ── Preset gallery ──────────────────────────────────────────────────────────── + +test.describe("preset gallery", () => { + test("detected preset shows Detected badge", async ({ page }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [HERMES_AVAILABLE, OPENCLAW_NOT_INSTALLED], + }); + await openHarnessSettings(page); + + const hermesCard = page.getByTestId("harness-preset-hermes"); + await expect(hermesCard).toBeVisible(); + await expect(hermesCard.getByText("Detected")).toBeVisible(); + // Not-installed preset must NOT show Detected badge. + const openclawCard = page.getByTestId("harness-preset-openclaw"); + await expect(openclawCard).toBeVisible(); + await expect(openclawCard.getByText("Detected")).not.toBeVisible(); + }); + + test("not-installed preset shows Install link, not Detected badge", async ({ + page, + }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [HERMES_AVAILABLE, OPENCLAW_NOT_INSTALLED], + }); + await openHarnessSettings(page); + + const openclawCard = page.getByTestId("harness-preset-openclaw"); + await expect(openclawCard).toBeVisible(); + await expect(openclawCard.getByText("Install")).toBeVisible(); + await expect(openclawCard.getByText("Detected")).not.toBeVisible(); + }); +}); + +// ── Custom harness add ──────────────────────────────────────────────────────── + +test.describe("add custom harness", () => { + test("form saves and row appears in list", async ({ page }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [HERMES_AVAILABLE, OPENCLAW_NOT_INSTALLED], + }); + await openHarnessSettings(page); + + // No custom rows yet. + // The list container may exist but have no row children. + await expect( + page.getByTestId("custom-harness-row-my-custom-agent"), + ).not.toBeVisible(); + + // Open the add form. + await page.getByTestId("harness-add-custom-button").click(); + await expect(page.getByTestId("custom-harness-form")).toBeVisible(); + + await fillHarnessForm(page, { + label: "My Custom Agent", + id: "my-custom-agent", + command: "my-custom-acp", + }); + + // Submit. + await page.getByRole("button", { name: "Save" }).click(); + + // Row must appear in the list after save. + await expect( + page.getByTestId("custom-harness-row-my-custom-agent"), + ).toBeVisible({ timeout: 5_000 }); + }); + + test("edit preserves env vars (definitionEnv round-trip)", async ({ + page, + }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [ + HERMES_AVAILABLE, + OPENCLAW_NOT_INSTALLED, + makeCustomEntry({ definition_env: { MY_API_KEY: "sk-test" } }), + ], + }); + await openHarnessSettings(page); + + // Open edit form for the existing custom entry. + await expect( + page.getByTestId("custom-harness-row-my-custom-agent"), + ).toBeVisible(); + await page.getByTestId("custom-harness-edit-my-custom-agent").click(); + await expect(page.getByTestId("custom-harness-form")).toBeVisible(); + + // Env KEY and value must be pre-populated. + await expect(page.locator('input[placeholder="KEY"]').first()).toHaveValue( + "MY_API_KEY", + ); + await expect( + page.locator('input[placeholder="value"]').first(), + ).toHaveValue("sk-test"); + }); + + test("same-ID edit replaces row — no duplicate", async ({ page }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [ + HERMES_AVAILABLE, + OPENCLAW_NOT_INSTALLED, + makeCustomEntry({ label: "V1 Label" }), + ], + }); + await openHarnessSettings(page); + + // Edit, keep same ID, change label. + await page.getByTestId("custom-harness-edit-my-custom-agent").click(); + await expect(page.getByTestId("custom-harness-form")).toBeVisible(); + await page.fill("#ch-label", "V2 Label"); + await page.getByRole("button", { name: "Save" }).click(); + + // Exactly one row with the same ID; label updated. + const rows = page.locator( + '[data-testid^="custom-harness-row-my-custom-agent"]', + ); + await expect(rows).toHaveCount(1); + await expect(rows.first()).toContainText("V2 Label"); + }); + + test("rename removes old row and inserts new row", async ({ page }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [ + HERMES_AVAILABLE, + OPENCLAW_NOT_INSTALLED, + makeCustomEntry({ id: "old-harness", label: "Old" }), + ], + }); + await openHarnessSettings(page); + + await page.getByTestId("custom-harness-edit-old-harness").click(); + await expect(page.getByTestId("custom-harness-form")).toBeVisible(); + await page.fill("#ch-label", "New Harness"); + await page.fill("#ch-id", "new-harness"); + await page.getByRole("button", { name: "Save" }).click(); + + // Old row gone; new row present. + await expect( + page.getByTestId("custom-harness-row-old-harness"), + ).not.toBeVisible({ timeout: 5_000 }); + await expect( + page.getByTestId("custom-harness-row-new-harness"), + ).toBeVisible({ timeout: 5_000 }); + }); +}); + +// ── Delete flow ─────────────────────────────────────────────────────────────── + +test.describe("delete custom harness", () => { + test("delete success removes the row", async ({ page }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [ + HERMES_AVAILABLE, + OPENCLAW_NOT_INSTALLED, + makeCustomEntry(), + ], + }); + await openHarnessSettings(page); + + // Enter confirm-delete mode. + await page.getByTestId("custom-harness-delete-my-custom-agent").click(); + // The confirm button must appear. + await expect( + page.getByTestId("custom-harness-delete-confirm-my-custom-agent"), + ).toBeVisible(); + await page + .getByTestId("custom-harness-delete-confirm-my-custom-agent") + .click(); + + // Row disappears after successful delete. + await expect( + page.getByTestId("custom-harness-row-my-custom-agent"), + ).not.toBeVisible({ timeout: 5_000 }); + }); + + test("delete failure shows inline error and keeps the row", async ({ + page, + }) => { + await installMockBridge(page, { + acpRuntimesCatalog: [ + HERMES_AVAILABLE, + OPENCLAW_NOT_INSTALLED, + makeCustomEntry(), + ], + deleteCustomHarnessError: "permission denied: could not remove file", + }); + await openHarnessSettings(page); + + await page.getByTestId("custom-harness-delete-my-custom-agent").click(); + await page + .getByTestId("custom-harness-delete-confirm-my-custom-agent") + .click(); + + // Error text visible; row still present. + await expect( + page.getByText("permission denied: could not remove file"), + ).toBeVisible({ timeout: 5_000 }); + await expect( + page.getByTestId("custom-harness-row-my-custom-agent"), + ).toBeVisible(); + }); +}); + +// ── PATH badge on custom harness row ───────────────────────────────────────── + +test("custom harness row shows Detected badge when command is on PATH", async ({ + page, +}) => { + await installMockBridge(page, { + acpRuntimesCatalog: [ + HERMES_AVAILABLE, + OPENCLAW_NOT_INSTALLED, + makeCustomEntry({ availability: "available" }), + ], + }); + await openHarnessSettings(page); + + const row = page.getByTestId("custom-harness-row-my-custom-agent"); + await expect(row).toBeVisible(); + await expect(row.getByText("Detected")).toBeVisible(); +}); + +// ── F8: onboarding navigate-after-complete ──────────────────────────────────── +// +// Verifies the parent-owned route intent introduced in B-8: +// 1. User reaches the machine-onboarding setup page. +// 2. Clicks "More harnesses" (onboarding-setup-more-harnesses). +// 3. App completes onboarding and immediately navigates to Settings → Agents. +// +// This test exercises the real App.tsx effect that gates router.navigate() on +// machine.stage === "ready", which the pure-logic tests in +// postOnboardingNav.test.mjs cannot cover (they simulate the predicate, not +// the real render path). + +test("onboarding setup More-harnesses click navigates to Settings → Agents", async ({ + page, +}) => { + // Start with a fresh machine (no machine-onboarding-complete flag). + // skipCommunitySeed: true so the user goes through machine onboarding. + // skipOnboardingSeed: true so the community/identity banner doesn't appear. + await installMockBridge(page, undefined, { + skipCommunitySeed: true, + skipOnboardingSeed: true, + }); + await page.goto("/"); + + // Reach the setup page: create a new identity key → skip backup step. + await page.getByRole("button", { name: "Create a new identity key" }).click(); + await expect(page.getByTestId("onboarding-page-backup")).toBeVisible({ + timeout: 10_000, + }); + await page.getByTestId("onboarding-next").click(); + + // Now on the setup page. + await expect( + page.getByRole("heading", { name: "Set up your agent harnesses" }), + ).toBeVisible({ timeout: 10_000 }); + + // Click the "More harnesses" link — fires navigateToAgentSettings. + await page.getByTestId("onboarding-setup-more-harnesses").click(); + + // After onboarding completes + router mounts, the app must land on + // Settings → Agents (harness management section visible). + await expect(page.getByTestId("settings-harness-management")).toBeVisible({ + timeout: 15_000, + }); +}); diff --git a/desktop/tests/helpers/bridge.ts b/desktop/tests/helpers/bridge.ts index 41fdeae6f..e9a067514 100644 --- a/desktop/tests/helpers/bridge.ts +++ b/desktop/tests/helpers/bridge.ts @@ -154,6 +154,8 @@ type MockBridgeOptions = { acpRuntimesDelayMs?: number; acpAuthMethods?: Record[] }>; acpAuthMethodsError?: string; + /** When set, the `delete_custom_harness` mock command throws with this message. */ + deleteCustomHarnessError?: string; connectAcpRuntimeResult?: { launched: boolean }; connectAcpRuntimeDelayMs?: number; connectAcpRuntimeError?: string;