fix(e2e): stop the packaged mac first-run gate failing on budget alone (#7899)

The `[P0] cold first Home run` packaged mac case has been red on almost every beta build for eleven days (1 pass in 12 attempts) and nobody saw it, because release-beta's smoke step is `continue-on-error: true`. Four changes, in the order they matter: 1. Capture evidence before cleanup. A failing case uninstalled the app and the next case deleted the runtime namespace, so the only surviving trace was `logs: {skipped: true}`. The failure path now writes the desktop log, a raw `/api/runs` diagnostics snapshot, and the error into the release report before anything is torn down. 2. Give the wait a budget that matches the chain, split in two. 15s had to cover cold project creation through rendered output; it is now 60s for "the daemon owns a finished run" plus 30s for "the output surfaced", so a timeout says which half broke instead of leaving three observations that cannot falsify each other. Polling moved inside the page: every sample used to cost a `tools-pack mac inspect` process, which ate about a third of the old budget on its own. 3. Drop the injected 503. `POST /api/runs` does not answer WORKSPACE_AUTHORITY_UNAVAILABLE — local run creation is deliberately independent of Workspace membership availability — so the case spent an extra create round trip plus a 500ms backoff on a branch production cannot reach. Run-create retry semantics stay covered by apps/web/tests/providers/sse.test.ts. 4. Make the exempted red audible. The exemption stays (at a 1/12 pass rate a blocking gate would stall the daily beta), but `tools-release write-report` now turns a failed smoke into a run annotation and a caution banner at the top of the step summary.
2026-10-27 09:51:12 +01:00 · 2026-09-08 11:38:41 +00:00
parent 057aa62ce1
commit 25475defab
11 changed files with 1296 additions and 85 deletions
@@ -738,7 +738,13 @@ jobs:
          RELEASE_REPORT_DIR: ${{ runner.temp }}/release-report/mac_arm64
          RELEASE_REPORT_JSON_PATH: ${{ runner.temp }}/release-report/mac_arm64/report.json
          RELEASE_REPORT_SUMMARY_PATH: ${{ runner.temp }}/release-report/mac_arm64/summary.md
+          # The smoke step is continue-on-error, so this job reports success even
+          # when the packaged gate is red. Feed the outcome to the report writer
+          # so the failure gets a run annotation and a summary banner instead of
+          # disappearing behind a green check.
+          RELEASE_SMOKE_EXEMPT: "true"
          RELEASE_SMOKE_MODE: ${{ inputs.mac_arm64_smoke_mode }}
+          RELEASE_SMOKE_OUTCOME: ${{ steps.mac_arm64_smoke.outcome }}
          RELEASE_TARGET: mac_arm64
          RELEASE_VERSION: ${{ needs.metadata.outputs.beta_version }}
          REPORT_TITLE: mac_arm64 beta build
@@ -1322,7 +1328,11 @@ jobs:
          RELEASE_REPORT_DIR: ${{ runner.temp }}\release-report\win_x64
          RELEASE_REPORT_JSON_PATH: ${{ runner.temp }}\release-report\win_x64\report.json
          RELEASE_REPORT_SUMMARY_PATH: ${{ runner.temp }}\release-report\win_x64\summary.md
+          # Same exemption as mac_arm64: keep the job green, but never let the
+          # red be silent.
+          RELEASE_SMOKE_EXEMPT: "true"
          RELEASE_SMOKE_MODE: ${{ inputs.win_x64_smoke_mode }}
+          RELEASE_SMOKE_OUTCOME: ${{ steps.win_x64_smoke.outcome }}
          RELEASE_TARGET: win_x64
          RELEASE_VERSION: ${{ needs.metadata.outputs.beta_version }}
          REPORT_TITLE: win_x64 beta build
@@ -0,0 +1,87 @@
+import type { E2eReport } from './report.ts';
+
+/**
+ * Report subdirectory that holds everything captured about one failed packaged
+ * smoke case. Kept separate from the passing report's files so a failure never
+ * overwrites the artifacts the release report already publishes.
+ */
+export const PACKAGED_FAILURE_EVIDENCE_DIR = 'first-run-failure';
+
+export type PackagedEvidenceSource = {
+  name: string;
+  read: () => Promise<string | Uint8Array>;
+};
+
+export type PackagedEvidenceEntry = {
+  detail: string;
+  name: string;
+  relpath: string;
+  status: 'failed' | 'saved';
+};
+
+/**
+ * Build the report-relative path for one evidence file.
+ *
+ * Names come from failure paths (log paths, error labels), so they are sanitized
+ * rather than trusted: an evidence writer must never be the reason a report
+ * write escapes its root.
+ */
+export function packagedEvidenceRelpath(dir: string, name: string): string {
+  return `${sanitizeSegment(dir)}/${sanitizeSegment(name)}`;
+}
+
+/**
+ * Save every source into the report, keeping going when individual captures
+ * fail, and never throwing.
+ *
+ * A failing packaged case is cleaned up immediately afterwards — uninstall, then
+ * the next case's `rm -rf` of the runtime namespace — so anything not copied out
+ * here is gone before anyone can look at it. That is why partial evidence beats
+ * a capture that aborts on its first unreadable source, and why the manifest
+ * records the failures too: "the desktop log could not be read" is itself a
+ * finding.
+ */
+export async function capturePackagedFailureEvidence(
+  report: Pick<E2eReport, 'json' | 'save'>,
+  dir: string,
+  sources: PackagedEvidenceSource[],
+): Promise<PackagedEvidenceEntry[]> {
+  const entries: PackagedEvidenceEntry[] = [];
+  for (const source of sources) {
+    const relpath = packagedEvidenceRelpath(dir, source.name);
+    try {
+      await report.save(relpath, await source.read());
+      entries.push({ detail: '', name: source.name, relpath, status: 'saved' });
+    } catch (error) {
+      entries.push({
+        detail: formatEvidenceError(error),
+        name: source.name,
+        relpath,
+        status: 'failed',
+      });
+    }
+  }
+  try {
+    await report.json(packagedEvidenceRelpath(dir, 'index.json'), {
+      capturedAt: new Date().toISOString(),
+      entries,
+    });
+  } catch {
+    // The manifest is a convenience; the captured files are the evidence.
+  }
+  return entries;
+}
+
+export function formatEvidenceError(value: unknown): string {
+  if (value instanceof Error) return `${value.name}: ${value.message}`;
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}
+
+function sanitizeSegment(value: string): string {
+  const cleaned = value.replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^[.-]+/, '');
+  return cleaned.length === 0 ? 'unnamed' : cleaned;
+}
@@ -12,20 +12,62 @@ export type PackagedHomeFirstRunResult = {
  daemonAssistantText: string;
  hrefAfter: string;
  hrefBefore: string;
+  inPageWaitedMs: number;
  inputTextBeforeSubmit: string;
-  injectedAuthorityOutageCount: number;
  navigationEntryCountAfter: number;
  navigationEntryCountBefore: number;
+  outputVisible: boolean;
  performanceTimeOriginAfter: number;
  performanceTimeOriginBefore: number;
  projectId: string;
  runEventRequestCount: number;
  runEventResponseStatuses: number[];
  runEventsContainExpectedOutput: boolean;
+  runReachedTerminal: boolean;
+  runStatuses: string[];
  submitClicked: boolean;
+  terminalRunStatus: string;
  workspaceTabClicksBeforeOutput: number;
 };

+/**
+ * The two things a cold packaged first Home run has to do, in order.
+ *
+ * They used to share one budget, so a timeout could not say whether the run was
+ * still working or had already failed. Splitting them makes the distinction
+ * structural: `run-terminal` only ends when the daemon owns a finished run row,
+ * so anything `assistant-output` reports afterwards is about surfacing, never
+ * about speed.
+ */
+export type PackagedHomeFirstRunStage = 'assistant-output' | 'run-terminal';
+
+/**
+ * Wall-clock budget per stage.
+ *
+ * `run-terminal` covers the whole cold chain — create project + conversation,
+ * route transition, ProjectView mount, auto-send, daemon agent-CLI probe, agent
+ * turn, persistence — on a cold packaged runtime with a cold daemon. The
+ * previous single 15s budget covered all of that plus rendering, and only the
+ * fastest CI machines made it, so this is sized for the chain rather than for
+ * the fastest observed sample.
+ */
+export const PACKAGED_HOME_FIRST_RUN_STAGE_TIMEOUT_MS: Record<PackagedHomeFirstRunStage, number> = {
+  'assistant-output': 30_000,
+  'run-terminal': 60_000,
+};
+
+/**
+ * How long one inspection may wait *inside* the page before returning.
+ *
+ * `tools-pack mac inspect --expr` resolves the sidecar EVAL over IPC with a hard
+ * 5s timeout, and each inspection also pays a full Node + tsx cold start. Polling
+ * from the Node side therefore spends most of the budget on process startup
+ * rather than on observation; polling inside the page spends almost none of it.
+ * Stay well under the IPC timeout so a slow daemon fetch on the last iteration
+ * cannot turn a normal observation into a transport error.
+ */
+export const PACKAGED_HOME_FIRST_RUN_IN_PAGE_WINDOW_MS = 2_500;
+
 export type PackagedHomeFirstRunReadiness = {
  composerContentEditable: boolean;
  composerFound: boolean;
@@ -218,7 +260,6 @@ export function packagedHomeFirstRunExpression(): string {
        hrefBefore: location.href,
        inputTextBeforeSubmit,
        instrumented: true,
-        injectedAuthorityOutageCount: 0,
        navigationEntryCountBefore: performance.getEntriesByType('navigation').length,
        performanceTimeOriginBefore: performance.timeOrigin,
        readiness,
@@ -257,20 +298,6 @@ export function packagedHomeFirstRunExpression(): string {
            ...(workspaceMemberId ? { 'x-od-workspace-member-id': workspaceMemberId } : {}),
          };
          state.createRunRequestCount += 1;
-          if (state.injectedAuthorityOutageCount === 0) {
-            state.injectedAuthorityOutageCount += 1;
-            state.createRunResponseStatuses.push(503);
-            return new Response(JSON.stringify({
-              error: {
-                code: 'WORKSPACE_AUTHORITY_UNAVAILABLE',
-                message: 'workspace membership authority is temporarily unavailable',
-                retryable: true,
-              },
-            }), {
-              status: 503,
-              headers: { 'Content-Type': 'application/json' },
-            });
-          }
        }
        if (isRunEvents) state.runEventRequestCount += 1;
        const response = await originalFetch(...args);
@@ -316,15 +343,176 @@ export function packagedHomeFirstRunSubmitExpression(): string {
  `;
 }

-export function packagedHomeFirstRunSnapshotExpression(): string {
+export type PackagedHomeFirstRunSnapshotOptions = {
+  /** How long the page may keep polling before returning its last observation. */
+  awaitMs?: number;
+  pollIntervalMs?: number;
+  /** Which stage's stop condition ends the in-page wait early. */
+  stage?: PackagedHomeFirstRunStage;
+};
+
+/**
+ * Observe the first packaged run, polling *inside* the page until this stage's
+ * stop condition holds or the caller's window expires.
+ *
+ * Every observation used to cost one `tools-pack mac inspect` process, so the
+ * sampling rate was bounded by Node + tsx startup rather than by the product.
+ * The wait now lives where the state does; the caller only pays process startup
+ * once per window.
+ */
+export function packagedHomeFirstRunSnapshotExpression(
+  options: PackagedHomeFirstRunSnapshotOptions = {},
+): string {
+  const stage: PackagedHomeFirstRunStage = options.stage ?? 'assistant-output';
+  const awaitMs = Math.max(0, Math.trunc(options.awaitMs ?? 0));
+  const pollIntervalMs = Math.max(0, Math.trunc(options.pollIntervalMs ?? 500));
  return `
    (async () => {
      const expectedOutput = ${JSON.stringify(PACKAGED_HOME_FIRST_RUN_OUTPUT)};
+      const stage = ${JSON.stringify(stage)};
+      const awaitMs = ${awaitMs};
+      const pollIntervalMs = ${pollIntervalMs};
      const state = globalThis.__odPackagedHomeFirstRun;
      const diagnosticFetch = typeof state?.originalFetch === 'function'
        ? state.originalFetch
        : globalThis.fetch.bind(globalThis);
      const diagnosticRequestInit = { headers: state?.workspaceRequestHeaders ?? {} };
+
+      async function collect() {
+        const [route, encodedProjectId, conversationsRoute, encodedConversationId] =
+          location.pathname.split('/').filter(Boolean);
+        const projectId = route === 'projects' && encodedProjectId
+          ? decodeURIComponent(encodedProjectId)
+          : '';
+        const conversationId = conversationsRoute === 'conversations' && encodedConversationId
+          ? decodeURIComponent(encodedConversationId)
+          : '';
+        const assistant = Array.from(document.querySelectorAll('[data-assistant-message-id]')).find(
+          (candidate) => candidate.textContent?.includes(expectedOutput),
+        );
+        const runsResponse = projectId
+          ? await diagnosticFetch(
+              '/api/runs?projectId=' + encodeURIComponent(projectId),
+              diagnosticRequestInit,
+            )
+          : null;
+        const runsBody = runsResponse?.ok ? await runsResponse.json() : { runs: [] };
+        const runs = Array.isArray(runsBody?.runs) ? runsBody.runs : [];
+        const runStatuses = runs.map((run) => String(run?.status ?? ''));
+        const terminalRun = runs.find((run) =>
+          ['succeeded', 'failed', 'canceled'].includes(String(run?.status)),
+        );
+        // Messages and events only become interesting once the daemon owns a
+        // finished run, so a still-running first stage stays a single cheap
+        // request instead of three.
+        const eventsResponse = terminalRun?.id
+          ? await diagnosticFetch(
+              '/api/runs/' + encodeURIComponent(terminalRun.id) + '/events',
+              diagnosticRequestInit,
+            )
+          : null;
+        const eventsText = eventsResponse?.ok ? await eventsResponse.text() : '';
+        const messagesResponse = terminalRun && projectId && conversationId
+          ? await diagnosticFetch(
+              '/api/projects/' + encodeURIComponent(projectId)
+                + '/conversations/' + encodeURIComponent(conversationId) + '/messages',
+              diagnosticRequestInit,
+            )
+          : null;
+        const messagesBody = messagesResponse?.ok
+          ? await messagesResponse.json()
+          : { messages: [] };
+        const messages = Array.isArray(messagesBody?.messages) ? messagesBody.messages : [];
+        const daemonAssistantText = messages
+          .filter((message) => message?.role === 'assistant')
+          .map((message) => String(message?.content ?? ''))
+          .join(String.fromCharCode(10));
+        const assistantText = assistant?.textContent ?? '';
+        const runEventsContainExpectedOutput = eventsText.includes(expectedOutput);
+
+        return {
+          assistantText,
+          conversationId,
+          createRunRequestCount: state?.createRunRequestCount ?? -1,
+          createRunResponseStatuses: state?.createRunResponseStatuses ?? [],
+          daemonAssistantText,
+          hrefAfter: location.href,
+          hrefBefore: state?.hrefBefore ?? '',
+          inPageWaitedMs: 0,
+          inputTextBeforeSubmit: state?.inputTextBeforeSubmit ?? '',
+          navigationEntryCountAfter: performance.getEntriesByType('navigation').length,
+          navigationEntryCountBefore: state?.navigationEntryCountBefore ?? -1,
+          outputVisible:
+            assistantText.includes(expectedOutput)
+            && daemonAssistantText.includes(expectedOutput)
+            && runEventsContainExpectedOutput,
+          performanceTimeOriginAfter: performance.timeOrigin,
+          performanceTimeOriginBefore: state?.performanceTimeOriginBefore ?? -1,
+          projectId,
+          runEventRequestCount: state?.runEventRequestCount ?? -1,
+          runEventResponseStatuses: state?.runEventResponseStatuses ?? [],
+          runEventsContainExpectedOutput,
+          runReachedTerminal: terminalRun != null,
+          runStatuses,
+          submitClicked: state?.submitClicked === true,
+          terminalRunStatus: terminalRun ? String(terminalRun.status ?? '') : '',
+          workspaceTabClicksBeforeOutput: state?.workspaceTabClicksBeforeOutput ?? -1,
+        };
+      }
+
+      function satisfied(snapshot) {
+        return stage === 'run-terminal' ? snapshot.runReachedTerminal : snapshot.outputVisible;
+      }
+
+      const startedAt = Date.now();
+      let snapshot = await collect();
+      while (!satisfied(snapshot) && Date.now() - startedAt < awaitMs) {
+        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+        snapshot = await collect();
+      }
+      snapshot.inPageWaitedMs = Date.now() - startedAt;
+      return snapshot;
+    })()
+  `;
+}
+
+/**
+ * Read everything a post-mortem needs out of the page in one inspection.
+ *
+ * Deliberately raw: the three observations a failed first run leaves behind (the
+ * rendered DOM, the daemon's conversation messages, the run event stream) each
+ * only say "no output" on their own, and none of them can falsify the others.
+ * The daemon's own run rows are what separate "still running" from "failed", so
+ * capture them verbatim rather than as a derived boolean.
+ */
+export function packagedHomeFirstRunDiagnosticsExpression(maxTextChars = 64_000): string {
+  const limit = Math.max(1_000, Math.trunc(maxTextChars));
+  return `
+    (async () => {
+      const limit = ${limit};
+      const state = globalThis.__odPackagedHomeFirstRun;
+      const diagnosticFetch = typeof state?.originalFetch === 'function'
+        ? state.originalFetch
+        : globalThis.fetch.bind(globalThis);
+      const diagnosticRequestInit = { headers: state?.workspaceRequestHeaders ?? {} };
+      const clip = (value) => {
+        const text = String(value ?? '');
+        return text.length > limit ? text.slice(0, limit) + '…[truncated]' : text;
+      };
+      const read = async (path) => {
+        try {
+          const response = await diagnosticFetch(path, diagnosticRequestInit);
+          return { path, status: response.status, body: clip(await response.text()) };
+        } catch (error) {
+          return {
+            path,
+            status: -1,
+            body: '',
+            error: error instanceof Error ? error.name + ': ' + error.message : String(error),
+          };
+        }
+      };
+
      const [route, encodedProjectId, conversationsRoute, encodedConversationId] =
        location.pathname.split('/').filter(Boolean);
      const projectId = route === 'projects' && encodedProjectId
@@ -333,62 +521,44 @@ export function packagedHomeFirstRunSnapshotExpression(): string {
      const conversationId = conversationsRoute === 'conversations' && encodedConversationId
        ? decodeURIComponent(encodedConversationId)
        : '';
-      const assistant = Array.from(document.querySelectorAll('[data-assistant-message-id]')).find(
-        (candidate) => candidate.textContent?.includes(expectedOutput),
+
+      const runs = await read(
+        projectId ? '/api/runs?projectId=' + encodeURIComponent(projectId) : '/api/runs',
      );
-      const runsResponse = projectId
-        ? await diagnosticFetch(
-            '/api/runs?projectId=' + encodeURIComponent(projectId),
-            diagnosticRequestInit,
-          )
-        : null;
-      const runsBody = runsResponse?.ok ? await runsResponse.json() : { runs: [] };
-      const runs = Array.isArray(runsBody?.runs) ? runsBody.runs : [];
-      const terminalRun = runs.find((run) =>
+      let parsedRuns = [];
+      try {
+        const parsed = JSON.parse(runs.body);
+        parsedRuns = Array.isArray(parsed?.runs) ? parsed.runs : [];
+      } catch {}
+      const terminalRun = parsedRuns.find((run) =>
        ['succeeded', 'failed', 'canceled'].includes(String(run?.status)),
      );
-      const eventsResponse = terminalRun?.id
-        ? await diagnosticFetch(
-            '/api/runs/' + encodeURIComponent(terminalRun.id) + '/events',
-            diagnosticRequestInit,
-          )
+      const events = terminalRun?.id
+        ? await read('/api/runs/' + encodeURIComponent(terminalRun.id) + '/events')
        : null;
-      const eventsText = eventsResponse?.ok ? await eventsResponse.text() : '';
-      const messagesResponse = projectId && conversationId
-        ? await diagnosticFetch(
+      const messages = projectId && conversationId
+        ? await read(
            '/api/projects/' + encodeURIComponent(projectId)
              + '/conversations/' + encodeURIComponent(conversationId) + '/messages',
-            diagnosticRequestInit,
          )
        : null;
-      const messagesBody = messagesResponse?.ok
-        ? await messagesResponse.json()
-        : { messages: [] };
-      const messages = Array.isArray(messagesBody?.messages) ? messagesBody.messages : [];
-      const daemonAssistantText = messages
-        .filter((message) => message?.role === 'assistant')
-        .map((message) => String(message?.content ?? ''))
-        .join(String.fromCharCode(10));

      return {
-        assistantText: assistant?.textContent ?? '',
+        capturedAt: new Date().toISOString(),
        conversationId,
        createRunRequestCount: state?.createRunRequestCount ?? -1,
        createRunResponseStatuses: state?.createRunResponseStatuses ?? [],
-        daemonAssistantText,
-        hrefAfter: location.href,
-        hrefBefore: state?.hrefBefore ?? '',
-        inputTextBeforeSubmit: state?.inputTextBeforeSubmit ?? '',
-        injectedAuthorityOutageCount: state?.injectedAuthorityOutageCount ?? -1,
-        navigationEntryCountAfter: performance.getEntriesByType('navigation').length,
-        navigationEntryCountBefore: state?.navigationEntryCountBefore ?? -1,
-        performanceTimeOriginAfter: performance.timeOrigin,
-        performanceTimeOriginBefore: state?.performanceTimeOriginBefore ?? -1,
+        events,
+        href: location.href,
+        instrumented: state?.instrumented === true,
+        messages,
        projectId,
        runEventRequestCount: state?.runEventRequestCount ?? -1,
        runEventResponseStatuses: state?.runEventResponseStatuses ?? [],
-        runEventsContainExpectedOutput: eventsText.includes(expectedOutput),
+        runStatuses: parsedRuns.map((run) => String(run?.status ?? '')),
+        runs,
        submitClicked: state?.submitClicked === true,
+        title: document.title,
        workspaceTabClicksBeforeOutput: state?.workspaceTabClicksBeforeOutput ?? -1,
      };
    })()
@@ -409,20 +579,95 @@ export function assertPackagedHomeFirstRunResult(
    || typeof candidate.daemonAssistantText !== 'string'
    || typeof candidate.hrefAfter !== 'string'
    || typeof candidate.hrefBefore !== 'string'
+    || typeof candidate.inPageWaitedMs !== 'number'
    || typeof candidate.inputTextBeforeSubmit !== 'string'
-    || typeof candidate.injectedAuthorityOutageCount !== 'number'
    || typeof candidate.navigationEntryCountAfter !== 'number'
    || typeof candidate.navigationEntryCountBefore !== 'number'
+    || typeof candidate.outputVisible !== 'boolean'
    || typeof candidate.performanceTimeOriginAfter !== 'number'
    || typeof candidate.performanceTimeOriginBefore !== 'number'
    || typeof candidate.projectId !== 'string'
    || typeof candidate.runEventRequestCount !== 'number'
    || !Array.isArray(candidate.runEventResponseStatuses)
    || typeof candidate.runEventsContainExpectedOutput !== 'boolean'
+    || typeof candidate.runReachedTerminal !== 'boolean'
+    || !Array.isArray(candidate.runStatuses)
    || typeof candidate.submitClicked !== 'boolean'
+    || typeof candidate.terminalRunStatus !== 'string'
    || typeof candidate.workspaceTabClicksBeforeOutput !== 'number'
  ) {
    throw new Error(`unexpected packaged first Home run value: ${JSON.stringify(value)}`);
  }
  return candidate as PackagedHomeFirstRunResult;
 }
+
+/**
+ * Clamp the caller's remaining budget into one in-page wait window.
+ *
+ * Returns 0 once the budget is spent so the caller stops issuing inspections
+ * instead of paying for one more process it has no time to use.
+ */
+export function packagedHomeFirstRunInPageAwaitMs(
+  remainingMs: number,
+  windowMs: number = PACKAGED_HOME_FIRST_RUN_IN_PAGE_WINDOW_MS,
+): number {
+  if (!Number.isFinite(remainingMs) || remainingMs <= 0) return 0;
+  return Math.min(Math.trunc(remainingMs), Math.max(0, Math.trunc(windowMs)));
+}
+
+export function packagedHomeFirstRunStageSatisfied(
+  stage: PackagedHomeFirstRunStage,
+  snapshot: PackagedHomeFirstRunResult,
+): boolean {
+  return stage === 'run-terminal' ? snapshot.runReachedTerminal : snapshot.outputVisible;
+}
+
+/**
+ * Name the one thing that did not happen, so a red gate is actionable without a
+ * packaged runtime in hand.
+ *
+ * Three observations (DOM, daemon messages, run events) all report "no output"
+ * when a run is merely slow AND when it has already failed. Ordering the checks
+ * from "never left Home" down to "succeeded but nothing rendered" makes the
+ * report say which of those it actually was.
+ */
+export function describePackagedHomeFirstRunStall(
+  stage: PackagedHomeFirstRunStage,
+  snapshot: PackagedHomeFirstRunResult | null,
+): string {
+  if (snapshot == null) {
+    return 'no readable snapshot was collected: every packaged inspection failed or returned an unexpected shape';
+  }
+  if (!snapshot.submitClicked) {
+    return 'the Home composer submit was never registered, so no run was ever requested';
+  }
+  if (stage === 'run-terminal') {
+    if (snapshot.projectId === '') {
+      return `the window never left Home (${snapshot.hrefAfter}): POST /api/projects did not route to /projects/:id`;
+    }
+    if (snapshot.createRunRequestCount <= 0) {
+      return 'the project route mounted but POST /api/runs was never sent, so the auto-send effect did not fire';
+    }
+    if (snapshot.runStatuses.length === 0) {
+      return `POST /api/runs was sent ${snapshot.createRunRequestCount} time(s) (statuses ${formatStatuses(snapshot.createRunResponseStatuses)}) but the daemon has no run row for this project`;
+    }
+    return `the run is still ${snapshot.runStatuses.join(', ')} and never reached a terminal status`;
+  }
+  if (!snapshot.runReachedTerminal) {
+    return `the run left its terminal status and is now ${snapshot.runStatuses.join(', ') || 'absent'}`;
+  }
+  if (snapshot.terminalRunStatus !== 'succeeded') {
+    return `the run finished as ${snapshot.terminalRunStatus}: this is a failed run, not a slow one`;
+  }
+  const missing = [
+    snapshot.assistantText.includes(PACKAGED_HOME_FIRST_RUN_OUTPUT) ? null : 'the rendered assistant message',
+    snapshot.daemonAssistantText.includes(PACKAGED_HOME_FIRST_RUN_OUTPUT) ? null : 'the daemon conversation messages',
+    snapshot.runEventsContainExpectedOutput ? null : 'the run event stream',
+  ].filter((entry): entry is string => entry != null);
+  if (missing.length === 0) return 'every observation carries the expected output';
+  return `the run succeeded but the expected output is missing from ${missing.join(' and ')}`;
+}
+
+function formatStatuses(statuses: number[]): string {
+  return statuses.length === 0 ? '(none)' : statuses.join(', ');
+}
@@ -10,14 +10,24 @@ import { afterAll, beforeAll, describe, expect, test } from 'vitest';

 import { createFakeAgentRuntimes } from '@/fake-agents';
 import { T } from '@/timeouts';
+import {
+  capturePackagedFailureEvidence,
+  PACKAGED_FAILURE_EVIDENCE_DIR,
+} from '@/vitest/packaged-failure-evidence';
 import {
  assertPackagedHomeFirstRunResult,
+  describePackagedHomeFirstRunStall,
  PACKAGED_HOME_FIRST_RUN_OUTPUT,
  PACKAGED_HOME_FIRST_RUN_PROMPT,
+  PACKAGED_HOME_FIRST_RUN_STAGE_TIMEOUT_MS,
+  packagedHomeFirstRunDiagnosticsExpression,
  packagedHomeFirstRunExpression,
+  packagedHomeFirstRunInPageAwaitMs,
  packagedHomeFirstRunSnapshotExpression,
+  packagedHomeFirstRunStageSatisfied,
  packagedHomeFirstRunSubmitExpression,
  type PackagedHomeFirstRunResult,
+  type PackagedHomeFirstRunStage,
  waitForPackagedHomeFirstRunSetup,
 } from '@/vitest/packaged-home-first-run';
 import { createPackagedSmokeReport } from '@/vitest/packaged-report';
@@ -428,6 +438,8 @@ macDescribe('packaged mac runtime smoke', () => {
    const fakeAgentRoot = join(toolsPackDir, 'fixtures', `home-first-run-${namespace}`);
    let firstRunInstalledAppPath: string | null = null;
    let firstRunStarted = false;
+    let firstRunDesktopLogPath: string | null = null;
+    let firstRunFailure: unknown = null;
    try {
      await resetPackagedRuntimeState();
      const fakeAgents = await createFakeAgentRuntimes({
@@ -440,6 +452,7 @@ macDescribe('packaged mac runtime smoke', () => {

      const start = await runToolsPackJson<MacStartResult>('start');
      firstRunStarted = true;
+      firstRunDesktopLogPath = start.logPath;
      expect(start.source).toBe('installed');
      await waitForHealthyDesktop();

@@ -459,16 +472,28 @@ macDescribe('packaged mac runtime smoke', () => {
      });

      await waitForPackagedHomeFirstRunSubmit();
-      const firstRun = await waitForPackagedHomeFirstRunOutput();
+      // Two stages, so a red gate can say which half broke. The first ends when
+      // the daemon owns a finished run row; only then is a missing message a
+      // rendering problem rather than an unfinished one.
+      const runFinished = await waitForPackagedHomeFirstRunStage('run-terminal');
+      expect(runFinished.terminalRunStatus).toBe('succeeded');
+
+      const firstRun = await waitForPackagedHomeFirstRunStage('assistant-output');
      expect(firstRun.submitClicked).toBe(true);
      expect(firstRun.projectId).toEqual(expect.any(String));
      expect(firstRun.hrefBefore).toMatch(/^(od:\/\/app\/|http:\/\/127\.0\.0\.1:\d+\/$)/);
      expect(firstRun.hrefAfter).toContain(`/projects/${firstRun.projectId}`);
-      expect(firstRun.injectedAuthorityOutageCount).toBe(1);
-      expect(firstRun.createRunRequestCount).toBeGreaterThanOrEqual(2);
-      expect(firstRun.createRunResponseStatuses[0]).toBe(503);
-      expect(firstRun.createRunResponseStatuses.at(-1)).toBeGreaterThanOrEqual(200);
-      expect(firstRun.createRunResponseStatuses.at(-1)).toBeLessThan(300);
+      // With nothing injected, a cold first run must reach the daemon cleanly:
+      // every create attempt answers 2xx, so no attempt was ever recovered from.
+      // The exact attempt count is deliberately not pinned — the injected outage
+      // made the real count unobservable, and guessing it here would trade a
+      // swallowed red for a false one.
+      expect(firstRun.createRunRequestCount).toBeGreaterThanOrEqual(1);
+      expect(firstRun.createRunResponseStatuses).toHaveLength(firstRun.createRunRequestCount);
+      for (const status of firstRun.createRunResponseStatuses) {
+        expect(status).toBeGreaterThanOrEqual(200);
+        expect(status).toBeLessThan(300);
+      }
      expect(firstRun.runEventRequestCount).toBeGreaterThan(0);
      expect(firstRun.runEventResponseStatuses).toContain(200);
      expect(firstRun.runEventsContainExpectedOutput).toBe(true);
@@ -477,7 +502,16 @@ macDescribe('packaged mac runtime smoke', () => {
      expect(firstRun.workspaceTabClicksBeforeOutput).toBe(0);
      expect(firstRun.navigationEntryCountAfter).toBe(firstRun.navigationEntryCountBefore);
      expect(firstRun.performanceTimeOriginAfter).toBe(firstRun.performanceTimeOriginBefore);
+    } catch (error) {
+      firstRunFailure = error;
+      throw error;
    } finally {
+      // Capture before uninstall: cleanup removes the installed app and the next
+      // case's reset deletes the runtime namespace, so evidence not copied out
+      // here no longer exists by the time anyone reads the report.
+      if (firstRunFailure != null) {
+        await capturePackagedHomeFirstRunFailure(firstRunFailure, firstRunDesktopLogPath);
+      }
      if (firstRunStarted || firstRunInstalledAppPath != null) {
        await runToolsPackJson<MacUninstallResult>('uninstall').catch((error: unknown) => {
          console.error('failed to uninstall packaged first-Home-run app during cleanup', error);
@@ -485,7 +519,12 @@ macDescribe('packaged mac runtime smoke', () => {
      }
      await rm(fakeAgentRoot, { force: true, recursive: true }).catch(() => undefined);
    }
-  }, 180_000);
+    // Budget: every wait above (desktop health 90s, composer readiness 45s, the
+    // two staged run waits 90s) must be able to expire and still leave room for
+    // the evidence capture in `finally`. A Vitest-level timeout aborts the case
+    // mid-cleanup, which is exactly how the earlier failures came back with
+    // `logs: {skipped: true}` and nothing else.
+  }, 300_000);

  test('installs, starts, inspects, stops, and uninstalls the built mac artifact', async () => {
    const report = await createPackagedSmokeReport('mac');
@@ -2310,36 +2349,106 @@ async function waitForHealthyDesktop(): Promise<MacInspectResult> {
  throw new Error(`packaged mac runtime did not become healthy: ${formatUnknown(lastResult)}`);
 }

-async function waitForPackagedHomeFirstRunOutput(): Promise<PackagedHomeFirstRunResult> {
-  const timeoutMs = 15_000;
+/**
+ * Wait for one stage of the cold first run, polling inside the page.
+ *
+ * Each `tools-pack mac inspect` pays a full Node + tsx cold start, so a Node-side
+ * poll loop spends most of its budget starting processes instead of observing
+ * the product. Handing the wait to the page turns one process into one window of
+ * continuous observation, and a transient inspect failure now costs a retry
+ * rather than the whole wait.
+ */
+async function waitForPackagedHomeFirstRunStage(
+  stage: PackagedHomeFirstRunStage,
+): Promise<PackagedHomeFirstRunResult> {
+  const timeoutMs = PACKAGED_HOME_FIRST_RUN_STAGE_TIMEOUT_MS[stage];
  const startedAt = Date.now();
-  let lastResult: unknown = null;
+  let snapshot: PackagedHomeFirstRunResult | null = null;
+  let lastError: unknown = null;

-  while (Date.now() - startedAt < timeoutMs) {
-    const inspect = await runToolsPackJson<MacInspectResult>('inspect', [
-      '--expr',
-      packagedHomeFirstRunSnapshotExpression(),
-    ]);
-    lastResult = inspect;
-    if (inspect.eval?.ok === true) {
-      const snapshot = assertPackagedHomeFirstRunResult(inspect.eval.value);
-      lastResult = snapshot;
-      if (
-        snapshot.assistantText.includes(PACKAGED_HOME_FIRST_RUN_OUTPUT)
-        && snapshot.daemonAssistantText.includes(PACKAGED_HOME_FIRST_RUN_OUTPUT)
-        && snapshot.runEventsContainExpectedOutput
-      ) {
-        return snapshot;
+  for (;;) {
+    const awaitMs = packagedHomeFirstRunInPageAwaitMs(timeoutMs - (Date.now() - startedAt));
+    try {
+      const inspect = await runToolsPackJson<MacInspectResult>('inspect', [
+        '--expr',
+        packagedHomeFirstRunSnapshotExpression({ awaitMs, stage }),
+      ]);
+      if (inspect.eval?.ok === true) {
+        snapshot = assertPackagedHomeFirstRunResult(inspect.eval.value);
+        lastError = null;
+        if (packagedHomeFirstRunStageSatisfied(stage, snapshot)) return snapshot;
+      } else {
+        lastError = inspect.eval ?? inspect;
      }
+    } catch (error) {
+      lastError = error;
    }
-    await delay(750);
+    if (Date.now() - startedAt >= timeoutMs) break;
+    await delay(250);
  }

  throw new Error(
-    `packaged first Home run did not render assistant output without recovery: ${formatUnknown(lastResult)}`,
+    [
+      `packaged first Home run stage "${stage}" timed out after ${Math.round((Date.now() - startedAt) / 1000)}s`,
+      describePackagedHomeFirstRunStall(stage, snapshot),
+      ...(lastError == null ? [] : [`last inspection error: ${formatUnknown(lastError)}`]),
+      `snapshot: ${formatUnknown(snapshot)}`,
+    ].join('\n'),
  );
 }

+/**
+ * Persist what a failed cold first run left behind, before cleanup destroys it.
+ *
+ * The daemon's own run rows are the only observation that separates "the run is
+ * still going" from "the run failed"; the DOM, the conversation messages, and
+ * the run event stream all read as "no output" in both cases and cannot falsify
+ * one another. Capture runs while the desktop is still alive, then the desktop
+ * log last so it also covers the diagnostic requests themselves.
+ */
+async function capturePackagedHomeFirstRunFailure(
+  error: unknown,
+  desktopLogPath: string | null,
+): Promise<void> {
+  try {
+    const { report } = await createPackagedSmokeReport('mac');
+    const entries = await capturePackagedFailureEvidence(report, PACKAGED_FAILURE_EVIDENCE_DIR, [
+      { name: 'error.txt', read: async () => `${formatUnknown(error)}\n` },
+      {
+        name: 'runs.json',
+        read: async () => {
+          const inspect = await runToolsPackJson<MacInspectResult>('inspect', [
+            '--expr',
+            packagedHomeFirstRunDiagnosticsExpression(),
+          ]);
+          if (inspect.eval?.ok !== true) {
+            throw new Error(`packaged diagnostics eval failed: ${formatUnknown(inspect.eval)}`);
+          }
+          return `${JSON.stringify(inspect.eval.value, null, 2)}\n`;
+        },
+      },
+      {
+        name: 'tools-pack-logs.json',
+        read: async () => `${JSON.stringify(await runToolsPackJson<LogsResult>('logs'), null, 2)}\n`,
+      },
+      {
+        name: 'desktop-latest.log',
+        read: async () => {
+          if (desktopLogPath == null) {
+            throw new Error('the packaged desktop never started, so no log path was reported');
+          }
+          return await readFile(desktopLogPath);
+        },
+      },
+    ]);
+    console.error(
+      `packaged first Home run failure evidence in ${report.root}: ${JSON.stringify(entries, null, 2)}`,
+    );
+  } catch (captureError) {
+    console.error('failed to capture packaged first Home run failure evidence', captureError);
+  }
+}
+
 async function waitForPackagedHomeFirstRunSubmit(): Promise<void> {
  const timeoutMs = 15_000;
  const startedAt = Date.now();
@@ -2731,6 +2731,34 @@ process.stdin.on("end", () => {
    expect(dailyWorkflow).toContain("RELEASE_STATE: ${{ needs.build.outputs.release_state }}");
  });

+  it("[P1] gives every exempted beta smoke step a visible failure signal", async () => {
+    // `continue-on-error: true` is deliberate — a red packaged smoke must not
+    // block the daily beta channel — but it also turns the job green, which is
+    // how a release-gating mac_arm64 case stayed red for eleven days. The
+    // exemption stays; the silence does not. Each exempted smoke step must feed
+    // its outcome to `tools-release write-report`, which turns a failure into a
+    // run annotation plus a caution banner at the top of the step summary.
+    const workflow = await readFile(releaseBetaWorkflowPath, "utf8");
+    const steps = workflow.split(/^ {6}- name: /m).slice(1);
+    const exemptedSmokeStepIds = steps
+      .filter((step) => /^ {8}continue-on-error: true$/m.test(step))
+      .map((step) => /^ {8}id: (\w+_smoke)$/m.exec(step)?.[1])
+      .filter((id): id is string => id != null);
+
+    // Guard the guard: an empty list would make every assertion below vacuous.
+    expect(exemptedSmokeStepIds).toEqual(["mac_arm64_smoke", "win_x64_smoke"]);
+
+    for (const stepId of exemptedSmokeStepIds) {
+      const reportStep = steps.find((step) =>
+        step.includes(`RELEASE_SMOKE_OUTCOME: \${{ steps.${stepId}.outcome }}`),
+      );
+      expect(reportStep, `no report step consumes ${stepId}.outcome`).toBeDefined();
+      expect(reportStep).toContain("pnpm exec tools-release write-report");
+      expect(reportStep).toContain('RELEASE_SMOKE_EXEMPT: "true"');
+      expect(reportStep).toContain("if: ${{ always() }}");
+    }
+  });
+
  it("[P2] daily beta resolve defaults to main and preserves the ref override", async () => {
    // Beta is the daily R&D channel and must track the development tip (main).
    // Selecting the highest-semver release/vX.Y.Z branch stalls the build: once
@@ -0,0 +1,112 @@
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import {
+  capturePackagedFailureEvidence,
+  packagedEvidenceRelpath,
+  PACKAGED_FAILURE_EVIDENCE_DIR,
+} from '@/vitest/packaged-failure-evidence';
+import { createReport } from '@/vitest/report';
+
+const scratchRoots: string[] = [];
+
+async function createScratchReport() {
+  const root = await mkdtemp(join(tmpdir(), 'od-packaged-evidence-'));
+  scratchRoots.push(root);
+  return { report: await createReport(root), root };
+}
+
+afterEach(async () => {
+  while (scratchRoots.length > 0) {
+    const root = scratchRoots.pop();
+    if (root != null) await rm(root, { force: true, recursive: true });
+  }
+});
+
+describe('packaged failure evidence paths', () => {
+  it('keeps every capture inside the report root even for hostile names', () => {
+    expect(packagedEvidenceRelpath(PACKAGED_FAILURE_EVIDENCE_DIR, 'runs.json'))
+      .toBe('first-run-failure/runs.json');
+    expect(packagedEvidenceRelpath('first-run-failure', '../../escape.log'))
+      .toBe('first-run-failure/escape.log');
+    expect(packagedEvidenceRelpath('first-run-failure', '/etc/passwd'))
+      .toBe('first-run-failure/etc-passwd');
+    expect(packagedEvidenceRelpath('first-run-failure', '')).toBe('first-run-failure/unnamed');
+  });
+});
+
+describe('packaged failure evidence capture', () => {
+  it('writes every source into the report and records them in a manifest', async () => {
+    const { report, root } = await createScratchReport();
+
+    const entries = await capturePackagedFailureEvidence(report, PACKAGED_FAILURE_EVIDENCE_DIR, [
+      { name: 'error.txt', read: async () => 'stage timed out\n' },
+      { name: 'runs.json', read: async () => '{"runs":[]}\n' },
+      { name: 'desktop-latest.log', read: async () => Uint8Array.from([104, 105]) },
+    ]);
+
+    expect(entries.map((entry) => entry.status)).toEqual(['saved', 'saved', 'saved']);
+    expect(await readFile(join(root, 'first-run-failure', 'error.txt'), 'utf8'))
+      .toBe('stage timed out\n');
+    expect(await readFile(join(root, 'first-run-failure', 'desktop-latest.log'), 'utf8')).toBe('hi');
+    const manifest = JSON.parse(
+      await readFile(join(root, 'first-run-failure', 'index.json'), 'utf8'),
+    ) as { entries: Array<{ name: string; relpath: string }> };
+    expect(manifest.entries.map((entry) => entry.relpath)).toEqual([
+      'first-run-failure/error.txt',
+      'first-run-failure/runs.json',
+      'first-run-failure/desktop-latest.log',
+    ]);
+  });
+
+  it('keeps capturing after an unreadable source and records why it failed', async () => {
+    // A failed packaged case is usually failing *because* something is broken,
+    // so the first source that cannot be read is the normal case, not the
+    // exceptional one. Losing the remaining evidence to it would reproduce the
+    // exact blind spot this capture exists to remove.
+    const { report, root } = await createScratchReport();
+
+    const entries = await capturePackagedFailureEvidence(report, PACKAGED_FAILURE_EVIDENCE_DIR, [
+      {
+        name: 'runs.json',
+        read: async () => {
+          throw new Error('desktop IPC socket is gone');
+        },
+      },
+      { name: 'error.txt', read: async () => 'stage timed out\n' },
+    ]);
+
+    expect(entries[0]).toMatchObject({ name: 'runs.json', status: 'failed' });
+    expect(entries[0]?.detail).toContain('desktop IPC socket is gone');
+    expect(entries[1]).toMatchObject({ name: 'error.txt', status: 'saved' });
+    expect(await readFile(join(root, 'first-run-failure', 'error.txt'), 'utf8'))
+      .toBe('stage timed out\n');
+  });
+
+  it('never throws out of the cleanup path, even when the report itself is broken', async () => {
+    const brokenReport = {
+      json: async () => {
+        throw new Error('report root is read-only');
+      },
+      save: async () => {
+        throw new Error('report root is read-only');
+      },
+    };
+
+    const entries = await capturePackagedFailureEvidence(brokenReport, PACKAGED_FAILURE_EVIDENCE_DIR, [
+      { name: 'error.txt', read: async () => 'stage timed out\n' },
+    ]);
+
+    expect(entries).toEqual([
+      {
+        detail: 'Error: report root is read-only',
+        name: 'error.txt',
+        relpath: 'first-run-failure/error.txt',
+        status: 'failed',
+      },
+    ]);
+  });
+});
@@ -0,0 +1,338 @@
+import { runInNewContext } from 'node:vm';
+
+import { describe, expect, it } from 'vitest';
+
+import {
+  assertPackagedHomeFirstRunResult,
+  describePackagedHomeFirstRunStall,
+  PACKAGED_HOME_FIRST_RUN_IN_PAGE_WINDOW_MS,
+  PACKAGED_HOME_FIRST_RUN_OUTPUT,
+  PACKAGED_HOME_FIRST_RUN_STAGE_TIMEOUT_MS,
+  packagedHomeFirstRunExpression,
+  packagedHomeFirstRunInPageAwaitMs,
+  packagedHomeFirstRunSnapshotExpression,
+  packagedHomeFirstRunStageSatisfied,
+  type PackagedHomeFirstRunResult,
+} from '@/vitest/packaged-home-first-run';
+
+type FakeResponse = {
+  json: () => Promise<unknown>;
+  ok: boolean;
+  status: number;
+  text: () => Promise<string>;
+};
+
+function jsonResponse(value: unknown): FakeResponse {
+  const body = JSON.stringify(value);
+  return { json: async () => JSON.parse(body), ok: true, status: 200, text: async () => body };
+}
+
+function textResponse(body: string): FakeResponse {
+  return { json: async () => JSON.parse(body), ok: true, status: 200, text: async () => body };
+}
+
+type SnapshotFixtureOptions = {
+  assistantText?: string;
+  daemonAssistantText?: string;
+  eventsText?: string;
+  /** `/api/runs` status per call; the last entry repeats once exhausted. */
+  runStatuses: string[];
+};
+
+function createSnapshotFixture(options: SnapshotFixtureOptions) {
+  const requests: string[] = [];
+  let runsCalls = 0;
+
+  const sandbox: Record<string, unknown> = {
+    document: {
+      querySelectorAll: (selector: string) =>
+        selector === '[data-assistant-message-id]' && options.assistantText != null
+          ? [{ textContent: options.assistantText }]
+          : [],
+      title: 'Open Design',
+    },
+    fetch: async (path: string): Promise<FakeResponse> => {
+      const requestPath = String(path);
+      requests.push(requestPath);
+      if (requestPath.startsWith('/api/runs?')) {
+        const index = Math.min(runsCalls, options.runStatuses.length - 1);
+        runsCalls += 1;
+        const status = options.runStatuses[index];
+        return jsonResponse({ runs: status == null ? [] : [{ id: 'run-1', status }] });
+      }
+      if (requestPath.endsWith('/events')) return textResponse(options.eventsText ?? '');
+      if (requestPath.endsWith('/messages')) {
+        return jsonResponse({
+          messages: [{ content: options.daemonAssistantText ?? '', role: 'assistant' }],
+        });
+      }
+      return jsonResponse({});
+    },
+    location: {
+      href: 'od://app/projects/p1/conversations/c1',
+      pathname: '/projects/p1/conversations/c1',
+    },
+    performance: { getEntriesByType: () => [{}], timeOrigin: 1234 },
+    setTimeout,
+  };
+
+  return {
+    requests,
+    runsCalls: () => runsCalls,
+    sandbox,
+    evaluate: async (expression: string): Promise<PackagedHomeFirstRunResult> =>
+      assertPackagedHomeFirstRunResult(await runInNewContext(expression, sandbox)),
+  };
+}
+
+function snapshotFor(overrides: Partial<PackagedHomeFirstRunResult>): PackagedHomeFirstRunResult {
+  return {
+    assistantText: '',
+    conversationId: 'c1',
+    createRunRequestCount: 1,
+    createRunResponseStatuses: [202],
+    daemonAssistantText: '',
+    hrefAfter: 'od://app/projects/p1/conversations/c1',
+    hrefBefore: 'od://app/',
+    inPageWaitedMs: 0,
+    inputTextBeforeSubmit: 'prompt',
+    navigationEntryCountAfter: 1,
+    navigationEntryCountBefore: 1,
+    outputVisible: false,
+    performanceTimeOriginAfter: 1,
+    performanceTimeOriginBefore: 1,
+    projectId: 'p1',
+    runEventRequestCount: 1,
+    runEventResponseStatuses: [200],
+    runEventsContainExpectedOutput: false,
+    runReachedTerminal: false,
+    runStatuses: ['running'],
+    submitClicked: true,
+    terminalRunStatus: '',
+    workspaceTabClicksBeforeOutput: 0,
+    ...overrides,
+  };
+}
+
+describe('packaged Home first-run budget', () => {
+  it('sizes each stage for its own chain instead of sharing one budget', () => {
+    // The single 15s budget had to cover cold project creation *and* rendering,
+    // which is why only the fastest machines passed. Both stages must now be
+    // large enough that a slower-but-working runtime still finishes.
+    expect(PACKAGED_HOME_FIRST_RUN_STAGE_TIMEOUT_MS['run-terminal']).toBeGreaterThanOrEqual(60_000);
+    expect(PACKAGED_HOME_FIRST_RUN_STAGE_TIMEOUT_MS['assistant-output']).toBeGreaterThanOrEqual(30_000);
+  });
+
+  it('keeps one in-page wait comfortably inside the 5s inspect IPC timeout', () => {
+    expect(PACKAGED_HOME_FIRST_RUN_IN_PAGE_WINDOW_MS).toBeLessThan(5_000);
+  });
+
+  it('clamps the remaining budget into one window and stops asking once it is spent', () => {
+    expect(packagedHomeFirstRunInPageAwaitMs(60_000, 2_500)).toBe(2_500);
+    expect(packagedHomeFirstRunInPageAwaitMs(900, 2_500)).toBe(900);
+    expect(packagedHomeFirstRunInPageAwaitMs(0, 2_500)).toBe(0);
+    expect(packagedHomeFirstRunInPageAwaitMs(-1_000, 2_500)).toBe(0);
+    expect(packagedHomeFirstRunInPageAwaitMs(Number.NaN, 2_500)).toBe(0);
+  });
+});
+
+describe('packaged Home first-run stage predicate', () => {
+  it('ends the first stage on a terminal run and the second on visible output', () => {
+    const running = snapshotFor({ runReachedTerminal: false });
+    const finished = snapshotFor({ runReachedTerminal: true, terminalRunStatus: 'succeeded' });
+    const rendered = snapshotFor({ outputVisible: true, runReachedTerminal: true });
+
+    expect(packagedHomeFirstRunStageSatisfied('run-terminal', running)).toBe(false);
+    expect(packagedHomeFirstRunStageSatisfied('run-terminal', finished)).toBe(true);
+    expect(packagedHomeFirstRunStageSatisfied('assistant-output', finished)).toBe(false);
+    expect(packagedHomeFirstRunStageSatisfied('assistant-output', rendered)).toBe(true);
+  });
+});
+
+describe('packaged Home first-run stall report', () => {
+  it('separates a run that is still going from a run that already failed', () => {
+    expect(
+      describePackagedHomeFirstRunStall('run-terminal', snapshotFor({ runStatuses: ['running'] })),
+    ).toContain('still running');
+    expect(
+      describePackagedHomeFirstRunStall(
+        'assistant-output',
+        snapshotFor({ runReachedTerminal: true, runStatuses: ['failed'], terminalRunStatus: 'failed' }),
+      ),
+    ).toContain('failed run, not a slow one');
+  });
+
+  it('names the earliest missing step in the cold chain', () => {
+    expect(
+      describePackagedHomeFirstRunStall('run-terminal', snapshotFor({ submitClicked: false })),
+    ).toContain('submit was never registered');
+    expect(
+      describePackagedHomeFirstRunStall(
+        'run-terminal',
+        snapshotFor({ hrefAfter: 'od://app/', projectId: '' }),
+      ),
+    ).toContain('never left Home');
+    expect(
+      describePackagedHomeFirstRunStall(
+        'run-terminal',
+        snapshotFor({ createRunRequestCount: 0, createRunResponseStatuses: [] }),
+      ),
+    ).toContain('POST /api/runs was never sent');
+    expect(
+      describePackagedHomeFirstRunStall('run-terminal', snapshotFor({ runStatuses: [] })),
+    ).toContain('no run row for this project');
+  });
+
+  it('lists exactly which observation is missing once the run succeeded', () => {
+    const message = describePackagedHomeFirstRunStall(
+      'assistant-output',
+      snapshotFor({
+        daemonAssistantText: PACKAGED_HOME_FIRST_RUN_OUTPUT,
+        runEventsContainExpectedOutput: true,
+        runReachedTerminal: true,
+        runStatuses: ['succeeded'],
+        terminalRunStatus: 'succeeded',
+      }),
+    );
+
+    expect(message).toContain('the rendered assistant message');
+    expect(message).not.toContain('the daemon conversation messages');
+    expect(message).not.toContain('the run event stream');
+  });
+
+  it('says so when no inspection ever produced a snapshot', () => {
+    expect(describePackagedHomeFirstRunStall('run-terminal', null)).toContain('no readable snapshot');
+  });
+});
+
+describe('packaged Home first-run in-page polling', () => {
+  it('keeps polling inside one inspection until the run reaches a terminal status', async () => {
+    const fixture = createSnapshotFixture({ runStatuses: ['running', 'running', 'succeeded'] });
+
+    const snapshot = await fixture.evaluate(
+      packagedHomeFirstRunSnapshotExpression({ awaitMs: 2_000, pollIntervalMs: 1, stage: 'run-terminal' }),
+    );
+
+    // One inspection, three observations: the sampling rate is no longer bounded
+    // by how fast a `tools-pack mac inspect` process can start.
+    expect(fixture.runsCalls()).toBe(3);
+    expect(snapshot.runReachedTerminal).toBe(true);
+    expect(snapshot.terminalRunStatus).toBe('succeeded');
+    expect(snapshot.runStatuses).toEqual(['succeeded']);
+  });
+
+  it('skips the message and event reads while the run is still going', async () => {
+    const fixture = createSnapshotFixture({ runStatuses: ['running'] });
+
+    await fixture.evaluate(
+      packagedHomeFirstRunSnapshotExpression({ awaitMs: 20, pollIntervalMs: 1, stage: 'run-terminal' }),
+    );
+
+    expect(fixture.requests.every((path) => path.startsWith('/api/runs?'))).toBe(true);
+  });
+
+  it('returns its last observation when the window expires instead of throwing', async () => {
+    const fixture = createSnapshotFixture({ runStatuses: ['running'] });
+
+    const snapshot = await fixture.evaluate(
+      packagedHomeFirstRunSnapshotExpression({ awaitMs: 30, pollIntervalMs: 1, stage: 'run-terminal' }),
+    );
+
+    expect(snapshot.runReachedTerminal).toBe(false);
+    expect(snapshot.runStatuses).toEqual(['running']);
+    expect(snapshot.inPageWaitedMs).toBeGreaterThanOrEqual(30);
+    expect(fixture.runsCalls()).toBeGreaterThan(1);
+  });
+
+  it('reports output as visible only when all three observations carry it', async () => {
+    const partial = createSnapshotFixture({
+      daemonAssistantText: PACKAGED_HOME_FIRST_RUN_OUTPUT,
+      eventsText: 'nothing useful',
+      assistantText: PACKAGED_HOME_FIRST_RUN_OUTPUT,
+      runStatuses: ['succeeded'],
+    });
+    const complete = createSnapshotFixture({
+      assistantText: PACKAGED_HOME_FIRST_RUN_OUTPUT,
+      daemonAssistantText: PACKAGED_HOME_FIRST_RUN_OUTPUT,
+      eventsText: `data: ${PACKAGED_HOME_FIRST_RUN_OUTPUT}`,
+      runStatuses: ['succeeded'],
+    });
+    const expression = packagedHomeFirstRunSnapshotExpression({
+      awaitMs: 20,
+      pollIntervalMs: 1,
+      stage: 'assistant-output',
+    });
+
+    await expect(partial.evaluate(expression)).resolves.toMatchObject({
+      outputVisible: false,
+      runEventsContainExpectedOutput: false,
+    });
+    await expect(complete.evaluate(expression)).resolves.toMatchObject({
+      outputVisible: true,
+      runEventsContainExpectedOutput: true,
+    });
+  });
+});
+
+describe('packaged Home first-run instrumentation', () => {
+  it('observes run creation without faking a failure the daemon cannot produce', async () => {
+    // `POST /api/runs` never answers WORKSPACE_AUTHORITY_UNAVAILABLE: local run
+    // creation does not synchronously depend on Workspace membership authority
+    // (apps/daemon/src/collab/project-request-authority.ts). Injecting that 503
+    // here tested a branch production cannot reach and spent budget the cold
+    // chain needed. Run-create retry semantics stay covered by the web unit test
+    // in apps/web/tests/providers/sse.test.ts.
+    const upstream: Array<{ method: string; url: string }> = [];
+    class FixtureComposer {
+      __lexicalEditor = {
+        parseEditorState: (value: string) => JSON.parse(value) as unknown,
+        setEditorState: (value: unknown) => {
+          const state = value as { root?: { children?: Array<{ children?: Array<{ text?: string }> }> } };
+          this.textContent = state.root?.children?.[0]?.children?.[0]?.text ?? '';
+        },
+      };
+
+      isContentEditable = true;
+      textContent = '';
+
+      focus(): void {}
+
+      getClientRects(): ArrayLike<unknown> {
+        return [{}];
+      }
+    }
+    const composer = new FixtureComposer();
+    const sandbox: Record<string, unknown> = {
+      Element: FixtureComposer,
+      HTMLElement: FixtureComposer,
+      Headers,
+      Request,
+      URL,
+      document: {
+        addEventListener: () => undefined,
+        querySelector: (selector: string) =>
+          selector === '[data-testid="home-hero-input"]' ? composer : null,
+      },
+      fetch: async (url: string, init?: { method?: string }) => {
+        upstream.push({ method: init?.method ?? 'GET', url: String(url) });
+        return jsonResponse({ runId: 'run-1' });
+      },
+      location: { href: 'od://app/', pathname: '/' },
+      performance: { getEntriesByType: () => [{}], timeOrigin: 7 },
+      setTimeout,
+    };
+
+    await runInNewContext(packagedHomeFirstRunExpression(), sandbox);
+    const wrappedFetch = sandbox.fetch as (url: string, init?: { method?: string }) => Promise<FakeResponse>;
+    const response = await wrappedFetch('/api/runs', { method: 'POST' });
+    const state = sandbox.__odPackagedHomeFirstRun as {
+      createRunRequestCount: number;
+      createRunResponseStatuses: number[];
+    };
+
+    expect(response.status).toBe(200);
+    expect(upstream).toEqual([{ method: 'POST', url: '/api/runs' }]);
+    expect(state.createRunRequestCount).toBe(1);
+    expect(state.createRunResponseStatuses).toEqual([200]);
+  });
+});
@@ -0,0 +1,81 @@
+/**
+ * Make a swallowed packaged-smoke failure impossible to miss.
+ *
+ * The release smoke steps run with `continue-on-error: true` on purpose: a red
+ * smoke must not block the daily beta channel. The cost of that exemption is
+ * that the job reports success, so the only trace a failure leaves is a `status`
+ * line buried in a per-target report — which is how a release-gating packaged
+ * case stayed red for eleven days without anyone noticing.
+ *
+ * This restores the signal without restoring the block: a `::error` annotation
+ * on the run and a caution banner at the top of the target's step summary, while
+ * the job conclusion stays exactly as it was.
+ */
+
+export type SwallowedSmokeInput = {
+  /** `true` when the smoke step carries `continue-on-error`, so the job stays green. */
+  exempt: boolean;
+  /** GitHub step outcome of the smoke step: success | failure | cancelled | skipped. */
+  outcome: string;
+  reportPath: string;
+  /** Status derived from the suite result, used when no step outcome is wired. */
+  suiteStatus: string;
+  target: string;
+  title: string;
+  version: string;
+};
+
+export type SwallowedSmokeAnnouncement = {
+  /** A GitHub workflow `::error` command, or null when the smoke did not fail. */
+  annotation: string | null;
+  /** Markdown lines to place above the report body, or empty when it did not fail. */
+  banner: string[];
+  failed: boolean;
+};
+
+export function smokeFailed(outcome: string, suiteStatus: string): boolean {
+  return outcome.trim() === "failure" || suiteStatus.trim() === "failed";
+}
+
+export function announceSwallowedSmoke(input: SwallowedSmokeInput): SwallowedSmokeAnnouncement {
+  if (!smokeFailed(input.outcome, input.suiteStatus)) {
+    return { annotation: null, banner: [], failed: false };
+  }
+
+  const label = input.title.trim().length > 0 ? input.title.trim() : `${input.target} packaged smoke`;
+  const version = input.version.trim().length > 0 ? input.version.trim() : "(unknown version)";
+  const exemptSentence = input.exempt
+    ? "This step runs with `continue-on-error: true`, so the job below still reports success and the release was not blocked. The failure is real — a green check on this job is not a pass."
+    : "This step is a blocking gate, so the job is red.";
+  const banner = [
+    "> [!CAUTION]",
+    `> **${label}: packaged smoke FAILED** (version \`${version}\`, target \`${input.target}\`).`,
+    ">",
+    `> ${exemptSentence}`,
+  ];
+  if (input.reportPath.trim().length > 0) {
+    banner.push(">", `> Evidence: \`${input.reportPath.trim()}\``);
+  }
+  banner.push("");
+
+  const annotationTitle = `${input.target} packaged smoke failed`;
+  const annotationMessage = input.exempt
+    ? `${label} failed for ${version}. continue-on-error kept the job green on purpose; the failure still needs an owner.`
+    : `${label} failed for ${version}.`;
+
+  return {
+    annotation: `::error title=${escapeAnnotationProperty(annotationTitle)}::${escapeAnnotationData(annotationMessage)}`,
+    banner,
+    failed: true,
+  };
+}
+
+/** GitHub workflow-command escaping for the message body. */
+function escapeAnnotationData(value: string): string {
+  return value.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
+}
+
+/** Property values additionally escape the command's own delimiters. */
+function escapeAnnotationProperty(value: string): string {
+  return escapeAnnotationData(value).replace(/:/g, "%3A").replace(/,/g, "%2C");
+}
@@ -1,6 +1,8 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";

+import { announceSwallowedSmoke } from "./smoke-annotation.ts";
+
 type JsonRecord = Record<string, unknown>;

 function required(name: string): string {
@@ -124,6 +126,17 @@ const cacheEntries = arrayOrEmpty(cacheReport?.entries ?? index?.cache);
 const buildSegments = arrayOrEmpty(build?.segments ?? index?.buildSegments);
 const totalDurationMs = numberOrNull(index?.durationMs) ?? numberOrNull(suiteResult?.durationMs) ?? null;

+const reportTitle = optional("REPORT_TITLE", `${releaseTarget} release report`);
+const smoke = announceSwallowedSmoke({
+  exempt: optional("RELEASE_SMOKE_EXEMPT") === "true",
+  outcome: optional("RELEASE_SMOKE_OUTCOME"),
+  reportPath: reportRoot,
+  suiteStatus: sourceStatus,
+  target: releaseTarget,
+  title: reportTitle,
+  version: optional("RELEASE_VERSION", String(index?.releaseVersion ?? manifest?.releaseVersion ?? "")),
+});
+
 const report = {
  version: 1,
  generatedAt: new Date().toISOString(),
@@ -145,6 +158,11 @@ const report = {
    smokeMode: optional("RELEASE_SMOKE_MODE", String(index?.smokeMode ?? "")),
    target: optional("RELEASE_BUILD_TARGET", String(index?.target ?? "")),
  },
+  smoke: {
+    exempt: optional("RELEASE_SMOKE_EXEMPT") === "true",
+    failed: smoke.failed,
+    outcome: optional("RELEASE_SMOKE_OUTCOME"),
+  },
  paths: {
    reportRoot,
    ...(buildJsonPath.length === 0 ? {} : { buildJsonPath: resolvePath(buildJsonPath) }),
@@ -176,7 +194,10 @@ mkdirSync(dirname(reportJsonPath), { recursive: true });
 writeFileSync(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");

 const lines = [
-  `### ${optional("REPORT_TITLE", `${releaseTarget} release report`)}`,
+  // The banner leads: a swallowed smoke failure must be the first thing anyone
+  // reading this job's summary sees, not a `status` line inside the report body.
+  ...smoke.banner,
+  `### ${reportTitle}`,
  "",
  `- status: ${code(sourceStatus)}`,
  `- target: ${code(releaseTarget)}`,
@@ -203,4 +224,8 @@ if (cacheEntries.length > 0) {

 mkdirSync(dirname(reportSummaryPath), { recursive: true });
 writeFileSync(reportSummaryPath, `${lines.join("\n")}\n`, "utf8");
+// Annotations surface at the top of the run page and in the job's annotation
+// list. `::error` does not fail a step, so the continue-on-error exemption is
+// preserved: the release still ships, but nobody can claim the red was silent.
+if (smoke.annotation != null) console.log(smoke.annotation);
 console.log(`wrote release report ${reportJsonPath}`);
@@ -0,0 +1,75 @@
+import { describe, expect, it } from "vitest";
+
+import { announceSwallowedSmoke, smokeFailed } from "../src/report/smoke-annotation.js";
+
+const base = {
+  exempt: true,
+  outcome: "failure",
+  reportPath: "/tmp/release-report/mac_arm64",
+  suiteStatus: "failed",
+  target: "mac_arm64",
+  title: "mac_arm64 beta build",
+  version: "0.21.6-beta.7",
+};
+
+describe("swallowed smoke detection", () => {
+  it("treats a failed step outcome or a failed suite result as a failure", () => {
+    expect(smokeFailed("failure", "unknown")).toBe(true);
+    expect(smokeFailed("", "failed")).toBe(true);
+    expect(smokeFailed("success", "success")).toBe(false);
+    expect(smokeFailed("skipped", "unknown")).toBe(false);
+    expect(smokeFailed("", "")).toBe(false);
+  });
+});
+
+describe("swallowed smoke announcement", () => {
+  it("stays silent when the smoke passed or never ran", () => {
+    expect(announceSwallowedSmoke({ ...base, outcome: "success", suiteStatus: "success" }))
+      .toEqual({ annotation: null, banner: [], failed: false });
+    expect(announceSwallowedSmoke({ ...base, outcome: "skipped", suiteStatus: "unknown" }))
+      .toEqual({ annotation: null, banner: [], failed: false });
+  });
+
+  it("emits a run annotation and a leading banner when the failure was exempted", () => {
+    const announcement = announceSwallowedSmoke(base);
+
+    expect(announcement.failed).toBe(true);
+    expect(announcement.annotation).toContain("::error title=mac_arm64 packaged smoke failed::");
+    expect(announcement.annotation).toContain("continue-on-error kept the job green");
+    expect(announcement.banner[0]).toBe("> [!CAUTION]");
+    expect(announcement.banner.join("\n")).toContain("packaged smoke FAILED");
+    expect(announcement.banner.join("\n")).toContain("0.21.6-beta.7");
+    // The exemption is the whole reason this signal exists: a reader who sees a
+    // green job must be told, in the summary itself, that green is not a pass.
+    expect(announcement.banner.join("\n")).toContain("a green check on this job is not a pass");
+    expect(announcement.banner.join("\n")).toContain("/tmp/release-report/mac_arm64");
+  });
+
+  it("does not claim the job stayed green when the smoke was a blocking gate", () => {
+    const announcement = announceSwallowedSmoke({ ...base, exempt: false });
+
+    expect(announcement.failed).toBe(true);
+    expect(announcement.banner.join("\n")).toContain("blocking gate");
+    expect(announcement.banner.join("\n")).not.toContain("continue-on-error");
+    expect(announcement.annotation).not.toContain("continue-on-error");
+  });
+
+  it("escapes workflow-command delimiters so the annotation cannot be truncated", () => {
+    const announcement = announceSwallowedSmoke({
+      ...base,
+      target: "mac:arm64,x64",
+      version: "1.0.0\nsecond line",
+    });
+
+    expect(announcement.annotation).toContain("title=mac%3Aarm64%2Cx64 packaged smoke failed::");
+    expect(announcement.annotation).toContain("1.0.0%0Asecond line");
+    expect(announcement.annotation).not.toContain("\n");
+  });
+
+  it("falls back to the target when no report title is configured", () => {
+    const announcement = announceSwallowedSmoke({ ...base, reportPath: "", title: "  " });
+
+    expect(announcement.banner.join("\n")).toContain("mac_arm64 packaged smoke");
+    expect(announcement.banner.join("\n")).not.toContain("Evidence:");
+  });
+});
@@ -0,0 +1,101 @@
+import { execFile as execFileCallback } from "node:child_process";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+
+import { describe, expect, it } from "vitest";
+
+const execFileAsync = promisify(execFileCallback);
+const require = createRequire(import.meta.url);
+const testDir = dirname(fileURLToPath(import.meta.url));
+const workspaceRoot = resolve(testDir, "..", "..", "..");
+const tsxCliPath = require.resolve("tsx/cli");
+
+type WriteReportRun = {
+  report: Record<string, unknown>;
+  stdout: string;
+  summary: string;
+};
+
+async function runWriteReport(
+  env: Record<string, string>,
+  suiteStatus: string,
+): Promise<WriteReportRun> {
+  const root = await mkdtemp(join(tmpdir(), "od-write-report-"));
+  const reportRoot = join(root, "release-report");
+  try {
+    await mkdir(reportRoot, { recursive: true });
+    await writeFile(
+      join(reportRoot, "suite-result.json"),
+      `${JSON.stringify({ durationMs: 1234, exitCode: suiteStatus === "failed" ? 1 : 0, status: suiteStatus })}\n`,
+      "utf8",
+    );
+    const result = await execFileAsync(
+      process.execPath,
+      [tsxCliPath, "tools/release/src/index.ts", "write-report"],
+      {
+        cwd: workspaceRoot,
+        env: {
+          ...process.env,
+          RELEASE_REPORT_DIR: reportRoot,
+          RELEASE_REPORT_JSON_PATH: join(reportRoot, "report.json"),
+          RELEASE_REPORT_SUMMARY_PATH: join(reportRoot, "summary.md"),
+          RELEASE_TARGET: "mac_arm64",
+          RELEASE_VERSION: "0.21.6-beta.7",
+          REPORT_TITLE: "mac_arm64 beta build",
+          ...env,
+        },
+      },
+    );
+    return {
+      report: JSON.parse(await readFile(join(reportRoot, "report.json"), "utf8")) as Record<string, unknown>,
+      stdout: result.stdout,
+      summary: await readFile(join(reportRoot, "summary.md"), "utf8"),
+    };
+  } finally {
+    await rm(root, { force: true, recursive: true });
+  }
+}
+
+describe("release report smoke signal", () => {
+  it("leads the step summary with the failure and annotates the run", async () => {
+    // This is the whole point of the change: the summary is published to
+    // $GITHUB_STEP_SUMMARY by the very next workflow step, and the annotation
+    // lands on the run page — so a continue-on-error smoke failure can no longer
+    // be reduced to one `status` line inside a report nobody scrolls to.
+    const run = await runWriteReport(
+      { RELEASE_SMOKE_EXEMPT: "true", RELEASE_SMOKE_OUTCOME: "failure" },
+      "failed",
+    );
+
+    expect(run.summary.startsWith("> [!CAUTION]")).toBe(true);
+    expect(run.summary).toContain("packaged smoke FAILED");
+    expect(run.summary).toContain("0.21.6-beta.7");
+    expect(run.summary).toContain("### mac_arm64 beta build");
+    expect(run.stdout).toContain("::error title=mac_arm64 packaged smoke failed::");
+    expect(run.report.smoke).toEqual({ exempt: true, failed: true, outcome: "failure" });
+  }, 60_000);
+
+  it("keeps a passing smoke summary unchanged", async () => {
+    const run = await runWriteReport(
+      { RELEASE_SMOKE_EXEMPT: "true", RELEASE_SMOKE_OUTCOME: "success" },
+      "success",
+    );
+
+    expect(run.summary.startsWith("### mac_arm64 beta build")).toBe(true);
+    expect(run.summary).not.toContain("[!CAUTION]");
+    expect(run.stdout).not.toContain("::error");
+    expect(run.report.smoke).toEqual({ exempt: true, failed: false, outcome: "success" });
+  }, 60_000);
+
+  it("still announces a failure for callers that only wire the suite result", async () => {
+    const run = await runWriteReport({}, "failed");
+
+    expect(run.summary.startsWith("> [!CAUTION]")).toBe(true);
+    expect(run.stdout).toContain("::error");
+    expect(run.report.smoke).toEqual({ exempt: false, failed: true, outcome: "" });
+  }, 60_000);
+});