feat(analytics): mark clarification-turn runs in run_finished (#3384)

The first session of a task does intent clarification by raising an
AskUserQuestion card, which inherently produces no artifact. These
clarification turns were indistinguishable from artifact-generation
failures on the dashboard, inflating the "run finished -> has artifact"
dropoff.

Add a required `asked_user_question: boolean` to RunFinishedProps,
populated daemon-side via a new `runAskedUserQuestion` helper that scans
the run's agent events for an AskUserQuestion / ask_user_question
tool_use. The dashboard can now exclude clarification turns from the
artifact funnel instead of scoring them as failures.

Co-authored-by: qiongyu1999 <2694684348@qq.com>

apps/daemon/src/run-artifacts.ts

@@ -36,6 +36,16 @@ const WRITE_OR_EDIT_TOOL_NAMES: ReadonlySet<string> = new Set([
   'multi_edit',
 ]);
 
+// Tool names the daemon recognizes as an intent-clarification card. Claude
+// emits `AskUserQuestion`; the ACP/MCP snake_case proxy shape emits
+// `ask_user_question`. Keep aligned with the AskUserQuestion detection in
+// `apps/daemon/src/server.ts` and the card rendering in
+// `apps/web/src/components/ToolCard.tsx`.
+const ASK_USER_QUESTION_TOOL_NAMES: ReadonlySet<string> = new Set([
+  'AskUserQuestion',
+  'ask_user_question',
+]);
+
 function extractToolFilePath(input: unknown): string | null {
   if (!input || typeof input !== 'object') return null;
   const obj = input as { file_path?: unknown; path?: unknown };
@@ -197,3 +207,26 @@ export function countNewHtmlArtifacts(events: readonly RunEventLike[]): number {
   }
   return writtenPaths.size;
 }
+
+// True iff the run raised an AskUserQuestion clarification card. Fed into
+// `run_finished.asked_user_question`. A clarification turn is the agent
+// stopping to ask the user a finite-choice question; it inherently produces
+// no artifact, so the dashboard uses this flag to exclude such runs from the
+// "run finished -> has artifact" funnel rather than scoring them as
+// artifact-generation failures.
+export function runAskedUserQuestion(
+  events: readonly RunEventLike[],
+): boolean {
+  if (!events || events.length === 0) return false;
+  for (const rec of events) {
+    if (rec?.event !== 'agent') continue;
+    const data = rec.data as
+      | { type?: string; name?: unknown }
+      | null
+      | undefined;
+    if (data?.type !== 'tool_use') continue;
+    if (typeof data.name !== 'string') continue;
+    if (ASK_USER_QUESTION_TOOL_NAMES.has(data.name)) return true;
+  }
+  return false;
+}

apps/daemon/src/server.ts

@@ -241,6 +241,7 @@ import {
   countDesignSystemPreviewModules,
   countNewHtmlArtifacts,
   didRunCreateDesignSystemFile,
+  runAskedUserQuestion,
 } from './run-artifacts.js';
 import {
   reportRunCompletedFromDaemon,
@@ -13532,6 +13533,12 @@ export async function startServer({
             // for the dedup semantics; tested in
             // `tests/run-artifacts.test.ts`.
             artifact_count: countNewHtmlArtifacts(run.events),
+            // True when the run raised an AskUserQuestion clarification
+            // card. Clarification turns inherently produce no artifact, so
+            // the dashboard excludes them from the "run finished -> has
+            // artifact" funnel instead of counting them as failures. See
+            // `run-artifacts.ts`; tested in `tests/run-artifacts.test.ts`.
+            asked_user_question: runAskedUserQuestion(run.events),
             ...(isDesignSystemRun ? {
               // DS runs land a `DESIGN.md` write when generation
               // succeeded; the run-artifacts inspector reuses the

apps/daemon/tests/run-artifacts.test.ts

@@ -20,6 +20,7 @@ import {
   countDesignSystemPreviewModules,
   countNewHtmlArtifacts,
   didRunCreateDesignSystemFile,
+  runAskedUserQuestion,
 } from '../src/run-artifacts.js';
 
 let nextId = 0;
@@ -288,3 +289,47 @@ describe('countDesignSystemPreviewModules', () => {
     ).toBe(1);
   });
 });
+
+// Helper: emit a bare tool_use (no result) for a named tool. Clarification
+// detection only needs the tool_use to appear; AskUserQuestion is answered
+// out-of-band via POST /api/runs/:id/tool-result, not a stream tool_result.
+function toolUse(name: string, id = freshId()) {
+  return [
+    {
+      event: 'agent',
+      data: { type: 'tool_use', id, name, input: {} },
+    },
+  ];
+}
+
+describe('runAskedUserQuestion', () => {
+  it('returns false for an empty event list', () => {
+    expect(runAskedUserQuestion([])).toBe(false);
+  });
+
+  it('returns true when the run raised an AskUserQuestion card', () => {
+    expect(runAskedUserQuestion(toolUse('AskUserQuestion'))).toBe(true);
+  });
+
+  it('matches the snake_case ask_user_question proxy shape', () => {
+    expect(runAskedUserQuestion(toolUse('ask_user_question'))).toBe(true);
+  });
+
+  it('returns false for a run that only wrote artifacts', () => {
+    expect(
+      runAskedUserQuestion([
+        ...pair('Write', '/proj/index.html'),
+        ...toolUse('Read'),
+      ]),
+    ).toBe(false);
+  });
+
+  it('detects the card even when mixed with other tool calls', () => {
+    expect(
+      runAskedUserQuestion([
+        ...pair('Write', '/proj/index.html'),
+        ...toolUse('AskUserQuestion'),
+      ]),
+    ).toBe(true);
+  });
+});

packages/contracts/src/analytics/events.ts

@@ -1757,6 +1757,12 @@ export interface RunFinishedProps extends Omit<RunCreatedProps, 'area'> {
   result: TrackingRunResult;
   error_code?: string;
   artifact_count: number;
+  // True when the run raised an AskUserQuestion clarification card. Such runs
+  // are intent-clarification turns (the agent stops to ask the user a question)
+  // and therefore inherently produce no artifact, so the dashboard can exclude
+  // them from the "run finished -> has artifact" funnel instead of counting
+  // them as artifact-generation failures.
+  asked_user_question: boolean;
   input_tokens?: number;
   output_tokens?: number;
   total_tokens?: number;