From 6bcc5e99eeccff1c3948fcb86338215751ef5f10 Mon Sep 17 00:00:00 2001 From: elihahah666 Date: Sun, 31 May 2026 23:03:57 +0800 Subject: [PATCH] feat(analytics): mark clarification-turn runs in run_finished (#3384) The first session of a task does intent clarification by raising an AskUserQuestion card, which inherently produces no artifact. These clarification turns were indistinguishable from artifact-generation failures on the dashboard, inflating the "run finished -> has artifact" dropoff. Add a required `asked_user_question: boolean` to RunFinishedProps, populated daemon-side via a new `runAskedUserQuestion` helper that scans the run's agent events for an AskUserQuestion / ask_user_question tool_use. The dashboard can now exclude clarification turns from the artifact funnel instead of scoring them as failures. Co-authored-by: qiongyu1999 <2694684348@qq.com> --- apps/daemon/src/run-artifacts.ts | 33 ++++++++++++++++ apps/daemon/src/server.ts | 7 ++++ apps/daemon/tests/run-artifacts.test.ts | 45 ++++++++++++++++++++++ packages/contracts/src/analytics/events.ts | 6 +++ 4 files changed, 91 insertions(+) diff --git a/apps/daemon/src/run-artifacts.ts b/apps/daemon/src/run-artifacts.ts index f6980086a0..cf8708f774 100644 --- a/apps/daemon/src/run-artifacts.ts +++ b/apps/daemon/src/run-artifacts.ts @@ -36,6 +36,16 @@ const WRITE_OR_EDIT_TOOL_NAMES: ReadonlySet = new Set([ 'multi_edit', ]); +// Tool names the daemon recognizes as an intent-clarification card. Claude +// emits `AskUserQuestion`; the ACP/MCP snake_case proxy shape emits +// `ask_user_question`. Keep aligned with the AskUserQuestion detection in +// `apps/daemon/src/server.ts` and the card rendering in +// `apps/web/src/components/ToolCard.tsx`. +const ASK_USER_QUESTION_TOOL_NAMES: ReadonlySet = new Set([ + 'AskUserQuestion', + 'ask_user_question', +]); + function extractToolFilePath(input: unknown): string | null { if (!input || typeof input !== 'object') return null; const obj = input as { file_path?: unknown; path?: unknown }; @@ -197,3 +207,26 @@ export function countNewHtmlArtifacts(events: readonly RunEventLike[]): number { } return writtenPaths.size; } + +// True iff the run raised an AskUserQuestion clarification card. Fed into +// `run_finished.asked_user_question`. A clarification turn is the agent +// stopping to ask the user a finite-choice question; it inherently produces +// no artifact, so the dashboard uses this flag to exclude such runs from the +// "run finished -> has artifact" funnel rather than scoring them as +// artifact-generation failures. +export function runAskedUserQuestion( + events: readonly RunEventLike[], +): boolean { + if (!events || events.length === 0) return false; + for (const rec of events) { + if (rec?.event !== 'agent') continue; + const data = rec.data as + | { type?: string; name?: unknown } + | null + | undefined; + if (data?.type !== 'tool_use') continue; + if (typeof data.name !== 'string') continue; + if (ASK_USER_QUESTION_TOOL_NAMES.has(data.name)) return true; + } + return false; +} diff --git a/apps/daemon/src/server.ts b/apps/daemon/src/server.ts index 6ea8614544..3629320b67 100644 --- a/apps/daemon/src/server.ts +++ b/apps/daemon/src/server.ts @@ -241,6 +241,7 @@ import { countDesignSystemPreviewModules, countNewHtmlArtifacts, didRunCreateDesignSystemFile, + runAskedUserQuestion, } from './run-artifacts.js'; import { reportRunCompletedFromDaemon, @@ -13532,6 +13533,12 @@ export async function startServer({ // for the dedup semantics; tested in // `tests/run-artifacts.test.ts`. artifact_count: countNewHtmlArtifacts(run.events), + // True when the run raised an AskUserQuestion clarification + // card. Clarification turns inherently produce no artifact, so + // the dashboard excludes them from the "run finished -> has + // artifact" funnel instead of counting them as failures. See + // `run-artifacts.ts`; tested in `tests/run-artifacts.test.ts`. + asked_user_question: runAskedUserQuestion(run.events), ...(isDesignSystemRun ? { // DS runs land a `DESIGN.md` write when generation // succeeded; the run-artifacts inspector reuses the diff --git a/apps/daemon/tests/run-artifacts.test.ts b/apps/daemon/tests/run-artifacts.test.ts index 1aa3fd83b0..d14b39bb08 100644 --- a/apps/daemon/tests/run-artifacts.test.ts +++ b/apps/daemon/tests/run-artifacts.test.ts @@ -20,6 +20,7 @@ import { countDesignSystemPreviewModules, countNewHtmlArtifacts, didRunCreateDesignSystemFile, + runAskedUserQuestion, } from '../src/run-artifacts.js'; let nextId = 0; @@ -288,3 +289,47 @@ describe('countDesignSystemPreviewModules', () => { ).toBe(1); }); }); + +// Helper: emit a bare tool_use (no result) for a named tool. Clarification +// detection only needs the tool_use to appear; AskUserQuestion is answered +// out-of-band via POST /api/runs/:id/tool-result, not a stream tool_result. +function toolUse(name: string, id = freshId()) { + return [ + { + event: 'agent', + data: { type: 'tool_use', id, name, input: {} }, + }, + ]; +} + +describe('runAskedUserQuestion', () => { + it('returns false for an empty event list', () => { + expect(runAskedUserQuestion([])).toBe(false); + }); + + it('returns true when the run raised an AskUserQuestion card', () => { + expect(runAskedUserQuestion(toolUse('AskUserQuestion'))).toBe(true); + }); + + it('matches the snake_case ask_user_question proxy shape', () => { + expect(runAskedUserQuestion(toolUse('ask_user_question'))).toBe(true); + }); + + it('returns false for a run that only wrote artifacts', () => { + expect( + runAskedUserQuestion([ + ...pair('Write', '/proj/index.html'), + ...toolUse('Read'), + ]), + ).toBe(false); + }); + + it('detects the card even when mixed with other tool calls', () => { + expect( + runAskedUserQuestion([ + ...pair('Write', '/proj/index.html'), + ...toolUse('AskUserQuestion'), + ]), + ).toBe(true); + }); +}); diff --git a/packages/contracts/src/analytics/events.ts b/packages/contracts/src/analytics/events.ts index 5aff30b426..5b43bdacc6 100644 --- a/packages/contracts/src/analytics/events.ts +++ b/packages/contracts/src/analytics/events.ts @@ -1757,6 +1757,12 @@ export interface RunFinishedProps extends Omit { result: TrackingRunResult; error_code?: string; artifact_count: number; + // True when the run raised an AskUserQuestion clarification card. Such runs + // are intent-clarification turns (the agent stops to ask the user a question) + // and therefore inherently produce no artifact, so the dashboard can exclude + // them from the "run finished -> has artifact" funnel instead of counting + // them as artifact-generation failures. + asked_user_question: boolean; input_tokens?: number; output_tokens?: number; total_tokens?: number;