fix(daemon): classify deterministic failures and preserve retry attribution (#5882)

apps/daemon/src/run-failure-classification.ts

@@ -257,16 +257,50 @@ function isSessionResumeExpiredText(text: string): boolean {
     /\bsession [\w-]+ not found\b/i.test(text);
 }
 
-function isPromptTooLargeText(text: string): boolean {
+function promptTooLargeDetail(text: string): TrackingRunFailureDetail | null {
+  if (
+    /\b(?:Payload Too Large|Request Entity Too Large|request entity too large|request body exceeds configured limit)\b/i.test(text) ||
+    /\[code=request_too_large\]/i.test(text)
+  ) {
+    return 'request_too_large';
+  }
   // `prefill context too large` is the local-runtime (MLX) shape of the same
   // "the prompt does not fit" failure that currently leaks into execution_failed.
-  return /\b(context window|prompt too large|maximum context|too many tokens|input.*too large|request (?:body )?exceeds configured limit|output token maximum|maximum output tokens|CLAUDE_CODE_MAX_OUTPUT_TOKENS|exceeds the safe size|composed prompt exceeds|prompt token count .* exceeds|maximum context length|context too large|prefill context too large|reduce the length of (?:the )?(?:messages|input prompt)|request \(\d+ tokens\) exceeds the available context size|n_keep:\s*\d+\s*>=\s*n_ctx)\b/i
-    .test(text);
+  if (
+    /\b(context window|context size (?:has been )?exceeded|prompt too large|maximum context|too many tokens|input.*too large|request (?:body )?exceeds configured limit|output token maximum|maximum output tokens|CLAUDE_CODE_MAX_OUTPUT_TOKENS|exceeds the safe size|composed prompt exceeds|prompt token count .* exceeds|maximum context length|context too large|prefill context too large|reduce the length of (?:the )?(?:messages|input prompt)|request \(\d+ tokens\) exceeds the available context size|n_keep:\s*\d+\s*>=\s*n_ctx)\b/i.test(text)
+  ) {
+    return 'prompt_too_large';
+  }
+  return null;
+}
+
+function clientRequestFailureDetail(text: string): TrackingRunFailureDetail | null {
+  if (
+    /\bsource\.media_type\b[\s\S]*\bInvalid enum value\b[\s\S]*\bapplication\/pdf\b/i.test(text) ||
+    /\bapplication\/pdf\b[\s\S]*\bexpected\b[\s\S]*\bimage\/(?:jpeg|png|gif|webp)\b/i.test(text)
+  ) {
+    return 'attachment_media_type_unsupported';
+  }
+  if (
+    /\bfunction_declarations\[\d+\]\.name\b[\s\S]*\bInvalid function name\b/i.test(text)
+  ) {
+    return 'tool_schema_invalid';
+  }
+  if (/\bFailed to tokenize (?:the )?prompt\b/i.test(text)) {
+    return 'prompt_tokenization_failed';
+  }
+  if (
+    /["']?status["']?\s*:\s*404\b[\s\S]*\bFunction\s+["'][^"']+["']\s+Not found for account\b/i.test(text) ||
+    /\bFunction\s+["'][^"']+["']\s+Not found for account\b[\s\S]*["']?status["']?\s*:\s*404\b/i.test(text)
+  ) {
+    return 'provider_resource_not_found';
+  }
+  return null;
 }
 
 function isUpstreamDetailText(text: string): boolean {
   return isUpstreamClientErrorText(text) ||
-    /\b(stream disconnected before completion|response\.completed|Transport error: network error|Upstream request failed|websocket closed|socket connection was closed unexpectedly|tls handshake eof|Connection reset by (?:peer|server)|TLS close_notify|Broken pipe|remote host|远程主机强迫关闭|No route to host|Connection refused|ConnectionRefused|error sending request|Provider returned error|high demand|model is at capacity|selected model is at capacity|temporarily unavailable|upstream_error|http2: response body closed|peer closed connection|incomplete chunked read|Client network socket disconnected before secure TLS connection|Connection failed repeatedly|lost its connection to (?:the Anthropic API|the configured custom Anthropic endpoint)|Server error mid-response|empty or malformed response|Unexpected server error|Streaming response failed|Failed to process error response|AMR model catalog is (?:temporarily )?unavailable)\b/i
+    /\b(stream disconnected before completion|(?:stream|upstream) idle timeout|response\.completed|Transport error: network error|Upstream request failed|websocket closed|socket connection was closed unexpectedly|tls handshake eof|Connection reset by (?:peer|server)|TLS close_notify|Broken pipe|remote host|远程主机强迫关闭|No route to host|Connection refused|ConnectionRefused|error sending request|Provider returned error|high demand|model is at capacity|selected model is at capacity|temporarily unavailable|upstream_error|http2: response body closed|peer closed connection|incomplete chunked read|Client network socket disconnected before secure TLS connection|Connection failed repeatedly|lost its connection to (?:the Anthropic API|the configured custom Anthropic endpoint)|Server error mid-response|empty or malformed response|Unexpected server error|Streaming response failed|Failed to process error response|AMR model catalog is (?:temporarily )?unavailable)\b/i
       .test(text);
 }
 
@@ -310,7 +344,7 @@ function modelUnavailableDetail(text: string): TrackingRunFailureDetail | null {
   if (/\b(no endpoints found that support tool use|provider routing)\b/i.test(text)) {
     return 'provider_routing_error';
   }
-  if (/\b(model .*not supported|requested model is not supported|supported api model names|not supported when using codex)\b/i.test(text)) {
+  if (/\b(model .*not supported|not supported model\b|requested model is not supported|supported api model names|not supported when using codex)\b/i.test(text)) {
     return 'model_not_supported';
   }
   if (/\b(model (?:is )?(?:unavailable|not available|unsupported|not found)|selected model is not available|not have access|no access|model .*not found|no healthy deployments|model .*not in (?:the )?allowed list)\b/i.test(text)) {
@@ -346,7 +380,7 @@ function upstreamDetail(text: string): TrackingRunFailureDetail {
     return 'provider_routing_error';
   }
   if (/\bhigh demand|temporary errors|model is at capacity|selected model is at capacity\b/i.test(text)) return 'provider_high_demand';
-  if (/\b(stream disconnected before completion|stream idle timeout|response\.completed|websocket closed|socket connection was closed unexpectedly|connection reset|ConnectionRefused|tls handshake eof|tls close_notify|broken pipe|peer closed connection|remote host|远程主机强迫关闭|http2: response body closed|incomplete chunked read|Client network socket disconnected before secure TLS connection|Connection failed repeatedly|lost its connection to (?:the Anthropic API|the configured custom Anthropic endpoint)|Server error mid-response|empty or malformed response|Streaming response failed)\b/i
+  if (/\b(stream disconnected before completion|(?:stream|upstream) idle timeout|response\.completed|websocket closed|socket connection was closed unexpectedly|connection reset|ConnectionRefused|tls handshake eof|tls close_notify|broken pipe|peer closed connection|remote host|远程主机强迫关闭|http2: response body closed|incomplete chunked read|Client network socket disconnected before secure TLS connection|Connection failed repeatedly|lost its connection to (?:the Anthropic API|the configured custom Anthropic endpoint)|Server error mid-response|empty or malformed response|Streaming response failed)\b/i
     .test(text)) {
     return 'stream_disconnected';
   }
@@ -516,10 +550,16 @@ function executionFailedDetail(
 export function isResumableFailure(
   failure: RunFailureClassification | undefined,
 ): boolean {
-  if (!failure) return false;
+  if (!failure?.retryable) return false;
   if (
     failure.failure_category === 'upstream_unavailable' &&
-    failure.failure_detail !== 'upstream_client_error'
+    (
+      failure.failure_detail === 'stream_disconnected' ||
+      failure.failure_detail === 'upstream_5xx' ||
+      failure.failure_detail === 'network_error' ||
+      failure.failure_detail === 'provider_high_demand' ||
+      failure.failure_detail === 'provider_routing_error'
+    )
   ) {
     return true;
   }
@@ -612,10 +652,11 @@ export function classifyRunFailure(
     );
   }
 
-  if (errorCode === 'AGENT_PROMPT_TOO_LARGE' || isPromptTooLargeText(text)) {
+  const promptSizeDetail = promptTooLargeDetail(text);
+  if (errorCode === 'AGENT_PROMPT_TOO_LARGE' || promptSizeDetail) {
     return classification(
       'prompt_too_large',
-      'prompt_too_large',
+      promptSizeDetail ?? 'prompt_too_large',
       'prompt_send',
       false,
       'reduce_context',
@@ -635,6 +676,17 @@ export function classifyRunFailure(
     );
   }
 
+  const clientRequestDetail = clientRequestFailureDetail(text);
+  if (clientRequestDetail) {
+    return classification(
+      'upstream_unavailable',
+      clientRequestDetail,
+      'prompt_send',
+      false,
+      'none',
+    );
+  }
+
   // A `--resume <id>` whose stored session no longer resolves (Claude's 30-day
   // cleanupPeriodDays prune, a CLAUDE_CONFIG_DIR change, a cwd/worktree change,
   // or a prior run killed before the session was flushed). The daemon already
@@ -746,12 +798,15 @@ export function classifyRunFailure(
     isUpstreamDetailText(text) ||
     byokOpenCodeProviderNotFound
   ) {
-    const retryable = byokOpenCodeProviderNotFound
-      ? false
-      : retryableHint ?? !isUpstreamClientErrorText(text);
+    const upstreamClientError =
+      byokOpenCodeProviderNotFound || isUpstreamClientErrorText(text);
+    // A provider/SDK 4xx or request-shape rejection will deterministically fail
+    // again with the same payload. Do not let a coarse SDK isRetryable=true hint
+    // override the text-level client-error evidence.
+    const retryable = upstreamClientError ? false : retryableHint ?? true;
     return classification(
       'upstream_unavailable',
-      byokOpenCodeProviderNotFound ? 'upstream_client_error' : upstreamDetail(text),
+      upstreamClientError ? 'upstream_client_error' : upstreamDetail(text),
       inferFailureStageFromEvents(input.events, 'first_token_wait'),
       retryable,
       retryable ? 'retry' : 'none',

apps/daemon/src/runtimes/runs.ts

@@ -103,6 +103,11 @@ export function createChatRunService({
       errorCode: null,
       cancelRequested: false,
       retryRestartTimer: null,
+      // First failure that triggered a same-run retry. The next attempt creates
+      // a fresh startChatRun closure and clears run.error/errorCode, so keep the
+      // compact analytics snapshot on the shared run until terminal telemetry.
+      retryOriginFailure: null,
+      retryOriginErrorCode: null,
       stdinOpen: false,
       // Work-completeness signals (#1247 / #1060), folded from agent events by
       // captureRunWorkCompletenessSignals (server.ts). `lastTodoSnapshot` is the

apps/daemon/src/server.ts

@@ -5327,10 +5327,20 @@ export async function startServer({
         attemptCount > 0
           ? { ...decision, retryAttemptIndex: attemptCount }
           : decision;
+      // A successful retry has no current failure classification or error code.
+      // Fall back to the failure that caused attempt 0 to be retried so success
+      // recovery can still be attributed by root cause. Failed/suppressed retry
+      // events retain their existing current-attempt semantics.
+      const eventFailure = retryResult === 'success'
+        ? run.retryOriginFailure ?? failure
+        : failure;
+      const eventErrorCode = retryResult === 'success'
+        ? run.retryOriginErrorCode ?? errorCode
+        : errorCode;
       run.retryFinalResult = retryResult;
       run.retrySuppressedReason = retrySuppressedReason;
       design.runs.emit(run, 'run_retry_finished', {
-        ...retryAnalyticsBase(eventDecision, failure, errorCode),
+        ...retryAnalyticsBase(eventDecision, eventFailure, eventErrorCode),
         retry_result: retryResult,
         ...(retrySuppressedReason
           ? { retry_suppressed_reason: retrySuppressedReason }
@@ -5396,6 +5406,10 @@ export async function startServer({
         sideEffects,
       });
       if (decision.shouldRetry && !design.runs.isTerminal(run.status)) {
+        if ((run.retryAttemptCount ?? 0) === 0) {
+          run.retryOriginFailure = failure ? { ...failure } : null;
+          run.retryOriginErrorCode = errorCode ?? null;
+        }
         run.retryAttemptCount = decision.retryAttemptIndex;
         run.retryFinalResult = undefined;
         run.retrySuppressedReason = undefined;

apps/daemon/tests/run-failure-classification.test.ts

@@ -1328,7 +1328,7 @@ describe('classifyRunFailure — batch A reclassification out of execution_faile
       'json-rpc id 4: opencode event stream: {"properties":{"error":{"data":{"message":"[code=request_too_large] request body exceeds configured limit"}}}}',
     );
     expect(result?.failure_category).toBe('prompt_too_large');
-    expect(result?.failure_detail).toBe('prompt_too_large');
+    expect(result?.failure_detail).toBe('request_too_large');
     expect(result?.user_action).toBe('reduce_context');
   });
 
@@ -1517,6 +1517,45 @@ describe('classifyRunFailure — BYOK OpenCode reclassification out of stream_er
     });
   });
 
+  it('does not let a coarse SDK retry hint override a provider client error', () => {
+    const message = 'API Error: 400 Bad Request: Invalid Responses API request';
+    const result = classifyForAgent(
+      'byok-opencode',
+      'AGENT_EXECUTION_FAILED',
+      message,
+      [
+        errorEvent('AGENT_EXECUTION_FAILED', message, true),
+        runtimeCloseEvent('stream_error'),
+      ],
+    );
+
+    expect(result).toMatchObject({
+      failure_category: 'upstream_unavailable',
+      failure_detail: 'upstream_client_error',
+      retryable: false,
+      user_action: 'none',
+    });
+    expect(isResumableFailure(result)).toBe(false);
+  });
+
+  it('prefers provider client-error evidence over mixed stream-disconnect text', () => {
+    const message = 'stream disconnected before completion: statusCode:404';
+    const result = classifyForAgent(
+      'byok-opencode',
+      'AGENT_EXECUTION_FAILED',
+      message,
+      [errorEvent('AGENT_EXECUTION_FAILED', message, true)],
+    );
+
+    expect(result).toMatchObject({
+      failure_category: 'upstream_unavailable',
+      failure_detail: 'upstream_client_error',
+      retryable: false,
+      user_action: 'none',
+    });
+    expect(isResumableFailure(result)).toBe(false);
+  });
+
   it('classifies BYOK OpenCode config directory permission errors as fixable agent config', () => {
     const result = classifyForAgent(
       'byok-opencode',
@@ -1581,3 +1620,130 @@ describe('classifyRunFailure — AMR sampled failures', () => {
     });
   });
 });
+
+describe('classifyRunFailure — sampled 0.15.1 provider request failures', () => {
+  it.each([
+    {
+      name: 'HTTP 413 request body rejection',
+      agentId: 'claude',
+      message: 'Payload Too Large: request entity too large',
+      expected: {
+        failure_category: 'prompt_too_large',
+        failure_detail: 'request_too_large',
+        failure_stage: 'prompt_send',
+        retryable: false,
+        user_action: 'reduce_context',
+      },
+      resumable: false,
+    },
+    {
+      name: 'unsupported PDF attachment media type',
+      agentId: 'claude',
+      message: "request.messages.2.content.0.content.1.source.media_type: Invalid enum value. Expected 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp', received 'application/pdf'",
+      expected: {
+        failure_category: 'upstream_unavailable',
+        failure_detail: 'attachment_media_type_unsupported',
+        failure_stage: 'prompt_send',
+        retryable: false,
+        user_action: 'none',
+      },
+      resumable: false,
+    },
+    {
+      name: 'invalid Gemini function declaration name',
+      agentId: 'byok-opencode',
+      message: 'GenerateContentRequest.tools[0].function_declarations[0].name: Invalid function name. Must start with a letter or underscore.',
+      expected: {
+        failure_category: 'upstream_unavailable',
+        failure_detail: 'tool_schema_invalid',
+        failure_stage: 'prompt_send',
+        retryable: false,
+        user_action: 'none',
+      },
+      resumable: false,
+    },
+    {
+      name: 'prompt tokenization rejection',
+      agentId: 'byok-opencode',
+      message: '400: {"code":400,"message":"Failed to tokenize prompt","type":"invalid_request_error"}',
+      expected: {
+        failure_category: 'upstream_unavailable',
+        failure_detail: 'prompt_tokenization_failed',
+        failure_stage: 'prompt_send',
+        retryable: false,
+        user_action: 'none',
+      },
+      resumable: false,
+    },
+    {
+      name: 'context size exceeded',
+      agentId: 'byok-opencode',
+      message: 'Context size has been exceeded.',
+      expected: {
+        failure_category: 'prompt_too_large',
+        failure_detail: 'prompt_too_large',
+        failure_stage: 'prompt_send',
+        retryable: false,
+        user_action: 'reduce_context',
+      },
+      resumable: false,
+    },
+    {
+      name: 'unsupported model',
+      agentId: 'byok-opencode',
+      message: 'Not supported model mimo-v2.5-pro-ultraspeed',
+      expected: {
+        failure_category: 'model_unavailable',
+        failure_detail: 'model_not_supported',
+        failure_stage: 'model_select',
+        retryable: false,
+        user_action: 'switch_model',
+      },
+      resumable: false,
+    },
+    {
+      name: 'provider account function not found',
+      agentId: 'byok-opencode',
+      message: 'Not Found: {"status":404,"detail":"Function \'chat-completions\' Not found for account acct_123"}',
+      expected: {
+        failure_category: 'upstream_unavailable',
+        failure_detail: 'provider_resource_not_found',
+        failure_stage: 'prompt_send',
+        retryable: false,
+        user_action: 'none',
+      },
+      resumable: false,
+    },
+    {
+      name: 'genuine upstream idle timeout',
+      agentId: 'byok-opencode',
+      message: 'Upstream idle timeout exceeded',
+      expected: {
+        failure_category: 'upstream_unavailable',
+        failure_detail: 'stream_disconnected',
+        failure_stage: 'first_token_wait',
+        retryable: true,
+        user_action: 'retry',
+      },
+      resumable: true,
+    },
+  ])('classifies $name before the generic stream close fallback', ({
+    agentId,
+    message,
+    expected,
+    resumable,
+  }) => {
+    const result = classifyForAgent(
+      agentId,
+      'AGENT_EXECUTION_FAILED',
+      message,
+      [
+        errorEvent('AGENT_EXECUTION_FAILED', message, true),
+        runtimeCloseEvent('stream_error'),
+      ],
+    );
+
+    expect(result).toMatchObject(expected);
+    expect(isResumableFailure(result)).toBe(resumable);
+  });
+});

apps/daemon/tests/run-retry-policy.test.ts

@@ -129,6 +129,30 @@ describe('decideSafeRunRetry', () => {
     });
   });
 
+  it.each([
+    'request_too_large',
+    'attachment_media_type_unsupported',
+    'tool_schema_invalid',
+    'prompt_tokenization_failed',
+    'provider_resource_not_found',
+  ] as const)('keeps %s out of the transient retry allowlist', (failure_detail) => {
+    expect(
+      decide({
+        failure: {
+          failure_category: failure_detail === 'request_too_large'
+            ? 'prompt_too_large'
+            : 'upstream_unavailable',
+          failure_detail,
+          failure_stage: 'prompt_send',
+          retryable: true,
+        },
+      }),
+    ).toMatchObject({
+      shouldRetry: false,
+      retrySuppressedReason: 'non_retryable_category',
+    });
+  });
+
   it('uses unsafe_failure_stage for transient categories after unsafe output phases', () => {
     expect(
       decide({

apps/daemon/tests/run-retry-runtime.test.ts

@@ -112,6 +112,10 @@ describe('same-run retry runtime', () => {
       retry_of_run_id: run.id,
       retry_attempt_index: 1,
       retry_result: 'success',
+      failure_category: 'upstream_unavailable',
+      failure_detail: 'upstream_5xx',
+      failure_stage: 'first_token_wait',
+      error_code: 'UPSTREAM_UNAVAILABLE',
     });
   });
 

packages/contracts/src/analytics/events/shared-enums.ts

@@ -225,6 +225,11 @@ export type TrackingRunFailureDetail =
   | 'local_model_not_loaded'
   | 'cli_version_incompatible'
   | 'prompt_too_large'
+  | 'request_too_large'
+  | 'attachment_media_type_unsupported'
+  | 'tool_schema_invalid'
+  | 'prompt_tokenization_failed'
+  | 'provider_resource_not_found'
   | 'upstream_5xx'
   | 'upstream_client_error'
   | 'stream_disconnected'