mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(agent): recognise every provider's image-rejection, on every status
A text-only model that is handed an image is a recoverable condition: the
turn loop strips the images out of history and continues. Reaching that
recovery depended on `is_unsupported_image_input_error`, which matched only
OpenRouter's sentence ("no endpoints found that support image input") and was
only consulted on a 404.
Fireworks says "This model does not support image inputs" and says it with a
400, so on the direct Fireworks route neither condition held. The rejection
stayed a terminal `AgentError::Llm`, buzz-acp requeued the batch with
exponential backoff, every retry replayed the same image and failed
identically, and after 10 attempts the batch dead-lettered. The agent then had
no work left and sat idle until the harness killed it, so the trial scored 0
however much budget remained. Measured on the 2026-08-07 max-effort cell: 6 of
the first 66 trials, all 6 at reward 0.0.
Match on substrings covering both phrasings, and check on any non-success
status rather than only 404 — in `openrouter_post` (which the Fireworks and
OpenRouter routes share) and in the generic `post`, since any
OpenAI-compatible provider fronting a text-only model can 400 the same way.
Signed-off-by: Atish Patel <atish@squareup.com>
Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Code
parent
c7e3409cb0
commit
8dda992ee3
@@ -1940,9 +1940,29 @@ fn classify_body_read_error(
|
||||
}
|
||||
}
|
||||
|
||||
/// Does this error body mean "the model cannot accept images", as opposed to
|
||||
/// any other 4xx?
|
||||
///
|
||||
/// PROVIDERS PHRASE THIS COMPLETELY DIFFERENTLY, and matching only one of them
|
||||
/// is not a cosmetic miss. OpenRouter reports it as a *routing* failure ("No
|
||||
/// endpoints found that support image input"); Fireworks reports it as a
|
||||
/// *capability* failure ("This model does not support image inputs"). Both must
|
||||
/// reach `AgentError::UnsupportedImageInput`, because that is the only error the
|
||||
/// turn loop recovers from — it strips the images out of history and continues.
|
||||
/// Anything not recognised here is terminal, and a terminal error on a
|
||||
/// text-only model is unrecoverable by construction: buzz-acp requeues the
|
||||
/// batch with exponential backoff, every retry replays the same image and
|
||||
/// fails identically, and after 10 attempts the batch dead-letters. The agent
|
||||
/// then has no work left and idles until the harness kills it, so the trial
|
||||
/// scores 0 no matter how much budget remained. Measured on the 2026-08-07
|
||||
/// Fireworks cell: 6 of the first 66 trials, all 6 scoring 0.0.
|
||||
///
|
||||
/// Matched case-insensitively on substrings rather than the full sentence so a
|
||||
/// singular/plural or punctuation change upstream does not silently reopen this.
|
||||
fn is_unsupported_image_input_error(body: &str) -> bool {
|
||||
body.to_ascii_lowercase()
|
||||
.contains("no endpoints found that support image input")
|
||||
let body = body.to_ascii_lowercase();
|
||||
body.contains("no endpoints found that support image input")
|
||||
|| body.contains("does not support image input")
|
||||
}
|
||||
|
||||
/// Build the terminal `AgentError::Llm` for a `post()` exit that has given up
|
||||
@@ -2137,6 +2157,13 @@ where
|
||||
"{status}: {body}"
|
||||
))));
|
||||
}
|
||||
// Likewise recoverable, and likewise not only a 404: any
|
||||
// OpenAI-compatible provider serving a text-only model can reject
|
||||
// an image with a plain 400. Without this the turn loop never gets
|
||||
// the chance to strip the image and retry.
|
||||
if is_unsupported_image_input_error(&body) {
|
||||
return Err(PostError::Agent(AgentError::UnsupportedImageInput(body)));
|
||||
}
|
||||
return Err(PostError::Agent(AgentError::Llm(format!(
|
||||
"{status}: {body}"
|
||||
))));
|
||||
@@ -2509,6 +2536,14 @@ async fn openrouter_post(
|
||||
if status == 400 && is_context_length_error(&body) {
|
||||
return Err(AgentError::LlmContextExceeded(format!("{status}: {body}")));
|
||||
}
|
||||
// Image rejection is checked on EVERY non-success status, not just
|
||||
// the 404 handled above. OpenRouter reports it as a 404 routing
|
||||
// failure; Fireworks reports it as a plain 400. Checking only the
|
||||
// 404 left the direct-Fireworks route with no recovery path at all
|
||||
// — see `is_unsupported_image_input_error` for what that costs.
|
||||
if is_unsupported_image_input_error(&body) {
|
||||
return Err(AgentError::UnsupportedImageInput(body));
|
||||
}
|
||||
return Err(AgentError::Llm(format!("{status}: {body}")));
|
||||
}
|
||||
if let Some(len) = resp.content_length() {
|
||||
@@ -7728,6 +7763,44 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The same condition, as FIREWORKS words it — a 400 with a capability
|
||||
/// message rather than a 404 with a routing message. This is a regression
|
||||
/// test for a real cell: the detector matched only OpenRouter's sentence, so
|
||||
/// on the direct Fireworks route the rejection stayed terminal, buzz-acp
|
||||
/// requeued and dead-lettered the batch, and the trial idled out its
|
||||
/// deadline at reward 0. Six of the first sixty-six trials went that way.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn openrouter_post_400_fireworks_unsupported_image_is_typed() {
|
||||
let (url, _captured, attempts) = spawn_openrouter_stub(vec![CannedResponse::new(
|
||||
400,
|
||||
r#"{"error":{"object":"error","type":"invalid_request_error","code":"invalid_request_error","message":"This model does not support image inputs"}}"#,
|
||||
)])
|
||||
.await;
|
||||
let http = Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
let err = openrouter_post(
|
||||
&http,
|
||||
&format!("{url}/x"),
|
||||
&json!({}),
|
||||
"key",
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, AgentError::UnsupportedImageInput(s) if s.contains("does not support image inputs")),
|
||||
"Fireworks' capability 400 must reach the history-recovery path, \
|
||||
not fall through to a terminal error: got {err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
attempts.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"a deterministic capability rejection must not be retried"
|
||||
);
|
||||
}
|
||||
|
||||
/// Every other 404 still maps to `LlmModelNotFound`, including one that
|
||||
/// shares the `No endpoints found` prefix but is about the model rather than
|
||||
/// the parameters — the discriminator is narrow enough that a genuinely
|
||||
|
||||
Reference in New Issue
Block a user