diff --git a/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openrouter-live.json b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openrouter-live.json new file mode 100644 index 000000000..2580144c4 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/testbed/endpoints/openrouter-live.json @@ -0,0 +1,16 @@ +{ + "moonshotai/kimi-k3": { + "provider": "openrouter", + "api_key_env": "OPENROUTER_API_KEY", + "env": { + "OPENROUTER_PROVIDER_ORDER": "moonshotai/mxfp4" + } + }, + "deepseek/deepseek-v4-flash-0731": { + "provider": "openrouter", + "api_key_env": "OPENROUTER_API_KEY", + "env": { + "OPENROUTER_PROVIDER_ORDER": "gmicloud/fp8" + } + } +} diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs index 2bb36b6d5..a31241084 100644 --- a/crates/buzz-agent/src/config.rs +++ b/crates/buzz-agent/src/config.rs @@ -778,6 +778,28 @@ pub struct Config { /// Databricks gateway does not auto-cache, so without this the surfaced /// `cache_read_input_tokens` is structurally always 0. pub prompt_caching: bool, + /// OpenRouter upstream pin: the `provider.order` list, from + /// `OPENROUTER_PROVIDER_ORDER` (comma-separated). Empty = let OpenRouter + /// route, which is the right default for interactive use and the wrong one + /// for a benchmark. + /// + /// An OpenRouter model id is not one deployment. `deepseek-v4-flash-0731` + /// is served by nine upstreams spanning fp4 to fp8, 262K to 1M context, and + /// a 1.6x price spread; `kimi-k3` by ten spanning a 1.5x spread. Unpinned, + /// consecutive requests land on different ones, so "the model" is a mixture + /// whose composition moves with provider load. + /// + /// It also decides whether prompt caching happens at all. Measured + /// 2026-08-01: pinned to `gmicloud/fp8`, deepseek serves a repeated prefix + /// from cache on every call; pinned to `siliconflow/fp8` it never does; + /// unpinned it did on one call in three. That is a ~7x swing on input cost + /// for identical work. (The `/endpoints` metadata is no guide here -- it + /// advertises `supports_implicit_caching: false` for every endpoint that + /// was then measured caching.) + /// + /// Set with `allow_fallbacks: false` in the request, so an unavailable pin + /// fails loudly instead of silently redefining the condition mid-run. + pub openrouter_provider_order: Vec, } impl Config { @@ -886,6 +908,9 @@ impl Config { hints_enabled: parse_env("BUZZ_AGENT_NO_HINTS", 0u8)? == 0, thinking_effort: parse_thinking_effort(env("BUZZ_AGENT_THINKING_EFFORT").as_deref())?, prompt_caching: parse_env("BUZZ_AGENT_PROMPT_CACHING", 1u8)? != 0, + openrouter_provider_order: parse_provider_order( + env("OPENROUTER_PROVIDER_ORDER").as_deref(), + ), }; cfg.validate()?; Ok(cfg) @@ -929,6 +954,7 @@ impl Config { hints_enabled: false, thinking_effort: None, prompt_caching: false, + openrouter_provider_order: Vec::new(), } } @@ -1146,6 +1172,26 @@ fn parse_hook_servers_env(key: &str) -> HookServers { parse_hook_servers(env(key).as_deref()) } +/// Parse `OPENROUTER_PROVIDER_ORDER` into a `provider.order` list. +/// +/// Comma-separated OpenRouter provider tags, in preference order — either a +/// bare slug (`moonshotai`) or the slug/quantization form shown in +/// `/api/v1/models/{id}/endpoints` (`gmicloud/fp8`). The quantized form is +/// worth preferring in a benchmark: one upstream can serve the same model id at +/// several quantizations, and fp4 versus fp8 is a different set of weights. +/// +/// Blank entries are dropped rather than passed through, so a trailing comma or +/// an accidentally-empty variable degrades to "no pin" instead of asking +/// OpenRouter to route to a provider named "". +pub fn parse_provider_order(raw: Option<&str>) -> Vec { + raw.unwrap_or("") + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect() +} + /// Pure parser exposed for unit tests. `None` (env unset) and `Some("")` /// (env set but empty) both yield `HookServers::None`. fn parse_hook_servers(raw: Option<&str>) -> HookServers { diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index e763c5c13..34c1ebdcd 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -156,6 +156,7 @@ impl Llm { cfg.thinking_effort, effective_model, cfg.prompt_caching, + &cfg.openrouter_provider_order, ); self.post_openrouter(cfg, &body) .await @@ -2542,6 +2543,7 @@ fn apply_openrouter_mutations( effort: Option, effective_model: &str, prompt_caching: bool, + provider_order: &[String], ) { if let Some(obj) = body.as_object_mut() { // OpenRouter's Chat Completions API spells the output cap `max_tokens`; @@ -2571,6 +2573,24 @@ fn apply_openrouter_mutations( ); } + // Upstream pin. Only ever set when the operator asked for one, which is + // why the "no body shape adds a provider routing filter" tests still + // hold: an unset OPENROUTER_PROVIDER_ORDER leaves the body untouched. + // + // `order` alone is only a *preference* — OpenRouter still falls back to + // another upstream when the named one is busy, which is precisely the + // silent condition-change a benchmark must not have. `allow_fallbacks: + // false` turns that into a 404 the run can see. Note this is NOT + // `require_parameters`, the filter the reasoning block above explains + // staying away from; that one screens on advertised parameters and + // 404s valid models, whereas this names the endpoint outright. + if !provider_order.is_empty() { + obj.insert( + "provider".into(), + json!({ "order": provider_order, "allow_fallbacks": false }), + ); + } + // A7: Anthropic cache_control injection for anthropic/* models. Gated on // `prompt_caching` (`BUZZ_AGENT_PROMPT_CACHING`) for the same reason as // the native Anthropic route: these are Anthropic-dialect breakpoints on @@ -2683,6 +2703,9 @@ mod tests { hints_enabled: true, thinking_effort: None, prompt_caching: true, + // Unpinned by default so the existing "no body shape adds a + // provider routing filter" assertions keep testing what they say. + openrouter_provider_order: Vec::new(), } } @@ -5973,6 +5996,7 @@ mod tests { c.thinking_effort, "anthropic/claude-opus-4-7", true, + &[], ); assert_eq!(body["reasoning"]["effort"], "high"); // `openai_body` is always called with `effort=None` on the OpenRouter @@ -6011,7 +6035,7 @@ mod tests { "anthropic/claude-opus-4-7", None, ); - apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true); + apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]); assert!( body.get("reasoning").is_none(), "reasoning must be absent when effort is None" @@ -6043,6 +6067,7 @@ mod tests { c.thinking_effort, "anthropic/claude-opus-4-7", true, + &[], ); assert_eq!(body["reasoning"]["effort"], "medium"); assert!( @@ -6063,7 +6088,7 @@ mod tests { "anthropic/claude-opus-4-7", None, ); - apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true); + apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]); assert!(body.get("reasoning").is_none()); assert!( body.get("provider").is_none(), @@ -6071,6 +6096,82 @@ mod tests { ); } + /// An OpenRouter model id is a family of upstream deployments, not one + /// deployment: `deepseek-v4-flash-0731` is served at both fp4 and fp8, over + /// a 1.6x price spread, and — measured 2026-08-01 — some of those endpoints + /// serve a repeated prefix from cache while others never do. A benchmark + /// cell that does not pin is therefore not one condition, and its input + /// cost swings ~7x on routing luck. + #[test] + fn openrouter_body_pins_upstream_when_order_is_set() { + let c = cfg(Provider::OpenRouter); + let order = vec!["gmicloud/fp8".to_string()]; + let mut body = openai_body( + &c, + "system", + &[HistoryItem::User("hi".into())], + &tools_vec(), + "deepseek/deepseek-v4-flash-0731", + None, + ); + apply_openrouter_mutations( + &mut body, + None, + "deepseek/deepseek-v4-flash-0731", + true, + &order, + ); + assert_eq!(body["provider"]["order"][0], "gmicloud/fp8"); + // The half that makes the pin mean anything. With fallbacks left on, + // `order` is only a preference: OpenRouter quietly serves from a + // different upstream when the named one is busy, which is the silent + // mid-run condition change this exists to prevent. Failing loudly is + // the point. + assert_eq!( + body["provider"]["allow_fallbacks"], + serde_json::Value::Bool(false), + "a pin that silently falls back is not a pin" + ); + } + + /// The pin must stay strictly opt-in. Every other OpenRouter body test + /// asserts `provider` is absent, and those assertions are load-bearing: + /// a stray routing filter 404s model ids that are otherwise fine. + #[test] + fn openrouter_body_unpinned_when_order_is_empty() { + let c = cfg(Provider::OpenRouter); + let mut body = openai_body( + &c, + "system", + &[HistoryItem::User("hi".into())], + &tools_vec(), + "deepseek/deepseek-v4-flash-0731", + None, + ); + apply_openrouter_mutations( + &mut body, + None, + "deepseek/deepseek-v4-flash-0731", + true, + &[], + ); + assert!(body.get("provider").is_none()); + } + + /// A trailing comma or a set-but-empty variable must degrade to "no pin", + /// not to a request asking OpenRouter to route to a provider named "". + #[test] + fn provider_order_parsing_drops_blanks() { + use crate::config::parse_provider_order; + assert_eq!(parse_provider_order(None), Vec::::new()); + assert_eq!(parse_provider_order(Some("")), Vec::::new()); + assert_eq!(parse_provider_order(Some(" , ")), Vec::::new()); + assert_eq!( + parse_provider_order(Some(" gmicloud/fp8 , deepinfra/fp4 ,")), + vec!["gmicloud/fp8".to_string(), "deepinfra/fp4".to_string()] + ); + } + /// `BUZZ_AGENT_PROMPT_CACHING=0` must reach the OpenRouter `anthropic/*` /// route too, not just the native Anthropic Messages routes. The switch and /// this route landed in separate changes, so nothing but this test stops the @@ -6087,7 +6188,7 @@ mod tests { "anthropic/claude-opus-4-7", None, ); - apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", false); + apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", false, &[]); assert!( !body.to_string().contains("cache_control"), "BUZZ_AGENT_PROMPT_CACHING=0 must suppress every breakpoint: {body}" @@ -6115,7 +6216,7 @@ mod tests { "anthropic/claude-opus-4-7", None, ); - apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true); + apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]); assert!( body.to_string().contains("cache_control"), "caching on must still emit breakpoints: {body}" @@ -6127,7 +6228,7 @@ mod tests { #[test] fn openrouter_body_without_token_limit_gains_none() { let mut body = json!({ "model": "vendor/model", "messages": [] }); - apply_openrouter_mutations(&mut body, None, "vendor/model", true); + apply_openrouter_mutations(&mut body, None, "vendor/model", true, &[]); assert!(body.get("max_tokens").is_none()); assert!(body.get("max_completion_tokens").is_none()); } @@ -6445,7 +6546,7 @@ mod tests { "anthropic/claude-opus-4-7", None, ); - apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true); + apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]); let messages = body["messages"].as_array().unwrap(); // System message should have cache_control @@ -6542,7 +6643,7 @@ mod tests { "anthropic/claude-opus-4-7", None, ); - apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true); + apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]); let messages = body["messages"].as_array().unwrap(); // Count text user messages that got cache_control