bench: pin the OpenRouter upstream so a cell is one condition

An OpenRouter model id is a family of upstream deployments, not one
deployment. `deepseek-v4-flash-0731` is served by nine upstreams spanning
fp4 to fp8 quantization, 262K to 1M context and a 1.6x price spread;
`kimi-k3` by ten spanning 1.5x. Unpinned, consecutive requests land on
different ones, so a benchmark cell measures a mixture whose composition
moves with provider load rather than a fixed condition.

It also decides whether prompt caching happens at all. Measured today
against a fixed 21.8K-token prefix, three calls each:

  deepseek, unpinned        cached 0, 0, 21760   (GMICloud, Cloudflare x2)
  deepseek gmicloud/fp8     cached 21760 x3      $0.00059/call
  deepseek siliconflow/fp8  cached 0 x3          $0.00306/call
  kimi, unpinned            cached 0, 0, 20480   (Fireworks, Moonshot x2)
  kimi moonshotai/mxfp4     cached 20480 x3      $0.00715 vs $0.06245

That is a 5-9x swing in input cost for identical work, decided by
routing luck. The `/endpoints` metadata is no guide: it advertises
`supports_implicit_caching: false` for every endpoint measured caching
above, and `true` only for one that is not routable on this account.

So: `OPENROUTER_PROVIDER_ORDER` (comma-separated, accepting both the bare
`moonshotai` slug and the `gmicloud/fp8` slug/quantization form) sets
`provider.order`. It is strictly opt-in -- unset leaves the body byte
identical, which is what keeps the existing "no body shape adds a provider
routing filter" assertions honest.

Paired with `allow_fallbacks: false`, because `order` alone is only a
preference: OpenRouter still serves from elsewhere when the named upstream
is busy, which is exactly the silent mid-run condition change this exists
to prevent. A pin that quietly falls back is not a pin. This is not
`require_parameters` -- that filter screens on advertised parameters and
404s otherwise-valid model ids, which is why the reasoning block nearby
stays away from it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Atish Patel <atish@squareup.com>

benchmarks/harbor-buzz-orchestra/testbed/endpoints/openrouter-live.json

@@ -0,0 +1,16 @@
+{
+  "moonshotai/kimi-k3": {
+    "provider": "openrouter",
+    "api_key_env": "OPENROUTER_API_KEY",
+    "env": {
+      "OPENROUTER_PROVIDER_ORDER": "moonshotai/mxfp4"
+    }
+  },
+  "deepseek/deepseek-v4-flash-0731": {
+    "provider": "openrouter",
+    "api_key_env": "OPENROUTER_API_KEY",
+    "env": {
+      "OPENROUTER_PROVIDER_ORDER": "gmicloud/fp8"
+    }
+  }
+}

crates/buzz-agent/src/config.rs

@@ -778,6 +778,28 @@ pub struct Config {
     /// Databricks gateway does not auto-cache, so without this the surfaced
     /// `cache_read_input_tokens` is structurally always 0.
     pub prompt_caching: bool,
+    /// OpenRouter upstream pin: the `provider.order` list, from
+    /// `OPENROUTER_PROVIDER_ORDER` (comma-separated). Empty = let OpenRouter
+    /// route, which is the right default for interactive use and the wrong one
+    /// for a benchmark.
+    ///
+    /// An OpenRouter model id is not one deployment. `deepseek-v4-flash-0731`
+    /// is served by nine upstreams spanning fp4 to fp8, 262K to 1M context, and
+    /// a 1.6x price spread; `kimi-k3` by ten spanning a 1.5x spread. Unpinned,
+    /// consecutive requests land on different ones, so "the model" is a mixture
+    /// whose composition moves with provider load.
+    ///
+    /// It also decides whether prompt caching happens at all. Measured
+    /// 2026-08-01: pinned to `gmicloud/fp8`, deepseek serves a repeated prefix
+    /// from cache on every call; pinned to `siliconflow/fp8` it never does;
+    /// unpinned it did on one call in three. That is a ~7x swing on input cost
+    /// for identical work. (The `/endpoints` metadata is no guide here -- it
+    /// advertises `supports_implicit_caching: false` for every endpoint that
+    /// was then measured caching.)
+    ///
+    /// Set with `allow_fallbacks: false` in the request, so an unavailable pin
+    /// fails loudly instead of silently redefining the condition mid-run.
+    pub openrouter_provider_order: Vec<String>,
 }
 
 impl Config {
@@ -886,6 +908,9 @@ impl Config {
             hints_enabled: parse_env("BUZZ_AGENT_NO_HINTS", 0u8)? == 0,
             thinking_effort: parse_thinking_effort(env("BUZZ_AGENT_THINKING_EFFORT").as_deref())?,
             prompt_caching: parse_env("BUZZ_AGENT_PROMPT_CACHING", 1u8)? != 0,
+            openrouter_provider_order: parse_provider_order(
+                env("OPENROUTER_PROVIDER_ORDER").as_deref(),
+            ),
         };
         cfg.validate()?;
         Ok(cfg)
@@ -929,6 +954,7 @@ impl Config {
             hints_enabled: false,
             thinking_effort: None,
             prompt_caching: false,
+            openrouter_provider_order: Vec::new(),
         }
     }
 
@@ -1146,6 +1172,26 @@ fn parse_hook_servers_env(key: &str) -> HookServers {
     parse_hook_servers(env(key).as_deref())
 }
 
+/// Parse `OPENROUTER_PROVIDER_ORDER` into a `provider.order` list.
+///
+/// Comma-separated OpenRouter provider tags, in preference order — either a
+/// bare slug (`moonshotai`) or the slug/quantization form shown in
+/// `/api/v1/models/{id}/endpoints` (`gmicloud/fp8`). The quantized form is
+/// worth preferring in a benchmark: one upstream can serve the same model id at
+/// several quantizations, and fp4 versus fp8 is a different set of weights.
+///
+/// Blank entries are dropped rather than passed through, so a trailing comma or
+/// an accidentally-empty variable degrades to "no pin" instead of asking
+/// OpenRouter to route to a provider named "".
+pub fn parse_provider_order(raw: Option<&str>) -> Vec<String> {
+    raw.unwrap_or("")
+        .split(',')
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+        .map(str::to_owned)
+        .collect()
+}
+
 /// Pure parser exposed for unit tests. `None` (env unset) and `Some("")`
 /// (env set but empty) both yield `HookServers::None`.
 fn parse_hook_servers(raw: Option<&str>) -> HookServers {

crates/buzz-agent/src/llm.rs

@@ -156,6 +156,7 @@ impl Llm {
                     cfg.thinking_effort,
                     effective_model,
                     cfg.prompt_caching,
+                    &cfg.openrouter_provider_order,
                 );
                 self.post_openrouter(cfg, &body)
                     .await
@@ -2542,6 +2543,7 @@ fn apply_openrouter_mutations(
     effort: Option<ThinkingEffort>,
     effective_model: &str,
     prompt_caching: bool,
+    provider_order: &[String],
 ) {
     if let Some(obj) = body.as_object_mut() {
         // OpenRouter's Chat Completions API spells the output cap `max_tokens`;
@@ -2571,6 +2573,24 @@ fn apply_openrouter_mutations(
             );
         }
 
+        // Upstream pin. Only ever set when the operator asked for one, which is
+        // why the "no body shape adds a provider routing filter" tests still
+        // hold: an unset OPENROUTER_PROVIDER_ORDER leaves the body untouched.
+        //
+        // `order` alone is only a *preference* — OpenRouter still falls back to
+        // another upstream when the named one is busy, which is precisely the
+        // silent condition-change a benchmark must not have. `allow_fallbacks:
+        // false` turns that into a 404 the run can see. Note this is NOT
+        // `require_parameters`, the filter the reasoning block above explains
+        // staying away from; that one screens on advertised parameters and
+        // 404s valid models, whereas this names the endpoint outright.
+        if !provider_order.is_empty() {
+            obj.insert(
+                "provider".into(),
+                json!({ "order": provider_order, "allow_fallbacks": false }),
+            );
+        }
+
         // A7: Anthropic cache_control injection for anthropic/* models. Gated on
         // `prompt_caching` (`BUZZ_AGENT_PROMPT_CACHING`) for the same reason as
         // the native Anthropic route: these are Anthropic-dialect breakpoints on
@@ -2683,6 +2703,9 @@ mod tests {
             hints_enabled: true,
             thinking_effort: None,
             prompt_caching: true,
+            // Unpinned by default so the existing "no body shape adds a
+            // provider routing filter" assertions keep testing what they say.
+            openrouter_provider_order: Vec::new(),
         }
     }
 
@@ -5973,6 +5996,7 @@ mod tests {
             c.thinking_effort,
             "anthropic/claude-opus-4-7",
             true,
+            &[],
         );
         assert_eq!(body["reasoning"]["effort"], "high");
         // `openai_body` is always called with `effort=None` on the OpenRouter
@@ -6011,7 +6035,7 @@ mod tests {
             "anthropic/claude-opus-4-7",
             None,
         );
-        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true);
+        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]);
         assert!(
             body.get("reasoning").is_none(),
             "reasoning must be absent when effort is None"
@@ -6043,6 +6067,7 @@ mod tests {
             c.thinking_effort,
             "anthropic/claude-opus-4-7",
             true,
+            &[],
         );
         assert_eq!(body["reasoning"]["effort"], "medium");
         assert!(
@@ -6063,7 +6088,7 @@ mod tests {
             "anthropic/claude-opus-4-7",
             None,
         );
-        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true);
+        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]);
         assert!(body.get("reasoning").is_none());
         assert!(
             body.get("provider").is_none(),
@@ -6071,6 +6096,82 @@ mod tests {
         );
     }
 
+    /// An OpenRouter model id is a family of upstream deployments, not one
+    /// deployment: `deepseek-v4-flash-0731` is served at both fp4 and fp8, over
+    /// a 1.6x price spread, and — measured 2026-08-01 — some of those endpoints
+    /// serve a repeated prefix from cache while others never do. A benchmark
+    /// cell that does not pin is therefore not one condition, and its input
+    /// cost swings ~7x on routing luck.
+    #[test]
+    fn openrouter_body_pins_upstream_when_order_is_set() {
+        let c = cfg(Provider::OpenRouter);
+        let order = vec!["gmicloud/fp8".to_string()];
+        let mut body = openai_body(
+            &c,
+            "system",
+            &[HistoryItem::User("hi".into())],
+            &tools_vec(),
+            "deepseek/deepseek-v4-flash-0731",
+            None,
+        );
+        apply_openrouter_mutations(
+            &mut body,
+            None,
+            "deepseek/deepseek-v4-flash-0731",
+            true,
+            &order,
+        );
+        assert_eq!(body["provider"]["order"][0], "gmicloud/fp8");
+        // The half that makes the pin mean anything. With fallbacks left on,
+        // `order` is only a preference: OpenRouter quietly serves from a
+        // different upstream when the named one is busy, which is the silent
+        // mid-run condition change this exists to prevent. Failing loudly is
+        // the point.
+        assert_eq!(
+            body["provider"]["allow_fallbacks"],
+            serde_json::Value::Bool(false),
+            "a pin that silently falls back is not a pin"
+        );
+    }
+
+    /// The pin must stay strictly opt-in. Every other OpenRouter body test
+    /// asserts `provider` is absent, and those assertions are load-bearing:
+    /// a stray routing filter 404s model ids that are otherwise fine.
+    #[test]
+    fn openrouter_body_unpinned_when_order_is_empty() {
+        let c = cfg(Provider::OpenRouter);
+        let mut body = openai_body(
+            &c,
+            "system",
+            &[HistoryItem::User("hi".into())],
+            &tools_vec(),
+            "deepseek/deepseek-v4-flash-0731",
+            None,
+        );
+        apply_openrouter_mutations(
+            &mut body,
+            None,
+            "deepseek/deepseek-v4-flash-0731",
+            true,
+            &[],
+        );
+        assert!(body.get("provider").is_none());
+    }
+
+    /// A trailing comma or a set-but-empty variable must degrade to "no pin",
+    /// not to a request asking OpenRouter to route to a provider named "".
+    #[test]
+    fn provider_order_parsing_drops_blanks() {
+        use crate::config::parse_provider_order;
+        assert_eq!(parse_provider_order(None), Vec::<String>::new());
+        assert_eq!(parse_provider_order(Some("")), Vec::<String>::new());
+        assert_eq!(parse_provider_order(Some("  ,  ")), Vec::<String>::new());
+        assert_eq!(
+            parse_provider_order(Some(" gmicloud/fp8 , deepinfra/fp4 ,")),
+            vec!["gmicloud/fp8".to_string(), "deepinfra/fp4".to_string()]
+        );
+    }
+
     /// `BUZZ_AGENT_PROMPT_CACHING=0` must reach the OpenRouter `anthropic/*`
     /// route too, not just the native Anthropic Messages routes. The switch and
     /// this route landed in separate changes, so nothing but this test stops the
@@ -6087,7 +6188,7 @@ mod tests {
             "anthropic/claude-opus-4-7",
             None,
         );
-        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", false);
+        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", false, &[]);
         assert!(
             !body.to_string().contains("cache_control"),
             "BUZZ_AGENT_PROMPT_CACHING=0 must suppress every breakpoint: {body}"
@@ -6115,7 +6216,7 @@ mod tests {
             "anthropic/claude-opus-4-7",
             None,
         );
-        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true);
+        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]);
         assert!(
             body.to_string().contains("cache_control"),
             "caching on must still emit breakpoints: {body}"
@@ -6127,7 +6228,7 @@ mod tests {
     #[test]
     fn openrouter_body_without_token_limit_gains_none() {
         let mut body = json!({ "model": "vendor/model", "messages": [] });
-        apply_openrouter_mutations(&mut body, None, "vendor/model", true);
+        apply_openrouter_mutations(&mut body, None, "vendor/model", true, &[]);
         assert!(body.get("max_tokens").is_none());
         assert!(body.get("max_completion_tokens").is_none());
     }
@@ -6445,7 +6546,7 @@ mod tests {
             "anthropic/claude-opus-4-7",
             None,
         );
-        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true);
+        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]);
         let messages = body["messages"].as_array().unwrap();
 
         // System message should have cache_control
@@ -6542,7 +6643,7 @@ mod tests {
             "anthropic/claude-opus-4-7",
             None,
         );
-        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true);
+        apply_openrouter_mutations(&mut body, None, "anthropic/claude-opus-4-7", true, &[]);
         let messages = body["messages"].as_array().unwrap();
 
         // Count text user messages that got cache_control