External/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-08-18 06:50:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(buzz-agent): support max effort for gpt-5.6 family (#1806)

Browse Source 
Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
Co-authored-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@sprout-oss.stage.blox.sqprod.co>
This commit is contained in:
Will Pfleger
2026-07-13 12:57:21 -04:00
committed by GitHub
co-authored by npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7
parent bc14052d39
commit e34e0974a1
4 changed files with 182 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/buzz-agent/src/config.rs
+121 -71
View File
@@ -12,8 +12,8 @@ pub const PROTOCOL_VERSION: u32 = 2;
/// - **Anthropic adaptive**: `low|medium|high|xhigh|max` (model-dependent; see `anthropic_thinking_config`).
/// `none`/`minimal` are not Anthropic values — rejected at startup.
/// - **Anthropic manual budget** (claude-3*, opus-4-5): `low|medium|high`; `xhigh`/`max` clamp to high budget.
/// - **OpenAI Responses / Chat Completions**: `none|minimal|low|medium|high|xhigh` (provider pass-through).
/// `max` is not an OpenAI value — rejected at startup.
/// - **OpenAI Responses / Chat Completions**: effort support is model-dependent and normalized at
/// request time; `max` is valid for documented max-supporting families such as GPT-5.6.
/// - **Databricks**: routed by model family (Claude → Anthropic mapping, GPT-5 → Responses, MLflow → Chat).
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ThinkingEffort {
@@ -310,27 +310,37 @@ fn gpt5_base_matches(lower_model: &str, token: &str) -> bool {
/// | Model | Supported effort values |
/// |-------------|-------------------------------------------|
/// | gpt-5-pro | `high` only |
/// | gpt-5.6 | `none, low, medium, high, xhigh, max` |
/// | gpt-5.5 | `none, low, medium, high, xhigh` |
/// | gpt-5.4 | `none, low, medium, high, xhigh` |
/// | gpt-5.1 | `none, low, medium, high` |
/// | gpt-5 (base)| `minimal, low, medium, high` |
/// | unknown | not doc-verified — pass through unchanged |
/// | unknown | not doc-verified — `max` clamps to `xhigh` |
///
/// Note the `none` vs `minimal` split: `gpt-5` (base) supports `minimal` but not `none`;
/// `gpt-5.1`/`gpt-5.4`/`gpt-5.5` support `none` but not `minimal`. These are matched via
/// `gpt-5.1`/`gpt-5.4`/`gpt-5.5`/`gpt-5.6` support `none` but not `minimal`. These are matched via
/// nearest-supported fallback in `normalize_effort_for_openai_route`.
///
/// Match order: `-pro` variant checked before versioned strings to prevent `gpt-5-pro` from
/// falling into the `gpt-5` base bucket (substring "gpt-5" is shared).
///
/// `model` is a raw model name (may include Databricks gateway prefixes or date suffixes).
/// Unknown models return `None` — caller treats `None` as "server-validated pass-through".
/// Unknown models return `None` — callers pass through values except `max`, which clamps to
/// `xhigh` until support is confirmed.
/// Versioned tokens use `gpt5_token_matches` (end-of-string or `-` boundary, blocking digit
/// and letter continuations). The base token uses `gpt5_base_matches`, which additionally
/// rejects short `-<1-3 digit>` suffixes that look like two-digit version numbers.
fn openai_efforts_for_model(model: &str) -> Option<&'static [ThinkingEffort]> {
// Effort ordered from lowest to highest for each family.
const GPT5_PRO: &[ThinkingEffort] = &[ThinkingEffort::High];
const GPT5_6: &[ThinkingEffort] = &[
ThinkingEffort::None,
ThinkingEffort::Low,
ThinkingEffort::Medium,
ThinkingEffort::High,
ThinkingEffort::XHigh,
ThinkingEffort::Max,
];
const GPT5_5_AND_5_4: &[ThinkingEffort] = &[
ThinkingEffort::None,
ThinkingEffort::Low,
@@ -356,6 +366,12 @@ fn openai_efforts_for_model(model: &str) -> Option<&'static [ThinkingEffort]> {
// matching the base "gpt-5" prefix first.
if gpt5_token_matches(&lower, "gpt-5-pro") || gpt5_token_matches(&lower, "gpt5-pro") {
Some(GPT5_PRO)
} else if gpt5_token_matches(&lower, "gpt-5.6")
|| gpt5_token_matches(&lower, "gpt5.6")
|| gpt5_token_matches(&lower, "gpt-5-6")
|| gpt5_token_matches(&lower, "gpt5-6")
{
Some(GPT5_6)
} else if gpt5_token_matches(&lower, "gpt-5.5")
|| gpt5_token_matches(&lower, "gpt5.5")
|| gpt5_token_matches(&lower, "gpt-5-5")
@@ -443,8 +459,8 @@ pub fn anthropic_efforts_for_model(
/// model families means the "closest analogue" is the other form before jumping to `low`).
/// - Above that pair: upward clamp first, then downward (prefer more thinking over less).
/// - `xhigh` falls back to `high` when not supported (no model skips from `high` to `xhigh`).
/// - `max` is first clamped to `xhigh` by `normalize_effort_for_openai_route` before this
/// function is reached; this function never sees `max`.
/// - `max` passes through for model families whose table includes it; otherwise it resolves to
/// the nearest supported level.
///
/// Logs a `warn!` on every substitution.
fn resolve_openai_effort(
@@ -505,42 +521,32 @@ fn resolve_openai_effort(
/// Normalize the effort value for an OpenAI-shaped request body (Chat Completions or Responses).
///
/// Two normalizations are applied in order:
///
/// 1. **`max` → `xhigh` clamp**: `max` is not a valid OpenAI reasoning effort value; clamped
/// at this step. The pure-OpenAI startup validator already rejects `max` at startup, so this
/// clamp only fires on `DatabricksV2` sessions that routed to the OpenAI path.
///
/// 2. **Per-model effort availability**: for doc-verified OpenAI model families, the requested
/// level (post-clamp) is checked against the model's supported set. If not supported, the
/// nearest supported level is substituted (see `resolve_openai_effort` for preference order).
/// Unknown/unverified models are passed through unchanged — the server validates.
/// Per-model effort availability is applied for doc-verified OpenAI model families. A requested
/// level not in the model's supported set is substituted with the nearest supported level (see
/// `resolve_openai_effort` for preference order). For unknown/unverified models, `max` is clamped
/// to `xhigh` because its support cannot be confirmed; all other values pass through unchanged.
///
/// Applies to pure-OpenAI request paths AND DBv2 OpenAI-shaped routes.
///
/// Doc-verified model table (July 2025):
/// - `gpt-5-pro`: `high` only
/// - `gpt-5.6`: `none, low, medium, high, xhigh, max`
/// - `gpt-5.5`, `gpt-5.4`: `none, low, medium, high, xhigh`
/// - `gpt-5.1`: `none, low, medium, high`
/// - `gpt-5` (base): `minimal, low, medium, high`
/// - unknown: pass through (server-validated)
/// - unknown: `max` clamps to `xhigh`; other values pass through
pub fn normalize_effort_for_openai_route(effort: ThinkingEffort, model: &str) -> ThinkingEffort {
// Step 1: clamp max → xhigh (max is not a valid OpenAI value).
let clamped = if effort == ThinkingEffort::Max {
tracing::warn!(
requested = "max",
resolved = "xhigh",
"BUZZ_AGENT_THINKING_EFFORT=max is not valid for OpenAI-shaped requests; clamping to xhigh"
);
ThinkingEffort::XHigh
} else {
effort
};
// Step 2: per-model effort availability check.
match openai_efforts_for_model(model) {
Some(supported) => resolve_openai_effort(model, clamped, supported),
None => clamped, // unknown model — pass through
Some(supported) => resolve_openai_effort(model, effort, supported),
None if effort == ThinkingEffort::Max => {
tracing::warn!(
requested = "max",
resolved = "xhigh",
"BUZZ_AGENT_THINKING_EFFORT=max not confirmed for unknown OpenAI model; clamping to xhigh"
);
ThinkingEffort::XHigh
}
None => effort,
}
}
@@ -925,19 +931,13 @@ impl Config {
}
// Provider-level effort validation (fail-fast, clear error).
// `none`/`minimal` are not Anthropic values — rejected at startup.
// `max` is not an OpenAI value — rejected at startup.
// Model-level clamping (e.g. xhigh on Opus 4.6) is dynamic: happens at request
// build time because `session/set_model` can change the model after startup.
//
// DatabricksV2 is intentionally EXCLUDED from startup validation: it dispatches
// across Anthropic Messages, OpenAI Responses, and MLflow Chat routes at request
// build time based on the effective model. No single effort value is invalid for
// all three routes, so provider-wide startup rejection is wrong. Route-aware effort
// normalization is applied instead via `normalize_effort_for_openai_route` /
// `normalize_effort_for_anthropic_route` at request build time in `llm.rs`.
// OpenAI, Databricks, and DatabricksV2 defer effort validation to request-time routing:
// availability is model-dependent, and `session/set_model` can change the effective model
// after startup. `normalize_effort_for_openai_route` / `normalize_effort_for_anthropic_route`
// apply route-aware normalization in `llm.rs` when building each request.
if let Some(effort) = self.thinking_effort {
let is_pure_anthropic = matches!(self.provider, Provider::Anthropic);
let is_pure_openai = matches!(self.provider, Provider::OpenAi | Provider::Databricks);
if is_pure_anthropic && matches!(effort, ThinkingEffort::None | ThinkingEffort::Minimal)
{
return Err(format!(
@@ -946,13 +946,6 @@ impl Config {
effort.openai_effort_str()
));
}
if is_pure_openai && matches!(effort, ThinkingEffort::Max) {
return Err(
"config: BUZZ_AGENT_THINKING_EFFORT=max is not valid for OpenAI/Databricks \
providers (allowed: none|minimal|low|medium|high|xhigh)"
.into(),
);
}
}
Ok(())
}
@@ -1910,30 +1903,43 @@ mod tests {
}
#[test]
fn validate_rejects_max_effort_for_openai() {
let cfg = make_config_for_validation(Provider::OpenAi, Some(ThinkingEffort::Max));
let err = cfg.validate().unwrap_err();
assert!(
err.contains("BUZZ_AGENT_THINKING_EFFORT=max"),
"error must name the value: {err}"
);
assert!(
err.contains("not valid for OpenAI"),
"error must name the provider: {err}"
);
assert!(
err.contains("none|minimal|low|medium|high|xhigh"),
"error must name allowed values: {err}"
);
fn validate_accepts_all_efforts_for_openai() {
// OpenAI effort support is model-dependent and normalized at request build time.
for effort in [
ThinkingEffort::None,
ThinkingEffort::Minimal,
ThinkingEffort::Low,
ThinkingEffort::Medium,
ThinkingEffort::High,
ThinkingEffort::XHigh,
ThinkingEffort::Max,
] {
let cfg = make_config_for_validation(Provider::OpenAi, Some(effort));
assert!(
cfg.validate().is_ok(),
"OpenAI must accept {effort:?} at startup (route-aware normalization at request build)"
);
}
}
#[test]
fn validate_rejects_max_effort_for_databricks() {
// Databricks legacy uses OpenAI Chat wire format — same rejection.
let cfg = make_config_for_validation(Provider::Databricks, Some(ThinkingEffort::Max));
let err = cfg.validate().unwrap_err();
assert!(err.contains("BUZZ_AGENT_THINKING_EFFORT=max"), "{err}");
assert!(err.contains("not valid for OpenAI/Databricks"), "{err}");
fn validate_accepts_all_efforts_for_databricks() {
// Legacy Databricks effort support is model-dependent and normalized at request build time.
for effort in [
ThinkingEffort::None,
ThinkingEffort::Minimal,
ThinkingEffort::Low,
ThinkingEffort::Medium,
ThinkingEffort::High,
ThinkingEffort::XHigh,
ThinkingEffort::Max,
] {
let cfg = make_config_for_validation(Provider::Databricks, Some(effort));
assert!(
cfg.validate().is_ok(),
"Databricks must accept {effort:?} at startup (route-aware normalization at request build)"
);
}
}
#[test]
@@ -2195,6 +2201,26 @@ mod tests {
);
}
#[test]
fn openai_efforts_for_model_gpt5_6_includes_max() {
let expected: &[ThinkingEffort] = &[
ThinkingEffort::None,
ThinkingEffort::Low,
ThinkingEffort::Medium,
ThinkingEffort::High,
ThinkingEffort::XHigh,
ThinkingEffort::Max,
];
for model in ["gpt-5.6", "gpt-5.6-sol", "gpt-5-6-sol", "goose-gpt-5-6-sol"] {
assert_eq!(
openai_efforts_for_model(model),
Some(expected),
"{model} must match the gpt-5.6 effort table"
);
}
}
#[test]
fn openai_efforts_for_model_gpt5_5_includes_xhigh() {
let supported = openai_efforts_for_model("gpt-5.5").expect("gpt-5.5 must be in table");
@@ -2420,6 +2446,26 @@ mod tests {
);
}
#[test]
fn normalize_openai_route_passes_max_through_for_gpt5_6() {
for model in ["gpt-5.6", "gpt-5-6-sol"] {
assert_eq!(
normalize_effort_for_openai_route(ThinkingEffort::Max, model),
ThinkingEffort::Max,
"{model} must preserve max"
);
}
}
#[test]
fn normalize_openai_route_gpt5_5_max_becomes_xhigh() {
assert_eq!(
normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.5"),
ThinkingEffort::XHigh,
"gpt-5.5 must clamp max to xhigh"
);
}
#[test]
fn normalize_openai_route_gpt5_5_minimal_becomes_none() {
// gpt-5.5 supports none but not minimal. minimal → none (peer fallback).
@@ -2576,6 +2622,10 @@ mod tests {
// gpt-5 family check mirrors gpt5FamilyModel in TS.
let is_gpt5 = gpt5_token_matches(&m, "gpt-5-pro")
|| gpt5_token_matches(&m, "gpt5-pro")
|| gpt5_token_matches(&m, "gpt-5.6")
|| gpt5_token_matches(&m, "gpt5.6")
|| gpt5_token_matches(&m, "gpt-5-6")
|| gpt5_token_matches(&m, "gpt5-6")
|| gpt5_token_matches(&m, "gpt-5.5")
|| gpt5_token_matches(&m, "gpt5.5")
|| gpt5_token_matches(&m, "gpt-5.4")
crates/buzz-agent/src/llm.rs
+24 -5
View File
@@ -91,9 +91,10 @@ impl Llm {
}
Provider::OpenAi | Provider::Databricks => {
self.openai_request(cfg, effective_model, |use_responses| {
// Normalize effort for model-specific availability (per-model table; max is
// already rejected at startup for pure OpenAI, but other per-model corrections
// like none→minimal on gpt-5 base still apply).
// Normalize effort for model-specific availability. Startup no longer rejects
// `max` for pure OpenAI/Databricks; this per-model table is the single authority
// — it keeps `max` for gpt-5.6, clamps `max`→`xhigh` for other OpenAI-shaped
// models, and still applies corrections like none→minimal on the gpt-5 base.
let e = effort.map(|ef| normalize_effort_for_openai_route(ef, effective_model));
if use_responses {
(
@@ -112,7 +113,7 @@ impl Llm {
Provider::DatabricksV2 => {
self.databricks_v2_request(cfg, effective_model, |route| match route {
DatabricksV2Route::OpenAiResponses => {
// OpenAI Responses path: normalize effort (max → xhigh, per-model table).
// OpenAI Responses path: normalize effort against the per-model table.
let e =
effort.map(|ef| normalize_effort_for_openai_route(ef, effective_model));
(
@@ -129,7 +130,7 @@ impl Llm {
)
}
DatabricksV2Route::MlflowChatCompletions => {
// MLflow Chat path (OpenAI-shaped): normalize effort (max → xhigh, per-model table).
// MLflow Chat path (OpenAI-shaped): normalize effort against the per-model table.
let e =
effort.map(|ef| normalize_effort_for_openai_route(ef, effective_model));
(
@@ -1982,6 +1983,24 @@ mod tests {
);
}
#[test]
fn dbv2_openai_route_max_effort_passes_through_for_gpt5_6() {
let normalized =
crate::config::normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.6-sol");
let body = responses_body(
&cfg_responses(),
"system",
&[HistoryItem::User("hi".into())],
&[],
"gpt-5.6-sol",
Some(normalized),
);
assert_eq!(
body["reasoning"]["effort"], "max",
"DBv2 GPT-5.6 route must serialize max to the Responses API"
);
}
#[test]
fn dbv2_mlflow_route_max_effort_clamped_to_xhigh_in_openai_body() {
// DBv2 MLflow route (unknown model): max → clamped to xhigh by normalize_effort_for_openai_route.
desktop/src/features/agents/ui/buzzAgentConfig.ts
+16 -1
View File
@@ -227,6 +227,10 @@ function gpt5FamilyModel(m: string): boolean {
return (
gpt5TokenMatches(m, "gpt-5-pro") ||
gpt5TokenMatches(m, "gpt5-pro") ||
gpt5TokenMatches(m, "gpt-5.6") ||
gpt5TokenMatches(m, "gpt5.6") ||
gpt5TokenMatches(m, "gpt-5-6") ||
gpt5TokenMatches(m, "gpt5-6") ||
gpt5TokenMatches(m, "gpt-5.5") ||
gpt5TokenMatches(m, "gpt5.5") ||
gpt5TokenMatches(m, "gpt-5.4") ||
@@ -243,6 +247,17 @@ function openaiConfig(m: string): ProviderEffortConfig {
if (gpt5TokenMatches(m, "gpt-5-pro") || gpt5TokenMatches(m, "gpt5-pro")) {
return { validValues: ["high"], defaultValue: "high" };
}
if (
gpt5TokenMatches(m, "gpt-5.6") ||
gpt5TokenMatches(m, "gpt5.6") ||
gpt5TokenMatches(m, "gpt-5-6") ||
gpt5TokenMatches(m, "gpt5-6")
) {
return {
validValues: ["none", "low", "medium", "high", "xhigh", "max"],
defaultValue: "medium",
};
}
if (
gpt5TokenMatches(m, "gpt-5.5") ||
gpt5TokenMatches(m, "gpt5.5") ||
@@ -275,7 +290,7 @@ function openaiConfig(m: string): ProviderEffortConfig {
defaultValue: "medium",
};
}
// Unknown OpenAI model — show all except max (OpenAI doesn't accept max).
// Unknown OpenAI model — conservative fallback; max is enabled only for families whose table includes it.
return {
validValues: ["none", "minimal", "low", "medium", "high", "xhigh"],
defaultValue: "medium",
desktop/src/features/agents/ui/effortTable.fixture.json
+21
View File
@@ -83,6 +83,13 @@
"validValues": ["high"],
"defaultValue": "high"
},
{
"note": "OpenAI gpt-5.6: none/low/medium/high/xhigh/max",
"provider": "openai",
"model": "gpt-5.6",
"validValues": ["none", "low", "medium", "high", "xhigh", "max"],
"defaultValue": "medium"
},
{
"note": "OpenAI gpt-5.5: none/low/medium/high/xhigh",
"provider": "openai",
@@ -139,6 +146,20 @@
"validValues": ["low", "medium", "high", "xhigh", "max"],
"defaultValue": "high"
},
{
"note": "DatabricksV2 gpt-5.6-sol route: OpenAI max-capable table",
"provider": "databricks_v2",
"model": "gpt-5.6-sol",
"validValues": ["none", "low", "medium", "high", "xhigh", "max"],
"defaultValue": "medium"
},
{
"note": "DatabricksV2 gpt-5-6-sol route: dashed OpenAI max-capable table",
"provider": "databricks_v2",
"model": "gpt-5-6-sol",
"validValues": ["none", "low", "medium", "high", "xhigh", "max"],
"defaultValue": "medium"
},
{
"note": "DatabricksV2 gpt-5.4 route: OpenAI gpt-5.5/5.4 table",
"provider": "databricks_v2",