mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(desktop): stop abandoning healthy mesh prefills at four minutes
buzz-agent's `BUZZ_AGENT_LLM_TIMEOUT_SECS` defaults to 240s. MeshLLM's OpenAI frontend gives a backend call 600s (`OpenAiFrontendConfig::DEFAULT_BACKEND_TIMEOUT`), so on shared compute the client aborts work the server is still legitimately doing. Requests to a local mesh are `stream: false`, so a cold multi-ten-thousand token prefill is one silent request. Measured on an M5 Max serving Qwen3.8-27B Q4_K_M: an 88,318-token cold prompt returns HTTP 200 after 503s at 176 tok/s prefill. At the 240s default the client gives up at four minutes, then retries with an escalated budget (240 -> 480 -> 960), piling load onto a box that is already prefilling the first attempt. Seat the mesh client budget at 660s, just above the frontend's 600s, so the mesh's own error surfaces rather than a client abort racing it. Remote providers are untouched and keep buzz-agent's 240s default. Like the other mesh seeds this is a default, not policy: an explicit user value survives via `insert_default_if_unset` plus the copy-forward list in `relay_mesh_process_env`. Co-authored-by: Michael Neale <michael.neale@gmail.com> Signed-off-by: Michael Neale <michael.neale@gmail.com>
This commit is contained in:
@@ -66,6 +66,21 @@ pub fn apply_relay_mesh_env(
|
||||
// survives (see `insert_default_if_unset`, and the copy-forward list in
|
||||
// `relay_mesh_process_env` that preserves it through the spawn path).
|
||||
insert_default_if_unset(env, "BUZZ_AGENT_REQUIRE_REPLY", "1");
|
||||
// A local mesh prefills a cold multi-ten-thousand-token prompt at a few
|
||||
// hundred tokens/second, so the first turn of a large session legitimately
|
||||
// runs for minutes with no bytes on the wire (`stream: false`). MeshLLM's
|
||||
// own frontend gives such a request 600 s before it aborts
|
||||
// (`OpenAiFrontendConfig::DEFAULT_BACKEND_TIMEOUT`); buzz-agent's 240 s
|
||||
// default abandons it at four minutes, so the client gives up on work the
|
||||
// server is still doing and retries it — adding load to a box that is
|
||||
// already prefilling. Seat the client just above the server's budget so the
|
||||
// mesh's own error is what surfaces, rather than a client-side abort racing
|
||||
// it. Measured: an 88,318-token cold prompt on an M5 Max returns 200 at
|
||||
// 503 s. Remote providers keep the 240 s default; only mesh moves.
|
||||
// A default, not policy — an explicit user value survives via
|
||||
// `insert_default_if_unset` and the copy-forward list in
|
||||
// `relay_mesh_process_env`.
|
||||
insert_default_if_unset(env, "BUZZ_AGENT_LLM_TIMEOUT_SECS", "660");
|
||||
// Deliberately no BUZZ_AGENT_THINKING_EFFORT default: mesh translates
|
||||
// `reasoning_effort` into the chat template's `enable_thinking` flag, so any
|
||||
// value we pick overrides each model's own template default — and the right
|
||||
@@ -102,6 +117,10 @@ pub fn relay_mesh_process_env(
|
||||
for key in [
|
||||
"BUZZ_AGENT_MAX_OUTPUT_TOKENS",
|
||||
"BUZZ_AGENT_THINKING_EFFORT",
|
||||
// Same reason as `BUZZ_AGENT_REQUIRE_REPLY` below: without the
|
||||
// copy-forward, an explicit user timeout is re-defaulted to 660 by
|
||||
// `apply_relay_mesh_env`.
|
||||
"BUZZ_AGENT_LLM_TIMEOUT_SECS",
|
||||
// Must be copied forward for the user's value to survive: this map is
|
||||
// written onto the command *after* the layered user env, so a key absent
|
||||
// here is re-defaulted by `apply_relay_mesh_env` below and an explicit
|
||||
@@ -270,6 +289,74 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// MeshLLM's frontend allows a backend call 600 s
|
||||
/// (`OpenAiFrontendConfig::DEFAULT_BACKEND_TIMEOUT`). buzz-agent's 240 s
|
||||
/// default abandons a healthy cold prefill long before the server does and
|
||||
/// retries it, so the client budget must sit above the server's.
|
||||
#[test]
|
||||
fn native_provider_outlasts_the_mesh_backend_timeout() {
|
||||
let mut env = BTreeMap::new();
|
||||
apply_relay_mesh_env(
|
||||
&mut env,
|
||||
Some(RELAY_MESH_PROVIDER_ID),
|
||||
Some(RELAY_MESH_AUTO_MODEL_ID),
|
||||
);
|
||||
|
||||
let seconds: u64 = env
|
||||
.get("BUZZ_AGENT_LLM_TIMEOUT_SECS")
|
||||
.expect("mesh seeds a client timeout")
|
||||
.parse()
|
||||
.expect("timeout is whole seconds");
|
||||
assert!(
|
||||
seconds > 600,
|
||||
"client budget ({seconds}s) must outlast the mesh frontend's 600s backend timeout"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_provider_preserves_explicit_llm_timeout() {
|
||||
let mut env =
|
||||
BTreeMap::from([("BUZZ_AGENT_LLM_TIMEOUT_SECS".to_string(), "90".to_string())]);
|
||||
apply_relay_mesh_env(
|
||||
&mut env,
|
||||
Some(RELAY_MESH_PROVIDER_ID),
|
||||
Some(RELAY_MESH_AUTO_MODEL_ID),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
env.get("BUZZ_AGENT_LLM_TIMEOUT_SECS").map(String::as_str),
|
||||
Some("90"),
|
||||
"an explicit timeout is a user decision, not a value to re-default"
|
||||
);
|
||||
}
|
||||
|
||||
/// Same spawn-ordering hazard as the reply guard: without the copy-forward
|
||||
/// in `relay_mesh_process_env`, the user's value is re-defaulted here.
|
||||
#[test]
|
||||
fn process_env_preserves_explicit_llm_timeout() {
|
||||
let effective_env =
|
||||
BTreeMap::from([("BUZZ_AGENT_LLM_TIMEOUT_SECS".to_string(), "90".to_string())]);
|
||||
|
||||
let env = relay_mesh_process_env(&effective_env, "Gemma-4");
|
||||
|
||||
assert_eq!(
|
||||
env.get("BUZZ_AGENT_LLM_TIMEOUT_SECS").map(String::as_str),
|
||||
Some("90")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_mesh_provider_leaves_llm_timeout_unset() {
|
||||
let mut env = BTreeMap::new();
|
||||
apply_relay_mesh_env(&mut env, Some("anthropic"), Some("claude-haiku-4.5"));
|
||||
|
||||
assert_eq!(
|
||||
env.get("BUZZ_AGENT_LLM_TIMEOUT_SECS"),
|
||||
None,
|
||||
"remote providers keep buzz-agent's own default"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_mesh_provider_leaves_reply_guard_unset() {
|
||||
let mut env = BTreeMap::new();
|
||||
|
||||
Reference in New Issue
Block a user