mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(buzz-agent): budget summarizer reasoning separately so it cannot starve the handoff summary
Reasoning models spend output tokens thinking before any visible text, and that spend counts against the summary call's max_tokens. On deepseek-v4-flash this starved the handoff summarizer completely: in a 89-task benchmark run, 13 consecutive handoff attempts across 5 trials length-stopped inside the reasoning channel, returned empty content, and every one degraded to lossy history truncation (~40 minutes of burned reasoning) before a stochastically-short reasoning run finally fit. Give reasoning its own equal-sized budget on top of the text budget (reasoning.max_tokens) and exclude it from the response (reasoning.exclude) — summarize() only reads content. max_tokens is raised to cover both budgets so the text budget the caller asked for is actually available for text. Non-reasoning endpoints ignore the reasoning object; deliberately not paired with provider.require_parameters (see apply_openrouter_mutations). Verification: - cargo test -p buzz-agent (422 unit + 110 integration, all pass) - cargo fmt / clippy clean Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
This commit is contained in:
@@ -2249,10 +2249,26 @@ fn openrouter_summary_body(
|
|||||||
user_prompt: &str,
|
user_prompt: &str,
|
||||||
max_output_tokens: u32,
|
max_output_tokens: u32,
|
||||||
) -> Value {
|
) -> Value {
|
||||||
|
// Reasoning models spend output tokens thinking before emitting any
|
||||||
|
// visible text, and that spend counts against `max_tokens`. Left
|
||||||
|
// unseparated, a model can burn the entire cap mid-reasoning and return an
|
||||||
|
// empty `content` — observed with deepseek-v4-flash, where 13 consecutive
|
||||||
|
// handoff attempts length-stopped inside the reasoning channel and every
|
||||||
|
// one degraded to lossy history truncation. Give reasoning its own
|
||||||
|
// equal-sized budget on top of the text budget so `max_output_tokens`
|
||||||
|
// remains what the caller means: visible summary text. `exclude` keeps the
|
||||||
|
// reasoning out of the response body; `summarize()` only reads `content`.
|
||||||
|
// Non-reasoning endpoints ignore the `reasoning` object (see
|
||||||
|
// `apply_openrouter_mutations` on why it is never paired with
|
||||||
|
// `provider.require_parameters`).
|
||||||
json!({
|
json!({
|
||||||
"model": effective_model,
|
"model": effective_model,
|
||||||
"stream": false,
|
"stream": false,
|
||||||
"max_tokens": max_output_tokens,
|
"max_tokens": max_output_tokens.saturating_mul(2),
|
||||||
|
"reasoning": {
|
||||||
|
"max_tokens": max_output_tokens,
|
||||||
|
"exclude": true,
|
||||||
|
},
|
||||||
"messages": [
|
"messages": [
|
||||||
{ "role": "system", "content": system_prompt },
|
{ "role": "system", "content": system_prompt },
|
||||||
{ "role": "user", "content": user_prompt },
|
{ "role": "user", "content": user_prompt },
|
||||||
@@ -6263,8 +6279,14 @@ mod tests {
|
|||||||
assert!(body.get("max_completion_tokens").is_none());
|
assert!(body.get("max_completion_tokens").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The summary body reserves `max_output_tokens` for visible text by
|
||||||
|
/// granting reasoning a separate, equal budget on top and excluding it
|
||||||
|
/// from the response. Without the separation, a reasoning model can spend
|
||||||
|
/// the entire cap thinking and length-stop with empty `content`, which
|
||||||
|
/// `summarize()` reports as an empty summary and the handoff degrades to
|
||||||
|
/// lossy truncation.
|
||||||
#[test]
|
#[test]
|
||||||
fn openrouter_summary_carries_neither_reasoning_nor_provider() {
|
fn openrouter_summary_budgets_reasoning_separately_and_carries_no_provider() {
|
||||||
let body = openrouter_summary_body(
|
let body = openrouter_summary_body(
|
||||||
"anthropic/claude-opus-4-7",
|
"anthropic/claude-opus-4-7",
|
||||||
"summarize",
|
"summarize",
|
||||||
@@ -6274,15 +6296,26 @@ mod tests {
|
|||||||
assert_eq!(body["model"], "anthropic/claude-opus-4-7");
|
assert_eq!(body["model"], "anthropic/claude-opus-4-7");
|
||||||
assert_eq!(body["messages"][0]["role"], "system");
|
assert_eq!(body["messages"][0]["role"], "system");
|
||||||
assert_eq!(body["messages"][1]["content"], "text to summarize");
|
assert_eq!(body["messages"][1]["content"], "text to summarize");
|
||||||
assert_eq!(body["max_tokens"], 1024);
|
assert_eq!(
|
||||||
|
body["max_tokens"], 2048,
|
||||||
|
"total cap must cover the text budget plus the reasoning budget"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
body["reasoning"]["max_tokens"], 1024,
|
||||||
|
"reasoning gets its own budget so it cannot starve the summary text"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
body["reasoning"]["exclude"], true,
|
||||||
|
"reasoning must not be included in the response; summarize() reads only content"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
body["reasoning"].get("effort").is_none(),
|
||||||
|
"budget-based cap only; effort stays unset for the summary call"
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
body.get("max_completion_tokens").is_none(),
|
body.get("max_completion_tokens").is_none(),
|
||||||
"summary body must use OpenRouter's token-limit spelling"
|
"summary body must use OpenRouter's token-limit spelling"
|
||||||
);
|
);
|
||||||
assert!(
|
|
||||||
body.get("reasoning").is_none(),
|
|
||||||
"summary body must not carry reasoning"
|
|
||||||
);
|
|
||||||
assert!(
|
assert!(
|
||||||
body.get("provider").is_none(),
|
body.get("provider").is_none(),
|
||||||
"summary body must not carry provider"
|
"summary body must not carry provider"
|
||||||
|
|||||||
Reference in New Issue
Block a user