mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(agent): make stop-hook rejection budget per-prompt, fix stale hook docs (#1503)
Signed-off-by: Wes <wesbillman@users.noreply.github.com> Co-authored-by: Brain <21994759fc7a6fa6b965551d35cfd7897d262f2495467f2d78694ddcfa6a5c7e@sprout-oss.stage.blox.sqprod.co>
This commit is contained in:
@@ -48,7 +48,7 @@ All replies and delegations — including task assignments to other agents — g
|
||||
|
||||
- Respond promptly to @mentions. Be direct — no preamble. Name what you did, what you found, or what you need.
|
||||
- **Every turn that processes a user message MUST end with `buzz messages send`.** Your reasoning and tool calls are invisible to users — if you didn't send a message, they saw nothing. A turn that ends without a sent message is a silent failure.
|
||||
- For work that requires follow-up tools, create an open todo **before** sending the pickup acknowledgment. Keep it open until the deliverable is verified and you have sent a completion or blocker message; never end a turn while promised work remains open.
|
||||
- For work that requires follow-up tools, create an open todo **before** sending the pickup acknowledgment. Keep it open until the deliverable is verified and you have sent a completion or blocker message; never end a turn with open todo state unless you have posted that completion or blocker message.
|
||||
- Use GitHub-flavored Markdown. Fenced code blocks with language tags for syntax highlighting.
|
||||
- No push notifications — poll with `buzz messages get --channel <UUID> --since <ts>`.
|
||||
- Address people by the name in their own message header.
|
||||
|
||||
@@ -42,11 +42,6 @@ pub struct RunCtx<'a> {
|
||||
pub history: &'a mut Vec<HistoryItem>,
|
||||
pub original_task: &'a mut Option<String>,
|
||||
pub handoff_count: &'a mut usize,
|
||||
/// Cumulative `_Stop` objection count for this session (persists
|
||||
/// across `session/prompt` calls). Once it hits
|
||||
/// `cfg.stop_max_rejections` we stop calling `_Stop` for that
|
||||
/// session — a runaway hook can't burn rejections on every prompt.
|
||||
pub stop_rejections: &'a mut u32,
|
||||
/// Cache-summed input tokens reported by the provider on this session's
|
||||
/// most recent request (persists across `session/prompt` calls), or `None`
|
||||
/// before the first response and immediately after a handoff resets the
|
||||
@@ -75,6 +70,10 @@ impl RunCtx<'_> {
|
||||
self.history.push(HistoryItem::User(user_text));
|
||||
|
||||
let mut round = 0u32;
|
||||
// Per-prompt `_Stop` objection count. Bounded per prompt (not per
|
||||
// session) so a stubborn exchange can't permanently disable the stop
|
||||
// guard for a long-lived session; `max_rounds` still caps the loop.
|
||||
let mut stop_rejections = 0u32;
|
||||
loop {
|
||||
if self.cfg.max_rounds > 0 && round >= self.cfg.max_rounds {
|
||||
return Ok(StopReason::MaxTurnRequests);
|
||||
@@ -200,7 +199,7 @@ impl RunCtx<'_> {
|
||||
let stop = map_stop(response.stop);
|
||||
// Only gate genuine end_turn — don't override max_tokens/refusal.
|
||||
if stop == StopReason::EndTurn {
|
||||
if *self.stop_rejections >= self.cfg.stop_max_rejections {
|
||||
if stop_rejections >= self.cfg.stop_max_rejections {
|
||||
return Ok(stop);
|
||||
}
|
||||
let objections = self
|
||||
@@ -213,7 +212,7 @@ impl RunCtx<'_> {
|
||||
)
|
||||
.await;
|
||||
if !objections.is_empty() {
|
||||
*self.stop_rejections = self.stop_rejections.saturating_add(1);
|
||||
stop_rejections = stop_rejections.saturating_add(1);
|
||||
push_hook_outputs_as_tool_results(self.history, "_Stop", &objections);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -620,7 +620,7 @@ pub struct Config {
|
||||
pub max_handoffs: usize,
|
||||
pub max_parallel_tools: usize,
|
||||
pub hook_timeout: Duration,
|
||||
/// Maximum `_Stop` rejections per session. Default 3. Set to 0 to
|
||||
/// Maximum `_Stop` rejections per prompt. Default 3. Set to 0 to
|
||||
/// disable `_Stop` hooks entirely (agent always honors end_turn).
|
||||
pub stop_max_rejections: u32,
|
||||
/// Hook server allowlist. See [`HookServers`] for variant semantics.
|
||||
|
||||
@@ -67,7 +67,6 @@ struct Session {
|
||||
steer_tx: Option<mpsc::UnboundedSender<Vec<ContentBlock>>>,
|
||||
original_task: Option<String>,
|
||||
handoff_count: usize,
|
||||
stop_rejections: u32,
|
||||
/// Cache-summed input tokens the provider reported for this session's most
|
||||
/// recent request, or `None` before the first response (or after a handoff
|
||||
/// resets the context). Drives the token-based handoff gate; see
|
||||
@@ -401,7 +400,6 @@ async fn session_new(app: &Arc<App>, id: Value, params: Value, wire_tx: &WireSen
|
||||
steer_tx: None,
|
||||
original_task: None,
|
||||
handoff_count: 0,
|
||||
stop_rejections: 0,
|
||||
last_request_input_tokens: None,
|
||||
last_request_history_bytes: None,
|
||||
effective_system_prompt,
|
||||
@@ -616,7 +614,6 @@ async fn run_prompt(app: Arc<App>, id: Value, params: Value, wire_tx: WireSender
|
||||
mut history,
|
||||
mut original_task,
|
||||
mut handoff_count,
|
||||
mut stop_rejections,
|
||||
mut last_request_input_tokens,
|
||||
mut last_request_history_bytes,
|
||||
mut cancel_rx,
|
||||
@@ -665,7 +662,6 @@ async fn run_prompt(app: Arc<App>, id: Value, params: Value, wire_tx: WireSender
|
||||
history: &mut history,
|
||||
original_task: &mut original_task,
|
||||
handoff_count: &mut handoff_count,
|
||||
stop_rejections: &mut stop_rejections,
|
||||
last_request_input_tokens: &mut last_request_input_tokens,
|
||||
last_request_history_bytes: &mut last_request_history_bytes,
|
||||
};
|
||||
@@ -678,7 +674,6 @@ async fn run_prompt(app: Arc<App>, id: Value, params: Value, wire_tx: WireSender
|
||||
s.history = history;
|
||||
s.original_task = original_task;
|
||||
s.handoff_count = handoff_count;
|
||||
s.stop_rejections = stop_rejections;
|
||||
s.last_request_input_tokens = last_request_input_tokens;
|
||||
s.last_request_history_bytes = last_request_history_bytes;
|
||||
}
|
||||
@@ -705,7 +700,6 @@ async fn acquire_session(
|
||||
Vec<HistoryItem>,
|
||||
Option<String>,
|
||||
usize,
|
||||
u32,
|
||||
Option<u64>,
|
||||
Option<usize>,
|
||||
watch::Receiver<bool>,
|
||||
@@ -742,7 +736,6 @@ async fn acquire_session(
|
||||
std::mem::take(&mut s.history),
|
||||
s.original_task.take(),
|
||||
s.handoff_count,
|
||||
s.stop_rejections,
|
||||
s.last_request_input_tokens,
|
||||
s.last_request_history_bytes,
|
||||
rx,
|
||||
|
||||
@@ -756,7 +756,7 @@ async fn init_session_with_fake_mcp(h: &mut Harness, extra_mcp_env: &[(&str, &st
|
||||
async fn hook_stop_blocks_premature_end() {
|
||||
// LLM sequence:
|
||||
// 1. text "premature" (triggers _Stop objection — call #1)
|
||||
// 2. tool_call to fake__tool_0 (regular tool, resets latch)
|
||||
// 2. tool_call to fake__tool_0 (regular tool round)
|
||||
// 3. text "really done" (hook returns empty on call #2 → end)
|
||||
let llm = spawn_capturing_llm(vec![
|
||||
openai_text("premature"),
|
||||
@@ -841,7 +841,7 @@ async fn hook_stop_blocks_premature_end() {
|
||||
async fn hook_stop_budget_exhausted() {
|
||||
// LLM sequence:
|
||||
// 1. text → triggers _Stop objection (rejections: 0→1)
|
||||
// 2. tool_call (resets last_was_end_turn)
|
||||
// 2. tool_call (regular tool round)
|
||||
// 3. text → gate sees rejections>=max, returns end_turn (no _Stop call)
|
||||
let llm = spawn_capturing_llm(vec![
|
||||
openai_text("first"),
|
||||
@@ -941,6 +941,61 @@ async fn hook_stop_consecutive_end_turn_uses_rejection_budget() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
|
||||
/// The `_Stop` rejection budget is per prompt: exhausting it on one prompt
|
||||
/// must not disable the stop guard for the rest of the session. A second
|
||||
/// prompt gets a fresh budget and its end_turn is objected to again.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn hook_stop_budget_resets_per_prompt() {
|
||||
// Each prompt: text → objection (budget 0→1) → text → cap. With max=1,
|
||||
// both prompts take exactly 2 LLM calls; a session-cumulative budget
|
||||
// would accept prompt 2's first end_turn without calling _Stop (3 total).
|
||||
let llm = spawn_capturing_llm(vec![
|
||||
openai_text("p1-a"),
|
||||
openai_text("p1-b"),
|
||||
openai_text("p2-a"),
|
||||
openai_text("p2-b"),
|
||||
])
|
||||
.await;
|
||||
let mut h = Harness::spawn_with_env(
|
||||
&llm.url,
|
||||
&[
|
||||
("MCP_HOOK_SERVERS", "fake"),
|
||||
("BUZZ_AGENT_STOP_MAX_REJECTIONS", "1"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let sid = init_session_with_fake_mcp(
|
||||
&mut h,
|
||||
&[
|
||||
("FAKE_MCP_TOOL_COUNT", "1"),
|
||||
("FAKE_MCP_STOP_HOOK", "1"),
|
||||
("FAKE_MCP_STOP_TEXT", "keep going"),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
for prompt in ["one", "two"] {
|
||||
let p = h
|
||||
.send(
|
||||
"session/prompt",
|
||||
json!({"sessionId": sid, "prompt": [{"type":"text","text": prompt}]}),
|
||||
)
|
||||
.await;
|
||||
let r = h.recv_until(|v| v["id"] == json!(p)).await;
|
||||
assert!(r.get("result").is_some(), "errored: {r}");
|
||||
assert_eq!(r["result"]["stopReason"], "end_turn");
|
||||
}
|
||||
|
||||
let captured = llm.captured.lock().await;
|
||||
assert_eq!(
|
||||
captured.len(),
|
||||
4,
|
||||
"expected 4 LLM calls (fresh budget objected on both prompts), got {}",
|
||||
captured.len()
|
||||
);
|
||||
h.shutdown().await;
|
||||
}
|
||||
|
||||
/// Regression: an LLM that tries to call a hidden hook tool (e.g.
|
||||
/// `fake___Stop`) directly must get an "unknown tool" error result —
|
||||
/// the MCP server must NOT be invoked. This guarantees a malicious or
|
||||
|
||||
@@ -50,8 +50,7 @@ Hooks are advisory, not authoritative. The agent enforces:
|
||||
| Constraint | Behavior |
|
||||
|---|---|
|
||||
| Timeout (2.5s default) | Treated as no objection. Server killed only on second consecutive timeout (tolerates one-off slowness) |
|
||||
| Rejection budget (3/session) | After exhaustion, agent stops regardless |
|
||||
| Consecutive end_turn | If LLM ends again without tool calls after an objection, agent stops — the LLM heard and declined |
|
||||
| Rejection budget (3/prompt) | After exhaustion, agent stops regardless; the budget resets on the next prompt |
|
||||
|
||||
These constraints ensure a buggy or malicious hook cannot trap the agent.
|
||||
|
||||
@@ -61,7 +60,7 @@ These constraints ensure a buggy or malicious hook cannot trap the agent.
|
||||
|---|---|---|
|
||||
| `MCP_HOOK_SERVERS` | (unset = no hooks) | Allowlist: `*` for all servers, or comma-separated names |
|
||||
| `BUZZ_AGENT_HOOK_TIMEOUT_MS` | 2500 | Per-hook call timeout in milliseconds |
|
||||
| `BUZZ_AGENT_STOP_MAX_REJECTIONS` | 3 | Session-wide `_Stop` budget (0 = disable) |
|
||||
| `BUZZ_AGENT_STOP_MAX_REJECTIONS` | 3 | Per-prompt `_Stop` budget (0 = disable) |
|
||||
|
||||
Hooks are **off by default**. The operator must explicitly opt in via
|
||||
`MCP_HOOK_SERVERS`.
|
||||
|
||||
Reference in New Issue
Block a user