fix: agent reliability — no restart on channel-add, visible dead-letter notice (#1468)

Signed-off-by: Wes <wesbillman@users.noreply.github.com>
Co-authored-by: Brain <21994759fc7a6fa6b965551d35cfd7897d262f2495467f2d78694ddcfa6a5c7e@sprout-oss.stage.blox.sqprod.co>

crates/buzz-acp/src/lib.rs

@@ -2111,6 +2111,7 @@ async fn tokio_main() -> Result<()> {
                     &respawn_tx,
                     &mut respawn_tasks,
                     observer.clone(),
+                    Some(&ctx.rest_client),
                 ) == LoopAction::Exit
                 {
                     break;
@@ -2662,6 +2663,7 @@ fn handle_prompt_result(
     respawn_tx: &mpsc::Sender<RespawnResult>,
     respawn_tasks: &mut tokio::task::JoinSet<()>,
     observer: Option<observer::ObserverHandle>,
+    rest_client: Option<&relay::RestClient>,
 ) -> LoopAction {
     let before = pool.task_map().len();
     let agent_index = result.agent.index;
@@ -2674,7 +2676,7 @@ fn handle_prompt_result(
     // retry_counts. If mark_complete runs first, retry_counts is cleared and
     // every retry starts at attempt 1 — defeating exponential backoff and
     // dead-letter protection.
-    if let Some(batch) = result.batch {
+    if let Some(batch) = result.batch.take() {
         // Don't requeue batches for channels the agent was removed from —
         // those events are stale and should be silently dropped.
         if !removed_channels.contains(&batch.channel_id) {
@@ -2689,8 +2691,31 @@ fn handle_prompt_result(
                 // system default — rather than telling the agent to supersede.
                 let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer);
                 queue.requeue_as_cancelled(batch, reason);
-            } else {
-                queue.requeue(batch);
+            } else if let Some(dead) = queue.requeue(batch) {
+                // Dead-lettered: retries exhausted and the events are gone.
+                // Post a visible notice so the channel isn't left waiting on
+                // a turn that will never happen.
+                if let Some(rest) = rest_client {
+                    let thread_tags = dead
+                        .events
+                        .last()
+                        .map(|be| queue::parse_thread_tags(&be.event))
+                        .unwrap_or_default();
+                    let reason = match &result.outcome {
+                        PromptOutcome::Timeout => "the turn timed out".to_string(),
+                        PromptOutcome::AgentExited => "the agent process exited".to_string(),
+                        PromptOutcome::Error(e) => format!("{e}"),
+                        _ => "repeated failures".to_string(),
+                    };
+                    let content = format!(
+                        "⚠️ I couldn't process the last request after multiple retries ({reason}). Please re-send if it's still needed."
+                    );
+                    let rest = rest.clone();
+                    let channel_id = dead.channel_id;
+                    tokio::spawn(async move {
+                        pool::post_failure_notice(&rest, channel_id, &thread_tags, &content).await;
+                    });
+                }
             }
         } else {
             tracing::debug!(
@@ -2875,7 +2900,9 @@ fn recover_panicked_agent(
     if let Some(batch) = meta.recoverable_batch {
         if let Some(ch) = meta.channel_id {
             if !removed_channels.contains(&ch) {
-                queue.requeue(batch);
+                // Dead-letter on exhaustion is logged inside requeue(); a
+                // panic path has no outcome to report, so no notice here.
+                let _ = queue.requeue(batch);
                 tracing::warn!("requeued batch for panicked agent {i}");
             } else {
                 tracing::debug!(
@@ -4020,6 +4047,7 @@ mod error_outcome_emission_tests {
             &respawn_tx,
             &mut respawn_tasks,
             Some(observer.clone()),
+            None,
         );
 
         observer

crates/buzz-acp/src/pool.rs

@@ -36,7 +36,7 @@ use crate::config::{DedupMode, PermissionMode};
 use crate::observer;
 use crate::queue::{
     CancelReason, ContextMessage, ConversationContext, FlushBatch, PromptChannelInfo,
-    PromptProfile, PromptProfileLookup,
+    PromptProfile, PromptProfileLookup, ThreadTags,
 };
 use crate::relay::{ChannelInfo, RestClient};
 
@@ -2615,6 +2615,50 @@ pub(crate) async fn reaction_add(rest: &crate::relay::RestClient, event_id: &str
     }
 }
 
+/// Best-effort: post a visible failure notice (kind:9) to a channel after a
+/// batch is dead-lettered. Replies into the thread of `thread_tags` when the
+/// triggering event was threaded. Errors are logged and swallowed — the
+/// notice must never take down the main loop.
+pub(crate) async fn post_failure_notice(
+    rest: &crate::relay::RestClient,
+    channel_id: Uuid,
+    thread_tags: &ThreadTags,
+    content: &str,
+) {
+    let thread_ref = thread_tags.root_event_id.as_deref().and_then(|root| {
+        let root_id = nostr::EventId::from_hex(root).ok()?;
+        let parent_id = thread_tags
+            .parent_event_id
+            .as_deref()
+            .and_then(|p| nostr::EventId::from_hex(p).ok())
+            .unwrap_or(root_id);
+        Some(buzz_sdk::ThreadRef {
+            root_event_id: root_id,
+            parent_event_id: parent_id,
+        })
+    });
+    let builder =
+        match buzz_sdk::build_message(channel_id, content, thread_ref.as_ref(), &[], false, &[]) {
+            Ok(b) => b,
+            Err(e) => {
+                tracing::warn!(channel = %channel_id, "failure notice: build failed: {e}");
+                return;
+            }
+        };
+    let event = match builder.sign_with_keys(&rest.keys) {
+        Ok(e) => e,
+        Err(e) => {
+            tracing::warn!(channel = %channel_id, "failure notice: sign failed: {e}");
+            return;
+        }
+    };
+    match tokio::time::timeout(Duration::from_secs(5), rest.submit_event(&event)).await {
+        Ok(Ok(_)) => {}
+        Ok(Err(e)) => tracing::warn!(channel = %channel_id, "failure notice failed: {e}"),
+        Err(_) => tracing::warn!(channel = %channel_id, "failure notice timed out"),
+    }
+}
+
 /// Best-effort: remove a reaction via a signed kind:5 (NIP-09) deletion event.
 ///
 /// Queries kind:7 reactions by our pubkey targeting the event, finds the matching

crates/buzz-acp/src/queue.rs

@@ -128,7 +128,7 @@ pub struct FlushBatch {
 ///
 ///   requeue(batch):
 ///     increment retry_counts[channel]
-///     if retry_counts[channel] > MAX_RETRIES: dead-letter (log ERROR, discard)
+///     if retry_counts[channel] > MAX_RETRIES: dead-letter (log ERROR, return batch to caller)
 ///     else: push_front with original received_at, set exponential backoff retry_after with jitter
 /// ```
 pub struct EventQueue {
@@ -376,12 +376,13 @@ impl EventQueue {
     /// not from resetting received_at.
     ///
     /// After [`MAX_RETRIES`] attempts the batch is dead-lettered: logged at
-    /// ERROR and discarded rather than requeued. This prevents poison batches
-    /// from looping forever.
+    /// ERROR and returned to the caller (rather than requeued) so a visible
+    /// failure notice can be posted to the channel. Returns `None` when the
+    /// batch was requeued for another attempt.
     ///
     /// Note: does NOT remove from `in_flight_channels` — caller must call
     /// `mark_complete` separately.
-    pub fn requeue(&mut self, batch: FlushBatch) {
+    pub fn requeue(&mut self, batch: FlushBatch) -> Option<FlushBatch> {
         let channel_id = batch.channel_id;
         let attempt = {
             let count = self.retry_counts.entry(channel_id).or_insert(0);
@@ -402,7 +403,7 @@ impl EventQueue {
             // Also clear retry_after so fresh traffic on this channel isn't
             // throttled by stale backoff from the discarded poison batch.
             self.retry_after.remove(&channel_id);
-            return;
+            return Some(batch);
         }
 
         // Exponential backoff: BASE * 2^(attempt-1), capped at MAX, with ±20% jitter.
@@ -449,6 +450,7 @@ impl EventQueue {
             );
         }
         self.retry_after.insert(channel_id, Instant::now() + delay);
+        None
     }
 
     /// Re-queue a batch preserving original `received_at` timestamps.
@@ -2690,6 +2692,36 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_requeue_dead_letters_after_max_retries() {
+        let mut q = EventQueue::new(DedupMode::Queue);
+        let ch = Uuid::new_v4();
+
+        q.push(make_queued(ch, "poison"));
+        for attempt in 1..=MAX_RETRIES {
+            q.retry_after
+                .insert(ch, Instant::now() - Duration::from_secs(1));
+            let batch = q.flush_next().expect("flush");
+            assert!(
+                q.requeue(batch).is_none(),
+                "attempt {attempt} should requeue, not dead-letter"
+            );
+            q.mark_complete(ch);
+        }
+
+        // The MAX_RETRIES+1'th failure dead-letters: batch is returned.
+        q.retry_after
+            .insert(ch, Instant::now() - Duration::from_secs(1));
+        let batch = q.flush_next().expect("flush");
+        let dead = q.requeue(batch).expect("should dead-letter");
+        assert_eq!(dead.channel_id, ch);
+        assert_eq!(dead.events.len(), 1);
+        q.mark_complete(ch);
+        // Retry state is cleared so fresh traffic isn't throttled.
+        assert!(!q.retry_counts.contains_key(&ch));
+        assert!(!q.retry_after.contains_key(&ch));
+    }
+
     #[test]
     fn test_retry_throttle_blocks_requeue_channel() {
         let mut q = EventQueue::new(DedupMode::Queue);

crates/buzz-agent/README.md

@@ -147,7 +147,7 @@ Everything is environment variables. No flags, no config files. (We are a subpro
 | `BUZZ_AGENT_MAX_OUTPUT_TOKENS` | `32768` | Per LLM call. Headroom for large tool-call inputs (e.g. file writes via heredoc); Sonnet 4 / Opus 4 cap at 64K. |
 | `BUZZ_AGENT_MAX_CONTEXT_TOKENS` | `200000` | Provider context window used by the handoff gate. |
 | `BUZZ_AGENT_MAX_HANDOFFS` | `10` | Max context handoffs per session before falling back to truncation. |
-| `BUZZ_AGENT_LLM_TIMEOUT_SECS` | `120` | |
+| `BUZZ_AGENT_LLM_TIMEOUT_SECS` | `240` | |
 | `BUZZ_AGENT_TOOL_TIMEOUT_SECS` | `660` | Per-tool call timeout in seconds |
 | `BUZZ_AGENT_MAX_PARALLEL_TOOLS` | `8` | Max concurrent tool calls per turn (1 = sequential) |
 | `BUZZ_AGENT_MAX_SESSIONS` | unlimited | Max concurrent ACP sessions. Sessions are cheap; default has no cap. |
@@ -244,7 +244,7 @@ The trust boundary is **the operator who launched the agent**. The harness, MCP
 | Tool schema bytes | 4 KiB | `MAX_SCHEMA_BYTES` (oversize → replaced with `{}`) |
 | Tool calls per turn | 64 | `MAX_TOOL_CALLS_PER_TURN` |
 | Loop rounds | 0 (unlimited) | `BUZZ_AGENT_MAX_ROUNDS` |
-| LLM call timeout | 120 s | `BUZZ_AGENT_LLM_TIMEOUT_SECS` |
+| LLM call timeout | 240 s | `BUZZ_AGENT_LLM_TIMEOUT_SECS` |
 | Tool call timeout | 660 s | `BUZZ_AGENT_TOOL_TIMEOUT_SECS` |
 
 ## What This Is NOT

crates/buzz-agent/src/config.rs

@@ -164,7 +164,7 @@ impl Config {
             openai_api,
             max_rounds: parse_env("BUZZ_AGENT_MAX_ROUNDS", 0)?,
             max_output_tokens: parse_env("BUZZ_AGENT_MAX_OUTPUT_TOKENS", 32_768)?,
-            llm_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_LLM_TIMEOUT_SECS", 120)?),
+            llm_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_LLM_TIMEOUT_SECS", 240)?),
             tool_timeout: Duration::from_secs(parse_env("BUZZ_AGENT_TOOL_TIMEOUT_SECS", 660)?),
             mcp_init_timeout: Duration::from_secs(parse_env(
                 "BUZZ_AGENT_MCP_INIT_TIMEOUT_SECS",

desktop/src/features/agents/channelAgents.ts

@@ -12,7 +12,6 @@ import {
   getChannelMembers,
   listManagedAgents,
   startManagedAgent,
-  stopManagedAgent,
   updateManagedAgent,
   uploadMediaBytes,
 } from "@/shared/api/tauri";
@@ -38,7 +37,6 @@ export type AttachManagedAgentToChannelInput = {
 export type AttachManagedAgentToChannelResult = {
   agent: ManagedAgent;
   membershipAdded: boolean;
-  restarted: boolean;
   started: boolean;
 };
 
@@ -122,25 +120,16 @@ export async function attachManagedAgentToChannel(
 
   let agent = input.agent;
   let started = false;
-  let restarted = false;
 
   if (ensureRunning) {
-    // Provider-backed agents use startManagedAgent as the deploy operation.
-    // Already-deployed providers auto-discover new channel membership via the
-    // harness; not-yet-deployed providers need the deploy call before the
-    // first mention can reach them.
+    // Running agents (local or provider) auto-discover new channel membership
+    // via the harness's membership notifications — no restart needed. Only
+    // not-yet-running agents need a start/deploy call before the first
+    // mention can reach them.
     const isRemote = input.agent.backend.type === "provider";
     if (isRemote && input.agent.status !== "deployed") {
       agent = await startManagedAgent(input.agent.pubkey);
       started = true;
-    } else if (
-      !isRemote &&
-      membershipAdded &&
-      (input.agent.status === "running" || input.agent.status === "deployed")
-    ) {
-      await stopManagedAgent(input.agent.pubkey);
-      agent = await startManagedAgent(input.agent.pubkey);
-      restarted = true;
     } else if (
       !isRemote &&
       input.agent.status !== "running" &&
@@ -154,7 +143,6 @@ export async function attachManagedAgentToChannel(
   return {
     agent,
     membershipAdded,
-    restarted,
     started,
   } satisfies AttachManagedAgentToChannelResult;
 }

desktop/src/features/agents/hooks.ts

@@ -534,7 +534,6 @@ export function useEnsureGooseInChannelMutation(channelId: string | null) {
       return {
         agent: attached.agent,
         membershipAdded: attached.membershipAdded,
-        restarted: attached.restarted,
         started: attached.started,
         created: attached.created,
       };

desktop/src/features/agents/ui/AddAgentToChannelDialog.tsx

@@ -116,9 +116,8 @@ export function AddAgentToChannelDialog({
             <DialogTitle>Add agent to channel</DialogTitle>
             <DialogDescription>
               Add {agent?.name ?? "this agent"} to a channel so desktop chat can
-              `@mention` it. Running local agents are restarted automatically
-              when they join a new channel. Remote agents pick up new channels
-              automatically via membership notifications.
+              `@mention` it. Running agents pick up new channels automatically
+              via membership notifications.
             </DialogDescription>
           </DialogHeader>
 

desktop/src/features/agents/ui/useManagedAgentActions.ts

@@ -364,9 +364,6 @@ export function useManagedAgentActions() {
   ) {
     setActionErrorMessage(null);
     setActionNoticeMessage(() => {
-      if (result.restarted) {
-        return `Added ${result.agent.name} to ${channel.name} and restarted it so the new channel subscription is live.`;
-      }
       if (result.started) {
         return `Added ${result.agent.name} to ${channel.name} and spawned it.`;
       }

desktop/src/features/profile/ui/UserProfilePanel.tsx

@@ -662,11 +662,7 @@ export function UserProfilePanel({
 
   const handleAddedToChannel = React.useCallback(
     (channel: Channel, result: AttachManagedAgentToChannelResult) => {
-      if (result.restarted) {
-        toast.success(
-          `Added ${result.agent.name} to ${channel.name} and restarted it.`,
-        );
-      } else if (result.started) {
+      if (result.started) {
         toast.success(`Added ${result.agent.name} to ${channel.name}.`);
       } else if (result.membershipAdded) {
         toast.success(`Added ${result.agent.name} to ${channel.name}.`);