feat(acp): idle-based timeout with dual-deadline architecture (#188)

This commit is contained in:
tlongwell-block
2026-03-26 14:54:59 -04:00
committed by GitHub
parent 038f38373a
commit 39ecb6a442
12 changed files with 644 additions and 89 deletions
+3 -2
View File
@@ -103,12 +103,13 @@ All configuration is via environment variables (or CLI flags — every env var h
| `SPROUT_ACP_AGENT_COMMAND` | no | `goose` | Agent binary to spawn. |
| `SPROUT_ACP_AGENT_ARGS` | no | `acp` | Agent arguments (comma-separated). |
| `SPROUT_ACP_MCP_COMMAND` | no | `sprout-mcp-server` | Path to the Sprout MCP server binary. |
| `SPROUT_ACP_TURN_TIMEOUT` | no | `300` | Max seconds per agent turn before cancellation. |
| `SPROUT_ACP_IDLE_TIMEOUT` | no | `300` | Idle timeout: max seconds of silence before cancelling a turn. Resets on any agent stdout activity. |
| `SPROUT_ACP_MAX_TURN_DURATION` | no | `3600` | Absolute wall-clock cap per turn (safety valve). |
| `SPROUT_API_TOKEN` | no | — | API token (required if relay enforces token auth). |
**Note:** `SPROUT_ACP_AGENT_ARGS` splits on commas. For args with values, use: `-c,key="value"`.
**Legacy env vars:** `SPROUT_ACP_PRIVATE_KEY` and `SPROUT_ACP_API_TOKEN` are still accepted as fallbacks.
**Legacy env vars:** `SPROUT_ACP_PRIVATE_KEY`, `SPROUT_ACP_API_TOKEN`, and `SPROUT_ACP_TURN_TIMEOUT` (replaced by `SPROUT_ACP_IDLE_TIMEOUT`) are still accepted as fallbacks.
### Parallel Agents & Heartbeat
+321 -16
View File
@@ -5,7 +5,7 @@
//! 1. [`AcpClient::spawn`] — launch agent binary as subprocess
//! 2. [`AcpClient::initialize`] — protocol version negotiation
//! 3. [`AcpClient::session_new`] — create session with MCP server config
//! 4. [`AcpClient::session_prompt`] — send prompt, receive streaming updates, return stop reason
//! 4. [`AcpClient::session_prompt_with_idle_timeout`] — send prompt with idle/hard deadline, return stop reason
//! 5. [`AcpClient::session_cancel`] / [`AcpClient::cancel_with_cleanup`] — cancel in-flight turn
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
@@ -76,9 +76,11 @@ pub enum AcpError {
#[error("Agent process exited unexpectedly")]
AgentExited,
#[allow(dead_code)]
#[error("Turn timed out")]
Timeout,
#[error("Idle timeout — no agent activity for {0:?}")]
IdleTimeout(std::time::Duration),
#[error("Hard turn timeout exceeded")]
HardTimeout,
#[error("Protocol error: {0}")]
Protocol(String),
@@ -112,8 +114,12 @@ pub struct AcpClient {
permission_responded: bool,
/// The JSON-RPC id of the most recently sent `session/prompt` request.
/// Used by [`cancel_with_cleanup`] to drain the correct response.
/// Set in [`session_prompt`]; consumed in [`cancel_with_cleanup`].
/// Set in [`session_prompt_with_idle_timeout`]; consumed in [`cancel_with_cleanup`].
last_prompt_id: Option<u64>,
/// Hard deadline for the current turn, set by `session_prompt_with_idle_timeout`.
/// Inherited by `cancel_with_cleanup` so the drain loop shares the same budget
/// rather than starting a fresh timer (prevents double-jeopardy).
current_hard_deadline: Option<tokio::time::Instant>,
}
impl AcpClient {
@@ -161,6 +167,7 @@ impl AcpClient {
pending_permission_id: None,
permission_responded: false,
last_prompt_id: None,
current_hard_deadline: None,
})
}
@@ -248,14 +255,16 @@ impl AcpClient {
self.send_request("session/set_model", params).await
}
/// Send `session/prompt` and block until the agent returns a stop reason.
/// Send `session/prompt` with idle-based timeout instead of wall-clock.
///
/// While waiting, incoming `session/update` notifications are logged and
/// `session/request_permission` requests are auto-approved with `allow_once`.
pub async fn session_prompt(
/// The idle deadline resets on any stdout activity from the agent. The hard
/// deadline is an absolute wall-clock cap (safety valve).
pub async fn session_prompt_with_idle_timeout(
&mut self,
session_id: &str,
prompt_text: &str,
idle_timeout: std::time::Duration,
max_duration: std::time::Duration,
) -> Result<StopReason, AcpError> {
let params = serde_json::json!({
"sessionId": session_id,
@@ -263,12 +272,48 @@ impl AcpClient {
{ "type": "text", "text": prompt_text }
]
});
// Record the prompt request ID before send_request increments next_id.
// Used by cancel_with_cleanup to drain the correct response.
let hard_deadline = tokio::time::Instant::now() + max_duration;
self.current_hard_deadline = Some(hard_deadline);
self.last_prompt_id = Some(self.next_id);
let result = self.send_request("session/prompt", params).await?;
self.last_prompt_id = None; // Clear after normal completion.
self.parse_stop_reason(&result)
let id = self.next_id;
self.next_id += 1;
let msg = serde_json::json!({
"jsonrpc": "2.0",
"id": id,
"method": "session/prompt",
"params": params,
});
tracing::debug!(target: "acp::wire", "→ {}", &serde_json::to_string(&msg).unwrap_or_default());
if let Err(e) = self.write_ndjson(&msg).await {
self.last_prompt_id = None;
self.current_hard_deadline = None;
return Err(e);
}
let result = self
.read_until_response_with_idle_timeout(id, idle_timeout, hard_deadline)
.await;
// On timeout errors, leave current_hard_deadline set so cancel_with_cleanup
// can inherit the remaining budget. Clear it on all other outcomes.
match &result {
Ok(_) => {
self.last_prompt_id = None;
self.current_hard_deadline = None;
}
Err(AcpError::IdleTimeout(_) | AcpError::HardTimeout) => {
// Leave last_prompt_id and current_hard_deadline set —
// caller will invoke cancel_with_cleanup.
}
Err(_) => {
self.last_prompt_id = None;
self.current_hard_deadline = None;
}
}
self.parse_stop_reason(&result?)
}
/// Send a `session/cancel` **notification** (no `id` field, no response expected).
@@ -295,7 +340,33 @@ impl AcpClient {
/// 3. Continue reading until the `session/prompt` response arrives with `stopReason: "cancelled"`.
///
/// Returns the final [`StopReason`] (almost always [`StopReason::Cancelled`]).
pub async fn cancel_with_cleanup(&mut self, session_id: &str) -> Result<StopReason, AcpError> {
pub async fn cancel_with_cleanup(
&mut self,
session_id: &str,
idle_timeout: std::time::Duration,
) -> Result<StopReason, AcpError> {
// Inherit the hard deadline from the timed-out turn so the drain loop
// doesn't start a fresh timer (prevents double-jeopardy). If the original
// deadline is already expired or near-expired, grant a 30s floor so the
// cancel notification has time to propagate and the agent can respond.
let stored_deadline = self.current_hard_deadline.take();
let min_cleanup_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
let hard_deadline = match stored_deadline {
Some(d) if d > min_cleanup_deadline => d,
Some(_) => {
tracing::debug!(
"original hard deadline expired or near-expired — using 30s cleanup grace"
);
min_cleanup_deadline
}
None => {
tracing::warn!(
"cancel_with_cleanup called without current_hard_deadline — using 30s fallback"
);
min_cleanup_deadline
}
};
// Validate precondition before any side effects — fail fast if there's
// no in-flight prompt (prevents writing permission responses or cancel
// notifications to the agent when no prompt is active).
@@ -321,7 +392,12 @@ impl AcpClient {
// Step 2: send session/cancel notification (no id)
self.session_cancel(session_id).await?;
tracing::info!(target: "acp::cancel", "sent session/cancel for {session_id}");
let result = self.read_until_response(prompt_id).await?;
// Clamp idle timeout to at least 30s during cleanup — the cancel notification
// needs time to propagate and the agent may go silent while winding down.
let cleanup_idle = idle_timeout.max(std::time::Duration::from_secs(30));
let result = self
.read_until_response_with_idle_timeout(prompt_id, cleanup_idle, hard_deadline)
.await?;
self.parse_stop_reason(&result)
}
@@ -450,6 +526,104 @@ impl AcpClient {
}
}
/// Idle-aware message loop: like [`read_until_response`] but resets an idle
/// deadline on every stdout line. Fires [`AcpError::IdleTimeout`] on silence
/// or [`AcpError::HardTimeout`] on absolute wall-clock cap.
///
/// `hard_deadline` is an absolute `Instant` (pre-computed by the caller) so
/// that `cancel_with_cleanup` can inherit the remaining budget from the
/// original turn rather than starting a fresh timer.
async fn read_until_response_with_idle_timeout(
&mut self,
expected_id: u64,
idle_timeout: std::time::Duration,
hard_deadline: tokio::time::Instant,
) -> Result<serde_json::Value, AcpError> {
use tokio::time::Instant;
let mut idle_deadline = Instant::now() + idle_timeout;
loop {
// Determine which deadline fires first BEFORE sleeping — this is
// the classification we'll use on timeout, immune to scheduler jitter.
let idle_fires_first = idle_deadline < hard_deadline;
let next_deadline = if idle_fires_first {
idle_deadline
} else {
hard_deadline
};
let remaining = next_deadline.saturating_duration_since(Instant::now());
let read_result = tokio::time::timeout(remaining, async {
let mut line = String::new();
let n = self.reader.read_line(&mut line).await?;
Ok::<(usize, String), std::io::Error>((n, line))
})
.await;
match read_result {
Ok(Ok((0, _))) => return Err(AcpError::AgentExited),
Ok(Ok((_, line))) => {
// Any stdout activity resets the idle clock.
idle_deadline = Instant::now() + idle_timeout;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
tracing::debug!(target: "acp::wire", "← {trimmed}");
let msg: serde_json::Value = match serde_json::from_str(trimmed) {
Ok(v) => v,
Err(e) => {
tracing::warn!(
target: "acp::wire",
"failed to parse line as JSON: {e} — skipping"
);
continue;
}
};
// Check for matching response.
if let Some(id) = msg.get("id") {
if *id == serde_json::json!(expected_id) {
if let Some(error) = msg.get("error") {
return Err(AcpError::Protocol(error.to_string()));
}
return Ok(msg["result"].clone());
}
}
// Dispatch notifications.
if let Some(method) = msg.get("method").and_then(|v| v.as_str()) {
match method {
"session/update" => self.handle_session_update(&msg),
"session/request_permission" => {
self.handle_permission_request(&msg).await?;
}
other => {
tracing::debug!(target: "acp::wire", "ignoring unknown method: {other}");
}
}
}
}
Ok(Err(e)) => return Err(AcpError::Io(e)),
Err(_elapsed) => {
// Classification was determined before sleeping — not
// affected by scheduler jitter between deadline and wakeup.
if idle_fires_first {
tracing::warn!("idle timeout ({idle_timeout:?}) — no agent activity");
return Err(AcpError::IdleTimeout(idle_timeout));
} else {
tracing::warn!("hard turn timeout exceeded");
return Err(AcpError::HardTimeout);
}
}
}
}
}
/// Log a `session/update` notification via tracing.
///
/// The discriminator field is `sessionUpdate` (not `type`) per the ACP schema.
@@ -1194,4 +1368,135 @@ mod tests {
})
);
}
// ── Error variant display ─────────────────────────────────────────────
#[test]
fn idle_timeout_error_includes_duration() {
let err = AcpError::IdleTimeout(std::time::Duration::from_secs(300));
let msg = err.to_string();
assert!(
msg.contains("300"),
"IdleTimeout display should include duration: {msg}"
);
}
#[test]
fn hard_timeout_error_display() {
let err = AcpError::HardTimeout;
let msg = err.to_string();
assert!(
msg.contains("Hard turn timeout"),
"HardTimeout display: {msg}"
);
}
// ── Async integration tests with real subprocess ──────────────────────
async fn spawn_script(script: &str) -> AcpClient {
AcpClient::spawn("bash", &["-c".into(), script.into()])
.await
.expect("failed to spawn test script")
}
#[tokio::test]
async fn idle_timeout_fires_on_silent_process() {
let mut client = spawn_script("sleep 10").await;
let hard_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
let result = client
.read_until_response_with_idle_timeout(
999,
std::time::Duration::from_millis(100),
hard_deadline,
)
.await;
assert!(
matches!(result, Err(AcpError::IdleTimeout(_))),
"expected IdleTimeout, got {result:?}"
);
}
#[tokio::test]
async fn hard_timeout_fires_when_deadline_is_immediate() {
let mut client = spawn_script("while true; do echo 'noise'; sleep 0.01; done").await;
let hard_deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(1);
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
let result = client
.read_until_response_with_idle_timeout(
999,
std::time::Duration::from_secs(60),
hard_deadline,
)
.await;
assert!(
matches!(result, Err(AcpError::HardTimeout)),
"expected HardTimeout, got {result:?}"
);
}
#[tokio::test]
async fn idle_resets_on_stdout_activity() {
let mut client =
spawn_script("for i in $(seq 1 10); do echo 'keepalive'; sleep 0.05; done; sleep 10")
.await;
let hard_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
let start = std::time::Instant::now();
let result = client
.read_until_response_with_idle_timeout(
999,
std::time::Duration::from_millis(200),
hard_deadline,
)
.await;
let elapsed = start.elapsed();
assert!(elapsed >= std::time::Duration::from_millis(400));
assert!(elapsed < std::time::Duration::from_secs(3));
assert!(matches!(result, Err(AcpError::IdleTimeout(_))));
}
#[tokio::test]
async fn response_returned_when_matching_id_arrives() {
let mut client =
spawn_script(r#"echo '{"jsonrpc":"2.0","id":42,"result":{"stopReason":"end_turn"}}'"#)
.await;
let hard_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
let result = client
.read_until_response_with_idle_timeout(
42,
std::time::Duration::from_secs(2),
hard_deadline,
)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap()["stopReason"].as_str(), Some("end_turn"));
}
#[tokio::test]
async fn agent_exit_detected_as_eof() {
let mut client = spawn_script("exit 0").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let hard_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
let result = client
.read_until_response_with_idle_timeout(
999,
std::time::Duration::from_secs(2),
hard_deadline,
)
.await;
assert!(matches!(result, Err(AcpError::AgentExited)));
}
#[tokio::test]
async fn idle_fires_before_hard_when_idle_is_shorter() {
let mut client = spawn_script("sleep 10").await;
let idle = std::time::Duration::from_millis(100);
let hard_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
let result = client
.read_until_response_with_idle_timeout(999, idle, hard_deadline)
.await;
assert!(
matches!(result, Err(AcpError::IdleTimeout(_))),
"idle should fire before hard when idle << hard, got {result:?}"
);
}
}
+114 -7
View File
@@ -161,8 +161,18 @@ pub struct CliArgs {
)]
pub mcp_command: String,
#[arg(long, env = "SPROUT_ACP_TURN_TIMEOUT", default_value = "300")]
pub turn_timeout: u64,
/// Idle timeout: max seconds of silence before killing a turn.
/// Resets on any agent stdout activity.
#[arg(long, env = "SPROUT_ACP_IDLE_TIMEOUT")]
pub idle_timeout: Option<u64>,
/// Absolute wall-clock cap per turn (safety valve).
#[arg(long, env = "SPROUT_ACP_MAX_TURN_DURATION", default_value = "3600")]
pub max_turn_duration: u64,
/// Deprecated: alias for --idle-timeout. If both set, --idle-timeout wins.
#[arg(long, env = "SPROUT_ACP_TURN_TIMEOUT", hide = true)]
pub turn_timeout: Option<u64>,
#[arg(
long,
@@ -293,7 +303,8 @@ pub struct Config {
pub agent_command: String,
pub agent_args: Vec<String>,
pub mcp_command: String,
pub turn_timeout_secs: u64,
pub idle_timeout_secs: u64,
pub max_turn_duration_secs: u64,
pub agents: u32,
pub heartbeat_interval_secs: u64,
pub heartbeat_prompt: Option<String>,
@@ -436,7 +447,44 @@ impl Config {
agent_command,
agent_args,
mcp_command: args.mcp_command,
turn_timeout_secs: args.turn_timeout,
// Deprecated --turn-timeout is a fallback for backward compat.
// New deployments should use --idle-timeout exclusively.
// Precedence: explicit --idle-timeout > --turn-timeout (deprecated) > default 300.
idle_timeout_secs: {
let raw = match (args.idle_timeout, args.turn_timeout) {
(Some(idle), Some(_turn)) => {
tracing::warn!(
"--turn-timeout / SPROUT_ACP_TURN_TIMEOUT is deprecated and ignored \
when --idle-timeout / SPROUT_ACP_IDLE_TIMEOUT is also set"
);
idle
}
(Some(idle), None) => idle,
(None, Some(turn)) => {
tracing::warn!(
"--turn-timeout / SPROUT_ACP_TURN_TIMEOUT is deprecated; \
use --idle-timeout / SPROUT_ACP_IDLE_TIMEOUT instead"
);
turn
}
(None, None) => 300, // default
};
if raw == 0 {
tracing::warn!("idle timeout of 0 is invalid — using 1s minimum");
1
} else {
raw
}
},
max_turn_duration_secs: {
let raw = args.max_turn_duration;
if raw == 0 {
tracing::warn!("max turn duration of 0 is invalid — using 60s minimum");
60
} else {
raw
}
},
agents: args.agents,
heartbeat_interval_secs: args.heartbeat_interval,
heartbeat_prompt,
@@ -461,13 +509,14 @@ impl Config {
/// Human-readable summary (no secrets).
pub fn summary(&self) -> String {
format!(
"relay={} pubkey={} agent_cmd={} {} mcp_cmd={} timeout={}s agents={} heartbeat={}s subscribe={:?} dedup={:?} ignore_self={} context_limit={} max_turns_per_session={} presence={} typing={} model={} permission_mode={}",
"relay={} pubkey={} agent_cmd={} {} mcp_cmd={} idle_timeout={}s max_turn={}s agents={} heartbeat={}s subscribe={:?} dedup={:?} ignore_self={} context_limit={} max_turns_per_session={} presence={} typing={} model={} permission_mode={}",
self.relay_url,
self.keys.public_key().to_hex(),
self.agent_command,
self.agent_args.join(" "),
self.mcp_command,
self.turn_timeout_secs,
self.idle_timeout_secs,
self.max_turn_duration_secs,
self.agents,
self.heartbeat_interval_secs,
self.subscribe_mode,
@@ -759,7 +808,8 @@ mod tests {
agent_command: "goose".into(),
agent_args: vec!["acp".into()],
mcp_command: "sprout-mcp-server".into(),
turn_timeout_secs: 300,
idle_timeout_secs: 300,
max_turn_duration_secs: 3600,
agents: 1,
heartbeat_interval_secs: 0,
heartbeat_prompt: None,
@@ -1432,4 +1482,61 @@ channels = "ALL"
);
}
}
// ── Idle timeout config precedence ─────────────────────────────────────
/// Helper: resolve idle_timeout_secs using the same precedence logic as Config::from_args.
/// Precedence: explicit --idle-timeout > --turn-timeout (deprecated) > default 300.
fn resolve_idle_timeout(idle: Option<u64>, turn: Option<u64>) -> u64 {
let raw = match (idle, turn) {
(Some(idle), Some(_)) => idle,
(Some(idle), None) => idle,
(None, Some(turn)) => turn,
(None, None) => 300,
};
if raw == 0 {
1
} else {
raw
}
}
#[test]
fn idle_timeout_explicit_wins_over_deprecated() {
assert_eq!(resolve_idle_timeout(Some(120), Some(600)), 120);
}
#[test]
fn idle_timeout_falls_back_to_deprecated_turn_timeout() {
assert_eq!(resolve_idle_timeout(None, Some(600)), 600);
}
#[test]
fn idle_timeout_defaults_to_300_when_neither_set() {
assert_eq!(resolve_idle_timeout(None, None), 300);
}
#[test]
fn idle_timeout_zero_clamped_to_one() {
assert_eq!(resolve_idle_timeout(Some(0), None), 1);
}
#[test]
fn idle_timeout_zero_from_deprecated_clamped_to_one() {
assert_eq!(resolve_idle_timeout(None, Some(0)), 1);
}
#[test]
fn test_config_summary_includes_idle_and_max_turn() {
let config = test_config(SubscribeMode::Mentions);
let summary = config.summary();
assert!(
summary.contains("idle_timeout=300s"),
"summary should include idle_timeout: {summary}"
);
assert!(
summary.contains("max_turn=3600s"),
"summary should include max_turn: {summary}"
);
}
}
+8 -4
View File
@@ -218,7 +218,8 @@ async fn main() -> Result<()> {
let ctx = Arc::new(PromptContext {
mcp_servers: build_mcp_servers(&config),
initial_message: config.initial_message.clone(),
turn_timeout: Duration::from_secs(config.turn_timeout_secs),
idle_timeout: Duration::from_secs(config.idle_timeout_secs),
max_turn_duration: Duration::from_secs(config.max_turn_duration_secs),
dedup_mode: config.dedup_mode,
system_prompt: config.system_prompt.clone(),
heartbeat_prompt: config.heartbeat_prompt.clone(),
@@ -665,7 +666,9 @@ async fn main() -> Result<()> {
// ── Shutdown sequence ─────────────────────────────────────────────────────
tracing::info!("shutdown: waiting for in-flight prompts");
let grace = Duration::from_secs(config.turn_timeout_secs + 5);
// 30 s is generous for in-flight prompts to be cancelled; using
// max_turn_duration here would cause Ctrl+C to hang for up to an hour.
let grace = Duration::from_secs(30);
let shutdown_result = tokio::time::timeout(grace, async {
while let Some(result) = pool.join_set.join_next().await {
if let Err(e) = result {
@@ -824,11 +827,12 @@ async fn handle_prompt_result(
let agent_index = result.agent.index;
match result.outcome {
PromptOutcome::AgentExited => {
// Fatal outcomes: the agent subprocess is dead or poisoned — respawn it.
PromptOutcome::AgentExited | PromptOutcome::Timeout => {
tracing::debug!(
agent = agent_index,
outcome = outcome_label,
"agent_returned"
"agent_returned — respawning"
);
let index = result.agent.index;
match respawn_agent_into(result.agent, config).await {
+140 -53
View File
@@ -166,7 +166,8 @@ pub enum PromptOutcome {
pub struct PromptContext {
pub mcp_servers: Vec<McpServer>,
pub initial_message: Option<String>,
pub turn_timeout: Duration,
pub idle_timeout: Duration,
pub max_turn_duration: Duration,
pub dedup_mode: DedupMode,
pub system_prompt: Option<String>,
pub heartbeat_prompt: Option<String>,
@@ -357,9 +358,14 @@ async fn create_session_and_apply_model(
}
}
// Apply permission mode if not the agent's built-in default.
if !ctx.permission_mode.is_default() {
apply_permission_mode(&mut agent.acp, &resp.session_id, &ctx.permission_mode).await;
// Apply permission mode if not the agent's built-in default AND the agent
// advertises the requested mode in session/new. Agents that don't support
// the mode (e.g., goose crashes on unrecognized set_config_option values)
// are safely skipped — the harness auto-approves via handle_permission_request.
if !ctx.permission_mode.is_default()
&& agent_supports_mode(&resp.raw, ctx.permission_mode.as_wire_str())
{
apply_permission_mode(&mut agent.acp, &resp.session_id, &ctx.permission_mode).await?;
}
Ok(resp.session_id)
@@ -424,10 +430,33 @@ async fn apply_model_switch(
/// Set the session permission mode via `session/set_config_option`.
///
/// Non-fatal: logs and proceeds on timeout or error. The agent falls back
/// Non-fatal for most errors: logs and proceeds. The agent falls back
/// to its default permission mode (`"default"`), which still works via
/// Check if the agent's `session/new` response advertises a given mode ID
/// in `result.modes.availableModes[].id`. Returns `false` if the modes
/// field is absent or the mode isn't listed.
fn agent_supports_mode(session_new_result: &serde_json::Value, mode_wire: &str) -> bool {
session_new_result
.get("modes")
.and_then(|m| m.get("availableModes"))
.and_then(|a| a.as_array())
.map(|modes| {
modes
.iter()
.any(|m| m.get("id").and_then(|v| v.as_str()) == Some(mode_wire))
})
.unwrap_or(false)
}
/// per-tool auto-approval in `handle_permission_request`.
async fn apply_permission_mode(acp: &mut AcpClient, session_id: &str, mode: &PermissionMode) {
///
/// **Fatal exception:** if the agent process exits (e.g., goose crashes on
/// unrecognized methods), returns `Err(AgentExited)` so the caller can respawn.
async fn apply_permission_mode(
acp: &mut AcpClient,
session_id: &str,
mode: &PermissionMode,
) -> Result<(), AcpError> {
let wire = mode.as_wire_str();
let result = tokio::time::timeout(PERMISSION_MODE_TIMEOUT, async {
acp.session_set_config_option(session_id, "mode", wire)
@@ -442,6 +471,16 @@ async fn apply_permission_mode(acp: &mut AcpClient, session_id: &str, mode: &Per
"applied permission mode {wire:?} on session {session_id}"
);
}
Ok(Err(AcpError::AgentExited)) => {
// Fatal: the agent process crashed (e.g., goose doesn't support
// session/set_config_option and exits instead of returning an error).
// Propagate so the caller can respawn.
tracing::error!(
target: "pool::permission",
"agent exited while setting permission mode {wire:?} — process crashed"
);
return Err(AcpError::AgentExited);
}
Ok(Err(e)) => {
tracing::warn!(
target: "pool::permission",
@@ -455,6 +494,7 @@ async fn apply_permission_mode(acp: &mut AcpClient, session_id: &str, mode: &Per
);
}
}
Ok(())
}
/// Core async function spawned for each prompt.
@@ -578,20 +618,24 @@ pub async fn run_prompt_task(
target: "pool::session",
"sending initial_message to session {session_id} for channel {cid}"
);
let init_result = timeout(
ctx.turn_timeout,
agent.acp.session_prompt(&session_id, initial_msg),
)
.await;
let init_result = agent
.acp
.session_prompt_with_idle_timeout(
&session_id,
initial_msg,
ctx.idle_timeout,
ctx.max_turn_duration,
)
.await;
match init_result {
Ok(Ok(stop_reason)) => {
Ok(stop_reason) => {
tracing::info!(
target: "pool::session",
"initial_message complete for channel {cid}: {stop_reason:?}"
);
}
Ok(Err(AcpError::AgentExited)) => {
Err(AcpError::AgentExited) => {
agent.state.invalidate_all();
let _ = result_tx.send(PromptResult {
agent,
@@ -601,26 +645,17 @@ pub async fn run_prompt_task(
});
return;
}
Ok(Err(e)) => {
tracing::error!(
target: "pool::session",
"initial_message failed for channel {cid}: {e} — invalidating session"
);
agent.state.invalidate(&source);
let _ = result_tx.send(PromptResult {
agent,
source,
outcome: PromptOutcome::Error(e),
batch: requeue_batch_if_queue(&ctx, batch),
});
return;
}
Err(_elapsed) => {
Err(AcpError::IdleTimeout(_)) => {
tracing::warn!(
target: "pool::session",
"initial_message timed out for channel {cid} — cancelling"
"initial_message idle timeout ({}s) for channel {cid} — cancelling",
ctx.idle_timeout.as_secs()
);
match agent.acp.cancel_with_cleanup(&session_id).await {
match agent
.acp
.cancel_with_cleanup(&session_id, ctx.idle_timeout)
.await
{
Ok(_) => {
agent.state.invalidate(&source);
}
@@ -650,6 +685,35 @@ pub async fn run_prompt_task(
});
return;
}
Err(AcpError::HardTimeout) => {
tracing::error!(
target: "pool::session",
"hard timeout ({}s cap) during initial_message for channel {cid} — agent process is unrecoverable",
ctx.max_turn_duration.as_secs()
);
agent.state.invalidate_all();
let _ = result_tx.send(PromptResult {
agent,
source,
outcome: PromptOutcome::Timeout,
batch: requeue_batch_if_queue(&ctx, batch),
});
return;
}
Err(e) => {
tracing::error!(
target: "pool::session",
"initial_message failed for channel {cid}: {e} — invalidating session"
);
agent.state.invalidate(&source);
let _ = result_tx.send(PromptResult {
agent,
source,
outcome: PromptOutcome::Error(e),
batch: requeue_batch_if_queue(&ctx, batch),
});
return;
}
}
}
}
@@ -712,14 +776,18 @@ pub async fn run_prompt_task(
// ── Send the actual prompt ────────────────────────────────────────────
let prompt_result = timeout(
ctx.turn_timeout,
agent.acp.session_prompt(&session_id, &prompt_text),
)
.await;
let prompt_result = agent
.acp
.session_prompt_with_idle_timeout(
&session_id,
&prompt_text,
ctx.idle_timeout,
ctx.max_turn_duration,
)
.await;
match prompt_result {
Ok(Ok(stop_reason)) => {
Ok(stop_reason) => {
log_stop_reason(&source, &stop_reason);
// ── Session rotation on context exhaustion ────────────────
@@ -763,7 +831,7 @@ pub async fn run_prompt_task(
batch: None,
});
}
Ok(Err(AcpError::AgentExited)) => {
Err(AcpError::AgentExited) => {
tracing::error!(target: "pool::prompt", "agent {} exited during prompt", agent.index);
agent.state.invalidate_all();
let _ = result_tx.send(PromptResult {
@@ -773,27 +841,21 @@ pub async fn run_prompt_task(
batch: requeue_batch_if_queue(&ctx, batch),
});
}
Ok(Err(e)) => {
tracing::error!(target: "pool::prompt", "session_prompt error: {e}");
// Invalidate only the affected session.
agent.state.invalidate(&source);
let _ = result_tx.send(PromptResult {
agent,
source,
outcome: PromptOutcome::Error(e),
batch: requeue_batch_if_queue(&ctx, batch),
});
}
Err(_elapsed) => {
Err(AcpError::IdleTimeout(_)) => {
tracing::warn!(
target: "pool::prompt",
"turn timeout ({}s) — cancelling session {session_id}",
ctx.turn_timeout.as_secs()
"idle timeout ({}s) — cancelling session {session_id}",
ctx.idle_timeout.as_secs()
);
match agent.acp.cancel_with_cleanup(&session_id).await {
match agent
.acp
.cancel_with_cleanup(&session_id, ctx.idle_timeout)
.await
{
Ok(stop_reason) => {
log_stop_reason(&source, &stop_reason);
// Session is still valid after a clean cancel.
// Timeout triggers respawn in handle_prompt_result —
// session state will be discarded with the old agent.
let _ = result_tx.send(PromptResult {
agent,
source,
@@ -830,6 +892,31 @@ pub async fn run_prompt_task(
}
}
}
Err(AcpError::HardTimeout) => {
tracing::error!(
target: "pool::prompt",
"hard timeout ({}s cap) — agent process is unrecoverable, invalidating all sessions",
ctx.max_turn_duration.as_secs()
);
agent.state.invalidate_all();
let _ = result_tx.send(PromptResult {
agent,
source,
outcome: PromptOutcome::Timeout,
batch: requeue_batch_if_queue(&ctx, batch),
});
}
Err(e) => {
tracing::error!(target: "pool::prompt", "session_prompt error: {e}");
// Invalidate only the affected session.
agent.state.invalidate(&source);
let _ = result_tx.send(PromptResult {
agent,
source,
outcome: PromptOutcome::Error(e),
batch: requeue_batch_if_queue(&ctx, batch),
});
}
}
// _reaction_guard drops here → spawns clear_reactions for all exit paths.
}
+1 -1
View File
@@ -40,7 +40,7 @@ const overrides = new Map([
["src/features/tokens/ui/TokenSettingsCard.tsx", 800],
["src/shared/api/relayClientSession.ts", 740], // durable websocket session manager with reconnect/replay/recovery state + sendTypingIndicator
["src/shared/api/tauri.ts", 1100], // remote agent provider API bindings + canvas API functions
["src-tauri/src/commands/agents.rs", 845], // remote agent lifecycle routing (local + provider branches) + scope enforcement; rustfmt adds line breaks around long tuple/closure blocks
["src-tauri/src/commands/agents.rs", 849], // remote agent lifecycle routing (local + provider branches) + scope enforcement; rustfmt adds line breaks around long tuple/closure blocks
["src-tauri/src/managed_agents/backend.rs", 530], // provider IPC, validation, discovery, binary resolution + tests
["src/features/agents/ui/AgentsView.tsx", 790], // remote agent stop/delete + channel UUID resolution + presence-aware delete guard + persona/team import + provider/model fields
["src/features/agents/ui/CreateAgentDialog.tsx", 685], // provider selector + config form + schema-typed config coercion + required field validation + locked scopes
+6
View File
@@ -31,6 +31,8 @@ fn build_deploy_payload(record: &ManagedAgentRecord) -> serde_json::Value {
"system_prompt": &record.system_prompt,
"model": &record.model,
"turn_timeout_seconds": record.turn_timeout_seconds,
"idle_timeout_seconds": record.idle_timeout_seconds,
"max_turn_duration_seconds": record.max_turn_duration_seconds,
"parallelism": record.parallelism,
})
}
@@ -361,6 +363,10 @@ pub async fn create_managed_agent(
.turn_timeout_seconds
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_AGENT_TURN_TIMEOUT_SECONDS),
// 0 or None → harness uses its own default (300s idle, 3600s max).
// The harness CLI also clamps 0 → minimum, so both paths are safe.
idle_timeout_seconds: input.idle_timeout_seconds.filter(|s| *s > 0),
max_turn_duration_seconds: input.max_turn_duration_seconds.filter(|s| *s > 0),
parallelism: input
.parallelism
.filter(|count| (1..=32).contains(count))
@@ -200,6 +200,8 @@ pub fn build_managed_agent_summary(
agent_args: record.agent_args.clone(),
mcp_command: record.mcp_command.clone(),
turn_timeout_seconds: record.turn_timeout_seconds,
idle_timeout_seconds: record.idle_timeout_seconds,
max_turn_duration_seconds: record.max_turn_duration_seconds,
parallelism: record.parallelism,
system_prompt: record.system_prompt.clone(),
model: record.model.clone(),
@@ -294,10 +296,23 @@ pub fn start_managed_agent_process(
command.env("SPROUT_ACP_AGENT_COMMAND", &record.agent_command);
command.env("SPROUT_ACP_AGENT_ARGS", agent_args.join(","));
command.env("SPROUT_ACP_MCP_COMMAND", &resolved_mcp_command);
command.env(
"SPROUT_ACP_TURN_TIMEOUT",
record.turn_timeout_seconds.to_string(),
);
// Timeout configuration: always set both IDLE_TIMEOUT and the deprecated TURN_TIMEOUT
// so older harness binaries (which only read TURN_TIMEOUT) still get a value.
if let Some(idle) = record.idle_timeout_seconds {
command.env("SPROUT_ACP_IDLE_TIMEOUT", idle.to_string());
// Mirror to deprecated var for older harness binaries.
command.env("SPROUT_ACP_TURN_TIMEOUT", idle.to_string());
} else {
command.env(
"SPROUT_ACP_TURN_TIMEOUT",
record.turn_timeout_seconds.to_string(),
);
}
let max_dur = record
.max_turn_duration_seconds
.unwrap_or(super::types::DEFAULT_AGENT_MAX_TURN_DURATION_SECONDS);
command.env("SPROUT_ACP_MAX_TURN_DURATION", max_dur.to_string());
command.env("SPROUT_ACP_AGENTS", record.parallelism.to_string());
command.env(
"GOOSE_MODE",
+14 -2
View File
@@ -57,6 +57,12 @@ pub struct ManagedAgentRecord {
pub agent_args: Vec<String>,
pub mcp_command: String,
pub turn_timeout_seconds: u64,
/// Idle timeout in seconds. If set, overrides turn_timeout_seconds.
#[serde(default)]
pub idle_timeout_seconds: Option<u64>,
/// Absolute wall-clock cap per turn.
#[serde(default)]
pub max_turn_duration_seconds: Option<u64>,
#[serde(default = "default_agent_parallelism")]
pub parallelism: u32,
pub system_prompt: Option<String>,
@@ -100,6 +106,8 @@ pub struct ManagedAgentSummary {
pub agent_args: Vec<String>,
pub mcp_command: String,
pub turn_timeout_seconds: u64,
pub idle_timeout_seconds: Option<u64>,
pub max_turn_duration_seconds: Option<u64>,
pub parallelism: u32,
pub system_prompt: Option<String>,
pub model: Option<String>,
@@ -131,6 +139,8 @@ pub struct CreateManagedAgentRequest {
pub agent_args: Vec<String>,
pub mcp_command: Option<String>,
pub turn_timeout_seconds: Option<u64>,
pub idle_timeout_seconds: Option<u64>,
pub max_turn_duration_seconds: Option<u64>,
pub parallelism: Option<u32>,
pub system_prompt: Option<String>,
pub avatar_url: Option<String>,
@@ -310,8 +320,10 @@ pub const DEFAULT_ADMIN_COMMAND: &str = "sprout-admin";
pub const DEFAULT_AGENT_COMMAND: &str = "goose";
pub const DEFAULT_MCP_COMMAND: &str = "sprout-mcp-server";
pub const DEFAULT_AGENT_ARG: &str = "acp";
/// 10 min — agents with tool-heavy turns regularly exceed the previous 5 min default.
pub const DEFAULT_AGENT_TURN_TIMEOUT_SECONDS: u64 = 600;
/// 5 min — matches the CLI harness default (SPROUT_ACP_IDLE_TIMEOUT).
pub const DEFAULT_AGENT_TURN_TIMEOUT_SECONDS: u64 = 300;
/// 1 hour — absolute wall-clock safety cap per turn.
pub const DEFAULT_AGENT_MAX_TURN_DURATION_SECONDS: u64 = 3600;
pub const DEFAULT_AGENT_PARALLELISM: u32 = 1;
fn default_agent_parallelism() -> u32 {
+6
View File
@@ -233,6 +233,8 @@ export type RawManagedAgent = {
agent_args: string[];
mcp_command: string;
turn_timeout_seconds: number;
idle_timeout_seconds: number | null;
max_turn_duration_seconds: number | null;
parallelism: number;
system_prompt: string | null;
model: string | null;
@@ -810,6 +812,8 @@ export function fromRawManagedAgent(agent: RawManagedAgent): ManagedAgent {
agentArgs: agent.agent_args,
mcpCommand: agent.mcp_command,
turnTimeoutSeconds: agent.turn_timeout_seconds,
idleTimeoutSeconds: agent.idle_timeout_seconds,
maxTurnDurationSeconds: agent.max_turn_duration_seconds,
parallelism: agent.parallelism,
systemPrompt: agent.system_prompt,
model: agent.model,
@@ -910,6 +914,8 @@ export async function createManagedAgent(input: CreateManagedAgentInput) {
agentArgs: input.agentArgs,
mcpCommand: input.mcpCommand,
turnTimeoutSeconds: input.turnTimeoutSeconds,
idleTimeoutSeconds: input.idleTimeoutSeconds,
maxTurnDurationSeconds: input.maxTurnDurationSeconds,
parallelism: input.parallelism,
systemPrompt: input.systemPrompt,
avatarUrl: input.avatarUrl,
+4
View File
@@ -286,6 +286,8 @@ export type ManagedAgent = {
agentArgs: string[];
mcpCommand: string;
turnTimeoutSeconds: number;
idleTimeoutSeconds: number | null;
maxTurnDurationSeconds: number | null;
parallelism: number;
systemPrompt: string | null;
model: string | null;
@@ -326,6 +328,8 @@ export type CreateManagedAgentInput = {
agentArgs?: string[];
mcpCommand?: string;
turnTimeoutSeconds?: number;
idleTimeoutSeconds?: number;
maxTurnDurationSeconds?: number;
parallelism?: number;
systemPrompt?: string;
avatarUrl?: string;
+8
View File
@@ -226,6 +226,8 @@ type RawManagedAgent = {
agent_args: string[];
mcp_command: string;
turn_timeout_seconds: number;
idle_timeout_seconds: number | null;
max_turn_duration_seconds: number | null;
parallelism: number;
system_prompt: string | null;
model: string | null;
@@ -541,6 +543,8 @@ function cloneManagedAgent(agent: MockManagedAgent): RawManagedAgent {
agent_args: [...agent.agent_args],
mcp_command: agent.mcp_command,
turn_timeout_seconds: agent.turn_timeout_seconds,
idle_timeout_seconds: agent.idle_timeout_seconds ?? null,
max_turn_duration_seconds: agent.max_turn_duration_seconds ?? null,
parallelism: agent.parallelism,
system_prompt: agent.system_prompt,
model: agent.model,
@@ -2535,6 +2539,8 @@ async function handleCreateManagedAgent(args: {
agentArgs?: string[];
mcpCommand?: string;
turnTimeoutSeconds?: number;
idleTimeoutSeconds?: number;
maxTurnDurationSeconds?: number;
parallelism?: number;
systemPrompt?: string;
avatarUrl?: string;
@@ -2573,6 +2579,8 @@ async function handleCreateManagedAgent(args: {
: ["acp"],
mcp_command: args.input.mcpCommand ?? "sprout-mcp-server",
turn_timeout_seconds: args.input.turnTimeoutSeconds ?? 300,
idle_timeout_seconds: args.input.idleTimeoutSeconds ?? null,
max_turn_duration_seconds: args.input.maxTurnDurationSeconds ?? null,
parallelism: args.input.parallelism ?? 1,
system_prompt: args.input.systemPrompt?.trim() || null,
model: args.input.model?.trim() || null,