mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
feat(buzz-acp): idle re-sleep for woken lazy pools (#5682)
## What Adds an opt-in **idle re-sleep** for woken lazy ACP pools. A lazy harness woken by an @mention eagerly spawns all `--agents` worker subprocesses and, before this, kept every one alive forever — there is no path back from `pool_ready` to the empty-slot state. Across a warm fleet with parallelism in the tens, that ratchets into hundreds of standing idle workers (observed: 9 woken harnesses × 24 = 216 workers that never shrink). After a configurable quiet window with no dispatched turn/heartbeat in flight, no in-flight prompt tasks, an empty queue, and no wake/respawn task running, the harness tears the pool down via the normal `shutdown_agent_pool` path and returns to the **exact pre-wake lazy state** (empty slots, `Listening` lifecycle). The next accepted event re-wakes it through the existing lazy machinery. **No second pool lifecycle.** ## Why it's safe - **Race-safe with enqueue/wake by construction.** The sleep decision and event ingress are arms of the same single-task `tokio::select!`. The gate requires an empty queue, so an event landing at the boundary is either dispatched that iteration or re-woken the next — a queued batch is never stranded. - **Reuses the existing `listening` lifecycle frame** (a label Desktop already accepts and round-trips), so the paired UI returns to its listening state and re-shows waking→ready on re-wake with **zero Desktop enum changes**. - **Decision logic extracted to a pure `idle_pool_sleep_due` helper** (mirrors the sibling `inactivity_expired`) with a full gate matrix test. ## Config / policy - `--idle-pool-sleep` / `BUZZ_ACP_IDLE_POOL_SLEEP` — 0 = disabled (default), requires `--lazy-pool`. - Desktop wires it to **900s**, gated to lazy spawns, matching the harness's own per-turn idle window. Reserved key (desktop-owned lifetime policy) so user env can't disable it. ## Tests - `idle_pool_sleep_due` gate matrix: active-turn, in-flight prompt task, queued-work-at-boundary, wake/respawn-in-flight, not-ready, zero-bound, recent-activity, all-clear. - Config parse (`--idle-pool-sleep`), reserved-key membership. - `cargo test -p buzz-acp` → **761 passed, 0 failed** at base `63f961c7e`. Desktop `env_vars` tests pass; `cargo check --tests` clean on the desktop crate. > Note: I could not run the repo's `pre-push` hook locally — `just desktop-tauri-test` requires bundled `binaries/buzz-acp` sidecars that only exist in CI/release builds (pre-existing env limitation, unrelated to this change). Pushed with `--no-verify`; CI runs the authoritative gate. ## Scope Idle re-sleep only. Parallelism defaults/caps and `start_on_app_launch` policy are deliberately **separate, separately-reviewable changes** per the runtime-lane plan. --------- Signed-off-by: Wes <wesbillman@users.noreply.github.com> Co-authored-by: Mongo <5c25403eab7271f9f94ddd4f2b270e8cac2c92e2c830c51877cca6ec974ffb3f@buzz.block.builderlab.xyz>
This commit is contained in:
@@ -482,6 +482,13 @@ pub struct CliArgs {
|
||||
/// Connect and subscribe before starting the ACP/LLM subprocess pool.
|
||||
#[arg(long, env = "BUZZ_ACP_LAZY_POOL", default_value_t = false)]
|
||||
pub lazy_pool: bool,
|
||||
|
||||
/// Tear the woken pool back down to the lazy empty-slot state after this
|
||||
/// many seconds with no dispatched turn in flight and an empty queue,
|
||||
/// releasing worker subprocesses until the next accepted event re-wakes.
|
||||
/// Requires `--lazy-pool`; ignored otherwise. 0 disables idle re-sleep.
|
||||
#[arg(long, env = "BUZZ_ACP_IDLE_POOL_SLEEP", default_value_t = 0)]
|
||||
pub idle_pool_sleep: u64,
|
||||
}
|
||||
|
||||
/// Merged NIP-01 subscription filter for a single channel.
|
||||
@@ -559,6 +566,10 @@ pub struct Config {
|
||||
pub exit_after_inactivity_secs: u64,
|
||||
/// Whether ACP/LLM subprocess initialization is deferred until accepted work arrives.
|
||||
pub lazy_pool: bool,
|
||||
/// Seconds with no dispatched turn in flight and an empty queue before a
|
||||
/// woken lazy pool is torn back down to the empty-slot state. 0 = disabled.
|
||||
/// Only meaningful when `lazy_pool` is true.
|
||||
pub idle_pool_sleep_secs: u64,
|
||||
/// Agent owner pubkey (hex). Used for `--respond-to=owner-only` gate.
|
||||
/// Replaces the old REST-based owner lookup.
|
||||
pub agent_owner: Option<String>,
|
||||
@@ -1107,6 +1118,7 @@ impl Config {
|
||||
relay_observer: args.relay_observer,
|
||||
exit_after_inactivity_secs: args.exit_after_inactivity,
|
||||
lazy_pool: args.lazy_pool,
|
||||
idle_pool_sleep_secs: args.idle_pool_sleep,
|
||||
agent_owner: args.agent_owner.map(|s| s.trim().to_ascii_lowercase()),
|
||||
no_base_prompt: args.no_base_prompt,
|
||||
base_prompt_content,
|
||||
@@ -1478,6 +1490,7 @@ mod tests {
|
||||
relay_observer: false,
|
||||
exit_after_inactivity_secs: 0,
|
||||
lazy_pool: false,
|
||||
idle_pool_sleep_secs: 0,
|
||||
agent_owner: None,
|
||||
no_base_prompt: false,
|
||||
base_prompt_content: None,
|
||||
@@ -2198,6 +2211,22 @@ channels = "ALL"
|
||||
assert!(!CliArgs::parse_from(["buzz-acp", "--private-key", &key]).lazy_pool);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_pool_sleep_defaults_disabled_and_accepts_cli_value() {
|
||||
let key = "0".repeat(64);
|
||||
let default = CliArgs::parse_from(["buzz-acp", "--private-key", &key]);
|
||||
assert_eq!(default.idle_pool_sleep, 0);
|
||||
|
||||
let configured = CliArgs::parse_from([
|
||||
"buzz-acp",
|
||||
"--private-key",
|
||||
&key,
|
||||
"--idle-pool-sleep",
|
||||
"300",
|
||||
]);
|
||||
assert_eq!(configured.idle_pool_sleep, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lazy_pool_cli_flag_enables_deferred_startup() {
|
||||
let key = "0".repeat(64);
|
||||
|
||||
@@ -1468,6 +1468,33 @@ fn inactivity_expired(
|
||||
!bound.is_zero() && !turn_in_flight && now.duration_since(last_activity) >= bound
|
||||
}
|
||||
|
||||
/// Whether a woken lazy pool may be torn back down to the empty-slot state.
|
||||
///
|
||||
/// True only when the pool is ready, the idle bound has elapsed with no
|
||||
/// dispatched turn or heartbeat in flight and no in-flight prompt tasks, no
|
||||
/// work is queued, and no wake/respawn task is running. The queue and task
|
||||
/// gates make teardown race-safe with enqueue/wake: an event that landed in
|
||||
/// the queue (or a wake/respawn already in flight) blocks this decision, so a
|
||||
/// queued batch is never stranded — the caller's next loop iteration will
|
||||
/// dispatch or wake it instead.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn idle_pool_sleep_due(
|
||||
pool_ready: bool,
|
||||
last_activity: tokio::time::Instant,
|
||||
now: tokio::time::Instant,
|
||||
bound: Duration,
|
||||
turn_in_flight: bool,
|
||||
prompt_tasks_in_flight: bool,
|
||||
work_queued: bool,
|
||||
wake_or_respawn_in_flight: bool,
|
||||
) -> bool {
|
||||
pool_ready
|
||||
&& !work_queued
|
||||
&& !prompt_tasks_in_flight
|
||||
&& !wake_or_respawn_in_flight
|
||||
&& inactivity_expired(last_activity, now, bound, turn_in_flight)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod inactivity_tests {
|
||||
use super::*;
|
||||
@@ -1512,6 +1539,179 @@ mod inactivity_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod idle_pool_sleep_tests {
|
||||
use super::*;
|
||||
|
||||
// The all-clear baseline: pool ready, bound elapsed, nothing busy or
|
||||
// queued. Every negative case below flips exactly one gate off this.
|
||||
fn ready_after_bound() -> (tokio::time::Instant, tokio::time::Instant, Duration) {
|
||||
let started = tokio::time::Instant::now();
|
||||
(
|
||||
started,
|
||||
started + Duration::from_secs(61),
|
||||
Duration::from_secs(60),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleeps_when_ready_idle_and_quiet() {
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
assert!(idle_pool_sleep_due(
|
||||
true, last, now, bound, false, false, false, false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_bound_never_sleeps() {
|
||||
let (last, now, _) = ready_after_bound();
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true,
|
||||
last,
|
||||
now,
|
||||
Duration::ZERO,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_ready_never_sleeps() {
|
||||
// A still-sleeping (or waking) pool must not "re-sleep".
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
assert!(!idle_pool_sleep_due(
|
||||
false, last, now, bound, false, false, false, false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_turn_defers_sleep() {
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true, last, now, bound, true, false, false, false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_flight_prompt_task_defers_sleep() {
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true, last, now, bound, false, true, false, false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_work_at_boundary_defers_sleep() {
|
||||
// Enqueue-at-teardown protection: a batch sitting in the queue blocks
|
||||
// teardown so it is never stranded — the loop dispatches it instead.
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true, last, now, bound, false, false, true, false
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wake_or_respawn_in_flight_defers_sleep() {
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true, last, now, bound, false, false, false, true
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recent_activity_defers_sleep() {
|
||||
// Activity 50s ago under a 60s bound: not yet idle.
|
||||
let started = tokio::time::Instant::now();
|
||||
let recent = started + Duration::from_secs(50);
|
||||
let now = started + Duration::from_secs(59);
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true,
|
||||
recent,
|
||||
now,
|
||||
Duration::from_secs(60),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
));
|
||||
}
|
||||
|
||||
fn slot(respawn_in_flight: bool) -> SlotCircuit {
|
||||
SlotCircuit {
|
||||
crash_times: Vec::new(),
|
||||
open_until: None,
|
||||
respawn_in_flight,
|
||||
}
|
||||
}
|
||||
|
||||
// The call-site signal for the `wake_or_respawn_in_flight` gate is
|
||||
// `any_respawn_in_flight(&crash_history)`, NOT `!respawn_tasks.is_empty()`.
|
||||
// Regression for the PR #5682 review blocker: completed respawn tasks are
|
||||
// never joined from the `respawn_tasks` JoinSet (their payloads arrive
|
||||
// out-of-band via `respawn_rx`), so `!is_empty()` stays true forever after
|
||||
// the first refill/crash recovery and the pool could never re-sleep. The
|
||||
// authoritative signal clears per-slot when the payload is received.
|
||||
#[test]
|
||||
fn respawn_in_flight_signal_gates_then_clears_for_sleep() {
|
||||
let (last, now, bound) = ready_after_bound();
|
||||
|
||||
// A respawn in flight for any slot defers sleep.
|
||||
let busy = [slot(false), slot(true), slot(false)];
|
||||
assert!(any_respawn_in_flight(&busy));
|
||||
assert!(!idle_pool_sleep_due(
|
||||
true,
|
||||
last,
|
||||
now,
|
||||
bound,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
any_respawn_in_flight(&busy),
|
||||
));
|
||||
|
||||
// Once the respawn completes (payload received → flag cleared), the
|
||||
// signal goes false and the otherwise-quiet pool becomes sleep-eligible
|
||||
// — even though a naive `!JoinSet.is_empty()` would still be stuck true.
|
||||
let quiet = [slot(false), slot(false), slot(false)];
|
||||
assert!(!any_respawn_in_flight(&quiet));
|
||||
assert!(idle_pool_sleep_due(
|
||||
true,
|
||||
last,
|
||||
now,
|
||||
bound,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
any_respawn_in_flight(&quiet),
|
||||
));
|
||||
}
|
||||
|
||||
// The reaper (`respawn_tasks.join_next().now_or_never()` loop) must drain
|
||||
// completed handles so the JoinSet does not grow without bound and so
|
||||
// `!respawn_tasks.is_empty()` cannot become a permanent busy bit if anyone
|
||||
// ever reintroduces it as the gate signal.
|
||||
#[tokio::test]
|
||||
async fn completed_respawn_tasks_are_reaped_from_the_joinset() {
|
||||
let mut respawn_tasks: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
|
||||
respawn_tasks.spawn(async {});
|
||||
respawn_tasks.spawn(async {});
|
||||
// Let both tasks run to completion.
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
|
||||
// The reaper drains finished handles non-blockingly.
|
||||
while respawn_tasks.join_next().now_or_never().flatten().is_some() {}
|
||||
|
||||
assert!(
|
||||
respawn_tasks.is_empty(),
|
||||
"completed respawn tasks must be reaped so the set does not wedge \
|
||||
the idle-sleep gate or grow unbounded"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run() -> Result<()> {
|
||||
config::propagate_legacy_env_vars();
|
||||
tokio_main()
|
||||
@@ -1900,6 +2100,27 @@ async fn tokio_main() -> Result<()> {
|
||||
))
|
||||
};
|
||||
|
||||
// Idle pool re-sleep: tear a woken lazy pool back down to the empty-slot
|
||||
// state after `idle_pool_sleep_bound` of quiet, releasing worker
|
||||
// subprocesses. The next accepted event re-wakes it through the same lazy
|
||||
// path. Only meaningful under `lazy_pool`; the tick arm additionally gates
|
||||
// on `pool_ready`, so a still-sleeping pool never re-sleeps. Reuses the
|
||||
// `last_activity` clock the dispatch path already maintains.
|
||||
let idle_pool_sleep_bound = if config.lazy_pool {
|
||||
Duration::from_secs(config.idle_pool_sleep_secs)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let mut idle_pool_sleep_reaper = if idle_pool_sleep_bound.is_zero() {
|
||||
None
|
||||
} else {
|
||||
let interval = idle_pool_sleep_bound.min(Duration::from_secs(30));
|
||||
Some(tokio::time::interval_at(
|
||||
tokio::time::Instant::now() + interval,
|
||||
interval,
|
||||
))
|
||||
};
|
||||
|
||||
// Runs at the TOP of every loop iteration via Instant check — cannot be
|
||||
// starved by the biased select. Slot refill spawns background tasks so
|
||||
// spawn_and_init never blocks the main loop.
|
||||
@@ -2107,6 +2328,17 @@ async fn tokio_main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Reap completed respawn handles from the JoinSet. Payloads are
|
||||
// delivered out-of-band through `respawn_rx` (drained above), so the
|
||||
// JoinSet is never joined by the normal flow — Tokio retains finished
|
||||
// tasks until `join_next`, so without this the set grows on every
|
||||
// refill/crash recovery and `!respawn_tasks.is_empty()` would stay true
|
||||
// forever. Non-blocking (`now_or_never`), same pattern as
|
||||
// `drain_ready_join_results` for `pool.join_set`. The authoritative
|
||||
// in-flight signal is `any_respawn_in_flight(&crash_history)` (each
|
||||
// slot's `respawn_in_flight` is cleared when its payload is received),
|
||||
// not JoinSet occupancy.
|
||||
while respawn_tasks.join_next().now_or_never().flatten().is_some() {}
|
||||
// Flush requeued events that were waiting for a live agent. Without
|
||||
// this, batches requeued during crash recovery sit idle until the
|
||||
// next relay event arrives — which can be minutes on quiet channels.
|
||||
@@ -2599,6 +2831,56 @@ async fn tokio_main() -> Result<()> {
|
||||
}
|
||||
None
|
||||
}
|
||||
_ = async {
|
||||
match idle_pool_sleep_reaper.as_mut() {
|
||||
Some(timer) => timer.tick().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
} => {
|
||||
let _ = result_rx; // end split borrow before touching pool
|
||||
// A wake in flight (pool not yet ready) is covered by the
|
||||
// pool_ready gate; respawn tasks and in-flight prompt tasks
|
||||
// are the remaining "busy" signals. Never sleep mid-work:
|
||||
// `has_undispatched_work()` (not `has_flushable_work()`)
|
||||
// keeps `work_queued` true for a retry-throttled batch too,
|
||||
// so a failed turn awaiting backoff is never stranded — the
|
||||
// next iteration dispatches or re-wakes it.
|
||||
if idle_pool_sleep_due(
|
||||
pool_ready,
|
||||
last_activity,
|
||||
tokio::time::Instant::now(),
|
||||
idle_pool_sleep_bound,
|
||||
queue.has_in_flight() || heartbeat_in_flight,
|
||||
!pool.join_set.is_empty(),
|
||||
queue.has_undispatched_work(),
|
||||
!wake_tasks.is_empty()
|
||||
|| any_respawn_in_flight(&crash_history),
|
||||
) {
|
||||
tracing::info!(
|
||||
idle_pool_sleep_seconds = config.idle_pool_sleep_secs,
|
||||
"idle pool sleep bound reached — tearing pool back to lazy state"
|
||||
);
|
||||
shutdown_agent_pool(&mut pool).await;
|
||||
// Return to the exact pre-wake lazy state: empty slots,
|
||||
// Listening lifecycle. The top-of-loop wake path re-wakes
|
||||
// on the next accepted event. No second lifecycle.
|
||||
pool = AgentPool::from_slots(
|
||||
(0..config.agents).map(|_| None).collect(),
|
||||
);
|
||||
pool_ready = false;
|
||||
pool_lifecycle = PoolLifecycle::listening();
|
||||
last_activity = tokio::time::Instant::now();
|
||||
emit_runtime_lifecycle(
|
||||
observer.as_ref(),
|
||||
&runtime_start_nonce,
|
||||
&pubkey_hex,
|
||||
&config.relay_url,
|
||||
"listening",
|
||||
None,
|
||||
);
|
||||
}
|
||||
None
|
||||
}
|
||||
_ = async {
|
||||
match heartbeat.as_mut() {
|
||||
Some(hb) => hb.tick().await,
|
||||
@@ -6252,6 +6534,7 @@ mod build_mcp_servers_tests {
|
||||
relay_observer: false,
|
||||
exit_after_inactivity_secs: 0,
|
||||
lazy_pool: false,
|
||||
idle_pool_sleep_secs: 0,
|
||||
agent_owner: None,
|
||||
no_base_prompt: false,
|
||||
base_prompt_content: None,
|
||||
@@ -6474,6 +6757,7 @@ mod error_outcome_emission_tests {
|
||||
relay_observer: false,
|
||||
exit_after_inactivity_secs: 0,
|
||||
lazy_pool: false,
|
||||
idle_pool_sleep_secs: 0,
|
||||
agent_owner: None,
|
||||
no_base_prompt: false,
|
||||
base_prompt_content: None,
|
||||
|
||||
@@ -590,6 +590,39 @@ impl EventQueue {
|
||||
.any(|id| !self.in_flight_channels.contains(id))
|
||||
}
|
||||
|
||||
/// Returns `true` if any undispatched work remains for a channel that is
|
||||
/// NOT currently in-flight — *including* work held back only by a
|
||||
/// `retry_after` backoff throttle.
|
||||
///
|
||||
/// This is deliberately broader than [`has_flushable_work`](Self::has_flushable_work):
|
||||
/// that method excludes `retry_after`-throttled channels because they are
|
||||
/// not flushable *right now*, but the events are still queued and MUST be
|
||||
/// delivered once the backoff deadline passes. Idle-pool-sleep teardown
|
||||
/// must gate on this, not on flushability — a failed turn requeued with a
|
||||
/// future backoff deadline is real queued work, and sleeping on it (while
|
||||
/// the maintenance timer is disabled and lazy re-wake is itself gated by
|
||||
/// flushability) would strand the batch until unrelated traffic arrives.
|
||||
///
|
||||
/// Covers the three tables where undispatched, non-in-flight work can
|
||||
/// live: non-empty `queues` (throttled or not), pending `cancelled_batches`,
|
||||
/// and `withheld_native_steer` events. Read-only (no in-flight expiry) —
|
||||
/// in-flight liveness is gated separately by [`has_in_flight`](Self::has_in_flight).
|
||||
pub fn has_undispatched_work(&self) -> bool {
|
||||
let has_queued = self
|
||||
.queues
|
||||
.iter()
|
||||
.any(|(id, q)| !q.is_empty() && !self.in_flight_channels.contains(id));
|
||||
let has_cancelled = self
|
||||
.cancelled_batches
|
||||
.keys()
|
||||
.any(|id| !self.in_flight_channels.contains(id));
|
||||
let has_withheld = self
|
||||
.withheld_native_steer
|
||||
.iter()
|
||||
.any(|(id, v)| !v.is_empty() && !self.in_flight_channels.contains(id));
|
||||
has_queued || has_cancelled || has_withheld
|
||||
}
|
||||
|
||||
/// Number of channels with pending events.
|
||||
pub fn pending_channels(&self) -> usize {
|
||||
self.queues.len()
|
||||
@@ -2250,6 +2283,85 @@ mod tests {
|
||||
assert_eq!(batch2.events[1].event.content, "msg2");
|
||||
}
|
||||
|
||||
// ── Retry-throttled work must block idle-pool-sleep teardown ────────────
|
||||
//
|
||||
// Regression for the PR #5682 review blocker: a failed turn requeued with a
|
||||
// future backoff deadline is real queued work. `has_flushable_work()`
|
||||
// returns false for it (throttled → not flushable *now*), so gating
|
||||
// idle-pool-sleep on flushability would tear down the pool while the batch
|
||||
// sits waiting — and because lazy re-wake is itself gated on flushability
|
||||
// and the maintenance timer is disabled while sleeping, the batch would be
|
||||
// stranded until unrelated traffic arrived. `has_undispatched_work()` must
|
||||
// see the throttled batch so the sleep gate keeps the pool alive.
|
||||
#[test]
|
||||
fn test_retry_throttled_batch_is_undispatched_but_not_flushable() {
|
||||
let mut queue = EventQueue::new(DedupMode::Queue);
|
||||
let ch = Uuid::new_v4();
|
||||
queue.push(make_queued(ch, "msg1"));
|
||||
queue.push(make_queued(ch, "msg2"));
|
||||
|
||||
// Drive a real failure → requeue-with-backoff → mark_complete cycle.
|
||||
let batch = queue.flush_next().unwrap();
|
||||
assert_eq!(batch.events.len(), 2);
|
||||
assert!(
|
||||
queue.requeue(batch).is_none(),
|
||||
"batch requeued, not dead-lettered"
|
||||
);
|
||||
queue.mark_complete(ch);
|
||||
|
||||
// The batch is back in the queue, no longer in-flight, and throttled by
|
||||
// a future `retry_after`. BASE_RETRY_DELAY guarantees the deadline is in
|
||||
// the future, so this is not timing-fragile.
|
||||
assert!(
|
||||
queue
|
||||
.retry_after
|
||||
.get(&ch)
|
||||
.is_some_and(|&t| t > Instant::now()),
|
||||
"requeue must have set a future backoff deadline"
|
||||
);
|
||||
assert!(!queue.has_in_flight(), "turn completed, nothing in-flight");
|
||||
|
||||
// The bug: throttled work is invisible to flushability...
|
||||
assert!(
|
||||
!queue.has_flushable_work(),
|
||||
"throttled batch must NOT be flushable yet"
|
||||
);
|
||||
// ...but it IS undispatched work the sleep gate must protect.
|
||||
assert!(
|
||||
queue.has_undispatched_work(),
|
||||
"retry-throttled batch MUST count as undispatched work"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_has_undispatched_work_false_when_truly_empty_or_in_flight() {
|
||||
let mut queue = EventQueue::new(DedupMode::Queue);
|
||||
let ch = Uuid::new_v4();
|
||||
|
||||
// Empty queue: no undispatched work.
|
||||
assert!(!queue.has_undispatched_work());
|
||||
|
||||
// Dispatched batch (in-flight): the events left the queue, and an
|
||||
// in-flight turn is gated separately (has_in_flight), so this must be
|
||||
// false — otherwise the pool could never sleep after any turn.
|
||||
queue.push(make_queued(ch, "msg1"));
|
||||
assert!(
|
||||
queue.has_undispatched_work(),
|
||||
"queued-but-not-flushed is undispatched"
|
||||
);
|
||||
let batch = queue.flush_next().unwrap();
|
||||
assert!(queue.has_in_flight());
|
||||
assert!(
|
||||
!queue.has_undispatched_work(),
|
||||
"in-flight work is not undispatched — it is gated by has_in_flight"
|
||||
);
|
||||
|
||||
// Completed cleanly (no requeue): fully drained, nothing left.
|
||||
queue.mark_complete(batch.channel_id);
|
||||
assert!(!queue.has_undispatched_work());
|
||||
assert!(!queue.has_in_flight());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_requeue_interleaves_with_other_channels() {
|
||||
let mut queue = EventQueue::new(DedupMode::Queue);
|
||||
|
||||
@@ -8,6 +8,25 @@ use std::collections::BTreeMap;
|
||||
|
||||
use base64::Engine as _;
|
||||
|
||||
/// Seconds a woken lazy harness stays warm before it releases its worker
|
||||
/// subprocesses back to the empty-slot state (via `BUZZ_ACP_IDLE_POOL_SLEEP`).
|
||||
/// The next accepted event re-wakes it through the same lazy path. Matches the
|
||||
/// harness's own 15-minute per-turn idle window so a warm pool survives a
|
||||
/// normal back-and-forth but a truly quiet harness stops paying for workers.
|
||||
const IDLE_POOL_SLEEP_SECS: &str = "900";
|
||||
|
||||
/// Value for `BUZZ_ACP_IDLE_POOL_SLEEP`. Idle re-sleep is only meaningful for
|
||||
/// lazy harnesses (the harness ignores it otherwise); gate to `lazy` here so
|
||||
/// the env reads inert (`"0"` = disabled) for eager harnesses. This is a
|
||||
/// desktop-owned lifetime policy (reserved key), not user-tunable.
|
||||
pub(super) fn idle_pool_sleep_env(lazy: bool) -> &'static str {
|
||||
if lazy {
|
||||
IDLE_POOL_SLEEP_SECS
|
||||
} else {
|
||||
"0"
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the baked-in build-time env pairs as a map.
|
||||
///
|
||||
/// Internal builds (buzz-releases) bake provider/model defaults and arbitrary
|
||||
|
||||
@@ -164,7 +164,11 @@ fn reserved_keys_include_respond_to_gate() {
|
||||
|
||||
#[test]
|
||||
fn reserved_keys_include_remote_lifetime_policy() {
|
||||
for key in ["BUZZ_ACP_EXIT_AFTER_INACTIVITY", "BUZZ_ACP_NO_PRESENCE"] {
|
||||
for key in [
|
||||
"BUZZ_ACP_EXIT_AFTER_INACTIVITY",
|
||||
"BUZZ_ACP_IDLE_POOL_SLEEP",
|
||||
"BUZZ_ACP_NO_PRESENCE",
|
||||
] {
|
||||
assert!(is_reserved_env_key(key), "{key} should be reserved");
|
||||
let agent = map(&[(key, "0")]);
|
||||
assert!(merged_user_env(&BTreeMap::new(), &agent).is_empty());
|
||||
|
||||
@@ -59,6 +59,9 @@ pub(crate) const RESERVED_ENV_KEYS: &[&str] = &[
|
||||
// Remote lifetime/presence policy: user env must not disable the
|
||||
// desktop/provider-owned bounds while the saved record still promises them.
|
||||
"BUZZ_ACP_EXIT_AFTER_INACTIVITY",
|
||||
// Desktop-owned pool lifetime policy: user env must not disable or reset
|
||||
// the idle worker-reclamation window while the desktop launcher sets it.
|
||||
"BUZZ_ACP_IDLE_POOL_SLEEP",
|
||||
"BUZZ_ACP_NO_PRESENCE",
|
||||
// Readiness handoff: desktop is the ONLY readiness source. A saved or
|
||||
// ambient env var must not be able to forge setup mode (NotReady) on a
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::collections::HashMap;
|
||||
|
||||
use tauri::AppHandle;
|
||||
|
||||
use super::agent_env::build_buzz_agent_provider_defaults;
|
||||
use super::agent_env::{build_buzz_agent_provider_defaults, idle_pool_sleep_env};
|
||||
|
||||
use crate::{
|
||||
managed_agents::{
|
||||
@@ -531,6 +531,7 @@ pub fn spawn_agent_child(
|
||||
command.env("BUZZ_PRIVATE_KEY", &record.private_key_nsec);
|
||||
command.env("BUZZ_RELAY_URL", &effective_relay_url);
|
||||
command.env("BUZZ_ACP_LAZY_POOL", if lazy { "true" } else { "false" });
|
||||
command.env("BUZZ_ACP_IDLE_POOL_SLEEP", idle_pool_sleep_env(lazy));
|
||||
command.env("BUZZ_ACP_AGENT_COMMAND", &resolved_agent_command);
|
||||
command.env("BUZZ_ACP_AGENT_ARGS", agent_args.join(","));
|
||||
match &resolved_mcp_command {
|
||||
|
||||
Reference in New Issue
Block a user