[F045] rate-limit: loud activate-failure log + in-memory orphan-probe fallback

The in-verb i_am_blocked(rate_limited) path wrapped RateLimitStateTracker.activate
in a bare contextlib.suppress. A silent activate failure stranded the fleet:
agents were parked in _waiting_records but the provider never entered the tracker,
so the tracker-driven _sweep_rate_limit_probes never probed it and no
_on_probe_success ever resumed them — parked agents stuck in WAITING_LONG.

Fix: (1) replace the bare suppress with a try/except that logs an error event
naming the provider + affected agents; (2) in _sweep_rate_limit_probes, after
probing the tracker-listed set, scan _waiting_records for any rate_limit_lifted
provider the loop did NOT cover and probe it via the time-expiry fallback (empty
state -> probe now) so _on_probe_success resumes the parked agents. The fallback
reads only local memory, so it still resumes when Redis was down at park time
(list_rate_limited_providers failure now falls through to the orphan scan instead
of returning early).
This commit is contained in:
Renn F
2026-06-28 11:36:47 +02:00
parent 05a5852bb5
commit a6de1330aa
4 changed files with 172 additions and 2 deletions
@@ -20,6 +20,7 @@ from uuid import uuid4
from roboco.models.events import EventType
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
from structlog.testing import capture_logs
# ---------------------------------------------------------------------------
# Helpers
@@ -505,3 +506,31 @@ class TestRateLimitTrackerActivateOnParking:
assert env.error is None
assert env.status == "in_progress"
async def test_activate_failure_is_logged_not_silent(self) -> None:
"""F045: an activate() failure must be logged loudly, not bare-suppressed.
The probe-resume loop is tracker-driven, so a silent activate failure
strands every parked agent in WAITING_LONG with no probe ever running.
A loud error log makes the stranded-fleet condition visible to
operators (and pairs with the orchestrator's in-memory fallback sweep).
"""
agent_id = uuid4()
task_id = uuid4()
orch = _make_orchestrator(active_agents=["be-dev-1"], provider=_PROVIDER)
deps = _make_deps(agent_id, task_id, orchestrator=orch)
c = Choreographer(deps)
mock_tracker = AsyncMock()
mock_tracker.activate = AsyncMock(side_effect=RuntimeError("redis down"))
mock_tracker_cls = MagicMock(return_value=mock_tracker)
with patch(_TRACKER_PATCH, mock_tracker_cls), capture_logs() as logs:
env = await c.i_am_blocked(agent_id, task_id, "rate_limited")
assert env.error is None
assert any(
"activate" in str(e.get("event", "")).lower()
and e.get("log_level") == "error"
for e in logs
), f"expected an error log about activate failure; got {logs!r}"
@@ -414,6 +414,93 @@ class TestCEONotificationThreshold:
notify_mock.assert_awaited_once()
# ---------------------------------------------------------------------------
# Tests: orphan-provider fallback (F045)
# ---------------------------------------------------------------------------
class TestOrphanProviderFallback:
"""F045: an activate() failure in the in-verb ``i_am_blocked(rate_limited)``
path leaves agents parked in ``_waiting_records`` but the provider never
makes it into the tracker — so the tracker-driven loop never probes it and
the parked agents strand in WAITING_LONG forever. The sweep must scan the
in-memory records for any ``rate_limit_lifted`` provider the tracker-listed
set did NOT cover and probe it via the time-expiry fallback so
``_on_probe_success`` can resume them.
"""
async def test_orphan_parked_agent_resumed_when_tracker_lacks_provider(
self,
) -> None:
orch = _make_orchestrator()
provider = "anthropic"
agent = "be-dev-1"
orch._waiting_records = {agent: _waiting_record(agent, provider)}
tracker_mock = _make_tracker_mock()
resolve_mock = AsyncMock(return_value=None)
with (
patch.object(orch, "resolve_wait", new=resolve_mock),
patch.object(orch, "_make_tracker", return_value=tracker_mock),
patch.object(orch, "_do_probe", new=AsyncMock(return_value=True)),
patch.object(
RateLimitStateTracker,
"list_rate_limited_providers",
new=AsyncMock(return_value=[]),
),
patch("roboco.events.get_event_bus") as mock_bus_fn,
):
bus_mock = AsyncMock()
bus_mock.publish = AsyncMock()
mock_bus_fn.return_value = bus_mock
await orch._sweep_rate_limit_probes()
# The orphan provider was probed and the parked agent resumed.
assert resolve_mock.await_count == 1
assert resolve_mock.call_args.args[0] == agent
async def test_orphan_skipped_when_tracker_already_covers_provider(
self,
) -> None:
"""A provider the tracker lists must NOT be double-probed via the fallback."""
orch = _make_orchestrator()
provider = "anthropic"
agent = "be-dev-1"
orch._waiting_records = {agent: _waiting_record(agent, provider)}
tracker_mock = _make_tracker_mock()
resolve_mock = AsyncMock(return_value=None)
state = _make_active_state(provider, retry_after=None)
probe_calls: list[str] = []
async def fake_do_probe(p: str) -> bool:
probe_calls.append(p)
return True
with (
patch.object(orch, "resolve_wait", new=resolve_mock),
patch.object(orch, "_make_tracker", return_value=tracker_mock),
patch.object(orch, "_do_probe", new=fake_do_probe),
patch.object(
RateLimitStateTracker,
"list_rate_limited_providers",
new=AsyncMock(return_value=[(provider, state)]),
),
patch("roboco.events.get_event_bus") as mock_bus_fn,
):
bus_mock = AsyncMock()
bus_mock.publish = AsyncMock()
mock_bus_fn.return_value = bus_mock
await orch._sweep_rate_limit_probes()
# Probed exactly once (via the tracker-listed path), not twice.
assert probe_calls == [provider]
# ---------------------------------------------------------------------------
# Tests: list_rate_limited_providers
# ---------------------------------------------------------------------------