mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F045] rate-limit: loud activate-failure log + in-memory orphan-probe fallback
The in-verb i_am_blocked(rate_limited) path wrapped RateLimitStateTracker.activate in a bare contextlib.suppress. A silent activate failure stranded the fleet: agents were parked in _waiting_records but the provider never entered the tracker, so the tracker-driven _sweep_rate_limit_probes never probed it and no _on_probe_success ever resumed them — parked agents stuck in WAITING_LONG. Fix: (1) replace the bare suppress with a try/except that logs an error event naming the provider + affected agents; (2) in _sweep_rate_limit_probes, after probing the tracker-listed set, scan _waiting_records for any rate_limit_lifted provider the loop did NOT cover and probe it via the time-expiry fallback (empty state -> probe now) so _on_probe_success resumes the parked agents. The fallback reads only local memory, so it still resumes when Redis was down at park time (list_rate_limited_providers failure now falls through to the orphan scan instead of returning early).
This commit is contained in:
@@ -20,6 +20,7 @@ from uuid import uuid4
|
||||
|
||||
from roboco.models.events import EventType
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
from structlog.testing import capture_logs
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -505,3 +506,31 @@ class TestRateLimitTrackerActivateOnParking:
|
||||
|
||||
assert env.error is None
|
||||
assert env.status == "in_progress"
|
||||
|
||||
async def test_activate_failure_is_logged_not_silent(self) -> None:
|
||||
"""F045: an activate() failure must be logged loudly, not bare-suppressed.
|
||||
|
||||
The probe-resume loop is tracker-driven, so a silent activate failure
|
||||
strands every parked agent in WAITING_LONG with no probe ever running.
|
||||
A loud error log makes the stranded-fleet condition visible to
|
||||
operators (and pairs with the orchestrator's in-memory fallback sweep).
|
||||
"""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
orch = _make_orchestrator(active_agents=["be-dev-1"], provider=_PROVIDER)
|
||||
deps = _make_deps(agent_id, task_id, orchestrator=orch)
|
||||
c = Choreographer(deps)
|
||||
|
||||
mock_tracker = AsyncMock()
|
||||
mock_tracker.activate = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
mock_tracker_cls = MagicMock(return_value=mock_tracker)
|
||||
|
||||
with patch(_TRACKER_PATCH, mock_tracker_cls), capture_logs() as logs:
|
||||
env = await c.i_am_blocked(agent_id, task_id, "rate_limited")
|
||||
|
||||
assert env.error is None
|
||||
assert any(
|
||||
"activate" in str(e.get("event", "")).lower()
|
||||
and e.get("log_level") == "error"
|
||||
for e in logs
|
||||
), f"expected an error log about activate failure; got {logs!r}"
|
||||
|
||||
@@ -414,6 +414,93 @@ class TestCEONotificationThreshold:
|
||||
notify_mock.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: orphan-provider fallback (F045)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOrphanProviderFallback:
|
||||
"""F045: an activate() failure in the in-verb ``i_am_blocked(rate_limited)``
|
||||
path leaves agents parked in ``_waiting_records`` but the provider never
|
||||
makes it into the tracker — so the tracker-driven loop never probes it and
|
||||
the parked agents strand in WAITING_LONG forever. The sweep must scan the
|
||||
in-memory records for any ``rate_limit_lifted`` provider the tracker-listed
|
||||
set did NOT cover and probe it via the time-expiry fallback so
|
||||
``_on_probe_success`` can resume them.
|
||||
"""
|
||||
|
||||
async def test_orphan_parked_agent_resumed_when_tracker_lacks_provider(
|
||||
self,
|
||||
) -> None:
|
||||
orch = _make_orchestrator()
|
||||
provider = "anthropic"
|
||||
agent = "be-dev-1"
|
||||
orch._waiting_records = {agent: _waiting_record(agent, provider)}
|
||||
|
||||
tracker_mock = _make_tracker_mock()
|
||||
resolve_mock = AsyncMock(return_value=None)
|
||||
|
||||
with (
|
||||
patch.object(orch, "resolve_wait", new=resolve_mock),
|
||||
patch.object(orch, "_make_tracker", return_value=tracker_mock),
|
||||
patch.object(orch, "_do_probe", new=AsyncMock(return_value=True)),
|
||||
patch.object(
|
||||
RateLimitStateTracker,
|
||||
"list_rate_limited_providers",
|
||||
new=AsyncMock(return_value=[]),
|
||||
),
|
||||
patch("roboco.events.get_event_bus") as mock_bus_fn,
|
||||
):
|
||||
bus_mock = AsyncMock()
|
||||
bus_mock.publish = AsyncMock()
|
||||
mock_bus_fn.return_value = bus_mock
|
||||
|
||||
await orch._sweep_rate_limit_probes()
|
||||
|
||||
# The orphan provider was probed and the parked agent resumed.
|
||||
assert resolve_mock.await_count == 1
|
||||
assert resolve_mock.call_args.args[0] == agent
|
||||
|
||||
async def test_orphan_skipped_when_tracker_already_covers_provider(
|
||||
self,
|
||||
) -> None:
|
||||
"""A provider the tracker lists must NOT be double-probed via the fallback."""
|
||||
orch = _make_orchestrator()
|
||||
provider = "anthropic"
|
||||
agent = "be-dev-1"
|
||||
orch._waiting_records = {agent: _waiting_record(agent, provider)}
|
||||
|
||||
tracker_mock = _make_tracker_mock()
|
||||
resolve_mock = AsyncMock(return_value=None)
|
||||
state = _make_active_state(provider, retry_after=None)
|
||||
|
||||
probe_calls: list[str] = []
|
||||
|
||||
async def fake_do_probe(p: str) -> bool:
|
||||
probe_calls.append(p)
|
||||
return True
|
||||
|
||||
with (
|
||||
patch.object(orch, "resolve_wait", new=resolve_mock),
|
||||
patch.object(orch, "_make_tracker", return_value=tracker_mock),
|
||||
patch.object(orch, "_do_probe", new=fake_do_probe),
|
||||
patch.object(
|
||||
RateLimitStateTracker,
|
||||
"list_rate_limited_providers",
|
||||
new=AsyncMock(return_value=[(provider, state)]),
|
||||
),
|
||||
patch("roboco.events.get_event_bus") as mock_bus_fn,
|
||||
):
|
||||
bus_mock = AsyncMock()
|
||||
bus_mock.publish = AsyncMock()
|
||||
mock_bus_fn.return_value = bus_mock
|
||||
|
||||
await orch._sweep_rate_limit_probes()
|
||||
|
||||
# Probed exactly once (via the tracker-listed path), not twice.
|
||||
assert probe_calls == [provider]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: list_rate_limited_providers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user