mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F045] rate-limit: loud activate-failure log + in-memory orphan-probe fallback
The in-verb i_am_blocked(rate_limited) path wrapped RateLimitStateTracker.activate in a bare contextlib.suppress. A silent activate failure stranded the fleet: agents were parked in _waiting_records but the provider never entered the tracker, so the tracker-driven _sweep_rate_limit_probes never probed it and no _on_probe_success ever resumed them — parked agents stuck in WAITING_LONG. Fix: (1) replace the bare suppress with a try/except that logs an error event naming the provider + affected agents; (2) in _sweep_rate_limit_probes, after probing the tracker-listed set, scan _waiting_records for any rate_limit_lifted provider the loop did NOT cover and probe it via the time-expiry fallback (empty state -> probe now) so _on_probe_success resumes the parked agents. The fallback reads only local memory, so it still resumes when Redis was down at park time (list_rate_limited_providers failure now falls through to the orphan scan instead of returning early).
This commit is contained in:
@@ -20,6 +20,7 @@ from uuid import uuid4
|
||||
|
||||
from roboco.models.events import EventType
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
from structlog.testing import capture_logs
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -505,3 +506,31 @@ class TestRateLimitTrackerActivateOnParking:
|
||||
|
||||
assert env.error is None
|
||||
assert env.status == "in_progress"
|
||||
|
||||
async def test_activate_failure_is_logged_not_silent(self) -> None:
|
||||
"""F045: an activate() failure must be logged loudly, not bare-suppressed.
|
||||
|
||||
The probe-resume loop is tracker-driven, so a silent activate failure
|
||||
strands every parked agent in WAITING_LONG with no probe ever running.
|
||||
A loud error log makes the stranded-fleet condition visible to
|
||||
operators (and pairs with the orchestrator's in-memory fallback sweep).
|
||||
"""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
orch = _make_orchestrator(active_agents=["be-dev-1"], provider=_PROVIDER)
|
||||
deps = _make_deps(agent_id, task_id, orchestrator=orch)
|
||||
c = Choreographer(deps)
|
||||
|
||||
mock_tracker = AsyncMock()
|
||||
mock_tracker.activate = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
mock_tracker_cls = MagicMock(return_value=mock_tracker)
|
||||
|
||||
with patch(_TRACKER_PATCH, mock_tracker_cls), capture_logs() as logs:
|
||||
env = await c.i_am_blocked(agent_id, task_id, "rate_limited")
|
||||
|
||||
assert env.error is None
|
||||
assert any(
|
||||
"activate" in str(e.get("event", "")).lower()
|
||||
and e.get("log_level") == "error"
|
||||
for e in logs
|
||||
), f"expected an error log about activate failure; got {logs!r}"
|
||||
|
||||
Reference in New Issue
Block a user