mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
* [25aa5b24] Implement rate-limit Zustand store, Axios interceptor, WebSocket hook, banner component, and page-load sync (#99) (#101) * [25aa5b24] feat(rate-limits): add types, Zustand store, Axios 429 interceptor, WS hook, sync hook, and banner component - panel/src/types/rate-limits.ts: RateLimitEntry, RateLimitHitEvent, RateLimitLiftedEvent, RateLimitApiResponse - panel/src/store/rate-limit-store.ts: useRateLimitStore with Map state, hitRateLimit/liftRateLimit/syncFromApi - panel/src/lib/api/rate-limits.ts: GET /api/system/rate-limits with isMockMode guard - panel/src/lib/api/client.ts: 429 interceptor dispatches to store first, Sonner toast on retry exhaustion - panel/src/hooks/use-rate-limit-websocket.ts: RATE_LIMIT_HIT/LIFTED events + onReconnect callback - panel/src/hooks/use-rate-limit-sync.ts: mount sync + no-op with console.warn when endpoint unavailable - panel/src/components/rate-limit/rate-limit-banner.tsx: amber rows with countdown, no dismiss button - panel/src/app/(dashboard)/layout.tsx: RateLimitBanner mounted below Header - store/index.ts, hooks/index.ts: export new store and hooks * [25aa5b24] fix(rate-limit-banner): use lint-clean countdown pattern (computeSecondsLeft outside render) * [25aa5b24] fix(client): add real retry loop to 429 interceptor so Sonner toast fires on exhaustion - Increment error.config._retryCount and return api(error.config) when retryCount < RATE_LIMIT_MAX_RETRIES, actually retrying the request. - Toast fires only when retryCount >= RATE_LIMIT_MAX_RETRIES (3 attempts). - Fixes AC4: toast was dead code because without return api(error.config) every 429 saw retryCount=1, permanently below the threshold of 3. --------- Co-authored-by: Frontend Developer 1 <fe-dev-1@agents.roboco.dev> * [4112cd34] feat(rate-limit): add RateLimitError with 5-retry exponential backoff at all LLM call sites (#102) (#103) - Create roboco/services/exceptions.py with RateLimitError(provider, retry_after), HTTP_TOO_MANY_REQUESTS, MAX_RATE_LIMIT_RETRIES constants, and parse_retry_after_header() helper - extraction.py: extract _call_anthropic_with_retry() helper; retry Anthropic call 5x on 429 with exponential backoff; re-raise RateLimitError from outer except instead of swallowing it - ollama_embedder.py: 5-retry outer loop (429) wrapping existing 3-retry inner loop (ConnectError/Timeout) for all 4 call sites; two concerns kept isolated - indexes/base.py, mentor.py, validator.py: replace magic 429 literals with HTTP_TOO_MANY_REQUESTS; 5-retry loop on 429 for LLM calls - middleware.py: add rate_limit_exception_handler returning HTTP 429 with Retry-After response header - tests/unit/services/test_rate_limit_retry.py: 28 tests covering exhaustion, Retry-After header sleep, partial retries then success, ConnectError isolation Co-authored-by: Backend Developer 1 <be-dev-1@agents.roboco.dev> * [18107054] feat(rate-limit): Redis rate-limit state tracker + i_am_blocked rate_limited path (#105) (#106) - Add RateLimitStateTracker in roboco/services/gateway/rate_limit_tracker.py with activate(), clear(), is_rate_limited(), get_state(), increment_probe_failures(), reset_probe_failures() backed by redis.asyncio - Add RATE_LIMIT_HIT = "rate_limit.hit" to EventType StrEnum in events.py - Add _handle_rate_limited_parking() to Choreographer: intercepts i_am_blocked(reason='rate_limited') before block state transition, parks all active agents sharing affected provider via mark_waiting_long, publishes RATE_LIMIT_HIT event to StreamEventBus, task stays in_progress - Add get_provider_for_agent() and get_active_agent_slugs_for_provider() helper methods to AgentOrchestrator - Wire orchestrator and stream_bus into ChoreographerDeps via deps.py - Add test_rate_limit_tracker.py (basic ops, probe failures, cross-reconnection persistence, provider isolation) and test_i_am_blocked_rate_limited.py (AC3/AC4/AC5 coverage: task stays in_progress, mark_waiting_long call count equals active agent count, RATE_LIMIT_HIT event payload structure) Co-authored-by: Backend Developer 1 <be-dev-1@agents.roboco.dev> * [5501e4b4] Wire RateLimitStateTracker into live orchestrator paths — 4 CEO-identified integration gaps (#109) * [8451ca50] feat(gateway): wire RateLimitStateTracker.activate() into i_am_blocked rate-limited path and add provider-rate-limit gate to decide_spawn() (#107) - Add provider/provider_rate_limited optional fields to TriggerContext (backward-compatible defaults) - Insert rule 2 in decide_spawn(): QUEUE when trigger.provider_rate_limited is True with reason 'provider X rate-limited' - Call RateLimitStateTracker(provider).activate() in _handle_rate_limited_parking() after mark_waiting_long loop (wrapped in contextlib.suppress for Redis fault tolerance) - Extend gateway_pre_spawn_check() with optional provider param; check RateLimitStateTracker.is_rate_limited() when provider is known - Pass provider=self.get_provider_for_agent(agent_id) from orchestrator call site - Add TestProviderRateLimitGate (6 tests) to test_trigger_filter.py - Add TestRateLimitTrackerActivateOnParking (6 tests) to test_i_am_blocked_rate_limited.py - All 38 unit tests pass; ruff and mypy clean on changed files Co-authored-by: Backend Developer 1 <be-dev-1@agents.roboco.dev> * [e9cef0f0] feat(rate-limits): sweeper probe loop, CEO notification, and GET /api/system/rate-limits endpoint (AC4, AC8, AC9) (#108) - Add RATE_LIMIT_LIFTED event type to EventType enum in models/events.py - Add RateLimitStateTracker.list_rate_limited_providers() classmethod to scan Redis for all currently rate-limited providers (used by the new endpoint) - Add orchestrator._rate_limit_probe_loop(): background task started/stopped in start()/stop(), runs _sweep_rate_limit_probes() every 30s - Add orchestrator._probe_one_provider(): checks estimated_lift_at gate, calls _do_probe(); on success: tracker.clear(), resolve_wait() for all parked agents with waiting_for='rate_limit_lifted' matching the provider, publishes RATE_LIMIT_LIFTED event; on failure: increments probe_failures counter, sends CEO notification at threshold 10 (once per episode via _rate_limit_ceo_notified) - Add orchestrator._make_tracker(): injectable factory for RateLimitStateTracker - Add orchestrator._do_probe(): overridable async bool probe (default: True) - Add orchestrator._notify_rate_limit_ceo(): high-priority notification to CEO containing provider name, duration since activation, and paused agent count - Add roboco/api/routes/system.py with GET /rate-limits endpoint (AC9) - Register system_router in app.py under /api/system prefix - Add 17 unit tests in tests/unit/runtime/test_rate_limit_sweep.py covering all AC4/AC8/AC9 paths: probe success/failure, CEO threshold, endpoint schema Co-authored-by: Backend Developer 1 <be-dev-1@agents.roboco.dev> --------- Co-authored-by: Backend Developer 1 <be-dev-1@agents.roboco.dev> --------- Co-authored-by: Frontend Developer 1 <fe-dev-1@agents.roboco.dev> Co-authored-by: Backend Developer 1 <be-dev-1@agents.roboco.dev> Co-authored-by: Renn F <rennf93@users.noreply.github.com>
245 lines
9.4 KiB
Python
245 lines
9.4 KiB
Python
"""Unit tests for RateLimitStateTracker.
|
|
|
|
These tests use mock Redis clients (no real Redis server required) to
|
|
verify the state-management logic. The cross-reconnection persistence
|
|
test constructs *two* RateLimitStateTracker instances that share the
|
|
same mock Redis store, proving that state written by one instance is
|
|
visible to a fresh instance.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from roboco.services.gateway.rate_limit_tracker import RateLimitStateTracker
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_redis_mock(initial_store: dict[str, Any] | None = None) -> AsyncMock:
|
|
"""Build an async Redis mock backed by a plain dict.
|
|
|
|
The mock supports ``get``, ``set``, and ``delete`` with the same
|
|
semantics as the real redis.asyncio.Redis client.
|
|
"""
|
|
# Use the dict AS-IS (no copy) so that two mocks sharing the same
|
|
# dict object see each other's writes and deletes — this is what the
|
|
# cross-reconnection persistence tests rely on.
|
|
store: dict[str, Any] = initial_store if initial_store is not None else {}
|
|
|
|
async def _get(key: str) -> bytes | None:
|
|
val = store.get(key)
|
|
if val is None:
|
|
return None
|
|
if isinstance(val, bytes):
|
|
return val
|
|
return str(val).encode()
|
|
|
|
async def _set(key: str, value: Any) -> None:
|
|
store[key] = value
|
|
|
|
async def _delete(key: str) -> int:
|
|
return 1 if store.pop(key, None) is not None else 0
|
|
|
|
mock = AsyncMock()
|
|
mock.get = AsyncMock(side_effect=_get)
|
|
mock.set = AsyncMock(side_effect=_set)
|
|
mock.delete = AsyncMock(side_effect=_delete)
|
|
# Stash the backing store so tests can inspect raw state
|
|
mock._store = store
|
|
return mock
|
|
|
|
|
|
def _make_tracker(
|
|
provider: str = "anthropic",
|
|
redis_mock: AsyncMock | None = None,
|
|
) -> RateLimitStateTracker:
|
|
"""Build a tracker with an injected mock Redis client."""
|
|
tracker = RateLimitStateTracker(provider=provider, redis_url="redis://unused")
|
|
if redis_mock is not None:
|
|
tracker._redis = redis_mock # type: ignore[assignment]
|
|
return tracker
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: basic operations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestActivateAndRead:
|
|
async def test_is_rate_limited_false_by_default(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
assert await tracker.is_rate_limited() is False
|
|
|
|
async def test_get_state_empty_by_default(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
assert await tracker.get_state() == {}
|
|
|
|
async def test_activate_sets_rate_limited_true(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate()
|
|
assert await tracker.is_rate_limited() is True
|
|
|
|
async def test_activate_stores_retry_after(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate(retry_after=30.0)
|
|
state = await tracker.get_state()
|
|
assert state["retry_after"] == 30.0
|
|
|
|
async def test_activate_stores_affected_agents(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate(affected_agents=["be-dev-1", "be-dev-2"])
|
|
state = await tracker.get_state()
|
|
assert state["affected_agents"] == ["be-dev-1", "be-dev-2"]
|
|
|
|
async def test_activate_initialises_probe_failures_zero(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate()
|
|
state = await tracker.get_state()
|
|
assert state["probe_failures"] == 0
|
|
|
|
async def test_clear_removes_state(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate()
|
|
await tracker.clear()
|
|
assert await tracker.is_rate_limited() is False
|
|
assert await tracker.get_state() == {}
|
|
|
|
|
|
class TestProbeFailures:
|
|
async def test_increment_starts_from_zero(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate()
|
|
count = await tracker.increment_probe_failures()
|
|
assert count == 1
|
|
|
|
async def test_increment_accumulates(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate()
|
|
await tracker.increment_probe_failures()
|
|
await tracker.increment_probe_failures()
|
|
count = await tracker.increment_probe_failures()
|
|
assert count == 3
|
|
|
|
async def test_reset_sets_zero(self) -> None:
|
|
mock = _make_redis_mock()
|
|
tracker = _make_tracker(redis_mock=mock)
|
|
await tracker.activate()
|
|
await tracker.increment_probe_failures()
|
|
await tracker.increment_probe_failures()
|
|
await tracker.reset_probe_failures()
|
|
state = await tracker.get_state()
|
|
assert state["probe_failures"] == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: cross-reconnection persistence
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# AC2: "State persists across client reconnection: a test writes state via
|
|
# activate(), creates a new RateLimitStateTracker instance pointing at the
|
|
# same Redis URL, calls is_rate_limited() and get_state() and gets back the
|
|
# same values — proving state survives a process restart."
|
|
#
|
|
# We simulate this by sharing the same backing dict between two mock Redis
|
|
# clients — one injected into the first tracker and one injected into the
|
|
# second. Both clients read from and write to the same dict, so the second
|
|
# tracker "sees" everything the first wrote.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestStatePersistsAcrossReconnection:
|
|
async def test_is_rate_limited_survives_reconnection(self) -> None:
|
|
shared_store: dict[str, Any] = {}
|
|
|
|
# First "connection": write rate-limit state
|
|
mock_a = _make_redis_mock(initial_store=shared_store)
|
|
tracker_a = _make_tracker(provider="anthropic", redis_mock=mock_a)
|
|
await tracker_a.activate(retry_after=60.0, affected_agents=["be-dev-1"])
|
|
|
|
# The mock writes into shared_store directly (our _set stores raw).
|
|
# We need to seed the second mock from the same backing store.
|
|
# Because mock_a._store IS shared_store (same dict object), we only
|
|
# need to give mock_b access to the same dict.
|
|
mock_b = _make_redis_mock(initial_store=mock_a._store)
|
|
tracker_b = RateLimitStateTracker(
|
|
provider="anthropic", redis_url="redis://unused"
|
|
)
|
|
tracker_b._redis = mock_b # type: ignore[assignment]
|
|
|
|
assert await tracker_b.is_rate_limited() is True
|
|
|
|
async def test_get_state_survives_reconnection(self) -> None:
|
|
shared_store: dict[str, Any] = {}
|
|
|
|
mock_a = _make_redis_mock(initial_store=shared_store)
|
|
tracker_a = _make_tracker(provider="anthropic", redis_mock=mock_a)
|
|
await tracker_a.activate(retry_after=45.0, affected_agents=["be-dev-2"])
|
|
|
|
mock_b = _make_redis_mock(initial_store=mock_a._store)
|
|
tracker_b = RateLimitStateTracker(
|
|
provider="anthropic", redis_url="redis://unused"
|
|
)
|
|
tracker_b._redis = mock_b # type: ignore[assignment]
|
|
|
|
state = await tracker_b.get_state()
|
|
assert state["rate_limited"] is True
|
|
assert state["retry_after"] == 45.0
|
|
assert state["affected_agents"] == ["be-dev-2"]
|
|
|
|
async def test_clear_via_first_instance_visible_to_second(self) -> None:
|
|
shared_store: dict[str, Any] = {}
|
|
|
|
mock_a = _make_redis_mock(initial_store=shared_store)
|
|
tracker_a = _make_tracker(provider="anthropic", redis_mock=mock_a)
|
|
await tracker_a.activate()
|
|
|
|
# Second instance points at the same store
|
|
mock_b = _make_redis_mock(initial_store=mock_a._store)
|
|
tracker_b = RateLimitStateTracker(
|
|
provider="anthropic", redis_url="redis://unused"
|
|
)
|
|
tracker_b._redis = mock_b # type: ignore[assignment]
|
|
|
|
# Write clear via tracker_a
|
|
await tracker_a.clear()
|
|
|
|
# tracker_b observes the cleared state
|
|
assert await tracker_b.is_rate_limited() is False
|
|
assert await tracker_b.get_state() == {}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: different providers are isolated
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestProviderIsolation:
|
|
async def test_activating_one_provider_does_not_affect_another(self) -> None:
|
|
store: dict[str, Any] = {}
|
|
mock_a = _make_redis_mock(initial_store=store)
|
|
mock_b = _make_redis_mock(initial_store=store)
|
|
|
|
tracker_anthropic = _make_tracker(provider="anthropic", redis_mock=mock_a)
|
|
tracker_ollama = _make_tracker(provider="ollama_cloud", redis_mock=mock_b)
|
|
|
|
await tracker_anthropic.activate()
|
|
assert await tracker_anthropic.is_rate_limited() is True
|
|
assert await tracker_ollama.is_rate_limited() is False
|