mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[067ce5d1] fix(runtime): gate all spawns while provider is parked
Generalize the GROK-only _grok_spawn_parked guard to _provider_spawn_parked. spawn_agent now consults the RateLimitStateTracker for every provider, so Anthropic session/overload parking suppresses container launches instead of letting the dispatcher re-spawn every tick. Fail-open on tracker errors. - Rename _grok_spawn_parked -> _provider_spawn_parked (any provider) - Update spawn_agent log + gate - Update grok-rate-limit tests to cover general provider behavior
This commit is contained in:
@@ -1869,19 +1869,21 @@ class AgentOrchestrator:
|
||||
config, instance, agent_settings_path = await self._prepare_agent_spawn(
|
||||
agent_id, task_id, model, git_context
|
||||
)
|
||||
# Grok 429 loop-breaker (B4): while the xAI provider is parked
|
||||
# rate-limited, do NOT launch another grok container — the dispatcher
|
||||
# would otherwise re-spawn the same task every tick, 429, and burn
|
||||
# cost. The probe-resume loop clears the park after the retry window and
|
||||
# the next tick spawns normally. Grok-only; the Claude path is untouched.
|
||||
# Provider-parking loop-breaker: while this agent's provider is parked
|
||||
# (rate-limited or overloaded), do NOT launch another container — the
|
||||
# dispatcher would otherwise re-spawn the same task every tick, hit the
|
||||
# limit again, and burn cost. The probe-resume loop clears the park when
|
||||
# the provider recovers and the next tick spawns normally. Covers both
|
||||
# the GROK 429 path and the Claude session/overload paths.
|
||||
# Fail-open: a tracker read error must never block spawning.
|
||||
if await self._grok_spawn_parked(config.provider_type):
|
||||
if await self._provider_spawn_parked(config.provider_type):
|
||||
self._mark_task_handled(task_id)
|
||||
instance.state = AgentState.OFFLINE
|
||||
logger.info(
|
||||
"Grok spawn skipped: provider rate-limited (parked)",
|
||||
"Spawn skipped: provider rate-limited (parked)",
|
||||
agent_id=agent_id,
|
||||
task_id=task_id,
|
||||
provider=config.provider_type,
|
||||
)
|
||||
return instance
|
||||
# Record the task as handled so later dispatchers in the same
|
||||
@@ -5837,23 +5839,23 @@ Start by:
|
||||
|
||||
return RateLimitStateTracker(provider)
|
||||
|
||||
async def _grok_spawn_parked(self, provider_type: str | None) -> bool:
|
||||
"""True when *provider_type* is GROK and the provider is parked rate-limited.
|
||||
async def _provider_spawn_parked(self, provider_type: str | None) -> bool:
|
||||
"""True when *provider_type*'s provider is parked (rate-limited/overloaded).
|
||||
|
||||
The grok 429 loop-breaker consults this before launching a grok
|
||||
container. Grok-only and fail-open: any error reading the tracker
|
||||
returns False so a Redis hiccup can never block spawning.
|
||||
The spawn loop-breaker consults this before launching any container.
|
||||
Fail-open: any error reading the tracker returns False so a Redis hiccup
|
||||
can never block spawning.
|
||||
"""
|
||||
from roboco.models.base import ModelProvider
|
||||
|
||||
if provider_type != ModelProvider.GROK.value:
|
||||
if provider_type is None:
|
||||
return False
|
||||
try:
|
||||
tracker = self._make_tracker(ModelProvider.GROK.value)
|
||||
tracker = self._make_tracker(provider_type)
|
||||
return bool(await tracker.is_rate_limited())
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"grok rate-limit check failed; allowing spawn", error=str(exc)
|
||||
"provider rate-limit check failed; allowing spawn",
|
||||
provider=provider_type,
|
||||
error=str(exc),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ A one-shot grok run that hits an xAI 429 exits 75; the orchestrator parks the
|
||||
grok provider rate-limited instead of crash-retrying, and the spawn guard
|
||||
suppresses re-spawns until the probe-resume loop clears the park. These tests
|
||||
exercise the decision points deterministically (tracker + finalize stubbed).
|
||||
The same spawn guard now protects every provider (not just GROK), so the
|
||||
Claude session/overload paths get the same loop-break for free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -61,44 +63,43 @@ def test_is_grok_rate_limit_exit() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grok_spawn_parked_true_when_limited(
|
||||
async def test_provider_spawn_parked_true_when_limited(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True))
|
||||
assert await orch._grok_spawn_parked("grok") is True
|
||||
assert await orch._provider_spawn_parked("grok") is True
|
||||
assert await orch._provider_spawn_parked("anthropic") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grok_spawn_parked_false_when_not_limited(
|
||||
async def test_provider_spawn_parked_false_when_not_limited(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False))
|
||||
assert await orch._grok_spawn_parked("grok") is False
|
||||
assert await orch._provider_spawn_parked("grok") is False
|
||||
assert await orch._provider_spawn_parked("anthropic") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grok_spawn_parked_false_for_non_grok(
|
||||
async def test_provider_spawn_parked_false_when_provider_unknown() -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
assert await orch._provider_spawn_parked(None) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_spawn_parked_fails_open(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
# Non-grok never consults the tracker (a tracker call would error here).
|
||||
monkeypatch.setattr(
|
||||
orch, "_make_tracker", lambda _p: (_ for _ in ()).throw(AssertionError)
|
||||
)
|
||||
assert await orch._grok_spawn_parked("anthropic") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grok_spawn_parked_fails_open(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# A tracker error must never block spawning (fail-open -> False).
|
||||
def _boom(_p: str) -> object:
|
||||
raise RuntimeError("redis down")
|
||||
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
monkeypatch.setattr(orch, "_make_tracker", _boom)
|
||||
assert await orch._grok_spawn_parked("grok") is False
|
||||
assert await orch._provider_spawn_parked("grok") is False
|
||||
assert await orch._provider_spawn_parked("anthropic") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user