[067ce5d1] fix(runtime): gate all spawns while provider is parked

Generalize the GROK-only _grok_spawn_parked guard to _provider_spawn_parked.
spawn_agent now consults the RateLimitStateTracker for every provider, so
Anthropic session/overload parking suppresses container launches instead of
letting the dispatcher re-spawn every tick. Fail-open on tracker errors.

- Rename _grok_spawn_parked -> _provider_spawn_parked (any provider)
- Update spawn_agent log + gate
- Update grok-rate-limit tests to cover general provider behavior
This commit is contained in:
Renn F
2026-06-25 03:34:40 +02:00
parent 75788f519c
commit 88ad03c8cb
2 changed files with 36 additions and 33 deletions
+19 -17
View File
@@ -1869,19 +1869,21 @@ class AgentOrchestrator:
config, instance, agent_settings_path = await self._prepare_agent_spawn( config, instance, agent_settings_path = await self._prepare_agent_spawn(
agent_id, task_id, model, git_context agent_id, task_id, model, git_context
) )
# Grok 429 loop-breaker (B4): while the xAI provider is parked # Provider-parking loop-breaker: while this agent's provider is parked
# rate-limited, do NOT launch another grok container — the dispatcher # (rate-limited or overloaded), do NOT launch another container — the
# would otherwise re-spawn the same task every tick, 429, and burn # dispatcher would otherwise re-spawn the same task every tick, hit the
# cost. The probe-resume loop clears the park after the retry window and # limit again, and burn cost. The probe-resume loop clears the park when
# the next tick spawns normally. Grok-only; the Claude path is untouched. # the provider recovers and the next tick spawns normally. Covers both
# the GROK 429 path and the Claude session/overload paths.
# Fail-open: a tracker read error must never block spawning. # Fail-open: a tracker read error must never block spawning.
if await self._grok_spawn_parked(config.provider_type): if await self._provider_spawn_parked(config.provider_type):
self._mark_task_handled(task_id) self._mark_task_handled(task_id)
instance.state = AgentState.OFFLINE instance.state = AgentState.OFFLINE
logger.info( logger.info(
"Grok spawn skipped: provider rate-limited (parked)", "Spawn skipped: provider rate-limited (parked)",
agent_id=agent_id, agent_id=agent_id,
task_id=task_id, task_id=task_id,
provider=config.provider_type,
) )
return instance return instance
# Record the task as handled so later dispatchers in the same # Record the task as handled so later dispatchers in the same
@@ -5837,23 +5839,23 @@ Start by:
return RateLimitStateTracker(provider) return RateLimitStateTracker(provider)
async def _grok_spawn_parked(self, provider_type: str | None) -> bool: async def _provider_spawn_parked(self, provider_type: str | None) -> bool:
"""True when *provider_type* is GROK and the provider is parked rate-limited. """True when *provider_type*'s provider is parked (rate-limited/overloaded).
The grok 429 loop-breaker consults this before launching a grok The spawn loop-breaker consults this before launching any container.
container. Grok-only and fail-open: any error reading the tracker Fail-open: any error reading the tracker returns False so a Redis hiccup
returns False so a Redis hiccup can never block spawning. can never block spawning.
""" """
from roboco.models.base import ModelProvider if provider_type is None:
if provider_type != ModelProvider.GROK.value:
return False return False
try: try:
tracker = self._make_tracker(ModelProvider.GROK.value) tracker = self._make_tracker(provider_type)
return bool(await tracker.is_rate_limited()) return bool(await tracker.is_rate_limited())
except Exception as exc: except Exception as exc:
logger.warning( logger.warning(
"grok rate-limit check failed; allowing spawn", error=str(exc) "provider rate-limit check failed; allowing spawn",
provider=provider_type,
error=str(exc),
) )
return False return False
+17 -16
View File
@@ -4,6 +4,8 @@ A one-shot grok run that hits an xAI 429 exits 75; the orchestrator parks the
grok provider rate-limited instead of crash-retrying, and the spawn guard grok provider rate-limited instead of crash-retrying, and the spawn guard
suppresses re-spawns until the probe-resume loop clears the park. These tests suppresses re-spawns until the probe-resume loop clears the park. These tests
exercise the decision points deterministically (tracker + finalize stubbed). exercise the decision points deterministically (tracker + finalize stubbed).
The same spawn guard now protects every provider (not just GROK), so the
Claude session/overload paths get the same loop-break for free.
""" """
from __future__ import annotations from __future__ import annotations
@@ -61,44 +63,43 @@ def test_is_grok_rate_limit_exit() -> None:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_grok_spawn_parked_true_when_limited( async def test_provider_spawn_parked_true_when_limited(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator) orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True)) monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True))
assert await orch._grok_spawn_parked("grok") is True assert await orch._provider_spawn_parked("grok") is True
assert await orch._provider_spawn_parked("anthropic") is True
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_grok_spawn_parked_false_when_not_limited( async def test_provider_spawn_parked_false_when_not_limited(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator) orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False)) monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False))
assert await orch._grok_spawn_parked("grok") is False assert await orch._provider_spawn_parked("grok") is False
assert await orch._provider_spawn_parked("anthropic") is False
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_grok_spawn_parked_false_for_non_grok( async def test_provider_spawn_parked_false_when_provider_unknown() -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
assert await orch._provider_spawn_parked(None) is False
@pytest.mark.asyncio
async def test_provider_spawn_parked_fails_open(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
# Non-grok never consults the tracker (a tracker call would error here).
monkeypatch.setattr(
orch, "_make_tracker", lambda _p: (_ for _ in ()).throw(AssertionError)
)
assert await orch._grok_spawn_parked("anthropic") is False
@pytest.mark.asyncio
async def test_grok_spawn_parked_fails_open(monkeypatch: pytest.MonkeyPatch) -> None:
# A tracker error must never block spawning (fail-open -> False). # A tracker error must never block spawning (fail-open -> False).
def _boom(_p: str) -> object: def _boom(_p: str) -> object:
raise RuntimeError("redis down") raise RuntimeError("redis down")
orch = AgentOrchestrator.__new__(AgentOrchestrator) orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(orch, "_make_tracker", _boom) monkeypatch.setattr(orch, "_make_tracker", _boom)
assert await orch._grok_spawn_parked("grok") is False assert await orch._provider_spawn_parked("grok") is False
assert await orch._provider_spawn_parked("anthropic") is False
@pytest.mark.asyncio @pytest.mark.asyncio