[067ce5d1] fix(runtime): gate all spawns while provider is parked

Generalize the GROK-only _grok_spawn_parked guard to _provider_spawn_parked.
spawn_agent now consults the RateLimitStateTracker for every provider, so
Anthropic session/overload parking suppresses container launches instead of
letting the dispatcher re-spawn every tick. Fail-open on tracker errors.

- Rename _grok_spawn_parked -> _provider_spawn_parked (any provider)
- Update spawn_agent log + gate
- Update grok-rate-limit tests to cover general provider behavior
This commit is contained in:
Renn F
2026-06-25 03:34:40 +02:00
parent 75788f519c
commit 88ad03c8cb
2 changed files with 36 additions and 33 deletions
+19 -17
View File
@@ -1869,19 +1869,21 @@ class AgentOrchestrator:
config, instance, agent_settings_path = await self._prepare_agent_spawn(
agent_id, task_id, model, git_context
)
# Grok 429 loop-breaker (B4): while the xAI provider is parked
# rate-limited, do NOT launch another grok container — the dispatcher
# would otherwise re-spawn the same task every tick, 429, and burn
# cost. The probe-resume loop clears the park after the retry window and
# the next tick spawns normally. Grok-only; the Claude path is untouched.
# Provider-parking loop-breaker: while this agent's provider is parked
# (rate-limited or overloaded), do NOT launch another container — the
# dispatcher would otherwise re-spawn the same task every tick, hit the
# limit again, and burn cost. The probe-resume loop clears the park when
# the provider recovers and the next tick spawns normally. Covers both
# the GROK 429 path and the Claude session/overload paths.
# Fail-open: a tracker read error must never block spawning.
if await self._grok_spawn_parked(config.provider_type):
if await self._provider_spawn_parked(config.provider_type):
self._mark_task_handled(task_id)
instance.state = AgentState.OFFLINE
logger.info(
"Grok spawn skipped: provider rate-limited (parked)",
"Spawn skipped: provider rate-limited (parked)",
agent_id=agent_id,
task_id=task_id,
provider=config.provider_type,
)
return instance
# Record the task as handled so later dispatchers in the same
@@ -5837,23 +5839,23 @@ Start by:
return RateLimitStateTracker(provider)
async def _grok_spawn_parked(self, provider_type: str | None) -> bool:
"""True when *provider_type* is GROK and the provider is parked rate-limited.
async def _provider_spawn_parked(self, provider_type: str | None) -> bool:
"""True when *provider_type*'s provider is parked (rate-limited/overloaded).
The grok 429 loop-breaker consults this before launching a grok
container. Grok-only and fail-open: any error reading the tracker
returns False so a Redis hiccup can never block spawning.
The spawn loop-breaker consults this before launching any container.
Fail-open: any error reading the tracker returns False so a Redis hiccup
can never block spawning.
"""
from roboco.models.base import ModelProvider
if provider_type != ModelProvider.GROK.value:
if provider_type is None:
return False
try:
tracker = self._make_tracker(ModelProvider.GROK.value)
tracker = self._make_tracker(provider_type)
return bool(await tracker.is_rate_limited())
except Exception as exc:
logger.warning(
"grok rate-limit check failed; allowing spawn", error=str(exc)
"provider rate-limit check failed; allowing spawn",
provider=provider_type,
error=str(exc),
)
return False
+17 -16
View File
@@ -4,6 +4,8 @@ A one-shot grok run that hits an xAI 429 exits 75; the orchestrator parks the
grok provider rate-limited instead of crash-retrying, and the spawn guard
suppresses re-spawns until the probe-resume loop clears the park. These tests
exercise the decision points deterministically (tracker + finalize stubbed).
The same spawn guard now protects every provider (not just GROK), so the
Claude session/overload paths get the same loop-break for free.
"""
from __future__ import annotations
@@ -61,44 +63,43 @@ def test_is_grok_rate_limit_exit() -> None:
@pytest.mark.asyncio
async def test_grok_spawn_parked_true_when_limited(
async def test_provider_spawn_parked_true_when_limited(
monkeypatch: pytest.MonkeyPatch,
) -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True))
assert await orch._grok_spawn_parked("grok") is True
assert await orch._provider_spawn_parked("grok") is True
assert await orch._provider_spawn_parked("anthropic") is True
@pytest.mark.asyncio
async def test_grok_spawn_parked_false_when_not_limited(
async def test_provider_spawn_parked_false_when_not_limited(
monkeypatch: pytest.MonkeyPatch,
) -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False))
assert await orch._grok_spawn_parked("grok") is False
assert await orch._provider_spawn_parked("grok") is False
assert await orch._provider_spawn_parked("anthropic") is False
@pytest.mark.asyncio
async def test_grok_spawn_parked_false_for_non_grok(
async def test_provider_spawn_parked_false_when_provider_unknown() -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
assert await orch._provider_spawn_parked(None) is False
@pytest.mark.asyncio
async def test_provider_spawn_parked_fails_open(
monkeypatch: pytest.MonkeyPatch,
) -> None:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
# Non-grok never consults the tracker (a tracker call would error here).
monkeypatch.setattr(
orch, "_make_tracker", lambda _p: (_ for _ in ()).throw(AssertionError)
)
assert await orch._grok_spawn_parked("anthropic") is False
@pytest.mark.asyncio
async def test_grok_spawn_parked_fails_open(monkeypatch: pytest.MonkeyPatch) -> None:
# A tracker error must never block spawning (fail-open -> False).
def _boom(_p: str) -> object:
raise RuntimeError("redis down")
orch = AgentOrchestrator.__new__(AgentOrchestrator)
monkeypatch.setattr(orch, "_make_tracker", _boom)
assert await orch._grok_spawn_parked("grok") is False
assert await orch._provider_spawn_parked("grok") is False
assert await orch._provider_spawn_parked("anthropic") is False
@pytest.mark.asyncio