mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[067ce5d1] fix(runtime): gate all spawns while provider is parked
Generalize the GROK-only _grok_spawn_parked guard to _provider_spawn_parked. spawn_agent now consults the RateLimitStateTracker for every provider, so Anthropic session/overload parking suppresses container launches instead of letting the dispatcher re-spawn every tick. Fail-open on tracker errors. - Rename _grok_spawn_parked -> _provider_spawn_parked (any provider) - Update spawn_agent log + gate - Update grok-rate-limit tests to cover general provider behavior
This commit is contained in:
@@ -1869,19 +1869,21 @@ class AgentOrchestrator:
|
|||||||
config, instance, agent_settings_path = await self._prepare_agent_spawn(
|
config, instance, agent_settings_path = await self._prepare_agent_spawn(
|
||||||
agent_id, task_id, model, git_context
|
agent_id, task_id, model, git_context
|
||||||
)
|
)
|
||||||
# Grok 429 loop-breaker (B4): while the xAI provider is parked
|
# Provider-parking loop-breaker: while this agent's provider is parked
|
||||||
# rate-limited, do NOT launch another grok container — the dispatcher
|
# (rate-limited or overloaded), do NOT launch another container — the
|
||||||
# would otherwise re-spawn the same task every tick, 429, and burn
|
# dispatcher would otherwise re-spawn the same task every tick, hit the
|
||||||
# cost. The probe-resume loop clears the park after the retry window and
|
# limit again, and burn cost. The probe-resume loop clears the park when
|
||||||
# the next tick spawns normally. Grok-only; the Claude path is untouched.
|
# the provider recovers and the next tick spawns normally. Covers both
|
||||||
|
# the GROK 429 path and the Claude session/overload paths.
|
||||||
# Fail-open: a tracker read error must never block spawning.
|
# Fail-open: a tracker read error must never block spawning.
|
||||||
if await self._grok_spawn_parked(config.provider_type):
|
if await self._provider_spawn_parked(config.provider_type):
|
||||||
self._mark_task_handled(task_id)
|
self._mark_task_handled(task_id)
|
||||||
instance.state = AgentState.OFFLINE
|
instance.state = AgentState.OFFLINE
|
||||||
logger.info(
|
logger.info(
|
||||||
"Grok spawn skipped: provider rate-limited (parked)",
|
"Spawn skipped: provider rate-limited (parked)",
|
||||||
agent_id=agent_id,
|
agent_id=agent_id,
|
||||||
task_id=task_id,
|
task_id=task_id,
|
||||||
|
provider=config.provider_type,
|
||||||
)
|
)
|
||||||
return instance
|
return instance
|
||||||
# Record the task as handled so later dispatchers in the same
|
# Record the task as handled so later dispatchers in the same
|
||||||
@@ -5837,23 +5839,23 @@ Start by:
|
|||||||
|
|
||||||
return RateLimitStateTracker(provider)
|
return RateLimitStateTracker(provider)
|
||||||
|
|
||||||
async def _grok_spawn_parked(self, provider_type: str | None) -> bool:
|
async def _provider_spawn_parked(self, provider_type: str | None) -> bool:
|
||||||
"""True when *provider_type* is GROK and the provider is parked rate-limited.
|
"""True when *provider_type*'s provider is parked (rate-limited/overloaded).
|
||||||
|
|
||||||
The grok 429 loop-breaker consults this before launching a grok
|
The spawn loop-breaker consults this before launching any container.
|
||||||
container. Grok-only and fail-open: any error reading the tracker
|
Fail-open: any error reading the tracker returns False so a Redis hiccup
|
||||||
returns False so a Redis hiccup can never block spawning.
|
can never block spawning.
|
||||||
"""
|
"""
|
||||||
from roboco.models.base import ModelProvider
|
if provider_type is None:
|
||||||
|
|
||||||
if provider_type != ModelProvider.GROK.value:
|
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
tracker = self._make_tracker(ModelProvider.GROK.value)
|
tracker = self._make_tracker(provider_type)
|
||||||
return bool(await tracker.is_rate_limited())
|
return bool(await tracker.is_rate_limited())
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"grok rate-limit check failed; allowing spawn", error=str(exc)
|
"provider rate-limit check failed; allowing spawn",
|
||||||
|
provider=provider_type,
|
||||||
|
error=str(exc),
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ A one-shot grok run that hits an xAI 429 exits 75; the orchestrator parks the
|
|||||||
grok provider rate-limited instead of crash-retrying, and the spawn guard
|
grok provider rate-limited instead of crash-retrying, and the spawn guard
|
||||||
suppresses re-spawns until the probe-resume loop clears the park. These tests
|
suppresses re-spawns until the probe-resume loop clears the park. These tests
|
||||||
exercise the decision points deterministically (tracker + finalize stubbed).
|
exercise the decision points deterministically (tracker + finalize stubbed).
|
||||||
|
The same spawn guard now protects every provider (not just GROK), so the
|
||||||
|
Claude session/overload paths get the same loop-break for free.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -61,44 +63,43 @@ def test_is_grok_rate_limit_exit() -> None:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_grok_spawn_parked_true_when_limited(
|
async def test_provider_spawn_parked_true_when_limited(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||||
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True))
|
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True))
|
||||||
assert await orch._grok_spawn_parked("grok") is True
|
assert await orch._provider_spawn_parked("grok") is True
|
||||||
|
assert await orch._provider_spawn_parked("anthropic") is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_grok_spawn_parked_false_when_not_limited(
|
async def test_provider_spawn_parked_false_when_not_limited(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||||
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False))
|
monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False))
|
||||||
assert await orch._grok_spawn_parked("grok") is False
|
assert await orch._provider_spawn_parked("grok") is False
|
||||||
|
assert await orch._provider_spawn_parked("anthropic") is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_grok_spawn_parked_false_for_non_grok(
|
async def test_provider_spawn_parked_false_when_provider_unknown() -> None:
|
||||||
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||||
|
assert await orch._provider_spawn_parked(None) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_provider_spawn_parked_fails_open(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
|
||||||
# Non-grok never consults the tracker (a tracker call would error here).
|
|
||||||
monkeypatch.setattr(
|
|
||||||
orch, "_make_tracker", lambda _p: (_ for _ in ()).throw(AssertionError)
|
|
||||||
)
|
|
||||||
assert await orch._grok_spawn_parked("anthropic") is False
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_grok_spawn_parked_fails_open(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
||||||
# A tracker error must never block spawning (fail-open -> False).
|
# A tracker error must never block spawning (fail-open -> False).
|
||||||
def _boom(_p: str) -> object:
|
def _boom(_p: str) -> object:
|
||||||
raise RuntimeError("redis down")
|
raise RuntimeError("redis down")
|
||||||
|
|
||||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||||
monkeypatch.setattr(orch, "_make_tracker", _boom)
|
monkeypatch.setattr(orch, "_make_tracker", _boom)
|
||||||
assert await orch._grok_spawn_parked("grok") is False
|
assert await orch._provider_spawn_parked("grok") is False
|
||||||
|
assert await orch._provider_spawn_parked("anthropic") is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
Reference in New Issue
Block a user