"""GROK 429 parking: break the 429 -> exit -> respawn cost loop (B4). A one-shot grok run that hits an xAI 429 exits 75; the orchestrator parks the grok provider rate-limited instead of crash-retrying, and the spawn guard suppresses re-spawns until the probe-resume loop clears the park. These tests exercise the decision points deterministically (tracker + finalize stubbed). The same spawn guard now protects every provider (not just GROK), so the Claude session/overload paths get the same loop-break for free. """ from __future__ import annotations from datetime import UTC, datetime, timedelta from unittest.mock import AsyncMock import pytest from roboco.models.runtime import AgentInstance from roboco.runtime.orchestrator import ( _GROK_AUTH_EXIT_CODE, _GROK_RATE_LIMIT_EXIT_CODE, _GROK_RATE_LIMIT_RETRY_AFTER_S, _GROK_REPARK_BACKOFF_CAP, AgentOrchestrator, AgentState, ) def _grok_instance(provider_type: str = "grok") -> AgentInstance: cfg = type("C", (), {"provider_type": provider_type, "model": "grok-build-0.1"})() inst = AgentInstance(agent_id="be-dev-1", state=AgentState.ACTIVE, config=cfg) inst.current_task_id = "task-1" inst.container_id = "cid" return inst class _FakeTracker: def __init__(self, *, limited: bool = False) -> None: self._limited = limited self.activated_with: dict[str, object] | None = None async def is_rate_limited(self) -> bool: return self._limited async def activate( self, *, retry_after: float, affected_agents: list[str], kind: str = "rate_limited", ) -> None: self.activated_with = { "retry_after": retry_after, "affected_agents": affected_agents, "kind": kind, } def test_is_grok_rate_limit_exit() -> None: inst = _grok_instance() assert AgentOrchestrator._is_grok_rate_limit_exit(inst, _GROK_RATE_LIMIT_EXIT_CODE) # Wrong exit code, or a non-grok provider, is not a grok-429 exit. assert not AgentOrchestrator._is_grok_rate_limit_exit(inst, 0) assert not AgentOrchestrator._is_grok_rate_limit_exit(inst, 1) assert not AgentOrchestrator._is_grok_rate_limit_exit( _grok_instance(provider_type="anthropic"), _GROK_RATE_LIMIT_EXIT_CODE ) @pytest.mark.asyncio async def test_provider_spawn_parked_true_when_limited( monkeypatch: pytest.MonkeyPatch, ) -> None: orch = AgentOrchestrator.__new__(AgentOrchestrator) monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=True)) assert await orch._provider_spawn_parked("grok") is True assert await orch._provider_spawn_parked("anthropic") is True @pytest.mark.asyncio async def test_provider_spawn_parked_false_when_not_limited( monkeypatch: pytest.MonkeyPatch, ) -> None: orch = AgentOrchestrator.__new__(AgentOrchestrator) monkeypatch.setattr(orch, "_make_tracker", lambda _p: _FakeTracker(limited=False)) assert await orch._provider_spawn_parked("grok") is False assert await orch._provider_spawn_parked("anthropic") is False @pytest.mark.asyncio async def test_provider_spawn_parked_false_when_provider_unknown() -> None: orch = AgentOrchestrator.__new__(AgentOrchestrator) assert await orch._provider_spawn_parked(None) is False @pytest.mark.asyncio async def test_provider_spawn_parked_fails_open( monkeypatch: pytest.MonkeyPatch, ) -> None: # A tracker error must never block spawning (fail-open -> False). def _boom(_p: str) -> object: raise RuntimeError("redis down") orch = AgentOrchestrator.__new__(AgentOrchestrator) monkeypatch.setattr(orch, "_make_tracker", _boom) assert await orch._provider_spawn_parked("grok") is False assert await orch._provider_spawn_parked("anthropic") is False @pytest.mark.asyncio async def test_park_grok_rate_limited_activates_and_offlines( monkeypatch: pytest.MonkeyPatch, ) -> None: orch = AgentOrchestrator.__new__(AgentOrchestrator) # _park_provider_unavailable registers a WaitingRecord (F035) so the # probe-resume loop can revive the task; the bare __new__ orchestrator # needs the dict + persist stub to exercise that without AttributeError. orch._waiting_records = {} orch._rate_limit_ceo_notified = set() # Backoff state — the constructor (skipped here) initializes these. orch._grok_last_park_at = None orch._grok_repark_count = 0 inst = _grok_instance() inst.error_count = 2 # pretend prior crashes — parking must NOT count one tracker = _FakeTracker() monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker) finalize = AsyncMock() monkeypatch.setattr(orch, "_finalize_spawn_session", finalize) monkeypatch.setattr(orch, "_persist_waiting_record", AsyncMock()) await orch._park_grok_rate_limited("be-dev-1", inst) finalize.assert_awaited_once() assert inst.state == AgentState.OFFLINE assert inst.container_id is None assert inst.error_count == 0 # a 429 is not a crash assert tracker.activated_with == { "retry_after": pytest.approx(60.0), "affected_agents": ["be-dev-1"], "kind": "rate_limited", } @pytest.mark.asyncio async def test_handle_stopped_container_parks_on_grok_429( monkeypatch: pytest.MonkeyPatch, ) -> None: orch = AgentOrchestrator.__new__(AgentOrchestrator) inst = _grok_instance() park = AsyncMock() finalize = AsyncMock() monkeypatch.setattr(orch, "_park_grok_rate_limited", park) monkeypatch.setattr(orch, "_finalize_spawn_session", finalize) await orch._handle_stopped_container("be-dev-1", inst, _GROK_RATE_LIMIT_EXIT_CODE) park.assert_awaited_once_with("be-dev-1", inst) # Early-return: the normal crash/graceful finalize path never runs. finalize.assert_not_awaited() # --------------------------------------------------------------------------- # Exit 78 (auth missing/expired) parks instead of crash-retrying # --------------------------------------------------------------------------- def test_is_grok_auth_exit() -> None: inst = _grok_instance() assert AgentOrchestrator._is_grok_auth_exit(inst, _GROK_AUTH_EXIT_CODE) # Wrong exit code, or a non-grok provider, is not a grok-auth exit. assert not AgentOrchestrator._is_grok_auth_exit(inst, 0) assert not AgentOrchestrator._is_grok_auth_exit(inst, 1) assert not AgentOrchestrator._is_grok_auth_exit( _grok_instance(provider_type="anthropic"), _GROK_AUTH_EXIT_CODE ) @pytest.mark.asyncio async def test_handle_stopped_container_parks_on_grok_auth_exit( monkeypatch: pytest.MonkeyPatch, ) -> None: # A grok container whose entrypoint ran `grok_auth --check` and found the # token missing/expired exits 78 (EX_CONFIG); park it (like the 429 exit-75 # path) so the probe-resume loop revives the task once a fresh token is minted. orch = AgentOrchestrator.__new__(AgentOrchestrator) inst = _grok_instance() park = AsyncMock() finalize = AsyncMock() monkeypatch.setattr(orch, "_park_grok_auth_unavailable", park) monkeypatch.setattr(orch, "_finalize_spawn_session", finalize) await orch._handle_stopped_container("be-dev-1", inst, _GROK_AUTH_EXIT_CODE) park.assert_awaited_once_with("be-dev-1", inst) # Early-return: the crash-retry path never runs (no token burn). finalize.assert_not_awaited() @pytest.mark.asyncio async def test_park_grok_auth_unavailable_activates_with_auth_missing_kind( monkeypatch: pytest.MonkeyPatch, ) -> None: orch = AgentOrchestrator.__new__(AgentOrchestrator) orch._waiting_records = {} orch._rate_limit_ceo_notified = set() inst = _grok_instance() inst.error_count = 2 # prior crashes — parking must NOT count one tracker = _FakeTracker() monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker) monkeypatch.setattr(orch, "_finalize_spawn_session", AsyncMock()) monkeypatch.setattr(orch, "_persist_waiting_record", AsyncMock()) await orch._park_grok_auth_unavailable("be-dev-1", inst) assert inst.state == AgentState.OFFLINE assert inst.container_id is None assert inst.error_count == 0 # an auth-missing exit is not a crash assert tracker.activated_with == { "retry_after": pytest.approx(60.0), "affected_agents": ["be-dev-1"], "kind": "auth_missing", } # --------------------------------------------------------------------------- # # Grok has no real probe, so an optimistic clear respawns into a still-active # xAI 429 every ~90s; back off the re-park retry_after within one rate-limit # episode so the churn dampens instead of spinning flat at 60s. # --------------------------------------------------------------------------- # class _RecordingTracker: """Records every activate() retry_after across multiple re-parks.""" def __init__(self) -> None: self.retry_afters: list[float] = [] self.kinds: list[str] = [] self.agents_lists: list[list[str]] = [] async def activate( self, *, retry_after: float, affected_agents: list[str], kind: str ) -> None: self.retry_afters.append(retry_after) self.kinds.append(kind) self.agents_lists.append(affected_agents) def _backoff_orchestrator() -> AgentOrchestrator: orch = AgentOrchestrator.__new__(AgentOrchestrator) orch._waiting_records = {} orch._rate_limit_ceo_notified = set() # Backoff state — the constructor (skipped here) initializes these. orch._grok_last_park_at = None orch._grok_repark_count = 0 return orch @pytest.mark.asyncio async def test_grok_repark_backs_off_within_episode( monkeypatch: pytest.MonkeyPatch, ) -> None: """Three re-parks within one episode (microseconds apart) must grow the retry_after — 60 -> 120 -> 240 — not stay flat at 60s (the ~90s crash-retry cycle the optimistic clear produces today).""" orch = _backoff_orchestrator() tracker = _RecordingTracker() monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker) monkeypatch.setattr(orch, "_finalize_spawn_session", AsyncMock()) monkeypatch.setattr(orch, "_persist_waiting_record", AsyncMock()) inst = _grok_instance() await orch._park_grok_rate_limited("be-dev-1", inst) await orch._park_grok_rate_limited("be-dev-1", inst) await orch._park_grok_rate_limited("be-dev-1", inst) assert tracker.retry_afters == [60.0, 120.0, 240.0] assert tracker.kinds == ["rate_limited", "rate_limited", "rate_limited"] @pytest.mark.asyncio async def test_grok_repark_resets_after_episode_gap( monkeypatch: pytest.MonkeyPatch, ) -> None: """A re-park AFTER the episode gap is a fresh episode — the retry_after resets to the base 60s even if the prior episode had backed off.""" orch = _backoff_orchestrator() orch._grok_repark_count = 3 # pretend a prior episode backed off hard orch._grok_last_park_at = datetime.now(UTC) - timedelta(hours=2) tracker = _RecordingTracker() monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker) monkeypatch.setattr(orch, "_finalize_spawn_session", AsyncMock()) monkeypatch.setattr(orch, "_persist_waiting_record", AsyncMock()) inst = _grok_instance() await orch._park_grok_rate_limited("be-dev-1", inst) # Fresh episode -> base retry_after, no carried-over backoff. assert tracker.retry_afters == [60.0] assert orch._grok_repark_count == 0 @pytest.mark.asyncio async def test_grok_repark_backoff_caps(monkeypatch: pytest.MonkeyPatch) -> None: """The backoff is capped so a long xAI rate-limit window doesn't push the retry_after toward infinity (bounded cycle, recovery still reachable).""" orch = _backoff_orchestrator() tracker = _RecordingTracker() monkeypatch.setattr(orch, "_make_tracker", lambda _p: tracker) monkeypatch.setattr(orch, "_finalize_spawn_session", AsyncMock()) monkeypatch.setattr(orch, "_persist_waiting_record", AsyncMock()) inst = _grok_instance() # Park once, then re-park well past the cap. for _ in range(_GROK_REPARK_BACKOFF_CAP + 3): await orch._park_grok_rate_limited("be-dev-1", inst) max_expected = _GROK_RATE_LIMIT_RETRY_AFTER_S * (2**_GROK_REPARK_BACKOFF_CAP) # Every retry_after from the cap onward is the same capped value. assert all( r == max_expected for r in tracker.retry_afters[_GROK_REPARK_BACKOFF_CAP:] ) assert max(tracker.retry_afters) == max_expected