diff --git a/roboco/runtime/orchestrator.py b/roboco/runtime/orchestrator.py index 078beaf2..dcb2da54 100644 --- a/roboco/runtime/orchestrator.py +++ b/roboco/runtime/orchestrator.py @@ -110,8 +110,13 @@ _CEO_NOTIFY_THRESHOLD = 10 # survives to kill the run. When it does, park the provider like a 429 instead # of crash-retrying into the overload. These markers are matched (lowercased, # substring) against the tail of the dead container's own output, so they are -# kept specific to how the API surfaces an overload — bare "500"/"529" would -# false-match an agent that merely writes about HTTP status codes. +# kept specific to how the API surfaces an overload. Bare "error 529"/"error +# 500"/"error 503" were dropped (F037): an agent that merely writes about an +# HTTP status code in its own notes ("the endpoint returned error 500, +# retrying") would false-match and park the whole Anthropic fleet. The SDK +# error formatter emits "API Error: NNN" + a JSON error type, so the +# ``api error: NNN`` and type-string markers below cover every real overload +# without that false-match surface. _OVERLOAD_RETRY_AFTER_S = 45.0 _ANTHROPIC_OVERLOAD_MARKERS: tuple[str, ...] = ( "overloaded_error", @@ -119,9 +124,6 @@ _ANTHROPIC_OVERLOAD_MARKERS: tuple[str, ...] = ( "api error: 529", "api error: 500", "api error: 503", - "error 529", - "error 500", - "error 503", ) # Session / usage-limit parking (HTTP 429). The Claude session ("5-hour") limit diff --git a/tests/unit/runtime/test_provider_overload_break.py b/tests/unit/runtime/test_provider_overload_break.py index 4ecda593..e8606f0e 100644 --- a/tests/unit/runtime/test_provider_overload_break.py +++ b/tests/unit/runtime/test_provider_overload_break.py @@ -123,6 +123,29 @@ async def test_detects_overload_marker_in_transcript( ) +@pytest.mark.asyncio +async def test_agent_writing_about_error_500_does_not_park( + orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch +) -> None: + # F037: an agent that merely writes about an HTTP error code in its own + # notes ("the endpoint returned error 500, retrying") must NOT trip the + # overload detector and park the whole Anthropic fleet. Markers must be + # specific to the API error formatter, not bare "error NNN". + monkeypatch.setattr(settings, "overload_break_enabled", True) + agent_note = ( + "be-dev-1: the /health endpoint returned error 500 on retry; " + "adding a backoff. Also saw error 529 and error 503 upstream. " + "Not a model-API issue." + ) + monkeypatch.setattr( + orch, "_tail_container_logs", AsyncMock(return_value=agent_note) + ) + monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "") + assert ( + await orch._provider_overload_park_target("be-dev-1", _instance()) is None + ) + + @pytest.mark.asyncio async def test_disabled_flag_never_parks( orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch