fix(run-hardening): park the workforce on a session-limit + PR-review verdict colour (#249)

* fix(orchestrator): park the provider on a Claude session-limit 429, not crash-loop

When the org Claude usage ("5-hour") session limit is hit, an agent container
exits non-zero with a 0-token 429 rejection. The provider-unavailable break only
recognized 5xx overload signatures (529/500/503), so a session-limit crash fell
through to the normal crash-retry path — the orchestrator respawned the agent
straight back into the limit, fleet-wide, until the window reset.

Add a sibling detector _provider_rate_limit_park_target that matches the
session-limit markers ("hit your session limit", "five_hour") in the dead
container's output and parks the provider with kind="rate_limited" (a longer
probe cadence), checked before the overload path in _handle_stopped_container.
Reuses the existing park-and-probe machinery, so the background probe loop
revives the parked tasks when the quota resets — no churn. Gated by the same
overload_break_enabled flag.

Also backfills the CHANGELOG Fixed entry for the orchestrator self-call auth fix
(merged in #248 without one).

* fix(panel): PR Reviewer Notes card colour reflects the verdict

The card was hardcoded teal/green regardless of the review verdict, so a Failed
review sat inside a green card and read as passing at a glance. Derive the card
background from the verdict (red on failed, green on approved/passed, amber on
changes-requested, neutral teal before a verdict) — mirroring the QA Notes card.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-06-24 03:20:14 +02:00
committed by GitHub
co-authored by Renn F
parent 889f3689e7
commit 40d685bd9f
4 changed files with 180 additions and 1 deletions
@@ -16,6 +16,7 @@ from roboco.config import settings
from roboco.models.runtime import AgentInstance
from roboco.runtime.orchestrator import (
_OVERLOAD_RETRY_AFTER_S,
_RATE_LIMIT_RETRY_AFTER_S,
AgentOrchestrator,
AgentState,
)
@@ -25,6 +26,11 @@ _OVERLOAD_LOG = (
'"message":"Overloaded"}}'
)
_CLEAN_LOG = "be-dev-1 finished editing src/app.py; all checks passed"
_SESSION_LIMIT_LOG = (
'{"type":"result","is_error":true,"api_error_status":429,'
'"result":"You have hit your session limit - resets 1am (UTC)",'
'"rate_limit_info":{"rateLimitType":"five_hour"}}'
)
def _instance(provider_type: str | None = "anthropic") -> AgentInstance:
@@ -155,6 +161,9 @@ async def test_stopped_container_parks_on_overload(
park = AsyncMock()
spawn = AsyncMock()
monkeypatch.setattr(orch, "_is_grok_rate_limit_exit", lambda _i, _e: False)
monkeypatch.setattr(
orch, "_provider_rate_limit_park_target", AsyncMock(return_value=None)
)
monkeypatch.setattr(
orch, "_provider_overload_park_target", AsyncMock(return_value="anthropic")
)
@@ -182,6 +191,9 @@ async def test_stopped_container_crash_retries_when_not_overload(
inst.error_count = 0
spawn = AsyncMock()
monkeypatch.setattr(orch, "_is_grok_rate_limit_exit", lambda _i, _e: False)
monkeypatch.setattr(
orch, "_provider_rate_limit_park_target", AsyncMock(return_value=None)
)
monkeypatch.setattr(
orch, "_provider_overload_park_target", AsyncMock(return_value=None)
)
@@ -192,3 +204,74 @@ async def test_stopped_container_crash_retries_when_not_overload(
# Not an overload → the normal crash-retry path runs.
spawn.assert_awaited_once()
# ---------------------------------------------------------------------------
# Session/usage-limit (429) parking — the same break for a different signal
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_detects_session_limit_marker_for_anthropic(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(settings, "overload_break_enabled", True)
monkeypatch.setattr(
orch, "_tail_container_logs", AsyncMock(return_value=_SESSION_LIMIT_LOG)
)
assert (
await orch._provider_rate_limit_park_target("be-dev-1", _instance())
== "anthropic"
)
@pytest.mark.asyncio
async def test_clean_output_is_not_a_session_limit(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(settings, "overload_break_enabled", True)
monkeypatch.setattr(
orch, "_tail_container_logs", AsyncMock(return_value=_CLEAN_LOG)
)
assert await orch._provider_rate_limit_park_target("be-dev-1", _instance()) is None
@pytest.mark.asyncio
async def test_session_limit_disabled_flag_never_parks(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(settings, "overload_break_enabled", False)
tail = AsyncMock(return_value=_SESSION_LIMIT_LOG)
monkeypatch.setattr(orch, "_tail_container_logs", tail)
assert await orch._provider_rate_limit_park_target("be-dev-1", _instance()) is None
tail.assert_not_awaited() # short-circuits before reading logs
@pytest.mark.asyncio
async def test_stopped_container_parks_on_session_limit(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
inst = _instance()
park = AsyncMock()
spawn = AsyncMock()
overload = AsyncMock(return_value=None)
monkeypatch.setattr(orch, "_is_grok_rate_limit_exit", lambda _i, _e: False)
monkeypatch.setattr(
orch, "_provider_rate_limit_park_target", AsyncMock(return_value="anthropic")
)
monkeypatch.setattr(orch, "_provider_overload_park_target", overload)
monkeypatch.setattr(orch, "_park_provider_unavailable", park)
monkeypatch.setattr(orch, "_finalize_spawn_session", AsyncMock())
monkeypatch.setattr(orch, "spawn_agent", spawn)
await orch._handle_stopped_container("be-dev-1", inst, exit_code=1)
park.assert_awaited_once_with(
"be-dev-1",
inst,
provider="anthropic",
retry_after=_RATE_LIMIT_RETRY_AFTER_S,
kind="rate_limited",
)
spawn.assert_not_awaited() # crash-retry short-circuited
overload.assert_not_awaited() # session-limit checked before the overload path