diff --git a/roboco/runtime/orchestrator.py b/roboco/runtime/orchestrator.py index 38ea828b..11f3ebb3 100644 --- a/roboco/runtime/orchestrator.py +++ b/roboco/runtime/orchestrator.py @@ -5890,6 +5890,35 @@ Start by: return "" return out.decode(errors="replace") + def _transcript_tail_text(self, agent_id: str, lines: int = 80) -> str: + """Return the last ``lines`` of the newest Claude transcript for *agent_id*. + + The SDK server redirects its runtime log to ``/tmp/sdk-server.log`` inside + the container, so session-limit markers such as "hit your session limit" + and "five_hour" do not reach ``docker logs``. The durable Claude + transcript on the host (mounted into the orchestrator at ``~/.claude``) + contains those same events, so we search it as a fallback when deciding + whether to park the provider. Returns "" when no transcript is found or it + cannot be read. + """ + from pathlib import Path + + projects = Path.home() / ".claude" / "projects" + try: + jsonl = [ + f + for d in projects.glob(f"*-{agent_id}") + if d.is_dir() + for f in d.glob("*.jsonl") + ] + if not jsonl: + return "" + newest = max(jsonl, key=lambda f: f.stat().st_mtime) + text = newest.read_text(encoding="utf-8", errors="replace") + return "\n".join(text.splitlines()[-lines:]) + except OSError: + return "" + async def _provider_overload_park_target( self, agent_id: str, instance: Any ) -> str | None: @@ -5933,7 +5962,11 @@ Start by: if provider_type not in (None, ModelProvider.ANTHROPIC.value): return None tail = await self._tail_container_logs(f"roboco-agent-{agent_id}") - lowered = tail.lower() + # The SDK server writes to /tmp/sdk-server.log, not stdout, so the + # session-limit markers may not appear in docker logs. Search the durable + # Claude transcript on the host as well. + transcript_tail = self._transcript_tail_text(agent_id) + lowered = (tail + "\n" + transcript_tail).lower() if any(marker in lowered for marker in _ANTHROPIC_RATE_LIMIT_MARKERS): return ModelProvider.ANTHROPIC.value return None diff --git a/tests/unit/runtime/test_provider_overload_break.py b/tests/unit/runtime/test_provider_overload_break.py index c4191cba..96f7e829 100644 --- a/tests/unit/runtime/test_provider_overload_break.py +++ b/tests/unit/runtime/test_provider_overload_break.py @@ -46,8 +46,13 @@ def _instance(provider_type: str | None = "anthropic") -> AgentInstance: @pytest.fixture -def orch() -> AgentOrchestrator: - return AgentOrchestrator.__new__(AgentOrchestrator) +def orch(monkeypatch: pytest.MonkeyPatch) -> AgentOrchestrator: + orch = AgentOrchestrator.__new__(AgentOrchestrator) + # Tests for parking detection control docker logs directly; keep the durable + # transcript fallback empty by default so dev environments with stray + # transcripts do not make the tests flaky. + monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "") + return orch class _FakeTracker: @@ -225,6 +230,22 @@ async def test_detects_session_limit_marker_for_anthropic( ) +@pytest.mark.asyncio +async def test_detects_session_limit_marker_in_transcript( + orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch +) -> None: + """Docker logs miss the SDK-server log; the durable transcript still has it.""" + monkeypatch.setattr(settings, "overload_break_enabled", True) + monkeypatch.setattr(orch, "_tail_container_logs", AsyncMock(return_value="")) + monkeypatch.setattr( + orch, "_transcript_tail_text", lambda _a, _lines=80: _SESSION_LIMIT_LOG + ) + assert ( + await orch._provider_rate_limit_park_target("be-dev-1", _instance()) + == "anthropic" + ) + + @pytest.mark.asyncio async def test_clean_output_is_not_a_session_limit( orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch