mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[067ce5d1] fix(runtime): detect session-limit 429 in Claude transcript for provider parking
The SDK server writes runtime output to /tmp/sdk-server.log inside the agent container, so the session-limit markers never appeared in docker logs. Read the newest durable Claude transcript from ~/.claude/projects as a fallback so the provider gets parked and auto-revived instead of crash-retrying. - Add _transcript_tail_text to read the agent's transcript tail - Use it in _provider_rate_limit_park_target alongside docker logs - Add regression test for transcript-only detection
This commit is contained in:
@@ -5890,6 +5890,35 @@ Start by:
|
||||
return ""
|
||||
return out.decode(errors="replace")
|
||||
|
||||
def _transcript_tail_text(self, agent_id: str, lines: int = 80) -> str:
|
||||
"""Return the last ``lines`` of the newest Claude transcript for *agent_id*.
|
||||
|
||||
The SDK server redirects its runtime log to ``/tmp/sdk-server.log`` inside
|
||||
the container, so session-limit markers such as "hit your session limit"
|
||||
and "five_hour" do not reach ``docker logs``. The durable Claude
|
||||
transcript on the host (mounted into the orchestrator at ``~/.claude``)
|
||||
contains those same events, so we search it as a fallback when deciding
|
||||
whether to park the provider. Returns "" when no transcript is found or it
|
||||
cannot be read.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
projects = Path.home() / ".claude" / "projects"
|
||||
try:
|
||||
jsonl = [
|
||||
f
|
||||
for d in projects.glob(f"*-{agent_id}")
|
||||
if d.is_dir()
|
||||
for f in d.glob("*.jsonl")
|
||||
]
|
||||
if not jsonl:
|
||||
return ""
|
||||
newest = max(jsonl, key=lambda f: f.stat().st_mtime)
|
||||
text = newest.read_text(encoding="utf-8", errors="replace")
|
||||
return "\n".join(text.splitlines()[-lines:])
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
async def _provider_overload_park_target(
|
||||
self, agent_id: str, instance: Any
|
||||
) -> str | None:
|
||||
@@ -5933,7 +5962,11 @@ Start by:
|
||||
if provider_type not in (None, ModelProvider.ANTHROPIC.value):
|
||||
return None
|
||||
tail = await self._tail_container_logs(f"roboco-agent-{agent_id}")
|
||||
lowered = tail.lower()
|
||||
# The SDK server writes to /tmp/sdk-server.log, not stdout, so the
|
||||
# session-limit markers may not appear in docker logs. Search the durable
|
||||
# Claude transcript on the host as well.
|
||||
transcript_tail = self._transcript_tail_text(agent_id)
|
||||
lowered = (tail + "\n" + transcript_tail).lower()
|
||||
if any(marker in lowered for marker in _ANTHROPIC_RATE_LIMIT_MARKERS):
|
||||
return ModelProvider.ANTHROPIC.value
|
||||
return None
|
||||
|
||||
@@ -46,8 +46,13 @@ def _instance(provider_type: str | None = "anthropic") -> AgentInstance:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def orch() -> AgentOrchestrator:
|
||||
return AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
def orch(monkeypatch: pytest.MonkeyPatch) -> AgentOrchestrator:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
# Tests for parking detection control docker logs directly; keep the durable
|
||||
# transcript fallback empty by default so dev environments with stray
|
||||
# transcripts do not make the tests flaky.
|
||||
monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "")
|
||||
return orch
|
||||
|
||||
|
||||
class _FakeTracker:
|
||||
@@ -225,6 +230,22 @@ async def test_detects_session_limit_marker_for_anthropic(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detects_session_limit_marker_in_transcript(
|
||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Docker logs miss the SDK-server log; the durable transcript still has it."""
|
||||
monkeypatch.setattr(settings, "overload_break_enabled", True)
|
||||
monkeypatch.setattr(orch, "_tail_container_logs", AsyncMock(return_value=""))
|
||||
monkeypatch.setattr(
|
||||
orch, "_transcript_tail_text", lambda _a, _lines=80: _SESSION_LIMIT_LOG
|
||||
)
|
||||
assert (
|
||||
await orch._provider_rate_limit_park_target("be-dev-1", _instance())
|
||||
== "anthropic"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clean_output_is_not_a_session_limit(
|
||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||
|
||||
Reference in New Issue
Block a user