mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[067ce5d1] fix(runtime): detect session-limit 429 in Claude transcript for provider parking
The SDK server writes runtime output to /tmp/sdk-server.log inside the agent container, so the session-limit markers never appeared in docker logs. Read the newest durable Claude transcript from ~/.claude/projects as a fallback so the provider gets parked and auto-revived instead of crash-retrying. - Add _transcript_tail_text to read the agent's transcript tail - Use it in _provider_rate_limit_park_target alongside docker logs - Add regression test for transcript-only detection
This commit is contained in:
@@ -5890,6 +5890,35 @@ Start by:
|
|||||||
return ""
|
return ""
|
||||||
return out.decode(errors="replace")
|
return out.decode(errors="replace")
|
||||||
|
|
||||||
|
def _transcript_tail_text(self, agent_id: str, lines: int = 80) -> str:
|
||||||
|
"""Return the last ``lines`` of the newest Claude transcript for *agent_id*.
|
||||||
|
|
||||||
|
The SDK server redirects its runtime log to ``/tmp/sdk-server.log`` inside
|
||||||
|
the container, so session-limit markers such as "hit your session limit"
|
||||||
|
and "five_hour" do not reach ``docker logs``. The durable Claude
|
||||||
|
transcript on the host (mounted into the orchestrator at ``~/.claude``)
|
||||||
|
contains those same events, so we search it as a fallback when deciding
|
||||||
|
whether to park the provider. Returns "" when no transcript is found or it
|
||||||
|
cannot be read.
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
projects = Path.home() / ".claude" / "projects"
|
||||||
|
try:
|
||||||
|
jsonl = [
|
||||||
|
f
|
||||||
|
for d in projects.glob(f"*-{agent_id}")
|
||||||
|
if d.is_dir()
|
||||||
|
for f in d.glob("*.jsonl")
|
||||||
|
]
|
||||||
|
if not jsonl:
|
||||||
|
return ""
|
||||||
|
newest = max(jsonl, key=lambda f: f.stat().st_mtime)
|
||||||
|
text = newest.read_text(encoding="utf-8", errors="replace")
|
||||||
|
return "\n".join(text.splitlines()[-lines:])
|
||||||
|
except OSError:
|
||||||
|
return ""
|
||||||
|
|
||||||
async def _provider_overload_park_target(
|
async def _provider_overload_park_target(
|
||||||
self, agent_id: str, instance: Any
|
self, agent_id: str, instance: Any
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
@@ -5933,7 +5962,11 @@ Start by:
|
|||||||
if provider_type not in (None, ModelProvider.ANTHROPIC.value):
|
if provider_type not in (None, ModelProvider.ANTHROPIC.value):
|
||||||
return None
|
return None
|
||||||
tail = await self._tail_container_logs(f"roboco-agent-{agent_id}")
|
tail = await self._tail_container_logs(f"roboco-agent-{agent_id}")
|
||||||
lowered = tail.lower()
|
# The SDK server writes to /tmp/sdk-server.log, not stdout, so the
|
||||||
|
# session-limit markers may not appear in docker logs. Search the durable
|
||||||
|
# Claude transcript on the host as well.
|
||||||
|
transcript_tail = self._transcript_tail_text(agent_id)
|
||||||
|
lowered = (tail + "\n" + transcript_tail).lower()
|
||||||
if any(marker in lowered for marker in _ANTHROPIC_RATE_LIMIT_MARKERS):
|
if any(marker in lowered for marker in _ANTHROPIC_RATE_LIMIT_MARKERS):
|
||||||
return ModelProvider.ANTHROPIC.value
|
return ModelProvider.ANTHROPIC.value
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -46,8 +46,13 @@ def _instance(provider_type: str | None = "anthropic") -> AgentInstance:
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def orch() -> AgentOrchestrator:
|
def orch(monkeypatch: pytest.MonkeyPatch) -> AgentOrchestrator:
|
||||||
return AgentOrchestrator.__new__(AgentOrchestrator)
|
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||||
|
# Tests for parking detection control docker logs directly; keep the durable
|
||||||
|
# transcript fallback empty by default so dev environments with stray
|
||||||
|
# transcripts do not make the tests flaky.
|
||||||
|
monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "")
|
||||||
|
return orch
|
||||||
|
|
||||||
|
|
||||||
class _FakeTracker:
|
class _FakeTracker:
|
||||||
@@ -225,6 +230,22 @@ async def test_detects_session_limit_marker_for_anthropic(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_detects_session_limit_marker_in_transcript(
|
||||||
|
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""Docker logs miss the SDK-server log; the durable transcript still has it."""
|
||||||
|
monkeypatch.setattr(settings, "overload_break_enabled", True)
|
||||||
|
monkeypatch.setattr(orch, "_tail_container_logs", AsyncMock(return_value=""))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
orch, "_transcript_tail_text", lambda _a, _lines=80: _SESSION_LIMIT_LOG
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
await orch._provider_rate_limit_park_target("be-dev-1", _instance())
|
||||||
|
== "anthropic"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_clean_output_is_not_a_session_limit(
|
async def test_clean_output_is_not_a_session_limit(
|
||||||
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
|
||||||
|
|||||||
Reference in New Issue
Block a user