[067ce5d1] fix(runtime): detect session-limit 429 in Claude transcript for provider parking

The SDK server writes runtime output to /tmp/sdk-server.log inside the agent
container, so the session-limit markers never appeared in docker logs. Read
the newest durable Claude transcript from ~/.claude/projects as a fallback so
the provider gets parked and auto-revived instead of crash-retrying.

- Add _transcript_tail_text to read the agent's transcript tail
- Use it in _provider_rate_limit_park_target alongside docker logs
- Add regression test for transcript-only detection
This commit is contained in:
Renn F
2026-06-25 03:28:03 +02:00
parent 60c64c70e8
commit 75788f519c
2 changed files with 57 additions and 3 deletions
+34 -1
View File
@@ -5890,6 +5890,35 @@ Start by:
return ""
return out.decode(errors="replace")
def _transcript_tail_text(self, agent_id: str, lines: int = 80) -> str:
"""Return the last ``lines`` of the newest Claude transcript for *agent_id*.
The SDK server redirects its runtime log to ``/tmp/sdk-server.log`` inside
the container, so session-limit markers such as "hit your session limit"
and "five_hour" do not reach ``docker logs``. The durable Claude
transcript on the host (mounted into the orchestrator at ``~/.claude``)
contains those same events, so we search it as a fallback when deciding
whether to park the provider. Returns "" when no transcript is found or it
cannot be read.
"""
from pathlib import Path
projects = Path.home() / ".claude" / "projects"
try:
jsonl = [
f
for d in projects.glob(f"*-{agent_id}")
if d.is_dir()
for f in d.glob("*.jsonl")
]
if not jsonl:
return ""
newest = max(jsonl, key=lambda f: f.stat().st_mtime)
text = newest.read_text(encoding="utf-8", errors="replace")
return "\n".join(text.splitlines()[-lines:])
except OSError:
return ""
async def _provider_overload_park_target(
self, agent_id: str, instance: Any
) -> str | None:
@@ -5933,7 +5962,11 @@ Start by:
if provider_type not in (None, ModelProvider.ANTHROPIC.value):
return None
tail = await self._tail_container_logs(f"roboco-agent-{agent_id}")
lowered = tail.lower()
# The SDK server writes to /tmp/sdk-server.log, not stdout, so the
# session-limit markers may not appear in docker logs. Search the durable
# Claude transcript on the host as well.
transcript_tail = self._transcript_tail_text(agent_id)
lowered = (tail + "\n" + transcript_tail).lower()
if any(marker in lowered for marker in _ANTHROPIC_RATE_LIMIT_MARKERS):
return ModelProvider.ANTHROPIC.value
return None
@@ -46,8 +46,13 @@ def _instance(provider_type: str | None = "anthropic") -> AgentInstance:
@pytest.fixture
def orch() -> AgentOrchestrator:
return AgentOrchestrator.__new__(AgentOrchestrator)
def orch(monkeypatch: pytest.MonkeyPatch) -> AgentOrchestrator:
orch = AgentOrchestrator.__new__(AgentOrchestrator)
# Tests for parking detection control docker logs directly; keep the durable
# transcript fallback empty by default so dev environments with stray
# transcripts do not make the tests flaky.
monkeypatch.setattr(orch, "_transcript_tail_text", lambda _a, _lines=80: "")
return orch
class _FakeTracker:
@@ -225,6 +230,22 @@ async def test_detects_session_limit_marker_for_anthropic(
)
@pytest.mark.asyncio
async def test_detects_session_limit_marker_in_transcript(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Docker logs miss the SDK-server log; the durable transcript still has it."""
monkeypatch.setattr(settings, "overload_break_enabled", True)
monkeypatch.setattr(orch, "_tail_container_logs", AsyncMock(return_value=""))
monkeypatch.setattr(
orch, "_transcript_tail_text", lambda _a, _lines=80: _SESSION_LIMIT_LOG
)
assert (
await orch._provider_rate_limit_park_target("be-dev-1", _instance())
== "anthropic"
)
@pytest.mark.asyncio
async def test_clean_output_is_not_a_session_limit(
orch: AgentOrchestrator, monkeypatch: pytest.MonkeyPatch