Feat/provider overload break (#242)

* feat(conventions): standard schema models + effective-map merge

* feat(orchestrator): park provider on persistent server overload (529/500)

A 429 rate limit already parks a provider — queue its spawns, probe until it
recovers — but a persistent 529/500/503 overload had no such break: the run
died and the orchestrator crash-retried straight back into the overload,
burning tokens in a respawn loop.

Generalize the park to provider-unavailability. On a non-graceful Anthropic
agent exit, match the API's overload markers (overloaded_error /
internal_server_error / "API Error: 5xx") against the dead container's own
output and park the provider with kind="overloaded"; the existing spawn gate
already queues any parked provider, and the probe-resume loop revives the task
when it recovers. Grok keeps its exit-75 path; both now route through one
_park_provider_unavailable helper. Markers are kept specific so an agent that
merely writes about HTTP 500/529 can't trip the break.

Fix the recovery probe to require a 2xx: it treated any non-429 as recovered,
so a probe that itself got a 529 would have resumed agents straight back into
the overload — wrong for the new path and for a 429 that lifts into a 5xx.

Gated by ROBOCO_OVERLOAD_BREAK_ENABLED (default on; off => crash-retry).

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-06-22 03:49:45 +02:00
committed by GitHub
co-authored by Renn F
parent ba8b877a05
commit 01e10ad693
14 changed files with 826 additions and 29 deletions
Generated
+13
View File
@@ -3003,6 +3003,7 @@ dependencies = [
{ name = "python-jose", extra = ["cryptography"] },
{ name = "python-multipart" },
{ name = "python-toon" },
{ name = "pyyaml" },
{ name = "redis" },
{ name = "sqlalchemy", extra = ["asyncio"] },
{ name = "sse-starlette" },
@@ -3035,6 +3036,7 @@ dev = [
{ name = "ruff" },
{ name = "types-passlib" },
{ name = "types-python-jose" },
{ name = "types-pyyaml" },
{ name = "vulture" },
{ name = "xenon" },
]
@@ -3086,6 +3088,7 @@ requires-dist = [
{ name = "python-jose", extras = ["cryptography"] },
{ name = "python-multipart" },
{ name = "python-toon" },
{ name = "pyyaml" },
{ name = "radon", marker = "extra == 'dev'" },
{ name = "redis" },
{ name = "rich", marker = "extra == 'dev'" },
@@ -3098,6 +3101,7 @@ requires-dist = [
{ name = "tomli-w" },
{ name = "types-passlib", marker = "extra == 'dev'" },
{ name = "types-python-jose", marker = "extra == 'dev'" },
{ name = "types-pyyaml", marker = "extra == 'dev'" },
{ name = "uvicorn", extras = ["standard"] },
{ name = "vulture", marker = "extra == 'dev'" },
{ name = "websockets" },
@@ -3721,6 +3725,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d3/83/df2b34e64f0a674935d718471cf10fb392a7e5bdb0e9e7c739885b62d274/types_python_jose-3.5.0.20260408-py3-none-any.whl", hash = "sha256:968d8a8eac1ff9da249d6335a2bb9f82288d59ba23afe91fcc2662eb9f485e2a", size = 14694, upload-time = "2026-04-08T04:34:09.747Z" },
]
[[package]]
name = "types-pyyaml"
version = "6.0.12.20260518"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850, upload-time = "2026-05-18T06:01:58.675Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312, upload-time = "2026-05-18T06:01:57.368Z" },
]
[[package]]
name = "typing-extensions"
version = "4.15.0"