Feat/provider overload break (#242)

* feat(conventions): standard schema models + effective-map merge

* feat(orchestrator): park provider on persistent server overload (529/500)

A 429 rate limit already parks a provider — queue its spawns, probe until it
recovers — but a persistent 529/500/503 overload had no such break: the run
died and the orchestrator crash-retried straight back into the overload,
burning tokens in a respawn loop.

Generalize the park to provider-unavailability. On a non-graceful Anthropic
agent exit, match the API's overload markers (overloaded_error /
internal_server_error / "API Error: 5xx") against the dead container's own
output and park the provider with kind="overloaded"; the existing spawn gate
already queues any parked provider, and the probe-resume loop revives the task
when it recovers. Grok keeps its exit-75 path; both now route through one
_park_provider_unavailable helper. Markers are kept specific so an agent that
merely writes about HTTP 500/529 can't trip the break.

Fix the recovery probe to require a 2xx: it treated any non-429 as recovered,
so a probe that itself got a 529 would have resumed agents straight back into
the overload — wrong for the new path and for a 429 that lifts into a 5xx.

Gated by ROBOCO_OVERLOAD_BREAK_ENABLED (default on; off => crash-retry).

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-06-22 03:49:45 +02:00
committed by GitHub
co-authored by Renn F
parent ba8b877a05
commit 01e10ad693
14 changed files with 826 additions and 29 deletions
@@ -0,0 +1,99 @@
"""Schema-model + YAML-parse tests for the architectural-conventions standard."""
from __future__ import annotations
import pytest
from roboco.foundation.policy.conventions.models import (
BUILTIN_RULES,
ConventionsParseError,
ConventionsStandard,
CustomRule,
Module,
Rule,
Waiver,
)
_VALID_YAML = """
version: 1
languages: [python, typescript]
modules:
- path: app/routers
purpose: HTTP routes
forbidden: [model, helper]
- path: app/models
purpose: Pydantic / ORM models
rules:
no_models_in_routers: { level: block }
no_inline_comments: { level: warn }
custom:
- id: no-print
pattern: '\\bprint\\('
message: use the logger
level: warn
languages: [python]
waivers:
- path: app/routers/legacy.py
rule: no_models_in_routers
reason: extraction tracked separately
"""
def test_valid_yaml_parses_to_standard() -> None:
std = ConventionsStandard.parse_yaml(_VALID_YAML)
assert std.version == 1
assert std.languages == ["python", "typescript"]
assert std.modules[0].path == "app/routers"
assert std.modules[0].forbidden == ["model", "helper"]
assert std.rules["no_models_in_routers"].level == "block"
assert std.rules["no_models_in_routers"].name == "no_models_in_routers"
assert std.custom[0].id == "no-print"
assert std.custom[0].languages == ["python"]
assert std.waivers[0].rule == "no_models_in_routers"
def test_empty_yaml_yields_default_standard() -> None:
std = ConventionsStandard.parse_yaml("")
assert std == ConventionsStandard()
assert std.version == 1
def test_unknown_rule_level_raises_parse_error() -> None:
with pytest.raises(ConventionsParseError):
ConventionsStandard.parse_yaml(
"rules:\n no_models_in_routers: { level: explode }\n"
)
def test_malformed_yaml_raises_parse_error() -> None:
with pytest.raises(ConventionsParseError):
ConventionsStandard.parse_yaml("modules: [unterminated\n")
def test_non_mapping_top_level_raises_parse_error() -> None:
with pytest.raises(ConventionsParseError):
ConventionsStandard.parse_yaml("- just\n- a\n- list\n")
def test_unknown_definition_kind_in_forbidden_raises() -> None:
with pytest.raises(ConventionsParseError):
ConventionsStandard.parse_yaml(
"modules:\n - path: x\n purpose: y\n forbidden: [wizard]\n"
)
def test_builtin_rules_cover_the_org_defaults() -> None:
assert BUILTIN_RULES["no_models_in_routers"] == "block"
assert BUILTIN_RULES["no_helpers_in_routers"] == "block"
assert BUILTIN_RULES["no_lint_suppressions"] == "block"
assert BUILTIN_RULES["no_inline_comments"] == "warn"
def test_models_construct_directly() -> None:
mod = Module(path="app/services", purpose="logic", forbidden=["route"])
assert mod.forbidden == ["route"]
rule = Rule(name="no_print", level="warn")
assert rule.level == "warn"
custom = CustomRule(id="x", pattern="y", message="z", level="block")
assert custom.languages == []
waiver = Waiver(path="a.py", rule="no_models_in_routers", reason="r")
assert waiver.path == "a.py"
@@ -0,0 +1,89 @@
"""Effective-map merge tests: auto-derived defaults overlaid by the file."""
from __future__ import annotations
from roboco.foundation.policy.conventions.effective_map import effective_map
from roboco.foundation.policy.conventions.models import (
ConventionsStandard,
CustomRule,
Module,
Rule,
Waiver,
)
def test_effective_map_applies_builtin_rules_when_file_absent() -> None:
eff = effective_map(ConventionsStandard(), None)
assert eff.rules["no_models_in_routers"].level == "block"
assert eff.rules["no_inline_comments"].level == "warn"
def test_file_module_overrides_derived_by_path() -> None:
derived = ConventionsStandard(
modules=[Module(path="app/routers", purpose="routes")]
)
file = ConventionsStandard(
modules=[Module(path="app/routers", purpose="routes", forbidden=["model"])]
)
eff = effective_map(derived, file)
assert len(eff.modules) == 1
assert eff.modules[0].forbidden == ["model"]
def test_file_module_appends_new_path() -> None:
derived = ConventionsStandard(
modules=[Module(path="app/routers", purpose="routes")]
)
file = ConventionsStandard(modules=[Module(path="app/models", purpose="models")])
eff = effective_map(derived, file)
assert [m.path for m in eff.modules] == ["app/routers", "app/models"]
def test_file_rule_overrides_builtin_level() -> None:
file = ConventionsStandard(
rules={"no_inline_comments": Rule(name="no_inline_comments", level="block")}
)
eff = effective_map(ConventionsStandard(), file)
assert eff.rules["no_inline_comments"].level == "block"
def test_derived_rule_overrides_builtin_then_file_overrides_derived() -> None:
derived = ConventionsStandard(
rules={"no_inline_comments": Rule(name="no_inline_comments", level="block")}
)
eff_no_file = effective_map(derived, None)
assert eff_no_file.rules["no_inline_comments"].level == "block"
file = ConventionsStandard(
rules={"no_inline_comments": Rule(name="no_inline_comments", level="warn")}
)
eff = effective_map(derived, file)
assert eff.rules["no_inline_comments"].level == "warn"
def test_languages_are_unioned() -> None:
derived = ConventionsStandard(languages=["python"])
file = ConventionsStandard(languages=["python", "typescript"])
eff = effective_map(derived, file)
assert eff.languages == ["python", "typescript"]
def test_file_custom_and_waivers_replace_derived() -> None:
derived = ConventionsStandard(
custom=[CustomRule(id="d", pattern="d", message="d", level="warn")],
waivers=[Waiver(path="d.py", rule="no_models_in_routers", reason="d")],
)
file = ConventionsStandard(
custom=[CustomRule(id="f", pattern="f", message="f", level="block")],
waivers=[Waiver(path="f.py", rule="no_helpers_in_routers", reason="f")],
)
eff = effective_map(derived, file)
assert [c.id for c in eff.custom] == ["f"]
assert [w.path for w in eff.waivers] == ["f.py"]
def test_file_none_keeps_derived_custom_and_waivers() -> None:
derived = ConventionsStandard(
custom=[CustomRule(id="d", pattern="d", message="d", level="warn")],
)
eff = effective_map(derived, None)
assert [c.id for c in eff.custom] == ["d"]