feat(routing): cost-tiered complexity routing + saved presets (#656)

The 08-31 lever: model_assignments gains one compound rung —
AGENT_SLUG > ROLE('{role}:{complexity}') > ROLE > GLOBAL — so a
low-complexity task can route to a cheaper tier while coordinators stay
pinned. Structurally opt-in: zero rows means byte-identical routing
(pinned by a named test across every precedence case), the cost_tiered
apply-mode (seeds developer:low→haiku) is reachable only from the
explicit PM-gated endpoint — verified no startup path can apply it.
Overrides are downgrade-only (input-price comparator), allowlisted to
{developer, qa, documenter} — cell_pm excluded per the org's own
coordinator definition and its documented weak-model incidents — and
validated at write time (disabled/unconfigured provider rejected with
remediation; cross-provider-family overrides warn explicitly).

Per adversarial review: the four mode-switch applies now spare compound
rows exactly like agent pins (the 2026-07-17 unscoped-wipe class, new
victim, same fix extended via one shared wipe helper) with panel cache
invalidation + truthful confirm dialogs; preset apply validates the
entire payload BEFORE the wipe (validate-all-first), with a savepoint
crash test proving rollback.

Presets (CEO request): routing_presets table (migration 082) snapshots
the full mix — mode, per-agent overrides, complexity rows — with
save/apply/delete endpoints and a panel preset bar; applying skips
since-removed models with per-entry notes, never silently.

Task complexity threads task_id through _resolve_agent_route at both
call sites; taskless spawns unchanged. 235 backend + 23 panel tests.

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-07-23 03:04:18 +02:00
committed by GitHub
co-authored by Renn F
parent 10f039c36f
commit 165892dc62
17 changed files with 2812 additions and 70 deletions
+43
View File
@@ -21,6 +21,7 @@ from roboco.billing.pricing import (
_is_anthropic_model,
calculate_cost,
calculate_cost_result,
input_price_per_million,
)
# ---------------------------------------------------------------------------
@@ -539,3 +540,45 @@ def test_sonnet5_reverts_to_list_rate_after_2026_08_31(
_SONNET_CACHE_READ,
_SONNET_CACHE_WRITE,
)
# ---------------------------------------------------------------------------
# input_price_per_million — the cost-tiered complexity-override comparator
# ---------------------------------------------------------------------------
class TestInputPricePerMillion:
"""The downgrade-only comparator for complexity overrides (no explicit
tier ordering exists in the model catalog, so the input rate stands in
for "which tier is costlier")."""
def test_orders_haiku_below_sonnet_below_opus(self) -> None:
assert (
input_price_per_million("haiku")
< input_price_per_million("sonnet")
< input_price_per_million("opus")
)
def test_matches_pricing_table_value(self) -> None:
assert input_price_per_million("haiku") == _HAIKU_INPUT
assert input_price_per_million("sonnet") == _SONNET_INPUT
assert input_price_per_million("opus") == _OPUS_INPUT
def test_grok_priced_below_sonnet(self) -> None:
"""Grok legitimately downgrades-from sonnet under this comparator."""
assert input_price_per_million("grok-build-0.1") < input_price_per_million(
"sonnet"
)
def test_unpriced_non_anthropic_model_is_free_tier(self) -> None:
"""A self-hosted / Ollama Cloud model has no per-token rate — treated
as the cheapest possible tier, so it can never be rejected as
"costlier" by the downgrade-only policy."""
assert input_price_per_million("glm-5.2:cloud") == 0.0
assert input_price_per_million("my-custom-self-hosted-model:7b") == 0.0
def test_empty_model_returns_zero(self) -> None:
assert input_price_per_million("") == 0.0
def test_case_insensitive(self) -> None:
assert input_price_per_million("HAIKU") == input_price_per_million("haiku")
@@ -0,0 +1,166 @@
"""Task complexity threads into `_resolve_agent_route` -> `resolve_for_agent`.
Cost-tiered routing (roboco/services/llm.py) reads a task's
`estimated_complexity` to try a compound ROLE(":"complexity) row before
falling to the plain ROLE row. The orchestrator owns the one indexed Task
lookup and threads the lowercase complexity string through. This is the pure
wiring test (`_resolve_agent_route` -> `resolve_for_agent`); the precedence
logic itself is covered in tests/integration/test_llm_routing.py.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
import roboco.db.base as db_base
import roboco.services.llm as llm_module
from roboco.models.base import Complexity, ModelProvider
from roboco.runtime.orchestrator import AgentOrchestrator
from roboco.services.llm import AgentRoute
# Sentinel route the mocked resolve_for_agent returns — a real AgentRoute
# instance (not a bare string) so `result is _SENTINEL_ROUTE` type-checks
# cleanly against `_resolve_agent_route`'s declared AgentRoute return type.
_SENTINEL_ROUTE = AgentRoute(
provider_id=None,
provider_type=ModelProvider.ANTHROPIC,
base_url=None,
auth_token=None,
model_name="sentinel",
)
class _ScalarResult:
def __init__(self, value: Any) -> None:
self._value = value
def scalar_one_or_none(self) -> Any:
return self._value
class _FakeSession:
"""Minimal async-context-manager session returning a fixed complexity."""
def __init__(self, complexity_value: Any) -> None:
self._complexity_value = complexity_value
def __call__(self) -> _FakeSession:
return self
async def __aenter__(self) -> _FakeSession:
return self
async def __aexit__(self, *exc: object) -> None:
return None
async def execute(self, _stmt: Any) -> _ScalarResult:
return _ScalarResult(self._complexity_value)
class _BoomSession(_FakeSession):
"""A session whose `execute` always raises — models a task-lookup failure
(bad/unresolvable task id) distinct from a genuine DB/session outage."""
async def execute(self, _stmt: Any) -> _ScalarResult:
raise RuntimeError("bad task id")
def _orch() -> AgentOrchestrator:
# __new__ + skip __init__: avoid all constructor I/O — this method is pure
# w.r.t. instance state (it only touches module-level imports + args).
return AgentOrchestrator.__new__(AgentOrchestrator)
def _wire(monkeypatch: pytest.MonkeyPatch, fake_session: Any) -> AsyncMock:
"""Patch get_session_factory + get_model_routing_service; return the
resolve_for_agent mock so the test can assert on its call."""
monkeypatch.setattr(
db_base,
"get_session_factory",
lambda: MagicMock(return_value=fake_session),
)
resolve_mock = AsyncMock(return_value=_SENTINEL_ROUTE)
fake_router = MagicMock(resolve_for_agent=resolve_mock)
monkeypatch.setattr(
llm_module, "get_model_routing_service", lambda _db: fake_router
)
return resolve_mock
@pytest.mark.asyncio
async def test_task_with_high_complexity_threads_lowercase_string(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A task with estimated_complexity=HIGH resolves the compound
'role:high' row — i.e. resolve_for_agent is called with complexity='high'
(lowercased from the Complexity enum's value)."""
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
orch = _orch()
result = await orch._resolve_agent_route("be-dev-1", "task-123")
assert result is _SENTINEL_ROUTE
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="high")
@pytest.mark.asyncio
async def test_task_with_low_complexity_threads_lowercase_string(
monkeypatch: pytest.MonkeyPatch,
) -> None:
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.LOW))
orch = _orch()
await orch._resolve_agent_route("be-dev-1", "task-456")
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="low")
@pytest.mark.asyncio
async def test_taskless_spawn_threads_none_complexity_unchanged(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A no-task spawn (idle PM bootstrap, Intake/Secretary chats, ...) never
even attempts a task lookup — complexity=None, byte-identical to the
pre-cost-tiering call shape."""
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
orch = _orch()
result = await orch._resolve_agent_route("be-dev-1", None)
assert result is _SENTINEL_ROUTE
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
@pytest.mark.asyncio
async def test_missing_task_row_degrades_to_none_complexity_silently(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""scalar_one_or_none() returning None (task not found / deleted) is not
an error — complexity falls back to None and routing still proceeds
through the router (not the hardcoded legacy path)."""
resolve_mock = _wire(monkeypatch, _FakeSession(None))
orch = _orch()
result = await orch._resolve_agent_route("be-dev-1", "ghost-task-id")
assert result is _SENTINEL_ROUTE
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
@pytest.mark.asyncio
async def test_task_lookup_failure_degrades_silently_not_to_full_legacy_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A task-lookup-specific failure (bad id, transient query error) must
NOT escalate to the full DB-failure downgrade (hardcoded ROLE_MODEL_MAP,
bypassing model_assignments entirely) — only the complexity lookup is
skipped; AGENT_SLUG/ROLE/GLOBAL resolution still runs via the router."""
resolve_mock = _wire(monkeypatch, _BoomSession(None))
orch = _orch()
result = await orch._resolve_agent_route("be-dev-1", "bad-task-id")
assert result is _SENTINEL_ROUTE
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
@@ -47,7 +47,7 @@ def _wire(monitor: dict[str, Any]) -> Any:
async def _git_context(_gc: Any, _tid: str | None) -> None:
return None
async def _route(_aid: str) -> Any:
async def _route(_aid: str, _tid: str | None = None) -> Any:
monitor["route_calls"] += 1
return SimpleNamespace(
provider_type=SimpleNamespace(value="anthropic"),