mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(routing): cost-tiered complexity routing + saved presets (#656)
The 08-31 lever: model_assignments gains one compound rung —
AGENT_SLUG > ROLE('{role}:{complexity}') > ROLE > GLOBAL — so a
low-complexity task can route to a cheaper tier while coordinators stay
pinned. Structurally opt-in: zero rows means byte-identical routing
(pinned by a named test across every precedence case), the cost_tiered
apply-mode (seeds developer:low→haiku) is reachable only from the
explicit PM-gated endpoint — verified no startup path can apply it.
Overrides are downgrade-only (input-price comparator), allowlisted to
{developer, qa, documenter} — cell_pm excluded per the org's own
coordinator definition and its documented weak-model incidents — and
validated at write time (disabled/unconfigured provider rejected with
remediation; cross-provider-family overrides warn explicitly).
Per adversarial review: the four mode-switch applies now spare compound
rows exactly like agent pins (the 2026-07-17 unscoped-wipe class, new
victim, same fix extended via one shared wipe helper) with panel cache
invalidation + truthful confirm dialogs; preset apply validates the
entire payload BEFORE the wipe (validate-all-first), with a savepoint
crash test proving rollback.
Presets (CEO request): routing_presets table (migration 082) snapshots
the full mix — mode, per-agent overrides, complexity rows — with
save/apply/delete endpoints and a panel preset bar; applying skips
since-removed models with per-entry notes, never silently.
Task complexity threads task_id through _resolve_agent_route at both
call sites; taskless spawns unchanged. 235 backend + 23 panel tests.
Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from roboco.billing.pricing import (
|
||||
_is_anthropic_model,
|
||||
calculate_cost,
|
||||
calculate_cost_result,
|
||||
input_price_per_million,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -539,3 +540,45 @@ def test_sonnet5_reverts_to_list_rate_after_2026_08_31(
|
||||
_SONNET_CACHE_READ,
|
||||
_SONNET_CACHE_WRITE,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# input_price_per_million — the cost-tiered complexity-override comparator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInputPricePerMillion:
|
||||
"""The downgrade-only comparator for complexity overrides (no explicit
|
||||
tier ordering exists in the model catalog, so the input rate stands in
|
||||
for "which tier is costlier")."""
|
||||
|
||||
def test_orders_haiku_below_sonnet_below_opus(self) -> None:
|
||||
assert (
|
||||
input_price_per_million("haiku")
|
||||
< input_price_per_million("sonnet")
|
||||
< input_price_per_million("opus")
|
||||
)
|
||||
|
||||
def test_matches_pricing_table_value(self) -> None:
|
||||
assert input_price_per_million("haiku") == _HAIKU_INPUT
|
||||
assert input_price_per_million("sonnet") == _SONNET_INPUT
|
||||
assert input_price_per_million("opus") == _OPUS_INPUT
|
||||
|
||||
def test_grok_priced_below_sonnet(self) -> None:
|
||||
"""Grok legitimately downgrades-from sonnet under this comparator."""
|
||||
assert input_price_per_million("grok-build-0.1") < input_price_per_million(
|
||||
"sonnet"
|
||||
)
|
||||
|
||||
def test_unpriced_non_anthropic_model_is_free_tier(self) -> None:
|
||||
"""A self-hosted / Ollama Cloud model has no per-token rate — treated
|
||||
as the cheapest possible tier, so it can never be rejected as
|
||||
"costlier" by the downgrade-only policy."""
|
||||
assert input_price_per_million("glm-5.2:cloud") == 0.0
|
||||
assert input_price_per_million("my-custom-self-hosted-model:7b") == 0.0
|
||||
|
||||
def test_empty_model_returns_zero(self) -> None:
|
||||
assert input_price_per_million("") == 0.0
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
assert input_price_per_million("HAIKU") == input_price_per_million("haiku")
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Task complexity threads into `_resolve_agent_route` -> `resolve_for_agent`.
|
||||
|
||||
Cost-tiered routing (roboco/services/llm.py) reads a task's
|
||||
`estimated_complexity` to try a compound ROLE(":"complexity) row before
|
||||
falling to the plain ROLE row. The orchestrator owns the one indexed Task
|
||||
lookup and threads the lowercase complexity string through. This is the pure
|
||||
wiring test (`_resolve_agent_route` -> `resolve_for_agent`); the precedence
|
||||
logic itself is covered in tests/integration/test_llm_routing.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import roboco.db.base as db_base
|
||||
import roboco.services.llm as llm_module
|
||||
from roboco.models.base import Complexity, ModelProvider
|
||||
from roboco.runtime.orchestrator import AgentOrchestrator
|
||||
from roboco.services.llm import AgentRoute
|
||||
|
||||
# Sentinel route the mocked resolve_for_agent returns — a real AgentRoute
|
||||
# instance (not a bare string) so `result is _SENTINEL_ROUTE` type-checks
|
||||
# cleanly against `_resolve_agent_route`'s declared AgentRoute return type.
|
||||
_SENTINEL_ROUTE = AgentRoute(
|
||||
provider_id=None,
|
||||
provider_type=ModelProvider.ANTHROPIC,
|
||||
base_url=None,
|
||||
auth_token=None,
|
||||
model_name="sentinel",
|
||||
)
|
||||
|
||||
|
||||
class _ScalarResult:
|
||||
def __init__(self, value: Any) -> None:
|
||||
self._value = value
|
||||
|
||||
def scalar_one_or_none(self) -> Any:
|
||||
return self._value
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Minimal async-context-manager session returning a fixed complexity."""
|
||||
|
||||
def __init__(self, complexity_value: Any) -> None:
|
||||
self._complexity_value = complexity_value
|
||||
|
||||
def __call__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
async def __aenter__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def execute(self, _stmt: Any) -> _ScalarResult:
|
||||
return _ScalarResult(self._complexity_value)
|
||||
|
||||
|
||||
class _BoomSession(_FakeSession):
|
||||
"""A session whose `execute` always raises — models a task-lookup failure
|
||||
(bad/unresolvable task id) distinct from a genuine DB/session outage."""
|
||||
|
||||
async def execute(self, _stmt: Any) -> _ScalarResult:
|
||||
raise RuntimeError("bad task id")
|
||||
|
||||
|
||||
def _orch() -> AgentOrchestrator:
|
||||
# __new__ + skip __init__: avoid all constructor I/O — this method is pure
|
||||
# w.r.t. instance state (it only touches module-level imports + args).
|
||||
return AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
|
||||
|
||||
def _wire(monkeypatch: pytest.MonkeyPatch, fake_session: Any) -> AsyncMock:
|
||||
"""Patch get_session_factory + get_model_routing_service; return the
|
||||
resolve_for_agent mock so the test can assert on its call."""
|
||||
monkeypatch.setattr(
|
||||
db_base,
|
||||
"get_session_factory",
|
||||
lambda: MagicMock(return_value=fake_session),
|
||||
)
|
||||
resolve_mock = AsyncMock(return_value=_SENTINEL_ROUTE)
|
||||
fake_router = MagicMock(resolve_for_agent=resolve_mock)
|
||||
monkeypatch.setattr(
|
||||
llm_module, "get_model_routing_service", lambda _db: fake_router
|
||||
)
|
||||
return resolve_mock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_with_high_complexity_threads_lowercase_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A task with estimated_complexity=HIGH resolves the compound
|
||||
'role:high' row — i.e. resolve_for_agent is called with complexity='high'
|
||||
(lowercased from the Complexity enum's value)."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "task-123")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="high")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_with_low_complexity_threads_lowercase_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.LOW))
|
||||
|
||||
orch = _orch()
|
||||
await orch._resolve_agent_route("be-dev-1", "task-456")
|
||||
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="low")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_taskless_spawn_threads_none_complexity_unchanged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A no-task spawn (idle PM bootstrap, Intake/Secretary chats, ...) never
|
||||
even attempts a task lookup — complexity=None, byte-identical to the
|
||||
pre-cost-tiering call shape."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", None)
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_task_row_degrades_to_none_complexity_silently(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""scalar_one_or_none() returning None (task not found / deleted) is not
|
||||
an error — complexity falls back to None and routing still proceeds
|
||||
through the router (not the hardcoded legacy path)."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(None))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "ghost-task-id")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_lookup_failure_degrades_silently_not_to_full_legacy_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A task-lookup-specific failure (bad id, transient query error) must
|
||||
NOT escalate to the full DB-failure downgrade (hardcoded ROLE_MODEL_MAP,
|
||||
bypassing model_assignments entirely) — only the complexity lookup is
|
||||
skipped; AGENT_SLUG/ROLE/GLOBAL resolution still runs via the router."""
|
||||
resolve_mock = _wire(monkeypatch, _BoomSession(None))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "bad-task-id")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
@@ -47,7 +47,7 @@ def _wire(monitor: dict[str, Any]) -> Any:
|
||||
async def _git_context(_gc: Any, _tid: str | None) -> None:
|
||||
return None
|
||||
|
||||
async def _route(_aid: str) -> Any:
|
||||
async def _route(_aid: str, _tid: str | None = None) -> Any:
|
||||
monitor["route_calls"] += 1
|
||||
return SimpleNamespace(
|
||||
provider_type=SimpleNamespace(value="anthropic"),
|
||||
|
||||
Reference in New Issue
Block a user