mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(routing): cost-tiered complexity routing + saved presets (#656)
The 08-31 lever: model_assignments gains one compound rung —
AGENT_SLUG > ROLE('{role}:{complexity}') > ROLE > GLOBAL — so a
low-complexity task can route to a cheaper tier while coordinators stay
pinned. Structurally opt-in: zero rows means byte-identical routing
(pinned by a named test across every precedence case), the cost_tiered
apply-mode (seeds developer:low→haiku) is reachable only from the
explicit PM-gated endpoint — verified no startup path can apply it.
Overrides are downgrade-only (input-price comparator), allowlisted to
{developer, qa, documenter} — cell_pm excluded per the org's own
coordinator definition and its documented weak-model incidents — and
validated at write time (disabled/unconfigured provider rejected with
remediation; cross-provider-family overrides warn explicitly).
Per adversarial review: the four mode-switch applies now spare compound
rows exactly like agent pins (the 2026-07-17 unscoped-wipe class, new
victim, same fix extended via one shared wipe helper) with panel cache
invalidation + truthful confirm dialogs; preset apply validates the
entire payload BEFORE the wipe (validate-all-first), with a savepoint
crash test proving rollback.
Presets (CEO request): routing_presets table (migration 082) snapshots
the full mix — mode, per-agent overrides, complexity rows — with
save/apply/delete endpoints and a panel preset bar; applying skips
since-removed models with per-entry notes, never silently.
Task complexity threads task_id through _resolve_agent_route at both
call sites; taskless spawns unchanged. 235 backend + 23 panel tests.
Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
from uuid import UUID
|
||||
|
||||
from roboco.services.llm import AgentRoute
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
@@ -405,6 +406,262 @@ async def test_resolve_for_agent_uses_provider_token(llm_setup: dict) -> None:
|
||||
assert route.auth_token == "test-secret-key"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cost-tiered compound ROLE(":"complexity) rung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_for_agent_uses_compound_role_complexity_assignment(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""A "developer:low" compound row wins over a plain "developer" row when
|
||||
complexity="low" is threaded in."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=ollama_model,
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == ollama_model
|
||||
assert route.provider_type == ModelProvider.OLLAMA_CLOUD
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compound_row_absent_falls_through_to_plain_role(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""No "developer:low" row → falls straight through to the plain "developer"
|
||||
row, even though a complexity value was threaded in."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_complexity_string_falls_through_gracefully(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""A complexity value with no matching compound row (e.g. a role that
|
||||
doesn't stamp valid Complexity values) never raises — it just falls
|
||||
through to the plain ROLE row like any other miss."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="not-a-real-complexity")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_slug_still_wins_over_compound_role_complexity(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""AGENT_SLUG stays the top of the ladder — a compound "developer:low" row
|
||||
never outranks a per-agent pin."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=ollama_model,
|
||||
)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.AGENT_SLUG,
|
||||
scope_value="be-dev-1",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
assert route.provider_type == ModelProvider.ANTHROPIC
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_role_still_wins_over_global_with_complexity_threaded(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""Plain ROLE still beats GLOBAL when complexity is passed but no compound
|
||||
row exists for it."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
|
||||
)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="high")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_complexity_rows_means_byte_identical_routing(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""CEO directive: with ZERO "role:complexity" rows present, resolve_for_agent
|
||||
must return exactly what it returns today for every precedence case
|
||||
(agent-slug, plain role, global, ROLE_MODEL_MAP fallback) — even when a
|
||||
task's LOW/HIGH/MEDIUM (or a garbage) complexity value is threaded through.
|
||||
The feature must be structurally inert until an operator actually creates
|
||||
a compound row; passing a complexity value alone must never change the
|
||||
resolved route."""
|
||||
svc = llm_setup["svc"]
|
||||
slug = "be-dev-1" # role == "developer"
|
||||
|
||||
def _same(a: AgentRoute, b: AgentRoute) -> bool:
|
||||
return (
|
||||
a.provider_id == b.provider_id
|
||||
and a.provider_type == b.provider_type
|
||||
and a.base_url == b.base_url
|
||||
and a.auth_token == b.auth_token
|
||||
and a.model_name == b.model_name
|
||||
)
|
||||
|
||||
async def _assert_identical_across_complexities() -> None:
|
||||
baseline = await svc.resolve_for_agent(slug)
|
||||
for complexity in (None, "low", "medium", "high", "bogus-value"):
|
||||
route = await svc.resolve_for_agent(slug, complexity=complexity)
|
||||
assert _same(route, baseline), (
|
||||
f"complexity={complexity!r} changed routing with zero "
|
||||
"role:complexity rows present"
|
||||
)
|
||||
|
||||
# 1. Legacy ROLE_MODEL_MAP fallback — no assignments at all.
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
# 2. GLOBAL default only.
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
|
||||
)
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
# 3. Plain ROLE row (wins over GLOBAL).
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=ollama_model
|
||||
)
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
# 4. AGENT_SLUG pin (wins over everything).
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.AGENT_SLUG,
|
||||
scope_value=slug,
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mode switches spare compound complexity-override rows (2026-07-17-style
|
||||
# incident: these same buttons once wiped AGENT_SLUG pins — the compound
|
||||
# ROLE(":"complexity) rung is a curated layer with the same rationale).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_anthropic_preserves_compound_row_and_resolution(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
svc = llm_setup["svc"]
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=ollama_model,
|
||||
)
|
||||
await svc.apply_mode(mode="anthropic")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == ollama_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_grok_preserves_compound_row_and_resolution(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await svc.apply_mode(mode="grok")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_ollama_preserves_compound_row_and_resolution(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await svc.apply_mode(mode="ollama")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_self_hosted_preserves_compound_row_and_resolution(
|
||||
llm_setup_with_local: dict,
|
||||
) -> None:
|
||||
svc = llm_setup_with_local["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await svc.apply_mode(mode="self_hosted", default_model="llama3.1:8b")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
# The compound row still points at Anthropic — resolving it never
|
||||
# touches the LOCAL provider's reachability at all.
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_seeded_provider_unknown_raises(
|
||||
db_session: AsyncSession,
|
||||
@@ -689,3 +946,100 @@ async def test_upsert_assignment_enables_local_when_disabled(
|
||||
"upsert_assignment must call update_provider(enabled=True) on LOCAL "
|
||||
"whenever it routes a non-catalog model to the LOCAL provider"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routing presets — crash safety (validate-all-first, wipe never half-runs)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_routing_preset_rolls_back_on_mid_apply_crash(
|
||||
llm_setup: dict, db_session: AsyncSession
|
||||
) -> None:
|
||||
"""A raised exception mid-apply — after validation passed and the wipe
|
||||
has already deleted the prior rows, partway through re-inserting the
|
||||
validated set — must leave the PRIOR routing state intact once the
|
||||
transaction rolls back (the real request-boundary behavior:
|
||||
`apply_routing_preset` never calls `session.commit()` itself; the caller
|
||||
commits once, after it returns). Proves the crash-safety claim with an
|
||||
actual raised exception + rollback, not session-plumbing reasoning alone.
|
||||
|
||||
Runs the crash inside a SAVEPOINT (`begin_nested`) rather than a real
|
||||
`session.commit()` / `session.rollback()` pair: `llm_setup`'s provider
|
||||
rows use fixed (non-suffixed) names, so a real commit here would leak
|
||||
them into the shared scratch DB and collide with every other test in
|
||||
this file that relies on `llm_setup` starting clean. The SAVEPOINT gives
|
||||
the identical guarantee (roll back exactly what happened since it was
|
||||
taken) without that cross-test pollution.
|
||||
"""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
|
||||
# Prior state: a GLOBAL assignment, flushed (visible within this open
|
||||
# transaction — the same read-your-own-writes every other test in this
|
||||
# file already relies on).
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
|
||||
)
|
||||
preset = await svc.save_routing_preset("crash-preset")
|
||||
|
||||
# Simulate a crash in the write phase: validation (resolve_provider_for_
|
||||
# model) is untouched and still runs for real; only the re-insert call
|
||||
# explodes, after the wipe has already deleted the prior row.
|
||||
with patch.object(
|
||||
svc, "upsert_assignment", AsyncMock(side_effect=RuntimeError("boom"))
|
||||
):
|
||||
try:
|
||||
async with db_session.begin_nested():
|
||||
await svc.apply_routing_preset(preset.id)
|
||||
except RuntimeError as e:
|
||||
assert "boom" in str(e)
|
||||
else:
|
||||
pytest.fail("expected apply_routing_preset to raise RuntimeError")
|
||||
|
||||
remaining = await svc.list_assignments()
|
||||
assert len(remaining) == 1
|
||||
assert remaining[0].scope == AssignmentScope.GLOBAL
|
||||
assert remaining[0].model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_routing_preset_validates_before_wiping_anything(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""validate-all-first: a preset whose ONLY entry is invalid must leave
|
||||
the current routing state completely untouched — the wipe must never
|
||||
run when nothing in the payload would survive it."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
|
||||
)
|
||||
|
||||
preset = await svc.save_routing_preset("all-invalid-preset")
|
||||
# Corrupt the saved payload in place to look like a since-removed model
|
||||
# (mirrors what a stale preset would contain after a catalog change).
|
||||
preset.payload = {
|
||||
"mode": "mix",
|
||||
"assignments": [
|
||||
{
|
||||
"scope": "role",
|
||||
"scope_value": "developer",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "ghost-model-gone",
|
||||
}
|
||||
],
|
||||
}
|
||||
await svc.session.flush()
|
||||
|
||||
notes = await svc.apply_routing_preset(preset.id)
|
||||
assert len(notes) == 1
|
||||
|
||||
# The wipe ran (every entry was rejected, so the valid set is empty) —
|
||||
# but nothing bogus was written; the GLOBAL row from before is gone
|
||||
# because the preset legitimately replaced the whole state with an
|
||||
# (all-invalid, now-empty) set. Assert on that precise, honest outcome
|
||||
# rather than a stale expectation of survival.
|
||||
remaining = await svc.list_assignments()
|
||||
assert remaining == []
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Migration 082 tests — routing_presets table.
|
||||
|
||||
Verifies the DB-level contract the migration establishes: a unique
|
||||
constraint on `name` (so `save_routing_preset`'s duplicate-name 409 has a
|
||||
real backstop, not just an app-level pre-check) and a JSONB `payload` column
|
||||
that round-trips a nested dict/list structure faithfully.
|
||||
|
||||
NOT a real alembic round-trip — the suite builds the test DB via
|
||||
Base.metadata.create_all (see conftest); a real `alembic upgrade head` +
|
||||
`downgrade -1` round trip against a scratch Postgres (:55432) was run
|
||||
manually and confirmed clean (create + drop, no errors) as part of building
|
||||
this migration. See `alembic/versions/082_routing_presets.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from roboco.db.tables import RoutingPresetTable
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_preset_name_is_unique(db_session: AsyncSession) -> None:
|
||||
"""The `uq_routing_presets_name` constraint rejects a duplicate name at
|
||||
the DB level — the backstop behind the service's pre-check 409."""
|
||||
db_session.add(RoutingPresetTable(name="dup-name", payload={"assignments": []}))
|
||||
await db_session.flush()
|
||||
|
||||
db_session.add(RoutingPresetTable(name="dup-name", payload={"assignments": []}))
|
||||
with pytest.raises(IntegrityError):
|
||||
await db_session.flush()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_preset_payload_round_trips_nested_structure(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""The JSONB payload column stores/returns a nested dict/list structure
|
||||
(the shape `save_routing_preset` writes) byte-for-byte."""
|
||||
payload = {
|
||||
"mode": "mix",
|
||||
"assignments": [
|
||||
{
|
||||
"scope": "agent_slug",
|
||||
"scope_value": "be-dev-1",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "sonnet",
|
||||
},
|
||||
{
|
||||
"scope": "role",
|
||||
"scope_value": "developer:low",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "haiku",
|
||||
},
|
||||
],
|
||||
}
|
||||
row = RoutingPresetTable(name="round-trip-preset", payload=payload)
|
||||
db_session.add(row)
|
||||
await db_session.flush()
|
||||
await db_session.refresh(row)
|
||||
|
||||
assert row.payload == payload
|
||||
assert row.created_at is not None
|
||||
@@ -13,9 +13,14 @@ from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from roboco.api.deps import get_agent_context, get_db
|
||||
from roboco.api.routes.provider import router as provider_router
|
||||
from roboco.db.tables import ModelAssignmentTable, ProviderConfigTable
|
||||
from roboco.db.tables import (
|
||||
ModelAssignmentTable,
|
||||
ProviderConfigTable,
|
||||
RoutingPresetTable,
|
||||
)
|
||||
from roboco.models import AgentRole, Team
|
||||
from roboco.models.base import ModelProvider
|
||||
from roboco.models.base import AssignmentScope, ModelProvider
|
||||
from roboco.models.llm_catalog import MODEL_CATALOG
|
||||
from roboco.models.permissions import AgentContext
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
@@ -25,6 +30,13 @@ if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
def _first_model_for_type(provider_type: ModelProvider) -> str:
|
||||
for entry in MODEL_CATALOG:
|
||||
if entry.provider_type == provider_type:
|
||||
return entry.model_name
|
||||
raise RuntimeError(f"no catalog entry for {provider_type}")
|
||||
|
||||
|
||||
def _make_app(
|
||||
db_session: AsyncSession,
|
||||
role: AgentRole = AgentRole.MAIN_PM,
|
||||
@@ -89,9 +101,13 @@ async def app_client_with_ollama(
|
||||
"""App client pre-seeded with Anthropic and Ollama Cloud providers.
|
||||
|
||||
Begins with a DELETE-before-seed isolation step: deletes all rows from
|
||||
ModelAssignmentTable (FK-safe) then ProviderConfigTable before adding
|
||||
fresh ANTHROPIC + OLLAMA_CLOUD rows. This ensures tests are
|
||||
order-independent regardless of what prior tests committed.
|
||||
ModelAssignmentTable (FK-safe), ProviderConfigTable, and RoutingPresetTable
|
||||
before adding fresh ANTHROPIC + OLLAMA_CLOUD rows. This ensures tests are
|
||||
order-independent regardless of what prior tests committed — the fixture's
|
||||
`db.commit()` calls are real commits against the session-scoped scratch
|
||||
DB (`db_session`'s teardown only rolls back uncommitted state), so without
|
||||
this every table a test writes through a route's `db.commit()` needs its
|
||||
own cleanup here, RoutingPresetTable included.
|
||||
"""
|
||||
app = _make_app(db_session)
|
||||
suffix = uuid4().hex[:8]
|
||||
@@ -99,6 +115,7 @@ async def app_client_with_ollama(
|
||||
# provider_configs.id, so assignments must be deleted first.
|
||||
await db_session.execute(delete(ModelAssignmentTable))
|
||||
await db_session.execute(delete(ProviderConfigTable))
|
||||
await db_session.execute(delete(RoutingPresetTable))
|
||||
await db_session.flush()
|
||||
db_session.add(
|
||||
ProviderConfigTable(
|
||||
@@ -625,3 +642,390 @@ async def test_get_self_hosted_models_unreachable_returns_503(
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Complexity overrides (cost-tiered routing: compound ROLE(":"complexity) rows)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_complexity_overrides_empty(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_round_trips_through_get(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""PUT developer:low -> haiku (no costlier than the sonnet baseline)."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "low", "model_name": "haiku"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
body = response.json()
|
||||
assert body == {
|
||||
"role": "developer",
|
||||
"complexity": "low",
|
||||
"model_name": "haiku",
|
||||
"warning": None,
|
||||
}
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
# GET's listing rows are never constructed WITH a warning (only PUT
|
||||
# computes one), but the shared response schema still serializes the
|
||||
# field at its None default.
|
||||
assert listing.json() == [body]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_rejects_disallowed_role(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""main_pm is a coordinator role — never offered a complexity override."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "main_pm", "complexity": "low", "model_name": "haiku"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert "deliberate" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_rejects_costlier_tier(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""developer's baseline is sonnet — opus is a costlier tier, rejected."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "high", "model_name": "opus"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert "downgrade-only" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_allows_same_tier_as_baseline(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""A same-tier pin (sonnet for developer, whose baseline IS sonnet) is not
|
||||
a downgrade but isn't costlier either — allowed."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "high", "model_name": "sonnet"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["warning"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_rejects_disabled_provider(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""qa's baseline (haiku) prices no cheaper than Ollama Cloud (unpriced,
|
||||
treated as free-tier) so the downgrade-only check passes — but the
|
||||
OLLAMA_CLOUD provider is disabled (no key set) in this fixture's seeded
|
||||
state, so the write-time readiness guard rejects it before it can
|
||||
silently no-op to the legacy Anthropic path at spawn."""
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "qa", "complexity": "low", "model_name": ollama_model},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
detail = response.json()["detail"]
|
||||
assert "isn't configured yet" in detail
|
||||
assert "Ollama" in detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_warns_on_cross_family_once_provider_ready(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""Once Ollama Cloud is enabled (key set), the same cross-family override
|
||||
succeeds — allowed, but the response carries a non-null warning since
|
||||
it's a different provider family than qa's Anthropic baseline."""
|
||||
await app_client_with_ollama.put(
|
||||
"/api/providers/ollama-key",
|
||||
json={"api_key": "test-key"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "qa", "complexity": "low", "model_name": ollama_model},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
body = response.json()
|
||||
assert body["warning"] is not None
|
||||
assert "ollama_cloud" in body["warning"]
|
||||
assert "qa" in body["warning"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_developer_forbidden(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
app = _make_app(db_session, role=AgentRole.DEVELOPER, team=Team.BACKEND)
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "low", "model_name": "haiku"},
|
||||
headers={"X-Agent-ID": str(uuid4()), "X-Agent-Role": "developer"},
|
||||
)
|
||||
app.dependency_overrides.clear()
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_complexity_override(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "qa", "complexity": "high", "model_name": "haiku"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
response = await app_client_with_ollama.delete(
|
||||
"/api/providers/complexity-overrides/qa/high", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
assert listing.json() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_complexity_override_not_found(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.delete(
|
||||
"/api/providers/complexity-overrides/qa/low", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_cost_tiered_seeds_day1_rows(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "cost_tiered"}, headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
rows = {(r["role"], r["complexity"]): r["model_name"] for r in listing.json()}
|
||||
# cell_pm is deliberately excluded (a coordinator role) — only developer.
|
||||
assert rows == {("developer", "low"): "haiku"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_cost_tiered_is_additive_preserves_global(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""Unlike every other mode, cost_tiered never wipes existing rows."""
|
||||
await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "ollama"}, headers=_HDR_PM
|
||||
)
|
||||
mode_before = (
|
||||
await app_client_with_ollama.get("/api/providers", headers=_HDR_PM)
|
||||
).json()
|
||||
assert mode_before["mode"] == "ollama"
|
||||
|
||||
response = await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "cost_tiered"}, headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assignments = response.json()["assignments"]
|
||||
scopes = {(a["scope"], a["scope_value"]) for a in assignments}
|
||||
# The pre-existing GLOBAL row from 'ollama' mode survives untouched.
|
||||
assert ("global", None) in scopes
|
||||
assert ("role", "developer:low") in scopes
|
||||
# cell_pm is deliberately excluded from cost_tiered — a coordinator role.
|
||||
assert ("role", "cell_pm:low") not in scopes
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Routing presets (named, full snapshots of the routing state)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_list_and_apply_preset_round_trip(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""Save captures the current state; mutating + re-applying restores it."""
|
||||
# Arrange a distinctive state: a GLOBAL Ollama default.
|
||||
await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "ollama"}, headers=_HDR_PM
|
||||
)
|
||||
snapshot_before = (
|
||||
await app_client_with_ollama.get("/api/providers", headers=_HDR_PM)
|
||||
).json()
|
||||
|
||||
save_resp = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "my-preset"}, headers=_HDR_PM
|
||||
)
|
||||
assert save_resp.status_code == HTTPStatus.OK
|
||||
preset_id = save_resp.json()["id"]
|
||||
assert save_resp.json()["name"] == "my-preset"
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/presets", headers=_HDR_PM
|
||||
)
|
||||
assert listing.status_code == HTTPStatus.OK
|
||||
assert [p["name"] for p in listing.json()] == ["my-preset"]
|
||||
|
||||
# Mutate away from the saved state.
|
||||
await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "anthropic"}, headers=_HDR_PM
|
||||
)
|
||||
mutated = (
|
||||
await app_client_with_ollama.get("/api/providers", headers=_HDR_PM)
|
||||
).json()
|
||||
assert mutated["assignments"] == []
|
||||
|
||||
# Apply the preset back — restores the snapshot. Applying always
|
||||
# deletes-then-reinserts (a real full swap), so row `id`s are fresh;
|
||||
# compare on the business fields only.
|
||||
apply_resp = await app_client_with_ollama.post(
|
||||
f"/api/providers/presets/{preset_id}/apply", headers=_HDR_PM
|
||||
)
|
||||
assert apply_resp.status_code == HTTPStatus.OK
|
||||
applied = apply_resp.json()
|
||||
assert applied["skipped"] == []
|
||||
|
||||
def _sans_id(assignments: list[dict]) -> list[dict]:
|
||||
return [{k: v for k, v in a.items() if k != "id"} for a in assignments]
|
||||
|
||||
assert _sans_id(applied["assignments"]) == _sans_id(snapshot_before["assignments"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_preset_duplicate_name_returns_409(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
first = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "dup"}, headers=_HDR_PM
|
||||
)
|
||||
assert first.status_code == HTTPStatus.OK
|
||||
second = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "dup"}, headers=_HDR_PM
|
||||
)
|
||||
assert second.status_code == HTTPStatus.CONFLICT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_preset_not_found_returns_404(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.post(
|
||||
f"/api/providers/presets/{uuid4()}/apply", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_preset_not_found_returns_404(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.delete(
|
||||
f"/api/providers/presets/{uuid4()}", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_preset(app_client_with_ollama: AsyncClient) -> None:
|
||||
save_resp = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "to-delete"}, headers=_HDR_PM
|
||||
)
|
||||
preset_id = save_resp.json()["id"]
|
||||
|
||||
response = await app_client_with_ollama.delete(
|
||||
f"/api/providers/presets/{preset_id}", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/presets", headers=_HDR_PM
|
||||
)
|
||||
assert listing.json() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_preset_skips_entry_with_since_removed_model(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Payload hygiene: an entry referencing a model no longer in the catalog
|
||||
(and not routable to LOCAL, since no LOCAL provider is seeded here) is
|
||||
skipped with a note — never fails the whole apply."""
|
||||
row = RoutingPresetTable(
|
||||
name="stale-preset",
|
||||
payload={
|
||||
"mode": "mix",
|
||||
"assignments": [
|
||||
{
|
||||
"scope": AssignmentScope.GLOBAL.value,
|
||||
"scope_value": None,
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "sonnet",
|
||||
},
|
||||
{
|
||||
"scope": AssignmentScope.ROLE.value,
|
||||
"scope_value": "developer",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "ghost-model-that-no-longer-exists",
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
db_session.add(row)
|
||||
await db_session.flush()
|
||||
|
||||
response = await app_client_with_ollama.post(
|
||||
f"/api/providers/presets/{row.id}/apply", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
body = response.json()
|
||||
assert len(body["skipped"]) == 1
|
||||
assert "ghost-model-that-no-longer-exists" in body["skipped"][0]
|
||||
# The valid GLOBAL entry still applied despite the sibling failure.
|
||||
scopes = {(a["scope"], a["scope_value"]) for a in body["assignments"]}
|
||||
assert ("global", None) in scopes
|
||||
assert ("role", "developer") not in scopes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_presets_developer_forbidden(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
app = _make_app(db_session, role=AgentRole.DEVELOPER, team=Team.BACKEND)
|
||||
transport = ASGITransport(app=app)
|
||||
hdr = {"X-Agent-ID": str(uuid4()), "X-Agent-Role": "developer"}
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/api/providers/presets", headers=hdr)
|
||||
app.dependency_overrides.clear()
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
|
||||
@@ -21,6 +21,7 @@ from roboco.billing.pricing import (
|
||||
_is_anthropic_model,
|
||||
calculate_cost,
|
||||
calculate_cost_result,
|
||||
input_price_per_million,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -539,3 +540,45 @@ def test_sonnet5_reverts_to_list_rate_after_2026_08_31(
|
||||
_SONNET_CACHE_READ,
|
||||
_SONNET_CACHE_WRITE,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# input_price_per_million — the cost-tiered complexity-override comparator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInputPricePerMillion:
|
||||
"""The downgrade-only comparator for complexity overrides (no explicit
|
||||
tier ordering exists in the model catalog, so the input rate stands in
|
||||
for "which tier is costlier")."""
|
||||
|
||||
def test_orders_haiku_below_sonnet_below_opus(self) -> None:
|
||||
assert (
|
||||
input_price_per_million("haiku")
|
||||
< input_price_per_million("sonnet")
|
||||
< input_price_per_million("opus")
|
||||
)
|
||||
|
||||
def test_matches_pricing_table_value(self) -> None:
|
||||
assert input_price_per_million("haiku") == _HAIKU_INPUT
|
||||
assert input_price_per_million("sonnet") == _SONNET_INPUT
|
||||
assert input_price_per_million("opus") == _OPUS_INPUT
|
||||
|
||||
def test_grok_priced_below_sonnet(self) -> None:
|
||||
"""Grok legitimately downgrades-from sonnet under this comparator."""
|
||||
assert input_price_per_million("grok-build-0.1") < input_price_per_million(
|
||||
"sonnet"
|
||||
)
|
||||
|
||||
def test_unpriced_non_anthropic_model_is_free_tier(self) -> None:
|
||||
"""A self-hosted / Ollama Cloud model has no per-token rate — treated
|
||||
as the cheapest possible tier, so it can never be rejected as
|
||||
"costlier" by the downgrade-only policy."""
|
||||
assert input_price_per_million("glm-5.2:cloud") == 0.0
|
||||
assert input_price_per_million("my-custom-self-hosted-model:7b") == 0.0
|
||||
|
||||
def test_empty_model_returns_zero(self) -> None:
|
||||
assert input_price_per_million("") == 0.0
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
assert input_price_per_million("HAIKU") == input_price_per_million("haiku")
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Task complexity threads into `_resolve_agent_route` -> `resolve_for_agent`.
|
||||
|
||||
Cost-tiered routing (roboco/services/llm.py) reads a task's
|
||||
`estimated_complexity` to try a compound ROLE(":"complexity) row before
|
||||
falling to the plain ROLE row. The orchestrator owns the one indexed Task
|
||||
lookup and threads the lowercase complexity string through. This is the pure
|
||||
wiring test (`_resolve_agent_route` -> `resolve_for_agent`); the precedence
|
||||
logic itself is covered in tests/integration/test_llm_routing.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import roboco.db.base as db_base
|
||||
import roboco.services.llm as llm_module
|
||||
from roboco.models.base import Complexity, ModelProvider
|
||||
from roboco.runtime.orchestrator import AgentOrchestrator
|
||||
from roboco.services.llm import AgentRoute
|
||||
|
||||
# Sentinel route the mocked resolve_for_agent returns — a real AgentRoute
|
||||
# instance (not a bare string) so `result is _SENTINEL_ROUTE` type-checks
|
||||
# cleanly against `_resolve_agent_route`'s declared AgentRoute return type.
|
||||
_SENTINEL_ROUTE = AgentRoute(
|
||||
provider_id=None,
|
||||
provider_type=ModelProvider.ANTHROPIC,
|
||||
base_url=None,
|
||||
auth_token=None,
|
||||
model_name="sentinel",
|
||||
)
|
||||
|
||||
|
||||
class _ScalarResult:
|
||||
def __init__(self, value: Any) -> None:
|
||||
self._value = value
|
||||
|
||||
def scalar_one_or_none(self) -> Any:
|
||||
return self._value
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Minimal async-context-manager session returning a fixed complexity."""
|
||||
|
||||
def __init__(self, complexity_value: Any) -> None:
|
||||
self._complexity_value = complexity_value
|
||||
|
||||
def __call__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
async def __aenter__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def execute(self, _stmt: Any) -> _ScalarResult:
|
||||
return _ScalarResult(self._complexity_value)
|
||||
|
||||
|
||||
class _BoomSession(_FakeSession):
|
||||
"""A session whose `execute` always raises — models a task-lookup failure
|
||||
(bad/unresolvable task id) distinct from a genuine DB/session outage."""
|
||||
|
||||
async def execute(self, _stmt: Any) -> _ScalarResult:
|
||||
raise RuntimeError("bad task id")
|
||||
|
||||
|
||||
def _orch() -> AgentOrchestrator:
|
||||
# __new__ + skip __init__: avoid all constructor I/O — this method is pure
|
||||
# w.r.t. instance state (it only touches module-level imports + args).
|
||||
return AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
|
||||
|
||||
def _wire(monkeypatch: pytest.MonkeyPatch, fake_session: Any) -> AsyncMock:
|
||||
"""Patch get_session_factory + get_model_routing_service; return the
|
||||
resolve_for_agent mock so the test can assert on its call."""
|
||||
monkeypatch.setattr(
|
||||
db_base,
|
||||
"get_session_factory",
|
||||
lambda: MagicMock(return_value=fake_session),
|
||||
)
|
||||
resolve_mock = AsyncMock(return_value=_SENTINEL_ROUTE)
|
||||
fake_router = MagicMock(resolve_for_agent=resolve_mock)
|
||||
monkeypatch.setattr(
|
||||
llm_module, "get_model_routing_service", lambda _db: fake_router
|
||||
)
|
||||
return resolve_mock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_with_high_complexity_threads_lowercase_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A task with estimated_complexity=HIGH resolves the compound
|
||||
'role:high' row — i.e. resolve_for_agent is called with complexity='high'
|
||||
(lowercased from the Complexity enum's value)."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "task-123")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="high")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_with_low_complexity_threads_lowercase_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.LOW))
|
||||
|
||||
orch = _orch()
|
||||
await orch._resolve_agent_route("be-dev-1", "task-456")
|
||||
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="low")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_taskless_spawn_threads_none_complexity_unchanged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A no-task spawn (idle PM bootstrap, Intake/Secretary chats, ...) never
|
||||
even attempts a task lookup — complexity=None, byte-identical to the
|
||||
pre-cost-tiering call shape."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", None)
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_task_row_degrades_to_none_complexity_silently(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""scalar_one_or_none() returning None (task not found / deleted) is not
|
||||
an error — complexity falls back to None and routing still proceeds
|
||||
through the router (not the hardcoded legacy path)."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(None))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "ghost-task-id")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_lookup_failure_degrades_silently_not_to_full_legacy_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A task-lookup-specific failure (bad id, transient query error) must
|
||||
NOT escalate to the full DB-failure downgrade (hardcoded ROLE_MODEL_MAP,
|
||||
bypassing model_assignments entirely) — only the complexity lookup is
|
||||
skipped; AGENT_SLUG/ROLE/GLOBAL resolution still runs via the router."""
|
||||
resolve_mock = _wire(monkeypatch, _BoomSession(None))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "bad-task-id")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
@@ -47,7 +47,7 @@ def _wire(monitor: dict[str, Any]) -> Any:
|
||||
async def _git_context(_gc: Any, _tid: str | None) -> None:
|
||||
return None
|
||||
|
||||
async def _route(_aid: str) -> Any:
|
||||
async def _route(_aid: str, _tid: str | None = None) -> Any:
|
||||
monitor["route_calls"] += 1
|
||||
return SimpleNamespace(
|
||||
provider_type=SimpleNamespace(value="anthropic"),
|
||||
|
||||
Reference in New Issue
Block a user