Files
roboco/tests/integration/test_llm_routing.py
T

1379 lines
51 KiB
Python

"""ModelRoutingService coverage — assignment CRUD + mode application + resolution."""
from __future__ import annotations
from typing import TYPE_CHECKING, cast
from unittest.mock import AsyncMock, patch
import pytest
import pytest_asyncio
from roboco.db.tables import ProviderConfigTable
from roboco.models.base import AssignmentScope, ModelProvider
from roboco.models.llm_catalog import MODEL_CATALOG
from roboco.services.base import NotFoundError
from roboco.services.llm import ModelRoutingService, get_model_routing_service
from roboco.services.provider import ProviderService, ProviderUpdate
from roboco.utils.crypto import EncryptionError
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from uuid import UUID
from roboco.services.llm import AgentRoute
from sqlalchemy.ext.asyncio import AsyncSession
def _first_model_for_type(provider_type: ModelProvider) -> str:
for entry in MODEL_CATALOG:
if entry.provider_type == provider_type:
return entry.model_name
raise RuntimeError(f"no catalog entry for {provider_type}")
@pytest_asyncio.fixture
async def llm_setup(
db_session: AsyncSession,
) -> AsyncIterator[dict]:
"""Seed the two provider rows the routing service expects."""
anthropic = ProviderConfigTable(
name="anthropic-test",
type=ModelProvider.ANTHROPIC,
enabled=True,
)
grok = ProviderConfigTable(
name="grok-test",
type=ModelProvider.GROK,
enabled=True,
base_url="https://api.x.ai/v1",
)
ollama = ProviderConfigTable(
name="ollama-test",
type=ModelProvider.OLLAMA_CLOUD,
enabled=True,
base_url="https://ollama.example.com",
)
# Mirrors migration 083_seed_openai_provider's contract: enabled=True at
# seed time.
openai = ProviderConfigTable(
name="openai-test",
type=ModelProvider.OPENAI,
enabled=True,
base_url="https://api.openai.com/v1",
)
# Mirrors the post-086 seeded state (085 seeds enabled=false, 086 flips it
# true to match Codex) — no base_url, subscription OAuth auth only.
gemini = ProviderConfigTable(
name="gemini-test",
type=ModelProvider.GEMINI,
enabled=True,
)
db_session.add_all([anthropic, grok, ollama, openai, gemini])
await db_session.flush()
yield {"svc": ModelRoutingService(db_session)}
# ---------------------------------------------------------------------------
# Assignment CRUD
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_list_assignments_empty(llm_setup: dict) -> None:
svc = llm_setup["svc"]
assert await svc.list_assignments() == []
@pytest.mark.asyncio
async def test_upsert_global_assignment(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
row = await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=model
)
assert row.scope == AssignmentScope.GLOBAL
assert row.model_name == model
@pytest.mark.asyncio
async def test_upsert_role_assignment(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
row = await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value="developer", model_name=model
)
assert row.scope == AssignmentScope.ROLE
assert row.scope_value == "developer"
@pytest.mark.asyncio
async def test_upsert_replaces_existing(llm_setup: dict) -> None:
svc = llm_setup["svc"]
anth_model = _first_model_for_type(ModelProvider.ANTHROPIC)
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
a = await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anth_model
)
b = await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
)
assert a.id == b.id # Updated, not duplicated.
assert b.model_name == ollama_model
@pytest.mark.asyncio
async def test_upsert_unknown_model_raises(llm_setup: dict) -> None:
svc = llm_setup["svc"]
with pytest.raises(ValueError, match="Unknown model"):
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL,
scope_value=None,
model_name="ghost-model",
)
@pytest.mark.asyncio
async def test_upsert_invalid_global_scope_raises(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
with pytest.raises(ValueError, match="global scope must"):
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value="not-none", model_name=model
)
@pytest.mark.asyncio
async def test_upsert_role_requires_value(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
with pytest.raises(ValueError, match="requires a non-empty"):
await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value=None, model_name=model
)
@pytest.mark.asyncio
async def test_get_assignment_returns_none_when_missing(llm_setup: dict) -> None:
svc = llm_setup["svc"]
assert (
await svc.get_assignment(scope=AssignmentScope.AGENT_SLUG, scope_value="ghost")
is None
)
@pytest.mark.asyncio
async def test_delete_assignment(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=model
)
await svc.delete_assignment(scope=AssignmentScope.GLOBAL, scope_value=None)
assert (
await svc.get_assignment(scope=AssignmentScope.GLOBAL, scope_value=None) is None
)
@pytest.mark.asyncio
async def test_delete_assignment_raises_when_missing(llm_setup: dict) -> None:
svc = llm_setup["svc"]
with pytest.raises(NotFoundError):
await svc.delete_assignment(scope=AssignmentScope.GLOBAL, scope_value=None)
# ---------------------------------------------------------------------------
# derive_mode
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_derive_mode_anthropic_when_no_assignments(llm_setup: dict) -> None:
svc = llm_setup["svc"]
assert await svc.derive_mode() == "anthropic"
@pytest.mark.asyncio
async def test_derive_mode_ollama_when_only_ollama_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
)
assert await svc.derive_mode() == "ollama"
@pytest.mark.asyncio
async def test_derive_mode_grok_when_only_grok_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
grok_model = _first_model_for_type(ModelProvider.GROK)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=grok_model
)
assert await svc.derive_mode() == "grok"
@pytest.mark.asyncio
async def test_derive_mode_codex_when_only_openai_global(llm_setup: dict) -> None:
"""A pure-OPENAI global assignment reports "codex", not the catch-all
"mix" — the read-only branch derive_mode gained alongside the seed fix."""
svc = llm_setup["svc"]
codex_model = _first_model_for_type(ModelProvider.OPENAI)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=codex_model
)
assert await svc.derive_mode() == "codex"
@pytest.mark.asyncio
async def test_derive_mode_gemini_when_only_gemini_global(llm_setup: dict) -> None:
"""A pure-GEMINI global assignment reports "gemini", not the catch-all
"mix" — mirrors the codex branch derive_mode already carries."""
svc = llm_setup["svc"]
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=gemini_model
)
assert await svc.derive_mode() == "gemini"
@pytest.mark.asyncio
async def test_derive_mode_mix_with_per_agent(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name=model,
)
assert await svc.derive_mode() == "mix"
# ---------------------------------------------------------------------------
# apply_mode
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_apply_mode_anthropic_clears_all(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=model
)
await svc.apply_mode(mode="anthropic")
assert await svc.list_assignments() == []
@pytest.mark.asyncio
async def test_apply_mode_anthropic_preserves_agent_pin(llm_setup: dict) -> None:
"""A mode switch must not wipe per-agent pins — only role/global rows."""
svc = llm_setup["svc"]
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name=ollama_model,
)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
)
await svc.apply_mode(mode="anthropic")
assignments = await svc.list_assignments()
assert len(assignments) == 1 # GLOBAL row cleared, AGENT_SLUG pin survives.
assert assignments[0].scope == AssignmentScope.AGENT_SLUG
assert assignments[0].scope_value == "be-dev-1"
assert assignments[0].model_name == ollama_model
@pytest.mark.asyncio
async def test_apply_mode_ollama_sets_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.apply_mode(mode="ollama", default_model=ollama_model)
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
@pytest.mark.asyncio
async def test_apply_mode_ollama_preserves_agent_pin(llm_setup: dict) -> None:
"""The Ollama mode-switch button must not wipe per-agent model pins."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name=anthropic_model,
)
await svc.apply_mode(mode="ollama", default_model=ollama_model)
assignments = await svc.list_assignments()
assert len(assignments) == 2 # noqa: PLR2004 AGENT_SLUG pin kept + new GLOBAL row.
by_scope = {a.scope: a for a in assignments}
assert by_scope[AssignmentScope.AGENT_SLUG].scope_value == "be-dev-1"
assert by_scope[AssignmentScope.AGENT_SLUG].model_name == anthropic_model
assert by_scope[AssignmentScope.GLOBAL].model_name == ollama_model
@pytest.mark.asyncio
async def test_apply_mode_grok_sets_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
await svc.apply_mode(mode="grok")
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
assert assignments[0].provider.type == ModelProvider.GROK
@pytest.mark.asyncio
async def test_apply_mode_grok_enables_grok_provider(llm_setup: dict) -> None:
"""apply_mode('grok') enables the GROK provider row so resolve_for_agent
routes to the GrokCliProvider (SuperGrok auth, independent of the xAI key).
Mirrors self_hosted enabling LOCAL."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
grok = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GROK
)
# Model the real seeded state: GROK disabled because no xAI key was set.
await provider_svc.update_provider(
cast("UUID", grok.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
await svc.apply_mode(mode="grok")
refetched = await provider_svc.get_provider(cast("UUID", grok.id))
assert refetched is not None
assert refetched.enabled is True
@pytest.mark.asyncio
async def test_apply_mode_codex_sets_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
await svc.apply_mode(mode="codex")
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
assert assignments[0].provider.type == ModelProvider.OPENAI
assert assignments[0].model_name == "gpt-5.3-codex"
@pytest.mark.asyncio
async def test_apply_mode_codex_enables_openai_provider(llm_setup: dict) -> None:
"""apply_mode('codex') force-enables the OPENAI row — belt-and-suspenders
alongside migration 083's own enabled=true seed."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
openai = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.OPENAI
)
await provider_svc.update_provider(
cast("UUID", openai.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
await svc.apply_mode(mode="codex")
refetched = await provider_svc.get_provider(cast("UUID", openai.id))
assert refetched is not None
assert refetched.enabled is True
@pytest.mark.asyncio
async def test_apply_mode_gemini_sets_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
await svc.apply_mode(mode="gemini")
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
assert assignments[0].provider.type == ModelProvider.GEMINI
assert assignments[0].model_name == "gemini-2.5-pro"
@pytest.mark.asyncio
async def test_apply_mode_gemini_enables_gemini_provider(llm_setup: dict) -> None:
"""apply_mode('gemini') force-enables the GEMINI row — the exact gap this
fix closes (migration 085 seeds it disabled and nothing else ever flipped
it before this write path + migration 086 existed)."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
gemini = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GEMINI
)
await provider_svc.update_provider(
cast("UUID", gemini.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
await svc.apply_mode(mode="gemini")
refetched = await provider_svc.get_provider(cast("UUID", gemini.id))
assert refetched is not None
assert refetched.enabled is True
@pytest.mark.asyncio
async def test_apply_mode_mix_requires_per_agent(llm_setup: dict) -> None:
svc = llm_setup["svc"]
with pytest.raises(ValueError, match="requires a per_agent"):
await svc.apply_mode(mode="mix")
@pytest.mark.asyncio
async def test_apply_mode_mix_empty_map_clears_all_pins(llm_setup: dict) -> None:
"""An EMPTY per_agent map is the explicit clear-all — the only way out of
a fully-pinned fleet, since mode switches deliberately spare pins (the
2026-07-23 live incident: 25/25 pins made every mode button a no-op)."""
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.apply_mode(mode="mix", per_agent={"be-dev-1": model, "be-qa": model})
assert any(
r.scope == AssignmentScope.AGENT_SLUG for r in await svc.list_assignments()
)
await svc.apply_mode(mode="mix", per_agent={})
rows = await svc.list_assignments()
assert not any(r.scope == AssignmentScope.AGENT_SLUG for r in rows)
# Nothing left at all -> the mode label escapes "mix".
assert await svc.derive_mode() == "anthropic"
@pytest.mark.asyncio
async def test_apply_mode_mix_writes_overrides(llm_setup: dict) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.apply_mode(
mode="mix",
per_agent={"be-dev-1": model, "be-dev-2": model, "skip-me": ""},
)
rows = await svc.list_assignments()
slugs = {r.scope_value for r in rows if r.scope == AssignmentScope.AGENT_SLUG}
assert "be-dev-1" in slugs
assert "be-dev-2" in slugs
# Empty model_name skipped.
assert "skip-me" not in slugs
@pytest.mark.asyncio
async def test_apply_mode_unknown_raises(llm_setup: dict) -> None:
svc = llm_setup["svc"]
with pytest.raises(ValueError, match="Unknown mode"):
await svc.apply_mode(mode="quantum")
# ---------------------------------------------------------------------------
# Ollama API key
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_set_ollama_api_key_encrypts(llm_setup: dict) -> None:
svc = llm_setup["svc"]
provider = await svc.set_ollama_api_key("secret-key-123")
assert provider.auth_token_encrypted is not None
assert provider.enabled is True
@pytest.mark.asyncio
async def test_set_ollama_api_key_clears(llm_setup: dict) -> None:
svc = llm_setup["svc"]
await svc.set_ollama_api_key("secret-key-123")
cleared = await svc.set_ollama_api_key("")
assert cleared.auth_token_encrypted is None
assert cleared.enabled is False
# ---------------------------------------------------------------------------
# Resolution
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_resolve_for_agent_legacy_fallback_when_no_assignments(
llm_setup: dict,
) -> None:
"""No assignments → Anthropic with a model from MODEL_MAP, no auth_token."""
svc = llm_setup["svc"]
route = await svc.resolve_for_agent("be-dev-1")
assert route.provider_type == ModelProvider.ANTHROPIC
assert route.auth_token is None # Container uses mounted creds.
assert route.model_name # Always resolved.
@pytest.mark.asyncio
async def test_resolve_for_agent_uses_global_assignment(
llm_setup: dict,
) -> None:
svc = llm_setup["svc"]
model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=model
)
route = await svc.resolve_for_agent("be-dev-1")
assert route.model_name == model
@pytest.mark.asyncio
async def test_upsert_and_resolve_openai_assignment_roundtrip(
llm_setup: dict,
) -> None:
"""gpt-5.3-codex through upsert_assignment -> resolve_for_agent, against
the seeded OPENAI row (migration 083). Before that seed existed,
upsert_assignment's `_get_seeded_provider(ModelProvider.OPENAI)` lookup
raised NotFoundError the moment anyone tried this — this is the round
trip that would have caught it."""
svc = llm_setup["svc"]
codex_model = _first_model_for_type(ModelProvider.OPENAI)
row = await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name=codex_model,
)
assert row.model_name == codex_model
route = await svc.resolve_for_agent("be-dev-1")
assert route.provider_type == ModelProvider.OPENAI
assert route.model_name == codex_model
# The seeded row carries no stored token — Codex authenticates via the
# mounted ~/.codex subscription dir, not a decrypted provider token.
assert route.auth_token is None
@pytest.mark.asyncio
async def test_upsert_and_resolve_gemini_assignment_roundtrip(
llm_setup: dict,
) -> None:
"""gemini-2.5-pro through upsert_assignment -> resolve_for_agent, against
the seeded GEMINI row. Proves resolve_for_agent actually returns a GEMINI
spawn route — not a silent Anthropic fallback — the exact gap left open
by the row seeding disabled with no enable path (migration 085 alone)."""
svc = llm_setup["svc"]
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
row = await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="ux-dev-1",
model_name=gemini_model,
)
assert row.model_name == gemini_model
route = await svc.resolve_for_agent("ux-dev-1")
assert route.provider_type == ModelProvider.GEMINI
assert route.model_name == gemini_model
# Subscription OAuth auth (~/.gemini), not a decrypted provider token.
assert route.auth_token is None
@pytest.mark.asyncio
async def test_upsert_assignment_enables_disabled_gemini_provider(
llm_setup: dict,
) -> None:
"""Belt-and-suspenders: assigning a Gemini model via Mix (upsert_assignment)
force-enables the row even if it was disabled — not just apply_mode('gemini')."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
gemini = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GEMINI
)
await provider_svc.update_provider(
cast("UUID", gemini.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="ux-dev-1",
model_name=gemini_model,
)
refetched = await provider_svc.get_provider(cast("UUID", gemini.id))
assert refetched is not None
assert refetched.enabled is True
# And the route actually resolves to GEMINI now that it's enabled.
route = await svc.resolve_for_agent("ux-dev-1")
assert route.provider_type == ModelProvider.GEMINI
@pytest.mark.asyncio
async def test_apply_mode_gemini_end_to_end_reachable(llm_setup: dict) -> None:
"""The full reachability chain the original drill missed: apply_mode
-> derive_mode reflects it -> resolve_for_agent actually spawns Gemini."""
svc = llm_setup["svc"]
await svc.apply_mode(mode="gemini")
assert await svc.derive_mode() == "gemini"
route = await svc.resolve_for_agent("ux-dev-1")
assert route.provider_type == ModelProvider.GEMINI
assert route.model_name == "gemini-2.5-pro"
@pytest.mark.asyncio
async def test_apply_mode_codex_end_to_end_reachable(llm_setup: dict) -> None:
"""Same reachability chain for Codex, mirroring the Gemini test above."""
svc = llm_setup["svc"]
await svc.apply_mode(mode="codex")
assert await svc.derive_mode() == "codex"
route = await svc.resolve_for_agent("be-dev-1")
assert route.provider_type == ModelProvider.OPENAI
assert route.model_name == "gpt-5.3-codex"
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["codex", "gemini"])
@pytest.mark.parametrize("interactive_slug", ["intake-1", "secretary-1"])
async def test_interactive_agents_exempt_from_delivery_only_global_mode(
llm_setup: dict, mode: str, interactive_slug: str
) -> None:
"""A fleet-wide Codex/Gemini mode must not capture Intake/Secretary —
they have no V1 support on those providers, so the resolver keeps them
on the legacy Anthropic path (the completeness-drill gap: previously
they resolved to the unsupported provider and the spawn guard left both
chats refusing to start after a one-click mode switch)."""
svc = llm_setup["svc"]
await svc.apply_mode(mode=mode)
# The mode still derives cleanly (single GLOBAL row — no extra pins).
assert await svc.derive_mode() == mode
route = await svc.resolve_for_agent(interactive_slug)
assert route.provider_type == ModelProvider.ANTHROPIC
@pytest.mark.asyncio
async def test_interactive_agent_explicit_pin_is_not_exempted(
llm_setup: dict,
) -> None:
"""An EXPLICIT AGENT_SLUG pin to a delivery-only provider is honored by
the resolver (the orchestrator's spawn guard refuses it loudly) — a
deliberate operator choice must error, never be silently overridden."""
svc = llm_setup["svc"]
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="intake-1",
model_name="gpt-5.3-codex",
)
route = await svc.resolve_for_agent("intake-1")
assert route.provider_type == ModelProvider.OPENAI
@pytest.mark.asyncio
async def test_resolve_for_agent_uses_provider_token(llm_setup: dict) -> None:
"""When provider has auth_token_encrypted, it's decrypted (lines 345-346)."""
svc = llm_setup["svc"]
await svc.set_ollama_api_key("test-secret-key")
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
)
route = await svc.resolve_for_agent("be-dev-1")
assert route.auth_token == "test-secret-key"
# ---------------------------------------------------------------------------
# Cost-tiered compound ROLE(":"complexity) rung
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_resolve_for_agent_uses_compound_role_complexity_assignment(
llm_setup: dict,
) -> None:
"""A "developer:low" compound row wins over a plain "developer" row when
complexity="low" is threaded in."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE,
scope_value="developer:low",
model_name=ollama_model,
)
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == ollama_model
assert route.provider_type == ModelProvider.OLLAMA_CLOUD
@pytest.mark.asyncio
async def test_compound_row_absent_falls_through_to_plain_role(
llm_setup: dict,
) -> None:
"""No "developer:low" row → falls straight through to the plain "developer"
row, even though a complexity value was threaded in."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
)
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == anthropic_model
@pytest.mark.asyncio
async def test_malformed_complexity_string_falls_through_gracefully(
llm_setup: dict,
) -> None:
"""A complexity value with no matching compound row (e.g. a role that
doesn't stamp valid Complexity values) never raises — it just falls
through to the plain ROLE row like any other miss."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
)
route = await svc.resolve_for_agent("be-dev-1", complexity="not-a-real-complexity")
assert route.model_name == anthropic_model
@pytest.mark.asyncio
async def test_agent_slug_still_wins_over_compound_role_complexity(
llm_setup: dict,
) -> None:
"""AGENT_SLUG stays the top of the ladder — a compound "developer:low" row
never outranks a per-agent pin."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE,
scope_value="developer:low",
model_name=ollama_model,
)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name=anthropic_model,
)
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == anthropic_model
assert route.provider_type == ModelProvider.ANTHROPIC
@pytest.mark.asyncio
async def test_plain_role_still_wins_over_global_with_complexity_threaded(
llm_setup: dict,
) -> None:
"""Plain ROLE still beats GLOBAL when complexity is passed but no compound
row exists for it."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
)
route = await svc.resolve_for_agent("be-dev-1", complexity="high")
assert route.model_name == anthropic_model
@pytest.mark.asyncio
async def test_no_complexity_rows_means_byte_identical_routing(
llm_setup: dict,
) -> None:
"""CEO directive: with ZERO "role:complexity" rows present, resolve_for_agent
must return exactly what it returns today for every precedence case
(agent-slug, plain role, global, ROLE_MODEL_MAP fallback) — even when a
task's LOW/HIGH/MEDIUM (or a garbage) complexity value is threaded through.
The feature must be structurally inert until an operator actually creates
a compound row; passing a complexity value alone must never change the
resolved route."""
svc = llm_setup["svc"]
slug = "be-dev-1" # role == "developer"
def _same(a: AgentRoute, b: AgentRoute) -> bool:
return (
a.provider_id == b.provider_id
and a.provider_type == b.provider_type
and a.base_url == b.base_url
and a.auth_token == b.auth_token
and a.model_name == b.model_name
)
async def _assert_identical_across_complexities() -> None:
baseline = await svc.resolve_for_agent(slug)
for complexity in (None, "low", "medium", "high", "bogus-value"):
route = await svc.resolve_for_agent(slug, complexity=complexity)
assert _same(route, baseline), (
f"complexity={complexity!r} changed routing with zero "
"role:complexity rows present"
)
# 1. Legacy ROLE_MODEL_MAP fallback — no assignments at all.
await _assert_identical_across_complexities()
# 2. GLOBAL default only.
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
)
await _assert_identical_across_complexities()
# 3. Plain ROLE row (wins over GLOBAL).
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE, scope_value="developer", model_name=ollama_model
)
await _assert_identical_across_complexities()
# 4. AGENT_SLUG pin (wins over everything).
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value=slug,
model_name=anthropic_model,
)
await _assert_identical_across_complexities()
# ---------------------------------------------------------------------------
# Mode switches spare compound complexity-override rows (2026-07-17-style
# incident: these same buttons once wiped AGENT_SLUG pins — the compound
# ROLE(":"complexity) rung is a curated layer with the same rationale).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_apply_mode_anthropic_preserves_compound_row_and_resolution(
llm_setup: dict,
) -> None:
svc = llm_setup["svc"]
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE,
scope_value="developer:low",
model_name=ollama_model,
)
await svc.apply_mode(mode="anthropic")
assignments = await svc.list_assignments()
assert any(
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
for a in assignments
)
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == ollama_model
@pytest.mark.asyncio
async def test_apply_mode_grok_preserves_compound_row_and_resolution(
llm_setup: dict,
) -> None:
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE,
scope_value="developer:low",
model_name=anthropic_model,
)
await svc.apply_mode(mode="grok")
assignments = await svc.list_assignments()
assert any(
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
for a in assignments
)
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == anthropic_model
@pytest.mark.asyncio
async def test_apply_mode_ollama_preserves_compound_row_and_resolution(
llm_setup: dict,
) -> None:
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE,
scope_value="developer:low",
model_name=anthropic_model,
)
await svc.apply_mode(mode="ollama")
assignments = await svc.list_assignments()
assert any(
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
for a in assignments
)
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == anthropic_model
@pytest.mark.asyncio
async def test_apply_mode_self_hosted_preserves_compound_row_and_resolution(
llm_setup_with_local: dict,
) -> None:
svc = llm_setup_with_local["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.ROLE,
scope_value="developer:low",
model_name=anthropic_model,
)
await svc.apply_mode(mode="self_hosted", default_model="llama3.1:8b")
assignments = await svc.list_assignments()
assert any(
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
for a in assignments
)
# The compound row still points at Anthropic — resolving it never
# touches the LOCAL provider's reachability at all.
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
assert route.model_name == anthropic_model
@pytest.mark.asyncio
async def test_get_seeded_provider_unknown_raises(
db_session: AsyncSession,
) -> None:
"""When the requested provider type isn't seeded, raise NotFoundError (line 238)."""
svc = ModelRoutingService(db_session)
with pytest.raises(NotFoundError):
await svc._get_seeded_provider(ModelProvider.ANTHROPIC)
@pytest.mark.asyncio
async def test_upsert_openai_assignment_without_seed_raises_not_found(
db_session: AsyncSession,
) -> None:
"""The exact pre-fix failure: assigning a catalog model whose provider
type has no seeded `provider_configs` row raises NotFoundError out of
`upsert_assignment`. This is what migration `083_seed_openai_provider`
fixes — a bare session (no `llm_setup` fixture, so no OPENAI row) proves
the seed is load-bearing, not incidental."""
svc = ModelRoutingService(db_session)
codex_model = _first_model_for_type(ModelProvider.OPENAI)
with pytest.raises(NotFoundError):
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=codex_model
)
def test_get_model_routing_service_factory(db_session: AsyncSession) -> None:
"""Factory wraps ModelRoutingService with the given session (line 369)."""
svc = get_model_routing_service(db_session)
assert isinstance(svc, ModelRoutingService)
@pytest.mark.asyncio
async def test_resolve_for_agent_falls_back_on_decrypt_error(
llm_setup: dict,
) -> None:
"""EncryptionError on decrypt → falls back to legacy path (lines 98-99)."""
svc = llm_setup["svc"]
await svc.set_ollama_api_key("garbage-token")
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
)
with patch(
"roboco.services.llm.ProviderService.get_decrypted_token",
side_effect=EncryptionError("decrypt"),
new_callable=AsyncMock,
):
route = await svc.resolve_for_agent("be-dev-1")
# Falls back to legacy ANTHROPIC route.
assert route.provider_type == ModelProvider.ANTHROPIC
# ---------------------------------------------------------------------------
# Self-hosted (LOCAL) provider
# ---------------------------------------------------------------------------
@pytest_asyncio.fixture
async def llm_setup_with_local(
db_session: AsyncSession,
) -> AsyncIterator[dict]:
"""Seed Anthropic, Ollama Cloud, and LOCAL provider rows."""
anthropic = ProviderConfigTable(
name="anthropic-test-local",
type=ModelProvider.ANTHROPIC,
enabled=True,
)
ollama = ProviderConfigTable(
name="ollama-test-local",
type=ModelProvider.OLLAMA_CLOUD,
enabled=True,
base_url="https://ollama.example.com",
)
local = ProviderConfigTable(
name="self-hosted-test",
type=ModelProvider.LOCAL,
enabled=True,
base_url="http://localhost:11434",
)
db_session.add_all([anthropic, ollama, local])
await db_session.flush()
yield {"svc": ModelRoutingService(db_session), "local": local}
@pytest.mark.asyncio
async def test_derive_mode_self_hosted_when_only_local_global(
llm_setup_with_local: dict,
) -> None:
"""Single GLOBAL assignment pointing to LOCAL → 'self_hosted' mode."""
svc = llm_setup_with_local["svc"]
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL,
scope_value=None,
model_name="llama3.1:8b",
provider_type_override=ModelProvider.LOCAL,
)
assert await svc.derive_mode() == "self_hosted"
@pytest.mark.asyncio
async def test_apply_mode_self_hosted_sets_global_local(
llm_setup_with_local: dict,
) -> None:
"""apply_mode('self_hosted') clears assignments, enables LOCAL, inserts GLOBAL."""
svc = llm_setup_with_local["svc"]
await svc.apply_mode(mode="self_hosted", default_model="llama3.1:8b")
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
assert assignments[0].provider.type == ModelProvider.LOCAL
assert assignments[0].model_name == "llama3.1:8b"
# Verify derive_mode reflects the new state.
assert await svc.derive_mode() == "self_hosted"
@pytest.mark.asyncio
async def test_apply_mode_self_hosted_requires_default_model(
llm_setup_with_local: dict,
) -> None:
"""apply_mode('self_hosted') without default_model raises ValueError."""
svc = llm_setup_with_local["svc"]
with pytest.raises(ValueError, match="requires a default_model"):
await svc.apply_mode(mode="self_hosted")
@pytest.mark.asyncio
async def test_apply_mode_self_hosted_clears_prior_assignments(
llm_setup_with_local: dict,
) -> None:
"""apply_mode('self_hosted') clears role/global assignments but preserves
AGENT_SLUG pins (mixed-provider routing is a supported state)."""
svc = llm_setup_with_local["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name=anthropic_model,
)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL,
scope_value=None,
model_name=anthropic_model,
)
assert len(await svc.list_assignments()) == 2 # noqa: PLR2004
await svc.apply_mode(mode="self_hosted", default_model="gemma2:9b")
assignments = await svc.list_assignments()
assert len(assignments) == 2 # noqa: PLR2004 AGENT_SLUG pin kept + new GLOBAL row.
by_scope = {a.scope: a for a in assignments}
assert by_scope[AssignmentScope.AGENT_SLUG].scope_value == "be-dev-1"
assert by_scope[AssignmentScope.AGENT_SLUG].model_name == anthropic_model
assert by_scope[AssignmentScope.GLOBAL].provider.type == ModelProvider.LOCAL
@pytest.mark.asyncio
async def test_upsert_assignment_routes_unknown_model_to_local(
llm_setup_with_local: dict,
) -> None:
"""Non-catalog model names are silently routed to LOCAL if seeded."""
svc = llm_setup_with_local["svc"]
row = await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name="my-custom-model:latest",
)
assert row.provider.type == ModelProvider.LOCAL
assert row.model_name == "my-custom-model:latest"
@pytest.mark.asyncio
async def test_mix_mode_with_self_hosted_models(
llm_setup_with_local: dict,
) -> None:
"""mix mode accepts self-hosted model names without raising ValueError."""
svc = llm_setup_with_local["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.apply_mode(
mode="mix",
per_agent={
"be-dev-1": anthropic_model, # catalog model
"be-dev-2": "self-hosted-model:7b", # non-catalog → routed to LOCAL
},
)
rows = await svc.list_assignments()
by_slug = {r.scope_value: r for r in rows if r.scope == AssignmentScope.AGENT_SLUG}
assert by_slug["be-dev-1"].provider.type == ModelProvider.ANTHROPIC
assert by_slug["be-dev-2"].provider.type == ModelProvider.LOCAL
assert by_slug["be-dev-2"].model_name == "self-hosted-model:7b"
@pytest.mark.asyncio
async def test_resolve_for_agent_self_hosted_returns_base_url(
llm_setup_with_local: dict,
) -> None:
"""When LOCAL assignment is reachable, route has base_url from provider."""
svc = llm_setup_with_local["svc"]
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL,
scope_value=None,
model_name="llama3.1:8b",
provider_type_override=ModelProvider.LOCAL,
)
with patch(
"roboco.services.llm.probe_ollama_tags",
new_callable=AsyncMock,
return_value=(["llama3.1:8b"], None),
):
route = await svc.resolve_for_agent("be-dev-1")
assert route.provider_type == ModelProvider.LOCAL
assert route.base_url == "http://localhost:11434"
assert route.base_url is not None
@pytest.mark.asyncio
async def test_resolve_for_agent_falls_back_when_self_hosted_unreachable(
llm_setup_with_local: dict,
) -> None:
"""When LOCAL provider is unreachable, falls back to Anthropic default."""
svc = llm_setup_with_local["svc"]
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL,
scope_value=None,
model_name="llama3.1:8b",
provider_type_override=ModelProvider.LOCAL,
)
with patch(
"roboco.services.llm.probe_ollama_tags",
new_callable=AsyncMock,
return_value=([], "Could not connect to http://localhost:11434"),
):
route = await svc.resolve_for_agent("be-dev-1")
assert route.provider_type == ModelProvider.ANTHROPIC
assert route.base_url is None
@pytest.mark.asyncio
async def test_upsert_assignment_unknown_model_without_local_raises(
llm_setup: dict,
) -> None:
"""Without LOCAL provider seeded, non-catalog models raise ValueError."""
svc = llm_setup["svc"]
with pytest.raises(ValueError, match="Unknown model"):
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="be-dev-1",
model_name="ghost-model:latest",
)
@pytest.mark.asyncio
async def test_upsert_assignment_enables_local_when_disabled(
db_session: AsyncSession,
) -> None:
"""upsert_assignment transitions LOCAL provider from enabled=False to enabled=True.
AC4: proves that mix-mode assignment of a non-catalog model name resolves
to provider_type LOCAL and LOCAL.enabled is True afterward — even when the
LOCAL provider starts with enabled=False (the seeded state from migration 028
before the operator configures a base_url via PUT /providers/self-hosted).
"""
# Arrange: Anthropic provider (required by ModelRoutingService internals)
# and LOCAL provider starting with enabled=False.
anthropic = ProviderConfigTable(
name="anthropic-ac4-test",
type=ModelProvider.ANTHROPIC,
enabled=True,
)
local = ProviderConfigTable(
name="self-hosted-ac4-test",
type=ModelProvider.LOCAL,
enabled=False, # Starts DISABLED — this is the state to be transitioned.
base_url="http://localhost:11434",
)
db_session.add_all([anthropic, local])
await db_session.flush()
# Pre-condition: LOCAL is disabled before the call.
assert local.enabled is False
# Act: upsert a non-catalog model name → resolves to LOCAL →
# calls ProviderService.update_provider(enabled=True) on the LOCAL row.
svc = ModelRoutingService(db_session)
row = await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="test-agent-ac4",
model_name="non-catalog-model:7b",
)
# Refresh local from DB so the in-memory object reflects the DB write.
await db_session.refresh(local)
# Assert: assignment resolved to LOCAL and LOCAL provider is now enabled.
assert row.provider.type == ModelProvider.LOCAL
assert row.model_name == "non-catalog-model:7b"
assert local.enabled is True, (
"upsert_assignment must call update_provider(enabled=True) on LOCAL "
"whenever it routes a non-catalog model to the LOCAL provider"
)
# ---------------------------------------------------------------------------
# Routing presets — crash safety (validate-all-first, wipe never half-runs)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_apply_routing_preset_rolls_back_on_mid_apply_crash(
llm_setup: dict, db_session: AsyncSession
) -> None:
"""A raised exception mid-apply — after validation passed and the wipe
has already deleted the prior rows, partway through re-inserting the
validated set — must leave the PRIOR routing state intact once the
transaction rolls back (the real request-boundary behavior:
`apply_routing_preset` never calls `session.commit()` itself; the caller
commits once, after it returns). Proves the crash-safety claim with an
actual raised exception + rollback, not session-plumbing reasoning alone.
Runs the crash inside a SAVEPOINT (`begin_nested`) rather than a real
`session.commit()` / `session.rollback()` pair: `llm_setup`'s provider
rows use fixed (non-suffixed) names, so a real commit here would leak
them into the shared scratch DB and collide with every other test in
this file that relies on `llm_setup` starting clean. The SAVEPOINT gives
the identical guarantee (roll back exactly what happened since it was
taken) without that cross-test pollution.
"""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
# Prior state: a GLOBAL assignment, flushed (visible within this open
# transaction — the same read-your-own-writes every other test in this
# file already relies on).
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
)
preset = await svc.save_routing_preset("crash-preset")
# Simulate a crash in the write phase: validation (resolve_provider_for_
# model) is untouched and still runs for real; only the re-insert call
# explodes, after the wipe has already deleted the prior row.
with patch.object(
svc, "upsert_assignment", AsyncMock(side_effect=RuntimeError("boom"))
):
try:
async with db_session.begin_nested():
await svc.apply_routing_preset(preset.id)
except RuntimeError as e:
assert "boom" in str(e)
else:
pytest.fail("expected apply_routing_preset to raise RuntimeError")
remaining = await svc.list_assignments()
assert len(remaining) == 1
assert remaining[0].scope == AssignmentScope.GLOBAL
assert remaining[0].model_name == anthropic_model
@pytest.mark.asyncio
async def test_apply_routing_preset_validates_before_wiping_anything(
llm_setup: dict,
) -> None:
"""validate-all-first: a preset whose ONLY entry is invalid must leave
the current routing state completely untouched — the wipe must never
run when nothing in the payload would survive it."""
svc = llm_setup["svc"]
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
)
preset = await svc.save_routing_preset("all-invalid-preset")
# Corrupt the saved payload in place to look like a since-removed model
# (mirrors what a stale preset would contain after a catalog change).
preset.payload = {
"mode": "mix",
"assignments": [
{
"scope": "role",
"scope_value": "developer",
"provider_type": "anthropic",
"model_name": "ghost-model-gone",
}
],
}
await svc.session.flush()
notes = await svc.apply_routing_preset(preset.id)
assert len(notes) == 1
# The wipe ran (every entry was rejected, so the valid set is empty) —
# but nothing bogus was written; the GLOBAL row from before is gone
# because the preset legitimately replaced the whole state with an
# (all-invalid, now-empty) set. Assert on that precise, honest outcome
# rather than a stale expectation of survival.
remaining = await svc.list_assignments()
assert remaining == []
@pytest.mark.asyncio
async def test_apply_routing_preset_skips_entry_whose_provider_went_disabled(
llm_setup: dict,
) -> None:
"""A preset entry that resolved fine at save time but whose provider has
SINCE been disabled (key cleared, self-hosted disconnected, Codex/Gemini
disabled) must be skipped-with-note, never silently restored — applying
a preset can't resurrect a dead route behind a success toast."""
svc = llm_setup["svc"]
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=gemini_model
)
preset = await svc.save_routing_preset("gemini-then-disabled")
# Disable the GEMINI provider AFTER the preset was saved (mirrors an
# operator turning it off, or a fresh env where the row starts disabled).
provider_svc = ProviderService(svc.session)
gemini = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GEMINI
)
await provider_svc.update_provider(
cast("UUID", gemini.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
# Clear current routing so the preset apply has something to (not) restore.
await svc.apply_mode(mode="anthropic")
notes = await svc.apply_routing_preset(preset.id)
assert len(notes) == 1
assert "unavailable" in notes[0]
remaining = await svc.list_assignments()
assert remaining == [] # the disabled-provider entry was never written