fix: post-finale completeness sweep — routing surface, provider config, budgets, compose env, interactive exemption (#661)

This commit is contained in:
Renzo F
2026-07-23 09:41:27 +02:00
committed by GitHub
parent 21d6730400
commit d4b7e1e7b8
45 changed files with 2064 additions and 256 deletions
+252 -3
View File
@@ -53,15 +53,21 @@ async def llm_setup(
base_url="https://ollama.example.com",
)
# Mirrors migration 083_seed_openai_provider's contract: enabled=True at
# seed time (no apply_mode="codex" write path exists to flip it later —
# see that migration's docstring).
# seed time.
openai = ProviderConfigTable(
name="openai-test",
type=ModelProvider.OPENAI,
enabled=True,
base_url="https://api.openai.com/v1",
)
db_session.add_all([anthropic, grok, ollama, openai])
# Mirrors the post-086 seeded state (085 seeds enabled=false, 086 flips it
# true to match Codex) — no base_url, subscription OAuth auth only.
gemini = ProviderConfigTable(
name="gemini-test",
type=ModelProvider.GEMINI,
enabled=True,
)
db_session.add_all([anthropic, grok, ollama, openai, gemini])
await db_session.flush()
yield {"svc": ModelRoutingService(db_session)}
@@ -217,6 +223,18 @@ async def test_derive_mode_codex_when_only_openai_global(llm_setup: dict) -> Non
assert await svc.derive_mode() == "codex"
@pytest.mark.asyncio
async def test_derive_mode_gemini_when_only_gemini_global(llm_setup: dict) -> None:
"""A pure-GEMINI global assignment reports "gemini", not the catch-all
"mix" — mirrors the codex branch derive_mode already carries."""
svc = llm_setup["svc"]
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=gemini_model
)
assert await svc.derive_mode() == "gemini"
@pytest.mark.asyncio
async def test_derive_mode_mix_with_per_agent(llm_setup: dict) -> None:
svc = llm_setup["svc"]
@@ -332,6 +350,75 @@ async def test_apply_mode_grok_enables_grok_provider(llm_setup: dict) -> None:
assert refetched.enabled is True
@pytest.mark.asyncio
async def test_apply_mode_codex_sets_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
await svc.apply_mode(mode="codex")
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
assert assignments[0].provider.type == ModelProvider.OPENAI
assert assignments[0].model_name == "gpt-5.3-codex"
@pytest.mark.asyncio
async def test_apply_mode_codex_enables_openai_provider(llm_setup: dict) -> None:
"""apply_mode('codex') force-enables the OPENAI row — belt-and-suspenders
alongside migration 083's own enabled=true seed."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
openai = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.OPENAI
)
await provider_svc.update_provider(
cast("UUID", openai.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
await svc.apply_mode(mode="codex")
refetched = await provider_svc.get_provider(cast("UUID", openai.id))
assert refetched is not None
assert refetched.enabled is True
@pytest.mark.asyncio
async def test_apply_mode_gemini_sets_global(llm_setup: dict) -> None:
svc = llm_setup["svc"]
await svc.apply_mode(mode="gemini")
assignments = await svc.list_assignments()
assert len(assignments) == 1
assert assignments[0].scope == AssignmentScope.GLOBAL
assert assignments[0].provider.type == ModelProvider.GEMINI
assert assignments[0].model_name == "gemini-2.5-pro"
@pytest.mark.asyncio
async def test_apply_mode_gemini_enables_gemini_provider(llm_setup: dict) -> None:
"""apply_mode('gemini') force-enables the GEMINI row — the exact gap this
fix closes (migration 085 seeds it disabled and nothing else ever flipped
it before this write path + migration 086 existed)."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
gemini = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GEMINI
)
await provider_svc.update_provider(
cast("UUID", gemini.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
await svc.apply_mode(mode="gemini")
refetched = await provider_svc.get_provider(cast("UUID", gemini.id))
assert refetched is not None
assert refetched.enabled is True
@pytest.mark.asyncio
async def test_apply_mode_mix_requires_per_agent(llm_setup: dict) -> None:
svc = llm_setup["svc"]
@@ -440,6 +527,129 @@ async def test_upsert_and_resolve_openai_assignment_roundtrip(
assert route.auth_token is None
@pytest.mark.asyncio
async def test_upsert_and_resolve_gemini_assignment_roundtrip(
llm_setup: dict,
) -> None:
"""gemini-2.5-pro through upsert_assignment -> resolve_for_agent, against
the seeded GEMINI row. Proves resolve_for_agent actually returns a GEMINI
spawn route — not a silent Anthropic fallback — the exact gap left open
by the row seeding disabled with no enable path (migration 085 alone)."""
svc = llm_setup["svc"]
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
row = await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="ux-dev-1",
model_name=gemini_model,
)
assert row.model_name == gemini_model
route = await svc.resolve_for_agent("ux-dev-1")
assert route.provider_type == ModelProvider.GEMINI
assert route.model_name == gemini_model
# Subscription OAuth auth (~/.gemini), not a decrypted provider token.
assert route.auth_token is None
@pytest.mark.asyncio
async def test_upsert_assignment_enables_disabled_gemini_provider(
llm_setup: dict,
) -> None:
"""Belt-and-suspenders: assigning a Gemini model via Mix (upsert_assignment)
force-enables the row even if it was disabled — not just apply_mode('gemini')."""
svc = llm_setup["svc"]
provider_svc = ProviderService(svc.session)
gemini = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GEMINI
)
await provider_svc.update_provider(
cast("UUID", gemini.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="ux-dev-1",
model_name=gemini_model,
)
refetched = await provider_svc.get_provider(cast("UUID", gemini.id))
assert refetched is not None
assert refetched.enabled is True
# And the route actually resolves to GEMINI now that it's enabled.
route = await svc.resolve_for_agent("ux-dev-1")
assert route.provider_type == ModelProvider.GEMINI
@pytest.mark.asyncio
async def test_apply_mode_gemini_end_to_end_reachable(llm_setup: dict) -> None:
"""The full reachability chain the original drill missed: apply_mode
-> derive_mode reflects it -> resolve_for_agent actually spawns Gemini."""
svc = llm_setup["svc"]
await svc.apply_mode(mode="gemini")
assert await svc.derive_mode() == "gemini"
route = await svc.resolve_for_agent("ux-dev-1")
assert route.provider_type == ModelProvider.GEMINI
assert route.model_name == "gemini-2.5-pro"
@pytest.mark.asyncio
async def test_apply_mode_codex_end_to_end_reachable(llm_setup: dict) -> None:
"""Same reachability chain for Codex, mirroring the Gemini test above."""
svc = llm_setup["svc"]
await svc.apply_mode(mode="codex")
assert await svc.derive_mode() == "codex"
route = await svc.resolve_for_agent("be-dev-1")
assert route.provider_type == ModelProvider.OPENAI
assert route.model_name == "gpt-5.3-codex"
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["codex", "gemini"])
@pytest.mark.parametrize("interactive_slug", ["intake-1", "secretary-1"])
async def test_interactive_agents_exempt_from_delivery_only_global_mode(
llm_setup: dict, mode: str, interactive_slug: str
) -> None:
"""A fleet-wide Codex/Gemini mode must not capture Intake/Secretary —
they have no V1 support on those providers, so the resolver keeps them
on the legacy Anthropic path (the completeness-drill gap: previously
they resolved to the unsupported provider and the spawn guard left both
chats refusing to start after a one-click mode switch)."""
svc = llm_setup["svc"]
await svc.apply_mode(mode=mode)
# The mode still derives cleanly (single GLOBAL row — no extra pins).
assert await svc.derive_mode() == mode
route = await svc.resolve_for_agent(interactive_slug)
assert route.provider_type == ModelProvider.ANTHROPIC
@pytest.mark.asyncio
async def test_interactive_agent_explicit_pin_is_not_exempted(
llm_setup: dict,
) -> None:
"""An EXPLICIT AGENT_SLUG pin to a delivery-only provider is honored by
the resolver (the orchestrator's spawn guard refuses it loudly) — a
deliberate operator choice must error, never be silently overridden."""
svc = llm_setup["svc"]
await svc.upsert_assignment(
scope=AssignmentScope.AGENT_SLUG,
scope_value="intake-1",
model_name="gpt-5.3-codex",
)
route = await svc.resolve_for_agent("intake-1")
assert route.provider_type == ModelProvider.OPENAI
@pytest.mark.asyncio
async def test_resolve_for_agent_uses_provider_token(llm_setup: dict) -> None:
"""When provider has auth_token_encrypted, it's decrypted (lines 345-346)."""
@@ -1107,3 +1317,42 @@ async def test_apply_routing_preset_validates_before_wiping_anything(
# rather than a stale expectation of survival.
remaining = await svc.list_assignments()
assert remaining == []
@pytest.mark.asyncio
async def test_apply_routing_preset_skips_entry_whose_provider_went_disabled(
llm_setup: dict,
) -> None:
"""A preset entry that resolved fine at save time but whose provider has
SINCE been disabled (key cleared, self-hosted disconnected, Codex/Gemini
disabled) must be skipped-with-note, never silently restored — applying
a preset can't resurrect a dead route behind a success toast."""
svc = llm_setup["svc"]
gemini_model = _first_model_for_type(ModelProvider.GEMINI)
await svc.upsert_assignment(
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=gemini_model
)
preset = await svc.save_routing_preset("gemini-then-disabled")
# Disable the GEMINI provider AFTER the preset was saved (mirrors an
# operator turning it off, or a fresh env where the row starts disabled).
provider_svc = ProviderService(svc.session)
gemini = next(
p
for p in await provider_svc.list_providers(include_disabled=True)
if p.type == ModelProvider.GEMINI
)
await provider_svc.update_provider(
cast("UUID", gemini.id), ProviderUpdate(enabled=False)
)
await svc.session.flush()
# Clear current routing so the preset apply has something to (not) restore.
await svc.apply_mode(mode="anthropic")
notes = await svc.apply_routing_preset(preset.id)
assert len(notes) == 1
assert "unavailable" in notes[0]
remaining = await svc.list_assignments()
assert remaining == [] # the disabled-provider entry was never written
+74
View File
@@ -13,6 +13,7 @@ from fastapi import FastAPI
from httpx import ASGITransport, AsyncClient
from roboco.api.deps import get_agent_context, get_db
from roboco.api.routes.project import router as project_router
from roboco.config import settings
from roboco.db.tables import AgentTable
from roboco.models import AgentRole, AgentStatus, Team
from roboco.models.permissions import AgentContext
@@ -166,6 +167,79 @@ async def test_update_project_explicit_null_clears_field(
assert cleared.json()["test_command"] is None
@pytest.mark.asyncio
async def test_update_project_rejects_zero_monthly_budget_usd(
project_client: AsyncClient,
) -> None:
"""#654: a 0 cap would block every claim immediately — rejected at the
request boundary, never stored."""
create = await project_client.post("/api/projects", json=_payload(), headers=_HDR)
pid = create.json()["id"]
response = await project_client.patch(
f"/api/projects/{pid}",
json={"monthly_budget_usd": 0},
headers=_HDR,
)
assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
@pytest.mark.asyncio
async def test_update_project_rejects_negative_monthly_budget_usd(
project_client: AsyncClient,
) -> None:
create = await project_client.post("/api/projects", json=_payload(), headers=_HDR)
pid = create.json()["id"]
response = await project_client.patch(
f"/api/projects/{pid}",
json={"monthly_budget_usd": -5},
headers=_HDR,
)
assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
@pytest.mark.asyncio
async def test_update_project_accepts_positive_monthly_budget_usd(
project_client: AsyncClient,
) -> None:
create = await project_client.post("/api/projects", json=_payload(), headers=_HDR)
pid = create.json()["id"]
cap = 100
response = await project_client.patch(
f"/api/projects/{pid}",
json={"monthly_budget_usd": cap},
headers=_HDR,
)
assert response.status_code == HTTPStatus.OK
assert response.json()["monthly_budget_usd"] == cap
@pytest.mark.asyncio
async def test_get_project_by_id_includes_spend_when_budgets_enabled(
project_client: AsyncClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""monthly_spend_usd is populated (0.0 with no spawn sessions yet) once
ROBOCO_TASK_BUDGETS_ENABLED is on — the extra DB read only runs then."""
monkeypatch.setattr(settings, "task_budgets_enabled", True)
create = await project_client.post("/api/projects", json=_payload(), headers=_HDR)
pid = create.json()["id"]
response = await project_client.get(f"/api/projects/{pid}", headers=_HDR)
assert response.status_code == HTTPStatus.OK
assert response.json()["monthly_spend_usd"] == 0.0
@pytest.mark.asyncio
async def test_get_project_by_id_omits_spend_when_budgets_disabled(
project_client: AsyncClient, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Flag off => monthly_spend_usd stays null, same as before this field existed."""
monkeypatch.setattr(settings, "task_budgets_enabled", False)
create = await project_client.post("/api/projects", json=_payload(), headers=_HDR)
pid = create.json()["id"]
response = await project_client.get(f"/api/projects/{pid}", headers=_HDR)
assert response.status_code == HTTPStatus.OK
assert response.json()["monthly_spend_usd"] is None
@pytest.mark.asyncio
async def test_update_project_not_found(project_client: AsyncClient) -> None:
response = await project_client.patch(
+119
View File
@@ -330,6 +330,117 @@ async def test_apply_mode_ollama_without_provider_returns_404(
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest_asyncio.fixture
async def app_client_with_codex_and_gemini(
db_session: AsyncSession,
) -> AsyncIterator[AsyncClient]:
"""App client pre-seeded with Anthropic + disabled OPENAI/GEMINI providers
(mirrors the real seeded state before an operator ever applies either
mode: OPENAI seeds enabled=true per migration 083, GEMINI seeds
enabled=false per migration 085 — deliberately seeded disabled here so the
apply-mode round trip below proves the force-enable, not a pre-enabled
no-op)."""
app = _make_app(db_session)
suffix = uuid4().hex[:8]
await db_session.execute(delete(ModelAssignmentTable))
await db_session.execute(delete(ProviderConfigTable))
await db_session.flush()
db_session.add(
ProviderConfigTable(
name=f"anthropic-cg-{suffix}", type=ModelProvider.ANTHROPIC, enabled=True
)
)
db_session.add(
ProviderConfigTable(
name=f"codex-cg-{suffix}",
type=ModelProvider.OPENAI,
enabled=False,
base_url="https://api.openai.com/v1",
)
)
db_session.add(
ProviderConfigTable(
name=f"gemini-cg-{suffix}", type=ModelProvider.GEMINI, enabled=False
)
)
await db_session.flush()
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
yield client
app.dependency_overrides.clear()
@pytest.mark.asyncio
async def test_apply_mode_codex_returns_200_reflects_mode_and_enables_provider(
app_client_with_codex_and_gemini: AsyncClient,
) -> None:
"""The full HTTP round trip: POST mode="codex" -> 200, GET reflects
mode="codex", and the assignment resolves through the now-enabled OPENAI
provider — proving the pydantic Literal + dispatch + enable chain end to
end, not just the service-layer call this mirrors."""
response = await app_client_with_codex_and_gemini.post(
"/api/providers", json={"mode": "codex"}, headers=_HDR_PM
)
assert response.status_code == HTTPStatus.OK
body = response.json()
assert body["mode"] == "codex"
assert body["assignments"][0]["provider_type"] == "openai"
assert body["assignments"][0]["model_name"] == "gpt-5.3-codex"
followup = await app_client_with_codex_and_gemini.get(
"/api/providers", headers=_HDR_PM
)
assert followup.json()["mode"] == "codex"
@pytest.mark.asyncio
async def test_apply_mode_gemini_returns_200_reflects_mode_and_enables_provider(
app_client_with_codex_and_gemini: AsyncClient,
) -> None:
"""Same round trip as Codex's, for Gemini — the exact reachability gap
this fix closes (the row seeds disabled and nothing else ever flipped it)."""
response = await app_client_with_codex_and_gemini.post(
"/api/providers", json={"mode": "gemini"}, headers=_HDR_PM
)
assert response.status_code == HTTPStatus.OK
body = response.json()
assert body["mode"] == "gemini"
assert body["assignments"][0]["provider_type"] == "gemini"
assert body["assignments"][0]["model_name"] == "gemini-2.5-pro"
followup = await app_client_with_codex_and_gemini.get(
"/api/providers", headers=_HDR_PM
)
assert followup.json()["mode"] == "gemini"
@pytest.mark.asyncio
async def test_apply_mode_gemini_without_provider_returns_404(
db_session: AsyncSession,
) -> None:
"""Apply 'gemini' mode without the GEMINI provider seeded raises
NotFoundError -> 404 (mirrors the ollama/grok equivalents)."""
# FK-safe: a prior test may have committed a real GEMINI assignment
# (model_assignments.provider_config_id references provider_configs.id),
# so assignments must be cleared before the provider row can be deleted.
await db_session.execute(delete(ModelAssignmentTable))
await db_session.execute(
delete(ProviderConfigTable).where(
ProviderConfigTable.type == ModelProvider.GEMINI
)
)
await db_session.flush()
app = _make_app(db_session)
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.post(
"/api/providers", json={"mode": "gemini"}, headers=_HDR_PM
)
app.dependency_overrides.clear()
assert response.status_code == HTTPStatus.NOT_FOUND
# =============================================================================
# Self-hosted endpoints
# =============================================================================
@@ -876,6 +987,14 @@ async def test_save_list_and_apply_preset_round_trip(
app_client_with_ollama: AsyncClient,
) -> None:
"""Save captures the current state; mutating + re-applying restores it."""
# Set the Ollama key first — the fixture seeds OLLAMA_CLOUD `enabled=False`
# (no key yet), and `_validate_preset_entry` now rejects (skip-with-note)
# any preset entry whose provider is disabled, so a meaningful round trip
# needs the provider actually live, same as the real UI's key-gated mode
# button.
await app_client_with_ollama.put(
"/api/providers/ollama-key", json={"api_key": "test-key"}, headers=_HDR_PM
)
# Arrange a distinctive state: a GLOBAL Ollama default.
await app_client_with_ollama.post(
"/api/providers", json={"mode": "ollama"}, headers=_HDR_PM
+76
View File
@@ -22,6 +22,7 @@ from roboco.api.routes.tasks import (
from roboco.api.routes.tasks import (
router as tasks_router,
)
from roboco.config import settings
from roboco.db.tables import AgentTable, ProjectTable, TaskTable, WorkSessionTable
from roboco.exceptions import GitError, TaskLifecycleError
from roboco.foundation.policy.lifecycle import STATUS_GRAPH
@@ -278,6 +279,35 @@ async def test_get_task_by_id(task_client: dict) -> None:
assert response.status_code == HTTPStatus.OK
@pytest.mark.asyncio
async def test_get_task_by_id_includes_spend_when_budgets_enabled(
task_client: dict, monkeypatch: pytest.MonkeyPatch
) -> None:
"""spend_usd is populated (0.0 with no spawn sessions yet) once
ROBOCO_TASK_BUDGETS_ENABLED is on the extra DB read only runs then."""
monkeypatch.setattr(settings, "task_budgets_enabled", True)
client = task_client["client"]
task = _seed_task(task_client)
await task_client["db"].flush()
response = await client.get(f"/api/tasks/{task.id}", headers=_HDR)
assert response.status_code == HTTPStatus.OK
assert response.json()["spend_usd"] == 0.0
@pytest.mark.asyncio
async def test_get_task_by_id_omits_spend_when_budgets_disabled(
task_client: dict, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Flag off => spend_usd stays null, the same as before this field existed."""
monkeypatch.setattr(settings, "task_budgets_enabled", False)
client = task_client["client"]
task = _seed_task(task_client)
await task_client["db"].flush()
response = await client.get(f"/api/tasks/{task.id}", headers=_HDR)
assert response.status_code == HTTPStatus.OK
assert response.json()["spend_usd"] is None
@pytest.mark.asyncio
async def test_update_task(task_client: dict) -> None:
client = task_client["client"]
@@ -291,6 +321,52 @@ async def test_update_task(task_client: dict) -> None:
assert response.status_code in (HTTPStatus.OK, HTTPStatus.UNPROCESSABLE_ENTITY)
@pytest.mark.asyncio
async def test_update_task_rejects_zero_budget_usd(task_client: dict) -> None:
"""#654: a 0 cap would block every claim immediately — rejected at the
request boundary, never stored."""
client = task_client["client"]
task = _seed_task(task_client)
await task_client["db"].flush()
response = await client.patch(
f"/api/tasks/{task.id}",
json={"budget_usd": 0},
headers=_HDR,
)
assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
@pytest.mark.asyncio
async def test_update_task_rejects_negative_budget_usd(task_client: dict) -> None:
client = task_client["client"]
task = _seed_task(task_client)
await task_client["db"].flush()
response = await client.patch(
f"/api/tasks/{task.id}",
json={"budget_usd": -5},
headers=_HDR,
)
assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
@pytest.mark.asyncio
async def test_update_task_accepts_positive_budget_usd(task_client: dict) -> None:
# budget_usd is a _PRIVILEGED_UPDATE_FIELDS / non-"PM lighter" field —
# a plain main_pm PATCH would 403 here, so exercise the CEO's full scope.
_as_ceo(task_client)
client = task_client["client"]
task = _seed_task(task_client)
await task_client["db"].flush()
budget = 12.5
response = await client.patch(
f"/api/tasks/{task.id}",
json={"budget_usd": budget},
headers=_HDR,
)
assert response.status_code == HTTPStatus.OK
assert response.json()["budget_usd"] == budget
@pytest.mark.asyncio
async def test_update_task_status_override_recovers_blocked(task_client: dict) -> None:
"""A privileged PATCH with ``status`` + ``force`` is applied as an audited