diff --git a/alembic/versions/082_routing_presets.py b/alembic/versions/082_routing_presets.py new file mode 100644 index 00000000..1b517b29 --- /dev/null +++ b/alembic/versions/082_routing_presets.py @@ -0,0 +1,48 @@ +"""Routing presets — named, full snapshots of the AI-routing state. + +Lets an operator save the current routing state (mode + every +`model_assignments` row: GLOBAL / plain ROLE / compound ROLE(":"complexity) +cost-tier overrides / AGENT_SLUG pins) under a name and re-apply it later in +one call, instead of re-picking every per-agent Select. ``payload`` mirrors +exactly what ``GET /providers`` + ``GET /providers/complexity-overrides`` +already serve, so a preset is "what the card currently shows" — see +``ModelRoutingService.save_routing_preset`` / ``apply_routing_preset`` +(roboco/services/llm.py). Pure additive new table; no backfill, no data +migration — applying a preset is the only thing that ever mutates +``model_assignments`` as a side effect, and only on an explicit call. + +Revision ID: 082_routing_presets +Revises: 081_doctrine_version +Create Date: 2026-07-23 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "082_routing_presets" +down_revision = "081_doctrine_version" +branch_labels: dict[str, str] | None = None +depends_on: dict[str, str] | None = None + + +def upgrade() -> None: + op.create_table( + "routing_presets", + sa.Column("id", sa.UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.String(length=100), nullable=False), + sa.Column("payload", postgresql.JSONB(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint("name", name="uq_routing_presets_name"), + ) + + +def downgrade() -> None: + op.drop_table("routing_presets") diff --git a/panel/src/components/settings/__tests__/ai-routing-card.test.tsx b/panel/src/components/settings/__tests__/ai-routing-card.test.tsx index 0f44898a..1706bcc8 100644 --- a/panel/src/components/settings/__tests__/ai-routing-card.test.tsx +++ b/panel/src/components/settings/__tests__/ai-routing-card.test.tsx @@ -1,7 +1,18 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { + fireEvent, + render, + screen, + waitFor, + within, +} from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import type { ReactNode } from "react"; +import React from "react"; +import type { + ComplexityOverride, + RoutingPreset, +} from "@/lib/api/providers"; const { catalog, @@ -15,6 +26,13 @@ const { saveSelfHostedConfig, testSelfHosted, getSelfHostedModels, + getComplexityOverrides, + setComplexityOverride, + deleteComplexityOverride, + listPresets, + savePreset, + applyPreset, + deletePreset, } = vi.hoisted(() => ({ catalog: vi.fn(async () => [ { @@ -53,6 +71,21 @@ const { error: null, })), getSelfHostedModels: vi.fn(async () => []), + getComplexityOverrides: vi.fn(async (): Promise => []), + setComplexityOverride: vi.fn(async (payload: ComplexityOverride) => payload), + deleteComplexityOverride: vi.fn(async () => undefined), + listPresets: vi.fn(async (): Promise => []), + savePreset: vi.fn(async (name: string) => ({ + id: "preset-1", + name, + created_at: "2026-07-23T00:00:00Z", + })), + applyPreset: vi.fn(async () => ({ + mode: "mix", + assignments: [], + skipped: [] as string[], + })), + deletePreset: vi.fn(async () => undefined), })); vi.mock("@/lib/api/providers", () => ({ @@ -68,10 +101,95 @@ vi.mock("@/lib/api/providers", () => ({ saveSelfHostedConfig, testSelfHosted, getSelfHostedModels, + getComplexityOverrides, + setComplexityOverride, + deleteComplexityOverride, + listPresets, + savePreset, + applyPreset, + deletePreset, }, + COMPLEXITY_OVERRIDE_ROLES: ["developer", "qa", "documenter"], })); -vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); +vi.mock("sonner", () => ({ + toast: { success: vi.fn(), error: vi.fn(), warning: vi.fn() }, +})); + +// Functional Select mock (mirrors select-repo-picker.test.tsx / +// a2a-reply-composer.test.tsx): SelectItem renders as a clickable button +// wired to onValueChange via context, so a real "pick a value" interaction +// can be simulated without Radix's portal/pointer machinery. SelectTrigger +// keeps `role="combobox"` so the pre-existing per-agent-table combobox-count +// assertion is unaffected, and stamps `data-value` from context so a test can +// `waitFor` an async-loaded value (e.g. a complexity override) landing before +// interacting further. +vi.mock("@/components/ui/select", () => { + const Ctx = React.createContext<{ + value?: string; + onValueChange: (v: string) => void; + }>({ onValueChange: () => {} }); + return { + Select: ({ + value, + onValueChange, + children, + }: { + value?: string; + onValueChange?: (v: string) => void; + children: React.ReactNode; + }) => ( + {}) }} + > + {children} + + ), + SelectTrigger: ({ children }: { children: React.ReactNode }) => { + const { value } = React.useContext(Ctx); + return ( + + ); + }, + SelectValue: () => null, + SelectGroup: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + SelectLabel: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + SelectContent: ({ children }: { children: React.ReactNode }) => ( +
{children}
+ ), + SelectItem: ({ + value, + children, + }: { + value: string; + children: React.ReactNode; + }) => { + const { onValueChange } = React.useContext(Ctx); + return ( + + ); + }, + }; +}); // Full live roster (minus CEO/system) — mirrors foundation/identity.py so the // per-agent override grid is exercised against the real 25-agent org chart, @@ -277,6 +395,13 @@ describe("AIRoutingCard", () => { saveSelfHostedConfig.mockClear(); testSelfHosted.mockClear(); getSelfHostedModels.mockClear(); + getComplexityOverrides.mockClear(); + setComplexityOverride.mockClear(); + deleteComplexityOverride.mockClear(); + listPresets.mockClear(); + savePreset.mockClear(); + applyPreset.mockClear(); + deletePreset.mockClear(); useAgentDefinitions.mockReturnValue({ data: FULL_ROSTER, isLoading: false, @@ -468,4 +593,298 @@ describe("AIRoutingCard", () => { ); expect(applyMode).not.toHaveBeenCalled(); }); + + // ------------------------------------------------------------------------- + // Complexity overrides (cost-tiered routing) + // ------------------------------------------------------------------------- + + describe("Complexity overrides section", () => { + it("renders one row per allowlisted role with Low/High selects, and no coordinator role", async () => { + render(withQueryClient()); + await screen.findByText("Complexity overrides"); + + const section = screen.getByText("Complexity overrides").closest("section")!; + for (const label of ["Developer", "QA", "Documenter"]) { + expect(within(section).getByText(label)).toBeInTheDocument(); + } + // Coordinator (cell_pm, main_pm)/pr_reviewer/board/CEO-facing roles are + // never offered a row here — scoped to this section, since "Cell PM"/ + // "Main PM"/"PR Reviewer" legitimately appear elsewhere (the per-agent + // mix table's roster). + expect(within(section).queryByText("Cell PM")).not.toBeInTheDocument(); + expect(within(section).queryByText("Main PM")).not.toBeInTheDocument(); + expect(within(section).queryByText("PR Reviewer")).not.toBeInTheDocument(); + + // 3 roles x 2 (low/high) selects. + expect(section.querySelectorAll('[role="combobox"]')).toHaveLength(6); + }); + + it("picking a model for a role+complexity calls setComplexityOverride with the right payload", async () => { + render(withQueryClient()); + await screen.findByText("Complexity overrides"); + + const devLow = screen.getByTestId("complexity-select-developer-low"); + fireEvent.click( + await within(devLow).findByRole("option", { name: "Claude Opus 4.6" }), + ); + + await waitFor(() => + expect(setComplexityOverride).toHaveBeenCalledWith({ + role: "developer", + complexity: "low", + model_name: "claude-opus-4-6", + }), + ); + }); + + it("surfaces a returned cross-family warning as its own warning toast", async () => { + setComplexityOverride.mockResolvedValueOnce({ + role: "developer", + complexity: "low", + model_name: "grok-build-0.1", + warning: + "'grok-build-0.1' routes through grok, a different provider family than developer's Anthropic baseline.", + }); + render(withQueryClient()); + await screen.findByText("Complexity overrides"); + + const devLow = screen.getByTestId("complexity-select-developer-low"); + fireEvent.click( + await within(devLow).findByRole("option", { name: "Grok Build 0.1" }), + ); + + await waitFor(() => + expect(toast.warning).toHaveBeenCalledWith( + expect.stringContaining("different provider family"), + ), + ); + // Still allowed — the success toast still fires alongside the warning. + expect(toast.success).toHaveBeenCalledWith( + "developer:low → grok-build-0.1", + ); + }); + + it("never shows a warning toast when the response carries none", async () => { + render(withQueryClient()); + await screen.findByText("Complexity overrides"); + + const devLow = screen.getByTestId("complexity-select-developer-low"); + fireEvent.click( + await within(devLow).findByRole("option", { name: "Claude Opus 4.6" }), + ); + + await waitFor(() => expect(setComplexityOverride).toHaveBeenCalled()); + expect(toast.warning).not.toHaveBeenCalled(); + }); + + it("picking '(none)' on a row with no existing override never calls deleteComplexityOverride", async () => { + render(withQueryClient()); + await screen.findByText("Complexity overrides"); + + const qaHigh = screen.getByTestId("complexity-select-qa-high"); + fireEvent.click( + await within(qaHigh).findByRole("option", { name: "(none)" }), + ); + + // No pre-existing row for qa:high (getComplexityOverrides returns [] + // by default) — clearing an already-empty selection is a no-op. + await waitFor(() => expect(setComplexityOverride).not.toHaveBeenCalled()); + expect(deleteComplexityOverride).not.toHaveBeenCalled(); + }); + + it("picking '(none)' on a row WITH an existing override calls deleteComplexityOverride", async () => { + getComplexityOverrides.mockResolvedValueOnce([ + { role: "documenter", complexity: "low", model_name: "grok-build-0.1" }, + ]); + render(withQueryClient()); + await screen.findByText("Complexity overrides"); + + const documenterLow = await screen.findByTestId( + "complexity-select-documenter-low", + ); + // Wait for the async-loaded override to actually land on the Select + // (its combobox reflects the current value via data-value) before + // clearing it — otherwise the click races the query resolving. + await waitFor(() => + expect( + within(documenterLow).getByRole("combobox"), + ).toHaveAttribute("data-value", "grok-build-0.1"), + ); + fireEvent.click( + within(documenterLow).getByRole("option", { name: "(none)" }), + ); + + // deleteComplexityOverride(role, complexity) — positional, per the + // providersApi signature (mirrors the real hook's mutationFn). + await waitFor(() => + expect(deleteComplexityOverride).toHaveBeenCalledWith( + "documenter", + "low", + ), + ); + }); + }); + + // ------------------------------------------------------------------------- + // Cost-Tiered mode button (additive seed, never wipes routing) + // ------------------------------------------------------------------------- + + describe("Cost-Tiered mode button", () => { + it("renders beside the other mode buttons and applies mode='cost_tiered' on confirm", async () => { + const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true); + render(withQueryClient()); + await screen.findByText("Grok (xAI) API key"); + + fireEvent.click(screen.getByText("Cost-Tiered")); + + await waitFor(() => + expect(applyMode).toHaveBeenCalledWith({ mode: "cost_tiered" }), + ); + confirmSpy.mockRestore(); + }); + + it("does nothing when the confirm dialog is declined", async () => { + const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(false); + render(withQueryClient()); + await screen.findByText("Grok (xAI) API key"); + + fireEvent.click(screen.getByText("Cost-Tiered")); + + await waitFor(() => expect(confirmSpy).toHaveBeenCalled()); + expect(applyMode).not.toHaveBeenCalled(); + confirmSpy.mockRestore(); + }); + }); + + // ------------------------------------------------------------------------- + // Mode switches preserve complexity overrides (2026-07-17-style incident: + // these same buttons once wiped AGENT_SLUG pins) — the confirm text says so + // and the query cache is refreshed for both, not just the mode snapshot. + // ------------------------------------------------------------------------- + + describe("Mode switches preserve complexity overrides", () => { + it("applying Anthropic mode also refetches complexity overrides, not just the mode snapshot", async () => { + const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true); + render(withQueryClient()); + await screen.findByText("Grok (xAI) API key"); + getComplexityOverrides.mockClear(); + + fireEvent.click(screen.getByText("Anthropic")); + + await waitFor(() => + expect(applyMode).toHaveBeenCalledWith({ mode: "anthropic" }), + ); + await waitFor(() => + expect(getComplexityOverrides.mock.calls.length).toBeGreaterThan(0), + ); + confirmSpy.mockRestore(); + }); + + it("the Anthropic confirm dialog states complexity overrides are kept", async () => { + const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(false); + render(withQueryClient()); + await screen.findByText("Grok (xAI) API key"); + + fireEvent.click(screen.getByText("Anthropic")); + + await waitFor(() => expect(confirmSpy).toHaveBeenCalled()); + expect(confirmSpy.mock.calls[0][0]).toContain("complexity"); + confirmSpy.mockRestore(); + }); + }); + + // ------------------------------------------------------------------------- + // Routing presets (named, full snapshots) + // ------------------------------------------------------------------------- + + describe("Routing presets bar", () => { + it("lists saved presets and disables Apply/Delete until one is picked", async () => { + listPresets.mockResolvedValueOnce([ + { id: "p1", name: "Cheap Fleet", created_at: "2026-07-01T00:00:00Z" }, + ]); + render(withQueryClient()); + await screen.findByText("Cheap Fleet"); + + expect(screen.getByRole("button", { name: "Apply" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Delete" })).toBeDisabled(); + }); + + it("save-as-preset flow: reveals a name input and Confirm calls savePreset", async () => { + render(withQueryClient()); + await screen.findByText("Routing presets"); + + fireEvent.click(screen.getByText("Save as preset…")); + const nameInput = screen.getByPlaceholderText("Preset name"); + fireEvent.change(nameInput, { target: { value: "My Setup" } }); + fireEvent.click(screen.getByText("Confirm")); + + await waitFor(() => expect(savePreset).toHaveBeenCalledWith("My Setup")); + await waitFor(() => + expect(toast.success).toHaveBeenCalledWith('Saved preset "My Setup"'), + ); + }); + + it("save-as-preset with an empty name shows an error and never calls savePreset", async () => { + render(withQueryClient()); + await screen.findByText("Routing presets"); + + fireEvent.click(screen.getByText("Save as preset…")); + fireEvent.click(screen.getByText("Confirm")); + + await waitFor(() => + expect(toast.error).toHaveBeenCalledWith("Enter a preset name first"), + ); + expect(savePreset).not.toHaveBeenCalled(); + }); + + it("applying a preset with skipped rows surfaces them in a toast error", async () => { + const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true); + listPresets.mockResolvedValueOnce([ + { id: "p1", name: "Cheap Fleet", created_at: "2026-07-01T00:00:00Z" }, + ]); + applyPreset.mockResolvedValueOnce({ + mode: "mix", + assignments: [], + skipped: ["Skipped role:developer (ghost-model) — Unknown model"], + }); + render(withQueryClient()); + const presetSection = (await screen.findByText("Routing presets")).closest( + "section", + )!; + fireEvent.click( + await within(presetSection).findByRole("option", { + name: "Cheap Fleet", + }), + ); + fireEvent.click(screen.getByRole("button", { name: "Apply" })); + + await waitFor(() => expect(applyPreset).toHaveBeenCalledWith("p1")); + await waitFor(() => + expect(toast.error).toHaveBeenCalledWith( + expect.stringContaining("1 row(s) skipped"), + ), + ); + confirmSpy.mockRestore(); + }); + + it("deleting a preset calls deletePreset and clears the selection", async () => { + const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true); + listPresets.mockResolvedValueOnce([ + { id: "p1", name: "Cheap Fleet", created_at: "2026-07-01T00:00:00Z" }, + ]); + render(withQueryClient()); + const presetSection = (await screen.findByText("Routing presets")).closest( + "section", + )!; + fireEvent.click( + await within(presetSection).findByRole("option", { + name: "Cheap Fleet", + }), + ); + fireEvent.click(screen.getByRole("button", { name: "Delete" })); + + await waitFor(() => expect(deletePreset).toHaveBeenCalledWith("p1")); + confirmSpy.mockRestore(); + }); + }); }); diff --git a/panel/src/components/settings/ai-routing-card.tsx b/panel/src/components/settings/ai-routing-card.tsx index c62d1fa9..78647eb6 100644 --- a/panel/src/components/settings/ai-routing-card.tsx +++ b/panel/src/components/settings/ai-routing-card.tsx @@ -3,10 +3,17 @@ import { useEffect, useMemo, useState, useCallback } from "react"; import { useApplyMode, + useApplyPreset, useCatalog, + useComplexityOverrides, + useDeleteComplexityOverride, + useDeletePreset, useGrokKey, useOllamaKey, useRoutingMode, + useRoutingPresets, + useSavePreset, + useSetComplexityOverride, useSetGrokKey, useSetOllamaKey, useSelfHostedModels, @@ -34,6 +41,7 @@ import { Separator } from "@/components/ui/separator"; import { AlertTriangle, Cpu, + Gauge, Key, KeyRound, Server, @@ -43,7 +51,11 @@ import { } from "lucide-react"; import { toast } from "sonner"; import { AssignmentScope, AgentRole, ModelProvider } from "@/types"; -import type { SelfHostedModel } from "@/lib/api/providers"; +import { + COMPLEXITY_OVERRIDE_ROLES, + type ComplexityLevel, + type SelfHostedModel, +} from "@/lib/api/providers"; import type { RoutingMode, SelfHostedTestResult } from "@/lib/api/providers"; import { SelfHostedSection } from "@/components/settings/self-hosted-section"; import { Badge } from "@/components/ui/badge"; @@ -126,11 +138,29 @@ function byOrgOrder(a: AgentDefinition, b: AgentDefinition): number { return ra !== rb ? ra - rb : a.id.localeCompare(b.id); } +// Display labels for the complexity-override allowlist — mirrors the +// backend's fixed `_COMPLEXITY_OVERRIDE_ROLES` (developer/qa/documenter). +// cell_pm is deliberately excluded (a coordinator role — see +// COMPLEXITY_OVERRIDE_ROLES), along with every other coordinator/board/ +// CEO-facing role. +const COMPLEXITY_ROLE_LABELS: Record = { + developer: "Developer", + qa: "QA", + documenter: "Documenter", +}; + export function AIRoutingCard() { const { data: catalog = [] } = useCatalog(); const { data: keyStatus } = useOllamaKey(); const { data: snapshot } = useRoutingMode(); const { data: selfHostedModels = [] } = useSelfHostedModels(); + const { data: complexityOverrides = [] } = useComplexityOverrides(); + const setComplexityOverride = useSetComplexityOverride(); + const deleteComplexityOverride = useDeleteComplexityOverride(); + const { data: presets = [] } = useRoutingPresets(); + const savePreset = useSavePreset(); + const applyPreset = useApplyPreset(); + const deletePreset = useDeletePreset(); const { data: agentDefs, isLoading: agentsLoading, @@ -255,14 +285,15 @@ export function AIRoutingCard() { const flipToAnthropic = async () => { if ( !confirm( - "Switch every agent to Anthropic? Per-agent pins are kept; role/global assignments are replaced.", + "Switch every agent to Anthropic? Per-agent pins and complexity " + + "overrides are kept; other role/global assignments are replaced.", ) ) return; try { await applyMode.mutateAsync({ mode: "anthropic" }); toast.success( - "Role/global routing now on Anthropic — per-agent pins kept", + "Role/global routing now on Anthropic — per-agent pins and complexity overrides kept", ); } catch (e) { toast.error("Switch failed: " + errMsg(e)); @@ -276,13 +307,16 @@ export function AIRoutingCard() { } if ( !confirm( - "Switch every agent to Grok? Per-agent pins are kept; role/global assignments are replaced.", + "Switch every agent to Grok? Per-agent pins and complexity " + + "overrides are kept; other role/global assignments are replaced.", ) ) return; try { await applyMode.mutateAsync({ mode: "grok" }); - toast.success("Role/global routing now on Grok — per-agent pins kept"); + toast.success( + "Role/global routing now on Grok — per-agent pins and complexity overrides kept", + ); } catch (e) { toast.error("Switch failed: " + errMsg(e)); } @@ -295,13 +329,16 @@ export function AIRoutingCard() { } if ( !confirm( - "Switch every agent to Ollama? Per-agent pins are kept; role/global assignments are replaced.", + "Switch every agent to Ollama? Per-agent pins and complexity " + + "overrides are kept; other role/global assignments are replaced.", ) ) return; try { await applyMode.mutateAsync({ mode: "ollama" }); - toast.success("Role/global routing now on Ollama — per-agent pins kept"); + toast.success( + "Role/global routing now on Ollama — per-agent pins and complexity overrides kept", + ); } catch (e) { toast.error("Switch failed: " + errMsg(e)); } @@ -314,7 +351,9 @@ export function AIRoutingCard() { } if ( !confirm( - "Switch every agent to the self-hosted LLM? Per-agent pins are kept; role/global assignments are replaced.", + "Switch every agent to the self-hosted LLM? Per-agent pins and " + + "complexity overrides are kept; other role/global assignments " + + "are replaced.", ) ) return; @@ -324,7 +363,7 @@ export function AIRoutingCard() { ...(selfHostedModel ? { default_model: selfHostedModel } : {}), }); toast.success( - "Role/global routing now on Self-Hosted LLM — per-agent pins kept", + "Role/global routing now on Self-Hosted LLM — per-agent pins and complexity overrides kept", ); } catch (e) { toast.error("Switch failed: " + errMsg(e)); @@ -382,6 +421,132 @@ export function AIRoutingCard() { } }; + // --- Cost-tiered defaults (additive seed — never wipes existing routing) --- + const flipToCostTiered = async () => { + if ( + !confirm( + "Seed the day-1 cost-tiered default (developer:low → Haiku)? " + + "Unlike the other buttons this is additive — it does not clear " + + "any existing routing.", + ) + ) + return; + try { + await applyMode.mutateAsync({ mode: "cost_tiered" }); + toast.success( + "Cost-tiered default seeded (developer:low → Haiku)", + ); + } catch (e) { + toast.error("Apply failed: " + errMsg(e)); + } + }; + + // --- Complexity overrides (compound ROLE(":"complexity) rows) --- + const complexityMap = useMemo(() => { + const map: Record>> = {}; + for (const o of complexityOverrides) { + map[o.role] = { ...map[o.role], [o.complexity]: o.model_name }; + } + return map; + }, [complexityOverrides]); + + const handleComplexityChange = async ( + role: string, + complexity: ComplexityLevel, + modelName: string, + ) => { + try { + if (!modelName) { + if (complexityMap[role]?.[complexity]) { + await deleteComplexityOverride.mutateAsync({ role, complexity }); + toast.success(`Cleared ${role}:${complexity} override`); + } + return; + } + const result = await setComplexityOverride.mutateAsync({ + role, + complexity, + model_name: modelName, + }); + toast.success(`${role}:${complexity} → ${modelName}`); + // Allowed but never silent: a cross-provider-family override (e.g. an + // Anthropic role pinned to a Grok/Ollama/self-hosted model) also gets + // its own warning toast. + if (result.warning) { + toast.warning(result.warning); + } + } catch (e) { + toast.error("Save failed: " + errMsg(e)); + } + }; + + // --- Routing presets (named, full snapshots) --- + const [selectedPresetId, setSelectedPresetId] = useState(""); + const [showPresetNameInput, setShowPresetNameInput] = useState(false); + const [presetNameDraft, setPresetNameDraft] = useState(""); + const selectedPreset = presets.find((p) => p.id === selectedPresetId); + + const handleSavePreset = async () => { + const name = presetNameDraft.trim(); + if (!name) { + toast.error("Enter a preset name first"); + return; + } + try { + const saved = await savePreset.mutateAsync(name); + toast.success(`Saved preset "${saved.name}"`); + setPresetNameDraft(""); + setShowPresetNameInput(false); + setSelectedPresetId(saved.id); + } catch (e) { + toast.error("Save failed: " + errMsg(e)); + } + }; + + const handleApplyPreset = async () => { + if (!selectedPresetId) { + toast.error("Pick a preset first"); + return; + } + if ( + !confirm( + `Apply preset "${selectedPreset?.name ?? selectedPresetId}"? This ` + + "replaces the ENTIRE current routing state (every per-agent pin, " + + "role row, and global default) with the saved snapshot.", + ) + ) + return; + try { + const result = await applyPreset.mutateAsync(selectedPresetId); + if (result.skipped.length > 0) { + toast.error( + `Applied with ${result.skipped.length} row(s) skipped: ` + + result.skipped.join("; "), + ); + } else { + toast.success("Preset applied"); + } + } catch (e) { + toast.error("Apply failed: " + errMsg(e)); + } + }; + + const handleDeletePreset = async () => { + if (!selectedPresetId) { + toast.error("Pick a preset first"); + return; + } + if (!confirm(`Delete preset "${selectedPreset?.name ?? selectedPresetId}"?`)) + return; + try { + await deletePreset.mutateAsync(selectedPresetId); + toast.success("Preset deleted"); + setSelectedPresetId(""); + } catch (e) { + toast.error("Delete failed: " + errMsg(e)); + } + }; + return ( @@ -534,7 +699,7 @@ export function AIRoutingCard() { -
+
} label="Anthropic" @@ -591,6 +756,15 @@ export function AIRoutingCard() { disabled={false} highlight={currentMode === "mix"} /> + } + label="Cost-Tiered" + description="Seed developer:low → Haiku (see below)." + active={false} + onClick={flipToCostTiered} + disabled={applyMode.isPending} + labelHint="Unlike every button to the left this never wipes existing routing — it's a one-time additive seed you can re-run anytime. Edit or remove individual rows in the Complexity overrides section below." + />
{currentMode === "mix" && !hasOllamaKey ? (

@@ -649,6 +823,95 @@ export function AIRoutingCard() { )} + {/* -------- Preset bar (compact, sits right above the per-agent table) -------- */} + +

+ + + +
+ + + + + + + + + {showPresetNameInput ? ( + <> + setPresetNameDraft(e.target.value)} + placeholder="Preset name" + className="h-8 w-40 text-xs" + /> + + + + ) : ( + + + + )} +
+
+ {/* -------- Mix-mode per-agent picker -------- */}
@@ -843,6 +1106,78 @@ export function AIRoutingCard() {

) : null}
+ + {/* -------- Complexity overrides (cost-tiered routing) -------- */} + +
+ + + +

+ Pin a role to a cheaper model for LOW- or HIGH-complexity tasks + specifically — wins over that role's plain default and the + global mode, still loses to a per-agent pin above. Leave a Select + blank to remove the override. Coordinator roles (cell_pm, main_pm) + aren't offered a row. +

+
+ {COMPLEXITY_OVERRIDE_ROLES.map((role) => ( +
+ +
+ {COMPLEXITY_ROLE_LABELS[role]} +
+
+ {(["low", "high"] as const).map((complexity) => ( +
+ +
+ {complexity} +
+
+ +
+ ))} +
+ ))} +
+
); diff --git a/panel/src/hooks/use-providers.ts b/panel/src/hooks/use-providers.ts index 47c9ca35..661b37a3 100644 --- a/panel/src/hooks/use-providers.ts +++ b/panel/src/hooks/use-providers.ts @@ -2,6 +2,8 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { providersApi, type ApplyModePayload, + type ComplexityLevel, + type ComplexityOverride, type SelfHostedConfigPayload, } from "@/lib/api/providers"; @@ -14,6 +16,9 @@ export const providerKeys = { selfHostedConfig: () => [...providerKeys.all, "self-hosted-config"] as const, selfHostedModels: () => [...providerKeys.all, "self-hosted-models"] as const, selfHostedTest: () => [...providerKeys.all, "self-hosted-test"] as const, + complexityOverrides: () => + [...providerKeys.all, "complexity-overrides"] as const, + presets: () => [...providerKeys.all, "presets"] as const, }; export function useCatalog() { @@ -77,6 +82,11 @@ export function useApplyMode() { mutationFn: (payload: ApplyModePayload) => providersApi.applyMode(payload), onSuccess: () => { qc.invalidateQueries({ queryKey: providerKeys.mode() }); + // Complexity overrides survive a mode switch (see the backend's + // _wipe_mode_switch_assignments carve-out) but the mode-apply response + // doesn't carry them — refresh separately so the card doesn't show a + // stale list. Mirrors useApplyPreset, which already invalidates both. + qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() }); }, }); } @@ -136,3 +146,87 @@ export function useRefreshSelfHostedModels() { }, }); } + +// --------------------------------------------------------------------------- +// Complexity overrides +// --------------------------------------------------------------------------- + +export function useComplexityOverrides() { + return useQuery({ + queryKey: providerKeys.complexityOverrides(), + queryFn: () => providersApi.getComplexityOverrides(), + staleTime: 30_000, + }); +} + +export function useSetComplexityOverride() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (payload: ComplexityOverride) => + providersApi.setComplexityOverride(payload), + onSuccess: () => { + qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() }); + }, + }); +} + +export function useDeleteComplexityOverride() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: ({ + role, + complexity, + }: { + role: string; + complexity: ComplexityLevel; + }) => providersApi.deleteComplexityOverride(role, complexity), + onSuccess: () => { + qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() }); + }, + }); +} + +// --------------------------------------------------------------------------- +// Routing presets +// --------------------------------------------------------------------------- + +export function useRoutingPresets() { + return useQuery({ + queryKey: providerKeys.presets(), + queryFn: () => providersApi.listPresets(), + staleTime: 30_000, + }); +} + +export function useSavePreset() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (name: string) => providersApi.savePreset(name), + onSuccess: () => { + qc.invalidateQueries({ queryKey: providerKeys.presets() }); + }, + }); +} + +export function useApplyPreset() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => providersApi.applyPreset(id), + onSuccess: () => { + // A preset fully replaces the routing state — refresh everything the + // card renders off of, not just the mode snapshot. + qc.invalidateQueries({ queryKey: providerKeys.mode() }); + qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() }); + }, + }); +} + +export function useDeletePreset() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => providersApi.deletePreset(id), + onSuccess: () => { + qc.invalidateQueries({ queryKey: providerKeys.presets() }); + }, + }); +} diff --git a/panel/src/lib/api/providers.ts b/panel/src/lib/api/providers.ts index 59fbbf92..d9c5837a 100644 --- a/panel/src/lib/api/providers.ts +++ b/panel/src/lib/api/providers.ts @@ -31,7 +31,8 @@ export type RoutingMode = | "grok" | "ollama" | "self_hosted" - | "mix"; + | "mix" + | "cost_tiered"; export interface ModeSnapshot { mode: RoutingMode; @@ -75,6 +76,47 @@ export interface SelfHostedConfigPayload { auth_token?: string; // omit to leave token unchanged; "" to clear } +// --------------------------------------------------------------------------- +// Complexity overrides (cost-tiered routing: compound ROLE(":"complexity) rows) +// --------------------------------------------------------------------------- + +export type ComplexityLevel = "low" | "high"; + +/** One active ROLE(":"complexity) cost-tiered override row. */ +export interface ComplexityOverride { + role: string; + complexity: ComplexityLevel; + model_name: string; + /** Set only on the PUT response, when the model crosses provider families + * relative to the role's Anthropic baseline — allowed, never silent. */ + warning?: string | null; +} + +/** Roles the complexity-override endpoint accepts a row for — mirrors the + * server allowlist in api/routes/provider.py. Coordinator (cell_pm, main_pm), + * pr_reviewer, and board/CEO-facing roles are never offered a row here; tier + * pinning for those is deliberate — cell_pm especially, since the org's + * documented weak-model incidents were precisely a cheap model on a PM role. */ +export const COMPLEXITY_OVERRIDE_ROLES = ["developer", "qa", "documenter"] as const; + +// --------------------------------------------------------------------------- +// Routing presets (named, full snapshots of the routing state) +// --------------------------------------------------------------------------- + +/** One saved preset — list view (no payload). */ +export interface RoutingPreset { + id: string; + name: string; + created_at: string; +} + +/** Result of applying a preset. */ +export interface RoutingPresetApplyResult { + mode: RoutingMode; + assignments: ModelAssignment[]; + skipped: string[]; +} + export const providersApi = { catalog: async (): Promise => { const { data } = await api.get("/providers/catalog"); @@ -147,4 +189,61 @@ export const providersApi = { ); return data; }, + + // ------------------------------------------------------------------------- + // Complexity overrides + // ------------------------------------------------------------------------- + + getComplexityOverrides: async (): Promise => { + const { data } = await api.get( + "/providers/complexity-overrides", + ); + return data; + }, + + setComplexityOverride: async ( + payload: ComplexityOverride, + ): Promise => { + const { data } = await api.put( + "/providers/complexity-overrides", + payload, + ); + return data; + }, + + deleteComplexityOverride: async ( + role: string, + complexity: ComplexityLevel, + ): Promise => { + await api.delete( + `/providers/complexity-overrides/${encodeURIComponent(role)}/${complexity}`, + ); + }, + + // ------------------------------------------------------------------------- + // Routing presets + // ------------------------------------------------------------------------- + + listPresets: async (): Promise => { + const { data } = await api.get("/providers/presets"); + return data; + }, + + savePreset: async (name: string): Promise => { + const { data } = await api.post("/providers/presets", { + name, + }); + return data; + }, + + applyPreset: async (id: string): Promise => { + const { data } = await api.post( + `/providers/presets/${id}/apply`, + ); + return data; + }, + + deletePreset: async (id: string): Promise => { + await api.delete(`/providers/presets/${id}`); + }, }; diff --git a/roboco/api/routes/provider.py b/roboco/api/routes/provider.py index a8d76280..06c546fa 100644 --- a/roboco/api/routes/provider.py +++ b/roboco/api/routes/provider.py @@ -9,15 +9,23 @@ providers (Anthropic, Ollama Cloud, Self-Hosted) are pre-seeded by migrations 004 and 028. """ +from typing import Literal +from uuid import UUID + from fastapi import APIRouter, HTTPException, status from roboco.api.deps import CurrentAgentContext, DbSession, require_pm_or_above from roboco.api.schemas.provider import ( ApplyModeRequest, CatalogEntryResponse, + ComplexityOverrideRequest, + ComplexityOverrideResponse, GrokKeyStatus, ModeResponse, OllamaKeyStatus, + RoutingPresetApplyResponse, + RoutingPresetSummary, + SaveRoutingPresetRequest, SelfHostedConfigRequest, SelfHostedConfigResponse, SelfHostedModelEntry, @@ -25,9 +33,12 @@ from roboco.api.schemas.provider import ( SetGrokKeyRequest, SetOllamaKeyRequest, assignment_to_response, + routing_preset_to_summary, ) -from roboco.models.base import ModelProvider +from roboco.billing.pricing import input_price_per_million +from roboco.models.base import AssignmentScope, ModelProvider from roboco.models.llm_catalog import MODEL_CATALOG +from roboco.models.runtime import ROLE_MODEL_MAP from roboco.security import guard_deco from roboco.services.base import NotFoundError from roboco.services.llm import get_model_routing_service, probe_ollama_tags @@ -36,6 +47,37 @@ from roboco.utils.converters import require_uuid router = APIRouter() +# Roles the complexity-override endpoint accepts a row for. Coordinator roles +# (cell_pm, main_pm — CLAUDE.md's own _COORDINATOR_ROLES) and pr_reviewer/ +# board/CEO-facing roles are never offered a row — tier pinning for those is +# deliberate (see set_complexity_override). cell_pm is deliberately excluded +# even though it's not board/CEO-facing: the org's documented weak-model +# incidents were precisely a cheap model landed on a PM/coordinator role, +# not a leaf developer — a coordinator is the last place to gamble a +# downgrade on. +_COMPLEXITY_OVERRIDE_ROLES: frozenset[str] = frozenset( + {"developer", "qa", "documenter"} +) + +# Human remediation hint per provider type, for a complexity override that +# resolves to a not-ready (disabled / unconfigured) provider. +_PROVIDER_REMEDIATION: dict[ModelProvider, str] = { + ModelProvider.GROK: "Save the Grok (xAI) API key first (PUT /providers/grok-key).", + ModelProvider.OLLAMA_CLOUD: ( + "Save an Ollama Cloud API key first (PUT /providers/ollama-key)." + ), + ModelProvider.LOCAL: ( + "Configure + test the self-hosted server first (PUT /providers/self-hosted)." + ), + ModelProvider.ANTHROPIC: "The Anthropic provider is disabled — re-enable it first.", +} + + +def _provider_remediation(provider_type: ModelProvider) -> str: + return _PROVIDER_REMEDIATION.get( + provider_type, f"The {provider_type.value} provider is not configured." + ) + # ============================================================================= # CATALOG @@ -408,3 +450,270 @@ async def apply_mode( mode=mode, assignments=[assignment_to_response(a) for a in assignments], ) + + +# ============================================================================= +# COMPLEXITY OVERRIDES (cost-tiered routing: compound ROLE(":"complexity) rows) +# ============================================================================= + + +def _parse_complexity_override( + scope_value: str, model_name: str +) -> ComplexityOverrideResponse | None: + """Parse a ROLE scope_value into a response row, or None if not a + well-formed "role:low"/"role:high" compound key (a plain role row, or a + malformed compound value, are both silently skipped).""" + role, sep, complexity = scope_value.partition(":") + if not sep or not role: + return None + if complexity == "low": + return ComplexityOverrideResponse( + role=role, complexity="low", model_name=model_name + ) + if complexity == "high": + return ComplexityOverrideResponse( + role=role, complexity="high", model_name=model_name + ) + return None + + +@router.get("/complexity-overrides", response_model=list[ComplexityOverrideResponse]) +async def get_complexity_overrides( + db: DbSession, + agent: CurrentAgentContext, +) -> list[ComplexityOverrideResponse]: + """List the active compound ROLE(":"complexity) cost-tiered rows.""" + require_pm_or_above(agent.role, "view complexity overrides") + routing = get_model_routing_service(db) + rows = await routing.list_assignments() + overrides = [] + for row in rows: + if row.scope != AssignmentScope.ROLE or not row.scope_value: + continue + parsed = _parse_complexity_override(row.scope_value, row.model_name) + if parsed is not None: + overrides.append(parsed) + return overrides + + +@router.put("/complexity-overrides", response_model=ComplexityOverrideResponse) +@guard_deco.rate_limit(requests=20, window=60) +@guard_deco.max_request_size(size_bytes=4096) +@guard_deco.block_clouds() +@guard_deco.content_type_filter(["application/json"]) +@guard_deco.honeypot_detection(["email", "phone", "website"]) +async def set_complexity_override( + data: ComplexityOverrideRequest, + db: DbSession, + agent: CurrentAgentContext, +) -> ComplexityOverrideResponse: + """Upsert one ROLE(":"complexity) cost-tiered override. + + Write-time guards enforce the policy (never at read/resolve time): + - role allowlist: only {developer, qa, documenter} are offered a row — + cell_pm/main_pm (coordinators), pr_reviewer, and board/CEO-facing + roles are rejected outright, tier pinning for those is deliberate. + - downgrade-only: `model_name` must not be a costlier tier than the + role's `ROLE_MODEL_MAP` baseline, compared via each model's + per-1M-token input price (`billing.pricing.input_price_per_million`) + since the model catalog carries no explicit tier ordering. + - provider readiness: `model_name` must resolve to a provider that is + both known (catalog or LOCAL self-hosted) AND enabled+configured — + an override to a disabled/unconfigured provider would otherwise + silently no-op at spawn (falling back to the legacy Anthropic path) + behind a success toast; mirrors the intent of the Mix section's + client-side needsGrok/needsKey/needsSelfHosted guards, enforced here + server-side where it can't be bypassed by a direct API call. + + A model that resolves to a different provider FAMILY than the role's + Anthropic baseline (e.g. an Anthropic role pinned to Grok/Ollama/ + self-hosted) is still ALLOWED — the CEO may want that deliberately — but + the response carries a non-null `warning` so it's never silent. + """ + require_pm_or_above(agent.role, "change complexity-based routing overrides") + if data.role not in _COMPLEXITY_OVERRIDE_ROLES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"'{data.role}' is not eligible for a complexity override — " + "tier pinning for coordinator/board/CEO-facing roles is " + "deliberate." + ), + ) + baseline_model = ROLE_MODEL_MAP.get(data.role, "sonnet") + if input_price_per_million(data.model_name) > input_price_per_million( + baseline_model + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"'{data.model_name}' is a costlier tier than {data.role}'s " + f"baseline ('{baseline_model}') — complexity overrides are " + "downgrade-only." + ), + ) + routing = get_model_routing_service(db) + provider = await routing.resolve_provider_for_model(data.model_name) + if provider is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Unknown model '{data.model_name}'. Use one from " + "GET /api/providers/catalog." + ), + ) + if not provider.enabled or ( + provider.type == ModelProvider.LOCAL and not provider.base_url + ): + remediation = _provider_remediation(provider.type) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"'{data.model_name}' routes through {provider.type.value}, " + "which isn't configured yet — it would silently fall back to " + f"the legacy Anthropic path at spawn. {remediation}" + ), + ) + warning = ( + f"'{data.model_name}' routes through {provider.type.value}, a " + f"different provider family than {data.role}'s Anthropic baseline " + f"('{baseline_model}') — allowed, but make sure that's deliberate." + if provider.type != ModelProvider.ANTHROPIC + else None + ) + try: + row = await routing.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value=f"{data.role}:{data.complexity}", + model_name=data.model_name, + ) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(e) + ) from e + await db.commit() + return ComplexityOverrideResponse( + role=data.role, + complexity=data.complexity, + model_name=row.model_name, + warning=warning, + ) + + +@router.delete( + "/complexity-overrides/{role}/{complexity}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_complexity_override( + role: str, + complexity: Literal["low", "high"], + db: DbSession, + agent: CurrentAgentContext, +) -> None: + """Remove one ROLE(":"complexity) cost-tiered override row.""" + require_pm_or_above(agent.role, "remove a complexity override") + routing = get_model_routing_service(db) + try: + await routing.delete_assignment( + scope=AssignmentScope.ROLE, scope_value=f"{role}:{complexity}" + ) + except NotFoundError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e + await db.commit() + + +# ============================================================================= +# ROUTING PRESETS (named, full snapshots of the routing state) +# ============================================================================= + + +@router.get("/presets", response_model=list[RoutingPresetSummary]) +async def list_routing_presets( + db: DbSession, + agent: CurrentAgentContext, +) -> list[RoutingPresetSummary]: + """List saved routing presets, newest first (no payloads — see + `POST /presets/{id}/apply` to inspect one by applying it).""" + require_pm_or_above(agent.role, "view routing presets") + routing = get_model_routing_service(db) + rows = await routing.list_routing_presets() + return [routing_preset_to_summary(r) for r in rows] + + +@router.post("/presets", response_model=RoutingPresetSummary) +@guard_deco.rate_limit(requests=20, window=60) +@guard_deco.max_request_size(size_bytes=4096) +@guard_deco.block_clouds() +@guard_deco.content_type_filter(["application/json"]) +@guard_deco.honeypot_detection(["email", "phone", "website"]) +async def save_routing_preset( + data: SaveRoutingPresetRequest, + db: DbSession, + agent: CurrentAgentContext, +) -> RoutingPresetSummary: + """Snapshot the CURRENT routing state under `data.name`. + + Same privilege level as the mode-apply / mix-save endpoints. A duplicate + name is a 409 (the panel names presets, so a collision is a UI mistake, + not routine traffic). + """ + require_pm_or_above(agent.role, "save a routing preset") + routing = get_model_routing_service(db) + try: + row = await routing.save_routing_preset(data.name) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) from e + await db.commit() + return routing_preset_to_summary(row) + + +@router.post("/presets/{preset_id}/apply", response_model=RoutingPresetApplyResponse) +@guard_deco.rate_limit(requests=20, window=60) +@guard_deco.block_clouds() +@guard_deco.content_type_filter(["application/json"]) +@guard_deco.honeypot_detection(["email", "phone", "website"]) +async def apply_routing_preset( + preset_id: UUID, + db: DbSession, + agent: CurrentAgentContext, +) -> RoutingPresetApplyResponse: + """Replace the current routing state with the preset's snapshot. + + Transactional: the delete + re-upserts of the new rows happen in this + request's session, committed once at the end — a mid-apply failure never + leaves a half-swapped state. A row referencing a model no longer in the + catalog (or any other now-invalid entry) is skipped and reported in + `skipped`, never silently dropped and never failing the rows that DID + validate. + """ + require_pm_or_above(agent.role, "apply a routing preset") + routing = get_model_routing_service(db) + try: + skipped = await routing.apply_routing_preset(preset_id) + except NotFoundError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e + await db.commit() + + mode = await routing.derive_mode() + assignments = await routing.list_assignments() + return RoutingPresetApplyResponse( + mode=mode, + assignments=[assignment_to_response(a) for a in assignments], + skipped=skipped, + ) + + +@router.delete("/presets/{preset_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_routing_preset( + preset_id: UUID, + db: DbSession, + agent: CurrentAgentContext, +) -> None: + """Delete a saved routing preset.""" + require_pm_or_above(agent.role, "delete a routing preset") + routing = get_model_routing_service(db) + try: + await routing.delete_routing_preset(preset_id) + except NotFoundError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e + await db.commit() diff --git a/roboco/api/schemas/provider.py b/roboco/api/schemas/provider.py index eb336a69..5b233271 100644 --- a/roboco/api/schemas/provider.py +++ b/roboco/api/schemas/provider.py @@ -6,11 +6,13 @@ Minimal surface that backs the Settings UI: - set / clear / check the single Ollama Cloud API key - configure / test / discover the self-hosted (LOCAL) Ollama server - read current routing assignments (so the UI renders Mix mode) - - apply a routing mode (anthropic | grok | ollama | mix | self_hosted) + - apply a routing mode (anthropic | grok | ollama | mix | self_hosted | cost_tiered) + - read/write/delete cost-tiered complexity overrides (compound ROLE rows) """ from __future__ import annotations +from datetime import datetime # noqa: TC003 (pydantic needs the type at runtime) from typing import TYPE_CHECKING, Literal from uuid import UUID # noqa: TC003 (pydantic needs the type at runtime) @@ -20,7 +22,7 @@ from roboco.models.base import AssignmentScope, ModelProvider # noqa: TC001 from roboco.utils.converters import require_uuid if TYPE_CHECKING: - from roboco.db.tables import ModelAssignmentTable + from roboco.db.tables import ModelAssignmentTable, RoutingPresetTable # ============================================================================= @@ -187,9 +189,13 @@ class ApplyModeRequest(BaseModel): `per_agent` are routed to the LOCAL provider automatically. - mode="self_hosted": clear every assignment; enable LOCAL provider; set GLOBAL default to `default_model` (a self-hosted model name). + - mode="cost_tiered": seed the day-1 cost-tiered compound ROLE(":"complexity) + rows (see `ModelRoutingService._COST_TIERED_SEED`). Unlike every mode + above, nothing is cleared first — purely additive on top of whatever + routing already exists. """ - mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted"] + mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted", "cost_tiered"] default_model: str | None = None per_agent: dict[str, str] | None = None @@ -197,5 +203,77 @@ class ApplyModeRequest(BaseModel): class ModeResponse(BaseModel): """Server-side view of the current mode + a snapshot of active rules.""" - mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted"] + mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted", "cost_tiered"] assignments: list[AssignmentResponse] + + +# ============================================================================= +# COMPLEXITY OVERRIDES (cost-tiered routing: compound ROLE(":"complexity) rows) +# ============================================================================= + + +class ComplexityOverrideRequest(BaseModel): + """Upsert one ROLE(":"complexity) cost-tiered override. + + `role` is validated at the route (not here) against a fixed allowlist — + a rejected coordinator/board/CEO-facing role gets the deliberate + tier-pinning message instead of a generic 422. `model_name` must resolve + to a tier no costlier than that role's `ROLE_MODEL_MAP` baseline + (downgrade-only by policy), also enforced at the route. + """ + + role: str + complexity: Literal["low", "high"] + model_name: str + + +class ComplexityOverrideResponse(BaseModel): + """One active ROLE(":"complexity) cost-tiered override row. + + `warning` is set only by the PUT response (never by GET's listing) when + the model crosses provider families relative to the role's Anthropic + baseline (e.g. an Anthropic role pinned to a Grok/Ollama/self-hosted + model) — allowed, but surfaced so it's never a silent switch. + """ + + role: str + complexity: Literal["low", "high"] + model_name: str + warning: str | None = None + + +# ============================================================================= +# ROUTING PRESETS (named, full snapshots of the routing state) +# ============================================================================= + + +class RoutingPresetSummary(BaseModel): + """One saved preset — list view. No `payload` here; the panel doesn't + need the snapshot contents until it actually applies one.""" + + id: UUID + name: str + created_at: datetime + + +class SaveRoutingPresetRequest(BaseModel): + """Snapshot the CURRENT routing state under `name`.""" + + name: str = Field(..., min_length=1, max_length=100) + + +class RoutingPresetApplyResponse(BaseModel): + """Result of applying a preset: the fresh mode snapshot (same shape as + `ModeResponse`) plus any per-entry skip notes (e.g. a since-removed + catalog model) — never a partial/silent apply.""" + + mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted", "cost_tiered"] + assignments: list[AssignmentResponse] + skipped: list[str] + + +def routing_preset_to_summary(row: RoutingPresetTable) -> RoutingPresetSummary: + """Convert a RoutingPresetTable row to its list-view summary.""" + return RoutingPresetSummary( + id=require_uuid(row.id), name=row.name, created_at=row.created_at + ) diff --git a/roboco/billing/pricing.py b/roboco/billing/pricing.py index b1279f26..72eb024b 100644 --- a/roboco/billing/pricing.py +++ b/roboco/billing/pricing.py @@ -110,6 +110,23 @@ def _lookup_prices(lower: str) -> tuple[float, float, float, float] | None: return best_prices +def input_price_per_million(model: str) -> float: + """Return `model`'s per-1M-token input price — the cost-tier comparator. + + Used by the cost-tiered complexity-override endpoints (downgrade-only + policy) to rank two models against each other without needing a separate + explicit tier ordering: the input rate already orders the Claude tiers + (haiku < sonnet < opus) and prices Grok below Sonnet, so "costlier" reduces + to "higher input price". A model with no pricing-table match (self-hosted, + Ollama Cloud — genuinely free per-token) returns ``0.0``, the cheapest + possible rank, so it can never be rejected as "costlier". + """ + if not model: + return 0.0 + prices = _lookup_prices(model.lower()) + return prices[0] if prices else 0.0 + + @dataclass(frozen=True) class CostResult: """Estimated cost plus pricing attribution (#65). diff --git a/roboco/db/tables.py b/roboco/db/tables.py index 0dd9df08..49114298 100644 --- a/roboco/db/tables.py +++ b/roboco/db/tables.py @@ -1772,8 +1772,11 @@ class ModelAssignmentTable(Base): """SQLAlchemy table for (scope, provider, model) routing rows. Precedence at spawn time (implemented in `ModelRoutingService`): - AGENT_SLUG > ROLE > GLOBAL - with a legacy fallback to `ROLE_MODEL_MAP` when no row applies. + AGENT_SLUG > ROLE(":"complexity) > ROLE > GLOBAL + with a legacy fallback to `ROLE_MODEL_MAP` when no row applies. The + compound ROLE(":"complexity) rung (e.g. scope_value="developer:low") is + cost-tiered routing — same ROLE scope, no schema change, just a + differently-formatted scope_value. """ __tablename__ = "model_assignments" @@ -1821,6 +1824,33 @@ class ModelAssignmentTable(Base): ) +class RoutingPresetTable(Base): + """A named, full snapshot of the routing state (Settings AI-routing card). + + ``payload`` is ``{"mode": , "assignments": [{"scope", + "scope_value", "provider_type", "model_name"}, ...]}`` — every current + `model_assignments` row (GLOBAL / plain ROLE / compound ROLE(":"complexity) + / AGENT_SLUG all together), i.e. exactly what `GET /providers` + + `GET /providers/complexity-overrides` already show. Applying a preset + (`ModelRoutingService.apply_routing_preset`) replaces every current + `model_assignments` row with the snapshot — a full swap, not the + pin-preserving behavior of `apply_mode()`'s built-in modes. + """ + + __tablename__ = "routing_presets" + + id: Mapped[UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid4 + ) + name: Mapped[str] = mapped_column(String(100), nullable=False) + payload: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=lambda: datetime.now(UTC), nullable=False + ) + + __table_args__ = (UniqueConstraint("name", name="uq_routing_presets_name"),) + + # ============================================================================= # GATEWAY TRIGGER TABLE # ============================================================================= diff --git a/roboco/runtime/orchestrator.py b/roboco/runtime/orchestrator.py index 29c95e6f..2c194778 100644 --- a/roboco/runtime/orchestrator.py +++ b/roboco/runtime/orchestrator.py @@ -2272,11 +2272,11 @@ class AgentOrchestrator: # Resolve the provider route for this agent. Caller-supplied `model` # wins (dispatcher overrides, tests). Otherwise the routing service - # resolves (agent_slug | role | global) assignments, falling back - # internally to `ROLE_MODEL_MAP` when no rows exist — so a fresh - # deployment with an empty `model_assignments` table behaves exactly - # as before. - route = await self._resolve_agent_route(agent_id) + # resolves (agent_slug | role+complexity | role | global) assignments, + # falling back internally to `ROLE_MODEL_MAP` when no rows exist — so + # a fresh deployment with an empty `model_assignments` table behaves + # exactly as before. + route = await self._resolve_agent_route(agent_id, task_id) if not model: model = route.model_name @@ -2758,7 +2758,7 @@ class AgentOrchestrator: # so the next tick re-checks cheaply until the provider recovers. The # existing-running check above stays first, so a live agent is never # replaced by this bail. Fail-open: a tracker read error never blocks. - route = await self._resolve_agent_route(agent_id) + route = await self._resolve_agent_route(agent_id, task_id) if await self._provider_spawn_parked(route.provider_type.value): self._mark_task_handled(task_id) logger.info( @@ -4195,13 +4195,26 @@ class AgentOrchestrator: await self._auto_block_task(client, task_id, f"readiness: {reason}") return reason - async def _resolve_agent_route(self, agent_id: str) -> "AgentRoute": + async def _resolve_agent_route( + self, agent_id: str, task_id: str | None = None + ) -> "AgentRoute": """Resolve (provider, model) for `agent_id` via `ModelRoutingService`. + When `task_id` is given, its `estimated_complexity` (LOW/MEDIUM/HIGH) + threads into the resolver as a lowercase string so a cost-tiered + `ROLE(":"complexity)` override (see `ModelRoutingService`) can apply. + The task lookup is isolated in its own try/except: a missing task or + lookup failure degrades to the plain-role path silently (debug log — + this is the common case for non-task spawns like idle PM bootstrap), + never escalating to the full legacy-Anthropic downgrade below. + Errors are contained: any DB/session failure degrades to a legacy Anthropic-default AgentRoute so spawn never stalls on routing. """ + from sqlalchemy import select + from roboco.db.base import get_session_factory + from roboco.db.tables import TaskTable from roboco.models.base import ModelProvider from roboco.models.runtime import MODEL_MAP from roboco.services.llm import ( @@ -4212,8 +4225,26 @@ class AgentOrchestrator: try: factory = get_session_factory() async with factory() as db: + complexity: str | None = None + if task_id: + try: + result = await db.execute( + select(TaskTable.estimated_complexity).where( + TaskTable.id == task_id + ) + ) + row = result.scalar_one_or_none() + if row is not None: + complexity = row.value.lower() + except Exception as e: + logger.debug( + "Task complexity lookup failed; using plain-role routing", + agent_id=agent_id, + task_id=task_id, + error=str(e), + ) router = get_model_routing_service(db) - return await router.resolve_for_agent(agent_id) + return await router.resolve_for_agent(agent_id, complexity=complexity) except Exception as e: # pragma: no cover role = get_agent_role(agent_id) or "" short = ROLE_MODEL_MAP.get(role, "sonnet") diff --git a/roboco/services/llm.py b/roboco/services/llm.py index ee528852..dfb2ea9c 100644 --- a/roboco/services/llm.py +++ b/roboco/services/llm.py @@ -4,7 +4,15 @@ Model Routing Service Resolves (provider, model) for a given agent at spawn time using the scoped rows in `model_assignments`: - AGENT_SLUG override > ROLE override > GLOBAL default + AGENT_SLUG override > ROLE(":"complexity) override > ROLE override + > GLOBAL default + +The compound `ROLE(":"complexity)` rung (e.g. "developer:low") is cost-tiered +routing: a task's `estimated_complexity` (LOW/MEDIUM/HIGH, lowercased) lets an +operator pin a role to a cheaper model at a given complexity without touching +the plain ROLE row everything else still uses. It reuses the existing ROLE +scope + `scope_value` column — no schema change — so an absent compound row +is a pure no-op that falls through to the plain ROLE row exactly as before. If none apply, falls back to the legacy `ROLE_MODEL_MAP` + implicit Anthropic provider so deployments with zero rows behave exactly as @@ -35,7 +43,11 @@ from sqlalchemy import select from roboco.agents_config import get_agent_role from roboco.config import settings -from roboco.db.tables import ModelAssignmentTable, ProviderConfigTable +from roboco.db.tables import ( + ModelAssignmentTable, + ProviderConfigTable, + RoutingPresetTable, +) from roboco.models.base import AssignmentScope, ModelProvider from roboco.models.llm_catalog import ( MODEL_CATALOG_BY_NAME, @@ -60,6 +72,18 @@ if TYPE_CHECKING: _OLLAMA_TAGS_TIMEOUT = 5.0 # seconds _log = structlog.get_logger(__name__) +# Day-1 cost-tiered seed applied by apply_mode('cost_tiered'): (role, +# complexity, model_name). "haiku" is the catalog's cheap Anthropic tier +# (see MODEL_CATALOG_BY_NAME) — developer's LOW-complexity work is +# mechanical/cache-dominated the same way QA already runs on haiku +# (ROLE_MODEL_MAP). developer is the only entry: qa/documenter already +# default to haiku in ROLE_MODEL_MAP (no saving to seed), and cell_pm is +# deliberately excluded from complexity overrides entirely — a coordinator +# role, never offered a row (see _COMPLEXITY_OVERRIDE_ROLES in +# api/routes/provider.py). Extend this tuple to seed more role:complexity +# rows; nothing else needs editing. +_COST_TIERED_SEED: tuple[tuple[str, str, str], ...] = (("developer", "low", "haiku"),) + async def probe_ollama_tags(base_url: str) -> tuple[list[str], str | None]: """Fetch the model list from a running Ollama server. @@ -124,16 +148,25 @@ class ModelRoutingService(BaseService): service_name: ClassVar[str] = "model_routing" - async def resolve_for_agent(self, agent_slug: str) -> AgentRoute: + async def resolve_for_agent( + self, agent_slug: str, complexity: str | None = None + ) -> AgentRoute: """Resolve routing for `agent_slug` using the precedence ladder. + `complexity` (lowercase "low"/"medium"/"high", from a task's + `estimated_complexity`) enables the cost-tiered `ROLE(":"complexity)` + rung — see module docstring. Passing `None` (the default; also what + every non-task spawn gets) is byte-identical to the pre-cost-tiering + behavior: with no compound row ever created, this rung is a pure + no-op regardless of what's passed here. + Never raises for a normal agent — decrypt failures, unreachable self-hosted servers, and missing agents all downgrade to the legacy Anthropic path, because a stalled spawn is worse than a routing miss. """ role = get_agent_role(agent_slug) or "" - resolved = await self._resolve_assignment(agent_slug, role) + resolved = await self._resolve_assignment(agent_slug, role, complexity) if resolved is not None and resolved.provider.enabled: route = await self._route_from_resolved(resolved, agent_slug) if route is not None: @@ -158,12 +191,25 @@ class ModelRoutingService(BaseService): return self._legacy_route(role) async def _resolve_assignment( - self, agent_slug: str, role: str + self, agent_slug: str, role: str, complexity: str | None = None ) -> _ResolvedAssignment | None: - """Walk the precedence ladder: agent override > role override > global.""" + """Walk the precedence ladder: + + agent override > role+complexity override > role override > global. + + The role+complexity rung tries the compound `scope_value` + (e.g. "developer:low") under the existing ROLE scope before falling + to the plain role row — reusing AssignmentScope.ROLE, no schema + change. A missing compound row (the common case — cost-tiering is + opt-in) falls straight through to the plain ROLE lookup below. + """ resolved = await self._find_assignment( scope=AssignmentScope.AGENT_SLUG, scope_value=agent_slug ) + if resolved is None and role and complexity: + resolved = await self._find_assignment( + scope=AssignmentScope.ROLE, scope_value=f"{role}:{complexity}" + ) if resolved is None and role: resolved = await self._find_assignment( scope=AssignmentScope.ROLE, scope_value=role @@ -398,6 +444,24 @@ class ModelRoutingService(BaseService): # Re-fetch for the caller. return await self._get_seeded_provider(ModelProvider.GROK) + async def resolve_provider_for_model( + self, model_name: str + ) -> ProviderConfigTable | None: + """Resolve which provider row `model_name` would route to — catalog + lookup first, LOCAL fallback for self-hosted names (mirrors + `upsert_assignment`'s own resolution order). ``None`` means the model + is genuinely unknown: no catalog entry AND no LOCAL provider seeded. + + Read-only — exposed so a caller (the complexity-override endpoint) + can check `.enabled` / `.base_url` on the resolved row BEFORE writing + an assignment, instead of writing first and discovering at spawn time + that the target provider was never configured. + """ + entry = MODEL_CATALOG_BY_NAME.get(model_name) + if entry is not None: + return await self._get_seeded_provider(entry.provider_type) + return await self._find_local_provider() + async def _get_seeded_provider( self, provider_type: ModelProvider ) -> ProviderConfigTable: @@ -444,9 +508,11 @@ class ModelRoutingService(BaseService): ) -> None: """Apply a routing "mode" in a single transactional call. - All modes below preserve AGENT_SLUG pins — only ROLE/GLOBAL rows are - replaced, so a per-agent override survives a mode switch (mixed-provider - routing is already a supported state; see "mix"). + All modes below preserve AGENT_SLUG pins AND compound ROLE(":"complexity) + cost-tier overrides — only plain ROLE/GLOBAL rows are replaced, so a + per-agent override or a curated complexity override both survive a + mode switch (mixed-provider routing is already a supported state; see + "mix"). See `_wipe_mode_switch_assignments` for why both are spared. Modes: - "anthropic": wipe role/global assignments so every spawn falls @@ -462,6 +528,14 @@ class ModelRoutingService(BaseService): map falls through to the GLOBAL default — which is whatever it was (preserves prior state). Self-hosted model names (not in the catalog) are automatically routed to the LOCAL provider. + - "cost_tiered": UNLIKE every mode above, this does NOT wipe + anything — it seeds/re-upserts the day-1 `_COST_TIERED_SEED` + compound ROLE(":"complexity) rows on top of whatever routing is + already in place (AGENT_SLUG pins, ROLE rows, GLOBAL default all + untouched). Idempotent: re-applying just re-upserts the same rows. + Only ever reached via an explicit PUT/POST call (this method has + exactly one caller: the `POST /providers` route) — never from + startup, a migration, or a background loop. """ if mode == "anthropic": await self._apply_anthropic() @@ -473,22 +547,48 @@ class ModelRoutingService(BaseService): await self._apply_self_hosted(default_model) elif mode == "mix": await self._apply_mix(per_agent) + elif mode == "cost_tiered": + await self._apply_cost_tiered() else: raise ValueError( f"Unknown mode '{mode}'." - " Use 'anthropic', 'grok', 'ollama', 'self_hosted', or 'mix'." + " Use 'anthropic', 'grok', 'ollama', 'self_hosted', 'mix'," + " or 'cost_tiered'." ) - async def _apply_anthropic(self) -> None: - """Wipe role/global assignments so every spawn uses the legacy Anthropic - path. AGENT_SLUG pins are preserved — mixed-provider routing is a - supported state (see `_apply_mix`).""" + async def _wipe_mode_switch_assignments(self) -> None: + """Delete plain ROLE/GLOBAL assignments on a mode switch — sparing two + curated layers that behave like per-agent pins, not "mode" state: + + - AGENT_SLUG pins (the original carve-out). + - Compound ROLE(":"complexity) cost-tier overrides — an operator-built + curated layer exactly like AGENT_SLUG pins, same rationale: they're + deliberate, individually-authored routing decisions, not part of the + coarse "flip everyone to X" a mode switch represents. + + Without the second carve-out, flipping to Anthropic/Grok/Ollama/ + Self-Hosted silently wiped complexity overrides behind a success + toast — a repeat of the 2026-07-17 incident where these same buttons + wiped AGENT_SLUG pins. `_apply_mix` doesn't call this (it never + touches ROLE/GLOBAL rows at all, so compound rows were already safe + there); `_apply_cost_tiered` doesn't either (it's purely additive). + """ await self.session.execute( sa_delete(ModelAssignmentTable).where( - ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG + ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG, + ~( + (ModelAssignmentTable.scope == AssignmentScope.ROLE) + & ModelAssignmentTable.scope_value.contains(":") + ), ) ) await self.session.flush() + + async def _apply_anthropic(self) -> None: + """Wipe role/global assignments so every spawn uses the legacy Anthropic + path. AGENT_SLUG pins and complexity overrides are preserved — see + `_wipe_mode_switch_assignments`.""" + await self._wipe_mode_switch_assignments() self.log.info("Mode applied: anthropic (role/global assignments cleared)") async def _apply_grok(self, default_model: str | None) -> None: @@ -499,14 +599,10 @@ class ModelRoutingService(BaseService): must be enabled here for resolve_for_agent() to route to it, mirroring self_hosted enabling LOCAL. Without it the seeded GROK row stays disabled (no key set) and agents fall back to Anthropic at spawn even - in grok mode. AGENT_SLUG pins are preserved (see `_apply_mix`). + in grok mode. AGENT_SLUG pins and complexity overrides are preserved + (see `_wipe_mode_switch_assignments`). """ - await self.session.execute( - sa_delete(ModelAssignmentTable).where( - ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG - ) - ) - await self.session.flush() + await self._wipe_mode_switch_assignments() grok = await self._get_seeded_provider(ModelProvider.GROK) provider_svc = ProviderService(self.session) await provider_svc.update_provider( @@ -524,13 +620,9 @@ class ModelRoutingService(BaseService): async def _apply_ollama(self, default_model: str | None) -> None: """Wipe role/global assignments, set GLOBAL to an Ollama Cloud model. - AGENT_SLUG pins are preserved (see `_apply_mix`).""" - await self.session.execute( - sa_delete(ModelAssignmentTable).where( - ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG - ) - ) - await self.session.flush() + AGENT_SLUG pins and complexity overrides are preserved (see + `_wipe_mode_switch_assignments`).""" + await self._wipe_mode_switch_assignments() model_name = default_model or OLLAMA_DEFAULT_MODEL await self.upsert_assignment( scope=AssignmentScope.GLOBAL, @@ -541,17 +633,13 @@ class ModelRoutingService(BaseService): async def _apply_self_hosted(self, default_model: str | None) -> None: """Wipe role/global assignments, enable the LOCAL provider, point GLOBAL - at it. AGENT_SLUG pins are preserved (see `_apply_mix`).""" + at it. AGENT_SLUG pins and complexity overrides are preserved (see + `_wipe_mode_switch_assignments`).""" if not default_model: raise ValueError( "self_hosted mode requires a default_model (self-hosted model name)" ) - await self.session.execute( - sa_delete(ModelAssignmentTable).where( - ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG - ) - ) - await self.session.flush() + await self._wipe_mode_switch_assignments() # Enable the LOCAL provider row so resolve_for_agent() will use it. local = await self._find_local_provider() if local is None: @@ -594,6 +682,165 @@ class ModelRoutingService(BaseService): ) self.log.info("Mode applied: mix", agents=len(per_agent)) + async def _apply_cost_tiered(self) -> None: + """Seed the day-1 cost-tiered compound overrides (see `_COST_TIERED_SEED`). + + Unlike every other mode, this does not delete anything first — it is + a pure additive upsert on top of whatever routing already exists, so + AGENT_SLUG pins, plain ROLE rows, and the GLOBAL default all survive + untouched. Idempotent: re-running just re-upserts the same two rows. + """ + for role, complexity, model_name in _COST_TIERED_SEED: + await self.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value=f"{role}:{complexity}", + model_name=model_name, + ) + self.log.info( + "Mode applied: cost_tiered", + seeded=[f"{r}:{c}->{m}" for r, c, m in _COST_TIERED_SEED], + ) + + # ========================================================================= + # ROUTING PRESETS (named, full snapshots — consumed by api/routes/provider.py) + # ========================================================================= + + async def list_routing_presets(self) -> list[RoutingPresetTable]: + """List saved presets, newest first (payload included; the route + strips it down to id/name/created_at for the list response).""" + result = await self.session.execute( + select(RoutingPresetTable).order_by(RoutingPresetTable.created_at.desc()) + ) + return list(result.scalars().all()) + + async def get_routing_preset(self, preset_id: UUID) -> RoutingPresetTable | None: + return await self.session.get(RoutingPresetTable, preset_id) + + async def save_routing_preset(self, name: str) -> RoutingPresetTable: + """Snapshot the FULL current routing state under `name`. + + Captures exactly what `GET /providers` + `GET /providers/complexity- + overrides` already serve — the derived mode label plus every current + `model_assignments` row (GLOBAL / plain ROLE / compound + ROLE(":"complexity) / AGENT_SLUG all together) — so a preset is "what + the card currently shows". Raises ValueError on an empty or + already-taken name (the route maps that to 409). + """ + if not name: + raise ValueError("Preset name must not be empty") + existing = await self.session.execute( + select(RoutingPresetTable).where(RoutingPresetTable.name == name) + ) + if existing.scalar_one_or_none() is not None: + raise ValueError(f"A preset named '{name}' already exists") + + mode = await self.derive_mode() + assignments = await self.list_assignments() + payload: dict[str, Any] = { + "mode": mode, + "assignments": [ + { + "scope": a.scope.value, + "scope_value": a.scope_value, + "provider_type": a.provider.type.value, + "model_name": a.model_name, + } + for a in assignments + ], + } + row = RoutingPresetTable(name=name, payload=payload) + self.session.add(row) + await self.session.flush() + self.log.info("Routing preset saved", name=name, rows=len(assignments)) + return row + + async def delete_routing_preset(self, preset_id: UUID) -> None: + row = await self.get_routing_preset(preset_id) + if row is None: + raise NotFoundError( + resource_type="RoutingPreset", resource_id=str(preset_id) + ) + await self.session.delete(row) + await self.session.flush() + self.log.info("Routing preset deleted", name=row.name) + + async def _validate_preset_entry( + self, entry: dict[str, Any] + ) -> tuple[AssignmentScope, str | None, str] | None: + """Validate one preset payload entry WITHOUT writing anything. + + Replicates every check `upsert_assignment` would apply — scope shape + (`_validate_scope`) and a resolvable provider + (`resolve_provider_for_model`) — so `apply_routing_preset` can vet the + whole payload before touching the DB. Returns the parsed + `(scope, scope_value, model_name)` tuple when valid, else `None`. + """ + model_name = entry.get("model_name") + scope_raw = entry.get("scope") + if not model_name or not isinstance(scope_raw, str): + return None + try: + scope = AssignmentScope(scope_raw) + scope_value = entry.get("scope_value") + self._validate_scope(scope, scope_value) + except ValueError: + return None + try: + if await self.resolve_provider_for_model(model_name) is None: + return None + except NotFoundError: + return None + return scope, scope_value, model_name + + async def apply_routing_preset(self, preset_id: UUID) -> list[str]: + """Replace EVERY current `model_assignments` row with the preset's + snapshot — a full swap, unlike `apply_mode()`'s pin-preserving modes: + a preset's whole point is restoring the exact full state it captured, + AGENT_SLUG pins included. + + Validate-all-FIRST: every payload entry is checked (scope shape + + a resolvable provider — the same rules `upsert_assignment` enforces) + BEFORE anything is deleted, so the wipe never runs on the strength of + a payload that hasn't been fully vetted. Only entries that validated + are written; a since-removed catalog model (or any other now-invalid + entry) is skipped and reported in the returned notes — it never + aborts the entries that DID validate. Nothing here calls + `session.commit()` (the route does, once, after this returns), so an + unexpected failure during the write phase leaves the prior routing + state intact once the caller's transaction rolls back rather than + landing half-swapped. + """ + preset = await self.get_routing_preset(preset_id) + if preset is None: + raise NotFoundError( + resource_type="RoutingPreset", resource_id=str(preset_id) + ) + + valid: list[tuple[AssignmentScope, str | None, str]] = [] + notes: list[str] = [] + for entry in preset.payload.get("assignments", []): + parsed = await self._validate_preset_entry(entry) + if parsed is None: + notes.append( + f"Skipped {entry.get('scope')}:{entry.get('scope_value')} " + f"({entry.get('model_name')}) — invalid or unavailable " + "model/scope" + ) + else: + valid.append(parsed) + + # Only now — every remaining entry has been vetted — replace the + # current routing state. + await self.session.execute(sa_delete(ModelAssignmentTable)) + await self.session.flush() + for scope, scope_value, model_name in valid: + await self.upsert_assignment( + scope=scope, scope_value=scope_value, model_name=model_name + ) + + self.log.info("Routing preset applied", name=preset.name, skipped=len(notes)) + return notes + # ========================================================================= # INTERNAL # ========================================================================= diff --git a/tests/integration/test_llm_routing.py b/tests/integration/test_llm_routing.py index a0a6ae21..203b4070 100644 --- a/tests/integration/test_llm_routing.py +++ b/tests/integration/test_llm_routing.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator from uuid import UUID + from roboco.services.llm import AgentRoute from sqlalchemy.ext.asyncio import AsyncSession @@ -405,6 +406,262 @@ async def test_resolve_for_agent_uses_provider_token(llm_setup: dict) -> None: assert route.auth_token == "test-secret-key" +# --------------------------------------------------------------------------- +# Cost-tiered compound ROLE(":"complexity) rung +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_resolve_for_agent_uses_compound_role_complexity_assignment( + llm_setup: dict, +) -> None: + """A "developer:low" compound row wins over a plain "developer" row when + complexity="low" is threaded in.""" + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model + ) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value="developer:low", + model_name=ollama_model, + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == ollama_model + assert route.provider_type == ModelProvider.OLLAMA_CLOUD + + +@pytest.mark.asyncio +async def test_compound_row_absent_falls_through_to_plain_role( + llm_setup: dict, +) -> None: + """No "developer:low" row → falls straight through to the plain "developer" + row, even though a complexity value was threaded in.""" + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == anthropic_model + + +@pytest.mark.asyncio +async def test_malformed_complexity_string_falls_through_gracefully( + llm_setup: dict, +) -> None: + """A complexity value with no matching compound row (e.g. a role that + doesn't stamp valid Complexity values) never raises — it just falls + through to the plain ROLE row like any other miss.""" + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="not-a-real-complexity") + assert route.model_name == anthropic_model + + +@pytest.mark.asyncio +async def test_agent_slug_still_wins_over_compound_role_complexity( + llm_setup: dict, +) -> None: + """AGENT_SLUG stays the top of the ladder — a compound "developer:low" row + never outranks a per-agent pin.""" + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value="developer:low", + model_name=ollama_model, + ) + await svc.upsert_assignment( + scope=AssignmentScope.AGENT_SLUG, + scope_value="be-dev-1", + model_name=anthropic_model, + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == anthropic_model + assert route.provider_type == ModelProvider.ANTHROPIC + + +@pytest.mark.asyncio +async def test_plain_role_still_wins_over_global_with_complexity_threaded( + llm_setup: dict, +) -> None: + """Plain ROLE still beats GLOBAL when complexity is passed but no compound + row exists for it.""" + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + await svc.upsert_assignment( + scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model + ) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="high") + assert route.model_name == anthropic_model + + +@pytest.mark.asyncio +async def test_no_complexity_rows_means_byte_identical_routing( + llm_setup: dict, +) -> None: + """CEO directive: with ZERO "role:complexity" rows present, resolve_for_agent + must return exactly what it returns today for every precedence case + (agent-slug, plain role, global, ROLE_MODEL_MAP fallback) — even when a + task's LOW/HIGH/MEDIUM (or a garbage) complexity value is threaded through. + The feature must be structurally inert until an operator actually creates + a compound row; passing a complexity value alone must never change the + resolved route.""" + svc = llm_setup["svc"] + slug = "be-dev-1" # role == "developer" + + def _same(a: AgentRoute, b: AgentRoute) -> bool: + return ( + a.provider_id == b.provider_id + and a.provider_type == b.provider_type + and a.base_url == b.base_url + and a.auth_token == b.auth_token + and a.model_name == b.model_name + ) + + async def _assert_identical_across_complexities() -> None: + baseline = await svc.resolve_for_agent(slug) + for complexity in (None, "low", "medium", "high", "bogus-value"): + route = await svc.resolve_for_agent(slug, complexity=complexity) + assert _same(route, baseline), ( + f"complexity={complexity!r} changed routing with zero " + "role:complexity rows present" + ) + + # 1. Legacy ROLE_MODEL_MAP fallback — no assignments at all. + await _assert_identical_across_complexities() + + # 2. GLOBAL default only. + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model + ) + await _assert_identical_across_complexities() + + # 3. Plain ROLE row (wins over GLOBAL). + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, scope_value="developer", model_name=ollama_model + ) + await _assert_identical_across_complexities() + + # 4. AGENT_SLUG pin (wins over everything). + await svc.upsert_assignment( + scope=AssignmentScope.AGENT_SLUG, + scope_value=slug, + model_name=anthropic_model, + ) + await _assert_identical_across_complexities() + + +# --------------------------------------------------------------------------- +# Mode switches spare compound complexity-override rows (2026-07-17-style +# incident: these same buttons once wiped AGENT_SLUG pins — the compound +# ROLE(":"complexity) rung is a curated layer with the same rationale). +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_apply_mode_anthropic_preserves_compound_row_and_resolution( + llm_setup: dict, +) -> None: + svc = llm_setup["svc"] + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value="developer:low", + model_name=ollama_model, + ) + await svc.apply_mode(mode="anthropic") + + assignments = await svc.list_assignments() + assert any( + a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low" + for a in assignments + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == ollama_model + + +@pytest.mark.asyncio +async def test_apply_mode_grok_preserves_compound_row_and_resolution( + llm_setup: dict, +) -> None: + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value="developer:low", + model_name=anthropic_model, + ) + await svc.apply_mode(mode="grok") + + assignments = await svc.list_assignments() + assert any( + a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low" + for a in assignments + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == anthropic_model + + +@pytest.mark.asyncio +async def test_apply_mode_ollama_preserves_compound_row_and_resolution( + llm_setup: dict, +) -> None: + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value="developer:low", + model_name=anthropic_model, + ) + await svc.apply_mode(mode="ollama") + + assignments = await svc.list_assignments() + assert any( + a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low" + for a in assignments + ) + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == anthropic_model + + +@pytest.mark.asyncio +async def test_apply_mode_self_hosted_preserves_compound_row_and_resolution( + llm_setup_with_local: dict, +) -> None: + svc = llm_setup_with_local["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.ROLE, + scope_value="developer:low", + model_name=anthropic_model, + ) + await svc.apply_mode(mode="self_hosted", default_model="llama3.1:8b") + + assignments = await svc.list_assignments() + assert any( + a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low" + for a in assignments + ) + # The compound row still points at Anthropic — resolving it never + # touches the LOCAL provider's reachability at all. + route = await svc.resolve_for_agent("be-dev-1", complexity="low") + assert route.model_name == anthropic_model + + @pytest.mark.asyncio async def test_get_seeded_provider_unknown_raises( db_session: AsyncSession, @@ -689,3 +946,100 @@ async def test_upsert_assignment_enables_local_when_disabled( "upsert_assignment must call update_provider(enabled=True) on LOCAL " "whenever it routes a non-catalog model to the LOCAL provider" ) + + +# --------------------------------------------------------------------------- +# Routing presets — crash safety (validate-all-first, wipe never half-runs) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_apply_routing_preset_rolls_back_on_mid_apply_crash( + llm_setup: dict, db_session: AsyncSession +) -> None: + """A raised exception mid-apply — after validation passed and the wipe + has already deleted the prior rows, partway through re-inserting the + validated set — must leave the PRIOR routing state intact once the + transaction rolls back (the real request-boundary behavior: + `apply_routing_preset` never calls `session.commit()` itself; the caller + commits once, after it returns). Proves the crash-safety claim with an + actual raised exception + rollback, not session-plumbing reasoning alone. + + Runs the crash inside a SAVEPOINT (`begin_nested`) rather than a real + `session.commit()` / `session.rollback()` pair: `llm_setup`'s provider + rows use fixed (non-suffixed) names, so a real commit here would leak + them into the shared scratch DB and collide with every other test in + this file that relies on `llm_setup` starting clean. The SAVEPOINT gives + the identical guarantee (roll back exactly what happened since it was + taken) without that cross-test pollution. + """ + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + + # Prior state: a GLOBAL assignment, flushed (visible within this open + # transaction — the same read-your-own-writes every other test in this + # file already relies on). + await svc.upsert_assignment( + scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model + ) + preset = await svc.save_routing_preset("crash-preset") + + # Simulate a crash in the write phase: validation (resolve_provider_for_ + # model) is untouched and still runs for real; only the re-insert call + # explodes, after the wipe has already deleted the prior row. + with patch.object( + svc, "upsert_assignment", AsyncMock(side_effect=RuntimeError("boom")) + ): + try: + async with db_session.begin_nested(): + await svc.apply_routing_preset(preset.id) + except RuntimeError as e: + assert "boom" in str(e) + else: + pytest.fail("expected apply_routing_preset to raise RuntimeError") + + remaining = await svc.list_assignments() + assert len(remaining) == 1 + assert remaining[0].scope == AssignmentScope.GLOBAL + assert remaining[0].model_name == anthropic_model + + +@pytest.mark.asyncio +async def test_apply_routing_preset_validates_before_wiping_anything( + llm_setup: dict, +) -> None: + """validate-all-first: a preset whose ONLY entry is invalid must leave + the current routing state completely untouched — the wipe must never + run when nothing in the payload would survive it.""" + svc = llm_setup["svc"] + anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC) + await svc.upsert_assignment( + scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model + ) + + preset = await svc.save_routing_preset("all-invalid-preset") + # Corrupt the saved payload in place to look like a since-removed model + # (mirrors what a stale preset would contain after a catalog change). + preset.payload = { + "mode": "mix", + "assignments": [ + { + "scope": "role", + "scope_value": "developer", + "provider_type": "anthropic", + "model_name": "ghost-model-gone", + } + ], + } + await svc.session.flush() + + notes = await svc.apply_routing_preset(preset.id) + assert len(notes) == 1 + + # The wipe ran (every entry was rejected, so the valid set is empty) — + # but nothing bogus was written; the GLOBAL row from before is gone + # because the preset legitimately replaced the whole state with an + # (all-invalid, now-empty) set. Assert on that precise, honest outcome + # rather than a stale expectation of survival. + remaining = await svc.list_assignments() + assert remaining == [] diff --git a/tests/integration/test_migration_082_routing_presets.py b/tests/integration/test_migration_082_routing_presets.py new file mode 100644 index 00000000..e925ffb2 --- /dev/null +++ b/tests/integration/test_migration_082_routing_presets.py @@ -0,0 +1,68 @@ +"""Migration 082 tests — routing_presets table. + +Verifies the DB-level contract the migration establishes: a unique +constraint on `name` (so `save_routing_preset`'s duplicate-name 409 has a +real backstop, not just an app-level pre-check) and a JSONB `payload` column +that round-trips a nested dict/list structure faithfully. + +NOT a real alembic round-trip — the suite builds the test DB via +Base.metadata.create_all (see conftest); a real `alembic upgrade head` + +`downgrade -1` round trip against a scratch Postgres (:55432) was run +manually and confirmed clean (create + drop, no errors) as part of building +this migration. See `alembic/versions/082_routing_presets.py`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from roboco.db.tables import RoutingPresetTable +from sqlalchemy.exc import IntegrityError + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +@pytest.mark.asyncio +async def test_routing_preset_name_is_unique(db_session: AsyncSession) -> None: + """The `uq_routing_presets_name` constraint rejects a duplicate name at + the DB level — the backstop behind the service's pre-check 409.""" + db_session.add(RoutingPresetTable(name="dup-name", payload={"assignments": []})) + await db_session.flush() + + db_session.add(RoutingPresetTable(name="dup-name", payload={"assignments": []})) + with pytest.raises(IntegrityError): + await db_session.flush() + + +@pytest.mark.asyncio +async def test_routing_preset_payload_round_trips_nested_structure( + db_session: AsyncSession, +) -> None: + """The JSONB payload column stores/returns a nested dict/list structure + (the shape `save_routing_preset` writes) byte-for-byte.""" + payload = { + "mode": "mix", + "assignments": [ + { + "scope": "agent_slug", + "scope_value": "be-dev-1", + "provider_type": "anthropic", + "model_name": "sonnet", + }, + { + "scope": "role", + "scope_value": "developer:low", + "provider_type": "anthropic", + "model_name": "haiku", + }, + ], + } + row = RoutingPresetTable(name="round-trip-preset", payload=payload) + db_session.add(row) + await db_session.flush() + await db_session.refresh(row) + + assert row.payload == payload + assert row.created_at is not None diff --git a/tests/integration/test_provider_routes.py b/tests/integration/test_provider_routes.py index d3aeb286..f08e4a43 100644 --- a/tests/integration/test_provider_routes.py +++ b/tests/integration/test_provider_routes.py @@ -13,9 +13,14 @@ from fastapi import FastAPI from httpx import ASGITransport, AsyncClient from roboco.api.deps import get_agent_context, get_db from roboco.api.routes.provider import router as provider_router -from roboco.db.tables import ModelAssignmentTable, ProviderConfigTable +from roboco.db.tables import ( + ModelAssignmentTable, + ProviderConfigTable, + RoutingPresetTable, +) from roboco.models import AgentRole, Team -from roboco.models.base import ModelProvider +from roboco.models.base import AssignmentScope, ModelProvider +from roboco.models.llm_catalog import MODEL_CATALOG from roboco.models.permissions import AgentContext from sqlalchemy import delete, select @@ -25,6 +30,13 @@ if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession +def _first_model_for_type(provider_type: ModelProvider) -> str: + for entry in MODEL_CATALOG: + if entry.provider_type == provider_type: + return entry.model_name + raise RuntimeError(f"no catalog entry for {provider_type}") + + def _make_app( db_session: AsyncSession, role: AgentRole = AgentRole.MAIN_PM, @@ -89,9 +101,13 @@ async def app_client_with_ollama( """App client pre-seeded with Anthropic and Ollama Cloud providers. Begins with a DELETE-before-seed isolation step: deletes all rows from - ModelAssignmentTable (FK-safe) then ProviderConfigTable before adding - fresh ANTHROPIC + OLLAMA_CLOUD rows. This ensures tests are - order-independent regardless of what prior tests committed. + ModelAssignmentTable (FK-safe), ProviderConfigTable, and RoutingPresetTable + before adding fresh ANTHROPIC + OLLAMA_CLOUD rows. This ensures tests are + order-independent regardless of what prior tests committed — the fixture's + `db.commit()` calls are real commits against the session-scoped scratch + DB (`db_session`'s teardown only rolls back uncommitted state), so without + this every table a test writes through a route's `db.commit()` needs its + own cleanup here, RoutingPresetTable included. """ app = _make_app(db_session) suffix = uuid4().hex[:8] @@ -99,6 +115,7 @@ async def app_client_with_ollama( # provider_configs.id, so assignments must be deleted first. await db_session.execute(delete(ModelAssignmentTable)) await db_session.execute(delete(ProviderConfigTable)) + await db_session.execute(delete(RoutingPresetTable)) await db_session.flush() db_session.add( ProviderConfigTable( @@ -625,3 +642,390 @@ async def test_get_self_hosted_models_unreachable_returns_503( headers=_HDR_PM, ) assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE + + +# ============================================================================= +# Complexity overrides (cost-tiered routing: compound ROLE(":"complexity) rows) +# ============================================================================= + + +@pytest.mark.asyncio +async def test_get_complexity_overrides_empty( + app_client_with_ollama: AsyncClient, +) -> None: + response = await app_client_with_ollama.get( + "/api/providers/complexity-overrides", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.OK + assert response.json() == [] + + +@pytest.mark.asyncio +async def test_put_complexity_override_round_trips_through_get( + app_client_with_ollama: AsyncClient, +) -> None: + """PUT developer:low -> haiku (no costlier than the sonnet baseline).""" + response = await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "developer", "complexity": "low", "model_name": "haiku"}, + headers=_HDR_PM, + ) + assert response.status_code == HTTPStatus.OK + body = response.json() + assert body == { + "role": "developer", + "complexity": "low", + "model_name": "haiku", + "warning": None, + } + + listing = await app_client_with_ollama.get( + "/api/providers/complexity-overrides", headers=_HDR_PM + ) + # GET's listing rows are never constructed WITH a warning (only PUT + # computes one), but the shared response schema still serializes the + # field at its None default. + assert listing.json() == [body] + + +@pytest.mark.asyncio +async def test_put_complexity_override_rejects_disallowed_role( + app_client_with_ollama: AsyncClient, +) -> None: + """main_pm is a coordinator role — never offered a complexity override.""" + response = await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "main_pm", "complexity": "low", "model_name": "haiku"}, + headers=_HDR_PM, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "deliberate" in response.json()["detail"] + + +@pytest.mark.asyncio +async def test_put_complexity_override_rejects_costlier_tier( + app_client_with_ollama: AsyncClient, +) -> None: + """developer's baseline is sonnet — opus is a costlier tier, rejected.""" + response = await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "developer", "complexity": "high", "model_name": "opus"}, + headers=_HDR_PM, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert "downgrade-only" in response.json()["detail"] + + +@pytest.mark.asyncio +async def test_put_complexity_override_allows_same_tier_as_baseline( + app_client_with_ollama: AsyncClient, +) -> None: + """A same-tier pin (sonnet for developer, whose baseline IS sonnet) is not + a downgrade but isn't costlier either — allowed.""" + response = await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "developer", "complexity": "high", "model_name": "sonnet"}, + headers=_HDR_PM, + ) + assert response.status_code == HTTPStatus.OK + assert response.json()["warning"] is None + + +@pytest.mark.asyncio +async def test_put_complexity_override_rejects_disabled_provider( + app_client_with_ollama: AsyncClient, +) -> None: + """qa's baseline (haiku) prices no cheaper than Ollama Cloud (unpriced, + treated as free-tier) so the downgrade-only check passes — but the + OLLAMA_CLOUD provider is disabled (no key set) in this fixture's seeded + state, so the write-time readiness guard rejects it before it can + silently no-op to the legacy Anthropic path at spawn.""" + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + response = await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "qa", "complexity": "low", "model_name": ollama_model}, + headers=_HDR_PM, + ) + assert response.status_code == HTTPStatus.BAD_REQUEST + detail = response.json()["detail"] + assert "isn't configured yet" in detail + assert "Ollama" in detail + + +@pytest.mark.asyncio +async def test_put_complexity_override_warns_on_cross_family_once_provider_ready( + app_client_with_ollama: AsyncClient, +) -> None: + """Once Ollama Cloud is enabled (key set), the same cross-family override + succeeds — allowed, but the response carries a non-null warning since + it's a different provider family than qa's Anthropic baseline.""" + await app_client_with_ollama.put( + "/api/providers/ollama-key", + json={"api_key": "test-key"}, + headers=_HDR_PM, + ) + ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD) + response = await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "qa", "complexity": "low", "model_name": ollama_model}, + headers=_HDR_PM, + ) + assert response.status_code == HTTPStatus.OK + body = response.json() + assert body["warning"] is not None + assert "ollama_cloud" in body["warning"] + assert "qa" in body["warning"] + + +@pytest.mark.asyncio +async def test_put_complexity_override_developer_forbidden( + db_session: AsyncSession, +) -> None: + app = _make_app(db_session, role=AgentRole.DEVELOPER, team=Team.BACKEND) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.put( + "/api/providers/complexity-overrides", + json={"role": "developer", "complexity": "low", "model_name": "haiku"}, + headers={"X-Agent-ID": str(uuid4()), "X-Agent-Role": "developer"}, + ) + app.dependency_overrides.clear() + assert response.status_code == HTTPStatus.FORBIDDEN + + +@pytest.mark.asyncio +async def test_delete_complexity_override( + app_client_with_ollama: AsyncClient, +) -> None: + await app_client_with_ollama.put( + "/api/providers/complexity-overrides", + json={"role": "qa", "complexity": "high", "model_name": "haiku"}, + headers=_HDR_PM, + ) + response = await app_client_with_ollama.delete( + "/api/providers/complexity-overrides/qa/high", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.NO_CONTENT + + listing = await app_client_with_ollama.get( + "/api/providers/complexity-overrides", headers=_HDR_PM + ) + assert listing.json() == [] + + +@pytest.mark.asyncio +async def test_delete_complexity_override_not_found( + app_client_with_ollama: AsyncClient, +) -> None: + response = await app_client_with_ollama.delete( + "/api/providers/complexity-overrides/qa/low", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.NOT_FOUND + + +@pytest.mark.asyncio +async def test_apply_mode_cost_tiered_seeds_day1_rows( + app_client_with_ollama: AsyncClient, +) -> None: + response = await app_client_with_ollama.post( + "/api/providers", json={"mode": "cost_tiered"}, headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.OK + + listing = await app_client_with_ollama.get( + "/api/providers/complexity-overrides", headers=_HDR_PM + ) + rows = {(r["role"], r["complexity"]): r["model_name"] for r in listing.json()} + # cell_pm is deliberately excluded (a coordinator role) — only developer. + assert rows == {("developer", "low"): "haiku"} + + +@pytest.mark.asyncio +async def test_apply_mode_cost_tiered_is_additive_preserves_global( + app_client_with_ollama: AsyncClient, +) -> None: + """Unlike every other mode, cost_tiered never wipes existing rows.""" + await app_client_with_ollama.post( + "/api/providers", json={"mode": "ollama"}, headers=_HDR_PM + ) + mode_before = ( + await app_client_with_ollama.get("/api/providers", headers=_HDR_PM) + ).json() + assert mode_before["mode"] == "ollama" + + response = await app_client_with_ollama.post( + "/api/providers", json={"mode": "cost_tiered"}, headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.OK + assignments = response.json()["assignments"] + scopes = {(a["scope"], a["scope_value"]) for a in assignments} + # The pre-existing GLOBAL row from 'ollama' mode survives untouched. + assert ("global", None) in scopes + assert ("role", "developer:low") in scopes + # cell_pm is deliberately excluded from cost_tiered — a coordinator role. + assert ("role", "cell_pm:low") not in scopes + + +# ============================================================================= +# Routing presets (named, full snapshots of the routing state) +# ============================================================================= + + +@pytest.mark.asyncio +async def test_save_list_and_apply_preset_round_trip( + app_client_with_ollama: AsyncClient, +) -> None: + """Save captures the current state; mutating + re-applying restores it.""" + # Arrange a distinctive state: a GLOBAL Ollama default. + await app_client_with_ollama.post( + "/api/providers", json={"mode": "ollama"}, headers=_HDR_PM + ) + snapshot_before = ( + await app_client_with_ollama.get("/api/providers", headers=_HDR_PM) + ).json() + + save_resp = await app_client_with_ollama.post( + "/api/providers/presets", json={"name": "my-preset"}, headers=_HDR_PM + ) + assert save_resp.status_code == HTTPStatus.OK + preset_id = save_resp.json()["id"] + assert save_resp.json()["name"] == "my-preset" + + listing = await app_client_with_ollama.get( + "/api/providers/presets", headers=_HDR_PM + ) + assert listing.status_code == HTTPStatus.OK + assert [p["name"] for p in listing.json()] == ["my-preset"] + + # Mutate away from the saved state. + await app_client_with_ollama.post( + "/api/providers", json={"mode": "anthropic"}, headers=_HDR_PM + ) + mutated = ( + await app_client_with_ollama.get("/api/providers", headers=_HDR_PM) + ).json() + assert mutated["assignments"] == [] + + # Apply the preset back — restores the snapshot. Applying always + # deletes-then-reinserts (a real full swap), so row `id`s are fresh; + # compare on the business fields only. + apply_resp = await app_client_with_ollama.post( + f"/api/providers/presets/{preset_id}/apply", headers=_HDR_PM + ) + assert apply_resp.status_code == HTTPStatus.OK + applied = apply_resp.json() + assert applied["skipped"] == [] + + def _sans_id(assignments: list[dict]) -> list[dict]: + return [{k: v for k, v in a.items() if k != "id"} for a in assignments] + + assert _sans_id(applied["assignments"]) == _sans_id(snapshot_before["assignments"]) + + +@pytest.mark.asyncio +async def test_save_preset_duplicate_name_returns_409( + app_client_with_ollama: AsyncClient, +) -> None: + first = await app_client_with_ollama.post( + "/api/providers/presets", json={"name": "dup"}, headers=_HDR_PM + ) + assert first.status_code == HTTPStatus.OK + second = await app_client_with_ollama.post( + "/api/providers/presets", json={"name": "dup"}, headers=_HDR_PM + ) + assert second.status_code == HTTPStatus.CONFLICT + + +@pytest.mark.asyncio +async def test_apply_preset_not_found_returns_404( + app_client_with_ollama: AsyncClient, +) -> None: + response = await app_client_with_ollama.post( + f"/api/providers/presets/{uuid4()}/apply", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.NOT_FOUND + + +@pytest.mark.asyncio +async def test_delete_preset_not_found_returns_404( + app_client_with_ollama: AsyncClient, +) -> None: + response = await app_client_with_ollama.delete( + f"/api/providers/presets/{uuid4()}", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.NOT_FOUND + + +@pytest.mark.asyncio +async def test_delete_preset(app_client_with_ollama: AsyncClient) -> None: + save_resp = await app_client_with_ollama.post( + "/api/providers/presets", json={"name": "to-delete"}, headers=_HDR_PM + ) + preset_id = save_resp.json()["id"] + + response = await app_client_with_ollama.delete( + f"/api/providers/presets/{preset_id}", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.NO_CONTENT + + listing = await app_client_with_ollama.get( + "/api/providers/presets", headers=_HDR_PM + ) + assert listing.json() == [] + + +@pytest.mark.asyncio +async def test_apply_preset_skips_entry_with_since_removed_model( + app_client_with_ollama: AsyncClient, + db_session: AsyncSession, +) -> None: + """Payload hygiene: an entry referencing a model no longer in the catalog + (and not routable to LOCAL, since no LOCAL provider is seeded here) is + skipped with a note — never fails the whole apply.""" + row = RoutingPresetTable( + name="stale-preset", + payload={ + "mode": "mix", + "assignments": [ + { + "scope": AssignmentScope.GLOBAL.value, + "scope_value": None, + "provider_type": "anthropic", + "model_name": "sonnet", + }, + { + "scope": AssignmentScope.ROLE.value, + "scope_value": "developer", + "provider_type": "anthropic", + "model_name": "ghost-model-that-no-longer-exists", + }, + ], + }, + ) + db_session.add(row) + await db_session.flush() + + response = await app_client_with_ollama.post( + f"/api/providers/presets/{row.id}/apply", headers=_HDR_PM + ) + assert response.status_code == HTTPStatus.OK + body = response.json() + assert len(body["skipped"]) == 1 + assert "ghost-model-that-no-longer-exists" in body["skipped"][0] + # The valid GLOBAL entry still applied despite the sibling failure. + scopes = {(a["scope"], a["scope_value"]) for a in body["assignments"]} + assert ("global", None) in scopes + assert ("role", "developer") not in scopes + + +@pytest.mark.asyncio +async def test_presets_developer_forbidden( + db_session: AsyncSession, +) -> None: + app = _make_app(db_session, role=AgentRole.DEVELOPER, team=Team.BACKEND) + transport = ASGITransport(app=app) + hdr = {"X-Agent-ID": str(uuid4()), "X-Agent-Role": "developer"} + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/providers/presets", headers=hdr) + app.dependency_overrides.clear() + assert response.status_code == HTTPStatus.FORBIDDEN diff --git a/tests/unit/billing/test_pricing.py b/tests/unit/billing/test_pricing.py index c9de3eab..b9956452 100644 --- a/tests/unit/billing/test_pricing.py +++ b/tests/unit/billing/test_pricing.py @@ -21,6 +21,7 @@ from roboco.billing.pricing import ( _is_anthropic_model, calculate_cost, calculate_cost_result, + input_price_per_million, ) # --------------------------------------------------------------------------- @@ -539,3 +540,45 @@ def test_sonnet5_reverts_to_list_rate_after_2026_08_31( _SONNET_CACHE_READ, _SONNET_CACHE_WRITE, ) + + +# --------------------------------------------------------------------------- +# input_price_per_million — the cost-tiered complexity-override comparator +# --------------------------------------------------------------------------- + + +class TestInputPricePerMillion: + """The downgrade-only comparator for complexity overrides (no explicit + tier ordering exists in the model catalog, so the input rate stands in + for "which tier is costlier").""" + + def test_orders_haiku_below_sonnet_below_opus(self) -> None: + assert ( + input_price_per_million("haiku") + < input_price_per_million("sonnet") + < input_price_per_million("opus") + ) + + def test_matches_pricing_table_value(self) -> None: + assert input_price_per_million("haiku") == _HAIKU_INPUT + assert input_price_per_million("sonnet") == _SONNET_INPUT + assert input_price_per_million("opus") == _OPUS_INPUT + + def test_grok_priced_below_sonnet(self) -> None: + """Grok legitimately downgrades-from sonnet under this comparator.""" + assert input_price_per_million("grok-build-0.1") < input_price_per_million( + "sonnet" + ) + + def test_unpriced_non_anthropic_model_is_free_tier(self) -> None: + """A self-hosted / Ollama Cloud model has no per-token rate — treated + as the cheapest possible tier, so it can never be rejected as + "costlier" by the downgrade-only policy.""" + assert input_price_per_million("glm-5.2:cloud") == 0.0 + assert input_price_per_million("my-custom-self-hosted-model:7b") == 0.0 + + def test_empty_model_returns_zero(self) -> None: + assert input_price_per_million("") == 0.0 + + def test_case_insensitive(self) -> None: + assert input_price_per_million("HAIKU") == input_price_per_million("haiku") diff --git a/tests/unit/runtime/test_complexity_routing_thread.py b/tests/unit/runtime/test_complexity_routing_thread.py new file mode 100644 index 00000000..b7846bf5 --- /dev/null +++ b/tests/unit/runtime/test_complexity_routing_thread.py @@ -0,0 +1,166 @@ +"""Task complexity threads into `_resolve_agent_route` -> `resolve_for_agent`. + +Cost-tiered routing (roboco/services/llm.py) reads a task's +`estimated_complexity` to try a compound ROLE(":"complexity) row before +falling to the plain ROLE row. The orchestrator owns the one indexed Task +lookup and threads the lowercase complexity string through. This is the pure +wiring test (`_resolve_agent_route` -> `resolve_for_agent`); the precedence +logic itself is covered in tests/integration/test_llm_routing.py. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +import roboco.db.base as db_base +import roboco.services.llm as llm_module +from roboco.models.base import Complexity, ModelProvider +from roboco.runtime.orchestrator import AgentOrchestrator +from roboco.services.llm import AgentRoute + +# Sentinel route the mocked resolve_for_agent returns — a real AgentRoute +# instance (not a bare string) so `result is _SENTINEL_ROUTE` type-checks +# cleanly against `_resolve_agent_route`'s declared AgentRoute return type. +_SENTINEL_ROUTE = AgentRoute( + provider_id=None, + provider_type=ModelProvider.ANTHROPIC, + base_url=None, + auth_token=None, + model_name="sentinel", +) + + +class _ScalarResult: + def __init__(self, value: Any) -> None: + self._value = value + + def scalar_one_or_none(self) -> Any: + return self._value + + +class _FakeSession: + """Minimal async-context-manager session returning a fixed complexity.""" + + def __init__(self, complexity_value: Any) -> None: + self._complexity_value = complexity_value + + def __call__(self) -> _FakeSession: + return self + + async def __aenter__(self) -> _FakeSession: + return self + + async def __aexit__(self, *exc: object) -> None: + return None + + async def execute(self, _stmt: Any) -> _ScalarResult: + return _ScalarResult(self._complexity_value) + + +class _BoomSession(_FakeSession): + """A session whose `execute` always raises — models a task-lookup failure + (bad/unresolvable task id) distinct from a genuine DB/session outage.""" + + async def execute(self, _stmt: Any) -> _ScalarResult: + raise RuntimeError("bad task id") + + +def _orch() -> AgentOrchestrator: + # __new__ + skip __init__: avoid all constructor I/O — this method is pure + # w.r.t. instance state (it only touches module-level imports + args). + return AgentOrchestrator.__new__(AgentOrchestrator) + + +def _wire(monkeypatch: pytest.MonkeyPatch, fake_session: Any) -> AsyncMock: + """Patch get_session_factory + get_model_routing_service; return the + resolve_for_agent mock so the test can assert on its call.""" + monkeypatch.setattr( + db_base, + "get_session_factory", + lambda: MagicMock(return_value=fake_session), + ) + resolve_mock = AsyncMock(return_value=_SENTINEL_ROUTE) + fake_router = MagicMock(resolve_for_agent=resolve_mock) + monkeypatch.setattr( + llm_module, "get_model_routing_service", lambda _db: fake_router + ) + return resolve_mock + + +@pytest.mark.asyncio +async def test_task_with_high_complexity_threads_lowercase_string( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A task with estimated_complexity=HIGH resolves the compound + 'role:high' row — i.e. resolve_for_agent is called with complexity='high' + (lowercased from the Complexity enum's value).""" + resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH)) + + orch = _orch() + result = await orch._resolve_agent_route("be-dev-1", "task-123") + + assert result is _SENTINEL_ROUTE + resolve_mock.assert_awaited_once_with("be-dev-1", complexity="high") + + +@pytest.mark.asyncio +async def test_task_with_low_complexity_threads_lowercase_string( + monkeypatch: pytest.MonkeyPatch, +) -> None: + resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.LOW)) + + orch = _orch() + await orch._resolve_agent_route("be-dev-1", "task-456") + + resolve_mock.assert_awaited_once_with("be-dev-1", complexity="low") + + +@pytest.mark.asyncio +async def test_taskless_spawn_threads_none_complexity_unchanged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A no-task spawn (idle PM bootstrap, Intake/Secretary chats, ...) never + even attempts a task lookup — complexity=None, byte-identical to the + pre-cost-tiering call shape.""" + resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH)) + + orch = _orch() + result = await orch._resolve_agent_route("be-dev-1", None) + + assert result is _SENTINEL_ROUTE + resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None) + + +@pytest.mark.asyncio +async def test_missing_task_row_degrades_to_none_complexity_silently( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """scalar_one_or_none() returning None (task not found / deleted) is not + an error — complexity falls back to None and routing still proceeds + through the router (not the hardcoded legacy path).""" + resolve_mock = _wire(monkeypatch, _FakeSession(None)) + + orch = _orch() + result = await orch._resolve_agent_route("be-dev-1", "ghost-task-id") + + assert result is _SENTINEL_ROUTE + resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None) + + +@pytest.mark.asyncio +async def test_task_lookup_failure_degrades_silently_not_to_full_legacy_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A task-lookup-specific failure (bad id, transient query error) must + NOT escalate to the full DB-failure downgrade (hardcoded ROLE_MODEL_MAP, + bypassing model_assignments entirely) — only the complexity lookup is + skipped; AGENT_SLUG/ROLE/GLOBAL resolution still runs via the router.""" + resolve_mock = _wire(monkeypatch, _BoomSession(None)) + + orch = _orch() + result = await orch._resolve_agent_route("be-dev-1", "bad-task-id") + + assert result is _SENTINEL_ROUTE + resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None) diff --git a/tests/unit/runtime/test_parked_spawn_shortcut.py b/tests/unit/runtime/test_parked_spawn_shortcut.py index c3a3aaf4..6bd5522c 100644 --- a/tests/unit/runtime/test_parked_spawn_shortcut.py +++ b/tests/unit/runtime/test_parked_spawn_shortcut.py @@ -47,7 +47,7 @@ def _wire(monitor: dict[str, Any]) -> Any: async def _git_context(_gc: Any, _tid: str | None) -> None: return None - async def _route(_aid: str) -> Any: + async def _route(_aid: str, _tid: str | None = None) -> Any: monitor["route_calls"] += 1 return SimpleNamespace( provider_type=SimpleNamespace(value="anthropic"),