mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(routing): cost-tiered complexity routing + saved presets (#656)
The 08-31 lever: model_assignments gains one compound rung —
AGENT_SLUG > ROLE('{role}:{complexity}') > ROLE > GLOBAL — so a
low-complexity task can route to a cheaper tier while coordinators stay
pinned. Structurally opt-in: zero rows means byte-identical routing
(pinned by a named test across every precedence case), the cost_tiered
apply-mode (seeds developer:low→haiku) is reachable only from the
explicit PM-gated endpoint — verified no startup path can apply it.
Overrides are downgrade-only (input-price comparator), allowlisted to
{developer, qa, documenter} — cell_pm excluded per the org's own
coordinator definition and its documented weak-model incidents — and
validated at write time (disabled/unconfigured provider rejected with
remediation; cross-provider-family overrides warn explicitly).
Per adversarial review: the four mode-switch applies now spare compound
rows exactly like agent pins (the 2026-07-17 unscoped-wipe class, new
victim, same fix extended via one shared wipe helper) with panel cache
invalidation + truthful confirm dialogs; preset apply validates the
entire payload BEFORE the wipe (validate-all-first), with a savepoint
crash test proving rollback.
Presets (CEO request): routing_presets table (migration 082) snapshots
the full mix — mode, per-agent overrides, complexity rows — with
save/apply/delete endpoints and a panel preset bar; applying skips
since-removed models with per-entry notes, never silently.
Task complexity threads task_id through _resolve_agent_route at both
call sites; taskless spawns unchanged. 235 backend + 23 panel tests.
Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
@@ -0,0 +1,48 @@
|
||||
"""Routing presets — named, full snapshots of the AI-routing state.
|
||||
|
||||
Lets an operator save the current routing state (mode + every
|
||||
`model_assignments` row: GLOBAL / plain ROLE / compound ROLE(":"complexity)
|
||||
cost-tier overrides / AGENT_SLUG pins) under a name and re-apply it later in
|
||||
one call, instead of re-picking every per-agent Select. ``payload`` mirrors
|
||||
exactly what ``GET /providers`` + ``GET /providers/complexity-overrides``
|
||||
already serve, so a preset is "what the card currently shows" — see
|
||||
``ModelRoutingService.save_routing_preset`` / ``apply_routing_preset``
|
||||
(roboco/services/llm.py). Pure additive new table; no backfill, no data
|
||||
migration — applying a preset is the only thing that ever mutates
|
||||
``model_assignments`` as a side effect, and only on an explicit call.
|
||||
|
||||
Revision ID: 082_routing_presets
|
||||
Revises: 081_doctrine_version
|
||||
Create Date: 2026-07-23
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "082_routing_presets"
|
||||
down_revision = "081_doctrine_version"
|
||||
branch_labels: dict[str, str] | None = None
|
||||
depends_on: dict[str, str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"routing_presets",
|
||||
sa.Column("id", sa.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("name", sa.String(length=100), nullable=False),
|
||||
sa.Column("payload", postgresql.JSONB(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("now()"),
|
||||
),
|
||||
sa.UniqueConstraint("name", name="uq_routing_presets_name"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("routing_presets")
|
||||
@@ -1,7 +1,18 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import {
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
waitFor,
|
||||
within,
|
||||
} from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import type { ReactNode } from "react";
|
||||
import React from "react";
|
||||
import type {
|
||||
ComplexityOverride,
|
||||
RoutingPreset,
|
||||
} from "@/lib/api/providers";
|
||||
|
||||
const {
|
||||
catalog,
|
||||
@@ -15,6 +26,13 @@ const {
|
||||
saveSelfHostedConfig,
|
||||
testSelfHosted,
|
||||
getSelfHostedModels,
|
||||
getComplexityOverrides,
|
||||
setComplexityOverride,
|
||||
deleteComplexityOverride,
|
||||
listPresets,
|
||||
savePreset,
|
||||
applyPreset,
|
||||
deletePreset,
|
||||
} = vi.hoisted(() => ({
|
||||
catalog: vi.fn(async () => [
|
||||
{
|
||||
@@ -53,6 +71,21 @@ const {
|
||||
error: null,
|
||||
})),
|
||||
getSelfHostedModels: vi.fn(async () => []),
|
||||
getComplexityOverrides: vi.fn(async (): Promise<ComplexityOverride[]> => []),
|
||||
setComplexityOverride: vi.fn(async (payload: ComplexityOverride) => payload),
|
||||
deleteComplexityOverride: vi.fn(async () => undefined),
|
||||
listPresets: vi.fn(async (): Promise<RoutingPreset[]> => []),
|
||||
savePreset: vi.fn(async (name: string) => ({
|
||||
id: "preset-1",
|
||||
name,
|
||||
created_at: "2026-07-23T00:00:00Z",
|
||||
})),
|
||||
applyPreset: vi.fn(async () => ({
|
||||
mode: "mix",
|
||||
assignments: [],
|
||||
skipped: [] as string[],
|
||||
})),
|
||||
deletePreset: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/api/providers", () => ({
|
||||
@@ -68,10 +101,95 @@ vi.mock("@/lib/api/providers", () => ({
|
||||
saveSelfHostedConfig,
|
||||
testSelfHosted,
|
||||
getSelfHostedModels,
|
||||
getComplexityOverrides,
|
||||
setComplexityOverride,
|
||||
deleteComplexityOverride,
|
||||
listPresets,
|
||||
savePreset,
|
||||
applyPreset,
|
||||
deletePreset,
|
||||
},
|
||||
COMPLEXITY_OVERRIDE_ROLES: ["developer", "qa", "documenter"],
|
||||
}));
|
||||
|
||||
vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } }));
|
||||
vi.mock("sonner", () => ({
|
||||
toast: { success: vi.fn(), error: vi.fn(), warning: vi.fn() },
|
||||
}));
|
||||
|
||||
// Functional Select mock (mirrors select-repo-picker.test.tsx /
|
||||
// a2a-reply-composer.test.tsx): SelectItem renders as a clickable button
|
||||
// wired to onValueChange via context, so a real "pick a value" interaction
|
||||
// can be simulated without Radix's portal/pointer machinery. SelectTrigger
|
||||
// keeps `role="combobox"` so the pre-existing per-agent-table combobox-count
|
||||
// assertion is unaffected, and stamps `data-value` from context so a test can
|
||||
// `waitFor` an async-loaded value (e.g. a complexity override) landing before
|
||||
// interacting further.
|
||||
vi.mock("@/components/ui/select", () => {
|
||||
const Ctx = React.createContext<{
|
||||
value?: string;
|
||||
onValueChange: (v: string) => void;
|
||||
}>({ onValueChange: () => {} });
|
||||
return {
|
||||
Select: ({
|
||||
value,
|
||||
onValueChange,
|
||||
children,
|
||||
}: {
|
||||
value?: string;
|
||||
onValueChange?: (v: string) => void;
|
||||
children: React.ReactNode;
|
||||
}) => (
|
||||
<Ctx.Provider
|
||||
value={{ value, onValueChange: onValueChange ?? (() => {}) }}
|
||||
>
|
||||
{children}
|
||||
</Ctx.Provider>
|
||||
),
|
||||
SelectTrigger: ({ children }: { children: React.ReactNode }) => {
|
||||
const { value } = React.useContext(Ctx);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
role="combobox"
|
||||
aria-expanded={false}
|
||||
aria-controls="mock-select-content"
|
||||
data-value={value ?? ""}
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
},
|
||||
SelectValue: () => null,
|
||||
SelectGroup: ({ children }: { children: React.ReactNode }) => (
|
||||
<div>{children}</div>
|
||||
),
|
||||
SelectLabel: ({ children }: { children: React.ReactNode }) => (
|
||||
<div>{children}</div>
|
||||
),
|
||||
SelectContent: ({ children }: { children: React.ReactNode }) => (
|
||||
<div>{children}</div>
|
||||
),
|
||||
SelectItem: ({
|
||||
value,
|
||||
children,
|
||||
}: {
|
||||
value: string;
|
||||
children: React.ReactNode;
|
||||
}) => {
|
||||
const { onValueChange } = React.useContext(Ctx);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
role="option"
|
||||
aria-selected={false}
|
||||
onClick={() => onValueChange(value)}
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// Full live roster (minus CEO/system) — mirrors foundation/identity.py so the
|
||||
// per-agent override grid is exercised against the real 25-agent org chart,
|
||||
@@ -277,6 +395,13 @@ describe("AIRoutingCard", () => {
|
||||
saveSelfHostedConfig.mockClear();
|
||||
testSelfHosted.mockClear();
|
||||
getSelfHostedModels.mockClear();
|
||||
getComplexityOverrides.mockClear();
|
||||
setComplexityOverride.mockClear();
|
||||
deleteComplexityOverride.mockClear();
|
||||
listPresets.mockClear();
|
||||
savePreset.mockClear();
|
||||
applyPreset.mockClear();
|
||||
deletePreset.mockClear();
|
||||
useAgentDefinitions.mockReturnValue({
|
||||
data: FULL_ROSTER,
|
||||
isLoading: false,
|
||||
@@ -468,4 +593,298 @@ describe("AIRoutingCard", () => {
|
||||
);
|
||||
expect(applyMode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Complexity overrides (cost-tiered routing)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
describe("Complexity overrides section", () => {
|
||||
it("renders one row per allowlisted role with Low/High selects, and no coordinator role", async () => {
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Complexity overrides");
|
||||
|
||||
const section = screen.getByText("Complexity overrides").closest("section")!;
|
||||
for (const label of ["Developer", "QA", "Documenter"]) {
|
||||
expect(within(section).getByText(label)).toBeInTheDocument();
|
||||
}
|
||||
// Coordinator (cell_pm, main_pm)/pr_reviewer/board/CEO-facing roles are
|
||||
// never offered a row here — scoped to this section, since "Cell PM"/
|
||||
// "Main PM"/"PR Reviewer" legitimately appear elsewhere (the per-agent
|
||||
// mix table's roster).
|
||||
expect(within(section).queryByText("Cell PM")).not.toBeInTheDocument();
|
||||
expect(within(section).queryByText("Main PM")).not.toBeInTheDocument();
|
||||
expect(within(section).queryByText("PR Reviewer")).not.toBeInTheDocument();
|
||||
|
||||
// 3 roles x 2 (low/high) selects.
|
||||
expect(section.querySelectorAll('[role="combobox"]')).toHaveLength(6);
|
||||
});
|
||||
|
||||
it("picking a model for a role+complexity calls setComplexityOverride with the right payload", async () => {
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Complexity overrides");
|
||||
|
||||
const devLow = screen.getByTestId("complexity-select-developer-low");
|
||||
fireEvent.click(
|
||||
await within(devLow).findByRole("option", { name: "Claude Opus 4.6" }),
|
||||
);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(setComplexityOverride).toHaveBeenCalledWith({
|
||||
role: "developer",
|
||||
complexity: "low",
|
||||
model_name: "claude-opus-4-6",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces a returned cross-family warning as its own warning toast", async () => {
|
||||
setComplexityOverride.mockResolvedValueOnce({
|
||||
role: "developer",
|
||||
complexity: "low",
|
||||
model_name: "grok-build-0.1",
|
||||
warning:
|
||||
"'grok-build-0.1' routes through grok, a different provider family than developer's Anthropic baseline.",
|
||||
});
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Complexity overrides");
|
||||
|
||||
const devLow = screen.getByTestId("complexity-select-developer-low");
|
||||
fireEvent.click(
|
||||
await within(devLow).findByRole("option", { name: "Grok Build 0.1" }),
|
||||
);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(toast.warning).toHaveBeenCalledWith(
|
||||
expect.stringContaining("different provider family"),
|
||||
),
|
||||
);
|
||||
// Still allowed — the success toast still fires alongside the warning.
|
||||
expect(toast.success).toHaveBeenCalledWith(
|
||||
"developer:low → grok-build-0.1",
|
||||
);
|
||||
});
|
||||
|
||||
it("never shows a warning toast when the response carries none", async () => {
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Complexity overrides");
|
||||
|
||||
const devLow = screen.getByTestId("complexity-select-developer-low");
|
||||
fireEvent.click(
|
||||
await within(devLow).findByRole("option", { name: "Claude Opus 4.6" }),
|
||||
);
|
||||
|
||||
await waitFor(() => expect(setComplexityOverride).toHaveBeenCalled());
|
||||
expect(toast.warning).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("picking '(none)' on a row with no existing override never calls deleteComplexityOverride", async () => {
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Complexity overrides");
|
||||
|
||||
const qaHigh = screen.getByTestId("complexity-select-qa-high");
|
||||
fireEvent.click(
|
||||
await within(qaHigh).findByRole("option", { name: "(none)" }),
|
||||
);
|
||||
|
||||
// No pre-existing row for qa:high (getComplexityOverrides returns []
|
||||
// by default) — clearing an already-empty selection is a no-op.
|
||||
await waitFor(() => expect(setComplexityOverride).not.toHaveBeenCalled());
|
||||
expect(deleteComplexityOverride).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("picking '(none)' on a row WITH an existing override calls deleteComplexityOverride", async () => {
|
||||
getComplexityOverrides.mockResolvedValueOnce([
|
||||
{ role: "documenter", complexity: "low", model_name: "grok-build-0.1" },
|
||||
]);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Complexity overrides");
|
||||
|
||||
const documenterLow = await screen.findByTestId(
|
||||
"complexity-select-documenter-low",
|
||||
);
|
||||
// Wait for the async-loaded override to actually land on the Select
|
||||
// (its combobox reflects the current value via data-value) before
|
||||
// clearing it — otherwise the click races the query resolving.
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
within(documenterLow).getByRole("combobox"),
|
||||
).toHaveAttribute("data-value", "grok-build-0.1"),
|
||||
);
|
||||
fireEvent.click(
|
||||
within(documenterLow).getByRole("option", { name: "(none)" }),
|
||||
);
|
||||
|
||||
// deleteComplexityOverride(role, complexity) — positional, per the
|
||||
// providersApi signature (mirrors the real hook's mutationFn).
|
||||
await waitFor(() =>
|
||||
expect(deleteComplexityOverride).toHaveBeenCalledWith(
|
||||
"documenter",
|
||||
"low",
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Cost-Tiered mode button (additive seed, never wipes routing)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
describe("Cost-Tiered mode button", () => {
|
||||
it("renders beside the other mode buttons and applies mode='cost_tiered' on confirm", async () => {
|
||||
const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Grok (xAI) API key");
|
||||
|
||||
fireEvent.click(screen.getByText("Cost-Tiered"));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(applyMode).toHaveBeenCalledWith({ mode: "cost_tiered" }),
|
||||
);
|
||||
confirmSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("does nothing when the confirm dialog is declined", async () => {
|
||||
const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(false);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Grok (xAI) API key");
|
||||
|
||||
fireEvent.click(screen.getByText("Cost-Tiered"));
|
||||
|
||||
await waitFor(() => expect(confirmSpy).toHaveBeenCalled());
|
||||
expect(applyMode).not.toHaveBeenCalled();
|
||||
confirmSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Mode switches preserve complexity overrides (2026-07-17-style incident:
|
||||
// these same buttons once wiped AGENT_SLUG pins) — the confirm text says so
|
||||
// and the query cache is refreshed for both, not just the mode snapshot.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
describe("Mode switches preserve complexity overrides", () => {
|
||||
it("applying Anthropic mode also refetches complexity overrides, not just the mode snapshot", async () => {
|
||||
const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Grok (xAI) API key");
|
||||
getComplexityOverrides.mockClear();
|
||||
|
||||
fireEvent.click(screen.getByText("Anthropic"));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(applyMode).toHaveBeenCalledWith({ mode: "anthropic" }),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(getComplexityOverrides.mock.calls.length).toBeGreaterThan(0),
|
||||
);
|
||||
confirmSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("the Anthropic confirm dialog states complexity overrides are kept", async () => {
|
||||
const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(false);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Grok (xAI) API key");
|
||||
|
||||
fireEvent.click(screen.getByText("Anthropic"));
|
||||
|
||||
await waitFor(() => expect(confirmSpy).toHaveBeenCalled());
|
||||
expect(confirmSpy.mock.calls[0][0]).toContain("complexity");
|
||||
confirmSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Routing presets (named, full snapshots)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
describe("Routing presets bar", () => {
|
||||
it("lists saved presets and disables Apply/Delete until one is picked", async () => {
|
||||
listPresets.mockResolvedValueOnce([
|
||||
{ id: "p1", name: "Cheap Fleet", created_at: "2026-07-01T00:00:00Z" },
|
||||
]);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Cheap Fleet");
|
||||
|
||||
expect(screen.getByRole("button", { name: "Apply" })).toBeDisabled();
|
||||
expect(screen.getByRole("button", { name: "Delete" })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("save-as-preset flow: reveals a name input and Confirm calls savePreset", async () => {
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Routing presets");
|
||||
|
||||
fireEvent.click(screen.getByText("Save as preset…"));
|
||||
const nameInput = screen.getByPlaceholderText("Preset name");
|
||||
fireEvent.change(nameInput, { target: { value: "My Setup" } });
|
||||
fireEvent.click(screen.getByText("Confirm"));
|
||||
|
||||
await waitFor(() => expect(savePreset).toHaveBeenCalledWith("My Setup"));
|
||||
await waitFor(() =>
|
||||
expect(toast.success).toHaveBeenCalledWith('Saved preset "My Setup"'),
|
||||
);
|
||||
});
|
||||
|
||||
it("save-as-preset with an empty name shows an error and never calls savePreset", async () => {
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
await screen.findByText("Routing presets");
|
||||
|
||||
fireEvent.click(screen.getByText("Save as preset…"));
|
||||
fireEvent.click(screen.getByText("Confirm"));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(toast.error).toHaveBeenCalledWith("Enter a preset name first"),
|
||||
);
|
||||
expect(savePreset).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("applying a preset with skipped rows surfaces them in a toast error", async () => {
|
||||
const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true);
|
||||
listPresets.mockResolvedValueOnce([
|
||||
{ id: "p1", name: "Cheap Fleet", created_at: "2026-07-01T00:00:00Z" },
|
||||
]);
|
||||
applyPreset.mockResolvedValueOnce({
|
||||
mode: "mix",
|
||||
assignments: [],
|
||||
skipped: ["Skipped role:developer (ghost-model) — Unknown model"],
|
||||
});
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
const presetSection = (await screen.findByText("Routing presets")).closest(
|
||||
"section",
|
||||
)!;
|
||||
fireEvent.click(
|
||||
await within(presetSection).findByRole("option", {
|
||||
name: "Cheap Fleet",
|
||||
}),
|
||||
);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Apply" }));
|
||||
|
||||
await waitFor(() => expect(applyPreset).toHaveBeenCalledWith("p1"));
|
||||
await waitFor(() =>
|
||||
expect(toast.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining("1 row(s) skipped"),
|
||||
),
|
||||
);
|
||||
confirmSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("deleting a preset calls deletePreset and clears the selection", async () => {
|
||||
const confirmSpy = vi.spyOn(window, "confirm").mockReturnValue(true);
|
||||
listPresets.mockResolvedValueOnce([
|
||||
{ id: "p1", name: "Cheap Fleet", created_at: "2026-07-01T00:00:00Z" },
|
||||
]);
|
||||
render(withQueryClient(<AIRoutingCard />));
|
||||
const presetSection = (await screen.findByText("Routing presets")).closest(
|
||||
"section",
|
||||
)!;
|
||||
fireEvent.click(
|
||||
await within(presetSection).findByRole("option", {
|
||||
name: "Cheap Fleet",
|
||||
}),
|
||||
);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Delete" }));
|
||||
|
||||
await waitFor(() => expect(deletePreset).toHaveBeenCalledWith("p1"));
|
||||
confirmSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,10 +3,17 @@
|
||||
import { useEffect, useMemo, useState, useCallback } from "react";
|
||||
import {
|
||||
useApplyMode,
|
||||
useApplyPreset,
|
||||
useCatalog,
|
||||
useComplexityOverrides,
|
||||
useDeleteComplexityOverride,
|
||||
useDeletePreset,
|
||||
useGrokKey,
|
||||
useOllamaKey,
|
||||
useRoutingMode,
|
||||
useRoutingPresets,
|
||||
useSavePreset,
|
||||
useSetComplexityOverride,
|
||||
useSetGrokKey,
|
||||
useSetOllamaKey,
|
||||
useSelfHostedModels,
|
||||
@@ -34,6 +41,7 @@ import { Separator } from "@/components/ui/separator";
|
||||
import {
|
||||
AlertTriangle,
|
||||
Cpu,
|
||||
Gauge,
|
||||
Key,
|
||||
KeyRound,
|
||||
Server,
|
||||
@@ -43,7 +51,11 @@ import {
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { AssignmentScope, AgentRole, ModelProvider } from "@/types";
|
||||
import type { SelfHostedModel } from "@/lib/api/providers";
|
||||
import {
|
||||
COMPLEXITY_OVERRIDE_ROLES,
|
||||
type ComplexityLevel,
|
||||
type SelfHostedModel,
|
||||
} from "@/lib/api/providers";
|
||||
import type { RoutingMode, SelfHostedTestResult } from "@/lib/api/providers";
|
||||
import { SelfHostedSection } from "@/components/settings/self-hosted-section";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
@@ -126,11 +138,29 @@ function byOrgOrder(a: AgentDefinition, b: AgentDefinition): number {
|
||||
return ra !== rb ? ra - rb : a.id.localeCompare(b.id);
|
||||
}
|
||||
|
||||
// Display labels for the complexity-override allowlist — mirrors the
|
||||
// backend's fixed `_COMPLEXITY_OVERRIDE_ROLES` (developer/qa/documenter).
|
||||
// cell_pm is deliberately excluded (a coordinator role — see
|
||||
// COMPLEXITY_OVERRIDE_ROLES), along with every other coordinator/board/
|
||||
// CEO-facing role.
|
||||
const COMPLEXITY_ROLE_LABELS: Record<string, string> = {
|
||||
developer: "Developer",
|
||||
qa: "QA",
|
||||
documenter: "Documenter",
|
||||
};
|
||||
|
||||
export function AIRoutingCard() {
|
||||
const { data: catalog = [] } = useCatalog();
|
||||
const { data: keyStatus } = useOllamaKey();
|
||||
const { data: snapshot } = useRoutingMode();
|
||||
const { data: selfHostedModels = [] } = useSelfHostedModels();
|
||||
const { data: complexityOverrides = [] } = useComplexityOverrides();
|
||||
const setComplexityOverride = useSetComplexityOverride();
|
||||
const deleteComplexityOverride = useDeleteComplexityOverride();
|
||||
const { data: presets = [] } = useRoutingPresets();
|
||||
const savePreset = useSavePreset();
|
||||
const applyPreset = useApplyPreset();
|
||||
const deletePreset = useDeletePreset();
|
||||
const {
|
||||
data: agentDefs,
|
||||
isLoading: agentsLoading,
|
||||
@@ -255,14 +285,15 @@ export function AIRoutingCard() {
|
||||
const flipToAnthropic = async () => {
|
||||
if (
|
||||
!confirm(
|
||||
"Switch every agent to Anthropic? Per-agent pins are kept; role/global assignments are replaced.",
|
||||
"Switch every agent to Anthropic? Per-agent pins and complexity " +
|
||||
"overrides are kept; other role/global assignments are replaced.",
|
||||
)
|
||||
)
|
||||
return;
|
||||
try {
|
||||
await applyMode.mutateAsync({ mode: "anthropic" });
|
||||
toast.success(
|
||||
"Role/global routing now on Anthropic — per-agent pins kept",
|
||||
"Role/global routing now on Anthropic — per-agent pins and complexity overrides kept",
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error("Switch failed: " + errMsg(e));
|
||||
@@ -276,13 +307,16 @@ export function AIRoutingCard() {
|
||||
}
|
||||
if (
|
||||
!confirm(
|
||||
"Switch every agent to Grok? Per-agent pins are kept; role/global assignments are replaced.",
|
||||
"Switch every agent to Grok? Per-agent pins and complexity " +
|
||||
"overrides are kept; other role/global assignments are replaced.",
|
||||
)
|
||||
)
|
||||
return;
|
||||
try {
|
||||
await applyMode.mutateAsync({ mode: "grok" });
|
||||
toast.success("Role/global routing now on Grok — per-agent pins kept");
|
||||
toast.success(
|
||||
"Role/global routing now on Grok — per-agent pins and complexity overrides kept",
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error("Switch failed: " + errMsg(e));
|
||||
}
|
||||
@@ -295,13 +329,16 @@ export function AIRoutingCard() {
|
||||
}
|
||||
if (
|
||||
!confirm(
|
||||
"Switch every agent to Ollama? Per-agent pins are kept; role/global assignments are replaced.",
|
||||
"Switch every agent to Ollama? Per-agent pins and complexity " +
|
||||
"overrides are kept; other role/global assignments are replaced.",
|
||||
)
|
||||
)
|
||||
return;
|
||||
try {
|
||||
await applyMode.mutateAsync({ mode: "ollama" });
|
||||
toast.success("Role/global routing now on Ollama — per-agent pins kept");
|
||||
toast.success(
|
||||
"Role/global routing now on Ollama — per-agent pins and complexity overrides kept",
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error("Switch failed: " + errMsg(e));
|
||||
}
|
||||
@@ -314,7 +351,9 @@ export function AIRoutingCard() {
|
||||
}
|
||||
if (
|
||||
!confirm(
|
||||
"Switch every agent to the self-hosted LLM? Per-agent pins are kept; role/global assignments are replaced.",
|
||||
"Switch every agent to the self-hosted LLM? Per-agent pins and " +
|
||||
"complexity overrides are kept; other role/global assignments " +
|
||||
"are replaced.",
|
||||
)
|
||||
)
|
||||
return;
|
||||
@@ -324,7 +363,7 @@ export function AIRoutingCard() {
|
||||
...(selfHostedModel ? { default_model: selfHostedModel } : {}),
|
||||
});
|
||||
toast.success(
|
||||
"Role/global routing now on Self-Hosted LLM — per-agent pins kept",
|
||||
"Role/global routing now on Self-Hosted LLM — per-agent pins and complexity overrides kept",
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error("Switch failed: " + errMsg(e));
|
||||
@@ -382,6 +421,132 @@ export function AIRoutingCard() {
|
||||
}
|
||||
};
|
||||
|
||||
// --- Cost-tiered defaults (additive seed — never wipes existing routing) ---
|
||||
const flipToCostTiered = async () => {
|
||||
if (
|
||||
!confirm(
|
||||
"Seed the day-1 cost-tiered default (developer:low → Haiku)? " +
|
||||
"Unlike the other buttons this is additive — it does not clear " +
|
||||
"any existing routing.",
|
||||
)
|
||||
)
|
||||
return;
|
||||
try {
|
||||
await applyMode.mutateAsync({ mode: "cost_tiered" });
|
||||
toast.success(
|
||||
"Cost-tiered default seeded (developer:low → Haiku)",
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error("Apply failed: " + errMsg(e));
|
||||
}
|
||||
};
|
||||
|
||||
// --- Complexity overrides (compound ROLE(":"complexity) rows) ---
|
||||
const complexityMap = useMemo(() => {
|
||||
const map: Record<string, Partial<Record<ComplexityLevel, string>>> = {};
|
||||
for (const o of complexityOverrides) {
|
||||
map[o.role] = { ...map[o.role], [o.complexity]: o.model_name };
|
||||
}
|
||||
return map;
|
||||
}, [complexityOverrides]);
|
||||
|
||||
const handleComplexityChange = async (
|
||||
role: string,
|
||||
complexity: ComplexityLevel,
|
||||
modelName: string,
|
||||
) => {
|
||||
try {
|
||||
if (!modelName) {
|
||||
if (complexityMap[role]?.[complexity]) {
|
||||
await deleteComplexityOverride.mutateAsync({ role, complexity });
|
||||
toast.success(`Cleared ${role}:${complexity} override`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const result = await setComplexityOverride.mutateAsync({
|
||||
role,
|
||||
complexity,
|
||||
model_name: modelName,
|
||||
});
|
||||
toast.success(`${role}:${complexity} → ${modelName}`);
|
||||
// Allowed but never silent: a cross-provider-family override (e.g. an
|
||||
// Anthropic role pinned to a Grok/Ollama/self-hosted model) also gets
|
||||
// its own warning toast.
|
||||
if (result.warning) {
|
||||
toast.warning(result.warning);
|
||||
}
|
||||
} catch (e) {
|
||||
toast.error("Save failed: " + errMsg(e));
|
||||
}
|
||||
};
|
||||
|
||||
// --- Routing presets (named, full snapshots) ---
|
||||
const [selectedPresetId, setSelectedPresetId] = useState("");
|
||||
const [showPresetNameInput, setShowPresetNameInput] = useState(false);
|
||||
const [presetNameDraft, setPresetNameDraft] = useState("");
|
||||
const selectedPreset = presets.find((p) => p.id === selectedPresetId);
|
||||
|
||||
const handleSavePreset = async () => {
|
||||
const name = presetNameDraft.trim();
|
||||
if (!name) {
|
||||
toast.error("Enter a preset name first");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const saved = await savePreset.mutateAsync(name);
|
||||
toast.success(`Saved preset "${saved.name}"`);
|
||||
setPresetNameDraft("");
|
||||
setShowPresetNameInput(false);
|
||||
setSelectedPresetId(saved.id);
|
||||
} catch (e) {
|
||||
toast.error("Save failed: " + errMsg(e));
|
||||
}
|
||||
};
|
||||
|
||||
const handleApplyPreset = async () => {
|
||||
if (!selectedPresetId) {
|
||||
toast.error("Pick a preset first");
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!confirm(
|
||||
`Apply preset "${selectedPreset?.name ?? selectedPresetId}"? This ` +
|
||||
"replaces the ENTIRE current routing state (every per-agent pin, " +
|
||||
"role row, and global default) with the saved snapshot.",
|
||||
)
|
||||
)
|
||||
return;
|
||||
try {
|
||||
const result = await applyPreset.mutateAsync(selectedPresetId);
|
||||
if (result.skipped.length > 0) {
|
||||
toast.error(
|
||||
`Applied with ${result.skipped.length} row(s) skipped: ` +
|
||||
result.skipped.join("; "),
|
||||
);
|
||||
} else {
|
||||
toast.success("Preset applied");
|
||||
}
|
||||
} catch (e) {
|
||||
toast.error("Apply failed: " + errMsg(e));
|
||||
}
|
||||
};
|
||||
|
||||
const handleDeletePreset = async () => {
|
||||
if (!selectedPresetId) {
|
||||
toast.error("Pick a preset first");
|
||||
return;
|
||||
}
|
||||
if (!confirm(`Delete preset "${selectedPreset?.name ?? selectedPresetId}"?`))
|
||||
return;
|
||||
try {
|
||||
await deletePreset.mutateAsync(selectedPresetId);
|
||||
toast.success("Preset deleted");
|
||||
setSelectedPresetId("");
|
||||
} catch (e) {
|
||||
toast.error("Delete failed: " + errMsg(e));
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
@@ -534,7 +699,7 @@ export function AIRoutingCard() {
|
||||
<HelpTip label="Anthropic / Grok / Ollama / Self-Hosted replace role/global routing with that provider; per-agent pins in the table below survive the switch. Mix keeps whatever's picked in the table.">
|
||||
<Label className="text-sm font-medium">Routing mode</Label>
|
||||
</HelpTip>
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-5 gap-2">
|
||||
<div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-6 gap-2">
|
||||
<ModeButton
|
||||
icon={<ShieldCheck className="h-4 w-4" />}
|
||||
label="Anthropic"
|
||||
@@ -591,6 +756,15 @@ export function AIRoutingCard() {
|
||||
disabled={false}
|
||||
highlight={currentMode === "mix"}
|
||||
/>
|
||||
<ModeButton
|
||||
icon={<Gauge className="h-4 w-4" />}
|
||||
label="Cost-Tiered"
|
||||
description="Seed developer:low → Haiku (see below)."
|
||||
active={false}
|
||||
onClick={flipToCostTiered}
|
||||
disabled={applyMode.isPending}
|
||||
labelHint="Unlike every button to the left this never wipes existing routing — it's a one-time additive seed you can re-run anytime. Edit or remove individual rows in the Complexity overrides section below."
|
||||
/>
|
||||
</div>
|
||||
{currentMode === "mix" && !hasOllamaKey ? (
|
||||
<p className="text-xs text-amber-600 flex items-center gap-1">
|
||||
@@ -649,6 +823,95 @@ export function AIRoutingCard() {
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* -------- Preset bar (compact, sits right above the per-agent table) -------- */}
|
||||
<Separator />
|
||||
<section className="space-y-2">
|
||||
<HelpTip label="A preset snapshots the ENTIRE current routing state — mode, every per-agent pin, every role row, and every complexity override — so you can switch between whole setups in one click instead of re-picking every Select.">
|
||||
<Label className="text-sm font-medium">Routing presets</Label>
|
||||
</HelpTip>
|
||||
<div className="flex flex-wrap items-center gap-2 rounded-md border border-dashed p-2">
|
||||
<Select value={selectedPresetId} onValueChange={setSelectedPresetId}>
|
||||
<SelectTrigger size="sm" className="w-48 text-xs">
|
||||
<SelectValue
|
||||
placeholder={
|
||||
presets.length ? "Choose a preset…" : "No presets saved"
|
||||
}
|
||||
/>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{presets.map((p) => (
|
||||
<SelectItem key={p.id} value={p.id}>
|
||||
{p.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<HelpTip label="Replaces the ENTIRE current routing state with this preset's snapshot — every per-agent pin, role row, and global default, not merged with what's here now.">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={handleApplyPreset}
|
||||
disabled={!selectedPresetId || applyPreset.isPending}
|
||||
>
|
||||
{applyPreset.isPending ? "Applying…" : "Apply"}
|
||||
</Button>
|
||||
</HelpTip>
|
||||
<HelpTip label="Deletes the saved snapshot only — has no effect on the routing currently applied.">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={handleDeletePreset}
|
||||
disabled={!selectedPresetId || deletePreset.isPending}
|
||||
>
|
||||
Delete
|
||||
</Button>
|
||||
</HelpTip>
|
||||
<Separator orientation="vertical" className="h-6" />
|
||||
{showPresetNameInput ? (
|
||||
<>
|
||||
<Input
|
||||
value={presetNameDraft}
|
||||
onChange={(e) => setPresetNameDraft(e.target.value)}
|
||||
placeholder="Preset name"
|
||||
className="h-8 w-40 text-xs"
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
onClick={handleSavePreset}
|
||||
disabled={savePreset.isPending}
|
||||
>
|
||||
{savePreset.isPending ? "Saving…" : "Confirm"}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
onClick={() => {
|
||||
setShowPresetNameInput(false);
|
||||
setPresetNameDraft("");
|
||||
}}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<HelpTip label="Snapshots exactly what this card currently shows — the applied mode, every per-agent pin, and every complexity override.">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => setShowPresetNameInput(true)}
|
||||
>
|
||||
Save as preset…
|
||||
</Button>
|
||||
</HelpTip>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* -------- Mix-mode per-agent picker -------- */}
|
||||
<Separator />
|
||||
<section className="space-y-3">
|
||||
@@ -843,6 +1106,78 @@ export function AIRoutingCard() {
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
{/* -------- Complexity overrides (cost-tiered routing) -------- */}
|
||||
<Separator />
|
||||
<section className="space-y-3">
|
||||
<HelpTip label="Downgrade-only by policy: a role+complexity override can never point to a costlier tier than that role's baseline model — this lever only saves cost, it never spends more. Coordinator roles (cell_pm, main_pm), pr_reviewer, and board/CEO-facing roles aren't offered a row here at all; tier pinning for those is deliberate — cell_pm especially, since a coordinator is the last place to gamble a downgrade.">
|
||||
<Label className="text-sm font-medium">
|
||||
Complexity overrides
|
||||
</Label>
|
||||
</HelpTip>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Pin a role to a cheaper model for LOW- or HIGH-complexity tasks
|
||||
specifically — wins over that role's plain default and the
|
||||
global mode, still loses to a per-agent pin above. Leave a Select
|
||||
blank to remove the override. Coordinator roles (cell_pm, main_pm)
|
||||
aren't offered a row.
|
||||
</p>
|
||||
<div className="divide-y rounded-md border">
|
||||
{COMPLEXITY_OVERRIDE_ROLES.map((role) => (
|
||||
<div
|
||||
key={role}
|
||||
className="grid grid-cols-[1fr_140px_140px] items-center gap-4 p-3"
|
||||
>
|
||||
<HelpTip
|
||||
label={`Applies only to ${COMPLEXITY_ROLE_LABELS[role]} agents; a per-agent pin above still wins over this.`}
|
||||
>
|
||||
<div className="text-xs font-medium">
|
||||
{COMPLEXITY_ROLE_LABELS[role]}
|
||||
</div>
|
||||
</HelpTip>
|
||||
{(["low", "high"] as const).map((complexity) => (
|
||||
<div
|
||||
key={complexity}
|
||||
className="space-y-1"
|
||||
data-testid={`complexity-select-${role}-${complexity}`}
|
||||
>
|
||||
<HelpTip
|
||||
label={`Model used for ${COMPLEXITY_ROLE_LABELS[role]} tasks a PM estimates as ${complexity.toUpperCase()} complexity. Must be no costlier than ${COMPLEXITY_ROLE_LABELS[role]}'s baseline model — the server rejects an upgrade attempt here.`}
|
||||
>
|
||||
<div className="text-[10px] uppercase tracking-wide text-muted-foreground">
|
||||
{complexity}
|
||||
</div>
|
||||
</HelpTip>
|
||||
<Select
|
||||
value={complexityMap[role]?.[complexity] ?? "__clear__"}
|
||||
onValueChange={(v: string) =>
|
||||
handleComplexityChange(
|
||||
role,
|
||||
complexity,
|
||||
v === "__clear__" ? "" : v,
|
||||
)
|
||||
}
|
||||
>
|
||||
<SelectTrigger size="sm" className="w-full text-xs">
|
||||
<SelectValue placeholder="(none)" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="__clear__">(none)</SelectItem>
|
||||
{catalogForMix.map(
|
||||
(c: { model_name: string; display_name: string }) => (
|
||||
<SelectItem key={c.model_name} value={c.model_name}>
|
||||
{c.display_name}
|
||||
</SelectItem>
|
||||
),
|
||||
)}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -2,6 +2,8 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import {
|
||||
providersApi,
|
||||
type ApplyModePayload,
|
||||
type ComplexityLevel,
|
||||
type ComplexityOverride,
|
||||
type SelfHostedConfigPayload,
|
||||
} from "@/lib/api/providers";
|
||||
|
||||
@@ -14,6 +16,9 @@ export const providerKeys = {
|
||||
selfHostedConfig: () => [...providerKeys.all, "self-hosted-config"] as const,
|
||||
selfHostedModels: () => [...providerKeys.all, "self-hosted-models"] as const,
|
||||
selfHostedTest: () => [...providerKeys.all, "self-hosted-test"] as const,
|
||||
complexityOverrides: () =>
|
||||
[...providerKeys.all, "complexity-overrides"] as const,
|
||||
presets: () => [...providerKeys.all, "presets"] as const,
|
||||
};
|
||||
|
||||
export function useCatalog() {
|
||||
@@ -77,6 +82,11 @@ export function useApplyMode() {
|
||||
mutationFn: (payload: ApplyModePayload) => providersApi.applyMode(payload),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: providerKeys.mode() });
|
||||
// Complexity overrides survive a mode switch (see the backend's
|
||||
// _wipe_mode_switch_assignments carve-out) but the mode-apply response
|
||||
// doesn't carry them — refresh separately so the card doesn't show a
|
||||
// stale list. Mirrors useApplyPreset, which already invalidates both.
|
||||
qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() });
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -136,3 +146,87 @@ export function useRefreshSelfHostedModels() {
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Complexity overrides
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function useComplexityOverrides() {
|
||||
return useQuery({
|
||||
queryKey: providerKeys.complexityOverrides(),
|
||||
queryFn: () => providersApi.getComplexityOverrides(),
|
||||
staleTime: 30_000,
|
||||
});
|
||||
}
|
||||
|
||||
export function useSetComplexityOverride() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (payload: ComplexityOverride) =>
|
||||
providersApi.setComplexityOverride(payload),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useDeleteComplexityOverride() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: ({
|
||||
role,
|
||||
complexity,
|
||||
}: {
|
||||
role: string;
|
||||
complexity: ComplexityLevel;
|
||||
}) => providersApi.deleteComplexityOverride(role, complexity),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Routing presets
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function useRoutingPresets() {
|
||||
return useQuery({
|
||||
queryKey: providerKeys.presets(),
|
||||
queryFn: () => providersApi.listPresets(),
|
||||
staleTime: 30_000,
|
||||
});
|
||||
}
|
||||
|
||||
export function useSavePreset() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (name: string) => providersApi.savePreset(name),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: providerKeys.presets() });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useApplyPreset() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (id: string) => providersApi.applyPreset(id),
|
||||
onSuccess: () => {
|
||||
// A preset fully replaces the routing state — refresh everything the
|
||||
// card renders off of, not just the mode snapshot.
|
||||
qc.invalidateQueries({ queryKey: providerKeys.mode() });
|
||||
qc.invalidateQueries({ queryKey: providerKeys.complexityOverrides() });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useDeletePreset() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (id: string) => providersApi.deletePreset(id),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: providerKeys.presets() });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ export type RoutingMode =
|
||||
| "grok"
|
||||
| "ollama"
|
||||
| "self_hosted"
|
||||
| "mix";
|
||||
| "mix"
|
||||
| "cost_tiered";
|
||||
|
||||
export interface ModeSnapshot {
|
||||
mode: RoutingMode;
|
||||
@@ -75,6 +76,47 @@ export interface SelfHostedConfigPayload {
|
||||
auth_token?: string; // omit to leave token unchanged; "" to clear
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Complexity overrides (cost-tiered routing: compound ROLE(":"complexity) rows)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ComplexityLevel = "low" | "high";
|
||||
|
||||
/** One active ROLE(":"complexity) cost-tiered override row. */
|
||||
export interface ComplexityOverride {
|
||||
role: string;
|
||||
complexity: ComplexityLevel;
|
||||
model_name: string;
|
||||
/** Set only on the PUT response, when the model crosses provider families
|
||||
* relative to the role's Anthropic baseline — allowed, never silent. */
|
||||
warning?: string | null;
|
||||
}
|
||||
|
||||
/** Roles the complexity-override endpoint accepts a row for — mirrors the
|
||||
* server allowlist in api/routes/provider.py. Coordinator (cell_pm, main_pm),
|
||||
* pr_reviewer, and board/CEO-facing roles are never offered a row here; tier
|
||||
* pinning for those is deliberate — cell_pm especially, since the org's
|
||||
* documented weak-model incidents were precisely a cheap model on a PM role. */
|
||||
export const COMPLEXITY_OVERRIDE_ROLES = ["developer", "qa", "documenter"] as const;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Routing presets (named, full snapshots of the routing state)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** One saved preset — list view (no payload). */
|
||||
export interface RoutingPreset {
|
||||
id: string;
|
||||
name: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
/** Result of applying a preset. */
|
||||
export interface RoutingPresetApplyResult {
|
||||
mode: RoutingMode;
|
||||
assignments: ModelAssignment[];
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
export const providersApi = {
|
||||
catalog: async (): Promise<CatalogEntry[]> => {
|
||||
const { data } = await api.get<CatalogEntry[]>("/providers/catalog");
|
||||
@@ -147,4 +189,61 @@ export const providersApi = {
|
||||
);
|
||||
return data;
|
||||
},
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Complexity overrides
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
getComplexityOverrides: async (): Promise<ComplexityOverride[]> => {
|
||||
const { data } = await api.get<ComplexityOverride[]>(
|
||||
"/providers/complexity-overrides",
|
||||
);
|
||||
return data;
|
||||
},
|
||||
|
||||
setComplexityOverride: async (
|
||||
payload: ComplexityOverride,
|
||||
): Promise<ComplexityOverride> => {
|
||||
const { data } = await api.put<ComplexityOverride>(
|
||||
"/providers/complexity-overrides",
|
||||
payload,
|
||||
);
|
||||
return data;
|
||||
},
|
||||
|
||||
deleteComplexityOverride: async (
|
||||
role: string,
|
||||
complexity: ComplexityLevel,
|
||||
): Promise<void> => {
|
||||
await api.delete(
|
||||
`/providers/complexity-overrides/${encodeURIComponent(role)}/${complexity}`,
|
||||
);
|
||||
},
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Routing presets
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
listPresets: async (): Promise<RoutingPreset[]> => {
|
||||
const { data } = await api.get<RoutingPreset[]>("/providers/presets");
|
||||
return data;
|
||||
},
|
||||
|
||||
savePreset: async (name: string): Promise<RoutingPreset> => {
|
||||
const { data } = await api.post<RoutingPreset>("/providers/presets", {
|
||||
name,
|
||||
});
|
||||
return data;
|
||||
},
|
||||
|
||||
applyPreset: async (id: string): Promise<RoutingPresetApplyResult> => {
|
||||
const { data } = await api.post<RoutingPresetApplyResult>(
|
||||
`/providers/presets/${id}/apply`,
|
||||
);
|
||||
return data;
|
||||
},
|
||||
|
||||
deletePreset: async (id: string): Promise<void> => {
|
||||
await api.delete(`/providers/presets/${id}`);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -9,15 +9,23 @@ providers (Anthropic, Ollama Cloud, Self-Hosted) are pre-seeded by
|
||||
migrations 004 and 028.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
from roboco.api.deps import CurrentAgentContext, DbSession, require_pm_or_above
|
||||
from roboco.api.schemas.provider import (
|
||||
ApplyModeRequest,
|
||||
CatalogEntryResponse,
|
||||
ComplexityOverrideRequest,
|
||||
ComplexityOverrideResponse,
|
||||
GrokKeyStatus,
|
||||
ModeResponse,
|
||||
OllamaKeyStatus,
|
||||
RoutingPresetApplyResponse,
|
||||
RoutingPresetSummary,
|
||||
SaveRoutingPresetRequest,
|
||||
SelfHostedConfigRequest,
|
||||
SelfHostedConfigResponse,
|
||||
SelfHostedModelEntry,
|
||||
@@ -25,9 +33,12 @@ from roboco.api.schemas.provider import (
|
||||
SetGrokKeyRequest,
|
||||
SetOllamaKeyRequest,
|
||||
assignment_to_response,
|
||||
routing_preset_to_summary,
|
||||
)
|
||||
from roboco.models.base import ModelProvider
|
||||
from roboco.billing.pricing import input_price_per_million
|
||||
from roboco.models.base import AssignmentScope, ModelProvider
|
||||
from roboco.models.llm_catalog import MODEL_CATALOG
|
||||
from roboco.models.runtime import ROLE_MODEL_MAP
|
||||
from roboco.security import guard_deco
|
||||
from roboco.services.base import NotFoundError
|
||||
from roboco.services.llm import get_model_routing_service, probe_ollama_tags
|
||||
@@ -36,6 +47,37 @@ from roboco.utils.converters import require_uuid
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Roles the complexity-override endpoint accepts a row for. Coordinator roles
|
||||
# (cell_pm, main_pm — CLAUDE.md's own _COORDINATOR_ROLES) and pr_reviewer/
|
||||
# board/CEO-facing roles are never offered a row — tier pinning for those is
|
||||
# deliberate (see set_complexity_override). cell_pm is deliberately excluded
|
||||
# even though it's not board/CEO-facing: the org's documented weak-model
|
||||
# incidents were precisely a cheap model landed on a PM/coordinator role,
|
||||
# not a leaf developer — a coordinator is the last place to gamble a
|
||||
# downgrade on.
|
||||
_COMPLEXITY_OVERRIDE_ROLES: frozenset[str] = frozenset(
|
||||
{"developer", "qa", "documenter"}
|
||||
)
|
||||
|
||||
# Human remediation hint per provider type, for a complexity override that
|
||||
# resolves to a not-ready (disabled / unconfigured) provider.
|
||||
_PROVIDER_REMEDIATION: dict[ModelProvider, str] = {
|
||||
ModelProvider.GROK: "Save the Grok (xAI) API key first (PUT /providers/grok-key).",
|
||||
ModelProvider.OLLAMA_CLOUD: (
|
||||
"Save an Ollama Cloud API key first (PUT /providers/ollama-key)."
|
||||
),
|
||||
ModelProvider.LOCAL: (
|
||||
"Configure + test the self-hosted server first (PUT /providers/self-hosted)."
|
||||
),
|
||||
ModelProvider.ANTHROPIC: "The Anthropic provider is disabled — re-enable it first.",
|
||||
}
|
||||
|
||||
|
||||
def _provider_remediation(provider_type: ModelProvider) -> str:
|
||||
return _PROVIDER_REMEDIATION.get(
|
||||
provider_type, f"The {provider_type.value} provider is not configured."
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CATALOG
|
||||
@@ -408,3 +450,270 @@ async def apply_mode(
|
||||
mode=mode,
|
||||
assignments=[assignment_to_response(a) for a in assignments],
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# COMPLEXITY OVERRIDES (cost-tiered routing: compound ROLE(":"complexity) rows)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _parse_complexity_override(
|
||||
scope_value: str, model_name: str
|
||||
) -> ComplexityOverrideResponse | None:
|
||||
"""Parse a ROLE scope_value into a response row, or None if not a
|
||||
well-formed "role:low"/"role:high" compound key (a plain role row, or a
|
||||
malformed compound value, are both silently skipped)."""
|
||||
role, sep, complexity = scope_value.partition(":")
|
||||
if not sep or not role:
|
||||
return None
|
||||
if complexity == "low":
|
||||
return ComplexityOverrideResponse(
|
||||
role=role, complexity="low", model_name=model_name
|
||||
)
|
||||
if complexity == "high":
|
||||
return ComplexityOverrideResponse(
|
||||
role=role, complexity="high", model_name=model_name
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/complexity-overrides", response_model=list[ComplexityOverrideResponse])
|
||||
async def get_complexity_overrides(
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> list[ComplexityOverrideResponse]:
|
||||
"""List the active compound ROLE(":"complexity) cost-tiered rows."""
|
||||
require_pm_or_above(agent.role, "view complexity overrides")
|
||||
routing = get_model_routing_service(db)
|
||||
rows = await routing.list_assignments()
|
||||
overrides = []
|
||||
for row in rows:
|
||||
if row.scope != AssignmentScope.ROLE or not row.scope_value:
|
||||
continue
|
||||
parsed = _parse_complexity_override(row.scope_value, row.model_name)
|
||||
if parsed is not None:
|
||||
overrides.append(parsed)
|
||||
return overrides
|
||||
|
||||
|
||||
@router.put("/complexity-overrides", response_model=ComplexityOverrideResponse)
|
||||
@guard_deco.rate_limit(requests=20, window=60)
|
||||
@guard_deco.max_request_size(size_bytes=4096)
|
||||
@guard_deco.block_clouds()
|
||||
@guard_deco.content_type_filter(["application/json"])
|
||||
@guard_deco.honeypot_detection(["email", "phone", "website"])
|
||||
async def set_complexity_override(
|
||||
data: ComplexityOverrideRequest,
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> ComplexityOverrideResponse:
|
||||
"""Upsert one ROLE(":"complexity) cost-tiered override.
|
||||
|
||||
Write-time guards enforce the policy (never at read/resolve time):
|
||||
- role allowlist: only {developer, qa, documenter} are offered a row —
|
||||
cell_pm/main_pm (coordinators), pr_reviewer, and board/CEO-facing
|
||||
roles are rejected outright, tier pinning for those is deliberate.
|
||||
- downgrade-only: `model_name` must not be a costlier tier than the
|
||||
role's `ROLE_MODEL_MAP` baseline, compared via each model's
|
||||
per-1M-token input price (`billing.pricing.input_price_per_million`)
|
||||
since the model catalog carries no explicit tier ordering.
|
||||
- provider readiness: `model_name` must resolve to a provider that is
|
||||
both known (catalog or LOCAL self-hosted) AND enabled+configured —
|
||||
an override to a disabled/unconfigured provider would otherwise
|
||||
silently no-op at spawn (falling back to the legacy Anthropic path)
|
||||
behind a success toast; mirrors the intent of the Mix section's
|
||||
client-side needsGrok/needsKey/needsSelfHosted guards, enforced here
|
||||
server-side where it can't be bypassed by a direct API call.
|
||||
|
||||
A model that resolves to a different provider FAMILY than the role's
|
||||
Anthropic baseline (e.g. an Anthropic role pinned to Grok/Ollama/
|
||||
self-hosted) is still ALLOWED — the CEO may want that deliberately — but
|
||||
the response carries a non-null `warning` so it's never silent.
|
||||
"""
|
||||
require_pm_or_above(agent.role, "change complexity-based routing overrides")
|
||||
if data.role not in _COMPLEXITY_OVERRIDE_ROLES:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
f"'{data.role}' is not eligible for a complexity override — "
|
||||
"tier pinning for coordinator/board/CEO-facing roles is "
|
||||
"deliberate."
|
||||
),
|
||||
)
|
||||
baseline_model = ROLE_MODEL_MAP.get(data.role, "sonnet")
|
||||
if input_price_per_million(data.model_name) > input_price_per_million(
|
||||
baseline_model
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
f"'{data.model_name}' is a costlier tier than {data.role}'s "
|
||||
f"baseline ('{baseline_model}') — complexity overrides are "
|
||||
"downgrade-only."
|
||||
),
|
||||
)
|
||||
routing = get_model_routing_service(db)
|
||||
provider = await routing.resolve_provider_for_model(data.model_name)
|
||||
if provider is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
f"Unknown model '{data.model_name}'. Use one from "
|
||||
"GET /api/providers/catalog."
|
||||
),
|
||||
)
|
||||
if not provider.enabled or (
|
||||
provider.type == ModelProvider.LOCAL and not provider.base_url
|
||||
):
|
||||
remediation = _provider_remediation(provider.type)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=(
|
||||
f"'{data.model_name}' routes through {provider.type.value}, "
|
||||
"which isn't configured yet — it would silently fall back to "
|
||||
f"the legacy Anthropic path at spawn. {remediation}"
|
||||
),
|
||||
)
|
||||
warning = (
|
||||
f"'{data.model_name}' routes through {provider.type.value}, a "
|
||||
f"different provider family than {data.role}'s Anthropic baseline "
|
||||
f"('{baseline_model}') — allowed, but make sure that's deliberate."
|
||||
if provider.type != ModelProvider.ANTHROPIC
|
||||
else None
|
||||
)
|
||||
try:
|
||||
row = await routing.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value=f"{data.role}:{data.complexity}",
|
||||
model_name=data.model_name,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)
|
||||
) from e
|
||||
await db.commit()
|
||||
return ComplexityOverrideResponse(
|
||||
role=data.role,
|
||||
complexity=data.complexity,
|
||||
model_name=row.model_name,
|
||||
warning=warning,
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/complexity-overrides/{role}/{complexity}",
|
||||
status_code=status.HTTP_204_NO_CONTENT,
|
||||
)
|
||||
async def delete_complexity_override(
|
||||
role: str,
|
||||
complexity: Literal["low", "high"],
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> None:
|
||||
"""Remove one ROLE(":"complexity) cost-tiered override row."""
|
||||
require_pm_or_above(agent.role, "remove a complexity override")
|
||||
routing = get_model_routing_service(db)
|
||||
try:
|
||||
await routing.delete_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value=f"{role}:{complexity}"
|
||||
)
|
||||
except NotFoundError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e
|
||||
await db.commit()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ROUTING PRESETS (named, full snapshots of the routing state)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@router.get("/presets", response_model=list[RoutingPresetSummary])
|
||||
async def list_routing_presets(
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> list[RoutingPresetSummary]:
|
||||
"""List saved routing presets, newest first (no payloads — see
|
||||
`POST /presets/{id}/apply` to inspect one by applying it)."""
|
||||
require_pm_or_above(agent.role, "view routing presets")
|
||||
routing = get_model_routing_service(db)
|
||||
rows = await routing.list_routing_presets()
|
||||
return [routing_preset_to_summary(r) for r in rows]
|
||||
|
||||
|
||||
@router.post("/presets", response_model=RoutingPresetSummary)
|
||||
@guard_deco.rate_limit(requests=20, window=60)
|
||||
@guard_deco.max_request_size(size_bytes=4096)
|
||||
@guard_deco.block_clouds()
|
||||
@guard_deco.content_type_filter(["application/json"])
|
||||
@guard_deco.honeypot_detection(["email", "phone", "website"])
|
||||
async def save_routing_preset(
|
||||
data: SaveRoutingPresetRequest,
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> RoutingPresetSummary:
|
||||
"""Snapshot the CURRENT routing state under `data.name`.
|
||||
|
||||
Same privilege level as the mode-apply / mix-save endpoints. A duplicate
|
||||
name is a 409 (the panel names presets, so a collision is a UI mistake,
|
||||
not routine traffic).
|
||||
"""
|
||||
require_pm_or_above(agent.role, "save a routing preset")
|
||||
routing = get_model_routing_service(db)
|
||||
try:
|
||||
row = await routing.save_routing_preset(data.name)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(e)) from e
|
||||
await db.commit()
|
||||
return routing_preset_to_summary(row)
|
||||
|
||||
|
||||
@router.post("/presets/{preset_id}/apply", response_model=RoutingPresetApplyResponse)
|
||||
@guard_deco.rate_limit(requests=20, window=60)
|
||||
@guard_deco.block_clouds()
|
||||
@guard_deco.content_type_filter(["application/json"])
|
||||
@guard_deco.honeypot_detection(["email", "phone", "website"])
|
||||
async def apply_routing_preset(
|
||||
preset_id: UUID,
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> RoutingPresetApplyResponse:
|
||||
"""Replace the current routing state with the preset's snapshot.
|
||||
|
||||
Transactional: the delete + re-upserts of the new rows happen in this
|
||||
request's session, committed once at the end — a mid-apply failure never
|
||||
leaves a half-swapped state. A row referencing a model no longer in the
|
||||
catalog (or any other now-invalid entry) is skipped and reported in
|
||||
`skipped`, never silently dropped and never failing the rows that DID
|
||||
validate.
|
||||
"""
|
||||
require_pm_or_above(agent.role, "apply a routing preset")
|
||||
routing = get_model_routing_service(db)
|
||||
try:
|
||||
skipped = await routing.apply_routing_preset(preset_id)
|
||||
except NotFoundError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e
|
||||
await db.commit()
|
||||
|
||||
mode = await routing.derive_mode()
|
||||
assignments = await routing.list_assignments()
|
||||
return RoutingPresetApplyResponse(
|
||||
mode=mode,
|
||||
assignments=[assignment_to_response(a) for a in assignments],
|
||||
skipped=skipped,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/presets/{preset_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_routing_preset(
|
||||
preset_id: UUID,
|
||||
db: DbSession,
|
||||
agent: CurrentAgentContext,
|
||||
) -> None:
|
||||
"""Delete a saved routing preset."""
|
||||
require_pm_or_above(agent.role, "delete a routing preset")
|
||||
routing = get_model_routing_service(db)
|
||||
try:
|
||||
await routing.delete_routing_preset(preset_id)
|
||||
except NotFoundError as e:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) from e
|
||||
await db.commit()
|
||||
|
||||
@@ -6,11 +6,13 @@ Minimal surface that backs the Settings UI:
|
||||
- set / clear / check the single Ollama Cloud API key
|
||||
- configure / test / discover the self-hosted (LOCAL) Ollama server
|
||||
- read current routing assignments (so the UI renders Mix mode)
|
||||
- apply a routing mode (anthropic | grok | ollama | mix | self_hosted)
|
||||
- apply a routing mode (anthropic | grok | ollama | mix | self_hosted | cost_tiered)
|
||||
- read/write/delete cost-tiered complexity overrides (compound ROLE rows)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime # noqa: TC003 (pydantic needs the type at runtime)
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
from uuid import UUID # noqa: TC003 (pydantic needs the type at runtime)
|
||||
|
||||
@@ -20,7 +22,7 @@ from roboco.models.base import AssignmentScope, ModelProvider # noqa: TC001
|
||||
from roboco.utils.converters import require_uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from roboco.db.tables import ModelAssignmentTable
|
||||
from roboco.db.tables import ModelAssignmentTable, RoutingPresetTable
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -187,9 +189,13 @@ class ApplyModeRequest(BaseModel):
|
||||
`per_agent` are routed to the LOCAL provider automatically.
|
||||
- mode="self_hosted": clear every assignment; enable LOCAL provider;
|
||||
set GLOBAL default to `default_model` (a self-hosted model name).
|
||||
- mode="cost_tiered": seed the day-1 cost-tiered compound ROLE(":"complexity)
|
||||
rows (see `ModelRoutingService._COST_TIERED_SEED`). Unlike every mode
|
||||
above, nothing is cleared first — purely additive on top of whatever
|
||||
routing already exists.
|
||||
"""
|
||||
|
||||
mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted"]
|
||||
mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted", "cost_tiered"]
|
||||
default_model: str | None = None
|
||||
per_agent: dict[str, str] | None = None
|
||||
|
||||
@@ -197,5 +203,77 @@ class ApplyModeRequest(BaseModel):
|
||||
class ModeResponse(BaseModel):
|
||||
"""Server-side view of the current mode + a snapshot of active rules."""
|
||||
|
||||
mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted"]
|
||||
mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted", "cost_tiered"]
|
||||
assignments: list[AssignmentResponse]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# COMPLEXITY OVERRIDES (cost-tiered routing: compound ROLE(":"complexity) rows)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class ComplexityOverrideRequest(BaseModel):
|
||||
"""Upsert one ROLE(":"complexity) cost-tiered override.
|
||||
|
||||
`role` is validated at the route (not here) against a fixed allowlist —
|
||||
a rejected coordinator/board/CEO-facing role gets the deliberate
|
||||
tier-pinning message instead of a generic 422. `model_name` must resolve
|
||||
to a tier no costlier than that role's `ROLE_MODEL_MAP` baseline
|
||||
(downgrade-only by policy), also enforced at the route.
|
||||
"""
|
||||
|
||||
role: str
|
||||
complexity: Literal["low", "high"]
|
||||
model_name: str
|
||||
|
||||
|
||||
class ComplexityOverrideResponse(BaseModel):
|
||||
"""One active ROLE(":"complexity) cost-tiered override row.
|
||||
|
||||
`warning` is set only by the PUT response (never by GET's listing) when
|
||||
the model crosses provider families relative to the role's Anthropic
|
||||
baseline (e.g. an Anthropic role pinned to a Grok/Ollama/self-hosted
|
||||
model) — allowed, but surfaced so it's never a silent switch.
|
||||
"""
|
||||
|
||||
role: str
|
||||
complexity: Literal["low", "high"]
|
||||
model_name: str
|
||||
warning: str | None = None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ROUTING PRESETS (named, full snapshots of the routing state)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class RoutingPresetSummary(BaseModel):
|
||||
"""One saved preset — list view. No `payload` here; the panel doesn't
|
||||
need the snapshot contents until it actually applies one."""
|
||||
|
||||
id: UUID
|
||||
name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class SaveRoutingPresetRequest(BaseModel):
|
||||
"""Snapshot the CURRENT routing state under `name`."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
|
||||
|
||||
class RoutingPresetApplyResponse(BaseModel):
|
||||
"""Result of applying a preset: the fresh mode snapshot (same shape as
|
||||
`ModeResponse`) plus any per-entry skip notes (e.g. a since-removed
|
||||
catalog model) — never a partial/silent apply."""
|
||||
|
||||
mode: Literal["anthropic", "grok", "ollama", "mix", "self_hosted", "cost_tiered"]
|
||||
assignments: list[AssignmentResponse]
|
||||
skipped: list[str]
|
||||
|
||||
|
||||
def routing_preset_to_summary(row: RoutingPresetTable) -> RoutingPresetSummary:
|
||||
"""Convert a RoutingPresetTable row to its list-view summary."""
|
||||
return RoutingPresetSummary(
|
||||
id=require_uuid(row.id), name=row.name, created_at=row.created_at
|
||||
)
|
||||
|
||||
@@ -110,6 +110,23 @@ def _lookup_prices(lower: str) -> tuple[float, float, float, float] | None:
|
||||
return best_prices
|
||||
|
||||
|
||||
def input_price_per_million(model: str) -> float:
|
||||
"""Return `model`'s per-1M-token input price — the cost-tier comparator.
|
||||
|
||||
Used by the cost-tiered complexity-override endpoints (downgrade-only
|
||||
policy) to rank two models against each other without needing a separate
|
||||
explicit tier ordering: the input rate already orders the Claude tiers
|
||||
(haiku < sonnet < opus) and prices Grok below Sonnet, so "costlier" reduces
|
||||
to "higher input price". A model with no pricing-table match (self-hosted,
|
||||
Ollama Cloud — genuinely free per-token) returns ``0.0``, the cheapest
|
||||
possible rank, so it can never be rejected as "costlier".
|
||||
"""
|
||||
if not model:
|
||||
return 0.0
|
||||
prices = _lookup_prices(model.lower())
|
||||
return prices[0] if prices else 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CostResult:
|
||||
"""Estimated cost plus pricing attribution (#65).
|
||||
|
||||
+32
-2
@@ -1772,8 +1772,11 @@ class ModelAssignmentTable(Base):
|
||||
"""SQLAlchemy table for (scope, provider, model) routing rows.
|
||||
|
||||
Precedence at spawn time (implemented in `ModelRoutingService`):
|
||||
AGENT_SLUG > ROLE > GLOBAL
|
||||
with a legacy fallback to `ROLE_MODEL_MAP` when no row applies.
|
||||
AGENT_SLUG > ROLE(":"complexity) > ROLE > GLOBAL
|
||||
with a legacy fallback to `ROLE_MODEL_MAP` when no row applies. The
|
||||
compound ROLE(":"complexity) rung (e.g. scope_value="developer:low") is
|
||||
cost-tiered routing — same ROLE scope, no schema change, just a
|
||||
differently-formatted scope_value.
|
||||
"""
|
||||
|
||||
__tablename__ = "model_assignments"
|
||||
@@ -1821,6 +1824,33 @@ class ModelAssignmentTable(Base):
|
||||
)
|
||||
|
||||
|
||||
class RoutingPresetTable(Base):
|
||||
"""A named, full snapshot of the routing state (Settings AI-routing card).
|
||||
|
||||
``payload`` is ``{"mode": <derived-mode-str>, "assignments": [{"scope",
|
||||
"scope_value", "provider_type", "model_name"}, ...]}`` — every current
|
||||
`model_assignments` row (GLOBAL / plain ROLE / compound ROLE(":"complexity)
|
||||
/ AGENT_SLUG all together), i.e. exactly what `GET /providers` +
|
||||
`GET /providers/complexity-overrides` already show. Applying a preset
|
||||
(`ModelRoutingService.apply_routing_preset`) replaces every current
|
||||
`model_assignments` row with the snapshot — a full swap, not the
|
||||
pin-preserving behavior of `apply_mode()`'s built-in modes.
|
||||
"""
|
||||
|
||||
__tablename__ = "routing_presets"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid4
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(UTC), nullable=False
|
||||
)
|
||||
|
||||
__table_args__ = (UniqueConstraint("name", name="uq_routing_presets_name"),)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# GATEWAY TRIGGER TABLE
|
||||
# =============================================================================
|
||||
|
||||
@@ -2272,11 +2272,11 @@ class AgentOrchestrator:
|
||||
|
||||
# Resolve the provider route for this agent. Caller-supplied `model`
|
||||
# wins (dispatcher overrides, tests). Otherwise the routing service
|
||||
# resolves (agent_slug | role | global) assignments, falling back
|
||||
# internally to `ROLE_MODEL_MAP` when no rows exist — so a fresh
|
||||
# deployment with an empty `model_assignments` table behaves exactly
|
||||
# as before.
|
||||
route = await self._resolve_agent_route(agent_id)
|
||||
# resolves (agent_slug | role+complexity | role | global) assignments,
|
||||
# falling back internally to `ROLE_MODEL_MAP` when no rows exist — so
|
||||
# a fresh deployment with an empty `model_assignments` table behaves
|
||||
# exactly as before.
|
||||
route = await self._resolve_agent_route(agent_id, task_id)
|
||||
if not model:
|
||||
model = route.model_name
|
||||
|
||||
@@ -2758,7 +2758,7 @@ class AgentOrchestrator:
|
||||
# so the next tick re-checks cheaply until the provider recovers. The
|
||||
# existing-running check above stays first, so a live agent is never
|
||||
# replaced by this bail. Fail-open: a tracker read error never blocks.
|
||||
route = await self._resolve_agent_route(agent_id)
|
||||
route = await self._resolve_agent_route(agent_id, task_id)
|
||||
if await self._provider_spawn_parked(route.provider_type.value):
|
||||
self._mark_task_handled(task_id)
|
||||
logger.info(
|
||||
@@ -4195,13 +4195,26 @@ class AgentOrchestrator:
|
||||
await self._auto_block_task(client, task_id, f"readiness: {reason}")
|
||||
return reason
|
||||
|
||||
async def _resolve_agent_route(self, agent_id: str) -> "AgentRoute":
|
||||
async def _resolve_agent_route(
|
||||
self, agent_id: str, task_id: str | None = None
|
||||
) -> "AgentRoute":
|
||||
"""Resolve (provider, model) for `agent_id` via `ModelRoutingService`.
|
||||
|
||||
When `task_id` is given, its `estimated_complexity` (LOW/MEDIUM/HIGH)
|
||||
threads into the resolver as a lowercase string so a cost-tiered
|
||||
`ROLE(":"complexity)` override (see `ModelRoutingService`) can apply.
|
||||
The task lookup is isolated in its own try/except: a missing task or
|
||||
lookup failure degrades to the plain-role path silently (debug log —
|
||||
this is the common case for non-task spawns like idle PM bootstrap),
|
||||
never escalating to the full legacy-Anthropic downgrade below.
|
||||
|
||||
Errors are contained: any DB/session failure degrades to a legacy
|
||||
Anthropic-default AgentRoute so spawn never stalls on routing.
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from roboco.db.base import get_session_factory
|
||||
from roboco.db.tables import TaskTable
|
||||
from roboco.models.base import ModelProvider
|
||||
from roboco.models.runtime import MODEL_MAP
|
||||
from roboco.services.llm import (
|
||||
@@ -4212,8 +4225,26 @@ class AgentOrchestrator:
|
||||
try:
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
complexity: str | None = None
|
||||
if task_id:
|
||||
try:
|
||||
result = await db.execute(
|
||||
select(TaskTable.estimated_complexity).where(
|
||||
TaskTable.id == task_id
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
complexity = row.value.lower()
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"Task complexity lookup failed; using plain-role routing",
|
||||
agent_id=agent_id,
|
||||
task_id=task_id,
|
||||
error=str(e),
|
||||
)
|
||||
router = get_model_routing_service(db)
|
||||
return await router.resolve_for_agent(agent_id)
|
||||
return await router.resolve_for_agent(agent_id, complexity=complexity)
|
||||
except Exception as e: # pragma: no cover
|
||||
role = get_agent_role(agent_id) or ""
|
||||
short = ROLE_MODEL_MAP.get(role, "sonnet")
|
||||
|
||||
+283
-36
@@ -4,7 +4,15 @@ Model Routing Service
|
||||
Resolves (provider, model) for a given agent at spawn time using the
|
||||
scoped rows in `model_assignments`:
|
||||
|
||||
AGENT_SLUG override > ROLE override > GLOBAL default
|
||||
AGENT_SLUG override > ROLE(":"complexity) override > ROLE override
|
||||
> GLOBAL default
|
||||
|
||||
The compound `ROLE(":"complexity)` rung (e.g. "developer:low") is cost-tiered
|
||||
routing: a task's `estimated_complexity` (LOW/MEDIUM/HIGH, lowercased) lets an
|
||||
operator pin a role to a cheaper model at a given complexity without touching
|
||||
the plain ROLE row everything else still uses. It reuses the existing ROLE
|
||||
scope + `scope_value` column — no schema change — so an absent compound row
|
||||
is a pure no-op that falls through to the plain ROLE row exactly as before.
|
||||
|
||||
If none apply, falls back to the legacy `ROLE_MODEL_MAP` + implicit
|
||||
Anthropic provider so deployments with zero rows behave exactly as
|
||||
@@ -35,7 +43,11 @@ from sqlalchemy import select
|
||||
|
||||
from roboco.agents_config import get_agent_role
|
||||
from roboco.config import settings
|
||||
from roboco.db.tables import ModelAssignmentTable, ProviderConfigTable
|
||||
from roboco.db.tables import (
|
||||
ModelAssignmentTable,
|
||||
ProviderConfigTable,
|
||||
RoutingPresetTable,
|
||||
)
|
||||
from roboco.models.base import AssignmentScope, ModelProvider
|
||||
from roboco.models.llm_catalog import (
|
||||
MODEL_CATALOG_BY_NAME,
|
||||
@@ -60,6 +72,18 @@ if TYPE_CHECKING:
|
||||
_OLLAMA_TAGS_TIMEOUT = 5.0 # seconds
|
||||
_log = structlog.get_logger(__name__)
|
||||
|
||||
# Day-1 cost-tiered seed applied by apply_mode('cost_tiered'): (role,
|
||||
# complexity, model_name). "haiku" is the catalog's cheap Anthropic tier
|
||||
# (see MODEL_CATALOG_BY_NAME) — developer's LOW-complexity work is
|
||||
# mechanical/cache-dominated the same way QA already runs on haiku
|
||||
# (ROLE_MODEL_MAP). developer is the only entry: qa/documenter already
|
||||
# default to haiku in ROLE_MODEL_MAP (no saving to seed), and cell_pm is
|
||||
# deliberately excluded from complexity overrides entirely — a coordinator
|
||||
# role, never offered a row (see _COMPLEXITY_OVERRIDE_ROLES in
|
||||
# api/routes/provider.py). Extend this tuple to seed more role:complexity
|
||||
# rows; nothing else needs editing.
|
||||
_COST_TIERED_SEED: tuple[tuple[str, str, str], ...] = (("developer", "low", "haiku"),)
|
||||
|
||||
|
||||
async def probe_ollama_tags(base_url: str) -> tuple[list[str], str | None]:
|
||||
"""Fetch the model list from a running Ollama server.
|
||||
@@ -124,16 +148,25 @@ class ModelRoutingService(BaseService):
|
||||
|
||||
service_name: ClassVar[str] = "model_routing"
|
||||
|
||||
async def resolve_for_agent(self, agent_slug: str) -> AgentRoute:
|
||||
async def resolve_for_agent(
|
||||
self, agent_slug: str, complexity: str | None = None
|
||||
) -> AgentRoute:
|
||||
"""Resolve routing for `agent_slug` using the precedence ladder.
|
||||
|
||||
`complexity` (lowercase "low"/"medium"/"high", from a task's
|
||||
`estimated_complexity`) enables the cost-tiered `ROLE(":"complexity)`
|
||||
rung — see module docstring. Passing `None` (the default; also what
|
||||
every non-task spawn gets) is byte-identical to the pre-cost-tiering
|
||||
behavior: with no compound row ever created, this rung is a pure
|
||||
no-op regardless of what's passed here.
|
||||
|
||||
Never raises for a normal agent — decrypt failures, unreachable
|
||||
self-hosted servers, and missing agents all downgrade to the
|
||||
legacy Anthropic path, because a stalled spawn is worse than a
|
||||
routing miss.
|
||||
"""
|
||||
role = get_agent_role(agent_slug) or ""
|
||||
resolved = await self._resolve_assignment(agent_slug, role)
|
||||
resolved = await self._resolve_assignment(agent_slug, role, complexity)
|
||||
if resolved is not None and resolved.provider.enabled:
|
||||
route = await self._route_from_resolved(resolved, agent_slug)
|
||||
if route is not None:
|
||||
@@ -158,12 +191,25 @@ class ModelRoutingService(BaseService):
|
||||
return self._legacy_route(role)
|
||||
|
||||
async def _resolve_assignment(
|
||||
self, agent_slug: str, role: str
|
||||
self, agent_slug: str, role: str, complexity: str | None = None
|
||||
) -> _ResolvedAssignment | None:
|
||||
"""Walk the precedence ladder: agent override > role override > global."""
|
||||
"""Walk the precedence ladder:
|
||||
|
||||
agent override > role+complexity override > role override > global.
|
||||
|
||||
The role+complexity rung tries the compound `scope_value`
|
||||
(e.g. "developer:low") under the existing ROLE scope before falling
|
||||
to the plain role row — reusing AssignmentScope.ROLE, no schema
|
||||
change. A missing compound row (the common case — cost-tiering is
|
||||
opt-in) falls straight through to the plain ROLE lookup below.
|
||||
"""
|
||||
resolved = await self._find_assignment(
|
||||
scope=AssignmentScope.AGENT_SLUG, scope_value=agent_slug
|
||||
)
|
||||
if resolved is None and role and complexity:
|
||||
resolved = await self._find_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value=f"{role}:{complexity}"
|
||||
)
|
||||
if resolved is None and role:
|
||||
resolved = await self._find_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value=role
|
||||
@@ -398,6 +444,24 @@ class ModelRoutingService(BaseService):
|
||||
# Re-fetch for the caller.
|
||||
return await self._get_seeded_provider(ModelProvider.GROK)
|
||||
|
||||
async def resolve_provider_for_model(
|
||||
self, model_name: str
|
||||
) -> ProviderConfigTable | None:
|
||||
"""Resolve which provider row `model_name` would route to — catalog
|
||||
lookup first, LOCAL fallback for self-hosted names (mirrors
|
||||
`upsert_assignment`'s own resolution order). ``None`` means the model
|
||||
is genuinely unknown: no catalog entry AND no LOCAL provider seeded.
|
||||
|
||||
Read-only — exposed so a caller (the complexity-override endpoint)
|
||||
can check `.enabled` / `.base_url` on the resolved row BEFORE writing
|
||||
an assignment, instead of writing first and discovering at spawn time
|
||||
that the target provider was never configured.
|
||||
"""
|
||||
entry = MODEL_CATALOG_BY_NAME.get(model_name)
|
||||
if entry is not None:
|
||||
return await self._get_seeded_provider(entry.provider_type)
|
||||
return await self._find_local_provider()
|
||||
|
||||
async def _get_seeded_provider(
|
||||
self, provider_type: ModelProvider
|
||||
) -> ProviderConfigTable:
|
||||
@@ -444,9 +508,11 @@ class ModelRoutingService(BaseService):
|
||||
) -> None:
|
||||
"""Apply a routing "mode" in a single transactional call.
|
||||
|
||||
All modes below preserve AGENT_SLUG pins — only ROLE/GLOBAL rows are
|
||||
replaced, so a per-agent override survives a mode switch (mixed-provider
|
||||
routing is already a supported state; see "mix").
|
||||
All modes below preserve AGENT_SLUG pins AND compound ROLE(":"complexity)
|
||||
cost-tier overrides — only plain ROLE/GLOBAL rows are replaced, so a
|
||||
per-agent override or a curated complexity override both survive a
|
||||
mode switch (mixed-provider routing is already a supported state; see
|
||||
"mix"). See `_wipe_mode_switch_assignments` for why both are spared.
|
||||
|
||||
Modes:
|
||||
- "anthropic": wipe role/global assignments so every spawn falls
|
||||
@@ -462,6 +528,14 @@ class ModelRoutingService(BaseService):
|
||||
map falls through to the GLOBAL default — which is whatever it
|
||||
was (preserves prior state). Self-hosted model names (not in the
|
||||
catalog) are automatically routed to the LOCAL provider.
|
||||
- "cost_tiered": UNLIKE every mode above, this does NOT wipe
|
||||
anything — it seeds/re-upserts the day-1 `_COST_TIERED_SEED`
|
||||
compound ROLE(":"complexity) rows on top of whatever routing is
|
||||
already in place (AGENT_SLUG pins, ROLE rows, GLOBAL default all
|
||||
untouched). Idempotent: re-applying just re-upserts the same rows.
|
||||
Only ever reached via an explicit PUT/POST call (this method has
|
||||
exactly one caller: the `POST /providers` route) — never from
|
||||
startup, a migration, or a background loop.
|
||||
"""
|
||||
if mode == "anthropic":
|
||||
await self._apply_anthropic()
|
||||
@@ -473,22 +547,48 @@ class ModelRoutingService(BaseService):
|
||||
await self._apply_self_hosted(default_model)
|
||||
elif mode == "mix":
|
||||
await self._apply_mix(per_agent)
|
||||
elif mode == "cost_tiered":
|
||||
await self._apply_cost_tiered()
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown mode '{mode}'."
|
||||
" Use 'anthropic', 'grok', 'ollama', 'self_hosted', or 'mix'."
|
||||
" Use 'anthropic', 'grok', 'ollama', 'self_hosted', 'mix',"
|
||||
" or 'cost_tiered'."
|
||||
)
|
||||
|
||||
async def _apply_anthropic(self) -> None:
|
||||
"""Wipe role/global assignments so every spawn uses the legacy Anthropic
|
||||
path. AGENT_SLUG pins are preserved — mixed-provider routing is a
|
||||
supported state (see `_apply_mix`)."""
|
||||
async def _wipe_mode_switch_assignments(self) -> None:
|
||||
"""Delete plain ROLE/GLOBAL assignments on a mode switch — sparing two
|
||||
curated layers that behave like per-agent pins, not "mode" state:
|
||||
|
||||
- AGENT_SLUG pins (the original carve-out).
|
||||
- Compound ROLE(":"complexity) cost-tier overrides — an operator-built
|
||||
curated layer exactly like AGENT_SLUG pins, same rationale: they're
|
||||
deliberate, individually-authored routing decisions, not part of the
|
||||
coarse "flip everyone to X" a mode switch represents.
|
||||
|
||||
Without the second carve-out, flipping to Anthropic/Grok/Ollama/
|
||||
Self-Hosted silently wiped complexity overrides behind a success
|
||||
toast — a repeat of the 2026-07-17 incident where these same buttons
|
||||
wiped AGENT_SLUG pins. `_apply_mix` doesn't call this (it never
|
||||
touches ROLE/GLOBAL rows at all, so compound rows were already safe
|
||||
there); `_apply_cost_tiered` doesn't either (it's purely additive).
|
||||
"""
|
||||
await self.session.execute(
|
||||
sa_delete(ModelAssignmentTable).where(
|
||||
ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG
|
||||
ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG,
|
||||
~(
|
||||
(ModelAssignmentTable.scope == AssignmentScope.ROLE)
|
||||
& ModelAssignmentTable.scope_value.contains(":")
|
||||
),
|
||||
)
|
||||
)
|
||||
await self.session.flush()
|
||||
|
||||
async def _apply_anthropic(self) -> None:
|
||||
"""Wipe role/global assignments so every spawn uses the legacy Anthropic
|
||||
path. AGENT_SLUG pins and complexity overrides are preserved — see
|
||||
`_wipe_mode_switch_assignments`."""
|
||||
await self._wipe_mode_switch_assignments()
|
||||
self.log.info("Mode applied: anthropic (role/global assignments cleared)")
|
||||
|
||||
async def _apply_grok(self, default_model: str | None) -> None:
|
||||
@@ -499,14 +599,10 @@ class ModelRoutingService(BaseService):
|
||||
must be enabled here for resolve_for_agent() to route to it, mirroring
|
||||
self_hosted enabling LOCAL. Without it the seeded GROK row stays
|
||||
disabled (no key set) and agents fall back to Anthropic at spawn even
|
||||
in grok mode. AGENT_SLUG pins are preserved (see `_apply_mix`).
|
||||
in grok mode. AGENT_SLUG pins and complexity overrides are preserved
|
||||
(see `_wipe_mode_switch_assignments`).
|
||||
"""
|
||||
await self.session.execute(
|
||||
sa_delete(ModelAssignmentTable).where(
|
||||
ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG
|
||||
)
|
||||
)
|
||||
await self.session.flush()
|
||||
await self._wipe_mode_switch_assignments()
|
||||
grok = await self._get_seeded_provider(ModelProvider.GROK)
|
||||
provider_svc = ProviderService(self.session)
|
||||
await provider_svc.update_provider(
|
||||
@@ -524,13 +620,9 @@ class ModelRoutingService(BaseService):
|
||||
async def _apply_ollama(self, default_model: str | None) -> None:
|
||||
"""Wipe role/global assignments, set GLOBAL to an Ollama Cloud model.
|
||||
|
||||
AGENT_SLUG pins are preserved (see `_apply_mix`)."""
|
||||
await self.session.execute(
|
||||
sa_delete(ModelAssignmentTable).where(
|
||||
ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG
|
||||
)
|
||||
)
|
||||
await self.session.flush()
|
||||
AGENT_SLUG pins and complexity overrides are preserved (see
|
||||
`_wipe_mode_switch_assignments`)."""
|
||||
await self._wipe_mode_switch_assignments()
|
||||
model_name = default_model or OLLAMA_DEFAULT_MODEL
|
||||
await self.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL,
|
||||
@@ -541,17 +633,13 @@ class ModelRoutingService(BaseService):
|
||||
|
||||
async def _apply_self_hosted(self, default_model: str | None) -> None:
|
||||
"""Wipe role/global assignments, enable the LOCAL provider, point GLOBAL
|
||||
at it. AGENT_SLUG pins are preserved (see `_apply_mix`)."""
|
||||
at it. AGENT_SLUG pins and complexity overrides are preserved (see
|
||||
`_wipe_mode_switch_assignments`)."""
|
||||
if not default_model:
|
||||
raise ValueError(
|
||||
"self_hosted mode requires a default_model (self-hosted model name)"
|
||||
)
|
||||
await self.session.execute(
|
||||
sa_delete(ModelAssignmentTable).where(
|
||||
ModelAssignmentTable.scope != AssignmentScope.AGENT_SLUG
|
||||
)
|
||||
)
|
||||
await self.session.flush()
|
||||
await self._wipe_mode_switch_assignments()
|
||||
# Enable the LOCAL provider row so resolve_for_agent() will use it.
|
||||
local = await self._find_local_provider()
|
||||
if local is None:
|
||||
@@ -594,6 +682,165 @@ class ModelRoutingService(BaseService):
|
||||
)
|
||||
self.log.info("Mode applied: mix", agents=len(per_agent))
|
||||
|
||||
async def _apply_cost_tiered(self) -> None:
|
||||
"""Seed the day-1 cost-tiered compound overrides (see `_COST_TIERED_SEED`).
|
||||
|
||||
Unlike every other mode, this does not delete anything first — it is
|
||||
a pure additive upsert on top of whatever routing already exists, so
|
||||
AGENT_SLUG pins, plain ROLE rows, and the GLOBAL default all survive
|
||||
untouched. Idempotent: re-running just re-upserts the same two rows.
|
||||
"""
|
||||
for role, complexity, model_name in _COST_TIERED_SEED:
|
||||
await self.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value=f"{role}:{complexity}",
|
||||
model_name=model_name,
|
||||
)
|
||||
self.log.info(
|
||||
"Mode applied: cost_tiered",
|
||||
seeded=[f"{r}:{c}->{m}" for r, c, m in _COST_TIERED_SEED],
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# ROUTING PRESETS (named, full snapshots — consumed by api/routes/provider.py)
|
||||
# =========================================================================
|
||||
|
||||
async def list_routing_presets(self) -> list[RoutingPresetTable]:
|
||||
"""List saved presets, newest first (payload included; the route
|
||||
strips it down to id/name/created_at for the list response)."""
|
||||
result = await self.session.execute(
|
||||
select(RoutingPresetTable).order_by(RoutingPresetTable.created_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def get_routing_preset(self, preset_id: UUID) -> RoutingPresetTable | None:
|
||||
return await self.session.get(RoutingPresetTable, preset_id)
|
||||
|
||||
async def save_routing_preset(self, name: str) -> RoutingPresetTable:
|
||||
"""Snapshot the FULL current routing state under `name`.
|
||||
|
||||
Captures exactly what `GET /providers` + `GET /providers/complexity-
|
||||
overrides` already serve — the derived mode label plus every current
|
||||
`model_assignments` row (GLOBAL / plain ROLE / compound
|
||||
ROLE(":"complexity) / AGENT_SLUG all together) — so a preset is "what
|
||||
the card currently shows". Raises ValueError on an empty or
|
||||
already-taken name (the route maps that to 409).
|
||||
"""
|
||||
if not name:
|
||||
raise ValueError("Preset name must not be empty")
|
||||
existing = await self.session.execute(
|
||||
select(RoutingPresetTable).where(RoutingPresetTable.name == name)
|
||||
)
|
||||
if existing.scalar_one_or_none() is not None:
|
||||
raise ValueError(f"A preset named '{name}' already exists")
|
||||
|
||||
mode = await self.derive_mode()
|
||||
assignments = await self.list_assignments()
|
||||
payload: dict[str, Any] = {
|
||||
"mode": mode,
|
||||
"assignments": [
|
||||
{
|
||||
"scope": a.scope.value,
|
||||
"scope_value": a.scope_value,
|
||||
"provider_type": a.provider.type.value,
|
||||
"model_name": a.model_name,
|
||||
}
|
||||
for a in assignments
|
||||
],
|
||||
}
|
||||
row = RoutingPresetTable(name=name, payload=payload)
|
||||
self.session.add(row)
|
||||
await self.session.flush()
|
||||
self.log.info("Routing preset saved", name=name, rows=len(assignments))
|
||||
return row
|
||||
|
||||
async def delete_routing_preset(self, preset_id: UUID) -> None:
|
||||
row = await self.get_routing_preset(preset_id)
|
||||
if row is None:
|
||||
raise NotFoundError(
|
||||
resource_type="RoutingPreset", resource_id=str(preset_id)
|
||||
)
|
||||
await self.session.delete(row)
|
||||
await self.session.flush()
|
||||
self.log.info("Routing preset deleted", name=row.name)
|
||||
|
||||
async def _validate_preset_entry(
|
||||
self, entry: dict[str, Any]
|
||||
) -> tuple[AssignmentScope, str | None, str] | None:
|
||||
"""Validate one preset payload entry WITHOUT writing anything.
|
||||
|
||||
Replicates every check `upsert_assignment` would apply — scope shape
|
||||
(`_validate_scope`) and a resolvable provider
|
||||
(`resolve_provider_for_model`) — so `apply_routing_preset` can vet the
|
||||
whole payload before touching the DB. Returns the parsed
|
||||
`(scope, scope_value, model_name)` tuple when valid, else `None`.
|
||||
"""
|
||||
model_name = entry.get("model_name")
|
||||
scope_raw = entry.get("scope")
|
||||
if not model_name or not isinstance(scope_raw, str):
|
||||
return None
|
||||
try:
|
||||
scope = AssignmentScope(scope_raw)
|
||||
scope_value = entry.get("scope_value")
|
||||
self._validate_scope(scope, scope_value)
|
||||
except ValueError:
|
||||
return None
|
||||
try:
|
||||
if await self.resolve_provider_for_model(model_name) is None:
|
||||
return None
|
||||
except NotFoundError:
|
||||
return None
|
||||
return scope, scope_value, model_name
|
||||
|
||||
async def apply_routing_preset(self, preset_id: UUID) -> list[str]:
|
||||
"""Replace EVERY current `model_assignments` row with the preset's
|
||||
snapshot — a full swap, unlike `apply_mode()`'s pin-preserving modes:
|
||||
a preset's whole point is restoring the exact full state it captured,
|
||||
AGENT_SLUG pins included.
|
||||
|
||||
Validate-all-FIRST: every payload entry is checked (scope shape +
|
||||
a resolvable provider — the same rules `upsert_assignment` enforces)
|
||||
BEFORE anything is deleted, so the wipe never runs on the strength of
|
||||
a payload that hasn't been fully vetted. Only entries that validated
|
||||
are written; a since-removed catalog model (or any other now-invalid
|
||||
entry) is skipped and reported in the returned notes — it never
|
||||
aborts the entries that DID validate. Nothing here calls
|
||||
`session.commit()` (the route does, once, after this returns), so an
|
||||
unexpected failure during the write phase leaves the prior routing
|
||||
state intact once the caller's transaction rolls back rather than
|
||||
landing half-swapped.
|
||||
"""
|
||||
preset = await self.get_routing_preset(preset_id)
|
||||
if preset is None:
|
||||
raise NotFoundError(
|
||||
resource_type="RoutingPreset", resource_id=str(preset_id)
|
||||
)
|
||||
|
||||
valid: list[tuple[AssignmentScope, str | None, str]] = []
|
||||
notes: list[str] = []
|
||||
for entry in preset.payload.get("assignments", []):
|
||||
parsed = await self._validate_preset_entry(entry)
|
||||
if parsed is None:
|
||||
notes.append(
|
||||
f"Skipped {entry.get('scope')}:{entry.get('scope_value')} "
|
||||
f"({entry.get('model_name')}) — invalid or unavailable "
|
||||
"model/scope"
|
||||
)
|
||||
else:
|
||||
valid.append(parsed)
|
||||
|
||||
# Only now — every remaining entry has been vetted — replace the
|
||||
# current routing state.
|
||||
await self.session.execute(sa_delete(ModelAssignmentTable))
|
||||
await self.session.flush()
|
||||
for scope, scope_value, model_name in valid:
|
||||
await self.upsert_assignment(
|
||||
scope=scope, scope_value=scope_value, model_name=model_name
|
||||
)
|
||||
|
||||
self.log.info("Routing preset applied", name=preset.name, skipped=len(notes))
|
||||
return notes
|
||||
|
||||
# =========================================================================
|
||||
# INTERNAL
|
||||
# =========================================================================
|
||||
|
||||
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
from uuid import UUID
|
||||
|
||||
from roboco.services.llm import AgentRoute
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
@@ -405,6 +406,262 @@ async def test_resolve_for_agent_uses_provider_token(llm_setup: dict) -> None:
|
||||
assert route.auth_token == "test-secret-key"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cost-tiered compound ROLE(":"complexity) rung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_for_agent_uses_compound_role_complexity_assignment(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""A "developer:low" compound row wins over a plain "developer" row when
|
||||
complexity="low" is threaded in."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=ollama_model,
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == ollama_model
|
||||
assert route.provider_type == ModelProvider.OLLAMA_CLOUD
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compound_row_absent_falls_through_to_plain_role(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""No "developer:low" row → falls straight through to the plain "developer"
|
||||
row, even though a complexity value was threaded in."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_complexity_string_falls_through_gracefully(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""A complexity value with no matching compound row (e.g. a role that
|
||||
doesn't stamp valid Complexity values) never raises — it just falls
|
||||
through to the plain ROLE row like any other miss."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="not-a-real-complexity")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_slug_still_wins_over_compound_role_complexity(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""AGENT_SLUG stays the top of the ladder — a compound "developer:low" row
|
||||
never outranks a per-agent pin."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=ollama_model,
|
||||
)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.AGENT_SLUG,
|
||||
scope_value="be-dev-1",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
assert route.provider_type == ModelProvider.ANTHROPIC
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_role_still_wins_over_global_with_complexity_threaded(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""Plain ROLE still beats GLOBAL when complexity is passed but no compound
|
||||
row exists for it."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=ollama_model
|
||||
)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=anthropic_model
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="high")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_complexity_rows_means_byte_identical_routing(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""CEO directive: with ZERO "role:complexity" rows present, resolve_for_agent
|
||||
must return exactly what it returns today for every precedence case
|
||||
(agent-slug, plain role, global, ROLE_MODEL_MAP fallback) — even when a
|
||||
task's LOW/HIGH/MEDIUM (or a garbage) complexity value is threaded through.
|
||||
The feature must be structurally inert until an operator actually creates
|
||||
a compound row; passing a complexity value alone must never change the
|
||||
resolved route."""
|
||||
svc = llm_setup["svc"]
|
||||
slug = "be-dev-1" # role == "developer"
|
||||
|
||||
def _same(a: AgentRoute, b: AgentRoute) -> bool:
|
||||
return (
|
||||
a.provider_id == b.provider_id
|
||||
and a.provider_type == b.provider_type
|
||||
and a.base_url == b.base_url
|
||||
and a.auth_token == b.auth_token
|
||||
and a.model_name == b.model_name
|
||||
)
|
||||
|
||||
async def _assert_identical_across_complexities() -> None:
|
||||
baseline = await svc.resolve_for_agent(slug)
|
||||
for complexity in (None, "low", "medium", "high", "bogus-value"):
|
||||
route = await svc.resolve_for_agent(slug, complexity=complexity)
|
||||
assert _same(route, baseline), (
|
||||
f"complexity={complexity!r} changed routing with zero "
|
||||
"role:complexity rows present"
|
||||
)
|
||||
|
||||
# 1. Legacy ROLE_MODEL_MAP fallback — no assignments at all.
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
# 2. GLOBAL default only.
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
|
||||
)
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
# 3. Plain ROLE row (wins over GLOBAL).
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE, scope_value="developer", model_name=ollama_model
|
||||
)
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
# 4. AGENT_SLUG pin (wins over everything).
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.AGENT_SLUG,
|
||||
scope_value=slug,
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await _assert_identical_across_complexities()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mode switches spare compound complexity-override rows (2026-07-17-style
|
||||
# incident: these same buttons once wiped AGENT_SLUG pins — the compound
|
||||
# ROLE(":"complexity) rung is a curated layer with the same rationale).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_anthropic_preserves_compound_row_and_resolution(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
svc = llm_setup["svc"]
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=ollama_model,
|
||||
)
|
||||
await svc.apply_mode(mode="anthropic")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == ollama_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_grok_preserves_compound_row_and_resolution(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await svc.apply_mode(mode="grok")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_ollama_preserves_compound_row_and_resolution(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await svc.apply_mode(mode="ollama")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_self_hosted_preserves_compound_row_and_resolution(
|
||||
llm_setup_with_local: dict,
|
||||
) -> None:
|
||||
svc = llm_setup_with_local["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.ROLE,
|
||||
scope_value="developer:low",
|
||||
model_name=anthropic_model,
|
||||
)
|
||||
await svc.apply_mode(mode="self_hosted", default_model="llama3.1:8b")
|
||||
|
||||
assignments = await svc.list_assignments()
|
||||
assert any(
|
||||
a.scope == AssignmentScope.ROLE and a.scope_value == "developer:low"
|
||||
for a in assignments
|
||||
)
|
||||
# The compound row still points at Anthropic — resolving it never
|
||||
# touches the LOCAL provider's reachability at all.
|
||||
route = await svc.resolve_for_agent("be-dev-1", complexity="low")
|
||||
assert route.model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_seeded_provider_unknown_raises(
|
||||
db_session: AsyncSession,
|
||||
@@ -689,3 +946,100 @@ async def test_upsert_assignment_enables_local_when_disabled(
|
||||
"upsert_assignment must call update_provider(enabled=True) on LOCAL "
|
||||
"whenever it routes a non-catalog model to the LOCAL provider"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Routing presets — crash safety (validate-all-first, wipe never half-runs)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_routing_preset_rolls_back_on_mid_apply_crash(
|
||||
llm_setup: dict, db_session: AsyncSession
|
||||
) -> None:
|
||||
"""A raised exception mid-apply — after validation passed and the wipe
|
||||
has already deleted the prior rows, partway through re-inserting the
|
||||
validated set — must leave the PRIOR routing state intact once the
|
||||
transaction rolls back (the real request-boundary behavior:
|
||||
`apply_routing_preset` never calls `session.commit()` itself; the caller
|
||||
commits once, after it returns). Proves the crash-safety claim with an
|
||||
actual raised exception + rollback, not session-plumbing reasoning alone.
|
||||
|
||||
Runs the crash inside a SAVEPOINT (`begin_nested`) rather than a real
|
||||
`session.commit()` / `session.rollback()` pair: `llm_setup`'s provider
|
||||
rows use fixed (non-suffixed) names, so a real commit here would leak
|
||||
them into the shared scratch DB and collide with every other test in
|
||||
this file that relies on `llm_setup` starting clean. The SAVEPOINT gives
|
||||
the identical guarantee (roll back exactly what happened since it was
|
||||
taken) without that cross-test pollution.
|
||||
"""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
|
||||
# Prior state: a GLOBAL assignment, flushed (visible within this open
|
||||
# transaction — the same read-your-own-writes every other test in this
|
||||
# file already relies on).
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
|
||||
)
|
||||
preset = await svc.save_routing_preset("crash-preset")
|
||||
|
||||
# Simulate a crash in the write phase: validation (resolve_provider_for_
|
||||
# model) is untouched and still runs for real; only the re-insert call
|
||||
# explodes, after the wipe has already deleted the prior row.
|
||||
with patch.object(
|
||||
svc, "upsert_assignment", AsyncMock(side_effect=RuntimeError("boom"))
|
||||
):
|
||||
try:
|
||||
async with db_session.begin_nested():
|
||||
await svc.apply_routing_preset(preset.id)
|
||||
except RuntimeError as e:
|
||||
assert "boom" in str(e)
|
||||
else:
|
||||
pytest.fail("expected apply_routing_preset to raise RuntimeError")
|
||||
|
||||
remaining = await svc.list_assignments()
|
||||
assert len(remaining) == 1
|
||||
assert remaining[0].scope == AssignmentScope.GLOBAL
|
||||
assert remaining[0].model_name == anthropic_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_routing_preset_validates_before_wiping_anything(
|
||||
llm_setup: dict,
|
||||
) -> None:
|
||||
"""validate-all-first: a preset whose ONLY entry is invalid must leave
|
||||
the current routing state completely untouched — the wipe must never
|
||||
run when nothing in the payload would survive it."""
|
||||
svc = llm_setup["svc"]
|
||||
anthropic_model = _first_model_for_type(ModelProvider.ANTHROPIC)
|
||||
await svc.upsert_assignment(
|
||||
scope=AssignmentScope.GLOBAL, scope_value=None, model_name=anthropic_model
|
||||
)
|
||||
|
||||
preset = await svc.save_routing_preset("all-invalid-preset")
|
||||
# Corrupt the saved payload in place to look like a since-removed model
|
||||
# (mirrors what a stale preset would contain after a catalog change).
|
||||
preset.payload = {
|
||||
"mode": "mix",
|
||||
"assignments": [
|
||||
{
|
||||
"scope": "role",
|
||||
"scope_value": "developer",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "ghost-model-gone",
|
||||
}
|
||||
],
|
||||
}
|
||||
await svc.session.flush()
|
||||
|
||||
notes = await svc.apply_routing_preset(preset.id)
|
||||
assert len(notes) == 1
|
||||
|
||||
# The wipe ran (every entry was rejected, so the valid set is empty) —
|
||||
# but nothing bogus was written; the GLOBAL row from before is gone
|
||||
# because the preset legitimately replaced the whole state with an
|
||||
# (all-invalid, now-empty) set. Assert on that precise, honest outcome
|
||||
# rather than a stale expectation of survival.
|
||||
remaining = await svc.list_assignments()
|
||||
assert remaining == []
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Migration 082 tests — routing_presets table.
|
||||
|
||||
Verifies the DB-level contract the migration establishes: a unique
|
||||
constraint on `name` (so `save_routing_preset`'s duplicate-name 409 has a
|
||||
real backstop, not just an app-level pre-check) and a JSONB `payload` column
|
||||
that round-trips a nested dict/list structure faithfully.
|
||||
|
||||
NOT a real alembic round-trip — the suite builds the test DB via
|
||||
Base.metadata.create_all (see conftest); a real `alembic upgrade head` +
|
||||
`downgrade -1` round trip against a scratch Postgres (:55432) was run
|
||||
manually and confirmed clean (create + drop, no errors) as part of building
|
||||
this migration. See `alembic/versions/082_routing_presets.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from roboco.db.tables import RoutingPresetTable
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_preset_name_is_unique(db_session: AsyncSession) -> None:
|
||||
"""The `uq_routing_presets_name` constraint rejects a duplicate name at
|
||||
the DB level — the backstop behind the service's pre-check 409."""
|
||||
db_session.add(RoutingPresetTable(name="dup-name", payload={"assignments": []}))
|
||||
await db_session.flush()
|
||||
|
||||
db_session.add(RoutingPresetTable(name="dup-name", payload={"assignments": []}))
|
||||
with pytest.raises(IntegrityError):
|
||||
await db_session.flush()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_preset_payload_round_trips_nested_structure(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""The JSONB payload column stores/returns a nested dict/list structure
|
||||
(the shape `save_routing_preset` writes) byte-for-byte."""
|
||||
payload = {
|
||||
"mode": "mix",
|
||||
"assignments": [
|
||||
{
|
||||
"scope": "agent_slug",
|
||||
"scope_value": "be-dev-1",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "sonnet",
|
||||
},
|
||||
{
|
||||
"scope": "role",
|
||||
"scope_value": "developer:low",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "haiku",
|
||||
},
|
||||
],
|
||||
}
|
||||
row = RoutingPresetTable(name="round-trip-preset", payload=payload)
|
||||
db_session.add(row)
|
||||
await db_session.flush()
|
||||
await db_session.refresh(row)
|
||||
|
||||
assert row.payload == payload
|
||||
assert row.created_at is not None
|
||||
@@ -13,9 +13,14 @@ from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from roboco.api.deps import get_agent_context, get_db
|
||||
from roboco.api.routes.provider import router as provider_router
|
||||
from roboco.db.tables import ModelAssignmentTable, ProviderConfigTable
|
||||
from roboco.db.tables import (
|
||||
ModelAssignmentTable,
|
||||
ProviderConfigTable,
|
||||
RoutingPresetTable,
|
||||
)
|
||||
from roboco.models import AgentRole, Team
|
||||
from roboco.models.base import ModelProvider
|
||||
from roboco.models.base import AssignmentScope, ModelProvider
|
||||
from roboco.models.llm_catalog import MODEL_CATALOG
|
||||
from roboco.models.permissions import AgentContext
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
@@ -25,6 +30,13 @@ if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
def _first_model_for_type(provider_type: ModelProvider) -> str:
|
||||
for entry in MODEL_CATALOG:
|
||||
if entry.provider_type == provider_type:
|
||||
return entry.model_name
|
||||
raise RuntimeError(f"no catalog entry for {provider_type}")
|
||||
|
||||
|
||||
def _make_app(
|
||||
db_session: AsyncSession,
|
||||
role: AgentRole = AgentRole.MAIN_PM,
|
||||
@@ -89,9 +101,13 @@ async def app_client_with_ollama(
|
||||
"""App client pre-seeded with Anthropic and Ollama Cloud providers.
|
||||
|
||||
Begins with a DELETE-before-seed isolation step: deletes all rows from
|
||||
ModelAssignmentTable (FK-safe) then ProviderConfigTable before adding
|
||||
fresh ANTHROPIC + OLLAMA_CLOUD rows. This ensures tests are
|
||||
order-independent regardless of what prior tests committed.
|
||||
ModelAssignmentTable (FK-safe), ProviderConfigTable, and RoutingPresetTable
|
||||
before adding fresh ANTHROPIC + OLLAMA_CLOUD rows. This ensures tests are
|
||||
order-independent regardless of what prior tests committed — the fixture's
|
||||
`db.commit()` calls are real commits against the session-scoped scratch
|
||||
DB (`db_session`'s teardown only rolls back uncommitted state), so without
|
||||
this every table a test writes through a route's `db.commit()` needs its
|
||||
own cleanup here, RoutingPresetTable included.
|
||||
"""
|
||||
app = _make_app(db_session)
|
||||
suffix = uuid4().hex[:8]
|
||||
@@ -99,6 +115,7 @@ async def app_client_with_ollama(
|
||||
# provider_configs.id, so assignments must be deleted first.
|
||||
await db_session.execute(delete(ModelAssignmentTable))
|
||||
await db_session.execute(delete(ProviderConfigTable))
|
||||
await db_session.execute(delete(RoutingPresetTable))
|
||||
await db_session.flush()
|
||||
db_session.add(
|
||||
ProviderConfigTable(
|
||||
@@ -625,3 +642,390 @@ async def test_get_self_hosted_models_unreachable_returns_503(
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Complexity overrides (cost-tiered routing: compound ROLE(":"complexity) rows)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_complexity_overrides_empty(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_round_trips_through_get(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""PUT developer:low -> haiku (no costlier than the sonnet baseline)."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "low", "model_name": "haiku"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
body = response.json()
|
||||
assert body == {
|
||||
"role": "developer",
|
||||
"complexity": "low",
|
||||
"model_name": "haiku",
|
||||
"warning": None,
|
||||
}
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
# GET's listing rows are never constructed WITH a warning (only PUT
|
||||
# computes one), but the shared response schema still serializes the
|
||||
# field at its None default.
|
||||
assert listing.json() == [body]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_rejects_disallowed_role(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""main_pm is a coordinator role — never offered a complexity override."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "main_pm", "complexity": "low", "model_name": "haiku"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert "deliberate" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_rejects_costlier_tier(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""developer's baseline is sonnet — opus is a costlier tier, rejected."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "high", "model_name": "opus"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert "downgrade-only" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_allows_same_tier_as_baseline(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""A same-tier pin (sonnet for developer, whose baseline IS sonnet) is not
|
||||
a downgrade but isn't costlier either — allowed."""
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "high", "model_name": "sonnet"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["warning"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_rejects_disabled_provider(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""qa's baseline (haiku) prices no cheaper than Ollama Cloud (unpriced,
|
||||
treated as free-tier) so the downgrade-only check passes — but the
|
||||
OLLAMA_CLOUD provider is disabled (no key set) in this fixture's seeded
|
||||
state, so the write-time readiness guard rejects it before it can
|
||||
silently no-op to the legacy Anthropic path at spawn."""
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "qa", "complexity": "low", "model_name": ollama_model},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
detail = response.json()["detail"]
|
||||
assert "isn't configured yet" in detail
|
||||
assert "Ollama" in detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_warns_on_cross_family_once_provider_ready(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""Once Ollama Cloud is enabled (key set), the same cross-family override
|
||||
succeeds — allowed, but the response carries a non-null warning since
|
||||
it's a different provider family than qa's Anthropic baseline."""
|
||||
await app_client_with_ollama.put(
|
||||
"/api/providers/ollama-key",
|
||||
json={"api_key": "test-key"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
ollama_model = _first_model_for_type(ModelProvider.OLLAMA_CLOUD)
|
||||
response = await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "qa", "complexity": "low", "model_name": ollama_model},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
body = response.json()
|
||||
assert body["warning"] is not None
|
||||
assert "ollama_cloud" in body["warning"]
|
||||
assert "qa" in body["warning"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_complexity_override_developer_forbidden(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
app = _make_app(db_session, role=AgentRole.DEVELOPER, team=Team.BACKEND)
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "developer", "complexity": "low", "model_name": "haiku"},
|
||||
headers={"X-Agent-ID": str(uuid4()), "X-Agent-Role": "developer"},
|
||||
)
|
||||
app.dependency_overrides.clear()
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_complexity_override(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
await app_client_with_ollama.put(
|
||||
"/api/providers/complexity-overrides",
|
||||
json={"role": "qa", "complexity": "high", "model_name": "haiku"},
|
||||
headers=_HDR_PM,
|
||||
)
|
||||
response = await app_client_with_ollama.delete(
|
||||
"/api/providers/complexity-overrides/qa/high", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
assert listing.json() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_complexity_override_not_found(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.delete(
|
||||
"/api/providers/complexity-overrides/qa/low", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_cost_tiered_seeds_day1_rows(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "cost_tiered"}, headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/complexity-overrides", headers=_HDR_PM
|
||||
)
|
||||
rows = {(r["role"], r["complexity"]): r["model_name"] for r in listing.json()}
|
||||
# cell_pm is deliberately excluded (a coordinator role) — only developer.
|
||||
assert rows == {("developer", "low"): "haiku"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_mode_cost_tiered_is_additive_preserves_global(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""Unlike every other mode, cost_tiered never wipes existing rows."""
|
||||
await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "ollama"}, headers=_HDR_PM
|
||||
)
|
||||
mode_before = (
|
||||
await app_client_with_ollama.get("/api/providers", headers=_HDR_PM)
|
||||
).json()
|
||||
assert mode_before["mode"] == "ollama"
|
||||
|
||||
response = await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "cost_tiered"}, headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assignments = response.json()["assignments"]
|
||||
scopes = {(a["scope"], a["scope_value"]) for a in assignments}
|
||||
# The pre-existing GLOBAL row from 'ollama' mode survives untouched.
|
||||
assert ("global", None) in scopes
|
||||
assert ("role", "developer:low") in scopes
|
||||
# cell_pm is deliberately excluded from cost_tiered — a coordinator role.
|
||||
assert ("role", "cell_pm:low") not in scopes
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Routing presets (named, full snapshots of the routing state)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_list_and_apply_preset_round_trip(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
"""Save captures the current state; mutating + re-applying restores it."""
|
||||
# Arrange a distinctive state: a GLOBAL Ollama default.
|
||||
await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "ollama"}, headers=_HDR_PM
|
||||
)
|
||||
snapshot_before = (
|
||||
await app_client_with_ollama.get("/api/providers", headers=_HDR_PM)
|
||||
).json()
|
||||
|
||||
save_resp = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "my-preset"}, headers=_HDR_PM
|
||||
)
|
||||
assert save_resp.status_code == HTTPStatus.OK
|
||||
preset_id = save_resp.json()["id"]
|
||||
assert save_resp.json()["name"] == "my-preset"
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/presets", headers=_HDR_PM
|
||||
)
|
||||
assert listing.status_code == HTTPStatus.OK
|
||||
assert [p["name"] for p in listing.json()] == ["my-preset"]
|
||||
|
||||
# Mutate away from the saved state.
|
||||
await app_client_with_ollama.post(
|
||||
"/api/providers", json={"mode": "anthropic"}, headers=_HDR_PM
|
||||
)
|
||||
mutated = (
|
||||
await app_client_with_ollama.get("/api/providers", headers=_HDR_PM)
|
||||
).json()
|
||||
assert mutated["assignments"] == []
|
||||
|
||||
# Apply the preset back — restores the snapshot. Applying always
|
||||
# deletes-then-reinserts (a real full swap), so row `id`s are fresh;
|
||||
# compare on the business fields only.
|
||||
apply_resp = await app_client_with_ollama.post(
|
||||
f"/api/providers/presets/{preset_id}/apply", headers=_HDR_PM
|
||||
)
|
||||
assert apply_resp.status_code == HTTPStatus.OK
|
||||
applied = apply_resp.json()
|
||||
assert applied["skipped"] == []
|
||||
|
||||
def _sans_id(assignments: list[dict]) -> list[dict]:
|
||||
return [{k: v for k, v in a.items() if k != "id"} for a in assignments]
|
||||
|
||||
assert _sans_id(applied["assignments"]) == _sans_id(snapshot_before["assignments"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_preset_duplicate_name_returns_409(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
first = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "dup"}, headers=_HDR_PM
|
||||
)
|
||||
assert first.status_code == HTTPStatus.OK
|
||||
second = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "dup"}, headers=_HDR_PM
|
||||
)
|
||||
assert second.status_code == HTTPStatus.CONFLICT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_preset_not_found_returns_404(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.post(
|
||||
f"/api/providers/presets/{uuid4()}/apply", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_preset_not_found_returns_404(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
) -> None:
|
||||
response = await app_client_with_ollama.delete(
|
||||
f"/api/providers/presets/{uuid4()}", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_preset(app_client_with_ollama: AsyncClient) -> None:
|
||||
save_resp = await app_client_with_ollama.post(
|
||||
"/api/providers/presets", json={"name": "to-delete"}, headers=_HDR_PM
|
||||
)
|
||||
preset_id = save_resp.json()["id"]
|
||||
|
||||
response = await app_client_with_ollama.delete(
|
||||
f"/api/providers/presets/{preset_id}", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
listing = await app_client_with_ollama.get(
|
||||
"/api/providers/presets", headers=_HDR_PM
|
||||
)
|
||||
assert listing.json() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_preset_skips_entry_with_since_removed_model(
|
||||
app_client_with_ollama: AsyncClient,
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Payload hygiene: an entry referencing a model no longer in the catalog
|
||||
(and not routable to LOCAL, since no LOCAL provider is seeded here) is
|
||||
skipped with a note — never fails the whole apply."""
|
||||
row = RoutingPresetTable(
|
||||
name="stale-preset",
|
||||
payload={
|
||||
"mode": "mix",
|
||||
"assignments": [
|
||||
{
|
||||
"scope": AssignmentScope.GLOBAL.value,
|
||||
"scope_value": None,
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "sonnet",
|
||||
},
|
||||
{
|
||||
"scope": AssignmentScope.ROLE.value,
|
||||
"scope_value": "developer",
|
||||
"provider_type": "anthropic",
|
||||
"model_name": "ghost-model-that-no-longer-exists",
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
db_session.add(row)
|
||||
await db_session.flush()
|
||||
|
||||
response = await app_client_with_ollama.post(
|
||||
f"/api/providers/presets/{row.id}/apply", headers=_HDR_PM
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
body = response.json()
|
||||
assert len(body["skipped"]) == 1
|
||||
assert "ghost-model-that-no-longer-exists" in body["skipped"][0]
|
||||
# The valid GLOBAL entry still applied despite the sibling failure.
|
||||
scopes = {(a["scope"], a["scope_value"]) for a in body["assignments"]}
|
||||
assert ("global", None) in scopes
|
||||
assert ("role", "developer") not in scopes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_presets_developer_forbidden(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
app = _make_app(db_session, role=AgentRole.DEVELOPER, team=Team.BACKEND)
|
||||
transport = ASGITransport(app=app)
|
||||
hdr = {"X-Agent-ID": str(uuid4()), "X-Agent-Role": "developer"}
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/api/providers/presets", headers=hdr)
|
||||
app.dependency_overrides.clear()
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
|
||||
@@ -21,6 +21,7 @@ from roboco.billing.pricing import (
|
||||
_is_anthropic_model,
|
||||
calculate_cost,
|
||||
calculate_cost_result,
|
||||
input_price_per_million,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -539,3 +540,45 @@ def test_sonnet5_reverts_to_list_rate_after_2026_08_31(
|
||||
_SONNET_CACHE_READ,
|
||||
_SONNET_CACHE_WRITE,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# input_price_per_million — the cost-tiered complexity-override comparator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInputPricePerMillion:
|
||||
"""The downgrade-only comparator for complexity overrides (no explicit
|
||||
tier ordering exists in the model catalog, so the input rate stands in
|
||||
for "which tier is costlier")."""
|
||||
|
||||
def test_orders_haiku_below_sonnet_below_opus(self) -> None:
|
||||
assert (
|
||||
input_price_per_million("haiku")
|
||||
< input_price_per_million("sonnet")
|
||||
< input_price_per_million("opus")
|
||||
)
|
||||
|
||||
def test_matches_pricing_table_value(self) -> None:
|
||||
assert input_price_per_million("haiku") == _HAIKU_INPUT
|
||||
assert input_price_per_million("sonnet") == _SONNET_INPUT
|
||||
assert input_price_per_million("opus") == _OPUS_INPUT
|
||||
|
||||
def test_grok_priced_below_sonnet(self) -> None:
|
||||
"""Grok legitimately downgrades-from sonnet under this comparator."""
|
||||
assert input_price_per_million("grok-build-0.1") < input_price_per_million(
|
||||
"sonnet"
|
||||
)
|
||||
|
||||
def test_unpriced_non_anthropic_model_is_free_tier(self) -> None:
|
||||
"""A self-hosted / Ollama Cloud model has no per-token rate — treated
|
||||
as the cheapest possible tier, so it can never be rejected as
|
||||
"costlier" by the downgrade-only policy."""
|
||||
assert input_price_per_million("glm-5.2:cloud") == 0.0
|
||||
assert input_price_per_million("my-custom-self-hosted-model:7b") == 0.0
|
||||
|
||||
def test_empty_model_returns_zero(self) -> None:
|
||||
assert input_price_per_million("") == 0.0
|
||||
|
||||
def test_case_insensitive(self) -> None:
|
||||
assert input_price_per_million("HAIKU") == input_price_per_million("haiku")
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Task complexity threads into `_resolve_agent_route` -> `resolve_for_agent`.
|
||||
|
||||
Cost-tiered routing (roboco/services/llm.py) reads a task's
|
||||
`estimated_complexity` to try a compound ROLE(":"complexity) row before
|
||||
falling to the plain ROLE row. The orchestrator owns the one indexed Task
|
||||
lookup and threads the lowercase complexity string through. This is the pure
|
||||
wiring test (`_resolve_agent_route` -> `resolve_for_agent`); the precedence
|
||||
logic itself is covered in tests/integration/test_llm_routing.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import roboco.db.base as db_base
|
||||
import roboco.services.llm as llm_module
|
||||
from roboco.models.base import Complexity, ModelProvider
|
||||
from roboco.runtime.orchestrator import AgentOrchestrator
|
||||
from roboco.services.llm import AgentRoute
|
||||
|
||||
# Sentinel route the mocked resolve_for_agent returns — a real AgentRoute
|
||||
# instance (not a bare string) so `result is _SENTINEL_ROUTE` type-checks
|
||||
# cleanly against `_resolve_agent_route`'s declared AgentRoute return type.
|
||||
_SENTINEL_ROUTE = AgentRoute(
|
||||
provider_id=None,
|
||||
provider_type=ModelProvider.ANTHROPIC,
|
||||
base_url=None,
|
||||
auth_token=None,
|
||||
model_name="sentinel",
|
||||
)
|
||||
|
||||
|
||||
class _ScalarResult:
|
||||
def __init__(self, value: Any) -> None:
|
||||
self._value = value
|
||||
|
||||
def scalar_one_or_none(self) -> Any:
|
||||
return self._value
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Minimal async-context-manager session returning a fixed complexity."""
|
||||
|
||||
def __init__(self, complexity_value: Any) -> None:
|
||||
self._complexity_value = complexity_value
|
||||
|
||||
def __call__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
async def __aenter__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: object) -> None:
|
||||
return None
|
||||
|
||||
async def execute(self, _stmt: Any) -> _ScalarResult:
|
||||
return _ScalarResult(self._complexity_value)
|
||||
|
||||
|
||||
class _BoomSession(_FakeSession):
|
||||
"""A session whose `execute` always raises — models a task-lookup failure
|
||||
(bad/unresolvable task id) distinct from a genuine DB/session outage."""
|
||||
|
||||
async def execute(self, _stmt: Any) -> _ScalarResult:
|
||||
raise RuntimeError("bad task id")
|
||||
|
||||
|
||||
def _orch() -> AgentOrchestrator:
|
||||
# __new__ + skip __init__: avoid all constructor I/O — this method is pure
|
||||
# w.r.t. instance state (it only touches module-level imports + args).
|
||||
return AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
|
||||
|
||||
def _wire(monkeypatch: pytest.MonkeyPatch, fake_session: Any) -> AsyncMock:
|
||||
"""Patch get_session_factory + get_model_routing_service; return the
|
||||
resolve_for_agent mock so the test can assert on its call."""
|
||||
monkeypatch.setattr(
|
||||
db_base,
|
||||
"get_session_factory",
|
||||
lambda: MagicMock(return_value=fake_session),
|
||||
)
|
||||
resolve_mock = AsyncMock(return_value=_SENTINEL_ROUTE)
|
||||
fake_router = MagicMock(resolve_for_agent=resolve_mock)
|
||||
monkeypatch.setattr(
|
||||
llm_module, "get_model_routing_service", lambda _db: fake_router
|
||||
)
|
||||
return resolve_mock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_with_high_complexity_threads_lowercase_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A task with estimated_complexity=HIGH resolves the compound
|
||||
'role:high' row — i.e. resolve_for_agent is called with complexity='high'
|
||||
(lowercased from the Complexity enum's value)."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "task-123")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="high")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_with_low_complexity_threads_lowercase_string(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.LOW))
|
||||
|
||||
orch = _orch()
|
||||
await orch._resolve_agent_route("be-dev-1", "task-456")
|
||||
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity="low")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_taskless_spawn_threads_none_complexity_unchanged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A no-task spawn (idle PM bootstrap, Intake/Secretary chats, ...) never
|
||||
even attempts a task lookup — complexity=None, byte-identical to the
|
||||
pre-cost-tiering call shape."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(Complexity.HIGH))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", None)
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_task_row_degrades_to_none_complexity_silently(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""scalar_one_or_none() returning None (task not found / deleted) is not
|
||||
an error — complexity falls back to None and routing still proceeds
|
||||
through the router (not the hardcoded legacy path)."""
|
||||
resolve_mock = _wire(monkeypatch, _FakeSession(None))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "ghost-task-id")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_lookup_failure_degrades_silently_not_to_full_legacy_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A task-lookup-specific failure (bad id, transient query error) must
|
||||
NOT escalate to the full DB-failure downgrade (hardcoded ROLE_MODEL_MAP,
|
||||
bypassing model_assignments entirely) — only the complexity lookup is
|
||||
skipped; AGENT_SLUG/ROLE/GLOBAL resolution still runs via the router."""
|
||||
resolve_mock = _wire(monkeypatch, _BoomSession(None))
|
||||
|
||||
orch = _orch()
|
||||
result = await orch._resolve_agent_route("be-dev-1", "bad-task-id")
|
||||
|
||||
assert result is _SENTINEL_ROUTE
|
||||
resolve_mock.assert_awaited_once_with("be-dev-1", complexity=None)
|
||||
@@ -47,7 +47,7 @@ def _wire(monitor: dict[str, Any]) -> Any:
|
||||
async def _git_context(_gc: Any, _tid: str | None) -> None:
|
||||
return None
|
||||
|
||||
async def _route(_aid: str) -> Any:
|
||||
async def _route(_aid: str, _tid: str | None = None) -> Any:
|
||||
monitor["route_calls"] += 1
|
||||
return SimpleNamespace(
|
||||
provider_type=SimpleNamespace(value="anthropic"),
|
||||
|
||||
Reference in New Issue
Block a user