From a3f84f31650035e6085f8f737c39459dac357a28 Mon Sep 17 00:00:00 2001 From: Renzo F <45401804+rennf93@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:14:13 +0200 Subject: [PATCH] chore(routing): retire haiku from delivery-lifecycle roles (#680) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Haiku can't reliably emit the structured envelopes the lifecycle now runs on — pass_review's per-AC criteria_verified, delegate's covers_parent_criteria, the findings ledger. A haiku QA/PM claims, gets validation-rejected, idles, respawns, and loops without advancing a task (2026-07-24 live: fe-qa on haiku looped four awaiting_qa tasks to zero progress). The per-token savings (~2x under the Sonnet-5 promo, 3x after) are dwarfed by the cost of a review that never completes. Three coordinated changes: ROLE_MODEL_MAP's qa/documenter defaults move haiku -> sonnet (the actual source of the live incident); the cost_tiered developer:low -> haiku seed retires to empty (the floor would upgrade it anyway); and a structured-verb capability floor upgrades any below-floor Anthropic assignment to sonnet at resolution — from a pin, a ROLE row, or a future map edit — in both the assignment and legacy paths. Non-Anthropic providers are untouched (an Anthropic-tier floor, not a provider policy). pr_reviewer/auditor stay on opus. Co-authored-by: Renn F --- roboco/models/runtime.py | 11 ++-- roboco/services/llm.py | 72 +++++++++++++++++++---- tests/integration/test_provider_routes.py | 12 ++-- tests/unit/llm/test_routing_downgrade.py | 54 +++++++++++++++++ tests/unit/models/test_model_map.py | 9 ++- 5 files changed, 132 insertions(+), 26 deletions(-) diff --git a/roboco/models/runtime.py b/roboco/models/runtime.py index e8aa8e98..7d13ce3b 100644 --- a/roboco/models/runtime.py +++ b/roboco/models/runtime.py @@ -119,13 +119,16 @@ MODEL_MAP: dict[str, str] = { # Default model by role ROLE_MODEL_MAP: dict[str, str] = { "developer": "sonnet", - # QA — mechanical gate work (read diff, run the gate, pass/fail); its cost is - # cache-dominated, so the cheapest tier fits. Haiku ignores effort (fine here). - "qa": "haiku", + # QA — emits structured review envelopes (pass_review's per-AC + # criteria_verified, the findings ledger) the haiku tier can't reliably + # produce: a haiku QA claims, gets validation-rejected, idles, respawns, + # and loops without passing (2026-07-24 live incident). Sonnet is the + # floor for every structured-verb lifecycle role. + "qa": "sonnet", # PR reviewer — reviews untrusted external/fork PRs and gates root→master; # highest-stakes review, so opus rather than the sonnet review tier. "pr_reviewer": "opus", - "documenter": "haiku", + "documenter": "sonnet", "cell_pm": "sonnet", # Main PM — cost is dominated by cache read/write of a large coordination # context; Sonnet 5's cache-write is ~12x cheaper than Opus. Experiment: watch diff --git a/roboco/services/llm.py b/roboco/services/llm.py index 4ebbb1df..8b046dea 100644 --- a/roboco/services/llm.py +++ b/roboco/services/llm.py @@ -33,7 +33,7 @@ Self-hosted (LOCAL) provider support: from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast import httpx @@ -72,17 +72,16 @@ if TYPE_CHECKING: _OLLAMA_TAGS_TIMEOUT = 5.0 # seconds _log = structlog.get_logger(__name__) -# Day-1 cost-tiered seed applied by apply_mode('cost_tiered'): (role, -# complexity, model_name). "haiku" is the catalog's cheap Anthropic tier -# (see MODEL_CATALOG_BY_NAME) — developer's LOW-complexity work is -# mechanical/cache-dominated the same way QA already runs on haiku -# (ROLE_MODEL_MAP). developer is the only entry: qa/documenter already -# default to haiku in ROLE_MODEL_MAP (no saving to seed), and cell_pm is -# deliberately excluded from complexity overrides entirely — a coordinator -# role, never offered a row (see _COMPLEXITY_OVERRIDE_ROLES in -# api/routes/provider.py). Extend this tuple to seed more role:complexity -# rows; nothing else needs editing. -_COST_TIERED_SEED: tuple[tuple[str, str, str], ...] = (("developer", "low", "haiku"),) +# Cost-tiered seed applied by apply_mode('cost_tiered'): (role, complexity, +# model_name). RETIRED to empty (2026-07-24): the one entry seeded +# developer:low -> haiku, but haiku can't reliably emit the structured +# lifecycle envelopes (see _below_capability_floor), so the "cheap tier for +# mechanical work" premise no longer holds — the capability floor would +# upgrade the seeded row back to sonnet anyway, making the seed a confusing +# no-op. cost_tiered mode stays wired (empty seed = inert) so an operator can +# re-seed a genuinely-above-floor role:complexity tier here later; nothing +# else needs editing. +_COST_TIERED_SEED: tuple[tuple[str, str, str], ...] = () # derive_mode()'s single-GLOBAL-assignment lookup — a provider type maps to # its "mode" label 1:1 for every mode `apply_mode` can set via a sole GLOBAL @@ -173,6 +172,29 @@ INTERACTIVE_UNSUPPORTED_PROVIDERS: tuple[ModelProvider, ...] = ( ) +# Capability floor: haiku-class models cannot reliably emit the structured +# review/coordination envelopes the lifecycle now requires (pass_review's +# criteria_verified, delegate's covers_parent_criteria, the findings ledger) +# — a haiku QA/PM/reviewer claims, gets validation-rejected, idles, respawns, +# and loops without ever advancing a task (2026-07-24 live: fe-qa on haiku +# looped 4 awaiting_qa tasks to zero progress). Every delivery-lifecycle role +# runs on structured verbs, so any Anthropic assignment below the floor is +# upgraded to the floor model at resolution, from whatever source (a pin, a +# ROLE row, or the retired cost_tiered seed). Non-Anthropic providers are +# untouched — this is an Anthropic-tier floor, not a provider policy. +_CAPABILITY_FLOOR_MODEL = "sonnet" +_BELOW_FLOOR_MODEL_MARKER = "haiku" + + +def _below_capability_floor(resolved: _ResolvedAssignment) -> bool: + """True iff an Anthropic assignment resolves below the structured-verb + floor (a haiku-class model). Provider-agnostic names never match.""" + return ( + resolved.provider.type is ModelProvider.ANTHROPIC + and _BELOW_FLOOR_MODEL_MARKER in resolved.model_name.lower() + ) + + def _interactive_exempt(agent_slug: str, resolved: _ResolvedAssignment) -> bool: """True iff a GLOBAL/ROLE row lands an interactive agent on a delivery-only provider — the resolver then keeps it on the legacy path. @@ -209,6 +231,7 @@ class ModelRoutingService(BaseService): """ role = get_agent_role(agent_slug) or "" resolved = await self._resolve_assignment(agent_slug, role, complexity) + resolved = self._floor_below_capability(agent_slug, resolved) if resolved is not None and _interactive_exempt(agent_slug, resolved): # A fleet-wide GLOBAL/ROLE row landed an interactive agent on a # delivery-only provider (e.g. the one-click Codex/Gemini mode). @@ -325,9 +348,32 @@ class ModelRoutingService(BaseService): ) return None + def _floor_below_capability( + self, agent_slug: str, resolved: _ResolvedAssignment | None + ) -> _ResolvedAssignment | None: + """Upgrade a below-floor Anthropic assignment to the structured-verb + floor (see `_below_capability_floor`); pass everything else through.""" + if resolved is None or not _below_capability_floor(resolved): + return resolved + self.log.info( + "Upgrading below-floor model to the structured-verb floor", + agent_slug=agent_slug, + from_model=resolved.model_name, + to_model=_CAPABILITY_FLOOR_MODEL, + ) + return replace(resolved, model_name=_CAPABILITY_FLOOR_MODEL) + def _legacy_route(self, role: str) -> AgentRoute: - """Legacy fallback: role-default short name through MODEL_MAP.""" + """Legacy fallback: role-default short name through MODEL_MAP. + + The structured-verb capability floor applies here too: any role whose + map entry is a below-floor tier is upgraded, so a future map edit + (or a role not covered above) can't silently reintroduce a haiku + lifecycle agent. + """ short = ROLE_MODEL_MAP.get(role, "sonnet") + if _BELOW_FLOOR_MODEL_MARKER in short.lower(): + short = _CAPABILITY_FLOOR_MODEL return AgentRoute( provider_id=None, provider_type=ModelProvider.ANTHROPIC, diff --git a/tests/integration/test_provider_routes.py b/tests/integration/test_provider_routes.py index 2ae54c0e..fb440a35 100644 --- a/tests/integration/test_provider_routes.py +++ b/tests/integration/test_provider_routes.py @@ -956,8 +956,10 @@ async def test_apply_mode_cost_tiered_seeds_day1_rows( "/api/providers/complexity-overrides", headers=_HDR_PM ) rows = {(r["role"], r["complexity"]): r["model_name"] for r in listing.json()} - # cell_pm is deliberately excluded (a coordinator role) — only developer. - assert rows == {("developer", "low"): "haiku"} + # Seed retired (2026-07-24): its only entry was developer:low -> haiku, + # which the structured-verb capability floor would upgrade to sonnet + # anyway. cost_tiered mode stays wired but seeds nothing now. + assert rows == {} @pytest.mark.asyncio @@ -979,11 +981,9 @@ async def test_apply_mode_cost_tiered_is_additive_preserves_global( assert response.status_code == HTTPStatus.OK assignments = response.json()["assignments"] scopes = {(a["scope"], a["scope_value"]) for a in assignments} - # The pre-existing GLOBAL row from 'ollama' mode survives untouched. + # The pre-existing GLOBAL row from 'ollama' mode survives untouched — the + # non-wiping contract holds even with the seed retired to empty. assert ("global", None) in scopes - assert ("role", "developer:low") in scopes - # cell_pm is deliberately excluded from cost_tiered — a coordinator role. - assert ("role", "cell_pm:low") not in scopes # ============================================================================= diff --git a/tests/unit/llm/test_routing_downgrade.py b/tests/unit/llm/test_routing_downgrade.py index f21ba7d8..a9f4e8ed 100644 --- a/tests/unit/llm/test_routing_downgrade.py +++ b/tests/unit/llm/test_routing_downgrade.py @@ -77,3 +77,57 @@ async def test_resolve_for_agent_no_assignment_is_silent_legacy_fallback() -> No assert route.provider_type == ModelProvider.ANTHROPIC svc.log.warning.assert_not_called() + + +@pytest.mark.asyncio +async def test_haiku_lifecycle_assignment_upgraded_to_the_floor() -> None: + """The structured-verb capability floor: a haiku-class Anthropic + assignment on any lifecycle role resolves to sonnet instead, so a QA/PM + agent can produce the structured review envelopes (2026-07-24 retirement). + """ + provider = MagicMock( + enabled=True, id="prov-anthropic", type=ModelProvider.ANTHROPIC + ) + resolved = _ResolvedAssignment( + provider=provider, + model_name="claude-haiku-4-5-20251001", + scope=AssignmentScope.ROLE, + ) + svc = _svc() + captured: dict[str, str] = {} + + async def _fake_route(res: _ResolvedAssignment, _slug: str) -> object: + captured["model"] = res.model_name + return object() + + with ( + patch.object(svc, "_resolve_assignment", AsyncMock(return_value=resolved)), + patch.object(svc, "_route_from_resolved", _fake_route), + ): + await svc.resolve_for_agent("be-qa") + + assert captured["model"] == "sonnet" + + +@pytest.mark.asyncio +async def test_non_anthropic_below_floor_name_is_untouched() -> None: + """The floor is an Anthropic-tier policy — a non-Anthropic model whose + name happens to contain the marker is never upgraded.""" + provider = MagicMock(enabled=True, id="prov-grok", type=ModelProvider.GROK) + resolved = _ResolvedAssignment( + provider=provider, model_name="grok-haiku-ish", scope=AssignmentScope.ROLE + ) + svc = _svc() + captured: dict[str, str] = {} + + async def _fake_route(res: _ResolvedAssignment, _slug: str) -> object: + captured["model"] = res.model_name + return object() + + with ( + patch.object(svc, "_resolve_assignment", AsyncMock(return_value=resolved)), + patch.object(svc, "_route_from_resolved", _fake_route), + ): + await svc.resolve_for_agent("be-qa") + + assert captured["model"] == "grok-haiku-ish" diff --git a/tests/unit/models/test_model_map.py b/tests/unit/models/test_model_map.py index 0b5586bc..efd12c76 100644 --- a/tests/unit/models/test_model_map.py +++ b/tests/unit/models/test_model_map.py @@ -23,9 +23,12 @@ def test_sonnet_5_is_priced() -> None: assert cost > 0.0 -def test_qa_role_routes_to_haiku() -> None: - # Phase 2: QA is mechanical gate work → cheapest tier. - assert ROLE_MODEL_MAP["qa"] == "haiku" +def test_no_lifecycle_role_routes_below_the_structured_verb_floor() -> None: + # Retired haiku from lifecycle roles (2026-07-24): a haiku QA/documenter + # can't emit the structured review envelopes (criteria_verified, findings) + # and loops without passing. No role's default may be a haiku-class tier. + for role, model in ROLE_MODEL_MAP.items(): + assert "haiku" not in model.lower(), f"{role} routes to a below-floor {model}" def test_main_pm_role_routes_to_sonnet() -> None: