mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
chore(routing): retire haiku from delivery-lifecycle roles (#680)
Haiku can't reliably emit the structured envelopes the lifecycle now runs on — pass_review's per-AC criteria_verified, delegate's covers_parent_criteria, the findings ledger. A haiku QA/PM claims, gets validation-rejected, idles, respawns, and loops without advancing a task (2026-07-24 live: fe-qa on haiku looped four awaiting_qa tasks to zero progress). The per-token savings (~2x under the Sonnet-5 promo, 3x after) are dwarfed by the cost of a review that never completes. Three coordinated changes: ROLE_MODEL_MAP's qa/documenter defaults move haiku -> sonnet (the actual source of the live incident); the cost_tiered developer:low -> haiku seed retires to empty (the floor would upgrade it anyway); and a structured-verb capability floor upgrades any below-floor Anthropic assignment to sonnet at resolution — from a pin, a ROLE row, or a future map edit — in both the assignment and legacy paths. Non-Anthropic providers are untouched (an Anthropic-tier floor, not a provider policy). pr_reviewer/auditor stay on opus. Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -77,3 +77,57 @@ async def test_resolve_for_agent_no_assignment_is_silent_legacy_fallback() -> No
|
||||
|
||||
assert route.provider_type == ModelProvider.ANTHROPIC
|
||||
svc.log.warning.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_haiku_lifecycle_assignment_upgraded_to_the_floor() -> None:
|
||||
"""The structured-verb capability floor: a haiku-class Anthropic
|
||||
assignment on any lifecycle role resolves to sonnet instead, so a QA/PM
|
||||
agent can produce the structured review envelopes (2026-07-24 retirement).
|
||||
"""
|
||||
provider = MagicMock(
|
||||
enabled=True, id="prov-anthropic", type=ModelProvider.ANTHROPIC
|
||||
)
|
||||
resolved = _ResolvedAssignment(
|
||||
provider=provider,
|
||||
model_name="claude-haiku-4-5-20251001",
|
||||
scope=AssignmentScope.ROLE,
|
||||
)
|
||||
svc = _svc()
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def _fake_route(res: _ResolvedAssignment, _slug: str) -> object:
|
||||
captured["model"] = res.model_name
|
||||
return object()
|
||||
|
||||
with (
|
||||
patch.object(svc, "_resolve_assignment", AsyncMock(return_value=resolved)),
|
||||
patch.object(svc, "_route_from_resolved", _fake_route),
|
||||
):
|
||||
await svc.resolve_for_agent("be-qa")
|
||||
|
||||
assert captured["model"] == "sonnet"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_anthropic_below_floor_name_is_untouched() -> None:
|
||||
"""The floor is an Anthropic-tier policy — a non-Anthropic model whose
|
||||
name happens to contain the marker is never upgraded."""
|
||||
provider = MagicMock(enabled=True, id="prov-grok", type=ModelProvider.GROK)
|
||||
resolved = _ResolvedAssignment(
|
||||
provider=provider, model_name="grok-haiku-ish", scope=AssignmentScope.ROLE
|
||||
)
|
||||
svc = _svc()
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
async def _fake_route(res: _ResolvedAssignment, _slug: str) -> object:
|
||||
captured["model"] = res.model_name
|
||||
return object()
|
||||
|
||||
with (
|
||||
patch.object(svc, "_resolve_assignment", AsyncMock(return_value=resolved)),
|
||||
patch.object(svc, "_route_from_resolved", _fake_route),
|
||||
):
|
||||
await svc.resolve_for_agent("be-qa")
|
||||
|
||||
assert captured["model"] == "grok-haiku-ish"
|
||||
|
||||
@@ -23,9 +23,12 @@ def test_sonnet_5_is_priced() -> None:
|
||||
assert cost > 0.0
|
||||
|
||||
|
||||
def test_qa_role_routes_to_haiku() -> None:
|
||||
# Phase 2: QA is mechanical gate work → cheapest tier.
|
||||
assert ROLE_MODEL_MAP["qa"] == "haiku"
|
||||
def test_no_lifecycle_role_routes_below_the_structured_verb_floor() -> None:
|
||||
# Retired haiku from lifecycle roles (2026-07-24): a haiku QA/documenter
|
||||
# can't emit the structured review envelopes (criteria_verified, findings)
|
||||
# and loops without passing. No role's default may be a haiku-class tier.
|
||||
for role, model in ROLE_MODEL_MAP.items():
|
||||
assert "haiku" not in model.lower(), f"{role} routes to a below-floor {model}"
|
||||
|
||||
|
||||
def test_main_pm_role_routes_to_sonnet() -> None:
|
||||
|
||||
Reference in New Issue
Block a user