mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(grok): cost-ceiling kill-switch (budget-guardrail parity)
Claude Code's per-agent token-budget hook fires against the SDK :9000 server; opencode exposes NO usage/budget hook to a plugin (confirmed against its plugin docs), so the budget kill-switch can't be a plugin/sidecar — the orchestrator enforces it instead. _enforce_grok_cost_budget runs each dispatch tick: for every ACTIVE GROK container it reads cumulative cost from the opencode store (the Phase-2 reader) and kills + evicts it past ROBOCO_GROK_MAX_COST_USD (0 = off), after which the reaper releases the freed task. This also catches a runaway loop that keeps firing verbs (so it evades the idle watchdog) but still burns cost. Covers the budget/runaway-burn slice of guardrail parity. The remaining Claude hooks (prompt-injection PRE-gate, stop-guard terminal-verb) have no blocking opencode equivalent — opencode's message/stop hooks are observe-only — and the interactive reasoning-variant has no opencode.json/serve knob (CLI-flag only); both are pinned for a live probe rather than shipped as a guess.
This commit is contained in:
@@ -678,6 +678,19 @@ class Settings(BaseSettings):
|
||||
"override via ROBOCO_GROK_IDLE_KILL_SECONDS"
|
||||
),
|
||||
)
|
||||
# Budget kill-switch parity for GROK. Claude Code's per-agent token-budget
|
||||
# hook fires against the SDK :9000 server; opencode exposes no usage hook to
|
||||
# a plugin, so the orchestrator enforces the cap by reading each live GROK
|
||||
# container's cumulative cost from its opencode store and killing it when it
|
||||
# crosses this ceiling (also catches runaway-loop token burn). USD; 0 = off.
|
||||
grok_max_cost_usd: float = Field(
|
||||
default=0.0,
|
||||
ge=0,
|
||||
description=(
|
||||
"Per-agent GROK cost ceiling (USD) before the container is killed; "
|
||||
"0 disables. Override via ROBOCO_GROK_MAX_COST_USD"
|
||||
),
|
||||
)
|
||||
# A task left CLAIMED/IN_PROGRESS with an assignee but no running container
|
||||
# (e.g. a reassignment that didn't spawn) is invisibly stuck — the heartbeat
|
||||
# reaper can't see it because its heartbeat was seeded fresh at claim time.
|
||||
|
||||
@@ -705,6 +705,10 @@ class AgentOrchestrator:
|
||||
# killed + evicted so the reaper can release its task; see
|
||||
# _maybe_kill_wedged_grok.
|
||||
self._grok_idle_kill_ttl: int = settings.grok_idle_kill_seconds
|
||||
# Cost ceiling (USD) before a live GROK container is killed — the budget
|
||||
# kill-switch parity (opencode exposes no usage hook). 0 disables. See
|
||||
# _enforce_grok_cost_budget.
|
||||
self._grok_max_cost_usd: float = settings.grok_max_cost_usd
|
||||
|
||||
# =========================================================================
|
||||
# LIFECYCLE
|
||||
@@ -3822,6 +3826,52 @@ class AgentOrchestrator:
|
||||
usage.tokens_cache_write,
|
||||
)
|
||||
|
||||
async def _enforce_grok_cost_budget(self) -> None:
|
||||
"""Kill a live GROK container whose cumulative opencode cost exceeds the cap.
|
||||
|
||||
opencode exposes no token/budget hook to a plugin, so the budget
|
||||
kill-switch (Claude Code parity for runaway token burn — a loop that
|
||||
keeps firing verbs evades the idle watchdog but still burns cost) lives
|
||||
here: read each ACTIVE GROK container's cumulative cost from its opencode
|
||||
store and kill + evict it past ``ROBOCO_GROK_MAX_COST_USD``. The reaper
|
||||
then releases the freed task. Disabled (no-op) when the cap is <= 0.
|
||||
"""
|
||||
cap = getattr(self, "_grok_max_cost_usd", 0.0)
|
||||
if cap <= 0:
|
||||
return
|
||||
from roboco.llm.providers.opencode_usage import cost_for_session
|
||||
from roboco.models.base import ModelProvider
|
||||
|
||||
for agent_id, instance in list(self._instances.items()):
|
||||
config = instance.config
|
||||
if (
|
||||
config is None
|
||||
or config.provider_type != ModelProvider.GROK.value
|
||||
or instance.state != AgentState.ACTIVE
|
||||
):
|
||||
continue
|
||||
_, cost = cost_for_session(
|
||||
config.model or "", self._opencode_db_path(agent_id)
|
||||
)
|
||||
if cost <= cap:
|
||||
continue
|
||||
try:
|
||||
await self._remove_container(f"roboco-agent-{agent_id}")
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"grok cost-cap kill failed; will retry next tick",
|
||||
agent_id=agent_id,
|
||||
error=str(exc),
|
||||
)
|
||||
continue
|
||||
self._instances.pop(agent_id, None)
|
||||
logger.warning(
|
||||
"grok container killed: cost ceiling exceeded",
|
||||
agent_id=agent_id,
|
||||
cost_usd=round(cost, 4),
|
||||
cap_usd=cap,
|
||||
)
|
||||
|
||||
async def _resolve_final_token_usage(
|
||||
self, agent_id: str
|
||||
) -> tuple[int, int, int, int]:
|
||||
@@ -6525,6 +6575,13 @@ Start now: evidence(task_id="{task_id}")
|
||||
except Exception as e:
|
||||
logger.error("Stale-claim reaper failed; continuing tick", error=str(e))
|
||||
|
||||
# Enforce the GROK cost ceiling (budget kill-switch parity). Wrapped so a
|
||||
# failure never blocks dispatch; the next tick retries.
|
||||
try:
|
||||
await self._enforce_grok_cost_budget()
|
||||
except Exception as e:
|
||||
logger.error("Grok cost-budget sweep failed; continuing tick", error=str(e))
|
||||
|
||||
# Orchestrator uses SYSTEM role for internal API calls
|
||||
# Using a well-known UUID for the orchestrator identity
|
||||
headers = {
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""GROK cost budget kill-switch: kill a live container over the cost ceiling.
|
||||
|
||||
opencode exposes no usage hook to a plugin, so the budget kill-switch lives in
|
||||
the orchestrator: it reads each live GROK container's cumulative opencode cost
|
||||
and kills + evicts it past ROBOCO_GROK_MAX_COST_USD (also catching runaway-loop
|
||||
token burn). The cost computation itself is covered in opencode_usage tests; here
|
||||
cost_for_session is stubbed so the kill DECISION is exercised deterministically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from roboco.models.runtime import AgentInstance
|
||||
from roboco.runtime.orchestrator import AgentOrchestrator, AgentState
|
||||
|
||||
_COST_FN = "roboco.llm.providers.opencode_usage.cost_for_session"
|
||||
|
||||
|
||||
def _grok_instance(provider_type: str = "grok") -> AgentInstance:
|
||||
cfg = type("C", (), {"provider_type": provider_type, "model": "grok-build-0.1"})()
|
||||
return AgentInstance(agent_id="be-dev-1", state=AgentState.ACTIVE, config=cfg)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cost_over_cap_kills_and_evicts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
orch._grok_max_cost_usd = 5.0
|
||||
orch._instances = {"be-dev-1": _grok_instance()}
|
||||
remove_mock = AsyncMock()
|
||||
monkeypatch.setattr(orch, "_remove_container", remove_mock)
|
||||
monkeypatch.setattr(_COST_FN, lambda *_a, **_k: (None, 7.5))
|
||||
|
||||
await orch._enforce_grok_cost_budget()
|
||||
|
||||
remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
|
||||
assert "be-dev-1" not in orch._instances
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cost_under_cap_spares(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
orch._grok_max_cost_usd = 5.0
|
||||
orch._instances = {"be-dev-1": _grok_instance()}
|
||||
remove_mock = AsyncMock()
|
||||
monkeypatch.setattr(orch, "_remove_container", remove_mock)
|
||||
monkeypatch.setattr(_COST_FN, lambda *_a, **_k: (None, 1.0))
|
||||
|
||||
await orch._enforce_grok_cost_budget()
|
||||
|
||||
remove_mock.assert_not_awaited()
|
||||
assert "be-dev-1" in orch._instances
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cap_zero_disables_the_sweep(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
orch._grok_max_cost_usd = 0.0
|
||||
orch._instances = {"be-dev-1": _grok_instance()}
|
||||
remove_mock = AsyncMock()
|
||||
monkeypatch.setattr(orch, "_remove_container", remove_mock)
|
||||
monkeypatch.setattr(_COST_FN, lambda *_a, **_k: (None, 999.0))
|
||||
|
||||
await orch._enforce_grok_cost_budget()
|
||||
|
||||
remove_mock.assert_not_awaited()
|
||||
assert "be-dev-1" in orch._instances
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_grok_container_is_ignored(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
orch = AgentOrchestrator.__new__(AgentOrchestrator)
|
||||
orch._grok_max_cost_usd = 5.0
|
||||
orch._instances = {"be-dev-1": _grok_instance(provider_type="anthropic")}
|
||||
remove_mock = AsyncMock()
|
||||
monkeypatch.setattr(orch, "_remove_container", remove_mock)
|
||||
monkeypatch.setattr(_COST_FN, lambda *_a, **_k: (None, 999.0))
|
||||
|
||||
await orch._enforce_grok_cost_budget()
|
||||
|
||||
remove_mock.assert_not_awaited()
|
||||
assert "be-dev-1" in orch._instances
|
||||
Reference in New Issue
Block a user