Files
roboco/tests/unit/gateway/test_budget_unblock_guard.py
T
7c8453e210 feat(budgets): per-task and per-project cost budgets (flag-gated) (#654)
* fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation

The sweep re-escalated every expired unacked ack-required notification
on every ~60s tick, forever — the live incident: 3 fresh blocker
escalations + Telegram DMs per minute from a static stale pile. Now
each notification carries reescalation_count / last_reescalated_at /
reescalation_delivered_count (migration 079): first fire at expiry,
then doubling intervals from 1h capped at 24h, hard stop after
ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent
log carrying attempts-vs-delivered so 'seen and ignored' is
distinguishable from 'route never worked'. The due/wait/capped decision
is a pure function in foundation/policy/communications.py.

Per adversarial review, the attempt slot is claimed by compare-and-set
(UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the
previous draft leaned on the 60s dedup window, which never engages for
BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent
sweeps would have double-delivered. A lost claim skips delivery
outright. Legacy rows read as count=0 and keep today's first-fire
semantics. 61 tests incl. a two-session CAS race and a real alembic
upgrade/downgrade round trip.

* feat(budgets): per-task and per-project cost budgets (flag-gated)

tasks.budget_usd + projects.monthly_budget_usd (migration 080, chained
on 079; adds ix_agent_spawn_sessions_task_id since both enforcement
seams filter on bare task_id). Behind ROBOCO_TASK_BUDGETS_ENABLED
(default off, feature-flags card) — verifiably inert when off.

Claim-time: a project-month-spend guard applies to WORK-STARTING claims
only (i_will_work_on / i_will_plan) — per adversarial review, review/
doc/gate/inbound-PR claims are exempt so in-flight work can always
finish reviewing and merging at cap. Spend counts closed sessions'
estimated_cost_usd PLUS open sessions priced live from token snapshots
(the original closed-only sum read parallel long sessions as $0).

Sweep-side: the existing budget sweep also prices the active task's
spend vs budget_usd (TaskType defaults when null); on breach the task
is BLOCKED (HUMAN resolver, budget marker) BEFORE the graceful stop so
the unclaim no-ops and the dispatcher never respawns onto it, and the
CEO notification names both recovery steps. unblock on a budget-blocked
task re-checks live spend and refuses while still over — no silent
re-breach loop. Panel: budget inputs in both dialogs (0 rejected — a
zero budget silently blocks everything), spend logic consolidated in
TaskService.task_spend_usd. 42 new tests incl. a real-DB spend-query
suite and a two-tick non-refire sweep test.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
2026-07-23 00:06:14 +02:00

166 lines
5.9 KiB
Python

"""unblock's budget-breach re-check (ROBOCO_TASK_BUDGETS_ENABLED).
A task the orchestrator's budget sweep BLOCKed carries the BUDGET_BLOCKED
marker (`_handle_task_budget_breach`). `unblock` re-checks spend-vs-cap
before letting it through: still over refuses (naming the budget
remediation), so a PM can't silently re-breach the same cap the next tick;
under (the CEO raised the cap) clears the marker and the unblock proceeds
exactly as before.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.foundation.policy.content import markers
from roboco.models.base import TaskType
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
def _make_deps(task_svc: AsyncMock) -> ChoreographerDeps:
base: dict[str, Any] = {
"task": task_svc,
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
base["journal"].has_decision_for_task.return_value = True
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
return ChoreographerDeps(**base)
def _budget_blocked_task(*, budget_usd: float | None = 5.0) -> MagicMock:
t = MagicMock(
id=uuid4(),
status="blocked",
pre_block_state="in_progress",
pre_block_assignee=uuid4(),
pre_block_metadata={},
dependency_ids=[],
task_type=TaskType.CODE,
budget_usd=budget_usd,
orchestration_markers=None,
)
markers.mark_budget_blocked(t)
return t
@pytest.fixture(autouse=True)
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", True)
@pytest.mark.asyncio
async def test_unblock_refuses_while_still_over_cap() -> None:
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.task_spend_usd.return_value = 7.0
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "attempting to resume")
body = env.as_dict()
assert body["error"] == "invalid_state", body
assert "5.00" in body["message"] and "7.00" in body["message"]
assert "budget" in body["remediate"].lower()
task_svc.unblock_with_restore.assert_not_awaited()
# Marker survives — a retry that hasn't actually cleared the cap must
# still be caught.
assert markers.is_budget_blocked(t) is True
@pytest.mark.asyncio
async def test_unblock_succeeds_when_already_under_cap() -> None:
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.task_spend_usd.return_value = 3.0
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "spend never actually breached")
body = env.as_dict()
assert body.get("error") is None, body
task_svc.unblock_with_restore.assert_awaited_once()
assert markers.is_budget_blocked(t) is False
@pytest.mark.asyncio
async def test_raise_then_unblock_succeeds_after_a_prior_refusal() -> None:
"""First attempt: still over -> refused. CEO raises budget_usd. Second
attempt: now under -> succeeds, marker cleared."""
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.task_spend_usd.return_value = 7.0
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
pm_id = uuid4()
refused = await c.unblock(pm_id, t.id, "attempting resume")
assert refused.as_dict()["error"] == "invalid_state"
task_svc.unblock_with_restore.assert_not_awaited()
# CEO raises the cap; re-block for a fresh attempt (the same task row, as
# it would be across two real requests).
t.budget_usd = 20.0
t.status = "blocked"
succeeded = await c.unblock(pm_id, t.id, "raised the budget, resuming")
body = succeeded.as_dict()
assert body.get("error") is None, body
task_svc.unblock_with_restore.assert_awaited_once()
assert markers.is_budget_blocked(t) is False
@pytest.mark.asyncio
async def test_flag_off_ignores_the_marker(monkeypatch: pytest.MonkeyPatch) -> None:
"""The marker alone must never gate anything with the flag off."""
monkeypatch.setattr(settings, "task_budgets_enabled", False)
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "flag is off")
assert env.as_dict().get("error") is None, env.as_dict()
task_svc.task_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_non_budget_block_never_reaches_the_spend_query() -> None:
"""A task blocked for an ordinary reason (no BUDGET_BLOCKED marker) skips
the guard entirely — it's not a budget block at all."""
t = MagicMock(
id=uuid4(),
status="blocked",
pre_block_state="in_progress",
pre_block_assignee=uuid4(),
pre_block_metadata={},
dependency_ids=[],
task_type=TaskType.CODE,
budget_usd=5.0,
orchestration_markers=None,
)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "manual escalation resolved")
assert env.as_dict().get("error") is None, env.as_dict()
task_svc.task_spend_usd.assert_not_awaited()