feat(budgets): per-task and per-project cost budgets (flag-gated) (#654)

* fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation

The sweep re-escalated every expired unacked ack-required notification
on every ~60s tick, forever — the live incident: 3 fresh blocker
escalations + Telegram DMs per minute from a static stale pile. Now
each notification carries reescalation_count / last_reescalated_at /
reescalation_delivered_count (migration 079): first fire at expiry,
then doubling intervals from 1h capped at 24h, hard stop after
ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent
log carrying attempts-vs-delivered so 'seen and ignored' is
distinguishable from 'route never worked'. The due/wait/capped decision
is a pure function in foundation/policy/communications.py.

Per adversarial review, the attempt slot is claimed by compare-and-set
(UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the
previous draft leaned on the 60s dedup window, which never engages for
BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent
sweeps would have double-delivered. A lost claim skips delivery
outright. Legacy rows read as count=0 and keep today's first-fire
semantics. 61 tests incl. a two-session CAS race and a real alembic
upgrade/downgrade round trip.

* feat(budgets): per-task and per-project cost budgets (flag-gated)

tasks.budget_usd + projects.monthly_budget_usd (migration 080, chained
on 079; adds ix_agent_spawn_sessions_task_id since both enforcement
seams filter on bare task_id). Behind ROBOCO_TASK_BUDGETS_ENABLED
(default off, feature-flags card) — verifiably inert when off.

Claim-time: a project-month-spend guard applies to WORK-STARTING claims
only (i_will_work_on / i_will_plan) — per adversarial review, review/
doc/gate/inbound-PR claims are exempt so in-flight work can always
finish reviewing and merging at cap. Spend counts closed sessions'
estimated_cost_usd PLUS open sessions priced live from token snapshots
(the original closed-only sum read parallel long sessions as $0).

Sweep-side: the existing budget sweep also prices the active task's
spend vs budget_usd (TaskType defaults when null); on breach the task
is BLOCKED (HUMAN resolver, budget marker) BEFORE the graceful stop so
the unclaim no-ops and the dispatcher never respawns onto it, and the
CEO notification names both recovery steps. unblock on a budget-blocked
task re-checks live spend and refuses while still over — no silent
re-breach loop. Panel: budget inputs in both dialogs (0 rejected — a
zero budget silently blocks everything), spend logic consolidated in
TaskService.task_spend_usd. 42 new tests incl. a real-DB spend-query
suite and a two-tick non-refire sweep test.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-07-23 00:06:14 +02:00
committed by GitHub
co-authored by Renn F
parent 1d5a8e846f
commit 7c8453e210
31 changed files with 2288 additions and 65 deletions
@@ -0,0 +1,112 @@
"""Per-project monthly-budget claim guard (ROBOCO_TASK_BUDGETS_ENABLED).
Two layers: the pure predicate ``project_budget_exceeded_guard`` (over/under/
null cap) and the Choreographer's ``_project_budget_claim_guard`` wiring
(flag-off inert, no-cap inert, spend query only reached when both are set).
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
from roboco.services.gateway.claim_guards import project_budget_exceeded_guard
# ---------------------------------------------------------------------------
# Pure predicate: project_budget_exceeded_guard
# ---------------------------------------------------------------------------
def test_predicate_null_cap_is_inert() -> None:
task = MagicMock(id=uuid4())
assert project_budget_exceeded_guard(task, None, 999.0) is None
def test_predicate_under_cap_passes() -> None:
task = MagicMock(id=uuid4())
assert project_budget_exceeded_guard(task, 10.0, 5.0) is None
def test_predicate_over_cap_refuses() -> None:
task = MagicMock(id=uuid4())
env = project_budget_exceeded_guard(task, 10.0, 15.0)
assert env is not None
body = env.as_dict()
assert body["error"] == "invalid_state"
assert "10.00" in body["message"]
assert "15.00" in body["message"]
assert "project settings" in body["remediate"]
def test_predicate_at_cap_refuses() -> None:
"""At-or-over refuses — the cap itself is not headroom."""
task = MagicMock(id=uuid4())
assert project_budget_exceeded_guard(task, 10.0, 10.0) is not None
# ---------------------------------------------------------------------------
# Choreographer wiring: _project_budget_claim_guard
# ---------------------------------------------------------------------------
def _make_choreographer(*, project_month_spend_usd: float = 0.0) -> Choreographer:
task_svc = AsyncMock()
task_svc.project_month_spend_usd.return_value = project_month_spend_usd
base = {
"task": task_svc,
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
return Choreographer(ChoreographerDeps(**base))
def _task_with_project(monthly_budget_usd: float | None) -> MagicMock:
project = MagicMock(id=uuid4(), monthly_budget_usd=monthly_budget_usd)
return MagicMock(id=uuid4(), project=project)
@pytest.mark.asyncio
async def test_flag_off_is_inert_even_over_cap(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", False)
c = _make_choreographer(project_month_spend_usd=999.0)
task = _task_with_project(monthly_budget_usd=10.0)
assert await c._project_budget_claim_guard(task) is None
c.task.project_month_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_no_project_cap_is_inert(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", True)
c = _make_choreographer(project_month_spend_usd=999.0)
task = _task_with_project(monthly_budget_usd=None)
assert await c._project_budget_claim_guard(task) is None
c.task.project_month_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_under_cap_passes(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", True)
c = _make_choreographer(project_month_spend_usd=4.0)
task = _task_with_project(monthly_budget_usd=10.0)
assert await c._project_budget_claim_guard(task) is None
@pytest.mark.asyncio
async def test_over_cap_refuses_naming_the_field(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", True)
c = _make_choreographer(project_month_spend_usd=12.0)
task = _task_with_project(monthly_budget_usd=10.0)
env = await c._project_budget_claim_guard(task)
assert env is not None
body = env.as_dict()
assert body["error"] == "invalid_state"
assert "Monthly Budget" in body["remediate"]
@@ -0,0 +1,335 @@
"""The project monthly-budget claim guard is scoped to work-STARTING claims.
A project at (or over) its monthly cap still has non-negotiable in-flight
work to finish: QA's ``claim_review``, the PR gate's ``claim_gate_review``,
``claim_doc_task``, and inbound ``claim_pr_review`` never even reach the
spend query (``check_project_budget`` defaults False at those four call
sites) — reviewing/documenting/merging what's already been paid for must
never wedge behind an exhausted cap. Only ``i_will_work_on`` / ``i_will_plan``
(``check_project_budget=True``) — genuinely starting new spend — refuse.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
_STEPS = [
{
"title": "Implement the change",
"description": (
"edit the target file, add tests, run them, and stage the "
"change for commit on the task branch"
),
}
]
@pytest.fixture(autouse=True)
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", True)
def _over_cap_project() -> MagicMock:
"""A project with a $10 cap — the spend stub below always reports $999."""
return MagicMock(id=uuid4(), monthly_budget_usd=10.0)
def _make_deps(task_svc: AsyncMock, **overrides: Any) -> ChoreographerDeps:
base: dict[str, Any] = {
"task": task_svc,
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
base.update(overrides)
return ChoreographerDeps(**base)
# ---------------------------------------------------------------------------
# Review-ish claims: guard skipped — each succeeds despite an over-cap project.
# project_month_spend_usd is asserted NOT awaited: the guard is skipped
# entirely, not merely lucky (e.g. a cache hit).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_claim_review_succeeds_at_cap() -> None:
task_svc = AsyncMock()
t = MagicMock(
id=uuid4(),
status="awaiting_qa",
assigned_to=uuid4(),
parent_task_id=uuid4(),
task_type="code",
dependency_ids=[],
team="backend",
pr_number=10,
pr_url="https://example/pr/10",
branch_name="feature/backend/abc",
batch_id=None,
project=_over_cap_project(),
)
task_svc.get.return_value = t
task_svc.agent_for.return_value = MagicMock(role="qa", slug="be-qa")
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.has_earlier_incomplete_code_sibling.return_value = False
task_svc.qa_claim = AsyncMock(return_value=t)
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
c = Choreographer(_make_deps(task_svc))
cc: Any = c
cc._build_qa_review_evidence = AsyncMock(return_value={})
env = await c.claim_review(uuid4(), t.id)
body = env.as_dict()
assert body.get("error") is None, body
task_svc.project_month_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_claim_gate_review_succeeds_at_cap() -> None:
task_svc = AsyncMock()
t = MagicMock(
id=uuid4(),
status="awaiting_pr_review",
assigned_to=uuid4(),
parent_task_id=uuid4(),
task_type="planning",
dependency_ids=[],
team="main_pm",
pr_number=139,
pr_url="https://example/pr/139",
branch_name="feature/main_pm/root",
batch_id=None,
project=_over_cap_project(),
)
task_svc.get.return_value = t
task_svc.agent_for.return_value = MagicMock(
role="pr_reviewer", slug="be-pr-reviewer"
)
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.pr_gate_claim = AsyncMock(return_value=t)
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
c = Choreographer(_make_deps(task_svc))
cc: Any = c
cc._build_gate_review_evidence = AsyncMock(return_value={"pr_number": 139})
env = await c.claim_gate_review(uuid4(), t.id)
body = env.as_dict()
assert body.get("error") is None, body
task_svc.project_month_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_claim_doc_task_succeeds_at_cap() -> None:
task_svc = AsyncMock()
t = MagicMock(
id=uuid4(),
status="awaiting_documentation",
assigned_to=None,
parent_task_id=None,
task_type="documentation",
team="backend",
branch_name="feature/backend/abc",
quick_context=None,
documents=[],
commits=[{"sha": "abc123", "message": "[x] work"}],
pr_number=7,
pr_url="https://github.com/x/y/pull/7",
dev_notes="done",
acceptance_criteria_status=[],
work_session_id=uuid4(),
dependency_ids=[],
batch_id=None,
project=_over_cap_project(),
)
task_svc.get.return_value = t
task_svc.agent_for.return_value = MagicMock(role="documenter", team="backend")
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.doc_claim.return_value = t
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
git_svc = AsyncMock()
git_svc.diff.return_value = "diff"
git_svc.list_changed_files.return_value = ["README.md"]
c = Choreographer(_make_deps(task_svc, git=git_svc))
env = await c.claim_doc_task(uuid4(), t.id)
body = env.as_dict()
assert body["error"] is None, body
task_svc.project_month_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_claim_pr_review_succeeds_at_cap() -> None:
task_svc = AsyncMock()
t = MagicMock(
id=uuid4(),
status="pending",
assigned_to=None,
parent_task_id=None,
task_type="code",
dependency_ids=[],
team="system",
batch_id=None,
project=_over_cap_project(),
)
task_svc.get.return_value = t
task_svc.agent_for.return_value = MagicMock(
role="pr_reviewer", slug="be-pr-reviewer"
)
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.has_earlier_incomplete_code_sibling.return_value = False
task_svc.pr_review_claim = AsyncMock(return_value=t)
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
c = Choreographer(_make_deps(task_svc))
cc: Any = c
cc._build_pr_review_evidence = AsyncMock(return_value={})
env = await c.claim_pr_review(uuid4(), t.id)
body = env.as_dict()
assert body.get("error") is None, body
task_svc.project_month_spend_usd.assert_not_awaited()
# ---------------------------------------------------------------------------
# Work-starting claims: guard ON — both refuse at cap.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_will_work_on_refuses_at_cap() -> None:
agent_id = uuid4()
task_id = uuid4()
target = MagicMock(
id=task_id,
status="pending",
plan=None,
assigned_to=None,
parent_task_id=None,
sequence=0,
task_type="code",
team="backend",
dependency_ids=[],
project=_over_cap_project(),
)
task_svc = AsyncMock()
task_svc.get.return_value = target
task_svc.agent_for.return_value = MagicMock(
id=agent_id, role="developer", team="backend", slug=None
)
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
c = Choreographer(_make_deps(task_svc))
env = await c.i_will_work_on(agent_id, task_id, plan="x", steps=_STEPS)
body = env.as_dict()
assert body["error"] == "invalid_state", body
assert "10.00" in body["message"] and "999.00" in body["message"]
task_svc.claim.assert_not_awaited()
task_svc.project_month_spend_usd.assert_awaited_once()
@pytest.mark.asyncio
async def test_i_will_plan_refuses_at_cap() -> None:
pm_id = uuid4()
task_id = uuid4()
target = MagicMock(
id=task_id,
status="pending",
plan=None,
assigned_to=None,
parent_task_id=None,
sequence=0,
task_type="planning",
team="backend",
dependency_ids=[],
project=_over_cap_project(),
)
task_svc = AsyncMock()
task_svc.get.return_value = target
task_svc.agent_for.return_value = MagicMock(
id=pm_id, role="cell_pm", team="backend", slug=None
)
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
c = Choreographer(_make_deps(task_svc))
env = await c.i_will_plan(
pm_id,
task_id,
plan="break it down",
rich_plan={
"approach": (
"Single-cell decomposition: backend handles the full scope; "
"no frontend or ux work required for this planning task. "
"be-dev-1 owns the change end to end; QA reviews after the "
"PR opens, documentation follows, then be-pm completes and "
"submits up. Strict sequencing, no cross-cell dependencies."
),
"sub_tasks": [
{
"title": "Backend planning slice",
"description": (
"scope the change, assign be-dev-1, who implements "
"with tests and opens the leaf PR for QA review."
),
}
],
},
)
body = env.as_dict()
assert body["error"] == "invalid_state", body
task_svc.claim.assert_not_awaited()
task_svc.project_month_spend_usd.assert_awaited_once()
# ---------------------------------------------------------------------------
# Plumbing contract on _run_claim_guards itself.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_run_claim_guards_only_checks_budget_when_asked() -> None:
task_svc = AsyncMock()
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
c = Choreographer(_make_deps(task_svc))
task = MagicMock(id=uuid4(), dependency_ids=[])
calls = {"n": 0}
async def _fake_guard(_t: Any) -> None:
calls["n"] += 1
cc: Any = c
cc._project_budget_claim_guard = _fake_guard
await c._run_claim_guards(agent_id=uuid4(), task=task, skip_dev_guards=True)
assert calls["n"] == 0, (
"budget guard must not run without check_project_budget=True"
)
await c._run_claim_guards(
agent_id=uuid4(), task=task, skip_dev_guards=True, check_project_budget=True
)
assert calls["n"] == 1
@@ -0,0 +1,165 @@
"""unblock's budget-breach re-check (ROBOCO_TASK_BUDGETS_ENABLED).
A task the orchestrator's budget sweep BLOCKed carries the BUDGET_BLOCKED
marker (`_handle_task_budget_breach`). `unblock` re-checks spend-vs-cap
before letting it through: still over refuses (naming the budget
remediation), so a PM can't silently re-breach the same cap the next tick;
under (the CEO raised the cap) clears the marker and the unblock proceeds
exactly as before.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.foundation.policy.content import markers
from roboco.models.base import TaskType
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
def _make_deps(task_svc: AsyncMock) -> ChoreographerDeps:
base: dict[str, Any] = {
"task": task_svc,
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
base["journal"].has_decision_for_task.return_value = True
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
return ChoreographerDeps(**base)
def _budget_blocked_task(*, budget_usd: float | None = 5.0) -> MagicMock:
t = MagicMock(
id=uuid4(),
status="blocked",
pre_block_state="in_progress",
pre_block_assignee=uuid4(),
pre_block_metadata={},
dependency_ids=[],
task_type=TaskType.CODE,
budget_usd=budget_usd,
orchestration_markers=None,
)
markers.mark_budget_blocked(t)
return t
@pytest.fixture(autouse=True)
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "task_budgets_enabled", True)
@pytest.mark.asyncio
async def test_unblock_refuses_while_still_over_cap() -> None:
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.task_spend_usd.return_value = 7.0
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "attempting to resume")
body = env.as_dict()
assert body["error"] == "invalid_state", body
assert "5.00" in body["message"] and "7.00" in body["message"]
assert "budget" in body["remediate"].lower()
task_svc.unblock_with_restore.assert_not_awaited()
# Marker survives — a retry that hasn't actually cleared the cap must
# still be caught.
assert markers.is_budget_blocked(t) is True
@pytest.mark.asyncio
async def test_unblock_succeeds_when_already_under_cap() -> None:
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.task_spend_usd.return_value = 3.0
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "spend never actually breached")
body = env.as_dict()
assert body.get("error") is None, body
task_svc.unblock_with_restore.assert_awaited_once()
assert markers.is_budget_blocked(t) is False
@pytest.mark.asyncio
async def test_raise_then_unblock_succeeds_after_a_prior_refusal() -> None:
"""First attempt: still over -> refused. CEO raises budget_usd. Second
attempt: now under -> succeeds, marker cleared."""
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.task_spend_usd.return_value = 7.0
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
pm_id = uuid4()
refused = await c.unblock(pm_id, t.id, "attempting resume")
assert refused.as_dict()["error"] == "invalid_state"
task_svc.unblock_with_restore.assert_not_awaited()
# CEO raises the cap; re-block for a fresh attempt (the same task row, as
# it would be across two real requests).
t.budget_usd = 20.0
t.status = "blocked"
succeeded = await c.unblock(pm_id, t.id, "raised the budget, resuming")
body = succeeded.as_dict()
assert body.get("error") is None, body
task_svc.unblock_with_restore.assert_awaited_once()
assert markers.is_budget_blocked(t) is False
@pytest.mark.asyncio
async def test_flag_off_ignores_the_marker(monkeypatch: pytest.MonkeyPatch) -> None:
"""The marker alone must never gate anything with the flag off."""
monkeypatch.setattr(settings, "task_budgets_enabled", False)
t = _budget_blocked_task(budget_usd=5.0)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "flag is off")
assert env.as_dict().get("error") is None, env.as_dict()
task_svc.task_spend_usd.assert_not_awaited()
@pytest.mark.asyncio
async def test_non_budget_block_never_reaches_the_spend_query() -> None:
"""A task blocked for an ordinary reason (no BUDGET_BLOCKED marker) skips
the guard entirely — it's not a budget block at all."""
t = MagicMock(
id=uuid4(),
status="blocked",
pre_block_state="in_progress",
pre_block_assignee=uuid4(),
pre_block_metadata={},
dependency_ids=[],
task_type=TaskType.CODE,
budget_usd=5.0,
orchestration_markers=None,
)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.unmet_dependency_ids.return_value = []
task_svc.unblock_with_restore.return_value = t
c = Choreographer(_make_deps(task_svc))
env = await c.unblock(uuid4(), t.id, "manual escalation resolved")
assert env.as_dict().get("error") is None, env.as_dict()
task_svc.task_spend_usd.assert_not_awaited()