mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
7c8453e210
* fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation The sweep re-escalated every expired unacked ack-required notification on every ~60s tick, forever — the live incident: 3 fresh blocker escalations + Telegram DMs per minute from a static stale pile. Now each notification carries reescalation_count / last_reescalated_at / reescalation_delivered_count (migration 079): first fire at expiry, then doubling intervals from 1h capped at 24h, hard stop after ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent log carrying attempts-vs-delivered so 'seen and ignored' is distinguishable from 'route never worked'. The due/wait/capped decision is a pure function in foundation/policy/communications.py. Per adversarial review, the attempt slot is claimed by compare-and-set (UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the previous draft leaned on the 60s dedup window, which never engages for BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent sweeps would have double-delivered. A lost claim skips delivery outright. Legacy rows read as count=0 and keep today's first-fire semantics. 61 tests incl. a two-session CAS race and a real alembic upgrade/downgrade round trip. * feat(budgets): per-task and per-project cost budgets (flag-gated) tasks.budget_usd + projects.monthly_budget_usd (migration 080, chained on 079; adds ix_agent_spawn_sessions_task_id since both enforcement seams filter on bare task_id). Behind ROBOCO_TASK_BUDGETS_ENABLED (default off, feature-flags card) — verifiably inert when off. Claim-time: a project-month-spend guard applies to WORK-STARTING claims only (i_will_work_on / i_will_plan) — per adversarial review, review/ doc/gate/inbound-PR claims are exempt so in-flight work can always finish reviewing and merging at cap. Spend counts closed sessions' estimated_cost_usd PLUS open sessions priced live from token snapshots (the original closed-only sum read parallel long sessions as $0). Sweep-side: the existing budget sweep also prices the active task's spend vs budget_usd (TaskType defaults when null); on breach the task is BLOCKED (HUMAN resolver, budget marker) BEFORE the graceful stop so the unclaim no-ops and the dispatcher never respawns onto it, and the CEO notification names both recovery steps. unblock on a budget-blocked task re-checks live spend and refuses while still over — no silent re-breach loop. Panel: budget inputs in both dialogs (0 rejected — a zero budget silently blocks everything), spend logic consolidated in TaskService.task_spend_usd. 42 new tests incl. a real-DB spend-query suite and a two-tick non-refire sweep test. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
113 lines
4.1 KiB
Python
113 lines
4.1 KiB
Python
"""Per-project monthly-budget claim guard (ROBOCO_TASK_BUDGETS_ENABLED).
|
|
|
|
Two layers: the pure predicate ``project_budget_exceeded_guard`` (over/under/
|
|
null cap) and the Choreographer's ``_project_budget_claim_guard`` wiring
|
|
(flag-off inert, no-cap inert, spend query only reached when both are set).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.config import settings
|
|
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
|
from roboco.services.gateway.claim_guards import project_budget_exceeded_guard
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pure predicate: project_budget_exceeded_guard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_predicate_null_cap_is_inert() -> None:
|
|
task = MagicMock(id=uuid4())
|
|
assert project_budget_exceeded_guard(task, None, 999.0) is None
|
|
|
|
|
|
def test_predicate_under_cap_passes() -> None:
|
|
task = MagicMock(id=uuid4())
|
|
assert project_budget_exceeded_guard(task, 10.0, 5.0) is None
|
|
|
|
|
|
def test_predicate_over_cap_refuses() -> None:
|
|
task = MagicMock(id=uuid4())
|
|
env = project_budget_exceeded_guard(task, 10.0, 15.0)
|
|
assert env is not None
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state"
|
|
assert "10.00" in body["message"]
|
|
assert "15.00" in body["message"]
|
|
assert "project settings" in body["remediate"]
|
|
|
|
|
|
def test_predicate_at_cap_refuses() -> None:
|
|
"""At-or-over refuses — the cap itself is not headroom."""
|
|
task = MagicMock(id=uuid4())
|
|
assert project_budget_exceeded_guard(task, 10.0, 10.0) is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Choreographer wiring: _project_budget_claim_guard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_choreographer(*, project_month_spend_usd: float = 0.0) -> Choreographer:
|
|
task_svc = AsyncMock()
|
|
task_svc.project_month_spend_usd.return_value = project_month_spend_usd
|
|
base = {
|
|
"task": task_svc,
|
|
"work_session": AsyncMock(),
|
|
"git": AsyncMock(),
|
|
"a2a": AsyncMock(),
|
|
"journal": AsyncMock(),
|
|
"audit": AsyncMock(),
|
|
"evidence_repo": AsyncMock(),
|
|
}
|
|
return Choreographer(ChoreographerDeps(**base))
|
|
|
|
|
|
def _task_with_project(monthly_budget_usd: float | None) -> MagicMock:
|
|
project = MagicMock(id=uuid4(), monthly_budget_usd=monthly_budget_usd)
|
|
return MagicMock(id=uuid4(), project=project)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flag_off_is_inert_even_over_cap(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", False)
|
|
c = _make_choreographer(project_month_spend_usd=999.0)
|
|
task = _task_with_project(monthly_budget_usd=10.0)
|
|
assert await c._project_budget_claim_guard(task) is None
|
|
c.task.project_month_spend_usd.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_project_cap_is_inert(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
|
c = _make_choreographer(project_month_spend_usd=999.0)
|
|
task = _task_with_project(monthly_budget_usd=None)
|
|
assert await c._project_budget_claim_guard(task) is None
|
|
c.task.project_month_spend_usd.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_under_cap_passes(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
|
c = _make_choreographer(project_month_spend_usd=4.0)
|
|
task = _task_with_project(monthly_budget_usd=10.0)
|
|
assert await c._project_budget_claim_guard(task) is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_over_cap_refuses_naming_the_field(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
|
c = _make_choreographer(project_month_spend_usd=12.0)
|
|
task = _task_with_project(monthly_budget_usd=10.0)
|
|
env = await c._project_budget_claim_guard(task)
|
|
assert env is not None
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state"
|
|
assert "Monthly Budget" in body["remediate"]
|