mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
* fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation The sweep re-escalated every expired unacked ack-required notification on every ~60s tick, forever — the live incident: 3 fresh blocker escalations + Telegram DMs per minute from a static stale pile. Now each notification carries reescalation_count / last_reescalated_at / reescalation_delivered_count (migration 079): first fire at expiry, then doubling intervals from 1h capped at 24h, hard stop after ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent log carrying attempts-vs-delivered so 'seen and ignored' is distinguishable from 'route never worked'. The due/wait/capped decision is a pure function in foundation/policy/communications.py. Per adversarial review, the attempt slot is claimed by compare-and-set (UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the previous draft leaned on the 60s dedup window, which never engages for BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent sweeps would have double-delivered. A lost claim skips delivery outright. Legacy rows read as count=0 and keep today's first-fire semantics. 61 tests incl. a two-session CAS race and a real alembic upgrade/downgrade round trip. * feat(budgets): per-task and per-project cost budgets (flag-gated) tasks.budget_usd + projects.monthly_budget_usd (migration 080, chained on 079; adds ix_agent_spawn_sessions_task_id since both enforcement seams filter on bare task_id). Behind ROBOCO_TASK_BUDGETS_ENABLED (default off, feature-flags card) — verifiably inert when off. Claim-time: a project-month-spend guard applies to WORK-STARTING claims only (i_will_work_on / i_will_plan) — per adversarial review, review/ doc/gate/inbound-PR claims are exempt so in-flight work can always finish reviewing and merging at cap. Spend counts closed sessions' estimated_cost_usd PLUS open sessions priced live from token snapshots (the original closed-only sum read parallel long sessions as $0). Sweep-side: the existing budget sweep also prices the active task's spend vs budget_usd (TaskType defaults when null); on breach the task is BLOCKED (HUMAN resolver, budget marker) BEFORE the graceful stop so the unclaim no-ops and the dispatcher never respawns onto it, and the CEO notification names both recovery steps. unblock on a budget-blocked task re-checks live spend and refuses while still over — no silent re-breach loop. Panel: budget inputs in both dialogs (0 rejected — a zero budget silently blocks everything), spend logic consolidated in TaskService.task_spend_usd. 42 new tests incl. a real-DB spend-query suite and a two-tick non-refire sweep test. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
166 lines
5.9 KiB
Python
166 lines
5.9 KiB
Python
"""unblock's budget-breach re-check (ROBOCO_TASK_BUDGETS_ENABLED).
|
|
|
|
A task the orchestrator's budget sweep BLOCKed carries the BUDGET_BLOCKED
|
|
marker (`_handle_task_budget_breach`). `unblock` re-checks spend-vs-cap
|
|
before letting it through: still over refuses (naming the budget
|
|
remediation), so a PM can't silently re-breach the same cap the next tick;
|
|
under (the CEO raised the cap) clears the marker and the unblock proceeds
|
|
exactly as before.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.config import settings
|
|
from roboco.foundation.policy.content import markers
|
|
from roboco.models.base import TaskType
|
|
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
|
|
|
|
|
def _make_deps(task_svc: AsyncMock) -> ChoreographerDeps:
|
|
base: dict[str, Any] = {
|
|
"task": task_svc,
|
|
"work_session": AsyncMock(),
|
|
"git": AsyncMock(),
|
|
"a2a": AsyncMock(),
|
|
"journal": AsyncMock(),
|
|
"audit": AsyncMock(),
|
|
"evidence_repo": AsyncMock(),
|
|
}
|
|
base["journal"].has_decision_for_task.return_value = True
|
|
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
|
return ChoreographerDeps(**base)
|
|
|
|
|
|
def _budget_blocked_task(*, budget_usd: float | None = 5.0) -> MagicMock:
|
|
t = MagicMock(
|
|
id=uuid4(),
|
|
status="blocked",
|
|
pre_block_state="in_progress",
|
|
pre_block_assignee=uuid4(),
|
|
pre_block_metadata={},
|
|
dependency_ids=[],
|
|
task_type=TaskType.CODE,
|
|
budget_usd=budget_usd,
|
|
orchestration_markers=None,
|
|
)
|
|
markers.mark_budget_blocked(t)
|
|
return t
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unblock_refuses_while_still_over_cap() -> None:
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.task_spend_usd.return_value = 7.0
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "attempting to resume")
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state", body
|
|
assert "5.00" in body["message"] and "7.00" in body["message"]
|
|
assert "budget" in body["remediate"].lower()
|
|
task_svc.unblock_with_restore.assert_not_awaited()
|
|
# Marker survives — a retry that hasn't actually cleared the cap must
|
|
# still be caught.
|
|
assert markers.is_budget_blocked(t) is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unblock_succeeds_when_already_under_cap() -> None:
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.task_spend_usd.return_value = 3.0
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "spend never actually breached")
|
|
body = env.as_dict()
|
|
assert body.get("error") is None, body
|
|
task_svc.unblock_with_restore.assert_awaited_once()
|
|
assert markers.is_budget_blocked(t) is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_raise_then_unblock_succeeds_after_a_prior_refusal() -> None:
|
|
"""First attempt: still over -> refused. CEO raises budget_usd. Second
|
|
attempt: now under -> succeeds, marker cleared."""
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.task_spend_usd.return_value = 7.0
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
pm_id = uuid4()
|
|
|
|
refused = await c.unblock(pm_id, t.id, "attempting resume")
|
|
assert refused.as_dict()["error"] == "invalid_state"
|
|
task_svc.unblock_with_restore.assert_not_awaited()
|
|
|
|
# CEO raises the cap; re-block for a fresh attempt (the same task row, as
|
|
# it would be across two real requests).
|
|
t.budget_usd = 20.0
|
|
t.status = "blocked"
|
|
succeeded = await c.unblock(pm_id, t.id, "raised the budget, resuming")
|
|
body = succeeded.as_dict()
|
|
assert body.get("error") is None, body
|
|
task_svc.unblock_with_restore.assert_awaited_once()
|
|
assert markers.is_budget_blocked(t) is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flag_off_ignores_the_marker(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The marker alone must never gate anything with the flag off."""
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", False)
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "flag is off")
|
|
assert env.as_dict().get("error") is None, env.as_dict()
|
|
task_svc.task_spend_usd.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_budget_block_never_reaches_the_spend_query() -> None:
|
|
"""A task blocked for an ordinary reason (no BUDGET_BLOCKED marker) skips
|
|
the guard entirely — it's not a budget block at all."""
|
|
t = MagicMock(
|
|
id=uuid4(),
|
|
status="blocked",
|
|
pre_block_state="in_progress",
|
|
pre_block_assignee=uuid4(),
|
|
pre_block_metadata={},
|
|
dependency_ids=[],
|
|
task_type=TaskType.CODE,
|
|
budget_usd=5.0,
|
|
orchestration_markers=None,
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "manual escalation resolved")
|
|
assert env.as_dict().get("error") is None, env.as_dict()
|
|
task_svc.task_spend_usd.assert_not_awaited()
|