mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(budgets): per-task and per-project cost budgets (flag-gated) (#654)
* fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation The sweep re-escalated every expired unacked ack-required notification on every ~60s tick, forever — the live incident: 3 fresh blocker escalations + Telegram DMs per minute from a static stale pile. Now each notification carries reescalation_count / last_reescalated_at / reescalation_delivered_count (migration 079): first fire at expiry, then doubling intervals from 1h capped at 24h, hard stop after ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent log carrying attempts-vs-delivered so 'seen and ignored' is distinguishable from 'route never worked'. The due/wait/capped decision is a pure function in foundation/policy/communications.py. Per adversarial review, the attempt slot is claimed by compare-and-set (UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the previous draft leaned on the 60s dedup window, which never engages for BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent sweeps would have double-delivered. A lost claim skips delivery outright. Legacy rows read as count=0 and keep today's first-fire semantics. 61 tests incl. a two-session CAS race and a real alembic upgrade/downgrade round trip. * feat(budgets): per-task and per-project cost budgets (flag-gated) tasks.budget_usd + projects.monthly_budget_usd (migration 080, chained on 079; adds ix_agent_spawn_sessions_task_id since both enforcement seams filter on bare task_id). Behind ROBOCO_TASK_BUDGETS_ENABLED (default off, feature-flags card) — verifiably inert when off. Claim-time: a project-month-spend guard applies to WORK-STARTING claims only (i_will_work_on / i_will_plan) — per adversarial review, review/ doc/gate/inbound-PR claims are exempt so in-flight work can always finish reviewing and merging at cap. Spend counts closed sessions' estimated_cost_usd PLUS open sessions priced live from token snapshots (the original closed-only sum read parallel long sessions as $0). Sweep-side: the existing budget sweep also prices the active task's spend vs budget_usd (TaskType defaults when null); on breach the task is BLOCKED (HUMAN resolver, budget marker) BEFORE the graceful stop so the unclaim no-ops and the dispatcher never respawns onto it, and the CEO notification names both recovery steps. unblock on a budget-blocked task re-checks live spend and refuses while still over — no silent re-breach loop. Panel: budget inputs in both dialogs (0 rejected — a zero budget silently blocks everything), spend logic consolidated in TaskService.task_spend_usd. 42 new tests incl. a real-DB spend-query suite and a two-tick non-refire sweep test. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
"""Per-project monthly-budget claim guard (ROBOCO_TASK_BUDGETS_ENABLED).
|
||||
|
||||
Two layers: the pure predicate ``project_budget_exceeded_guard`` (over/under/
|
||||
null cap) and the Choreographer's ``_project_budget_claim_guard`` wiring
|
||||
(flag-off inert, no-cap inert, spend query only reached when both are set).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
from roboco.services.gateway.claim_guards import project_budget_exceeded_guard
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pure predicate: project_budget_exceeded_guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_predicate_null_cap_is_inert() -> None:
|
||||
task = MagicMock(id=uuid4())
|
||||
assert project_budget_exceeded_guard(task, None, 999.0) is None
|
||||
|
||||
|
||||
def test_predicate_under_cap_passes() -> None:
|
||||
task = MagicMock(id=uuid4())
|
||||
assert project_budget_exceeded_guard(task, 10.0, 5.0) is None
|
||||
|
||||
|
||||
def test_predicate_over_cap_refuses() -> None:
|
||||
task = MagicMock(id=uuid4())
|
||||
env = project_budget_exceeded_guard(task, 10.0, 15.0)
|
||||
assert env is not None
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state"
|
||||
assert "10.00" in body["message"]
|
||||
assert "15.00" in body["message"]
|
||||
assert "project settings" in body["remediate"]
|
||||
|
||||
|
||||
def test_predicate_at_cap_refuses() -> None:
|
||||
"""At-or-over refuses — the cap itself is not headroom."""
|
||||
task = MagicMock(id=uuid4())
|
||||
assert project_budget_exceeded_guard(task, 10.0, 10.0) is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Choreographer wiring: _project_budget_claim_guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_choreographer(*, project_month_spend_usd: float = 0.0) -> Choreographer:
|
||||
task_svc = AsyncMock()
|
||||
task_svc.project_month_spend_usd.return_value = project_month_spend_usd
|
||||
base = {
|
||||
"task": task_svc,
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
return Choreographer(ChoreographerDeps(**base))
|
||||
|
||||
|
||||
def _task_with_project(monthly_budget_usd: float | None) -> MagicMock:
|
||||
project = MagicMock(id=uuid4(), monthly_budget_usd=monthly_budget_usd)
|
||||
return MagicMock(id=uuid4(), project=project)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flag_off_is_inert_even_over_cap(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", False)
|
||||
c = _make_choreographer(project_month_spend_usd=999.0)
|
||||
task = _task_with_project(monthly_budget_usd=10.0)
|
||||
assert await c._project_budget_claim_guard(task) is None
|
||||
c.task.project_month_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_project_cap_is_inert(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
||||
c = _make_choreographer(project_month_spend_usd=999.0)
|
||||
task = _task_with_project(monthly_budget_usd=None)
|
||||
assert await c._project_budget_claim_guard(task) is None
|
||||
c.task.project_month_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_under_cap_passes(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
||||
c = _make_choreographer(project_month_spend_usd=4.0)
|
||||
task = _task_with_project(monthly_budget_usd=10.0)
|
||||
assert await c._project_budget_claim_guard(task) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_over_cap_refuses_naming_the_field(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
||||
c = _make_choreographer(project_month_spend_usd=12.0)
|
||||
task = _task_with_project(monthly_budget_usd=10.0)
|
||||
env = await c._project_budget_claim_guard(task)
|
||||
assert env is not None
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state"
|
||||
assert "Monthly Budget" in body["remediate"]
|
||||
@@ -0,0 +1,335 @@
|
||||
"""The project monthly-budget claim guard is scoped to work-STARTING claims.
|
||||
|
||||
A project at (or over) its monthly cap still has non-negotiable in-flight
|
||||
work to finish: QA's ``claim_review``, the PR gate's ``claim_gate_review``,
|
||||
``claim_doc_task``, and inbound ``claim_pr_review`` never even reach the
|
||||
spend query (``check_project_budget`` defaults False at those four call
|
||||
sites) — reviewing/documenting/merging what's already been paid for must
|
||||
never wedge behind an exhausted cap. Only ``i_will_work_on`` / ``i_will_plan``
|
||||
(``check_project_budget=True``) — genuinely starting new spend — refuse.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
|
||||
_STEPS = [
|
||||
{
|
||||
"title": "Implement the change",
|
||||
"description": (
|
||||
"edit the target file, add tests, run them, and stage the "
|
||||
"change for commit on the task branch"
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
||||
|
||||
|
||||
def _over_cap_project() -> MagicMock:
|
||||
"""A project with a $10 cap — the spend stub below always reports $999."""
|
||||
return MagicMock(id=uuid4(), monthly_budget_usd=10.0)
|
||||
|
||||
|
||||
def _make_deps(task_svc: AsyncMock, **overrides: Any) -> ChoreographerDeps:
|
||||
base: dict[str, Any] = {
|
||||
"task": task_svc,
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
return ChoreographerDeps(**base)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Review-ish claims: guard skipped — each succeeds despite an over-cap project.
|
||||
# project_month_spend_usd is asserted NOT awaited: the guard is skipped
|
||||
# entirely, not merely lucky (e.g. a cache hit).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_review_succeeds_at_cap() -> None:
|
||||
task_svc = AsyncMock()
|
||||
t = MagicMock(
|
||||
id=uuid4(),
|
||||
status="awaiting_qa",
|
||||
assigned_to=uuid4(),
|
||||
parent_task_id=uuid4(),
|
||||
task_type="code",
|
||||
dependency_ids=[],
|
||||
team="backend",
|
||||
pr_number=10,
|
||||
pr_url="https://example/pr/10",
|
||||
branch_name="feature/backend/abc",
|
||||
batch_id=None,
|
||||
project=_over_cap_project(),
|
||||
)
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(role="qa", slug="be-qa")
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.has_earlier_incomplete_code_sibling.return_value = False
|
||||
task_svc.qa_claim = AsyncMock(return_value=t)
|
||||
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
cc: Any = c
|
||||
cc._build_qa_review_evidence = AsyncMock(return_value={})
|
||||
|
||||
env = await c.claim_review(uuid4(), t.id)
|
||||
body = env.as_dict()
|
||||
assert body.get("error") is None, body
|
||||
task_svc.project_month_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_gate_review_succeeds_at_cap() -> None:
|
||||
task_svc = AsyncMock()
|
||||
t = MagicMock(
|
||||
id=uuid4(),
|
||||
status="awaiting_pr_review",
|
||||
assigned_to=uuid4(),
|
||||
parent_task_id=uuid4(),
|
||||
task_type="planning",
|
||||
dependency_ids=[],
|
||||
team="main_pm",
|
||||
pr_number=139,
|
||||
pr_url="https://example/pr/139",
|
||||
branch_name="feature/main_pm/root",
|
||||
batch_id=None,
|
||||
project=_over_cap_project(),
|
||||
)
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
role="pr_reviewer", slug="be-pr-reviewer"
|
||||
)
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.pr_gate_claim = AsyncMock(return_value=t)
|
||||
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
cc: Any = c
|
||||
cc._build_gate_review_evidence = AsyncMock(return_value={"pr_number": 139})
|
||||
|
||||
env = await c.claim_gate_review(uuid4(), t.id)
|
||||
body = env.as_dict()
|
||||
assert body.get("error") is None, body
|
||||
task_svc.project_month_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_doc_task_succeeds_at_cap() -> None:
|
||||
task_svc = AsyncMock()
|
||||
t = MagicMock(
|
||||
id=uuid4(),
|
||||
status="awaiting_documentation",
|
||||
assigned_to=None,
|
||||
parent_task_id=None,
|
||||
task_type="documentation",
|
||||
team="backend",
|
||||
branch_name="feature/backend/abc",
|
||||
quick_context=None,
|
||||
documents=[],
|
||||
commits=[{"sha": "abc123", "message": "[x] work"}],
|
||||
pr_number=7,
|
||||
pr_url="https://github.com/x/y/pull/7",
|
||||
dev_notes="done",
|
||||
acceptance_criteria_status=[],
|
||||
work_session_id=uuid4(),
|
||||
dependency_ids=[],
|
||||
batch_id=None,
|
||||
project=_over_cap_project(),
|
||||
)
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(role="documenter", team="backend")
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.doc_claim.return_value = t
|
||||
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c = Choreographer(_make_deps(task_svc, git=git_svc))
|
||||
|
||||
env = await c.claim_doc_task(uuid4(), t.id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
task_svc.project_month_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_pr_review_succeeds_at_cap() -> None:
|
||||
task_svc = AsyncMock()
|
||||
t = MagicMock(
|
||||
id=uuid4(),
|
||||
status="pending",
|
||||
assigned_to=None,
|
||||
parent_task_id=None,
|
||||
task_type="code",
|
||||
dependency_ids=[],
|
||||
team="system",
|
||||
batch_id=None,
|
||||
project=_over_cap_project(),
|
||||
)
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
role="pr_reviewer", slug="be-pr-reviewer"
|
||||
)
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.has_earlier_incomplete_code_sibling.return_value = False
|
||||
task_svc.pr_review_claim = AsyncMock(return_value=t)
|
||||
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
cc: Any = c
|
||||
cc._build_pr_review_evidence = AsyncMock(return_value={})
|
||||
|
||||
env = await c.claim_pr_review(uuid4(), t.id)
|
||||
body = env.as_dict()
|
||||
assert body.get("error") is None, body
|
||||
task_svc.project_month_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Work-starting claims: guard ON — both refuse at cap.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_i_will_work_on_refuses_at_cap() -> None:
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
target = MagicMock(
|
||||
id=task_id,
|
||||
status="pending",
|
||||
plan=None,
|
||||
assigned_to=None,
|
||||
parent_task_id=None,
|
||||
sequence=0,
|
||||
task_type="code",
|
||||
team="backend",
|
||||
dependency_ids=[],
|
||||
project=_over_cap_project(),
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = target
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
id=agent_id, role="developer", team="backend", slug=None
|
||||
)
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
|
||||
env = await c.i_will_work_on(agent_id, task_id, plan="x", steps=_STEPS)
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state", body
|
||||
assert "10.00" in body["message"] and "999.00" in body["message"]
|
||||
task_svc.claim.assert_not_awaited()
|
||||
task_svc.project_month_spend_usd.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_i_will_plan_refuses_at_cap() -> None:
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
target = MagicMock(
|
||||
id=task_id,
|
||||
status="pending",
|
||||
plan=None,
|
||||
assigned_to=None,
|
||||
parent_task_id=None,
|
||||
sequence=0,
|
||||
task_type="planning",
|
||||
team="backend",
|
||||
dependency_ids=[],
|
||||
project=_over_cap_project(),
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = target
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
id=pm_id, role="cell_pm", team="backend", slug=None
|
||||
)
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.project_month_spend_usd = AsyncMock(return_value=999.0)
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
|
||||
env = await c.i_will_plan(
|
||||
pm_id,
|
||||
task_id,
|
||||
plan="break it down",
|
||||
rich_plan={
|
||||
"approach": (
|
||||
"Single-cell decomposition: backend handles the full scope; "
|
||||
"no frontend or ux work required for this planning task. "
|
||||
"be-dev-1 owns the change end to end; QA reviews after the "
|
||||
"PR opens, documentation follows, then be-pm completes and "
|
||||
"submits up. Strict sequencing, no cross-cell dependencies."
|
||||
),
|
||||
"sub_tasks": [
|
||||
{
|
||||
"title": "Backend planning slice",
|
||||
"description": (
|
||||
"scope the change, assign be-dev-1, who implements "
|
||||
"with tests and opens the leaf PR for QA review."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state", body
|
||||
task_svc.claim.assert_not_awaited()
|
||||
task_svc.project_month_spend_usd.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plumbing contract on _run_claim_guards itself.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_claim_guards_only_checks_budget_when_asked() -> None:
|
||||
task_svc = AsyncMock()
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
task = MagicMock(id=uuid4(), dependency_ids=[])
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _fake_guard(_t: Any) -> None:
|
||||
calls["n"] += 1
|
||||
|
||||
cc: Any = c
|
||||
cc._project_budget_claim_guard = _fake_guard
|
||||
|
||||
await c._run_claim_guards(agent_id=uuid4(), task=task, skip_dev_guards=True)
|
||||
assert calls["n"] == 0, (
|
||||
"budget guard must not run without check_project_budget=True"
|
||||
)
|
||||
|
||||
await c._run_claim_guards(
|
||||
agent_id=uuid4(), task=task, skip_dev_guards=True, check_project_budget=True
|
||||
)
|
||||
assert calls["n"] == 1
|
||||
@@ -0,0 +1,165 @@
|
||||
"""unblock's budget-breach re-check (ROBOCO_TASK_BUDGETS_ENABLED).
|
||||
|
||||
A task the orchestrator's budget sweep BLOCKed carries the BUDGET_BLOCKED
|
||||
marker (`_handle_task_budget_breach`). `unblock` re-checks spend-vs-cap
|
||||
before letting it through: still over refuses (naming the budget
|
||||
remediation), so a PM can't silently re-breach the same cap the next tick;
|
||||
under (the CEO raised the cap) clears the marker and the unblock proceeds
|
||||
exactly as before.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings
|
||||
from roboco.foundation.policy.content import markers
|
||||
from roboco.models.base import TaskType
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
|
||||
|
||||
def _make_deps(task_svc: AsyncMock) -> ChoreographerDeps:
|
||||
base: dict[str, Any] = {
|
||||
"task": task_svc,
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base["journal"].has_decision_for_task.return_value = True
|
||||
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
||||
return ChoreographerDeps(**base)
|
||||
|
||||
|
||||
def _budget_blocked_task(*, budget_usd: float | None = 5.0) -> MagicMock:
|
||||
t = MagicMock(
|
||||
id=uuid4(),
|
||||
status="blocked",
|
||||
pre_block_state="in_progress",
|
||||
pre_block_assignee=uuid4(),
|
||||
pre_block_metadata={},
|
||||
dependency_ids=[],
|
||||
task_type=TaskType.CODE,
|
||||
budget_usd=budget_usd,
|
||||
orchestration_markers=None,
|
||||
)
|
||||
markers.mark_budget_blocked(t)
|
||||
return t
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unblock_refuses_while_still_over_cap() -> None:
|
||||
t = _budget_blocked_task(budget_usd=5.0)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.unmet_dependency_ids.return_value = []
|
||||
task_svc.task_spend_usd.return_value = 7.0
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
|
||||
env = await c.unblock(uuid4(), t.id, "attempting to resume")
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state", body
|
||||
assert "5.00" in body["message"] and "7.00" in body["message"]
|
||||
assert "budget" in body["remediate"].lower()
|
||||
task_svc.unblock_with_restore.assert_not_awaited()
|
||||
# Marker survives — a retry that hasn't actually cleared the cap must
|
||||
# still be caught.
|
||||
assert markers.is_budget_blocked(t) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unblock_succeeds_when_already_under_cap() -> None:
|
||||
t = _budget_blocked_task(budget_usd=5.0)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.unmet_dependency_ids.return_value = []
|
||||
task_svc.task_spend_usd.return_value = 3.0
|
||||
task_svc.unblock_with_restore.return_value = t
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
|
||||
env = await c.unblock(uuid4(), t.id, "spend never actually breached")
|
||||
body = env.as_dict()
|
||||
assert body.get("error") is None, body
|
||||
task_svc.unblock_with_restore.assert_awaited_once()
|
||||
assert markers.is_budget_blocked(t) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_then_unblock_succeeds_after_a_prior_refusal() -> None:
|
||||
"""First attempt: still over -> refused. CEO raises budget_usd. Second
|
||||
attempt: now under -> succeeds, marker cleared."""
|
||||
t = _budget_blocked_task(budget_usd=5.0)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.unmet_dependency_ids.return_value = []
|
||||
task_svc.task_spend_usd.return_value = 7.0
|
||||
task_svc.unblock_with_restore.return_value = t
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
pm_id = uuid4()
|
||||
|
||||
refused = await c.unblock(pm_id, t.id, "attempting resume")
|
||||
assert refused.as_dict()["error"] == "invalid_state"
|
||||
task_svc.unblock_with_restore.assert_not_awaited()
|
||||
|
||||
# CEO raises the cap; re-block for a fresh attempt (the same task row, as
|
||||
# it would be across two real requests).
|
||||
t.budget_usd = 20.0
|
||||
t.status = "blocked"
|
||||
succeeded = await c.unblock(pm_id, t.id, "raised the budget, resuming")
|
||||
body = succeeded.as_dict()
|
||||
assert body.get("error") is None, body
|
||||
task_svc.unblock_with_restore.assert_awaited_once()
|
||||
assert markers.is_budget_blocked(t) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flag_off_ignores_the_marker(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The marker alone must never gate anything with the flag off."""
|
||||
monkeypatch.setattr(settings, "task_budgets_enabled", False)
|
||||
t = _budget_blocked_task(budget_usd=5.0)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.unmet_dependency_ids.return_value = []
|
||||
task_svc.unblock_with_restore.return_value = t
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
|
||||
env = await c.unblock(uuid4(), t.id, "flag is off")
|
||||
assert env.as_dict().get("error") is None, env.as_dict()
|
||||
task_svc.task_spend_usd.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_budget_block_never_reaches_the_spend_query() -> None:
|
||||
"""A task blocked for an ordinary reason (no BUDGET_BLOCKED marker) skips
|
||||
the guard entirely — it's not a budget block at all."""
|
||||
t = MagicMock(
|
||||
id=uuid4(),
|
||||
status="blocked",
|
||||
pre_block_state="in_progress",
|
||||
pre_block_assignee=uuid4(),
|
||||
pre_block_metadata={},
|
||||
dependency_ids=[],
|
||||
task_type=TaskType.CODE,
|
||||
budget_usd=5.0,
|
||||
orchestration_markers=None,
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.unmet_dependency_ids.return_value = []
|
||||
task_svc.unblock_with_restore.return_value = t
|
||||
c = Choreographer(_make_deps(task_svc))
|
||||
|
||||
env = await c.unblock(uuid4(), t.id, "manual escalation resolved")
|
||||
assert env.as_dict().get("error") is None, env.as_dict()
|
||||
task_svc.task_spend_usd.assert_not_awaited()
|
||||
Reference in New Issue
Block a user