mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
Renn F
e6c9dde2a9
Claim-shaped verbs were failing 7/7 (claim_review) and 6/6 (claim_doc_task) as silent 120s FlowVerbTimeout 504s on the NAS: the per-claim ownership repair walked the whole clone issuing two stat syscalls per entry (chown_ms 39502 vs git_ms 8 in the live log), several passes stacked per claim, and the claim transaction held the task row the whole time — so concurrent writers queued behind it into the 60s lock_timeout. The walk now does one stat per entry shared by the chown-skip and chmod-skip checks, and a .git/roboco-owned sentinel (worktree-aware via _resolve_clone_root, written only after a zero-failure pass) skips the walk entirely when the tree is already agent-owned. Every root-side git write invalidates the sentinel BEFORE its subprocess runs — GitService._run_git for scope != none, plus the three raw-subprocess paths inside WorkspaceService the adversarial pass proved bypass it deterministically on the common respawn shape (_worktree_git for mutating verbs, _fetch_branch_ref, _fetch_origin_best_effort) — so a live marker can never vouch for files a root write is about to create. One of those queued writers was the PM journal-decision auto-record: its INSERT hit the lock timeout, _ensure_pm_decision's catch-all swallowed it without rollback, and the poisoned session blew up escalate_up with PendingRollbackError (live incident). The helper's try body now runs in a savepoint — one fix covering all seven PM verbs that route through it — verified empirically against real Postgres in both directions: the failure path leaves the session healthy and the task object readable, and create_entry's internal commit inside the savepoint drains the transactional outbox exactly once.
175 lines
6.3 KiB
Python
175 lines
6.3 KiB
Python
"""unblock's budget-breach re-check (ROBOCO_TASK_BUDGETS_ENABLED).
|
|
|
|
A task the orchestrator's budget sweep BLOCKed carries the BUDGET_BLOCKED
|
|
marker (`_handle_task_budget_breach`). `unblock` re-checks spend-vs-cap
|
|
before letting it through: still over refuses (naming the budget
|
|
remediation), so a PM can't silently re-breach the same cap the next tick;
|
|
under (the CEO raised the cap) clears the marker and the unblock proceeds
|
|
exactly as before.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.config import settings
|
|
from roboco.foundation.policy.content import markers
|
|
from roboco.models.base import TaskType
|
|
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
|
|
|
|
|
def _make_deps(task_svc: AsyncMock) -> ChoreographerDeps:
|
|
base: dict[str, Any] = {
|
|
"task": task_svc,
|
|
"work_session": AsyncMock(),
|
|
"git": AsyncMock(),
|
|
"a2a": AsyncMock(),
|
|
"journal": AsyncMock(),
|
|
"audit": AsyncMock(),
|
|
"evidence_repo": AsyncMock(),
|
|
}
|
|
base["journal"].has_decision_for_task.return_value = True
|
|
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
|
# _ensure_pm_decision's journal write is savepoint-guarded — an
|
|
# unconfigured AsyncMock's begin_nested() call returns a raw unawaited
|
|
# coroutine, which `async with` cannot use.
|
|
base["task"].session.begin_nested = MagicMock(
|
|
return_value=MagicMock(
|
|
__aenter__=AsyncMock(return_value=None),
|
|
__aexit__=AsyncMock(return_value=False),
|
|
)
|
|
)
|
|
return ChoreographerDeps(**base)
|
|
|
|
|
|
def _budget_blocked_task(*, budget_usd: float | None = 5.0) -> MagicMock:
|
|
t = MagicMock(
|
|
id=uuid4(),
|
|
status="blocked",
|
|
pre_block_state="in_progress",
|
|
pre_block_assignee=uuid4(),
|
|
pre_block_metadata={},
|
|
dependency_ids=[],
|
|
task_type=TaskType.CODE,
|
|
budget_usd=budget_usd,
|
|
orchestration_markers=None,
|
|
)
|
|
markers.mark_budget_blocked(t)
|
|
return t
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _budgets_on(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unblock_refuses_while_still_over_cap() -> None:
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.task_spend_usd.return_value = 7.0
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "attempting to resume")
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state", body
|
|
assert "5.00" in body["message"] and "7.00" in body["message"]
|
|
assert "budget" in body["remediate"].lower()
|
|
task_svc.unblock_with_restore.assert_not_awaited()
|
|
# Marker survives — a retry that hasn't actually cleared the cap must
|
|
# still be caught.
|
|
assert markers.is_budget_blocked(t) is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unblock_succeeds_when_already_under_cap() -> None:
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.task_spend_usd.return_value = 3.0
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "spend never actually breached")
|
|
body = env.as_dict()
|
|
assert body.get("error") is None, body
|
|
task_svc.unblock_with_restore.assert_awaited_once()
|
|
assert markers.is_budget_blocked(t) is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_raise_then_unblock_succeeds_after_a_prior_refusal() -> None:
|
|
"""First attempt: still over -> refused. CEO raises budget_usd. Second
|
|
attempt: now under -> succeeds, marker cleared."""
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.task_spend_usd.return_value = 7.0
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
pm_id = uuid4()
|
|
|
|
refused = await c.unblock(pm_id, t.id, "attempting resume")
|
|
assert refused.as_dict()["error"] == "invalid_state"
|
|
task_svc.unblock_with_restore.assert_not_awaited()
|
|
|
|
# CEO raises the cap; re-block for a fresh attempt (the same task row, as
|
|
# it would be across two real requests).
|
|
t.budget_usd = 20.0
|
|
t.status = "blocked"
|
|
succeeded = await c.unblock(pm_id, t.id, "raised the budget, resuming")
|
|
body = succeeded.as_dict()
|
|
assert body.get("error") is None, body
|
|
task_svc.unblock_with_restore.assert_awaited_once()
|
|
assert markers.is_budget_blocked(t) is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flag_off_ignores_the_marker(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The marker alone must never gate anything with the flag off."""
|
|
monkeypatch.setattr(settings, "task_budgets_enabled", False)
|
|
t = _budget_blocked_task(budget_usd=5.0)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "flag is off")
|
|
assert env.as_dict().get("error") is None, env.as_dict()
|
|
task_svc.task_spend_usd.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_budget_block_never_reaches_the_spend_query() -> None:
|
|
"""A task blocked for an ordinary reason (no BUDGET_BLOCKED marker) skips
|
|
the guard entirely — it's not a budget block at all."""
|
|
t = MagicMock(
|
|
id=uuid4(),
|
|
status="blocked",
|
|
pre_block_state="in_progress",
|
|
pre_block_assignee=uuid4(),
|
|
pre_block_metadata={},
|
|
dependency_ids=[],
|
|
task_type=TaskType.CODE,
|
|
budget_usd=5.0,
|
|
orchestration_markers=None,
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.unmet_dependency_ids.return_value = []
|
|
task_svc.unblock_with_restore.return_value = t
|
|
c = Choreographer(_make_deps(task_svc))
|
|
|
|
env = await c.unblock(uuid4(), t.id, "manual escalation resolved")
|
|
assert env.as_dict().get("error") is None, env.as_dict()
|
|
task_svc.task_spend_usd.assert_not_awaited()
|