mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
* fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation The sweep re-escalated every expired unacked ack-required notification on every ~60s tick, forever — the live incident: 3 fresh blocker escalations + Telegram DMs per minute from a static stale pile. Now each notification carries reescalation_count / last_reescalated_at / reescalation_delivered_count (migration 079): first fire at expiry, then doubling intervals from 1h capped at 24h, hard stop after ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent log carrying attempts-vs-delivered so 'seen and ignored' is distinguishable from 'route never worked'. The due/wait/capped decision is a pure function in foundation/policy/communications.py. Per adversarial review, the attempt slot is claimed by compare-and-set (UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the previous draft leaned on the 60s dedup window, which never engages for BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent sweeps would have double-delivered. A lost claim skips delivery outright. Legacy rows read as count=0 and keep today's first-fire semantics. 61 tests incl. a two-session CAS race and a real alembic upgrade/downgrade round trip. * feat(budgets): per-task and per-project cost budgets (flag-gated) tasks.budget_usd + projects.monthly_budget_usd (migration 080, chained on 079; adds ix_agent_spawn_sessions_task_id since both enforcement seams filter on bare task_id). Behind ROBOCO_TASK_BUDGETS_ENABLED (default off, feature-flags card) — verifiably inert when off. Claim-time: a project-month-spend guard applies to WORK-STARTING claims only (i_will_work_on / i_will_plan) — per adversarial review, review/ doc/gate/inbound-PR claims are exempt so in-flight work can always finish reviewing and merging at cap. Spend counts closed sessions' estimated_cost_usd PLUS open sessions priced live from token snapshots (the original closed-only sum read parallel long sessions as $0). Sweep-side: the existing budget sweep also prices the active task's spend vs budget_usd (TaskType defaults when null); on breach the task is BLOCKED (HUMAN resolver, budget marker) BEFORE the graceful stop so the unclaim no-ops and the dispatcher never respawns onto it, and the CEO notification names both recovery steps. unblock on a budget-blocked task re-checks live spend and refuses while still over — no silent re-breach loop. Panel: budget inputs in both dialogs (0 rejected — a zero budget silently blocks everything), spend logic consolidated in TaskService.task_spend_usd. 42 new tests incl. a real-DB spend-query suite and a two-tick non-refire sweep test. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
138 lines
5.0 KiB
Python
138 lines
5.0 KiB
Python
"""Concurrency-invariant claim-time predicates.
|
|
|
|
These guards run BEFORE any task-status mutation in the claim verbs
|
|
(``i_will_work_on``, ``i_will_plan``, ``claim_review``, ``claim_doc_task``).
|
|
Each predicate returns a rejection ``Envelope`` if it fires; ``None`` if it
|
|
passes. The first non-None return short-circuits the claim.
|
|
|
|
Scope: only system-level concurrency invariants the lifecycle spec does
|
|
NOT model live here. Role/state/task_type checks (the former
|
|
``role_typed_claim_guard`` and ``pm_cannot_execute_code_guard``) now route
|
|
through ``spec.can_invoke_action``'s CLAIM_RULES + ``ActionSpec
|
|
.allowed_task_types`` and have been deleted.
|
|
|
|
Pre-gateway location at commit 0c3d15a:
|
|
roboco/mcp/tasks/handlers/_helpers.py:124-204
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from roboco.services.gateway.envelope import Envelope
|
|
|
|
if TYPE_CHECKING:
|
|
from uuid import UUID
|
|
|
|
# Statuses that count as "still actively worked".
|
|
# ``blocked`` is included: it is still owned by the dev and resumes to
|
|
# ``in_progress``; excluding it let a dev hold two in_progress tasks at once.
|
|
_ACTIVE_BLOCKING_STATUSES: frozenset[str] = frozenset(
|
|
{"claimed", "in_progress", "verifying", "blocked"}
|
|
)
|
|
|
|
|
|
def already_active_guard(
|
|
in_progress_tasks: list[Any], target_task_id: UUID
|
|
) -> Envelope | None:
|
|
"""Refuse claim if agent has any in_progress task other than this one.
|
|
|
|
Pre-gateway: _helpers.py:check_blocking_tasks 134-152.
|
|
"""
|
|
blocking = [
|
|
t
|
|
for t in in_progress_tasks
|
|
if str(t.status) in _ACTIVE_BLOCKING_STATUSES and t.id != target_task_id
|
|
]
|
|
if not blocking:
|
|
return None
|
|
blocker = blocking[0]
|
|
return Envelope.invalid_state(
|
|
message=(
|
|
f"You have a {blocker.status} task ({blocker.id}); "
|
|
"finish or pause it before claiming new work."
|
|
),
|
|
remediate=(
|
|
f"finish or pause {blocker.id} first via i_am_done(...) or i_am_idle()"
|
|
),
|
|
)
|
|
|
|
|
|
def paused_tasks_guard(
|
|
paused_tasks: list[Any], target_task_id: UUID | None = None
|
|
) -> Envelope | None:
|
|
"""Refuse claim if agent has a paused task OTHER than the one being claimed.
|
|
|
|
``target_task_id`` is excluded so a re-entry on the agent's own paused task
|
|
(e.g. a PM re-planning an umbrella that ``i_am_idle`` auto-paused) is never
|
|
self-blocked — mirroring ``already_active_guard``'s target exclusion.
|
|
|
|
Pre-gateway: _helpers.py:check_paused_tasks 154-165.
|
|
"""
|
|
blocking = [t for t in paused_tasks if t.id != target_task_id]
|
|
if not blocking:
|
|
return None
|
|
paused = blocking[0]
|
|
return Envelope.invalid_state(
|
|
message=(
|
|
f"You have {len(blocking)} paused task(s); resume before claiming new work."
|
|
),
|
|
remediate=(
|
|
f"resume {paused.id} (call i_will_work_on again) before starting new work"
|
|
),
|
|
)
|
|
|
|
|
|
def project_budget_exceeded_guard(
|
|
target_task: Any, monthly_budget_usd: float | None, month_spend_usd: float
|
|
) -> Envelope | None:
|
|
"""Refuse claim once the task's project has spent its monthly cap.
|
|
|
|
Only fires when ``monthly_budget_usd`` is set (``None`` = no cap, the
|
|
guard is inert) — the caller resolves both the project's cap and this
|
|
calendar month's summed agent-spawn spend (a DB read) so this predicate
|
|
stays pure, mirroring ``unmet_dependency_guard``. At-or-over the cap
|
|
refuses; strictly under it passes.
|
|
"""
|
|
if monthly_budget_usd is None or month_spend_usd < monthly_budget_usd:
|
|
return None
|
|
return Envelope.invalid_state(
|
|
message=(
|
|
f"task {target_task.id}'s project has reached its monthly budget "
|
|
f"(${monthly_budget_usd:,.2f} cap, ${month_spend_usd:,.2f} spent "
|
|
"this calendar month)."
|
|
),
|
|
remediate=(
|
|
"wait for next calendar month, or raise the project's Monthly "
|
|
"Budget (USD) field in project settings"
|
|
),
|
|
)
|
|
|
|
|
|
def unmet_dependency_guard(
|
|
target_task: Any, unmet_dependency_ids: list[UUID]
|
|
) -> Envelope | None:
|
|
"""Refuse claim while the task has non-terminal dependencies.
|
|
|
|
A task may not be claimed until every task it ``depends_on`` reaches a
|
|
terminal state (completed/cancelled). This holds the pre-assigned dev
|
|
that arrives via the claim verb directly — the dependency filter on the
|
|
unassigned claim pool (``list_pending(filter_by_dependencies=True)``)
|
|
never sees a pre-assigned task. ``unmet_dependency_ids`` is resolved by
|
|
the caller (it requires a DB read) so this predicate stays pure.
|
|
"""
|
|
if not unmet_dependency_ids:
|
|
return None
|
|
blockers = ", ".join(str(dep_id) for dep_id in unmet_dependency_ids)
|
|
return Envelope.invalid_state(
|
|
message=(
|
|
f"task {target_task.id} depends on unfinished work; "
|
|
f"{len(unmet_dependency_ids)} dependency(ies) not yet "
|
|
"completed/cancelled."
|
|
),
|
|
remediate=(
|
|
f"wait for dependency task(s) {blockers} to reach "
|
|
"completed/cancelled before claiming this task"
|
|
),
|
|
)
|