mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
* fix(run-hardening): stop three blocked-task respawn loops
Three independent fixes for blocked-task respawn loops observed in the live
run (the bleeders behind a wedged near-complete run):
- verb runner: re-check the working task after EACH composed atomic action,
not just at entry. A concurrent transition between a verb's precondition
gate and execution (e.g. a racing i_am_blocked moving a root from
needs_revision to blocked) made claim() return None mid-sequence; the next
composed step dereferenced None.id and crashed with the opaque
"'NoneType' object has no attribute 'id'", looping the PM. Now fails fast
with an actionable INVALID_STATE; the savepoint rolls the partial run back.
- blocker dispatch: never dispatch a Board role (product-owner / head-
marketing) as a blocker resolver. Board roles have no unblock verb, so the
dispatcher respawned one forever to "resolve" a blocker it could only
notify/triage about — one incident burned ~6400 tool calls on a single
mis-owned root. _blocker_resolver_slug now returns None for a Board
assignee so the dispatch skips it.
- git push: recover a missing local task-branch ref from origin/<branch>
before push-by-name. A re-provisioned shared clone can lack the branch
locally though its commits are on origin, so push died on
"src refspec <branch> does not match any" and the task wedged at i_am_done.
Now materializes the ref (no-op push when already on origin) or fails loud
with an unclaim+reclaim instruction when the work is on neither.
Adds regression tests for all three. Full no-DB gate green (ruff, reflow,
mypy, xenon); pytest+coverage validated by CI.
* fix(verb-runner): only raise on an INTERMEDIATE composed None, not the last
The mid-composition None-guard was too aggressive: it raised for a None
returned by the LAST composed action too (e.g. start()), preempting the
caller's existing `if task is None` handler that surfaces the verb-specific
message ("start failed for task ...", the board verb's decline envelope).
Three tests asserting those messages broke in CI.
Only an INTERMEDIATE None is fatal (the next action would deref None.id). A
None from the last action is the verb's own result and must flow out as the
runner's return value. Guard now fires only for position > 0, before the
next dispatch — still prevents the crash, preserves the last-action contract.
* fix(escalation): never hand a Main-PM coordination root to the Board
The upstream cause of the board catch-22 (which the orchestrator-side
blocker-dispatch guard only backstopped): the escalation chain points
main-pm -> product-owner, and i_am_blocked/escalate REASSIGNS the task to
that chain target. apply_escalation's board-advisory guard only refused
descendant cell tasks (both predicates require parent_task_id), so a
top-level Main-PM coordination root slipped through and the whole root was
reassigned to the Product Owner + marked blocked. The board has no unblock
verb, so it spam-notified the CEO and respawn-looped (~6400 tool calls on
one root).
Add _is_coordination_task (team == main_pm — covers a delivery root AND a
MegaTask root-subtask) and a shared _board_cannot_own predicate, applied at
all four board-refusal sites (escalation, reassign, reassign_active_claim,
dependency-revival). A main_pm coordination task escalated/reassigned onto a
board role is now diverted to the pool for a role-matched (Main-PM) reclaim.
Complements the blocker-dispatch backstop in the prior commits (defense in
depth). Tests: coordination-root predicate cases + apply_escalation divert;
existing teamless-root / board-root behavior unchanged.
---------
Co-authored-by: Renn F <rennf93@users.noreply.github.com>
263 lines
10 KiB
Python
263 lines
10 KiB
Python
"""Verb runner — wraps spec.composed_actions_for in a savepoint.
|
|
|
|
Atomicity invariant: preconditions checked BEFORE side effects.
|
|
A mid-sequence atomic-action failure rolls the DB back to the
|
|
pre-call state; git side effects are runs AFTER the savepoint
|
|
commits.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.foundation.policy import lifecycle as spec
|
|
from roboco.services.gateway.choreographer._verb_runner import (
|
|
VerbRunner,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_runs_composed_actions_in_order() -> None:
|
|
"""For i_will_work_on the runner runs claim, then set_plan, then start."""
|
|
task_svc = AsyncMock()
|
|
task_svc.session.begin_nested = MagicMock(
|
|
return_value=MagicMock(__aenter__=AsyncMock(), __aexit__=AsyncMock())
|
|
)
|
|
runner = VerbRunner(task_service=task_svc, git_service=AsyncMock())
|
|
|
|
calls: list[str] = []
|
|
|
|
def _record(name: str, status: str) -> Callable[..., MagicMock]:
|
|
def _inner(*_args: object, **_kwargs: object) -> MagicMock:
|
|
calls.append(name)
|
|
return MagicMock(status=status)
|
|
|
|
return _inner
|
|
|
|
task_svc.claim = AsyncMock(side_effect=_record("claim", "claimed"))
|
|
task_svc.set_plan = AsyncMock(side_effect=_record("set_plan", "claimed"))
|
|
task_svc.start = AsyncMock(side_effect=_record("start", "in_progress"))
|
|
|
|
task = MagicMock(id=uuid4(), status="pending", plan=None, commits=[])
|
|
agent = MagicMock(id=uuid4(), role="developer")
|
|
ctx = spec.Context(plan="my plan")
|
|
|
|
final_task = await runner.run_intent("i_will_work_on", task, agent, ctx)
|
|
assert calls == ["claim", "set_plan", "start"]
|
|
assert final_task.status == "in_progress"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_rejects_none_task_or_agent() -> None:
|
|
"""A None task/agent fails loud with a clean error, not a NoneType crash.
|
|
|
|
The atomic handlers dereference task.id / agent.id; without the guard a
|
|
missing one crashes with "'NoneType' object has no attribute 'id'" (observed
|
|
when a task was forced into an unexpected state out-of-band).
|
|
"""
|
|
runner = VerbRunner(task_service=AsyncMock(), git_service=AsyncMock())
|
|
ctx = spec.Context(plan="p")
|
|
agent = MagicMock(id=uuid4(), role="cell_pm")
|
|
task = MagicMock(id=uuid4(), status="in_progress")
|
|
|
|
with pytest.raises(ValueError, match="INVALID_STATE"):
|
|
await runner.run_intent("i_will_plan", None, agent, ctx)
|
|
with pytest.raises(ValueError, match="INVALID_STATE"):
|
|
await runner.run_intent("i_will_plan", task, None, ctx)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_rejects_none_returned_mid_composition() -> None:
|
|
"""A composed action returning None mid-sequence fails loud, not a crash.
|
|
|
|
Observed in prod: i_will_plan on a task a concurrent agent had just moved to
|
|
`blocked` — claim() returned None (no valid transition), then
|
|
_do_set_plan(None, ...) crashed with "'NoneType' object has no attribute
|
|
'id'". The choreographer surfaced it as a cryptic "verb runner failed" and
|
|
the PM respawn-looped. The entry guard only covers the INITIAL task, so the
|
|
loop body must re-check after each composed action.
|
|
"""
|
|
task_svc = AsyncMock()
|
|
# __aexit__ must return falsy so the savepoint context does not SUPPRESS the
|
|
# ValueError raised inside it (real SQLAlchemy begin_nested re-raises + rolls back).
|
|
task_svc.session.begin_nested = MagicMock(
|
|
return_value=MagicMock(
|
|
__aenter__=AsyncMock(), __aexit__=AsyncMock(return_value=False)
|
|
)
|
|
)
|
|
# claim() returns None — its source status was invalid (concurrent change).
|
|
task_svc.claim = AsyncMock(return_value=None)
|
|
task_svc.set_plan = AsyncMock()
|
|
task_svc.start = AsyncMock()
|
|
runner = VerbRunner(task_service=task_svc, git_service=AsyncMock())
|
|
|
|
task = MagicMock(id=uuid4(), status="needs_revision", plan="p", commits=[])
|
|
agent = MagicMock(id=uuid4(), role="main_pm")
|
|
ctx = spec.Context(plan="my plan")
|
|
|
|
with pytest.raises(ValueError, match="INVALID_STATE"):
|
|
await runner.run_intent("i_will_plan", task, agent, ctx)
|
|
# The downstream composed actions must NOT run on a None task.
|
|
task_svc.set_plan.assert_not_called()
|
|
task_svc.start.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_runs_side_effects_after_db_commit() -> None:
|
|
"""For open_pr: composes is empty; side_effects (push_branch, create_pr) run."""
|
|
task_svc = AsyncMock()
|
|
task_svc.session.begin_nested = MagicMock(
|
|
return_value=MagicMock(__aenter__=AsyncMock(), __aexit__=AsyncMock())
|
|
)
|
|
git_svc = AsyncMock()
|
|
git_svc.push_branch = AsyncMock()
|
|
git_svc.create_pr = AsyncMock(return_value={"pr_number": 42})
|
|
runner = VerbRunner(task_service=task_svc, git_service=git_svc)
|
|
|
|
task = MagicMock(
|
|
id=uuid4(),
|
|
status="in_progress",
|
|
commits=["abc"],
|
|
pr_number=None,
|
|
parent_task_id=None,
|
|
branch_name="feature/backend/ABC12345",
|
|
)
|
|
agent = MagicMock(id=uuid4(), role="developer")
|
|
ctx = spec.Context()
|
|
|
|
await runner.run_intent("open_pr", task, agent, ctx)
|
|
git_svc.push_branch.assert_awaited_once()
|
|
git_svc.create_pr.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submit_up_creates_pr_before_transition() -> None:
|
|
"""submit_up's create_pr (pre_side_effect) runs BEFORE the
|
|
submit_for_review transition.
|
|
|
|
submit_for_review needs the cell→root PR to already exist (the reviewer
|
|
reviews it); create_pr persists pr_number onto the task row. With a
|
|
composes→side_effects ordering the transition would run first and the
|
|
trailing create_pr would crash — so the pre-side-effect must run first.
|
|
"""
|
|
calls: list[str] = []
|
|
|
|
task_svc = AsyncMock()
|
|
task_svc.session.begin_nested = MagicMock(
|
|
return_value=MagicMock(__aenter__=AsyncMock(), __aexit__=AsyncMock())
|
|
)
|
|
|
|
def _submit_for_review(*_args: object, **_kwargs: object) -> MagicMock:
|
|
calls.append("submit_for_review")
|
|
return MagicMock(status="awaiting_pr_review")
|
|
|
|
task_svc.submit_for_review = AsyncMock(side_effect=_submit_for_review)
|
|
|
|
git_svc = AsyncMock()
|
|
|
|
def _create_pr(*_args: object, **_kwargs: object) -> dict[str, int]:
|
|
calls.append("create_pr")
|
|
return {"pr_number": 31}
|
|
|
|
git_svc.create_pr = AsyncMock(side_effect=_create_pr)
|
|
runner = VerbRunner(task_service=task_svc, git_service=git_svc)
|
|
|
|
task = MagicMock(
|
|
id=uuid4(),
|
|
status="in_progress",
|
|
parent_task_id=None,
|
|
branch_name="feature/backend/ABC12345--DEF67890",
|
|
)
|
|
agent = MagicMock(id=uuid4(), role="cell_pm")
|
|
ctx = spec.Context(notes="cell scope complete; bubbling up to main pm")
|
|
|
|
await runner.run_intent("submit_up", task, agent, ctx)
|
|
assert calls == ["create_pr", "submit_for_review"], (
|
|
f"create_pr must precede submit_for_review for submit_up; got {calls}"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_pr_base_is_parent_task_branch_across_team() -> None:
|
|
"""#181: the cell→root PR base is the PARENT task's actual branch_name,
|
|
not the team-preserving string derivation.
|
|
|
|
The cell branch is feature/backend/ROOT--CELL but the root branch is
|
|
feature/main_pm/ROOT (different team). parent_branch_for() would derive
|
|
feature/backend/ROOT — a ref that doesn't exist on the remote, which
|
|
GitHub rejects with base: invalid. The base must come from the parent
|
|
task's branch_name.
|
|
"""
|
|
task_svc = AsyncMock()
|
|
task_svc.session.begin_nested = MagicMock(
|
|
return_value=MagicMock(__aenter__=AsyncMock(), __aexit__=AsyncMock())
|
|
)
|
|
task_svc.submit_pm_review = AsyncMock(
|
|
return_value=MagicMock(status="awaiting_pm_review")
|
|
)
|
|
root_id = uuid4()
|
|
# Parent (root) task lives under a DIFFERENT team prefix.
|
|
task_svc.get = AsyncMock(
|
|
return_value=MagicMock(branch_name="feature/main_pm/ROOT0001")
|
|
)
|
|
|
|
git_svc = AsyncMock()
|
|
git_svc.create_pr = AsyncMock(return_value={"pr_number": 32})
|
|
runner = VerbRunner(task_service=task_svc, git_service=git_svc)
|
|
|
|
task = MagicMock(
|
|
id=uuid4(),
|
|
status="in_progress",
|
|
parent_task_id=root_id,
|
|
branch_name="feature/backend/ROOT0001--CELL0001",
|
|
)
|
|
agent = MagicMock(id=uuid4(), role="cell_pm")
|
|
ctx = spec.Context(notes="cell scope complete; bubbling up to main pm")
|
|
|
|
await runner.run_intent("submit_up", task, agent, ctx)
|
|
|
|
task_svc.get.assert_awaited_once_with(root_id)
|
|
_, kwargs = git_svc.create_pr.call_args
|
|
assert kwargs["parent"] == "feature/main_pm/ROOT0001", (
|
|
"cell→root PR base must be the parent task's real branch, not the "
|
|
f"team-derived name; got parent={kwargs['parent']!r}"
|
|
)
|
|
|
|
|
|
def test_submit_up_spec_pins_pr_before_transition() -> None:
|
|
"""The submit_up spec declares create_pr as a pre_side_effect, not a
|
|
trailing side_effect — the ordering fix lives in the spec."""
|
|
submit_up = spec._INTENT_VERBS["submit_up"]
|
|
assert submit_up.pre_side_effects == ("create_pr",)
|
|
assert "create_pr" not in submit_up.side_effects
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_does_not_run_side_effects_if_compose_fails() -> None:
|
|
"""If a composed atomic action raises, side effects must NOT run."""
|
|
task_svc = AsyncMock()
|
|
task_svc.session.begin_nested = MagicMock(
|
|
return_value=MagicMock(
|
|
__aenter__=AsyncMock(),
|
|
__aexit__=AsyncMock(side_effect=RuntimeError("rolled back")),
|
|
)
|
|
)
|
|
task_svc.claim = AsyncMock(side_effect=RuntimeError("workspace down"))
|
|
git_svc = AsyncMock()
|
|
runner = VerbRunner(task_service=task_svc, git_service=git_svc)
|
|
|
|
task = MagicMock(id=uuid4(), status="pending", plan=None, commits=[])
|
|
agent = MagicMock(id=uuid4(), role="developer")
|
|
ctx = spec.Context(plan="x")
|
|
|
|
with pytest.raises(RuntimeError):
|
|
await runner.run_intent("i_will_work_on", task, agent, ctx)
|
|
git_svc.push_branch.assert_not_called()
|
|
git_svc.create_pr.assert_not_called()
|