mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(lifecycle): stop PMs re-claiming tasks out of the closure queue — kills the i_will_plan review loop
A cell/main PM's i_will_plan could legally re-claim its own task from awaiting_pm_review (a CLAIM_RULES edge added for post-respawn recovery), resetting the task to in_progress and re-running submit_up -> pr_pass -> awaiting_pm_review forever: one Sentinel conventions child looped eleven full laps in four hours (14 reviews on one PR, 37 agent spawns) while its root's closure check fired eighteen times and the Main PM could never close anything. The claim edge is gone from every table that carried it — CLAIM_RULES, the claim ActionSpec's source statuses, the StatusTransition row, the service-layer _ROLE_CLAIM_STATUSES twin, and the legacy enforcement shim's operational-edge/role-gate entries (left divergent, it would be the same silent two-table drift that produced this bug). The respawn case the edge existed for is now served properly: _handle_pm_reentry gained a third contract — a PM calling i_will_plan on its own awaiting_pm_review task gets a steering envelope (no claim, no state change) pointing at complete/request_changes, and give_me_work's next hint for that status says the same instead of steering back into i_will_plan. The no-transition review-claim path and the pm-review dispatch prompt were already correct and are untouched, so closure still converges through them. A new bidirectional test asserts CLAIM_RULES and _ROLE_CLAIM_STATUSES stay identical per PM role (the old comment claimed a sync test existed; it checked one direction only). Lifecycle artifacts regenerated; the parity suite's three unshaped session mocks fixed, zero AsyncMock warnings remain.
This commit is contained in:
@@ -869,6 +869,26 @@ async def test_complete_matches_spec(role: str, status: str) -> None:
|
||||
id=task_id, status="awaiting_ceo_approval", assigned_to=None, team="backend"
|
||||
)
|
||||
task_svc.all_subtasks_terminal.return_value = True
|
||||
# complete's merge path runs _stamp_pm_findings_verified_or_rejection,
|
||||
# which opens a session.begin_nested() savepoint and reads the findings
|
||||
# ledger via ReviewFindingsRepository.list_for_task (session.execute ->
|
||||
# .scalars().all()). Unconfigured, both calls resolve to auto-generated
|
||||
# AsyncMock children: `async with <AsyncMock>():` fails the async
|
||||
# context-manager protocol and `<AsyncMock-result>.scalars()` returns an
|
||||
# unawaited coroutine — caught by the verb's own except Exception, but
|
||||
# the never-awaited coroutine leaks a RuntimeWarning at GC time.
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.pr_merge.return_value = {"merged": True, "merge_commit_sha": "x"}
|
||||
git_svc.create_pr.return_value = {"pr_number": 99, "pr_url": "x"}
|
||||
@@ -1268,6 +1288,18 @@ async def test_claim_review_matches_spec(role: str, status: str) -> None:
|
||||
task_svc.qa_claim.return_value = after
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
# claim_review's full=True briefing reads the findings ledger
|
||||
# (findings.open_findings_for_task -> session.execute -> .scalars().all()).
|
||||
# Unconfigured, session.execute auto-generates as an AsyncMock child whose
|
||||
# call returns another AsyncMock; .scalars() on that is itself an
|
||||
# unawaited coroutine — caught by open_findings_for_task's own fail-open
|
||||
# except, but the dangling coroutine leaks a RuntimeWarning at GC time.
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
deps = _make_deps(task_svc=task_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
@@ -1546,6 +1578,14 @@ async def test_claim_doc_task_matches_spec(role: str, status: str) -> None:
|
||||
task_svc.doc_claim.return_value = after
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
# claim_doc_task's full=True briefing reads the findings ledger the same
|
||||
# way claim_review's does (see that test's comment) — same fix.
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
deps = _make_deps(task_svc=task_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
|
||||
@@ -491,6 +491,11 @@ def test_claim_rules_match_pre_gateway_table() -> None:
|
||||
re-delegating fixes (scoped by give_me_work routing, which offers only the
|
||||
caller's own assigned tasks). BACKLOG → PENDING is a separate `activate`
|
||||
action (strict transitions; no implicit activate-on-claim).
|
||||
|
||||
AWAITING_PM_REVIEW is deliberately absent from both PM roles — a claim
|
||||
edge there let a respawned PM's i_will_plan legally re-claim its own
|
||||
review-queue task and loop the submit_up -> pr_pass -> awaiting_pm_review
|
||||
cycle forever. See test_awaiting_pm_review_not_claimable_by_any_role.
|
||||
"""
|
||||
assert spec.CLAIM_RULES[spec.Role.DEVELOPER] == frozenset(
|
||||
{spec.Status.PENDING, spec.Status.NEEDS_REVISION}
|
||||
@@ -500,21 +505,26 @@ def test_claim_rules_match_pre_gateway_table() -> None:
|
||||
{spec.Status.PENDING, spec.Status.AWAITING_DOCUMENTATION}
|
||||
)
|
||||
assert spec.CLAIM_RULES[spec.Role.CELL_PM] == frozenset(
|
||||
{
|
||||
spec.Status.PENDING,
|
||||
spec.Status.NEEDS_REVISION,
|
||||
spec.Status.AWAITING_PM_REVIEW,
|
||||
}
|
||||
{spec.Status.PENDING, spec.Status.NEEDS_REVISION}
|
||||
)
|
||||
assert spec.CLAIM_RULES[spec.Role.MAIN_PM] == frozenset(
|
||||
{
|
||||
spec.Status.PENDING,
|
||||
spec.Status.NEEDS_REVISION,
|
||||
spec.Status.AWAITING_PM_REVIEW,
|
||||
}
|
||||
{spec.Status.PENDING, spec.Status.NEEDS_REVISION}
|
||||
)
|
||||
|
||||
|
||||
def test_awaiting_pm_review_not_claimable_by_any_role() -> None:
|
||||
"""The claim edge that let a respawned PM's i_will_plan reset an
|
||||
awaiting_pm_review task (looping submit_up -> pr_pass -> awaiting_pm_review
|
||||
forever) is permanently closed: no role may claim from this status. A PM
|
||||
re-entering its own review task is steered by the choreographer directly
|
||||
to complete/request_changes, never through claim.
|
||||
"""
|
||||
for role, statuses in spec.CLAIM_RULES.items():
|
||||
assert spec.Status.AWAITING_PM_REVIEW not in statuses, role
|
||||
# PR_REVIEWER's own review-gate claim (a different status) is untouched.
|
||||
assert spec.Status.AWAITING_PR_REVIEW in spec.CLAIM_RULES[spec.Role.PR_REVIEWER]
|
||||
|
||||
|
||||
def test_team_rules_pin_team_for_seeded_agents() -> None:
|
||||
assert spec.ROLE_TEAM_RULES["be-dev-1"] == "backend"
|
||||
assert spec.ROLE_TEAM_RULES["be-pm"] == "backend"
|
||||
|
||||
@@ -277,6 +277,40 @@ def test_status_classification_is_mutually_disjoint() -> None:
|
||||
assert waiting & terminal == set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# awaiting_pm_review -> claimed is closed (the i_will_plan re-claim loop):
|
||||
# a PM re-entering its own review-queue task is steered by the choreographer
|
||||
# straight to complete/request_changes, never via a claim that resets the
|
||||
# task and re-runs submit_up -> pr_pass -> awaiting_pm_review forever. This
|
||||
# legacy shim (_LEGACY_OPERATIONAL_EDGES / _LEGACY_ROLE_GATES) used to grant
|
||||
# the same edge lifecycle.CLAIM_RULES had already closed — the identical
|
||||
# two-tables-drift shape that caused the incident.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_awaiting_pm_review_claim_no_longer_allowed_for_pm_roles() -> None:
|
||||
assert can_agent_transition("awaiting_pm_review", "claimed", "cell_pm") is False
|
||||
assert can_agent_transition("awaiting_pm_review", "claimed", "main_pm") is False
|
||||
with pytest.raises(TaskLifecycleError):
|
||||
validate_task_transition("awaiting_pm_review", "claimed", "cell_pm")
|
||||
with pytest.raises(TaskLifecycleError):
|
||||
validate_task_transition("awaiting_pm_review", "claimed", "main_pm")
|
||||
|
||||
|
||||
def test_awaiting_pm_review_needs_revision_still_allowed() -> None:
|
||||
"""The PM reject-back-to-dev path (request_changes) is untouched."""
|
||||
assert (
|
||||
can_agent_transition("awaiting_pm_review", "needs_revision", "cell_pm") is True
|
||||
)
|
||||
assert (
|
||||
can_agent_transition("awaiting_pm_review", "needs_revision", "main_pm") is True
|
||||
)
|
||||
assert (
|
||||
validate_task_transition("awaiting_pm_review", "needs_revision", "cell_pm")
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_status_classification_covers_every_enum_member() -> None:
|
||||
"""Every Status enum member must be classified by EXACTLY one of
|
||||
is_terminal_state / is_active_state / is_waiting_state — the coverage
|
||||
|
||||
@@ -86,6 +86,16 @@ def test_claim_verb_hint_pm_for_planning() -> None:
|
||||
assert "i_will_plan" in hint
|
||||
|
||||
|
||||
def test_claim_verb_hint_pm_for_awaiting_pm_review_steers_to_complete() -> None:
|
||||
"""A PM's own review-queue task must never hint i_will_plan — that verb
|
||||
used to legally re-claim and reset it, looping submit_up -> pr_pass ->
|
||||
awaiting_pm_review forever."""
|
||||
for role in ("cell_pm", "main_pm"):
|
||||
hint = Choreographer._claim_verb_hint(role, _task("awaiting_pm_review"))
|
||||
assert "complete" in hint
|
||||
assert "call i_will_plan(" not in hint
|
||||
|
||||
|
||||
def test_claim_verb_hint_dev_default() -> None:
|
||||
hint = Choreographer._claim_verb_hint("developer", _task("pending"))
|
||||
assert "i_will_work_on" in hint
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""PM re-entry on an awaiting_pm_review task must steer, never re-claim.
|
||||
|
||||
Live incident: an awaiting_pm_review task (already past the in-path PR gate)
|
||||
kept getting re-offered to its owning PM by give_me_work. The respawned PM
|
||||
called i_will_plan, and CLAIM_RULES used to grant CELL_PM/MAIN_PM a claim from
|
||||
AWAITING_PM_REVIEW — so the composed (claim, set_plan, start) sequence legally
|
||||
reset the task to in_progress and re-ran submit_up -> pr_pass ->
|
||||
awaiting_pm_review forever (one production task looped 11 cycles across 37
|
||||
spawns in 4h). ``_handle_pm_reentry`` now recognizes this status for the
|
||||
owning PM and returns a steering-only OK envelope (complete / request_changes)
|
||||
with no claim and no state change; CLAIM_RULES no longer permits the claim at
|
||||
all, so a non-owner (or any other caller) falls through to a normal spec
|
||||
rejection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared fixture helpers — same pattern as test_i_will_plan_sub_tasks_gate.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
base: dict[str, Any] = {
|
||||
"task": AsyncMock(),
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
task = base["task"]
|
||||
task.session = MagicMock()
|
||||
task.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
"list_unread_mentions",
|
||||
"list_pending_notifications",
|
||||
"task_metadata_gaps",
|
||||
"recent_team_activity",
|
||||
"blockers_in_lane",
|
||||
"journal_highlights_for_task",
|
||||
):
|
||||
getattr(repo, method).return_value = []
|
||||
_ldef = base["journal"].latest_decision_at.return_value
|
||||
if type(_ldef).__name__ in ("MagicMock", "AsyncMock"):
|
||||
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
||||
return ChoreographerDeps(**base)
|
||||
|
||||
|
||||
def _review_task_svc(task_id: object, pm_id: object, *, role: str) -> AsyncMock:
|
||||
"""TaskService mock for a PM re-entering its own awaiting_pm_review task."""
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = MagicMock(
|
||||
id=task_id,
|
||||
status="awaiting_pm_review",
|
||||
plan={"text": "already planned"},
|
||||
assigned_to=pm_id,
|
||||
task_type="planning",
|
||||
parent_task_id=None,
|
||||
sequence=0,
|
||||
team="backend",
|
||||
commits=["abc123"],
|
||||
pr_number=42,
|
||||
branch_name="feature/backend/abc",
|
||||
quick_context=None,
|
||||
)
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
id=pm_id, role=role, team="backend", slug=None
|
||||
)
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.get_subtasks.return_value = []
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
return task_svc
|
||||
|
||||
|
||||
async def _assert_steers_without_reclaiming(role: str) -> None:
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
task_svc = _review_task_svc(task_id, pm_id, role=role)
|
||||
deps = _make_deps(task=task_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
env = await c.i_will_plan(pm_id, task_id, plan="resume")
|
||||
body = env.as_dict()
|
||||
|
||||
assert body.get("error") is None, body
|
||||
assert body["status"] == "awaiting_pm_review", body
|
||||
assert "complete" in body["next"], body
|
||||
|
||||
task_svc.claim.assert_not_awaited()
|
||||
task_svc.set_plan.assert_not_awaited()
|
||||
task_svc.start.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cell_pm_reentry_awaiting_pm_review_steers_to_complete() -> None:
|
||||
await _assert_steers_without_reclaiming("cell_pm")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_main_pm_reentry_awaiting_pm_review_steers_to_complete() -> None:
|
||||
await _assert_steers_without_reclaiming("main_pm")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_owner_awaiting_pm_review_is_rejected_not_reclaimed() -> None:
|
||||
"""A PM that does NOT own the review task gets the normal spec rejection —
|
||||
CLAIM_RULES no longer grants a claim from awaiting_pm_review to anyone, so
|
||||
this falls straight through to invalid_state instead of resetting the task.
|
||||
"""
|
||||
pm_id = uuid4()
|
||||
other_pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
task_svc = _review_task_svc(task_id, other_pm_id, role="cell_pm")
|
||||
deps = _make_deps(task=task_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
env = await c.i_will_plan(pm_id, task_id, plan="resume")
|
||||
body = env.as_dict()
|
||||
|
||||
assert body.get("error") == "invalid_state", body
|
||||
task_svc.claim.assert_not_awaited()
|
||||
task_svc.set_plan.assert_not_awaited()
|
||||
task_svc.start.assert_not_awaited()
|
||||
@@ -22,7 +22,11 @@ from typing import TYPE_CHECKING, cast
|
||||
import pytest
|
||||
from roboco.foundation.policy import lifecycle as spec
|
||||
from roboco.models.base import TaskStatus
|
||||
from roboco.services.task import _default_claim_statuses, _get_valid_claim_statuses
|
||||
from roboco.services.task import (
|
||||
_ROLE_CLAIM_STATUSES,
|
||||
_default_claim_statuses,
|
||||
_get_valid_claim_statuses,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from roboco.db.tables import AgentTable
|
||||
@@ -52,3 +56,26 @@ def test_runtime_pm_claim_mapping_covers_spec_claim_rules(role: spec.Role) -> No
|
||||
f"runtime claim mapping for {role.value} is missing spec-allowed "
|
||||
f"status '{status.value}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", [spec.Role.CELL_PM, spec.Role.MAIN_PM])
|
||||
def test_claim_rules_and_role_statuses_are_identical(role: spec.Role) -> None:
|
||||
"""Genuine bidirectional cross-check between the two claim tables.
|
||||
|
||||
``test_runtime_pm_claim_mapping_covers_spec_claim_rules`` above only
|
||||
checks spec ⊆ runtime — it would still pass if ``_ROLE_CLAIM_STATUSES``
|
||||
carried an EXTRA status the spec doesn't grant (e.g. AWAITING_PM_REVIEW
|
||||
re-added to the runtime table alone, with lifecycle.CLAIM_RULES left
|
||||
untouched). That silent one-sided drift — "the service table already
|
||||
granted this on the belief that the spec granted it too" — is exactly the
|
||||
shape that caused the awaiting_pm_review re-claim loop
|
||||
(``lifecycle.py``'s ``CLAIM_RULES`` comment covers the incident). This
|
||||
test imports both tables directly and asserts the per-role sets are
|
||||
IDENTICAL, not just one-way-covering.
|
||||
"""
|
||||
spec_values = {s.value for s in spec.CLAIM_RULES[role]}
|
||||
runtime_values = {s.value for s in _ROLE_CLAIM_STATUSES[role.value]}
|
||||
assert spec_values == runtime_values, (
|
||||
f"spec.CLAIM_RULES[{role.value}]={spec_values} != "
|
||||
f"task._ROLE_CLAIM_STATUSES[{role.value!r}]={runtime_values}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user