mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
Renn F
e6c9dde2a9
Claim-shaped verbs were failing 7/7 (claim_review) and 6/6 (claim_doc_task) as silent 120s FlowVerbTimeout 504s on the NAS: the per-claim ownership repair walked the whole clone issuing two stat syscalls per entry (chown_ms 39502 vs git_ms 8 in the live log), several passes stacked per claim, and the claim transaction held the task row the whole time — so concurrent writers queued behind it into the 60s lock_timeout. The walk now does one stat per entry shared by the chown-skip and chmod-skip checks, and a .git/roboco-owned sentinel (worktree-aware via _resolve_clone_root, written only after a zero-failure pass) skips the walk entirely when the tree is already agent-owned. Every root-side git write invalidates the sentinel BEFORE its subprocess runs — GitService._run_git for scope != none, plus the three raw-subprocess paths inside WorkspaceService the adversarial pass proved bypass it deterministically on the common respawn shape (_worktree_git for mutating verbs, _fetch_branch_ref, _fetch_origin_best_effort) — so a live marker can never vouch for files a root write is about to create. One of those queued writers was the PM journal-decision auto-record: its INSERT hit the lock timeout, _ensure_pm_decision's catch-all swallowed it without rollback, and the poisoned session blew up escalate_up with PendingRollbackError (live incident). The helper's try body now runs in a savepoint — one fix covering all seven PM verbs that route through it — verified empirically against real Postgres in both directions: the failure path leaves the session healthy and the task object readable, and create_entry's internal commit inside the savepoint drains the transactional outbox exactly once.
407 lines
14 KiB
Python
407 lines
14 KiB
Python
"""Gate Set B: delegation-time guards in Choreographer.delegate.
|
|
|
|
Pre-gateway behavior: a PM could only call ``task_create`` while spawned
|
|
in-context, which only happened after the orchestrator saw them claim
|
|
and start a parent task. That implicit gate is restored explicitly here:
|
|
|
|
- PARENT_NOT_CLAIMED: parent must be in_progress AND assigned_to PM.
|
|
- SUBTASK_CAP: 8 children = soft warn (allowed), >12 = hard block.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.services.base import ValidationError
|
|
from roboco.services.gateway.choreographer import (
|
|
Choreographer,
|
|
ChoreographerDeps,
|
|
DelegateInputs,
|
|
)
|
|
|
|
|
|
def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
|
base: dict[str, Any] = {
|
|
"task": AsyncMock(),
|
|
"work_session": AsyncMock(),
|
|
"git": AsyncMock(),
|
|
"a2a": AsyncMock(),
|
|
"journal": AsyncMock(),
|
|
"audit": AsyncMock(),
|
|
"evidence_repo": AsyncMock(),
|
|
}
|
|
base.update(overrides)
|
|
repo = base["evidence_repo"]
|
|
for method in (
|
|
"list_unread_a2a",
|
|
"list_unread_mentions",
|
|
"list_pending_notifications",
|
|
"task_metadata_gaps",
|
|
"recent_team_activity",
|
|
"blockers_in_lane",
|
|
"journal_highlights_for_task",
|
|
):
|
|
getattr(repo, method).return_value = []
|
|
# C8: default-fresh journal:decision so PM-decision gate passes.
|
|
# Tests that exercise the gate boundary stub their own value.
|
|
# The check matches MagicMock and AsyncMock (the two default sentinel
|
|
# types pytest's unittest.mock leaves on un-stubbed return_values).
|
|
_ldef = base["journal"].latest_decision_at.return_value
|
|
if type(_ldef).__name__ in ("MagicMock", "AsyncMock"):
|
|
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
|
# _ensure_pm_decision's journal write is savepoint-guarded — an
|
|
# unconfigured AsyncMock's begin_nested() call returns a raw unawaited
|
|
# coroutine, which `async with` cannot use.
|
|
base["task"].session.begin_nested = MagicMock(
|
|
return_value=MagicMock(
|
|
__aenter__=AsyncMock(return_value=None),
|
|
__aexit__=AsyncMock(return_value=False),
|
|
)
|
|
)
|
|
return ChoreographerDeps(**base)
|
|
|
|
|
|
def _delegate_inputs() -> DelegateInputs:
|
|
return DelegateInputs(
|
|
title="Implement endpoint",
|
|
description="Add /v1/foo endpoint with tests",
|
|
assigned_to="be-dev-1",
|
|
team="backend",
|
|
task_type="code",
|
|
nature="technical",
|
|
acceptance_criteria=["GET /v1/foo returns 200 with body"],
|
|
intends_to_touch=["backend/api/routers/foo.py"],
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_blocks_when_parent_not_in_progress() -> None:
|
|
"""Parent in 'pending' status (PM never called i_will_plan) blocks delegate."""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="pending",
|
|
assigned_to=pm_id,
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = []
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
# The spec's create_subtask gate requires the parent in_progress.
|
|
# Per Section 6 of the design doc, rejection messages now come from
|
|
# the spec — the spec rejects with 'invalid_state' citing the
|
|
# source-status set, not the gateway-specific i_will_plan hint.
|
|
assert body["error"] == "invalid_state"
|
|
assert "in_progress" in body["message"]
|
|
task_svc.create_subtask.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_blocks_when_parent_assigned_to_other_agent() -> None:
|
|
"""Parent claimed by a different PM cannot be delegated against by us."""
|
|
pm_id = uuid4()
|
|
other_pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=other_pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = []
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] == "not_authorized"
|
|
assert "i_will_plan" in body["remediate"]
|
|
task_svc.create_subtask.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_allows_when_parent_in_progress_and_owned() -> None:
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
acceptance_criteria=[],
|
|
)
|
|
new_task = MagicMock(id=uuid4())
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = []
|
|
task_svc.create_subtask.return_value = new_task
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] is None
|
|
assert body["status"] == "created"
|
|
task_svc.create_subtask.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_blocks_when_subtask_cap_exceeded() -> None:
|
|
"""13+ subtasks = hard block (cap is 12)."""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
)
|
|
too_many = [MagicMock(id=uuid4()) for _ in range(13)]
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = too_many
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state"
|
|
assert "13" in body["message"] or "consolidate" in body["remediate"].lower()
|
|
task_svc.create_subtask.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_allows_when_subtask_cap_within_soft_zone() -> None:
|
|
"""8-12 subtasks: warn-but-allow."""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
acceptance_criteria=[],
|
|
)
|
|
many = [MagicMock(id=uuid4()) for _ in range(10)]
|
|
new_task = MagicMock(id=uuid4())
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = many
|
|
task_svc.create_subtask.return_value = new_task
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] is None
|
|
task_svc.create_subtask.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_allows_at_zero_subtasks() -> None:
|
|
"""Empty subtask list is the common case."""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
acceptance_criteria=[],
|
|
)
|
|
new_task = MagicMock(id=uuid4())
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = []
|
|
task_svc.create_subtask.return_value = new_task
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_blocks_at_exact_cap_plus_one() -> None:
|
|
"""Cap is 12; 13th attempt blocks."""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
)
|
|
# Already 12 children — adding the 13th must be blocked.
|
|
twelve = [MagicMock(id=uuid4()) for _ in range(12)]
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = twelve
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state"
|
|
task_svc.create_subtask.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_blocks_when_parent_quick_context_empty() -> None:
|
|
"""delegate obligates the PM's quick_context resumption section on the
|
|
parent; an empty quick_context (PM never called note(scope='handoff'))
|
|
fails the tracing gate before any subtask is created."""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="",
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = []
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] == "tracing_gap"
|
|
assert "quick_context>=min" in body["missing"]
|
|
task_svc.create_subtask.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_past_max_depth_returns_invalid_state_not_500() -> None:
|
|
"""Bug B: delegating past MAX_TASK_DEPTH raised a bare ValueError that
|
|
escaped uncaught as a 500 ExceptionGroup. ``_validate_parent_depth`` now
|
|
raises ``ValidationError`` (a ServiceError), and the choreographer
|
|
translates it to an ``invalid_state`` envelope whose remediate carries the
|
|
"create as a sibling" instruction — so the PM gets a clean, actionable
|
|
rejection instead of a crash, and the agent loop doesn't respawn-blind.
|
|
"""
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=uuid4(),
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Decomposition planned; cells implement their slice next.",
|
|
acceptance_criteria=[],
|
|
)
|
|
depth_msg = (
|
|
"Task hierarchy would exceed MAX_TASK_DEPTH=4. Create this work as a "
|
|
"sibling of the deepest task instead of a further nested subtask."
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = parent
|
|
task_svc.agent_for.return_value = MagicMock(role="cell_pm", team="backend")
|
|
task_svc.get_subtasks.return_value = []
|
|
# create_subtask -> create -> _validate_parent_depth raises ValidationError.
|
|
task_svc.create_subtask.side_effect = ValidationError(
|
|
depth_msg, field="parent_task_id"
|
|
)
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.delegate(pm_id, parent_id, _delegate_inputs())
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state"
|
|
assert "MAX_TASK_DEPTH" in body["message"]
|
|
# The remediation must reach the agent — the whole point of the fix.
|
|
assert "sibling" in body["remediate"]
|
|
task_svc.create_subtask.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_static_guards_allows_cell_map_root() -> None:
|
|
# A cross-cell MegaTask root-subtask carries a cell_projects map with
|
|
# project_id=None, product_id=None — a valid branchless-coordination
|
|
# shape (batch.is_branchless_coordination). The first project/product
|
|
# guard must NOT reject it.
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=None,
|
|
product_id=None,
|
|
cell_projects=[{"team": "backend", "project_id": uuid4()}],
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="Cross-cell fan-out planned.",
|
|
)
|
|
deps = _make_deps()
|
|
c = Choreographer(deps)
|
|
|
|
result = await c._delegate_static_guards(
|
|
pm_id, parent_id, parent, _delegate_inputs()
|
|
)
|
|
# The project/product guard must not fire for a cell-map root.
|
|
assert result is None or "neither a project_id nor a product_id" not in (
|
|
getattr(result, "message", "") or ""
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delegate_static_guards_rejects_bare_no_project_no_product() -> None:
|
|
# No project_id, no product_id, no cell_projects — still rejected.
|
|
pm_id = uuid4()
|
|
parent_id = uuid4()
|
|
parent = MagicMock(
|
|
id=parent_id,
|
|
project_id=None,
|
|
product_id=None,
|
|
cell_projects=None,
|
|
status="in_progress",
|
|
assigned_to=pm_id,
|
|
team="backend",
|
|
quick_context="x",
|
|
)
|
|
deps = _make_deps()
|
|
c = Choreographer(deps)
|
|
|
|
result = await c._delegate_static_guards(
|
|
pm_id, parent_id, parent, _delegate_inputs()
|
|
)
|
|
assert result is not None
|
|
assert "neither a project_id nor a product_id" in (result.message or "")
|