mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
Renn F
4829f93a68
Resolves the 100% claim-failure rate introduced by the gateway rewrite
(commit 62bda0c plus 78 follow-ups). Live smoke runs hit
`404 /api/v2/flow/developer/...` on every dev verb plus a manifest
fallback that silently exposed off-role verbs to PMs — confirmed
firing simultaneously in NAS agent logs (be-dev-1, be-pm, main-pm).
Audit reports under docs/internal/audit_2026_05_04/ catalogue 49
defects across gateway, services, prompts, MCP transport, substrate,
and tests (8 detail reports + master synthesis). Six smoking guns;
three proven in production logs.
Phase 0 — unblock claim:
- URL prefix /api/v2/flow/dev → /developer; slug-map board roles
(product_owner, head_marketing) → /board (D-01)
- _i_will_work_on AttributeError on None across pending /
needs_revision / claimed re-entry branches (D-02)
- Seed last_heartbeat_at in _qa_or_doc_claim (D-03)
- Drop misleading i_have_committed verb; dev flow uses commit() (D-04)
- Manifest mount via compose; flow_server + do_server fail loud
instead of exposing all-verbs fallback (D-12)
- MCP _post() surfaces envelope body on 4xx so agents see remediate
hints (D-13)
on git failure so retries aren't blocked by half-state (S-01)
Phase 1 — lifecycle stability:
- _resolve_skill falls back to AgentTable.capabilities (D-06)
- main_pm_complete uses kwargs for escalate_to_ceo (D-07)
- i_am_done auto-runs submit_verification when in_progress (D-08)
- active_claimant_id wired in claim/unclaim paths — single-claimant
invariant now functional (D-05)
- qa_pass/qa_fail assert claimed_by parity with qa_agent_id (D-18)
- Prompt-drift sweep: fail() shape, i_am_done(task_id, notes),
subtask cap (12 hard / 8 soft), error-code symbology rewritten in
base.md + per-role anti-patterns (D-10/11/29/30/31, D-37)
Phase 2 — invariants + architecture:
- Real-DB integration test exercising claim → in_progress → commit
→ submit_for_qa → i_am_done → awaiting_qa (P2-1)
- choreographer.py → package; 3 of 6 role mixins extracted
(board, doc, qa). _impl.py 2,526 → 2,080 lines (-18%). Continuation
plan in docs/internal/audit_2026_05_04/p2_2_decompose_plan.md (P2-2)
- Closure guards consolidated via _subtasks_not_terminal_envelope (P2-3)
- TaskService.unclaim_for_reaper routed through canonical
_validate_and_set_status; in_progress → pending added to
VALID_TRANSITIONS (P2-4)
- Dead code removed: i_am_done_with_catchup verb, _run_catch_up helper
(P2-5)
- 6 state-machine invariants asserted via property test (P2-6)
- attempt_id (uuid4) stamped on every gateway.rejected audit row (P2-7)
- _reconcile_orphan_claims_on_startup rolls back tasks left CLAIMED
with branch_name=NULL from prior crashes (P2-8)
- scripts/regenerate_verb_tables.py introspects Pydantic schemas +
role_config; compose_prompt injects per-role tables as a layer.
Eliminates the prompt-drift class structurally (P2-9)
Other:
- D-48: orchestrator mounts host's ~/.claude.json when present so
agents don't boot from backup recovery on every spawn
- D-49: dev dispatcher rejects role-mismatched spawns (e.g. doc task
assigned to dev agent)
Tests: 553 pass · ruff + mypy clean. Live NAS smoke verification
pending — needs the stack brought back up.
398 lines
13 KiB
Python
398 lines
13 KiB
Python
"""Tests for the developer-facing Choreographer methods."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
|
|
|
|
|
def _make_deps(**overrides: AsyncMock) -> ChoreographerDeps:
|
|
task = overrides.get("task", AsyncMock())
|
|
work_session = overrides.get("work_session", AsyncMock())
|
|
git = overrides.get("git", AsyncMock())
|
|
a2a = overrides.get("a2a", AsyncMock())
|
|
journal = overrides.get("journal", AsyncMock())
|
|
audit = overrides.get("audit", AsyncMock())
|
|
evidence_repo = overrides.get("evidence_repo", AsyncMock())
|
|
# Ensure evidence_repo returns empty lists by default
|
|
for method in (
|
|
"list_unread_a2a",
|
|
"list_unread_mentions",
|
|
"list_pending_notifications",
|
|
"task_metadata_gaps",
|
|
"recent_team_activity",
|
|
"blockers_in_lane",
|
|
):
|
|
getattr(evidence_repo, method).return_value = []
|
|
return ChoreographerDeps(
|
|
task=task,
|
|
work_session=work_session,
|
|
git=git,
|
|
a2a=a2a,
|
|
journal=journal,
|
|
audit=audit,
|
|
evidence_repo=evidence_repo,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_give_me_work_returns_assigned_task() -> None:
|
|
agent_id = uuid4()
|
|
task_obj = MagicMock(id=uuid4(), status="pending", title="t1")
|
|
task_svc = AsyncMock()
|
|
task_svc.list_assigned_for_agent.return_value = [task_obj]
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.give_me_work(agent_id)
|
|
body = env.as_dict()
|
|
assert body["status"] == "pending"
|
|
assert body["task_id"] == str(task_obj.id)
|
|
assert "i_will_work_on" in body["next"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_give_me_work_returns_paused_when_no_assigned() -> None:
|
|
agent_id = uuid4()
|
|
paused_obj = MagicMock(id=uuid4(), status="paused")
|
|
task_svc = AsyncMock()
|
|
task_svc.list_assigned_for_agent.return_value = []
|
|
task_svc.list_paused_for_agent.return_value = [paused_obj]
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.give_me_work(agent_id)
|
|
body = env.as_dict()
|
|
assert body["task_id"] == str(paused_obj.id)
|
|
assert "resume" in body["next"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_give_me_work_returns_idle_when_no_work() -> None:
|
|
agent_id = uuid4()
|
|
task_svc = AsyncMock()
|
|
task_svc.list_assigned_for_agent.return_value = []
|
|
task_svc.list_paused_for_agent.return_value = []
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.give_me_work(agent_id)
|
|
body = env.as_dict()
|
|
assert body["status"] == "idle"
|
|
assert "i_am_idle" in body["next"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_will_work_on_pending_with_plan() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
pending_task = MagicMock(
|
|
id=task_id,
|
|
status="pending",
|
|
plan=None,
|
|
assigned_to=None,
|
|
parent_task_id=None,
|
|
sequence=0,
|
|
task_type="code",
|
|
)
|
|
in_progress_task = MagicMock(
|
|
id=task_id, status="in_progress", plan={"text": "do x"}, assigned_to=agent_id
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = pending_task
|
|
task_svc.agent_for.return_value = MagicMock(role="developer", team="backend")
|
|
task_svc.list_in_progress_for_agent.return_value = []
|
|
task_svc.list_paused_for_agent.return_value = []
|
|
task_svc.get_subtasks.return_value = []
|
|
task_svc.claim.return_value = MagicMock(
|
|
id=task_id, status="claimed", plan=None, assigned_to=agent_id
|
|
)
|
|
task_svc.set_plan.return_value = MagicMock(
|
|
id=task_id, status="claimed", plan={"text": "do x"}, assigned_to=agent_id
|
|
)
|
|
task_svc.start.return_value = in_progress_task
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_will_work_on(agent_id, task_id, plan="do x then y")
|
|
assert env.error is None
|
|
assert env.status == "in_progress"
|
|
task_svc.claim.assert_awaited_once_with(task_id, agent_id)
|
|
task_svc.set_plan.assert_awaited_once()
|
|
task_svc.start.assert_awaited_once_with(task_id, agent_id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_will_work_on_pending_no_plan_returns_tracing_gap() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
pending_task = MagicMock(
|
|
id=task_id,
|
|
status="pending",
|
|
plan=None,
|
|
assigned_to=None,
|
|
description="task description",
|
|
parent_task_id=None,
|
|
sequence=0,
|
|
task_type="code",
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = pending_task
|
|
task_svc.agent_for.return_value = MagicMock(role="developer", team="backend")
|
|
task_svc.list_in_progress_for_agent.return_value = []
|
|
task_svc.list_paused_for_agent.return_value = []
|
|
task_svc.get_subtasks.return_value = []
|
|
task_svc.claim.return_value = MagicMock(
|
|
id=task_id, status="claimed", plan=None, assigned_to=agent_id
|
|
)
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_will_work_on(agent_id, task_id, plan=None)
|
|
body = env.as_dict()
|
|
assert body["error"] == "tracing_gap"
|
|
assert "plan" in body["missing"]
|
|
assert "i_will_work_on" in body["remediate"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_will_work_on_needs_revision_re_starts() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
nr_task = MagicMock(
|
|
id=task_id, status="needs_revision", assigned_to=agent_id, plan={"x": 1}
|
|
)
|
|
in_progress_task = MagicMock(
|
|
id=task_id, status="in_progress", assigned_to=agent_id, plan={"x": 1}
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = nr_task
|
|
task_svc.start.return_value = in_progress_task
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_will_work_on(agent_id, task_id)
|
|
assert env.status == "in_progress"
|
|
task_svc.claim.assert_not_awaited() # already assigned
|
|
task_svc.start.assert_awaited_once_with(task_id, agent_id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_will_work_on_task_not_found_returns_not_found() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = None
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_will_work_on(agent_id, task_id)
|
|
body = env.as_dict()
|
|
assert body["error"] == "not_found"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_will_work_on_invalid_state_returns_invalid_state() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
completed_task = MagicMock(id=task_id, status="completed", assigned_to=agent_id)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = completed_task
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_will_work_on(agent_id, task_id)
|
|
body = env.as_dict()
|
|
assert body["error"] == "invalid_state"
|
|
assert "completed" in body["message"]
|
|
|
|
|
|
# test_i_am_done_with_catchup_full_chain removed (audit P2-5/D-16):
|
|
# i_am_done_with_catchup verb deleted. submit_for_qa now does push + PR
|
|
# explicitly; i_am_done auto-runs submit_verification + submit_qa.
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_done_blocks_when_acceptance_criteria_unaddressed() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
t = MagicMock(
|
|
id=task_id,
|
|
status="in_progress",
|
|
assigned_to=agent_id,
|
|
plan={"x": 1},
|
|
branch_name="feature/backend/abc",
|
|
work_session_id=uuid4(),
|
|
self_verified=False,
|
|
progress_updates=[{"message": "p"}],
|
|
acceptance_criteria=["AC1", "AC2"],
|
|
acceptance_criteria_status=[
|
|
{"criterion": "AC1", "referencing_artifact_id": "c1"}
|
|
],
|
|
commits=[],
|
|
documents=[],
|
|
dev_notes="",
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
journal_svc = AsyncMock()
|
|
journal_svc.has_reflect_for_task.return_value = True
|
|
deps = _make_deps(task=task_svc, journal=journal_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_done(agent_id, task_id, "done")
|
|
body = env.as_dict()
|
|
assert body["error"] == "tracing_gap"
|
|
assert any("AC2" in m for m in body["missing"])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_done_blocks_when_journal_reflect_missing() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
t = MagicMock(
|
|
id=task_id,
|
|
status="in_progress",
|
|
assigned_to=agent_id,
|
|
plan={"x": 1},
|
|
branch_name="feature/backend/abc",
|
|
work_session_id=uuid4(),
|
|
self_verified=False,
|
|
progress_updates=[{"message": "p"}],
|
|
acceptance_criteria=["AC1"],
|
|
acceptance_criteria_status=[
|
|
{"criterion": "AC1", "referencing_artifact_id": "c1"}
|
|
],
|
|
commits=[],
|
|
documents=[],
|
|
dev_notes="",
|
|
)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
journal_svc = AsyncMock()
|
|
journal_svc.has_reflect_for_task.return_value = False # no reflect
|
|
deps = _make_deps(task=task_svc, journal=journal_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_done(agent_id, task_id, "done")
|
|
body = env.as_dict()
|
|
assert body["error"] == "tracing_gap"
|
|
assert "journal:reflect" in body["missing"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_done_not_assigned_returns_not_authorized() -> None:
|
|
agent_id = uuid4()
|
|
other_agent = uuid4()
|
|
task_id = uuid4()
|
|
t = MagicMock(id=task_id, status="in_progress", assigned_to=other_agent)
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_done(agent_id, task_id, "x")
|
|
body = env.as_dict()
|
|
assert body["error"] == "not_authorized"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_done_skill_resolution_picks_existing_skill() -> None:
|
|
"""Resolver picks first matching skill.
|
|
|
|
Falls back to the first preference entry when no match is found.
|
|
"""
|
|
deps = _make_deps()
|
|
c = Choreographer(deps)
|
|
|
|
qa = MagicMock(id=uuid4(), skills=[{"id": "qa_review"}, {"id": "test_validation"}])
|
|
skill = c._resolve_skill(qa, ["code_review", "qa_review"])
|
|
assert skill == "qa_review"
|
|
|
|
qa_with_canonical = MagicMock(id=uuid4(), skills=[{"id": "code_review"}])
|
|
skill2 = c._resolve_skill(qa_with_canonical, ["code_review", "qa_review"])
|
|
assert skill2 == "code_review"
|
|
|
|
qa_with_neither = MagicMock(id=uuid4(), skills=[{"id": "other_skill"}])
|
|
skill3 = c._resolve_skill(qa_with_neither, ["code_review", "qa_review"])
|
|
assert skill3 == "code_review" # fallback to first
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_blocked_escalates_and_journals() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
t = MagicMock(
|
|
id=task_id, status="in_progress", assigned_to=agent_id, pre_block_state=None
|
|
)
|
|
after = MagicMock(id=task_id, status="blocked")
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = t
|
|
task_svc.escalate.return_value = after
|
|
journal_svc = AsyncMock()
|
|
deps = _make_deps(task=task_svc, journal=journal_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_blocked(agent_id, task_id, "external API down")
|
|
assert env.error is None
|
|
assert env.status == "blocked"
|
|
journal_svc.write_struggle.assert_awaited_once()
|
|
task_svc.escalate.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_blocked_task_not_found_returns_not_found() -> None:
|
|
agent_id = uuid4()
|
|
task_id = uuid4()
|
|
task_svc = AsyncMock()
|
|
task_svc.get.return_value = None
|
|
deps = _make_deps(task=task_svc)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_blocked(agent_id, task_id, "x")
|
|
body = env.as_dict()
|
|
assert body["error"] == "not_found"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_idle_with_unread_a2a_soft_blocks() -> None:
|
|
agent_id = uuid4()
|
|
deps = _make_deps()
|
|
deps.evidence_repo.list_unread_a2a.return_value = [{"from": "x", "task_id": "t1"}]
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_idle(agent_id)
|
|
body = env.as_dict()
|
|
assert body["status"] == "idle_with_unread"
|
|
assert "address" in body["next"].lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_idle_with_unread_mentions_soft_blocks() -> None:
|
|
agent_id = uuid4()
|
|
deps = _make_deps()
|
|
deps.evidence_repo.list_unread_mentions.return_value = [
|
|
{"channel": "#x", "from": "y"}
|
|
]
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_idle(agent_id)
|
|
body = env.as_dict()
|
|
assert body["status"] == "idle_with_unread"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_i_am_idle_clean_returns_idle() -> None:
|
|
agent_id = uuid4()
|
|
task_svc = AsyncMock()
|
|
deps = _make_deps(task=task_svc)
|
|
# All evidence_repo lists are already empty per _make_deps default
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_am_idle(agent_id)
|
|
assert env.status == "idle"
|
|
task_svc.mark_agent_idle.assert_awaited_once_with(agent_id)
|