Files
roboco/tests/unit/gateway/test_choreographer_submit_qa_gates.py
T
Renn F 4829f93a68 fix(gateway): unblock task claim; full Phase 0/1/2 remediation
Resolves the 100% claim-failure rate introduced by the gateway rewrite
  (commit 62bda0c plus 78 follow-ups). Live smoke runs hit
  `404 /api/v2/flow/developer/...` on every dev verb plus a manifest
  fallback that silently exposed off-role verbs to PMs — confirmed
  firing simultaneously in NAS agent logs (be-dev-1, be-pm, main-pm).

  Audit reports under docs/internal/audit_2026_05_04/ catalogue 49
  defects across gateway, services, prompts, MCP transport, substrate,
  and tests (8 detail reports + master synthesis). Six smoking guns;
  three proven in production logs.

  Phase 0 — unblock claim:
  - URL prefix /api/v2/flow/dev → /developer; slug-map board roles
    (product_owner, head_marketing) → /board (D-01)
  - _i_will_work_on AttributeError on None across pending /
    needs_revision / claimed re-entry branches (D-02)
  - Seed last_heartbeat_at in _qa_or_doc_claim (D-03)
  - Drop misleading i_have_committed verb; dev flow uses commit() (D-04)
  - Manifest mount via compose; flow_server + do_server fail loud
    instead of exposing all-verbs fallback (D-12)
  - MCP _post() surfaces envelope body on 4xx so agents see remediate
    hints (D-13)
    on git failure so retries aren't blocked by half-state (S-01)

  Phase 1 — lifecycle stability:
  - _resolve_skill falls back to AgentTable.capabilities (D-06)
  - main_pm_complete uses kwargs for escalate_to_ceo (D-07)
  - i_am_done auto-runs submit_verification when in_progress (D-08)
  - active_claimant_id wired in claim/unclaim paths — single-claimant
    invariant now functional (D-05)
  - qa_pass/qa_fail assert claimed_by parity with qa_agent_id (D-18)
  - Prompt-drift sweep: fail() shape, i_am_done(task_id, notes),
    subtask cap (12 hard / 8 soft), error-code symbology rewritten in
    base.md + per-role anti-patterns (D-10/11/29/30/31, D-37)

  Phase 2 — invariants + architecture:
  - Real-DB integration test exercising claim → in_progress → commit
    → submit_for_qa → i_am_done → awaiting_qa (P2-1)
  - choreographer.py → package; 3 of 6 role mixins extracted
    (board, doc, qa). _impl.py 2,526 → 2,080 lines (-18%). Continuation
    plan in docs/internal/audit_2026_05_04/p2_2_decompose_plan.md (P2-2)
  - Closure guards consolidated via _subtasks_not_terminal_envelope (P2-3)
  - TaskService.unclaim_for_reaper routed through canonical
    _validate_and_set_status; in_progress → pending added to
    VALID_TRANSITIONS (P2-4)
  - Dead code removed: i_am_done_with_catchup verb, _run_catch_up helper
    (P2-5)
  - 6 state-machine invariants asserted via property test (P2-6)
  - attempt_id (uuid4) stamped on every gateway.rejected audit row (P2-7)
  - _reconcile_orphan_claims_on_startup rolls back tasks left CLAIMED
    with branch_name=NULL from prior crashes (P2-8)
  - scripts/regenerate_verb_tables.py introspects Pydantic schemas +
    role_config; compose_prompt injects per-role tables as a layer.
    Eliminates the prompt-drift class structurally (P2-9)

  Other:
  - D-48: orchestrator mounts host's ~/.claude.json when present so
    agents don't boot from backup recovery on every spawn
  - D-49: dev dispatcher rejects role-mismatched spawns (e.g. doc task
    assigned to dev agent)

  Tests: 553 pass · ruff + mypy clean. Live NAS smoke verification
  pending — needs the stack brought back up.
2026-05-04 23:43:55 +02:00

259 lines
8.9 KiB
Python

"""Gate Set E: submit-qa field-level gates in Choreographer.i_am_done.
Pre-gateway location: roboco/api/routes/tasks.py:903-940 (route layer).
The four field-level gates returned 400 errors when the dev tried to
submit for QA without:
- NOT_SELF_VERIFIED: task.self_verified must be true.
- NO_COMMITS: task.commits must be non-empty.
- NO_PR: task.pr_number must be set.
- NO_PROGRESS: task.progress_updates must have at least one entry.
The gateway's i_am_done previously called _run_catch_up which silently
auto-ran the full chain. That hid the missing-commits failure mode
(catch-up tried to push nothing, opened an empty PR, etc.).
Now i_am_done is strict and tells the dev exactly which prerequisite
is missing. A separate i_am_done_with_catchup verb retains the smart-
catch-up behavior for the explicit-opt-in case.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
def _make_deps(**overrides: Any) -> ChoreographerDeps:
base: dict[str, Any] = {
"task": AsyncMock(),
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
base.update(overrides)
repo = base["evidence_repo"]
for method in (
"list_unread_a2a",
"list_unread_mentions",
"list_pending_notifications",
"task_metadata_gaps",
"recent_team_activity",
"blockers_in_lane",
"journal_highlights_for_task",
):
getattr(repo, method).return_value = []
return ChoreographerDeps(**base)
def _ready_task(task_id: Any, agent_id: Any) -> MagicMock:
"""Build a task that satisfies tracing AND field-level gates."""
return MagicMock(
id=task_id,
status="in_progress",
assigned_to=agent_id,
plan={"x": 1},
branch_name="feature/backend/abc--def",
work_session_id=uuid4(),
self_verified=True,
pr_number=8,
pr_url="https://x/pr/8",
team="backend",
progress_updates=[{"message": "did x"}],
acceptance_criteria=["AC1"],
acceptance_criteria_status=[
{"criterion": "AC1", "referencing_artifact_id": "c1"}
],
commits=[{"sha": "abc"}],
documents=[],
dev_notes="",
)
# ---------------------------------------------------------------------------
# E.1 self_verified is no longer a gate (audit P1-3/D-08)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_am_done_auto_runs_submit_verification_when_in_progress() -> None:
"""Strict i_am_done auto-runs submit_verification (in_progress→verifying)
so the dev doesn't need a separate verb. The previous NOT_SELF_VERIFIED
gate required submit_for_verification which wasn't on any manifest.
"""
agent_id = uuid4()
task_id = uuid4()
t = _ready_task(task_id, agent_id)
t.self_verified = False
t.status = "in_progress"
after_verify = MagicMock(
**{**t.__dict__, "self_verified": True, "status": "verifying"}
)
after_submit = MagicMock(**{**after_verify.__dict__, "status": "awaiting_qa"})
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.submit_verification.return_value = after_verify
task_svc.submit_qa.return_value = after_submit
task_svc.qa_agent_for_team.return_value = MagicMock(
id=uuid4(), skills=[{"id": "code_review"}]
)
journal_svc = AsyncMock()
journal_svc.has_reflect_for_task.return_value = True
work_svc = AsyncMock()
work_svc.files_changed.return_value = ["foo.py"]
deps = _make_deps(task=task_svc, journal=journal_svc, work_session=work_svc)
c = Choreographer(deps)
env = await c.i_am_done(agent_id, task_id, "done")
body = env.as_dict()
assert body["error"] is None
task_svc.submit_verification.assert_awaited_once()
task_svc.submit_qa.assert_awaited_once()
# ---------------------------------------------------------------------------
# E.2 NO_COMMITS
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_am_done_blocks_when_no_commits() -> None:
agent_id = uuid4()
task_id = uuid4()
t = _ready_task(task_id, agent_id)
t.commits = []
task_svc = AsyncMock()
task_svc.get.return_value = t
journal_svc = AsyncMock()
journal_svc.has_reflect_for_task.return_value = True
deps = _make_deps(task=task_svc, journal=journal_svc)
c = Choreographer(deps)
env = await c.i_am_done(agent_id, task_id, "done")
body = env.as_dict()
assert body["error"] == "tracing_gap"
assert "NO_COMMITS" in body["missing"] or "commits" in body["missing"]
task_svc.submit_qa.assert_not_awaited()
# ---------------------------------------------------------------------------
# E.3 NO_PR
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_am_done_blocks_when_no_pr() -> None:
agent_id = uuid4()
task_id = uuid4()
t = _ready_task(task_id, agent_id)
t.pr_number = None
task_svc = AsyncMock()
task_svc.get.return_value = t
journal_svc = AsyncMock()
journal_svc.has_reflect_for_task.return_value = True
deps = _make_deps(task=task_svc, journal=journal_svc)
c = Choreographer(deps)
env = await c.i_am_done(agent_id, task_id, "done")
body = env.as_dict()
assert body["error"] == "tracing_gap"
assert "NO_PR" in body["missing"] or "pr_number" in body["missing"]
task_svc.submit_qa.assert_not_awaited()
# ---------------------------------------------------------------------------
# E.4 NO_PROGRESS
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_am_done_blocks_when_no_progress() -> None:
agent_id = uuid4()
task_id = uuid4()
t = _ready_task(task_id, agent_id)
t.progress_updates = []
task_svc = AsyncMock()
task_svc.get.return_value = t
journal_svc = AsyncMock()
journal_svc.has_reflect_for_task.return_value = True
deps = _make_deps(task=task_svc, journal=journal_svc)
c = Choreographer(deps)
env = await c.i_am_done(agent_id, task_id, "done")
body = env.as_dict()
assert body["error"] == "tracing_gap"
# progress>=1 is the existing tracing_gate Requirement key.
assert "progress>=1" in body["missing"] or "NO_PROGRESS" in body["missing"]
task_svc.submit_qa.assert_not_awaited()
# ---------------------------------------------------------------------------
# E.5 happy path: all gates pass → submit_qa runs (NO catch-up)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_am_done_proceeds_when_all_gates_pass() -> None:
agent_id = uuid4()
task_id = uuid4()
t = _ready_task(task_id, agent_id)
# Pre-verifying state (caller already ran submit_for_verification or
# task is already in `verifying`). i_am_done skips the auto-verify
# step and goes straight to submit_qa.
t.status = "verifying"
t.self_verified = True
after_submit = MagicMock(
**{**t.__dict__, "status": "awaiting_qa"},
)
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.submit_qa.return_value = after_submit
task_svc.qa_agent_for_team.return_value = MagicMock(
id=uuid4(), skills=[{"id": "code_review"}]
)
journal_svc = AsyncMock()
journal_svc.has_reflect_for_task.return_value = True
work_svc = AsyncMock()
work_svc.files_changed.return_value = ["foo.py"]
deps = _make_deps(task=task_svc, journal=journal_svc, work_session=work_svc)
c = Choreographer(deps)
env = await c.i_am_done(agent_id, task_id, "all done")
body = env.as_dict()
assert body["error"] is None
assert body["status"] == "awaiting_qa"
task_svc.submit_qa.assert_awaited_once()
# Already-verifying status: no auto-call to submit_verification.
task_svc.submit_verification.assert_not_awaited()
# ---------------------------------------------------------------------------
# E.6 — Removed: i_am_done_with_catchup verb deleted (audit P2-5/D-16).
# Its functionality is now split between submit_for_qa (push + PR) and
# i_am_done (auto-run submit_verification then submit_qa).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_i_am_done_blocks_unauthorized() -> None:
"""Existing not_authorized check still applies."""
agent_id = uuid4()
other_id = uuid4()
task_id = uuid4()
t = _ready_task(task_id, other_id)
task_svc = AsyncMock()
task_svc.get.return_value = t
deps = _make_deps(task=task_svc)
c = Choreographer(deps)
env = await c.i_am_done(agent_id, task_id, "done")
body = env.as_dict()
assert body["error"] == "not_authorized"