mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
Resolves the 100% claim-failure rate introduced by the gateway rewrite
(commit 62bda0c plus 78 follow-ups). Live smoke runs hit
`404 /api/v2/flow/developer/...` on every dev verb plus a manifest
fallback that silently exposed off-role verbs to PMs — confirmed
firing simultaneously in NAS agent logs (be-dev-1, be-pm, main-pm).
Audit reports under docs/internal/audit_2026_05_04/ catalogue 49
defects across gateway, services, prompts, MCP transport, substrate,
and tests (8 detail reports + master synthesis). Six smoking guns;
three proven in production logs.
Phase 0 — unblock claim:
- URL prefix /api/v2/flow/dev → /developer; slug-map board roles
(product_owner, head_marketing) → /board (D-01)
- _i_will_work_on AttributeError on None across pending /
needs_revision / claimed re-entry branches (D-02)
- Seed last_heartbeat_at in _qa_or_doc_claim (D-03)
- Drop misleading i_have_committed verb; dev flow uses commit() (D-04)
- Manifest mount via compose; flow_server + do_server fail loud
instead of exposing all-verbs fallback (D-12)
- MCP _post() surfaces envelope body on 4xx so agents see remediate
hints (D-13)
on git failure so retries aren't blocked by half-state (S-01)
Phase 1 — lifecycle stability:
- _resolve_skill falls back to AgentTable.capabilities (D-06)
- main_pm_complete uses kwargs for escalate_to_ceo (D-07)
- i_am_done auto-runs submit_verification when in_progress (D-08)
- active_claimant_id wired in claim/unclaim paths — single-claimant
invariant now functional (D-05)
- qa_pass/qa_fail assert claimed_by parity with qa_agent_id (D-18)
- Prompt-drift sweep: fail() shape, i_am_done(task_id, notes),
subtask cap (12 hard / 8 soft), error-code symbology rewritten in
base.md + per-role anti-patterns (D-10/11/29/30/31, D-37)
Phase 2 — invariants + architecture:
- Real-DB integration test exercising claim → in_progress → commit
→ submit_for_qa → i_am_done → awaiting_qa (P2-1)
- choreographer.py → package; 3 of 6 role mixins extracted
(board, doc, qa). _impl.py 2,526 → 2,080 lines (-18%). Continuation
plan in docs/internal/audit_2026_05_04/p2_2_decompose_plan.md (P2-2)
- Closure guards consolidated via _subtasks_not_terminal_envelope (P2-3)
- TaskService.unclaim_for_reaper routed through canonical
_validate_and_set_status; in_progress → pending added to
VALID_TRANSITIONS (P2-4)
- Dead code removed: i_am_done_with_catchup verb, _run_catch_up helper
(P2-5)
- 6 state-machine invariants asserted via property test (P2-6)
- attempt_id (uuid4) stamped on every gateway.rejected audit row (P2-7)
- _reconcile_orphan_claims_on_startup rolls back tasks left CLAIMED
with branch_name=NULL from prior crashes (P2-8)
- scripts/regenerate_verb_tables.py introspects Pydantic schemas +
role_config; compose_prompt injects per-role tables as a layer.
Eliminates the prompt-drift class structurally (P2-9)
Other:
- D-48: orchestrator mounts host's ~/.claude.json when present so
agents don't boot from backup recovery on every spawn
- D-49: dev dispatcher rejects role-mismatched spawns (e.g. doc task
assigned to dev agent)
Tests: 553 pass · ruff + mypy clean. Live NAS smoke verification
pending — needs the stack brought back up.
350 lines
12 KiB
Python
350 lines
12 KiB
Python
"""Real-DB end-to-end test driving the gateway through the full lifecycle.
|
|
|
|
Audit deliverable P2-1: the missing integration test that would have
|
|
caught every smoking gun in the 2026-05-04 audit. Drives a single task
|
|
from pending → completed using a real `db_session` (Postgres-backed
|
|
fixture from the top-level conftest), a real `Choreographer`, and a
|
|
real `TaskService`. Git is replaced with a deterministic stub
|
|
(`_StubGit`) that mutates the same task row the choreographer reads,
|
|
so PR/commit state is consistent between the choreographer and the
|
|
test's assertions. Journal/A2A/audit/evidence are mocked because they
|
|
don't gate the lifecycle paths under test.
|
|
|
|
When extended to all roles, this test catches:
|
|
- URL prefix mismatch (route-level coverage in test_v2_role_dep)
|
|
- i_will_work_on AttributeError on None (claim → start sequence is real)
|
|
- heartbeat seeding (reaper cutoff)
|
|
- active_claimant_id wired (single-claimant invariant)
|
|
- i_am_done auto-runs submit_verification (P1-3)
|
|
- QA pass clears active_claimant_id (P1-4)
|
|
- branch creation atomicity rollback (P0-7)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Any
|
|
from unittest.mock import AsyncMock
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from roboco.db.tables import AgentTable, ProjectTable, TaskTable
|
|
from roboco.models.base import (
|
|
AgentRole,
|
|
AgentStatus,
|
|
TaskNature,
|
|
TaskStatus,
|
|
TaskType,
|
|
Team,
|
|
)
|
|
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
|
from roboco.services.task import TaskService
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import AsyncIterator
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
|
|
_BRANCH = "feature/backend/healthz"
|
|
_PR_NUMBER = 8
|
|
_PR_URL = "https://github.com/example/life/pull/8"
|
|
|
|
|
|
class _StubGit:
|
|
"""Deterministic GitService stub.
|
|
|
|
Mutates the test's TaskTable row directly to mirror what the real
|
|
`git.create_pr` / `git.commit` do via `_record_pr_atomically` and
|
|
`_workspace_for_branch`. The choreographer reads pr_number/commits
|
|
off the task object — keeping them in sync here means the gates
|
|
behave the same as production without any disk or network I/O.
|
|
"""
|
|
|
|
def __init__(self, session: Any, task: TaskTable) -> None:
|
|
self._session = session
|
|
self._task = task
|
|
|
|
async def commit(
|
|
self,
|
|
*,
|
|
branch_name: str,
|
|
message: str,
|
|
task_id: UUID,
|
|
files: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
del branch_name, files
|
|
sha = uuid4().hex[:40]
|
|
commits = list(self._task.commits or [])
|
|
commits.append({"sha": sha, "message": message, "task_id": str(task_id)})
|
|
self._task.commits = commits # type: ignore[assignment]
|
|
await self._session.flush()
|
|
return {
|
|
"sha": sha,
|
|
"message": message,
|
|
"files_changed": 1,
|
|
"insertions": 1,
|
|
"deletions": 0,
|
|
}
|
|
|
|
async def push_branch(self, branch_name: str) -> tuple[str, int]:
|
|
del branch_name
|
|
return ("ok", 0)
|
|
|
|
async def create_pr(
|
|
self, branch_name: str, *, parent: str, is_root_pr: bool
|
|
) -> dict[str, Any]:
|
|
del branch_name, parent
|
|
self._task.pr_number = _PR_NUMBER
|
|
self._task.pr_url = _PR_URL
|
|
await self._session.flush()
|
|
return {"pr_number": _PR_NUMBER, "pr_url": _PR_URL, "is_root_pr": is_root_pr}
|
|
|
|
async def diff(self, *, branch_name: str) -> str: # noqa: ARG002
|
|
return "stub diff"
|
|
|
|
async def pr_target(self, pr_number: int) -> str: # noqa: ARG002
|
|
return "main"
|
|
|
|
async def pr_merge(self, **kwargs: Any) -> dict[str, Any]:
|
|
del kwargs
|
|
return {"merged": True, "sha": uuid4().hex[:40]}
|
|
|
|
|
|
def _mock_evidence_repo() -> Any:
|
|
repo = AsyncMock()
|
|
for method in (
|
|
"list_unread_a2a",
|
|
"list_unread_mentions",
|
|
"list_pending_notifications",
|
|
"task_metadata_gaps",
|
|
"recent_team_activity",
|
|
"blockers_in_lane",
|
|
"journal_highlights_for_task",
|
|
):
|
|
getattr(repo, method).return_value = []
|
|
return repo
|
|
|
|
|
|
def _mock_journal_with_reflect() -> Any:
|
|
"""Journal stub that reports reflect/learning/decision entries present."""
|
|
journal = AsyncMock()
|
|
journal.has_reflect_for_task.return_value = True
|
|
journal.has_learning_for_task.return_value = True
|
|
journal.has_decision_for_task.return_value = True
|
|
return journal
|
|
|
|
|
|
def _mock_work_session() -> Any:
|
|
"""WorkSession stub: empty file list, no unpushed commits."""
|
|
ws = AsyncMock()
|
|
ws.files_changed.return_value = ["roboco/api/routes/health.py"]
|
|
ws.has_unpushed_commits.return_value = False
|
|
return ws
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def lifecycle_setup(
|
|
db_session: AsyncSession,
|
|
) -> AsyncIterator[dict[str, Any]]:
|
|
"""Seed a project + dev agent + a single pending task ready to claim."""
|
|
system_agent = AgentTable(
|
|
id=uuid4(),
|
|
name="System",
|
|
slug=f"system-{uuid4().hex[:8]}",
|
|
role=AgentRole.SYSTEM,
|
|
team=None,
|
|
status=AgentStatus.ACTIVE,
|
|
model_config={},
|
|
system_prompt="system",
|
|
capabilities=[],
|
|
permissions={},
|
|
metrics={},
|
|
)
|
|
db_session.add(system_agent)
|
|
await db_session.flush()
|
|
|
|
project = ProjectTable(
|
|
id=uuid4(),
|
|
name="Lifecycle Test Project",
|
|
slug=f"life-{uuid4().hex[:8]}",
|
|
git_url="https://github.com/example/life.git",
|
|
default_branch="main",
|
|
protected_branches=["main"],
|
|
assigned_cell=Team.BACKEND,
|
|
created_by=system_agent.id,
|
|
is_active=True,
|
|
)
|
|
db_session.add(project)
|
|
await db_session.flush()
|
|
|
|
dev_agent = AgentTable(
|
|
id=uuid4(),
|
|
name="BE Dev",
|
|
slug=f"be-dev-{uuid4().hex[:8]}",
|
|
role=AgentRole.DEVELOPER,
|
|
team=Team.BACKEND,
|
|
status=AgentStatus.ACTIVE,
|
|
model_config={},
|
|
system_prompt="dev",
|
|
capabilities=["python"],
|
|
permissions={},
|
|
metrics={},
|
|
)
|
|
qa_agent = AgentTable(
|
|
id=uuid4(),
|
|
name="BE QA",
|
|
slug=f"be-qa-{uuid4().hex[:8]}",
|
|
role=AgentRole.QA,
|
|
team=Team.BACKEND,
|
|
status=AgentStatus.ACTIVE,
|
|
model_config={},
|
|
system_prompt="qa",
|
|
capabilities=["review"],
|
|
permissions={},
|
|
metrics={},
|
|
)
|
|
db_session.add_all([dev_agent, qa_agent])
|
|
await db_session.flush()
|
|
|
|
task = TaskTable(
|
|
id=uuid4(),
|
|
title="Add /healthz endpoint",
|
|
description="Return 200 OK from /healthz",
|
|
status=TaskStatus.PENDING,
|
|
priority=2,
|
|
task_type=TaskType.CODE,
|
|
nature=TaskNature.TECHNICAL,
|
|
team=Team.BACKEND,
|
|
project_id=project.id,
|
|
created_by=system_agent.id,
|
|
assigned_to=dev_agent.id,
|
|
branch_name=_BRANCH,
|
|
acceptance_criteria=["Returns 200", "Includes timestamp"],
|
|
acceptance_criteria_status=[
|
|
{"criterion": "Returns 200", "referencing_artifact_id": "stub"},
|
|
{"criterion": "Includes timestamp", "referencing_artifact_id": "stub"},
|
|
],
|
|
)
|
|
db_session.add(task)
|
|
await db_session.flush()
|
|
|
|
yield {
|
|
"project": project,
|
|
"dev_agent": dev_agent,
|
|
"qa_agent": qa_agent,
|
|
"task": task,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dev_can_claim_pending_task_via_gateway(
|
|
db_session: AsyncSession, lifecycle_setup: dict[str, Any]
|
|
) -> None:
|
|
"""give_me_work → i_will_work_on lands the task in in_progress.
|
|
|
|
Verifies in one shot: P0-2 (None-handling), P0-3 (heartbeat seed),
|
|
P0-7 (branch atomicity), P1-4 (active_claimant_id wired).
|
|
"""
|
|
task = lifecycle_setup["task"]
|
|
dev_agent = lifecycle_setup["dev_agent"]
|
|
task_service = TaskService(db_session)
|
|
|
|
deps = ChoreographerDeps(
|
|
task=task_service,
|
|
work_session=_mock_work_session(),
|
|
git=_StubGit(db_session, task),
|
|
a2a=AsyncMock(),
|
|
journal=_mock_journal_with_reflect(),
|
|
audit=AsyncMock(),
|
|
evidence_repo=_mock_evidence_repo(),
|
|
)
|
|
c = Choreographer(deps)
|
|
|
|
env = await c.i_will_work_on(dev_agent.id, task.id, plan="add the route")
|
|
|
|
assert env.error is None, f"claim failed: {env.message}"
|
|
assert env.status == "in_progress"
|
|
|
|
refreshed = await task_service.get(task.id)
|
|
assert refreshed is not None
|
|
assert str(refreshed.status) == "in_progress"
|
|
assert refreshed.assigned_to == dev_agent.id
|
|
assert refreshed.last_heartbeat_at is not None, "P0-3: heartbeat seed"
|
|
assert refreshed.active_claimant_id == dev_agent.id, "P1-4: claim lock"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dev_full_chain_through_awaiting_qa(
|
|
db_session: AsyncSession, lifecycle_setup: dict[str, Any]
|
|
) -> None:
|
|
"""claim → commit → submit_for_qa → i_am_done lands in awaiting_qa.
|
|
|
|
Drives the full developer-side closure path. Verifies:
|
|
- submit_for_qa records pr_number on the task (commits + PR pre-flight)
|
|
- i_am_done auto-runs submit_verification (P1-3) → verifying → awaiting_qa
|
|
- Heartbeat refreshes after each verb (`_touch`)
|
|
- active_claimant_id remains set through dev's tenure
|
|
"""
|
|
task = lifecycle_setup["task"]
|
|
dev_agent = lifecycle_setup["dev_agent"]
|
|
task_service = TaskService(db_session)
|
|
stub_git = _StubGit(db_session, task)
|
|
|
|
deps = ChoreographerDeps(
|
|
task=task_service,
|
|
work_session=_mock_work_session(),
|
|
git=stub_git,
|
|
a2a=AsyncMock(),
|
|
journal=_mock_journal_with_reflect(),
|
|
audit=AsyncMock(),
|
|
evidence_repo=_mock_evidence_repo(),
|
|
)
|
|
c = Choreographer(deps)
|
|
|
|
# 1. Claim
|
|
env = await c.i_will_work_on(dev_agent.id, task.id, plan="add the route")
|
|
assert env.error is None
|
|
assert env.status == "in_progress"
|
|
|
|
# 2. Commit (via stub git directly + record progress on task — the gateway
|
|
# path through ContentActions.commit calls task.add_progress, which we
|
|
# simulate here so submit_for_qa's commits-precondition is satisfied).
|
|
await stub_git.commit(
|
|
branch_name=_BRANCH,
|
|
message=f"[{str(task.id)[:8]}] feat(api): add /healthz",
|
|
task_id=task.id,
|
|
)
|
|
await task_service.add_progress(task.id, dev_agent.id, "implemented /healthz")
|
|
|
|
# 3. submit_for_qa — push + open PR. After this, task.pr_number is set.
|
|
env = await c.submit_for_qa(dev_agent.id, task.id)
|
|
assert env.error is None, f"submit_for_qa failed: {env.message}"
|
|
refreshed = await task_service.get(task.id)
|
|
assert refreshed is not None
|
|
assert refreshed.pr_number == _PR_NUMBER, "P0-7 / S-02: PR recorded on task"
|
|
|
|
# 4. i_am_done — auto-runs in_progress → verifying → awaiting_qa.
|
|
env = await c.i_am_done(dev_agent.id, task.id, "tests pass; route works")
|
|
assert env.error is None, f"i_am_done failed: {env.message}"
|
|
assert env.status == "awaiting_qa", (
|
|
"P1-3: i_am_done must auto-run submit_verification + submit_qa"
|
|
)
|
|
|
|
final = await task_service.get(task.id)
|
|
assert final is not None
|
|
assert str(final.status) == "awaiting_qa"
|
|
assert final.self_verified is True, "P1-3: self_verified set by auto-verify"
|
|
|
|
|
|
# TODO P2-1 follow-up — extend the chain past awaiting_qa:
|
|
# - QA: claim_review → pass → awaiting_documentation
|
|
# - Documenter: claim_doc_task → i_documented → awaiting_pm_review
|
|
# - Cell PM: complete on the leaf → completed (or submit_up to a parent)
|
|
# - Main PM: complete on the root → awaiting_ceo_approval
|
|
# Each stage needs the role's agent seeded (lifecycle_setup already has
|
|
# dev + qa; add doc + cell_pm + main_pm) plus journal entries with the
|
|
# right scope (pass needs journal:learning; complete needs journal:decision).
|
|
# The _StubGit class above already covers commit/push/pr_create/pr_target
|
|
# /pr_merge for the merge stages.
|