Files
roboco/tests/unit/api/routes/v1/test_flow_qa.py
T
cea3e56628 feat(lifecycle): revision findings ledger — structured failure feedback, persisted and delivered down the chain (#486)
* feat(lifecycle): revision findings ledger — structured QA/PR/PM/CEO failure feedback, persisted and delivered down the chain

Every bounce used to survive only as flattened prose: rounds overwrote each
other in notes_structured, request_changes persisted nothing, two raw
dev_notes appends were silently destroyed by the next handoff note, and the
dev prompt pointed at fields (qa_notes via evidence(), pm_notes) the API
never delivered. Agents re-interpreted and re-discovered every failure
before they could start fixing it.

- task_review_findings (migration 071, append-only): file/line/severity/
  criterion(AC-id-validated)/expected/actual/fix/evidence per finding, with
  origin (qa|pr_gate|pm|ceo), round, and an open->addressed->verified
  lifecycle (waived reserved); new tasks.pm_notes + PmReviewContent give
  request_changes a structured home
- producers: fail_review/pr_fail/request_changes take findings=[...] (prose
  issues shimmed+merged for one release, deprecation-logged); ceo_reject
  validates its reason (no 500), lands an origin=ceo finding, and bumps
  round+audit on branchless coordination roots; guardrails at the verb
  chokepoint (nudge >5, hard reject >10, field caps, traversal-safe file);
  the dev_notes data-loss appends are removed; new task.request_changes +
  task.ceo_reject audit events close rework attribution
- delivery: qa_notes/pr_reviewer_notes/pm_notes carry the deterministic
  [F-id8] rendering; claim briefings, evidence(), the REVISION_REQUIRED
  spawn prompt, PM triage bounced-blocks, and A2A bodies deliver open
  findings; round-N+1 QA and gate reviewers get the full prior ledger;
  panel Findings tab + bounced-xN chip; metrics pm_rejects/ceo_rejects +
  findings counts; vault task notes render a Findings section (fail-open)
- resolution closes for every origin: i_am_done and submit_up/submit_root
  take resolved_findings gated by FINDINGS_ADDRESSED (owner-gated so a
  stale non-owner PM can never mutate the ledger); pass_review/pr_pass/
  complete verify-stamp same-transaction; ceo_approve stamps best-effort
- 24 real-DB integration tests drive the full loop through the real
  choreographer; full suite 12856 green

* docs: revision findings ledger sweep — CLAUDE.md, map, RAG corpus

- CLAUDE.md: new ledger section + corrected request_changes row
- docs/map/review-findings.md (new subsystem map) + surgical updates to
  task-service/pr-gate-review/metrics-observability/vault/panel maps
- docs/rag: producers' findings contract across qa/pr-reviewer/developer/
  cell-pm/main-pm/ceo role docs (the PM docs were missing request_changes
  entirely), verb references, and a new architecture/review-findings.md
  disambiguating ledger findings from convention findings

* test(e2e): resubmit resolves the pr_fail finding per the ledger contract

The scripted pr_fail revision loop resubmitted submit_up without
resolved_findings — correctly rejected now that FINDINGS_ADDRESSED gates
the PM resubmit verbs (green locally, red only in CI since the e2e suite
skips without ROBOCO_E2E_SMOKE=1). The scripted PM now reads the open
ledger row pr_fail persisted (new open_finding_ids arc helper) and
resolves it on resubmit, asserting the open set drains — exercising the
coordinator half of the new contract end to end.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
2026-07-11 22:54:42 +02:00

262 lines
8.3 KiB
Python

"""Unit tests for /api/v1/flow/qa/* endpoints.
Uses a minimal FastAPI test client built from the new router only.
No DB required — Choreographer is mocked.
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from roboco.api.deps import get_choreographer
from roboco.api.routes.v1.flow_qa import router
_HTTP_200 = 200
_HTTP_422 = 422
_AGENT_ID = str(uuid4())
_TASK_ID = str(uuid4())
_HEADERS = {"X-Agent-ID": _AGENT_ID, "X-Agent-Role": "qa"}
def _make_envelope(
status: str = "ok", task_id: str | None = None, **extra: object
) -> MagicMock:
"""Return a mock Envelope whose as_dict() returns a predictable payload."""
env = MagicMock()
payload: dict[str, object] = {"status": status, "task_id": task_id, "next": "..."}
payload.update(extra)
env.as_dict.return_value = payload
return env
def _build_app(mock_choreographer: MagicMock) -> FastAPI:
"""Build minimal FastAPI app with the flow_qa router and a mocked dep."""
app = FastAPI()
app.include_router(router)
app.dependency_overrides[get_choreographer] = lambda: mock_choreographer
return app
@pytest.mark.asyncio
async def test_give_me_work_returns_envelope() -> None:
"""POST /api/v1/flow/qa/give_me_work returns 200 with envelope shape."""
mock_chore = MagicMock()
mock_chore.give_me_work = AsyncMock(return_value=_make_envelope(status="idle"))
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/give_me_work",
json={},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
body = resp.json()
assert body["status"] == "idle"
mock_chore.give_me_work.assert_awaited_once()
@pytest.mark.asyncio
async def test_claim_review_dispatches_task_id() -> None:
"""POST /api/v1/flow/qa/claim_review returns 200 with evidence.pr_url in body."""
mock_chore = MagicMock()
mock_chore.claim_review = AsyncMock(
return_value=_make_envelope(
status="claimed",
task_id=_TASK_ID,
evidence={"pr_url": "https://github.com/org/repo/pull/42"},
)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/claim_review",
json={"task_id": _TASK_ID},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
body = resp.json()
assert body["evidence"]["pr_url"] == "https://github.com/org/repo/pull/42"
mock_chore.claim_review.assert_awaited_once()
call_args = mock_chore.claim_review.call_args
assert str(call_args.args[1]) == _TASK_ID
@pytest.mark.asyncio
async def test_pass_review_with_notes_returns_awaiting_documentation() -> None:
"""POST /api/v1/flow/qa/pass with notes returns 200 with awaiting_documentation."""
mock_chore = MagicMock()
mock_chore.pass_review = AsyncMock(
return_value=_make_envelope(status="awaiting_documentation", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/pass",
json={
"task_id": _TASK_ID,
"notes": "All acceptance criteria met, tests green.",
},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
body = resp.json()
assert body["status"] == "awaiting_documentation"
mock_chore.pass_review.assert_awaited_once()
call_args = mock_chore.pass_review.call_args
assert call_args.args[2] == "All acceptance criteria met, tests green."
@pytest.mark.asyncio
async def test_pass_review_short_notes_returns_tracing_gap_envelope() -> None:
"""POST /api/v1/flow/qa/pass with minimal notes relies on choreographer to gate."""
mock_chore = MagicMock()
mock_chore.pass_review = AsyncMock(
return_value=_make_envelope(status="tracing_gap", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/pass",
json={"task_id": _TASK_ID, "notes": "ok"},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
body = resp.json()
assert body["status"] == "tracing_gap"
mock_chore.pass_review.assert_awaited_once()
@pytest.mark.asyncio
async def test_fail_review_with_issues_returns_needs_revision() -> None:
"""POST /api/v1/flow/qa/fail with issues returns 200 with needs_revision status."""
mock_chore = MagicMock()
mock_chore.fail_review = AsyncMock(
return_value=_make_envelope(status="needs_revision", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/fail",
json={
"task_id": _TASK_ID,
"issues": ["Missing error handling", "No unit tests"],
},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
body = resp.json()
assert body["status"] == "needs_revision"
mock_chore.fail_review.assert_awaited_once()
call_args = mock_chore.fail_review.call_args
assert call_args.args[2] == ["Missing error handling", "No unit tests"]
@pytest.mark.asyncio
async def test_fail_review_accepts_empty_issues_when_findings_given() -> None:
"""POST /api/v1/flow/qa/fail with issues=[] is schema-valid — issues is now
optional (deprecated free-text shim); findings carries the structured
revision-findings ledger entry instead. The "at least one of the two"
rule is enforced by the choreographer, not the HTTP schema."""
mock_chore = MagicMock()
mock_chore.fail_review = AsyncMock(
return_value=_make_envelope(status="needs_revision", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/fail",
json={
"task_id": _TASK_ID,
"issues": [],
"findings": [
{
"expected": "returns 200",
"actual": "returns 500",
"severity": "major",
}
],
},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
mock_chore.fail_review.assert_awaited_once()
call_args = mock_chore.fail_review.call_args
assert call_args.args[3][0]["actual"] == "returns 500"
@pytest.mark.asyncio
async def test_unclaim_dispatches_task_id() -> None:
mock_chore = MagicMock()
mock_chore.unclaim = AsyncMock(
return_value=_make_envelope(status="awaiting_qa", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/unclaim",
json={"task_id": _TASK_ID},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
mock_chore.unclaim.assert_awaited_once()
@pytest.mark.asyncio
async def test_resume_dispatches_task_id() -> None:
mock_chore = MagicMock()
mock_chore.resume = AsyncMock(
return_value=_make_envelope(status="claimed", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/resume",
json={"task_id": _TASK_ID},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
mock_chore.resume.assert_awaited_once()
@pytest.mark.asyncio
async def test_i_am_idle_dispatches_agent_id() -> None:
mock_chore = MagicMock()
mock_chore.i_am_idle = AsyncMock(return_value=_make_envelope(status="idle"))
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/i_am_idle",
json={},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
mock_chore.i_am_idle.assert_awaited_once()
@pytest.mark.asyncio
async def test_i_am_blocked_dispatches_to_choreographer() -> None:
"""POST /api/v1/flow/qa/i_am_blocked returns an envelope (the QA manifest
registers i_am_blocked) rather than a raw 404."""
mock_chore = MagicMock()
mock_chore.i_am_blocked = AsyncMock(
return_value=_make_envelope(status="blocked", task_id=_TASK_ID)
)
client = TestClient(_build_app(mock_chore))
resp = client.post(
"/api/v1/flow/qa/i_am_blocked",
json={"task_id": _TASK_ID, "reason": "PR diff won't load"},
headers=_HEADERS,
)
assert resp.status_code == _HTTP_200
body = resp.json()
assert body["status"] == "blocked"
mock_chore.i_am_blocked.assert_awaited_once()