feat(qa): require a per-acceptance-criterion verdict before pass_review

QA may no longer pass a task with a single gestalt approval — pass_review now
takes ac_verdicts (one verification entry per acceptance criterion) and the
gateway rejects a pass that does not cover every criterion. If a criterion does
not hold, QA fails the review instead. The verdicts are folded into the
persisted qa_notes for the audit trail. Threaded through the flow MCP tool,
the HTTP request schema, and the route; QA prompt updated.
This commit is contained in:
Renn F
2026-06-14 04:44:21 +02:00
parent 8affb283f5
commit 2db11c7833
9 changed files with 174 additions and 10 deletions
@@ -494,7 +494,12 @@ async def test_full_chain_through_doc_handoff(
"Reviewed the diff; route returns 200 OK with timestamp. Tests cover "
"both acceptance criteria. Approving."
)
env = await c.pass_review(qa_agent.id, task.id, notes=qa_notes)
env = await c.pass_review(
qa_agent.id,
task.id,
notes=qa_notes,
ac_verdicts=[f"verified: {crit}" for crit in task.acceptance_criteria],
)
assert env.error is None, f"pass_review failed: {env.message}"
assert env.status == "awaiting_documentation"
+6 -1
View File
@@ -495,7 +495,12 @@ async def test_qa_pass_path(
assert str(after_claim.status) == Status.AWAITING_QA.value
assert after_claim.assigned_to == qa_agent.id
env = await c.pass_review(qa_agent.id, task.id, notes=_QA_PASS_NOTES)
env = await c.pass_review(
qa_agent.id,
task.id,
notes=_QA_PASS_NOTES,
ac_verdicts=[f"verified: {crit}" for crit in task.acceptance_criteria],
)
assert env.error is None, f"pass_review failed: {env.message}"
assert env.status == Status.AWAITING_DOCUMENTATION.value
@@ -103,6 +103,7 @@ class _MockChoreographer:
_agent_id: object,
_task_id: object,
_notes: object,
_ac_verdicts: object = None,
) -> Envelope:
self._state["task_status"] = "awaiting_documentation"
return Envelope.ok(
+67
View File
@@ -0,0 +1,67 @@
"""pass_review requires a verification for every acceptance criterion.
A gestalt "looks good" approval is how a silently-unbuilt criterion slips
through QA; the coverage gate forces one verdict per criterion before a pass is
allowed. If a criterion does not hold, QA fails the review instead.
"""
from __future__ import annotations
from types import SimpleNamespace
from roboco.services.gateway.choreographer import Choreographer
def _task(criteria: list[str]) -> SimpleNamespace:
return SimpleNamespace(acceptance_criteria=criteria)
def test_no_criteria_imposes_no_requirement() -> None:
assert Choreographer._qa_ac_coverage_check(_task([]), None) is None
def test_full_coverage_passes() -> None:
t = _task(["returns 200", "includes timestamp"])
assert (
Choreographer._qa_ac_coverage_check(t, ["200 ok via test", "ts in diff"])
is None
)
def test_partial_coverage_is_rejected() -> None:
t = _task(["a", "b", "c"])
env = Choreographer._qa_ac_coverage_check(t, ["only one verified"])
assert env is not None
body = env.as_dict()
assert body["error"] == "invalid_state", body
assert "fail_review" in (env.remediate or "")
def test_missing_verdicts_entirely_is_rejected() -> None:
t = _task(["a", "b"])
env = Choreographer._qa_ac_coverage_check(t, None)
assert env is not None
def test_blank_verdicts_do_not_count() -> None:
t = _task(["a", "b"])
env = Choreographer._qa_ac_coverage_check(t, ["a ok", " "])
assert env is not None, "whitespace-only verdict must not satisfy a criterion"
def test_extra_verdicts_are_allowed() -> None:
t = _task(["a"])
assert Choreographer._qa_ac_coverage_check(t, ["a ok", "bonus note"]) is None
def test_verdicts_fold_into_persisted_notes() -> None:
merged = Choreographer._merge_ac_verdicts_into_notes(
"base review", ["a ok", "b ok"]
)
assert "Per-criterion verification" in merged
assert "- a ok" in merged
assert "- b ok" in merged
def test_merge_with_no_verdicts_returns_notes_unchanged() -> None:
assert Choreographer._merge_ac_verdicts_into_notes("base", None) == "base"