mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(qa): require a per-acceptance-criterion verdict before pass_review
QA may no longer pass a task with a single gestalt approval — pass_review now takes ac_verdicts (one verification entry per acceptance criterion) and the gateway rejects a pass that does not cover every criterion. If a criterion does not hold, QA fails the review instead. The verdicts are folded into the persisted qa_notes for the audit trail. Threaded through the flow MCP tool, the HTTP request schema, and the route; QA prompt updated.
This commit is contained in:
@@ -494,7 +494,12 @@ async def test_full_chain_through_doc_handoff(
|
||||
"Reviewed the diff; route returns 200 OK with timestamp. Tests cover "
|
||||
"both acceptance criteria. Approving."
|
||||
)
|
||||
env = await c.pass_review(qa_agent.id, task.id, notes=qa_notes)
|
||||
env = await c.pass_review(
|
||||
qa_agent.id,
|
||||
task.id,
|
||||
notes=qa_notes,
|
||||
ac_verdicts=[f"verified: {crit}" for crit in task.acceptance_criteria],
|
||||
)
|
||||
assert env.error is None, f"pass_review failed: {env.message}"
|
||||
assert env.status == "awaiting_documentation"
|
||||
|
||||
|
||||
@@ -495,7 +495,12 @@ async def test_qa_pass_path(
|
||||
assert str(after_claim.status) == Status.AWAITING_QA.value
|
||||
assert after_claim.assigned_to == qa_agent.id
|
||||
|
||||
env = await c.pass_review(qa_agent.id, task.id, notes=_QA_PASS_NOTES)
|
||||
env = await c.pass_review(
|
||||
qa_agent.id,
|
||||
task.id,
|
||||
notes=_QA_PASS_NOTES,
|
||||
ac_verdicts=[f"verified: {crit}" for crit in task.acceptance_criteria],
|
||||
)
|
||||
assert env.error is None, f"pass_review failed: {env.message}"
|
||||
assert env.status == Status.AWAITING_DOCUMENTATION.value
|
||||
|
||||
|
||||
@@ -103,6 +103,7 @@ class _MockChoreographer:
|
||||
_agent_id: object,
|
||||
_task_id: object,
|
||||
_notes: object,
|
||||
_ac_verdicts: object = None,
|
||||
) -> Envelope:
|
||||
self._state["task_status"] = "awaiting_documentation"
|
||||
return Envelope.ok(
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""pass_review requires a verification for every acceptance criterion.
|
||||
|
||||
A gestalt "looks good" approval is how a silently-unbuilt criterion slips
|
||||
through QA; the coverage gate forces one verdict per criterion before a pass is
|
||||
allowed. If a criterion does not hold, QA fails the review instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from roboco.services.gateway.choreographer import Choreographer
|
||||
|
||||
|
||||
def _task(criteria: list[str]) -> SimpleNamespace:
|
||||
return SimpleNamespace(acceptance_criteria=criteria)
|
||||
|
||||
|
||||
def test_no_criteria_imposes_no_requirement() -> None:
|
||||
assert Choreographer._qa_ac_coverage_check(_task([]), None) is None
|
||||
|
||||
|
||||
def test_full_coverage_passes() -> None:
|
||||
t = _task(["returns 200", "includes timestamp"])
|
||||
assert (
|
||||
Choreographer._qa_ac_coverage_check(t, ["200 ok via test", "ts in diff"])
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_partial_coverage_is_rejected() -> None:
|
||||
t = _task(["a", "b", "c"])
|
||||
env = Choreographer._qa_ac_coverage_check(t, ["only one verified"])
|
||||
assert env is not None
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state", body
|
||||
assert "fail_review" in (env.remediate or "")
|
||||
|
||||
|
||||
def test_missing_verdicts_entirely_is_rejected() -> None:
|
||||
t = _task(["a", "b"])
|
||||
env = Choreographer._qa_ac_coverage_check(t, None)
|
||||
assert env is not None
|
||||
|
||||
|
||||
def test_blank_verdicts_do_not_count() -> None:
|
||||
t = _task(["a", "b"])
|
||||
env = Choreographer._qa_ac_coverage_check(t, ["a ok", " "])
|
||||
assert env is not None, "whitespace-only verdict must not satisfy a criterion"
|
||||
|
||||
|
||||
def test_extra_verdicts_are_allowed() -> None:
|
||||
t = _task(["a"])
|
||||
assert Choreographer._qa_ac_coverage_check(t, ["a ok", "bonus note"]) is None
|
||||
|
||||
|
||||
def test_verdicts_fold_into_persisted_notes() -> None:
|
||||
merged = Choreographer._merge_ac_verdicts_into_notes(
|
||||
"base review", ["a ok", "b ok"]
|
||||
)
|
||||
assert "Per-criterion verification" in merged
|
||||
assert "- a ok" in merged
|
||||
assert "- b ok" in merged
|
||||
|
||||
|
||||
def test_merge_with_no_verdicts_returns_notes_unchanged() -> None:
|
||||
assert Choreographer._merge_ac_verdicts_into_notes("base", None) == "base"
|
||||
Reference in New Issue
Block a user