Files
roboco/tests/unit/gateway/test_possibilities_matrix.py
T
aa15dc40cc Feature/video artifact verification (#537)
* fix(release): CI wait polls the prod rung; escape the header tooltip apostrophe

get_latest_ci_conclusion defaults to the ladder's head rung, so
wait_for_ci searched slave for a release commit that lives on master
and timed out after 40 minutes with the run already green. The wait
now passes the prod branch explicitly. Also fixes the
react/no-unescaped-entities error that turned master's Panel CI red.

* fix(panel,video): dead dialog triggers behind tooltips; dotted composition ids render

HelpTip nested inside a Dialog/AlertDialog trigger puts the trigger's
click handler on the Tooltip root, which renders no DOM — the agents
Spawn item and the KB Reindex-All / Delete-index confirms were dead.
Tooltips now wrap the triggers. The video renderer accepts interior
single dots in composition ids (release-0.25.0) with '..' still
unrepresentable, and propose_video refuses an unrenderable id at
authoring time.

* fix(dispatch): restart-safe PM review turns

A leaf task in awaiting_pm_review had no periodic pickup: the closure
dispatcher bailed on childless tasks and skipped PR-bearing review
tasks as already-promoted, assuming the submit-time PM session was
still alive — an assumption every restart breaks. Proven live on the
docs-sync leaf after the 0.25.0 redeploy, which also dependency-blocked
its sibling dev task. Childless awaiting_pm_review tasks now flow to
the PM's review turn, and the merge turn respawns its PM when none is
active.

* feat(video): verify the rendered artifact, not the source

The 14s release-0.25.0 cut shipped with only one of four scenes visibly
registering: the dev authored DOM, the smoke asserted DOM, QA read code —
nobody consumed the rendered MP4 before the CEO did. Close that loop, and
the reject loop behind it:

- sidecar frames mode: POST /render with frames=1..32 renders the cut,
  ffprobes the REAL duration, extracts midpoint-sampled keyframe PNGs
  (timestamps in filenames), streams a tar.gz back with X-Video-Duration
- request_render do-verb (developer/QA, request_sandbox's shape): renders
  the caller's ACTUAL composition — dev's own worktree (head_sha/dirty
  provenance), QA a read-only git-archive export of the assembled branch —
  extracts frames to the container-shared .previews/ path, stamps the
  render_preview marker, returns the paths as envelope evidence
- gate: i_am_done on a source=video task refuses without a stamped
  render_preview (Requirement.RENDER_VERIFIED; canonical source string
  moved to foundation as markers.VIDEO_TASK_SOURCE; mirrored in the
  possibilities-matrix fast path so it cannot bypass the check)
- QA claim_review evidence carries video_context (composition id, the
  dev's preview, a re-render instruction) so review checks output
- dev spawn prompt block + a 4th authoring AC order Read-every-frame
  verification before submitting
- reject -> re-author: a CEO reject with a reason opens a fresh authoring
  task carrying the verbatim feedback + a revise-in-place pointer at the
  existing composition (best-effort, never fails the reject) — rejection
  feedback no longer dies on the cancelled draft

E2E: rendered the committed release-0.25.0 composition through the new
frames mode locally — the returned keyframes show exactly the reported
failure (blank frame at 5.8s, only 'Env ladder' by 12.8s), the check the
fleet was missing.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
2026-07-16 19:49:26 +02:00

523 lines
17 KiB
Python

"""W7 possibilities matrix: the ``_work_appears_done`` predicate + fast path.
A task whose work is already done (commits + PR open + every acceptance
criterion addressed + no open findings) qualifies for the fast path. These
tests pin the predicate's truth table and the schema it actually reads (the
per-criterion rows the writer persists use ``artifact_ref``; the predicate
unions ``artifact_ref`` / ``referencing_artifact_id`` / ``addressed`` so it
sees real data, not the latent reader/writer key drift).
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.foundation.policy.content import markers
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
from roboco.services.gateway.envelope import Envelope
def _deps() -> ChoreographerDeps:
return ChoreographerDeps(
task=AsyncMock(),
work_session=AsyncMock(),
git=AsyncMock(),
a2a=AsyncMock(),
journal=AsyncMock(),
audit=AsyncMock(),
evidence_repo=AsyncMock(),
)
async def _no_findings(_task_id: Any) -> tuple[()]:
return ()
async def _one_open(_task_id: Any) -> tuple[str, ...]:
return ("f1abcd12",)
def _t(
*,
status: str = "claimed",
commits: tuple[int, ...] = (1,),
pr_created: bool = True,
criteria: tuple[str, ...] = ("ac1", "ac2"),
ac_status: list[dict[str, Any]] | None = None,
) -> MagicMock:
if ac_status is None:
ac_status = [
{"criterion": "ac1", "addressed": True, "artifact_ref": "sha1"},
{"criterion": "ac2", "addressed": True, "artifact_ref": "sha2"},
]
t = MagicMock()
t.id = uuid4()
t.status = status
t.commits = commits
t.pr_created = pr_created
t.pr_number = 12345 if pr_created else None
t.acceptance_criteria = list(criteria)
t.acceptance_criteria_status = ac_status
t.source = "code"
t.orchestration_markers = None
return t
@pytest.mark.asyncio
async def test_work_appears_done_true_when_all_hold(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
assert await c._work_appears_done(_t()) is True
@pytest.mark.asyncio
async def test_work_appears_done_false_when_no_pr(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
assert await c._work_appears_done(_t(pr_created=False)) is False
@pytest.mark.asyncio
async def test_work_appears_done_false_when_no_commits(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
assert await c._work_appears_done(_t(commits=())) is False
@pytest.mark.asyncio
async def test_work_appears_done_false_when_ac_unaddressed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
t = _t(
ac_status=[
{"criterion": "ac1", "addressed": True, "artifact_ref": "sha1"},
{"criterion": "ac2", "addressed": False, "artifact_ref": None},
]
)
assert await c._work_appears_done(t) is False
@pytest.mark.asyncio
async def test_work_appears_done_true_with_no_criteria(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
assert await c._work_appears_done(_t(criteria=(), ac_status=[])) is True
@pytest.mark.asyncio
async def test_work_appears_done_false_when_open_finding(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _one_open)
assert await c._work_appears_done(_t()) is False
@pytest.mark.asyncio
async def test_work_appears_done_false_when_terminal_status(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
assert await c._work_appears_done(_t(status="awaiting_qa")) is False
assert await c._work_appears_done(_t(status="completed")) is False
assert await c._work_appears_done(_t(status="needs_revision")) is False
@pytest.mark.asyncio
async def test_work_appears_done_reads_referencing_artifact_id_schema(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_open_finding_ids", _no_findings)
t = _t(
ac_status=[
{"criterion": "ac1", "referencing_artifact_id": "sha1"},
{"criterion": "ac2", "referencing_artifact_id": "sha2"},
]
)
assert await c._work_appears_done(t) is True
# ---------------------------------------------------------------------------
# _fast_path_quality_verdict: CI-green proxy for the skipped local gate
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_quality_verdict_ci_success_skips_local_gate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(
c, "_resolve_ci_status", AsyncMock(return_value={"state": "success"})
)
local = AsyncMock(return_value=None)
monkeypatch.setattr(c, "_check_quality_gate", local)
rejection, ran_local = await c._fast_path_quality_verdict(
MagicMock(task_id=uuid4(), task=MagicMock(), briefing={})
)
assert rejection is None
assert ran_local is False
local.assert_not_awaited()
@pytest.mark.asyncio
async def test_quality_verdict_ci_failure_refuses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(
c,
"_resolve_ci_status",
AsyncMock(return_value={"state": "failure", "failing_checks": ["lint"]}),
)
local = AsyncMock(return_value=None)
monkeypatch.setattr(c, "_check_quality_gate", local)
rejection, ran_local = await c._fast_path_quality_verdict(
MagicMock(task_id=uuid4(), task=MagicMock(), briefing={})
)
assert rejection is not None
assert ran_local is False
local.assert_not_awaited()
@pytest.mark.asyncio
async def test_quality_verdict_no_ci_falls_back_to_local_gate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(
c, "_resolve_ci_status", AsyncMock(return_value={"state": "no_ci_configured"})
)
local = AsyncMock(return_value=None)
monkeypatch.setattr(c, "_check_quality_gate", local)
rejection, ran_local = await c._fast_path_quality_verdict(
MagicMock(task_id=uuid4(), task=MagicMock(), briefing={})
)
assert rejection is None
assert ran_local is True
local.assert_awaited_once()
@pytest.mark.asyncio
async def test_quality_verdict_unresolvable_falls_back_to_local_gate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
monkeypatch.setattr(c, "_resolve_ci_status", AsyncMock(return_value=None))
local = AsyncMock(return_value=None)
monkeypatch.setattr(c, "_check_quality_gate", local)
rejection, ran_local = await c._fast_path_quality_verdict(
MagicMock(task_id=uuid4(), task=MagicMock(), briefing={})
)
assert rejection is None
assert ran_local is True
local.assert_awaited_once()
# ---------------------------------------------------------------------------
# _i_am_done_fast_path: gate ordering + transition chain (skips rich plan)
# ---------------------------------------------------------------------------
def _ctx(status: str = "claimed") -> Any:
ctx = MagicMock()
ctx.agent_id = uuid4()
ctx.task_id = uuid4()
ctx.task = _t(status=status)
ctx.briefing = {}
ctx.notes = "done"
ctx.resolved_findings = None
ctx.role_str = "developer"
return ctx
@dataclass
class _FastPathMocks:
"""Typed handles to the patched verb-boundary mocks. mypy keeps the
declared method types on ``c.<method>`` even after ``monkeypatch.setattr``
(the runtime override is invisible to static analysis), so assertions
must go through these locals typed as ``AsyncMock`` — not ``c.<method>``."""
ok: AsyncMock
reject: AsyncMock
conventions: AsyncMock
open_findings: AsyncMock
record_milestone: AsyncMock
toolchain: AsyncMock
def _stub_fast_path(
c: Choreographer, monkeypatch: pytest.MonkeyPatch, quality_rejection: Any = None
) -> _FastPathMocks:
conventions = AsyncMock(return_value=None)
open_findings = AsyncMock(return_value=())
ok = AsyncMock(return_value="OK")
reject = AsyncMock(return_value="REJECT")
record_milestone = AsyncMock(return_value=None)
toolchain = AsyncMock(return_value=None)
monkeypatch.setattr(c, "_apply_resolved_findings", AsyncMock(return_value=None))
monkeypatch.setattr(c, "_check_submit_qa_field_gates", AsyncMock(return_value=None))
monkeypatch.setattr(c, "_behind_base_gate", AsyncMock(return_value=None))
monkeypatch.setattr(c, "_ensure_branch_pushed", AsyncMock(return_value=None))
monkeypatch.setattr(c, "_conventions_gate", conventions)
monkeypatch.setattr(c, "_open_finding_ids", open_findings)
monkeypatch.setattr(
c,
"_fast_path_quality_verdict",
AsyncMock(return_value=(quality_rejection, False)),
)
monkeypatch.setattr(c, "_toolchain_broken_guard", toolchain)
monkeypatch.setattr(c, "_notify_qa", AsyncMock(return_value=None))
monkeypatch.setattr(c, "_touch", AsyncMock(return_value=None))
monkeypatch.setattr(c, "_record_milestone_progress", record_milestone)
monkeypatch.setattr(c, "_build_i_am_done_ok", ok)
monkeypatch.setattr(c, "_reject_i_am_done", reject)
return _FastPathMocks(
ok=ok,
reject=reject,
conventions=conventions,
open_findings=open_findings,
record_milestone=record_milestone,
toolchain=toolchain,
)
@pytest.mark.asyncio
async def test_fast_path_claimed_starts_without_set_plan(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
await c._i_am_done_fast_path(_ctx(status="claimed"))
stubs.ok.assert_awaited_once() # OK path taken, no rejection
c.task.start.assert_awaited_once() # claimed -> in_progress
c.task.set_plan.assert_not_awaited() # rich plan SKIPPED (no set_plan)
c.task.submit_verification.assert_awaited_once()
c.task.submit_qa.assert_awaited_once()
stubs.conventions.assert_awaited_once() # conventions KEPT
stubs.record_milestone.assert_awaited_once()
@pytest.mark.asyncio
async def test_fast_path_in_progress_skips_start(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
await c._i_am_done_fast_path(_ctx(status="in_progress"))
stubs.ok.assert_awaited_once()
c.task.start.assert_not_awaited() # already in_progress
c.task.submit_verification.assert_awaited_once()
c.task.submit_qa.assert_awaited_once()
@pytest.mark.asyncio
async def test_fast_path_open_findings_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
monkeypatch.setattr(c, "_open_finding_ids", AsyncMock(return_value=("f1abcd12",)))
await c._i_am_done_fast_path(_ctx())
stubs.reject.assert_awaited_once()
c.task.submit_qa.assert_not_awaited()
@pytest.mark.asyncio
async def test_fast_path_video_task_without_render_preview_rejects(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A video-authoring task with no request_render preview must not be
able to fast-path around looking at the rendered artifact."""
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
tracing_gap = AsyncMock(
return_value=Envelope.tracing_gap(missing=["render_preview"], remediate="x")
)
monkeypatch.setattr(c, "_build_tracing_gap", tracing_gap)
ctx = _ctx()
ctx.task = _t()
ctx.task.source = markers.VIDEO_TASK_SOURCE
await c._i_am_done_fast_path(ctx)
stubs.reject.assert_awaited_once()
c.task.submit_qa.assert_not_awaited()
tracing_gap.assert_awaited_once_with(
ctx.agent_id, ctx.task_id, ["render_preview"], task=ctx.task
)
@pytest.mark.asyncio
async def test_fast_path_video_task_with_render_preview_passes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
ctx = _ctx()
ctx.task = _t()
ctx.task.source = markers.VIDEO_TASK_SOURCE
ctx.task.orchestration_markers = {markers.RENDER_PREVIEW: {"frames": ["a.png"]}}
await c._i_am_done_fast_path(ctx)
stubs.ok.assert_awaited_once()
c.task.submit_qa.assert_awaited_once()
@pytest.mark.asyncio
async def test_fast_path_conventions_block_rejects(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
monkeypatch.setattr(
c,
"_conventions_gate",
AsyncMock(return_value=Envelope.invalid_state(message="x", remediate="fix")),
)
await c._i_am_done_fast_path(_ctx())
stubs.reject.assert_awaited_once()
c.task.submit_qa.assert_not_awaited()
@pytest.mark.asyncio
async def test_fast_path_ci_failure_rejects_before_transition(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
monkeypatch.setattr(
c,
"_fast_path_quality_verdict",
AsyncMock(
return_value=(
Envelope.invalid_state(message="ci red", remediate="fix"),
False,
)
),
)
await c._i_am_done_fast_path(_ctx())
stubs.reject.assert_awaited_once()
c.task.submit_qa.assert_not_awaited()
# ---------------------------------------------------------------------------
# _i_am_done_fast_path: notes is the sole compensating control for the
# skipped journal gates — empty AND soup are rejected (unlike the standard
# path's soup-only check, which skips an empty value).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fast_path_empty_notes_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
ctx = _ctx()
ctx.notes = ""
await c._i_am_done_fast_path(ctx)
stubs.reject.assert_awaited_once()
c.task.submit_verification.assert_not_awaited()
c.task.submit_qa.assert_not_awaited()
@pytest.mark.asyncio
async def test_fast_path_trivial_notes_rejected(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
ctx = _ctx()
ctx.notes = "wip"
await c._i_am_done_fast_path(ctx)
stubs.reject.assert_awaited_once()
c.task.submit_verification.assert_not_awaited()
c.task.submit_qa.assert_not_awaited()
@pytest.mark.asyncio
async def test_fast_path_real_notes_passes(monkeypatch: pytest.MonkeyPatch) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
ctx = _ctx()
ctx.notes = "already implemented in a prior session"
await c._i_am_done_fast_path(ctx)
stubs.ok.assert_awaited_once()
c.task.submit_qa.assert_awaited_once()
# ---------------------------------------------------------------------------
# _i_am_done_fast_path: guard order — push before behind-base, mirroring the
# standard gate (_behind_base_gate reads origin, which must already reflect
# the pushed head).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fast_path_pushes_branch_before_behind_base_check(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
_stub_fast_path(c, monkeypatch)
order: list[str] = []
async def _push(_ctx: Any) -> None:
order.append("push")
async def _behind(_ctx: Any) -> None:
order.append("behind")
monkeypatch.setattr(c, "_ensure_branch_pushed", _push)
monkeypatch.setattr(c, "_behind_base_gate", _behind)
await c._i_am_done_fast_path(_ctx())
assert order == ["push", "behind"]
# ---------------------------------------------------------------------------
# _i_am_done_fast_path: toolchain backstop pairs with the local quality gate
# fallback (no CI signal) — never with the CI-green branch.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fast_path_toolchain_broken_blocks_on_local_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
monkeypatch.setattr(
c,
"_fast_path_quality_verdict",
AsyncMock(return_value=(None, True)), # local gate ran (no CI signal), passed
)
stubs.toolchain.return_value = Envelope.invalid_state(
message="toolchain broken", remediate="fix"
)
await c._i_am_done_fast_path(_ctx())
stubs.reject.assert_awaited_once()
c.task.submit_qa.assert_not_awaited()
@pytest.mark.asyncio
async def test_fast_path_toolchain_guard_skipped_on_ci_green(
monkeypatch: pytest.MonkeyPatch,
) -> None:
c = Choreographer(_deps())
stubs = _stub_fast_path(c, monkeypatch)
monkeypatch.setattr(
c, "_fast_path_quality_verdict", AsyncMock(return_value=(None, False))
)
await c._i_am_done_fast_path(_ctx())
stubs.ok.assert_awaited_once()
stubs.toolchain.assert_not_awaited()