roboco/tests/unit/gateway/test_qa_criteria_verified.py

"""pass_review requires a matched, evidenced verification per acceptance
criterion — not just a count of arbitrary strings.

Live failure this closes: QA passed a rendered video shipping 3 of the
brief's 4 named scenes because nothing forced the reviewer to walk each
acceptance criterion individually. Mirrors the test idiom in
test_qa_ac_coverage.py, one level stricter: criteria_verified entries must
each match a real AC (by id or exact text) and carry substantive evidence.
"""

from __future__ import annotations

from types import SimpleNamespace

from roboco.services.gateway.choreographer import Choreographer

_EVIDENCE_CAP = 500


def _task(criteria: list[str], ids: list[str] | None = None) -> SimpleNamespace:
    return SimpleNamespace(
        acceptance_criteria=criteria,
        acceptance_criteria_ids=ids or [],
    )


def test_no_criteria_imposes_no_requirement() -> None:
    pairs, rej = Choreographer._validate_criteria_verified(_task([]), None)
    assert pairs == []
    assert rej is None


def test_none_supplied_lists_every_criterion_verbatim() -> None:
    criteria = [
        "scene 1 renders",
        "scene 2 renders",
        "scene 3 renders",
        "scene 4 renders",
    ]
    t = _task(criteria)
    pairs, rej = Choreographer._validate_criteria_verified(t, None)
    assert pairs == []
    assert rej is not None
    body = rej.as_dict()
    assert body["error"] == "invalid_state", body
    for crit in criteria:
        assert crit in body["message"]


def test_empty_list_is_treated_as_none_supplied() -> None:
    t = _task(["a"])
    pairs, rej = Choreographer._validate_criteria_verified(t, [])
    assert pairs == []
    assert rej is not None


def test_partial_coverage_names_the_missing_criterion() -> None:
    t = _task(["a", "b", "c"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t,
        [
            {"criterion": "a", "evidence": "frame 1 shows a rendered"},
            {"criterion": "b", "evidence": "frame 2 shows b rendered"},
        ],
    )
    assert pairs == []
    assert rej is not None
    assert "c" in rej.as_dict()["message"]


def test_unmatched_criterion_is_rejected_naming_valid_ones() -> None:
    t = _task(["a", "b"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t,
        [
            {"criterion": "a", "evidence": "frame 1 shows a rendered"},
            {"criterion": "not-a-real-ac", "evidence": "frame 2 shows something"},
        ],
    )
    assert pairs == []
    assert rej is not None
    body = rej.as_dict()
    assert "not-a-real-ac" in body["message"]
    assert "a" in body["remediate"] and "b" in body["remediate"]


def test_missing_criterion_key_is_rejected() -> None:
    t = _task(["a"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t, [{"evidence": "frame 1 shows a rendered"}]
    )
    assert pairs == []
    assert rej is not None


def test_blank_evidence_is_rejected() -> None:
    t = _task(["a"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t, [{"criterion": "a", "evidence": "   "}]
    )
    assert pairs == []
    assert rej is not None


def test_soup_evidence_is_rejected() -> None:
    t = _task(["a"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t, [{"criterion": "a", "evidence": "wip"}]
    )
    assert pairs == []
    assert rej is not None


def test_overlong_evidence_is_rejected() -> None:
    t = _task(["a"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t, [{"criterion": "a", "evidence": "x" * (_EVIDENCE_CAP + 100)}]
    )
    assert pairs == []
    assert rej is not None
    assert str(_EVIDENCE_CAP) in rej.as_dict()["message"]


def test_full_coverage_by_exact_text_passes() -> None:
    t = _task(["a", "b"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t,
        [
            {"criterion": "a", "evidence": "frame 1 shows a rendered fully"},
            {"criterion": "b", "evidence": "frame 2 shows b rendered fully"},
        ],
    )
    assert rej is None
    assert pairs == [
        ("a", "frame 1 shows a rendered fully"),
        ("b", "frame 2 shows b rendered fully"),
    ]


def test_full_coverage_by_ac_id_passes() -> None:
    t = _task(["scene renders"], ids=["AC-1"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t, [{"criterion": "AC-1", "evidence": "rendered-frame path: out/frame3.png"}]
    )
    assert rej is None
    assert pairs == [("AC-1", "rendered-frame path: out/frame3.png")]


def test_extra_entries_beyond_the_ac_set_are_allowed() -> None:
    t = _task(["a"])
    pairs, rej = Choreographer._validate_criteria_verified(
        t,
        [{"criterion": "a", "evidence": "frame 1 shows a rendered fully"}],
    )
    assert rej is None
    assert len(pairs) == 1


def test_render_criteria_verified_matches_style() -> None:
    lines = Choreographer._render_criteria_verified(
        [("scene 1 renders", "frame 12 shows scene 1 fully")]
    )
    assert lines == ["[AC] scene 1 renders — verified: frame 12 shows scene 1 fully"]


def test_merge_criteria_verified_into_notes() -> None:
    merged = Choreographer._merge_criteria_verified_into_notes(
        "base review", [("a", "evidence a"), ("b", "evidence b")]
    )
    assert "[AC] a — verified: evidence a" in merged
    assert "[AC] b — verified: evidence b" in merged


def test_merge_with_no_pairs_returns_notes_unchanged() -> None:
    assert Choreographer._merge_criteria_verified_into_notes("base", []) == "base"