mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(toolchain): block QA/dev/PR gates when the suite cannot be executed
Adds _toolchain_broken_guard: when toolchain matching is on and the acting agent's workspace recorded a 'broken' status (the project's suite cannot be collected under the provisioned interpreter), i_am_done, pass_review, and pr_pass refuse with an i_am_blocked remediation instead of letting the role "pass" on a source read. fail_review / pr_fail stay available. git gains toolchain_status_for_task (resolves the actor's workspace, reads the marker, fail-open None). Inert when the flag is off; a missing/unknown status never strands a task.
This commit is contained in:
@@ -1663,6 +1663,8 @@ class Choreographer:
|
||||
return await self._reject_i_am_done(ctx, rejection)
|
||||
if rejection := await self._check_quality_gate(ctx):
|
||||
return await self._reject_i_am_done(ctx, rejection)
|
||||
if rejection := await self._toolchain_broken_guard(ctx.agent_id, ctx.task):
|
||||
return await self._reject_i_am_done(ctx, rejection)
|
||||
# Pre-gateway parity: persist per-criterion
|
||||
# status now that all gates have passed. The write runs AFTER the
|
||||
# verdict so it cannot change i_am_done's rejection behavior.
|
||||
@@ -1695,6 +1697,38 @@ class Choreographer:
|
||||
context_briefing=ctx.briefing,
|
||||
)
|
||||
|
||||
async def _toolchain_broken_guard(
|
||||
self, agent_id: UUID, task: Any
|
||||
) -> Envelope | None:
|
||||
"""Refuse a delivery gate when the acting agent's workspace cannot run
|
||||
the project's suite (interpreter mismatch).
|
||||
|
||||
A role that cannot execute the suite degrades to "verifying by reading
|
||||
source", which the QA + PR-review gates exist to prevent. Inert when the
|
||||
flag is off; only a recorded ``broken`` status blocks — a missing or
|
||||
``unknown`` status never strands a task (fail-open).
|
||||
"""
|
||||
from roboco.config import settings as _settings
|
||||
|
||||
if not _settings.toolchain_match_enabled:
|
||||
return None
|
||||
status = await self.git.toolchain_status_for_task(agent_id, task)
|
||||
if status != "broken":
|
||||
return None
|
||||
return Envelope.invalid_state(
|
||||
message=(
|
||||
"the project's test suite cannot be executed in this workspace "
|
||||
"(interpreter mismatch) — verifying by reading source is hollow"
|
||||
),
|
||||
remediate=(
|
||||
"the workspace Python does not match the project's requirement; "
|
||||
"call i_am_blocked(reason='toolchain') so the environment is "
|
||||
"rebuilt against the right interpreter — do NOT pass on a "
|
||||
"source read"
|
||||
),
|
||||
context_briefing={},
|
||||
)
|
||||
|
||||
async def _ensure_branch_pushed(self, ctx: _IAmDoneContext) -> Envelope | None:
|
||||
"""Push the task branch to origin before it reaches awaiting_qa.
|
||||
|
||||
|
||||
@@ -53,6 +53,11 @@ class ChoreographerHelpers:
|
||||
) -> Envelope:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _toolchain_broken_guard(
|
||||
self, agent_id: UUID, task: Any
|
||||
) -> Envelope | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _free_text_soup(
|
||||
cls, checks: tuple[tuple[str, Any, int], ...]
|
||||
|
||||
@@ -234,6 +234,17 @@ class PRGateMixin(_Base):
|
||||
)
|
||||
if gate is not None:
|
||||
return gate
|
||||
# A reviewer must not PASS an assembled PR whose suite cannot be run in
|
||||
# the workspace (interpreter mismatch). pr_fail stays available.
|
||||
if verb == "pr_pass" and (
|
||||
toolchain := await self._toolchain_broken_guard(reviewer_agent_id, t)
|
||||
):
|
||||
return await self._emit_rejection(
|
||||
toolchain.with_introspection(task=t, role=role_str),
|
||||
agent_id=reviewer_agent_id,
|
||||
task_id=task_id,
|
||||
verb=verb,
|
||||
)
|
||||
runner = self._verb_runner()
|
||||
try:
|
||||
t = await runner.run_intent(verb, t, agent, spec_ctx)
|
||||
|
||||
@@ -448,6 +448,37 @@ class QAMixin(_Base):
|
||||
verb=verb,
|
||||
)
|
||||
|
||||
async def _qa_pass_final_gates(
|
||||
self,
|
||||
qa_agent_id: UUID,
|
||||
task_id: UUID,
|
||||
t: Any,
|
||||
role_str: str,
|
||||
ac_verdicts: list[str] | None,
|
||||
) -> Envelope | None:
|
||||
"""AC-coverage + toolchain-runnability gates for pass_review.
|
||||
|
||||
Returns the first rejection (already emitted), else None. QA must not
|
||||
PASS on a workspace that cannot run the suite — that is a source-read
|
||||
"verification"; fail_review is unaffected.
|
||||
"""
|
||||
ac_rejection = self._qa_ac_coverage_check(t, ac_verdicts)
|
||||
if ac_rejection is not None:
|
||||
return await self._emit_rejection(
|
||||
ac_rejection.with_introspection(task=t, role=role_str),
|
||||
agent_id=qa_agent_id,
|
||||
task_id=task_id,
|
||||
verb="pass_review",
|
||||
)
|
||||
if toolchain := await self._toolchain_broken_guard(qa_agent_id, t):
|
||||
return await self._emit_rejection(
|
||||
toolchain.with_introspection(task=t, role=role_str),
|
||||
agent_id=qa_agent_id,
|
||||
task_id=task_id,
|
||||
verb="pass_review",
|
||||
)
|
||||
return None
|
||||
|
||||
async def pass_review(
|
||||
self,
|
||||
qa_agent_id: UUID,
|
||||
@@ -493,14 +524,10 @@ class QAMixin(_Base):
|
||||
soup_checks=(("notes", notes, 8),),
|
||||
):
|
||||
return gate_rejection
|
||||
ac_rejection = self._qa_ac_coverage_check(t, ac_verdicts)
|
||||
if ac_rejection is not None:
|
||||
return await self._emit_rejection(
|
||||
ac_rejection.with_introspection(task=t, role=role_str),
|
||||
agent_id=qa_agent_id,
|
||||
task_id=task_id,
|
||||
verb="pass_review",
|
||||
)
|
||||
if rej := await self._qa_pass_final_gates(
|
||||
qa_agent_id, task_id, t, role_str, ac_verdicts
|
||||
):
|
||||
return rej
|
||||
|
||||
briefing = await self._briefing_for(qa_agent_id, task_id)
|
||||
spec_ctx = spec_module.Context(
|
||||
|
||||
+24
-1
@@ -57,7 +57,11 @@ from roboco.services.gateway.quality_gate import GateResult, run_quality_command
|
||||
from roboco.services.project import get_project_service
|
||||
from roboco.services.task import TaskService, get_task_service
|
||||
from roboco.services.work_session import get_work_session_service
|
||||
from roboco.services.workspace import WorkspaceError, get_workspace_service
|
||||
from roboco.services.workspace import (
|
||||
WorkspaceError,
|
||||
WorkspaceService,
|
||||
get_workspace_service,
|
||||
)
|
||||
from roboco.templates.git import (
|
||||
BranchNameError,
|
||||
CommitContext,
|
||||
@@ -2751,6 +2755,25 @@ class GitService(BaseService):
|
||||
workspace = await self.get_workspace(project.slug, actor_agent_id)
|
||||
return await run_quality_commands(workspace, commands)
|
||||
|
||||
async def toolchain_status_for_task(
|
||||
self, actor_agent_id: UUID, task: Any
|
||||
) -> str | None:
|
||||
"""The recorded toolchain status (``ok`` | ``broken`` | ``unknown``) for
|
||||
the acting agent's workspace clone of the task's project, or ``None``.
|
||||
|
||||
``None`` on any resolution/read failure — the caller fails open and
|
||||
never blocks a gate on an inability to read the marker.
|
||||
"""
|
||||
try:
|
||||
project = await self._project_for_task(task)
|
||||
if project is None:
|
||||
return None
|
||||
workspace = await self.get_workspace(project.slug, actor_agent_id)
|
||||
_python, status = WorkspaceService.read_toolchain_status(workspace)
|
||||
return status
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def _project_slug_for_branch(self, branch_name: str) -> str | None:
|
||||
"""Resolve project slug via the task that owns the branch."""
|
||||
task = await self._task_for_branch(branch_name)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""The loud-fail guard: a delivery gate refuses when the suite can't run.
|
||||
|
||||
When toolchain matching is on and the acting agent's workspace recorded a
|
||||
``broken`` toolchain status (the project's suite cannot be collected under the
|
||||
provisioned interpreter), the dev/QA/PR gates must block — never let a role
|
||||
"pass" on a source read. Off, or any non-broken / unknown status, never blocks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
|
||||
|
||||
def _make_choreographer(*, status: str | None) -> Choreographer:
|
||||
base: dict[str, Any] = {
|
||||
"task": AsyncMock(),
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base["git"].toolchain_status_for_task.return_value = status
|
||||
return Choreographer(ChoreographerDeps(**base))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guard_blocks_when_broken_and_flag_on(monkeypatch) -> None:
|
||||
monkeypatch.setattr(settings, "toolchain_match_enabled", True)
|
||||
c = _make_choreographer(status="broken")
|
||||
env = await c._toolchain_broken_guard(uuid4(), MagicMock())
|
||||
assert env is not None
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "invalid_state"
|
||||
assert "i_am_blocked" in body["remediate"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guard_passes_when_ok(monkeypatch) -> None:
|
||||
monkeypatch.setattr(settings, "toolchain_match_enabled", True)
|
||||
c = _make_choreographer(status="ok")
|
||||
assert await c._toolchain_broken_guard(uuid4(), MagicMock()) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guard_passes_when_status_unknown_or_missing(monkeypatch) -> None:
|
||||
monkeypatch.setattr(settings, "toolchain_match_enabled", True)
|
||||
for status in ("unknown", None):
|
||||
c = _make_choreographer(status=status)
|
||||
assert await c._toolchain_broken_guard(uuid4(), MagicMock()) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guard_inert_when_flag_off(monkeypatch) -> None:
|
||||
monkeypatch.setattr(settings, "toolchain_match_enabled", False)
|
||||
c = _make_choreographer(status="broken")
|
||||
assert await c._toolchain_broken_guard(uuid4(), MagicMock()) is None
|
||||
# Flag off => the workspace is never consulted at all.
|
||||
c.git.toolchain_status_for_task.assert_not_awaited()
|
||||
Reference in New Issue
Block a user