mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
* fix(prompts): point agents at Makefile, drop raw uv run instructions backend.md:23-26 literally instructed raw uv run ruff/mypy/pytest (copied from the human-facing CLAUDE.md), so agents bypassed the Makefile's UV_NO_SYNC=1 + private UV_CACHE_DIR venv-corruption guard. Replace with make targets across backend/developer/qa/cell_pm + a universal rule in base.md. Regenerate verbs.md from the updated regen script (baked instruction now make foundation-check) and align the Makefile drift message. Ships with the bash-guard deny in the next commit so agents don't loop fighting the guard. * feat(bash-guard): deny raw uv/pip/conda/poetry, point at Makefile When a Makefile is present, deny raw uv run/uv pip/uv lock/add/remove, pip/pip3 install/uninstall, conda install/create/run, poetry run/install/add and remediate to make quality/gate/lint/test. Skipped when no Makefile (Makefile-less projects not blocked). ROBOCO_GUARD_SKIP_PM=1 (grok path) nudges exit 0 instead of the run-canceling exit 2. Overrides the prior bare-uv-run-allowed stance by CEO direction; the /app-targeted blocks above keep priority. * feat(grok): deny raw uv/pip/conda/poetry via native --deny + PM-skip nudge Add _RAW_PM_DENY (uv run/pip install/lock/add/remove, pip/pip3 install, conda install/create/run, poetry run/install/add) to _deny_rules so grok's graceful native --deny blocks raw package-manager commands (model adapts to make, run continues — unlike a hook deny which cancels the run). The bash-guard hook keeps the compound-command fallback (cd x && uv run) and nudges exit 0 there via ROBOCO_GUARD_SKIP_PM=1 in the grok hook env, never canceling. * test(bash-guard): align existing tests with W1 Makefile-gate policy Raw uv run / pip install are now Makefile-gated (W1, CEO item #15), so two existing bash-guard invariants reverse: - test_allows_pytest_even_if_suite_uses_requests keeps its HTTP-injection allow-path intent but uses bare `python -m pytest` (raw `uv run` is now denied); the deny case is covered by test_bash_guard_makefile_guardrail. - test_allows_pip_install_in_workspace -> test_denies_pip_install_when_makefile_ present: a workspace clone carries a Makefile, so bare pip install is now denied -> agents use `make` / `uv sync --extra dev`. Makefile-less skips stay covered. Gate: 12994 passed, 439 skipped, 94.81% cov (DB env :55432 user renzof); the lone flaky integration error passes in isolation (DB-state race, not W1). --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
"""Agent prompts must steer the fleet at the Makefile, not raw package managers.
|
||||
|
||||
`agents/prompts/teams/backend.md` used to literally instruct `uv run ruff ...`
|
||||
(copied from the human-facing CLAUDE.md), so agents ran raw `uv run` and bypassed
|
||||
the Makefile's `UV_NO_SYNC=1` + private `UV_CACHE_DIR` — the guard against the
|
||||
concurrent-venv-corruption race. This grep invariant keeps the prompts honest:
|
||||
no agent-facing prompt instructs a raw `uv run`/`pip`/`conda`/`poetry`, and the
|
||||
backend standards block names the make targets.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
PROMPTS = Path(__file__).resolve().parents[3] / "agents" / "prompts"
|
||||
RAW_PM = ["uv run ", "uv pip install", "pip install ", "conda install ", "poetry run "]
|
||||
|
||||
|
||||
def _prompt_texts() -> dict[str, str]:
|
||||
texts: dict[str, str] = {}
|
||||
for path in PROMPTS.rglob("*.md"):
|
||||
texts[str(path.relative_to(PROMPTS))] = path.read_text()
|
||||
return texts
|
||||
|
||||
|
||||
def test_no_prompt_instructs_raw_package_managers() -> None:
|
||||
"""No agent-facing prompt tells an agent to run raw uv/pip/conda/poetry."""
|
||||
offenders: list[str] = []
|
||||
for name, body in _prompt_texts().items():
|
||||
for bad in RAW_PM:
|
||||
if bad in body:
|
||||
offenders.append(f"{name}: '{bad.strip()}'")
|
||||
assert not offenders, f"prompts still instruct raw PM commands: {offenders}"
|
||||
|
||||
|
||||
def test_backend_prompt_names_make_targets() -> None:
|
||||
"""The backend standards block must point at make targets, not uv run."""
|
||||
body = (PROMPTS / "teams" / "backend.md").read_text()
|
||||
assert "make quality" in body
|
||||
assert "make gate" in body
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Grok native --deny covers raw package-manager commands (Makefile guardrail).
|
||||
|
||||
Native ``--deny`` is graceful on grok (the model gets a permission error and
|
||||
adapts to ``make``; the run continues — unlike a hook deny, which cancels the
|
||||
whole run). So raw-PM commands are denied via ``_RAW_PM_DENY`` in
|
||||
``_deny_rules()``, mirroring ``_GIT_MUTATE_DENY``. The bash-guard hook handles
|
||||
only the compound-command case the globs miss, and there
|
||||
``ROBOCO_GUARD_SKIP_PM=1`` nudges instead of canceling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from roboco.llm.providers.grok_cli_config import _RAW_PM_DENY, _deny_rules
|
||||
|
||||
|
||||
def test_raw_pm_deny_rules_present() -> None:
|
||||
rules = _deny_rules("developer")
|
||||
assert "Bash(uv run*)" in rules
|
||||
assert "Bash(uv pip install*)" in rules
|
||||
assert "Bash(pip install*)" in rules
|
||||
assert "Bash(conda install*)" in rules
|
||||
assert "Bash(poetry run*)" in rules
|
||||
|
||||
|
||||
def test_raw_pm_deny_is_subset_of_deny_rules() -> None:
|
||||
rules = set(_deny_rules("developer"))
|
||||
assert set(_RAW_PM_DENY) <= rules
|
||||
|
||||
|
||||
def test_raw_pm_deny_only_for_bash_roles() -> None:
|
||||
"""Non-bash roles have bash removed entirely — no deny rules at all."""
|
||||
assert _deny_rules("auditor") == []
|
||||
@@ -0,0 +1,86 @@
|
||||
"""bash-guard Makefile guardrail — deny raw uv/pip/conda/poetry, point at make.
|
||||
|
||||
CEO item: force agents to the Makefile. The existing hook deliberately allowed
|
||||
bare ``uv run`` (workspace .venv, cwd-relative); this guard overrides that by
|
||||
CEO direction when a ``Makefile`` is present, denying raw package-manager /
|
||||
test-runner commands and remediating to the make targets. Skipped when no
|
||||
Makefile exists so Makefile-less projects aren't blocked. On the grok path
|
||||
(``ROBOCO_GUARD_SKIP_PM=1``) a deny cancels the whole run, so it nudges (exit 0)
|
||||
instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
HOOK = REPO_ROOT / "docker" / "scripts" / "bash-guard-hook.sh"
|
||||
|
||||
# Hook exits 2 to deny, 0 to allow. Named (not magic) for ruff PLR2004.
|
||||
_DENIED = 2
|
||||
_ALLOWED = 0
|
||||
|
||||
|
||||
def _run_hook(
|
||||
command: str, cwd: Path, env_over: dict[str, str] | None = None
|
||||
) -> tuple[int, str]:
|
||||
payload = json.dumps({"tool_input": {"command": command}})
|
||||
env = dict(os.environ)
|
||||
if env_over:
|
||||
env.update(env_over)
|
||||
proc = subprocess.run(
|
||||
["bash", str(HOOK)],
|
||||
input=payload,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=str(cwd),
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
return proc.returncode, proc.stderr
|
||||
|
||||
|
||||
def test_denies_uv_run_when_makefile_present() -> None:
|
||||
rc, err = _run_hook("uv run pytest", REPO_ROOT)
|
||||
assert rc == _DENIED
|
||||
assert "make" in err.lower()
|
||||
|
||||
|
||||
def test_denies_pip_install() -> None:
|
||||
rc, _ = _run_hook("pip install requests", REPO_ROOT)
|
||||
assert rc == _DENIED
|
||||
|
||||
|
||||
def test_denies_compound_uv_run() -> None:
|
||||
rc, _ = _run_hook("cd svc && uv run ruff check .", REPO_ROOT)
|
||||
assert rc == _DENIED
|
||||
|
||||
|
||||
def test_denies_conda_and_poetry() -> None:
|
||||
assert _run_hook("conda install numpy", REPO_ROOT)[0] == _DENIED
|
||||
assert _run_hook("poetry run pytest", REPO_ROOT)[0] == _DENIED
|
||||
|
||||
|
||||
def test_allows_make_quality() -> None:
|
||||
rc, _ = _run_hook("make quality", REPO_ROOT)
|
||||
assert rc != _DENIED
|
||||
|
||||
|
||||
def test_allows_pnpm() -> None:
|
||||
rc, _ = _run_hook("pnpm lint", REPO_ROOT)
|
||||
assert rc != _DENIED
|
||||
|
||||
|
||||
def test_skips_deny_without_makefile(tmp_path: Path) -> None:
|
||||
rc, _ = _run_hook("uv run pytest", tmp_path)
|
||||
assert rc != _DENIED
|
||||
|
||||
|
||||
def test_grok_path_nudges_not_denies() -> None:
|
||||
"""ROBOCO_GUARD_SKIP_PM=1 (grok) -> exit 0 nudge, not run-canceling exit 2."""
|
||||
rc, err = _run_hook("uv run pytest", REPO_ROOT, {"ROBOCO_GUARD_SKIP_PM": "1"})
|
||||
assert rc == _ALLOWED
|
||||
assert "make" in err.lower()
|
||||
@@ -379,8 +379,13 @@ def test_allows_python_httpx_import_without_internal_host() -> None:
|
||||
|
||||
def test_allows_pytest_even_if_suite_uses_requests() -> None:
|
||||
"""The command string is just the runner — no http-client token and
|
||||
no internal host literal — so it must pass."""
|
||||
assert _run("uv run python -m pytest tests/unit/ -q") == _ALLOWED
|
||||
no internal host literal — so it must pass.
|
||||
|
||||
Bare ``python -m pytest`` (not ``uv run``): raw ``uv run`` is now
|
||||
Makefile-gated (W1), so the runner here is the bare interpreter to keep
|
||||
this test about the HTTP-injection allow path, not package-manager policy.
|
||||
"""
|
||||
assert _run("python -m pytest tests/unit/ -q") == _ALLOWED
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -435,8 +440,12 @@ def test_allows_uv_sync_in_workspace() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_allows_pip_install_in_workspace() -> None:
|
||||
assert _run("pip install -r requirements.txt") == _ALLOWED
|
||||
def test_denies_pip_install_when_makefile_present() -> None:
|
||||
"""W1: raw ``pip install`` is Makefile-gated. A workspace clone carries
|
||||
a ``Makefile`` (same repo), so a bare ``pip install`` is denied — agents
|
||||
use ``make`` / ``uv sync --extra dev``. Makefile-less projects skip the
|
||||
deny (covered in test_bash_guard_makefile_guardrail.py)."""
|
||||
assert _run("pip install -r requirements.txt") == _DENIED
|
||||
|
||||
|
||||
def test_allows_reading_files_under_app() -> None:
|
||||
|
||||
Reference in New Issue
Block a user