External/roboco
1
0
Fork 0
mirror of https://github.com/rennf93/roboco.git synced 2026-08-03 07:23:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(agents): stop instructing agents to ToolSearch built-in tools (#167)

Browse Source 
The system-prompt directive layer and the briefing block both opened
with "FIRST ACTION REQUIRED: run ToolSearch to activate deferred
Edit/Write". That premise is false: per Claude Code 2.1.114, ToolSearch
gates only deferred MCP tools, never built-ins — and it is not even a
callable tool in the agent runtime. Built-ins are loaded at spawn via
the `--tools` flag and gated solely by the per-role permission rules
(the actual Edit/Write breakage was the global Write(*)/Edit(*) deny +
single-slash path, fixed in c0ba335). So weak models dutifully chased a
nonexistent ToolSearch, concluded Edit/Write were unavailable, and
rewrote whole files via destructive shell redirection.

Both touch points now affirm the role's built-in tools are loaded and
ready, tell the agent NOT to call ToolSearch, and (for authoring roles)
explicitly steer away from whole-file shell redirection — directly
countering the clobber behaviour. Role prompt files (developer,
cell_pm, main_pm, board) updated to match. Dead
_read_tool_load_from_role_prompt (no callers) removed. Directive tests
rewritten to lock the corrected behaviour.
This commit is contained in:
Renn F
2026-05-16 03:53:52 +02:00
parent c0ba335470
commit 38dba74837
8 changed files with 175 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/agents/test_tool_load_directive_layer.py
+73 -60
View File
@@ -1,14 +1,17 @@
"""Smoke-7: system prompt opens with a ToolSearch directive that activates
deferred built-in tools (Edit, Write, Read, ...).
"""#167: the system prompt must NOT tell agents to ToolSearch built-ins.
Original bug: be-dev-1 called Edit and got "Edit exists but is not enabled
in this context" because Claude Code v2.1.69+ defers built-in tools behind
a ToolSearch call. The role prompts said "no ToolSearch needed" — a lie
for built-in tools — so weak models skipped the activation step.
Earlier (smoke-7) the prompt opened with a "# FIRST ACTION REQUIRED:
run ToolSearch to activate Edit/Write" block. That premise was false:
ToolSearch is MCP-only and never gates built-in tools, and it is not
even a callable tool in the agent runtime. Weak models chased the
nonexistent tool, concluded Edit/Write were unavailable, and rewrote
whole files via destructive shell redirection. The real cause of
"Edit exists but is not enabled in this context" was a permission bug
(global Write(*)/Edit(*) deny + single-slash path), fixed separately.
Fix: compose_prompt now prepends a tool-load directive layer that names
the exact ToolSearch call for the role. It's the highest-priority block in
the system prompt so even weak models follow it.
The directive layer now affirms the tools are loaded and ready, tells
agents NOT to call ToolSearch, and (for authoring roles) steers away
from whole-file shell redirection.
"""
from __future__ import annotations
@@ -18,74 +21,84 @@ from roboco.models import AgentRole, Team
def _composed_prompt_for(role: AgentRole, team: Team | None = None) -> str:
"""Compose the prompt for a role and team."""
return compose_prompt(role, team, agent_slug="test-agent")
def test_developer_prompt_starts_with_tool_load_directive() -> None:
"""Developer system prompt begins with the ToolSearch activation block."""
def test_prompt_no_longer_instructs_a_toolsearch_call() -> None:
"""No role prompt may instruct an actual ToolSearch(query=...) call."""
for role in (
AgentRole.DEVELOPER,
AgentRole.DOCUMENTER,
AgentRole.QA,
AgentRole.MAIN_PM,
AgentRole.CELL_PM,
):
prompt = _composed_prompt_for(role, Team.BACKEND)
assert "ToolSearch(query=" not in prompt, (
f"{role.value} prompt still instructs a ToolSearch call"
)
assert "are deferred" not in prompt, (
f"{role.value} prompt still claims built-ins are deferred"
)
def test_developer_prompt_starts_with_tools_ready_block() -> None:
"""Developer system prompt leads with the tools-ready affirmation."""
prompt = _composed_prompt_for(AgentRole.DEVELOPER, Team.BACKEND)
assert prompt.startswith("# FIRST ACTION REQUIRED"), (
f"Developer prompt must lead with the tool-load directive. "
assert prompt.startswith("# Your tools are ready"), (
f"Developer prompt must lead with the tools-ready block. "
f"Got first 80 chars: {prompt[:80]!r}"
)
def test_developer_directive_names_edit_and_write() -> None:
"""Developer ToolSearch call lists Edit + Write (the smoke-7 wedge)."""
def _tool_names(prompt: str) -> list[str]:
"""Exact tool tokens from the 'available now: <names>.' enumeration.
Exact tokens matter: a substring check would treat 'TodoWrite' as
containing 'Write'.
"""
line = next(ln for ln in prompt.splitlines() if "available now:" in ln)
seg = line.split("available now: ", 1)[1]
# _base layer ends the list with '.'; orchestrator continues with
# '. Use them...'. Either way the names stop at the first period.
return seg.split(".", 1)[0].split(", ")
def test_developer_block_lists_edit_and_write_as_available() -> None:
"""Authoring roles are told Edit + Write are loaded and available."""
for role in (AgentRole.DEVELOPER, AgentRole.DOCUMENTER):
names = _tool_names(_composed_prompt_for(role, Team.BACKEND))
assert "Edit" in names and "Write" in names, (
f"{role.value} tools-ready line must list Edit + Write: {names!r}"
)
def test_developer_block_steers_away_from_shell_redirection() -> None:
"""The exact failure mode (clobber a file via bash) is called out."""
prompt = _composed_prompt_for(AgentRole.DEVELOPER, Team.BACKEND)
assert 'ToolSearch(query="select:' in prompt
# Find the ToolSearch line
line = next(line for line in prompt.splitlines() if "ToolSearch(query=" in line)
assert "Edit" in line, f"Developer ToolSearch missing Edit: {line!r}"
assert "Write" in line
assert "shell redirection" in prompt
assert "Edit/Write" in prompt
def test_documenter_directive_names_edit_and_write() -> None:
"""Documenter needs Edit/Write too — they author docs."""
prompt = _composed_prompt_for(AgentRole.DOCUMENTER, Team.BACKEND)
line = next(line for line in prompt.splitlines() if "ToolSearch(query=" in line)
assert "Edit" in line
assert "Write" in line
def test_qa_block_excludes_edit_and_write() -> None:
"""QA reads/reviews — Edit/Write must not be listed as available."""
names = _tool_names(_composed_prompt_for(AgentRole.QA, Team.BACKEND))
assert "Edit" not in names and "Write" not in names, names
assert "Read" in names and "Bash" in names
def _tool_list_from_directive(prompt: str) -> list[str]:
"""Extract the comma-separated tool list from the ToolSearch call."""
line = next(line for line in prompt.splitlines() if "ToolSearch(query=" in line)
# `ToolSearch(query="select:Read,Bash,...")` — pull out the names.
start = line.find("select:") + len("select:")
end = line.find('"', start)
return line[start:end].split(",")
def test_qa_directive_excludes_edit_and_write() -> None:
"""QA reads but doesn't author — directive must NOT activate Edit/Write."""
tools = _tool_list_from_directive(_composed_prompt_for(AgentRole.QA, Team.BACKEND))
assert "Edit" not in tools, f"QA must not activate Edit: tools={tools}"
assert "Write" not in tools
assert "Read" in tools
assert "Bash" in tools
def test_pm_directives_exclude_edit_and_write() -> None:
"""PMs coordinate; they don't author code. No Edit/Write activation."""
def test_pm_blocks_exclude_edit_and_write() -> None:
for role in (AgentRole.MAIN_PM, AgentRole.CELL_PM):
tools = _tool_list_from_directive(_composed_prompt_for(role))
assert "Edit" not in tools, f"{role.value} must not activate Edit: {tools}"
assert "Write" not in tools
names = _tool_names(_composed_prompt_for(role))
assert "Edit" not in names and "Write" not in names, (
f"{role.value} must not list Edit/Write: {names}"
)
def test_directive_explains_why_it_matters() -> None:
"""The block must mention 'Edit exists but is not enabled' so the agent
understands what skipping the call causes."""
def test_block_is_first_layer_before_lifecycle() -> None:
"""Tools-ready block precedes the lifecycle and base layers."""
prompt = _composed_prompt_for(AgentRole.DEVELOPER, Team.BACKEND)
assert "Edit exists but is not enabled" in prompt
def test_directive_is_first_layer_before_lifecycle() -> None:
"""First Action block precedes the lifecycle and base layers."""
prompt = _composed_prompt_for(AgentRole.DEVELOPER, Team.BACKEND)
first_idx = prompt.find("# FIRST ACTION REQUIRED")
first_idx = prompt.find("# Your tools are ready")
lifecycle_idx = prompt.find("Lifecycle")
base_idx = prompt.find("RoboCo Agent — Base")
assert first_idx == 0
tests/unit/runtime/test_briefing_tool_load_block.py
+43 -47
View File
@@ -1,19 +1,19 @@
"""Smoke-8: briefing's _build_tool_load_block emits the ToolSearch directive
without depending on a "## Load on spawn" section in the role file.
"""#167: the briefing's _build_tool_load_block must not push ToolSearch.
The previous implementation scraped role prompts for that marker; since
no role file had it, the function returned "" and the briefing showed
agents no tool-load instruction. Combined with weak models skipping the
system-prompt directive, the agent went straight to Edit and hit "not
enabled in this context."
Earlier (smoke-8) the block instructed agents to run a ToolSearch call
to "activate deferred built-in tools". That premise was false —
ToolSearch is MCP-only, never gates built-ins, and is not callable in
the agent runtime — so weak models chased a nonexistent tool and fell
back to destructive shell file-writes. The real cause of "Edit not
enabled in this context" was a permission bug fixed separately.
Fix: per-role tool list lives in the orchestrator (mirrors the
factories layer). Pre-renders the directive directly — no file scrape.
The block now affirms the role's built-in tools are loaded and ready,
tells the agent NOT to call ToolSearch, and (for authoring roles)
steers away from whole-file shell redirection.
"""
from __future__ import annotations
import re
from unittest.mock import patch
from roboco.runtime.orchestrator import AgentOrchestrator
@@ -26,58 +26,54 @@ def _orch() -> AgentOrchestrator:
return orch
def test_developer_directive_includes_edit_and_write() -> None:
def _tool_names(block: str) -> list[str]:
"""Exact tool tokens (so 'TodoWrite' is not mistaken for 'Write')."""
line = next(ln for ln in block.splitlines() if "available now:" in ln)
seg = line.split("available now: ", 1)[1]
return seg.split(".", 1)[0].split(", ")
def test_developer_block_affirms_tools_no_toolsearch_call() -> None:
block = _orch()._build_tool_load_block("developer")
assert "First action required" in block
assert "ToolSearch" in block
assert "Edit" in block
assert "Write" in block
assert "Your tools are ready" in block
assert "ToolSearch(query=" not in block
assert "are deferred" not in block
names = _tool_names(block)
assert "Edit" in names and "Write" in names
def test_documenter_directive_includes_edit_and_write() -> None:
block = _orch()._build_tool_load_block("documenter")
assert "Edit" in block
assert "Write" in block
def test_developer_block_steers_away_from_shell_redirection() -> None:
block = _orch()._build_tool_load_block("developer")
assert "shell redirection" in block
assert "Edit/Write" in block
def test_qa_directive_excludes_edit_and_write() -> None:
def test_documenter_block_lists_edit_and_write() -> None:
names = _tool_names(_orch()._build_tool_load_block("documenter"))
assert "Edit" in names and "Write" in names
def test_qa_block_excludes_edit_and_write() -> None:
block = _orch()._build_tool_load_block("qa")
assert "First action required" in block
# The ToolSearch line itself: pull the comma list to be precise about
# which tools are listed (substring match would catch "TodoWrite").
match = re.search(r'select:([^"]+)"', block)
assert match is not None
tools = match.group(1).split(",")
assert "Edit" not in tools
assert "Write" not in tools
assert "Read" in tools
assert "Bash" in tools
assert "Your tools are ready" in block
names = _tool_names(block)
assert "Edit" not in names and "Write" not in names
assert "Read" in names and "Bash" in names
def test_pm_directives_exclude_edit_and_write() -> None:
def test_pm_blocks_exclude_edit_and_write() -> None:
for role in ("main_pm", "cell_pm", "product_owner", "head_marketing", "auditor"):
block = _orch()._build_tool_load_block(role)
match = re.search(r'select:([^"]+)"', block)
assert match is not None
tools = match.group(1).split(",")
assert "Edit" not in tools, f"{role} must not list Edit"
assert "Write" not in tools, f"{role} must not list Write"
names = _tool_names(_orch()._build_tool_load_block(role))
assert "Edit" not in names, f"{role} must not list Edit"
assert "Write" not in names, f"{role} must not list Write"
def test_unknown_role_returns_empty() -> None:
"""No directive for unknown roles (defensive)."""
block = _orch()._build_tool_load_block("nonexistent")
assert block == ""
def test_directive_warns_about_failure_mode() -> None:
block = _orch()._build_tool_load_block("developer")
assert "Edit exists but is not enabled" in block
assert _orch()._build_tool_load_block("nonexistent") == ""
def test_role_cache_works() -> None:
orch = _orch()
first = orch._build_tool_load_block("developer")
# Second call hits the cache (same return).
second = orch._build_tool_load_block("developer")
assert first is second # same string object
assert first is second