fix(bench): mention the orchestrator by pubkey when posting the task

The harness posts each trial's task via `buzz messages send`, relying on
`@<orchestrator-id>` name resolution. Task text is untrusted payload:
when it contains @-tokens of its own — TB 2.1's large-scale-text-editing
task embeds Vim macros like `:%normal! @a` — the CLI's mention resolver
rejects `@a` as an unknown channel member and refuses to send, killing
the trial with RuntimeLaunchError before the agent ever saw the task.

Pass the orchestrator's pubkey as an explicit --mention instead. The CLI
demotes unresolved @-tokens in the text to presentation-only when any
explicit identity is supplied, so delivery still targets exactly the
orchestrator and every @-token in the task statement becomes inert.

Live repro on the relay confirmed both halves: a fenced `@a` without
--mention still hard-fails (the resolver is not markdown-aware); the
same content with an explicit --mention sends clean.

- benchmarks/harbor-buzz-orchestra: full pytest suite, 35 passed
  (34 baseline + new test), ruff clean — validated against origin/main
  769ac70b with this patch applied
- testbed: full pytest suite, 23 passed / 1 skipped, ruff clean

Originating Buzz thread:
buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b

Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
(cherry picked from commit e1953e2198)
This commit is contained in:
Eva
2026-08-06 22:02:43 -04:00
parent 1cf841af13
commit 2944a88074
2 changed files with 52 additions and 6 deletions
@@ -365,9 +365,17 @@ class BuzzContainerRuntime:
) )
# The task arrives exactly as it would in production Buzz: a # The task arrives exactly as it would in production Buzz: a
# user prompt @mentioning the orchestrator. The harness never # user prompt @mentioning the orchestrator. The harness never
# speaks as any agent. # speaks as any agent. The orchestrator is mentioned by pubkey,
# not by name resolution: task text is untrusted payload, and any
# @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise
# fail member resolution and kill the trial before the agent
# ever saw the task. An explicit --mention demotes unresolved
# @-tokens in the text to presentation-only.
await self._send( await self._send(
trial.user, trial, f"@{orchestrator.agent_id} {instruction}" trial.user,
trial,
f"@{orchestrator.agent_id} {instruction}",
mention=orchestrator.nostr_pubkey,
) )
final_message = await asyncio.wait_for( final_message = await asyncio.wait_for(
self._wait_for_done( self._wait_for_done(
@@ -1223,13 +1231,20 @@ class BuzzContainerRuntime:
) )
async def _send( async def _send(
self, credential: AgentCredential, trial: TrialHandle, content: str self,
credential: AgentCredential,
trial: TrialHandle,
content: str,
*,
mention: str | None = None,
) -> None: ) -> None:
await self._buzz_json( args = [
credential, trial,
"messages", "send", "--channel", trial.channel_id, "messages", "send", "--channel", trial.channel_id,
"--content", content, "--content", content,
) ]
if mention is not None:
args += ["--mention", mention]
await self._buzz_json(credential, trial, *args)
async def _buzz_json( async def _buzz_json(
self, credential: AgentCredential, trial: TrialHandle, *args: str self, credential: AgentCredential, trial: TrialHandle, *args: str
@@ -915,6 +915,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped(
assert bool(probed) == (condition == "M1-hello-world") assert bool(probed) == (condition == "M1-hello-world")
async def test_send_mentions_by_pubkey_so_task_text_stays_inert(
tmp_path, monkeypatch
):
"""Task text is untrusted payload: `:%normal! @a` in a task statement must
not be fed to member-name resolution (it would fail and kill the trial).
An explicit --mention pins delivery to the orchestrator's pubkey."""
rt = runtime(tmp_path)
orch = credential("orch-1", "orchestrator", "orch-model")
trial = trial_handle((orch,))
calls = []
async def buzz_json(credential, trial, *args):
calls.append(args)
return {}
monkeypatch.setattr(rt, "_buzz_json", buzz_json)
await rt._send(
trial.user,
trial,
"@orch-1 run `:%normal! @a` on the file",
mention=orch.nostr_pubkey,
)
assert calls[-1][-2:] == ("--mention", "pubkey-orch-1")
# Without an explicit mention the send is unchanged (name resolution).
await rt._send(trial.user, trial, "plain content")
assert "--mention" not in calls[-1]
assert calls[-1][-2:] == ("--content", "plain content")
async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch): async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
rt = runtime(tmp_path, poll_seconds=0) rt = runtime(tmp_path, poll_seconds=0)
orch = credential("orch-1", "orchestrator", "orch-model") orch = credential("orch-1", "orchestrator", "orch-model")