From f53bbd1152464ecbb1de495e2d1d959e156138f0 Mon Sep 17 00:00:00 2001 From: Tyler <109685178+tlongwell-block@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:27:29 -0400 Subject: [PATCH] fix(bench): mention the orchestrator by pubkey when posting the task (#5136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The harness posts each trial's task via `buzz messages send`, relying on `@` name resolution. Task text is untrusted payload: when it contains @-tokens of its own, the CLI's mention resolver tries to resolve them as channel members, fails, and refuses to send — killing the trial with `RuntimeLaunchError` before the agent ever saw the task. Live occurrence: TB 2.1's `large-scale-text-editing` task embeds Vim macros (`:%normal! @a`). In the tb21-solo-1 run the trial died at launch: ``` RuntimeLaunchError: buzz messages send ... exited 1: {"error":"user_error","message":"mention '@a' does not match a current channel member; retry with --mention "} ``` Any TB task whose statement contains @-syntax is silently zeroed this way. ## Fix Pass the orchestrator's pubkey as an explicit `--mention` when posting the task. The CLI demotes unresolved @-tokens in the text to presentation-only when any explicit identity is supplied, so delivery still targets exactly the orchestrator and every @-token in the task statement becomes inert. The harness already holds the orchestrator's `AgentCredential` (it writes that pubkey into the worker roster tables), so no persistence is needed — fresh key per trial, fresh `--mention` per trial. Verified both halves against a live relay: a fenced `@a` without `--mention` still hard-fails (the resolver is not markdown-aware); the same content with an explicit `--mention` sends clean with `mention_pubkeys` containing only the target. ## Testing - `benchmarks/harbor-buzz-orchestra`: full pytest suite — 35 passed (34 baseline + new `test_send_mentions_by_pubkey_so_task_text_stays_inert`), ruff clean. Run against `origin/main` 769ac70b with exactly this patch applied. - `testbed`: full pytest suite — 23 passed, 1 skipped; ruff clean. ## Acceptance A task statement containing arbitrary @-tokens (Vim registers, emails, decorators) launches and delivers to the orchestrator instead of dying in `_send`. Originating Buzz thread: `buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b` Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> --- .../container_runtime.py | 28 ++++++++++++----- .../tests/test_container_runtime.py | 31 +++++++++++++++++++ 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index 149a5295a..ed883a820 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -157,9 +157,17 @@ class BuzzContainerRuntime: ) # The task arrives exactly as it would in production Buzz: a # user prompt @mentioning the orchestrator. The harness never - # speaks as any agent. + # speaks as any agent. The orchestrator is mentioned by pubkey, + # not by name resolution: task text is untrusted payload, and any + # @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise + # fail member resolution and kill the trial before the agent + # ever saw the task. An explicit --mention demotes unresolved + # @-tokens in the text to presentation-only. await self._send( - trial.user, trial, f"@{orchestrator.agent_id} {instruction}" + trial.user, + trial, + f"@{orchestrator.agent_id} {instruction}", + mention=orchestrator.nostr_pubkey, ) final_message = await asyncio.wait_for( self._wait_for_done(environment, orchestrator, trial, agents + infra), @@ -519,18 +527,24 @@ class BuzzContainerRuntime: ) async def _send( - self, credential: AgentCredential, trial: TrialHandle, content: str + self, + credential: AgentCredential, + trial: TrialHandle, + content: str, + *, + mention: str | None = None, ) -> None: - await self._buzz_json( - credential, - trial, + args = [ "messages", "send", "--channel", trial.channel_id, "--content", content, - ) + ] + if mention is not None: + args += ["--mention", mention] + await self._buzz_json(credential, trial, *args) async def _buzz_json( self, credential: AgentCredential, trial: TrialHandle, *args: str diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index ebf0eb4b5..5fc0e63e5 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -372,6 +372,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped( assert bool(probed) == (condition == "M1-hello-world") +async def test_send_mentions_by_pubkey_so_task_text_stays_inert( + tmp_path, monkeypatch +): + """Task text is untrusted payload: `:%normal! @a` in a task statement must + not be fed to member-name resolution (it would fail and kill the trial). + An explicit --mention pins delivery to the orchestrator's pubkey.""" + rt = runtime(tmp_path) + orch = credential("orch-1", "orchestrator", "orch-model") + trial = trial_handle((orch,)) + calls = [] + + async def buzz_json(credential, trial, *args): + calls.append(args) + return {} + + monkeypatch.setattr(rt, "_buzz_json", buzz_json) + + await rt._send( + trial.user, + trial, + "@orch-1 run `:%normal! @a` on the file", + mention=orch.nostr_pubkey, + ) + assert calls[-1][-2:] == ("--mention", "pubkey-orch-1") + + # Without an explicit mention the send is unchanged (name resolution). + await rt._send(trial.user, trial, "plain content") + assert "--mention" not in calls[-1] + assert calls[-1][-2:] == ("--content", "plain content") + + async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch): rt = runtime(tmp_path, poll_seconds=0) orch = credential("orch-1", "orchestrator", "orch-model")