fix(bench): mention the orchestrator by pubkey when posting the task (#5136)

## Problem

The harness posts each trial's task via `buzz messages send`, relying on
`@<orchestrator-id>` name resolution. Task text is untrusted payload:
when it contains @-tokens of its own, the CLI's mention resolver tries
to resolve them as channel members, fails, and refuses to send — killing
the trial with `RuntimeLaunchError` before the agent ever saw the task.

Live occurrence: TB 2.1's `large-scale-text-editing` task embeds Vim
macros (`:%normal! @a`). In the tb21-solo-1 run the trial died at
launch:

```
RuntimeLaunchError: buzz messages send ... exited 1:
{"error":"user_error","message":"mention '@a' does not match a current channel member; retry with --mention <pubkey>"}
```

Any TB task whose statement contains @-syntax is silently zeroed this
way.

## Fix

Pass the orchestrator's pubkey as an explicit `--mention` when posting
the task. The CLI demotes unresolved @-tokens in the text to
presentation-only when any explicit identity is supplied, so delivery
still targets exactly the orchestrator and every @-token in the task
statement becomes inert. The harness already holds the orchestrator's
`AgentCredential` (it writes that pubkey into the worker roster tables),
so no persistence is needed — fresh key per trial, fresh `--mention` per
trial.

Verified both halves against a live relay: a fenced `@a` without
`--mention` still hard-fails (the resolver is not markdown-aware); the
same content with an explicit `--mention` sends clean with
`mention_pubkeys` containing only the target.

## Testing

- `benchmarks/harbor-buzz-orchestra`: full pytest suite — 35 passed (34
baseline + new `test_send_mentions_by_pubkey_so_task_text_stays_inert`),
ruff clean. Run against `origin/main` 769ac70b with exactly this patch
applied.
- `testbed`: full pytest suite — 23 passed, 1 skipped; ruff clean.

## Acceptance

A task statement containing arbitrary @-tokens (Vim registers, emails,
decorators) launches and delivers to the orchestrator instead of dying
in `_send`.

Originating Buzz thread:

`buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b`

Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
This commit is contained in:
Tyler
2026-08-06 22:27:29 -04:00
committed by GitHub
co-authored by Eva
parent ad923353a2
commit f53bbd1152
2 changed files with 52 additions and 7 deletions
@@ -157,9 +157,17 @@ class BuzzContainerRuntime:
)
# The task arrives exactly as it would in production Buzz: a
# user prompt @mentioning the orchestrator. The harness never
# speaks as any agent.
# speaks as any agent. The orchestrator is mentioned by pubkey,
# not by name resolution: task text is untrusted payload, and any
# @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise
# fail member resolution and kill the trial before the agent
# ever saw the task. An explicit --mention demotes unresolved
# @-tokens in the text to presentation-only.
await self._send(
trial.user, trial, f"@{orchestrator.agent_id} {instruction}"
trial.user,
trial,
f"@{orchestrator.agent_id} {instruction}",
mention=orchestrator.nostr_pubkey,
)
final_message = await asyncio.wait_for(
self._wait_for_done(environment, orchestrator, trial, agents + infra),
@@ -519,18 +527,24 @@ class BuzzContainerRuntime:
)
async def _send(
self, credential: AgentCredential, trial: TrialHandle, content: str
self,
credential: AgentCredential,
trial: TrialHandle,
content: str,
*,
mention: str | None = None,
) -> None:
await self._buzz_json(
credential,
trial,
args = [
"messages",
"send",
"--channel",
trial.channel_id,
"--content",
content,
)
]
if mention is not None:
args += ["--mention", mention]
await self._buzz_json(credential, trial, *args)
async def _buzz_json(
self, credential: AgentCredential, trial: TrialHandle, *args: str
@@ -372,6 +372,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped(
assert bool(probed) == (condition == "M1-hello-world")
async def test_send_mentions_by_pubkey_so_task_text_stays_inert(
tmp_path, monkeypatch
):
"""Task text is untrusted payload: `:%normal! @a` in a task statement must
not be fed to member-name resolution (it would fail and kill the trial).
An explicit --mention pins delivery to the orchestrator's pubkey."""
rt = runtime(tmp_path)
orch = credential("orch-1", "orchestrator", "orch-model")
trial = trial_handle((orch,))
calls = []
async def buzz_json(credential, trial, *args):
calls.append(args)
return {}
monkeypatch.setattr(rt, "_buzz_json", buzz_json)
await rt._send(
trial.user,
trial,
"@orch-1 run `:%normal! @a` on the file",
mention=orch.nostr_pubkey,
)
assert calls[-1][-2:] == ("--mention", "pubkey-orch-1")
# Without an explicit mention the send is unchanged (name resolution).
await rt._send(trial.user, trial, "plain content")
assert "--mention" not in calls[-1]
assert calls[-1][-2:] == ("--content", "plain content")
async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
rt = runtime(tmp_path, poll_seconds=0)
orch = credential("orch-1", "orchestrator", "orch-model")