mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(bench): mention the orchestrator by pubkey when posting the task (#5136)
## Problem
The harness posts each trial's task via `buzz messages send`, relying on
`@<orchestrator-id>` name resolution. Task text is untrusted payload:
when it contains @-tokens of its own, the CLI's mention resolver tries
to resolve them as channel members, fails, and refuses to send — killing
the trial with `RuntimeLaunchError` before the agent ever saw the task.
Live occurrence: TB 2.1's `large-scale-text-editing` task embeds Vim
macros (`:%normal! @a`). In the tb21-solo-1 run the trial died at
launch:
```
RuntimeLaunchError: buzz messages send ... exited 1:
{"error":"user_error","message":"mention '@a' does not match a current channel member; retry with --mention <pubkey>"}
```
Any TB task whose statement contains @-syntax is silently zeroed this
way.
## Fix
Pass the orchestrator's pubkey as an explicit `--mention` when posting
the task. The CLI demotes unresolved @-tokens in the text to
presentation-only when any explicit identity is supplied, so delivery
still targets exactly the orchestrator and every @-token in the task
statement becomes inert. The harness already holds the orchestrator's
`AgentCredential` (it writes that pubkey into the worker roster tables),
so no persistence is needed — fresh key per trial, fresh `--mention` per
trial.
Verified both halves against a live relay: a fenced `@a` without
`--mention` still hard-fails (the resolver is not markdown-aware); the
same content with an explicit `--mention` sends clean with
`mention_pubkeys` containing only the target.
## Testing
- `benchmarks/harbor-buzz-orchestra`: full pytest suite — 35 passed (34
baseline + new `test_send_mentions_by_pubkey_so_task_text_stays_inert`),
ruff clean. Run against `origin/main` 769ac70b with exactly this patch
applied.
- `testbed`: full pytest suite — 23 passed, 1 skipped; ruff clean.
## Acceptance
A task statement containing arbitrary @-tokens (Vim registers, emails,
decorators) launches and delivers to the orchestrator instead of dying
in `_send`.
Originating Buzz thread:
`buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b`
Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
This commit is contained in:
@@ -157,9 +157,17 @@ class BuzzContainerRuntime:
|
|||||||
)
|
)
|
||||||
# The task arrives exactly as it would in production Buzz: a
|
# The task arrives exactly as it would in production Buzz: a
|
||||||
# user prompt @mentioning the orchestrator. The harness never
|
# user prompt @mentioning the orchestrator. The harness never
|
||||||
# speaks as any agent.
|
# speaks as any agent. The orchestrator is mentioned by pubkey,
|
||||||
|
# not by name resolution: task text is untrusted payload, and any
|
||||||
|
# @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise
|
||||||
|
# fail member resolution and kill the trial before the agent
|
||||||
|
# ever saw the task. An explicit --mention demotes unresolved
|
||||||
|
# @-tokens in the text to presentation-only.
|
||||||
await self._send(
|
await self._send(
|
||||||
trial.user, trial, f"@{orchestrator.agent_id} {instruction}"
|
trial.user,
|
||||||
|
trial,
|
||||||
|
f"@{orchestrator.agent_id} {instruction}",
|
||||||
|
mention=orchestrator.nostr_pubkey,
|
||||||
)
|
)
|
||||||
final_message = await asyncio.wait_for(
|
final_message = await asyncio.wait_for(
|
||||||
self._wait_for_done(environment, orchestrator, trial, agents + infra),
|
self._wait_for_done(environment, orchestrator, trial, agents + infra),
|
||||||
@@ -519,18 +527,24 @@ class BuzzContainerRuntime:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def _send(
|
async def _send(
|
||||||
self, credential: AgentCredential, trial: TrialHandle, content: str
|
self,
|
||||||
|
credential: AgentCredential,
|
||||||
|
trial: TrialHandle,
|
||||||
|
content: str,
|
||||||
|
*,
|
||||||
|
mention: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
await self._buzz_json(
|
args = [
|
||||||
credential,
|
|
||||||
trial,
|
|
||||||
"messages",
|
"messages",
|
||||||
"send",
|
"send",
|
||||||
"--channel",
|
"--channel",
|
||||||
trial.channel_id,
|
trial.channel_id,
|
||||||
"--content",
|
"--content",
|
||||||
content,
|
content,
|
||||||
)
|
]
|
||||||
|
if mention is not None:
|
||||||
|
args += ["--mention", mention]
|
||||||
|
await self._buzz_json(credential, trial, *args)
|
||||||
|
|
||||||
async def _buzz_json(
|
async def _buzz_json(
|
||||||
self, credential: AgentCredential, trial: TrialHandle, *args: str
|
self, credential: AgentCredential, trial: TrialHandle, *args: str
|
||||||
|
|||||||
@@ -372,6 +372,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped(
|
|||||||
assert bool(probed) == (condition == "M1-hello-world")
|
assert bool(probed) == (condition == "M1-hello-world")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_send_mentions_by_pubkey_so_task_text_stays_inert(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""Task text is untrusted payload: `:%normal! @a` in a task statement must
|
||||||
|
not be fed to member-name resolution (it would fail and kill the trial).
|
||||||
|
An explicit --mention pins delivery to the orchestrator's pubkey."""
|
||||||
|
rt = runtime(tmp_path)
|
||||||
|
orch = credential("orch-1", "orchestrator", "orch-model")
|
||||||
|
trial = trial_handle((orch,))
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
async def buzz_json(credential, trial, *args):
|
||||||
|
calls.append(args)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
monkeypatch.setattr(rt, "_buzz_json", buzz_json)
|
||||||
|
|
||||||
|
await rt._send(
|
||||||
|
trial.user,
|
||||||
|
trial,
|
||||||
|
"@orch-1 run `:%normal! @a` on the file",
|
||||||
|
mention=orch.nostr_pubkey,
|
||||||
|
)
|
||||||
|
assert calls[-1][-2:] == ("--mention", "pubkey-orch-1")
|
||||||
|
|
||||||
|
# Without an explicit mention the send is unchanged (name resolution).
|
||||||
|
await rt._send(trial.user, trial, "plain content")
|
||||||
|
assert "--mention" not in calls[-1]
|
||||||
|
assert calls[-1][-2:] == ("--content", "plain content")
|
||||||
|
|
||||||
|
|
||||||
async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
|
async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
|
||||||
rt = runtime(tmp_path, poll_seconds=0)
|
rt = runtime(tmp_path, poll_seconds=0)
|
||||||
orch = credential("orch-1", "orchestrator", "orch-model")
|
orch = credential("orch-1", "orchestrator", "orch-model")
|
||||||
|
|||||||
Reference in New Issue
Block a user