mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(bench): mention the orchestrator by pubkey when posting the task
The harness posts each trial's task via `buzz messages send`, relying on `@<orchestrator-id>` name resolution. Task text is untrusted payload: when it contains @-tokens of its own — TB 2.1's large-scale-text-editing task embeds Vim macros like `:%normal! @a` — the CLI's mention resolver rejects `@a` as an unknown channel member and refuses to send, killing the trial with RuntimeLaunchError before the agent ever saw the task. Pass the orchestrator's pubkey as an explicit --mention instead. The CLI demotes unresolved @-tokens in the text to presentation-only when any explicit identity is supplied, so delivery still targets exactly the orchestrator and every @-token in the task statement becomes inert. Live repro on the relay confirmed both halves: a fenced `@a` without --mention still hard-fails (the resolver is not markdown-aware); the same content with an explicit --mention sends clean. - benchmarks/harbor-buzz-orchestra: full pytest suite, 35 passed (34 baseline + new test), ruff clean — validated against origin/main769ac70bwith this patch applied - testbed: full pytest suite, 23 passed / 1 skipped, ruff clean Originating Buzz thread: buzz://message?channel=c3252dd2-0142-4e01-88c7-a2183c3960a5&id=74a65a0990fd2197882b66b5ea2707169d4a3dbd2020d1610c45150fb99f140b Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> (cherry picked from commite1953e2198)
This commit is contained in:
@@ -365,9 +365,17 @@ class BuzzContainerRuntime:
|
||||
)
|
||||
# The task arrives exactly as it would in production Buzz: a
|
||||
# user prompt @mentioning the orchestrator. The harness never
|
||||
# speaks as any agent.
|
||||
# speaks as any agent. The orchestrator is mentioned by pubkey,
|
||||
# not by name resolution: task text is untrusted payload, and any
|
||||
# @-token inside it (e.g. Vim's `:%normal! @a`) would otherwise
|
||||
# fail member resolution and kill the trial before the agent
|
||||
# ever saw the task. An explicit --mention demotes unresolved
|
||||
# @-tokens in the text to presentation-only.
|
||||
await self._send(
|
||||
trial.user, trial, f"@{orchestrator.agent_id} {instruction}"
|
||||
trial.user,
|
||||
trial,
|
||||
f"@{orchestrator.agent_id} {instruction}",
|
||||
mention=orchestrator.nostr_pubkey,
|
||||
)
|
||||
final_message = await asyncio.wait_for(
|
||||
self._wait_for_done(
|
||||
@@ -1223,13 +1231,20 @@ class BuzzContainerRuntime:
|
||||
)
|
||||
|
||||
async def _send(
|
||||
self, credential: AgentCredential, trial: TrialHandle, content: str
|
||||
self,
|
||||
credential: AgentCredential,
|
||||
trial: TrialHandle,
|
||||
content: str,
|
||||
*,
|
||||
mention: str | None = None,
|
||||
) -> None:
|
||||
await self._buzz_json(
|
||||
credential, trial,
|
||||
args = [
|
||||
"messages", "send", "--channel", trial.channel_id,
|
||||
"--content", content,
|
||||
)
|
||||
]
|
||||
if mention is not None:
|
||||
args += ["--mention", mention]
|
||||
await self._buzz_json(credential, trial, *args)
|
||||
|
||||
async def _buzz_json(
|
||||
self, credential: AgentCredential, trial: TrialHandle, *args: str
|
||||
|
||||
@@ -915,6 +915,37 @@ async def test_m1_output_probe_matches_grader_and_is_condition_scoped(
|
||||
assert bool(probed) == (condition == "M1-hello-world")
|
||||
|
||||
|
||||
async def test_send_mentions_by_pubkey_so_task_text_stays_inert(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""Task text is untrusted payload: `:%normal! @a` in a task statement must
|
||||
not be fed to member-name resolution (it would fail and kill the trial).
|
||||
An explicit --mention pins delivery to the orchestrator's pubkey."""
|
||||
rt = runtime(tmp_path)
|
||||
orch = credential("orch-1", "orchestrator", "orch-model")
|
||||
trial = trial_handle((orch,))
|
||||
calls = []
|
||||
|
||||
async def buzz_json(credential, trial, *args):
|
||||
calls.append(args)
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(rt, "_buzz_json", buzz_json)
|
||||
|
||||
await rt._send(
|
||||
trial.user,
|
||||
trial,
|
||||
"@orch-1 run `:%normal! @a` on the file",
|
||||
mention=orch.nostr_pubkey,
|
||||
)
|
||||
assert calls[-1][-2:] == ("--mention", "pubkey-orch-1")
|
||||
|
||||
# Without an explicit mention the send is unchanged (name resolution).
|
||||
await rt._send(trial.user, trial, "plain content")
|
||||
assert "--mention" not in calls[-1]
|
||||
assert calls[-1][-2:] == ("--content", "plain content")
|
||||
|
||||
|
||||
async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
|
||||
rt = runtime(tmp_path, poll_seconds=0)
|
||||
orch = credential("orch-1", "orchestrator", "orch-model")
|
||||
|
||||
Reference in New Issue
Block a user