From d403b716976f4e72d42dc0ae4117f9b6d16ced12 Mon Sep 17 00:00:00 2001 From: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Date: Tue, 11 Aug 2026 10:18:20 -0400 Subject: [PATCH] bench: expose task time budget to agents Add a manifest-hashed timing_signal opt-in that appends the effective Harbor agent timeout to the posted task. Resolve the task's own timeout_sec from its downloaded task.toml and include the run's timeout multiplier; existing manifests retain byte-identical prompts and condition hashes. Co-authored-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Signed-off-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> --- .../scripts/run_leaderboard.py | 3 ++ .../src/harbor_buzz_orchestra/agent.py | 32 ++++++++++++++- .../src/harbor_buzz_orchestra/manifest.py | 6 +++ .../harbor-buzz-orchestra/tests/test_agent.py | 40 +++++++++++++++++++ .../tests/test_container_runtime.py | 8 ---- .../tests/test_manifest.py | 11 +++++ .../tests/test_run_leaderboard.py | 1 + 7 files changed, 92 insertions(+), 9 deletions(-) diff --git a/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py b/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py index 8689007d4..c88a9d187 100755 --- a/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py +++ b/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py @@ -233,6 +233,9 @@ def build_command( "buzz_agent_binary": agent_binaries["buzz-agent"], "buzz_dev_mcp_binary": agent_binaries["buzz-dev-mcp"], "buzz_cli_binary": binaries["buzz"], + "timeout_multiplier": ( + args.timeout_multiplier if args.timeout_multiplier is not None else 1.0 + ), "run_id": args.job_name, } if args.relay_gateway: diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py index 3d1c81364..c3c80934a 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +import tomllib from typing import Any from harbor.agents.base import BaseAgent @@ -39,6 +40,7 @@ class BuzzOrchestraAgent(BaseAgent): buzz_cli_binary: str = "buzz", relay_gateway: str = "", forwarder_binary: str = "relay-forwarder", + timeout_multiplier: float = 1.0, run_id: str | None = None, **kwargs: Any, ) -> None: @@ -59,6 +61,9 @@ class BuzzOrchestraAgent(BaseAgent): forwarder_binary, ) self.run_id = run_id + self.timeout_multiplier = float(timeout_multiplier) + if self.timeout_multiplier <= 0: + raise ValueError("timeout_multiplier must be positive") @staticmethod def name() -> str: @@ -149,6 +154,31 @@ class BuzzOrchestraAgent(BaseAgent): if self.provisioner is not None: self.provisioner.healthcheck() + def _instruction_with_timing( + self, instruction: str, environment: BaseEnvironment + ) -> str: + if not self.manifest.timing_signal: + return instruction + task_config = Path(environment.environment_dir).parent / "task.toml" + try: + timeout = float( + tomllib.loads(task_config.read_text(encoding="utf-8"))["agent"][ + "timeout_sec" + ] + ) + except ( + OSError, + tomllib.TOMLDecodeError, + KeyError, + TypeError, + ValueError, + ) as error: + raise RuntimeError( + f"cannot resolve task agent timeout from {task_config}" + ) from error + total_seconds = round(timeout * self.timeout_multiplier) + return instruction + f"\n\n[Trial timing: {total_seconds}s total.]" + async def run( self, instruction: str, @@ -178,7 +208,7 @@ class BuzzOrchestraAgent(BaseAgent): raise RuntimeError("provisioner returned a handle for a different manifest") try: result = await self.runtime.run( - instruction=instruction, + instruction=self._instruction_with_timing(instruction, environment), environment=environment, manifest=self.manifest, trial=handle, diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py index f8f27f1fb..2a759b538 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py @@ -254,6 +254,10 @@ class ExperimentManifest(StrictModel): roster: tuple[AgentClass, ...] = Field(min_length=1) prices: dict[str, Price] trial_budget: TrialBudget + # Opt-in because the line changes the task prompt and therefore the condition. + # False is omitted from canonical bytes below so manifests written before this + # field existed retain their condition identity. + timing_signal: bool = False # Absent and empty are the same condition and hash identically, matching # how ``generation`` behaves on AgentClass. environment: EnvironmentOverrides = Field(default_factory=EnvironmentOverrides) @@ -285,6 +289,8 @@ class ExperimentManifest(StrictModel): def canonical_bytes(self) -> bytes: """Return stable UTF-8 JSON independent of YAML formatting and key order.""" data = self.model_dump(mode="json", exclude_none=False) + if not self.timing_signal: + data.pop("timing_signal", None) # An unpinned `thinking_effort` is dropped rather than serialised as # null, so that opening the effort axis did not re-identify every # condition that does not use it. diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_agent.py b/benchmarks/harbor-buzz-orchestra/tests/test_agent.py index b305344c5..ebb5e9400 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_agent.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_agent.py @@ -103,6 +103,46 @@ async def test_agent_lifecycle_and_context(tmp_path, manifest_data): assert context.metadata["trial_id"] == str(context_id) +async def test_timing_signal_uses_task_agent_timeout(tmp_path, manifest_data): + task_dir = tmp_path / "task" + environment_dir = task_dir / "environment" + environment_dir.mkdir(parents=True) + (task_dir / "task.toml").write_text("[agent]\ntimeout_sec = 900.0\n") + environment = SimpleNamespace( + context_id=uuid4(), + environment_name="hello-world", + environment_dir=environment_dir, + ) + manifest_data["timing_signal"] = True + agent = BuzzOrchestraAgent( + logs_dir=tmp_path, + manifest=manifest_data, + provisioner=Provisioner(), + runtime=Runtime(), + timeout_multiplier=3.0, + ) + await agent.run("solve it", environment, AgentContext()) + + assert agent.runtime.called["instruction"] == ( + "solve it\n\n[Trial timing: 2700s total.]" + ) + + +async def test_timing_signal_off_preserves_instruction(tmp_path, manifest_data): + provisioner, runtime = Provisioner(), Runtime() + environment = SimpleNamespace(context_id=uuid4(), environment_name="hello-world") + agent = BuzzOrchestraAgent( + logs_dir=tmp_path, + manifest=manifest_data, + provisioner=provisioner, + runtime=runtime, + ) + + await agent.run("solve it", environment, AgentContext()) + + assert runtime.called["instruction"] == "solve it" + + async def test_teardown_runs_when_runtime_fails(tmp_path, manifest_data): provisioner, runtime, context_id = ( Provisioner(), diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index 0053f93d2..ede731269 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -350,8 +350,6 @@ async def test_install_stack_requires_the_ca_bundle_on_disk(tmp_path): async def test_forwarder_bridges_the_canonical_relay_address(tmp_path): - from harbor_buzz_orchestra.container_runtime import FORWARDER - forwarder = tmp_path / "relay-forwarder" forwarder.write_text("ELF") rt = runtime( @@ -1461,8 +1459,6 @@ class _BindsAfter(Environment): self._log = "" async def exec(self, command, env=None, **kwargs): - from harbor_buzz_orchestra.container_runtime import FORWARDER - self.commands.append((command, env)) if command.startswith(": >"): self._log = "" @@ -1504,8 +1500,6 @@ async def test_forwarder_gives_up_after_the_attempt_budget(tmp_path, monkeypatch async def test_forwarder_does_not_retry_other_failures(tmp_path, monkeypatch): """Only EADDRINUSE is transient; a silent forwarder is a real fault.""" - from harbor_buzz_orchestra.container_runtime import FORWARDER - rt = _forwarder_runtime(tmp_path) monkeypatch.setattr(type(rt), "readiness_timeout_seconds", 0.0, raising=False) rt.readiness_timeout_seconds = 0.0 @@ -1545,8 +1539,6 @@ async def test_forwarder_from_an_earlier_phase_is_adopted(tmp_path): without SO_REUSEADDR, so its accepted sockets hold the port in TIME_WAIT for 60s after it exits. """ - from harbor_buzz_orchestra.container_runtime import FORWARDER - rt = _forwarder_runtime(tmp_path) environment = _HasLiveForwarder(pid=4242) diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py index 552b7d33f..b668a0171 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py @@ -49,6 +49,17 @@ def test_pinned_effort_is_part_of_the_condition_hash(manifest_data): assert pinned_manifest.sha256 != baseline.sha256 +def test_timing_signal_is_opt_in_and_part_of_the_condition_hash(manifest_data): + baseline = ExperimentManifest.load(copy.deepcopy(manifest_data)) + explicit_off = ExperimentManifest.load({**manifest_data, "timing_signal": False}) + enabled = ExperimentManifest.load({**manifest_data, "timing_signal": True}) + + assert b"timing_signal" not in baseline.canonical_bytes() + assert explicit_off.sha256 == baseline.sha256 + assert b'"timing_signal":true' in enabled.canonical_bytes() + assert enabled.sha256 != baseline.sha256 + + MANIFEST_DIR = pathlib.Path(__file__).resolve().parents[1] / "manifests" SHIPPED = sorted(MANIFEST_DIR.glob("*.yaml")) diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py b/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py index 67240f641..f4d00d409 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py @@ -107,6 +107,7 @@ def test_command_uses_standard_settings_only(args, binaries, agent_binaries): assert any(k.startswith("buzz_acp_binary=") for k in kwargs) assert any(k.startswith("buzz_agent_binary=") for k in kwargs) assert any(k.startswith("buzz_dev_mcp_binary=") for k in kwargs) + assert "timeout_multiplier=1.0" in kwargs def test_agent_binaries_must_exist(tmp_path):