bench: expose task time budget to agents

Add a manifest-hashed timing_signal opt-in that appends the effective Harbor agent timeout to the posted task. Resolve the task's own timeout_sec from its downloaded task.toml and include the run's timeout multiplier; existing manifests retain byte-identical prompts and condition hashes.

Co-authored-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz>
Signed-off-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz>

benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py

@@ -233,6 +233,9 @@ def build_command(
         "buzz_agent_binary": agent_binaries["buzz-agent"],
         "buzz_dev_mcp_binary": agent_binaries["buzz-dev-mcp"],
         "buzz_cli_binary": binaries["buzz"],
+        "timeout_multiplier": (
+            args.timeout_multiplier if args.timeout_multiplier is not None else 1.0
+        ),
         "run_id": args.job_name,
     }
     if args.relay_gateway:

benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+import tomllib
 from typing import Any
 
 from harbor.agents.base import BaseAgent
@@ -39,6 +40,7 @@ class BuzzOrchestraAgent(BaseAgent):
         buzz_cli_binary: str = "buzz",
         relay_gateway: str = "",
         forwarder_binary: str = "relay-forwarder",
+        timeout_multiplier: float = 1.0,
         run_id: str | None = None,
         **kwargs: Any,
     ) -> None:
@@ -59,6 +61,9 @@ class BuzzOrchestraAgent(BaseAgent):
             forwarder_binary,
         )
         self.run_id = run_id
+        self.timeout_multiplier = float(timeout_multiplier)
+        if self.timeout_multiplier <= 0:
+            raise ValueError("timeout_multiplier must be positive")
 
     @staticmethod
     def name() -> str:
@@ -149,6 +154,31 @@ class BuzzOrchestraAgent(BaseAgent):
         if self.provisioner is not None:
             self.provisioner.healthcheck()
 
+    def _instruction_with_timing(
+        self, instruction: str, environment: BaseEnvironment
+    ) -> str:
+        if not self.manifest.timing_signal:
+            return instruction
+        task_config = Path(environment.environment_dir).parent / "task.toml"
+        try:
+            timeout = float(
+                tomllib.loads(task_config.read_text(encoding="utf-8"))["agent"][
+                    "timeout_sec"
+                ]
+            )
+        except (
+            OSError,
+            tomllib.TOMLDecodeError,
+            KeyError,
+            TypeError,
+            ValueError,
+        ) as error:
+            raise RuntimeError(
+                f"cannot resolve task agent timeout from {task_config}"
+            ) from error
+        total_seconds = round(timeout * self.timeout_multiplier)
+        return instruction + f"\n\n[Trial timing: {total_seconds}s total.]"
+
     async def run(
         self,
         instruction: str,
@@ -178,7 +208,7 @@ class BuzzOrchestraAgent(BaseAgent):
             raise RuntimeError("provisioner returned a handle for a different manifest")
         try:
             result = await self.runtime.run(
-                instruction=instruction,
+                instruction=self._instruction_with_timing(instruction, environment),
                 environment=environment,
                 manifest=self.manifest,
                 trial=handle,

benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/manifest.py

@@ -254,6 +254,10 @@ class ExperimentManifest(StrictModel):
     roster: tuple[AgentClass, ...] = Field(min_length=1)
     prices: dict[str, Price]
     trial_budget: TrialBudget
+    # Opt-in because the line changes the task prompt and therefore the condition.
+    # False is omitted from canonical bytes below so manifests written before this
+    # field existed retain their condition identity.
+    timing_signal: bool = False
     # Absent and empty are the same condition and hash identically, matching
     # how ``generation`` behaves on AgentClass.
     environment: EnvironmentOverrides = Field(default_factory=EnvironmentOverrides)
@@ -285,6 +289,8 @@ class ExperimentManifest(StrictModel):
     def canonical_bytes(self) -> bytes:
         """Return stable UTF-8 JSON independent of YAML formatting and key order."""
         data = self.model_dump(mode="json", exclude_none=False)
+        if not self.timing_signal:
+            data.pop("timing_signal", None)
         # An unpinned `thinking_effort` is dropped rather than serialised as
         # null, so that opening the effort axis did not re-identify every
         # condition that does not use it.

benchmarks/harbor-buzz-orchestra/tests/test_agent.py

@@ -103,6 +103,46 @@ async def test_agent_lifecycle_and_context(tmp_path, manifest_data):
     assert context.metadata["trial_id"] == str(context_id)
 
 
+async def test_timing_signal_uses_task_agent_timeout(tmp_path, manifest_data):
+    task_dir = tmp_path / "task"
+    environment_dir = task_dir / "environment"
+    environment_dir.mkdir(parents=True)
+    (task_dir / "task.toml").write_text("[agent]\ntimeout_sec = 900.0\n")
+    environment = SimpleNamespace(
+        context_id=uuid4(),
+        environment_name="hello-world",
+        environment_dir=environment_dir,
+    )
+    manifest_data["timing_signal"] = True
+    agent = BuzzOrchestraAgent(
+        logs_dir=tmp_path,
+        manifest=manifest_data,
+        provisioner=Provisioner(),
+        runtime=Runtime(),
+        timeout_multiplier=3.0,
+    )
+    await agent.run("solve it", environment, AgentContext())
+
+    assert agent.runtime.called["instruction"] == (
+        "solve it\n\n[Trial timing: 2700s total.]"
+    )
+
+
+async def test_timing_signal_off_preserves_instruction(tmp_path, manifest_data):
+    provisioner, runtime = Provisioner(), Runtime()
+    environment = SimpleNamespace(context_id=uuid4(), environment_name="hello-world")
+    agent = BuzzOrchestraAgent(
+        logs_dir=tmp_path,
+        manifest=manifest_data,
+        provisioner=provisioner,
+        runtime=runtime,
+    )
+
+    await agent.run("solve it", environment, AgentContext())
+
+    assert runtime.called["instruction"] == "solve it"
+
+
 async def test_teardown_runs_when_runtime_fails(tmp_path, manifest_data):
     provisioner, runtime, context_id = (
         Provisioner(),

benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py

@@ -350,8 +350,6 @@ async def test_install_stack_requires_the_ca_bundle_on_disk(tmp_path):
 
 
 async def test_forwarder_bridges_the_canonical_relay_address(tmp_path):
-    from harbor_buzz_orchestra.container_runtime import FORWARDER
-
     forwarder = tmp_path / "relay-forwarder"
     forwarder.write_text("ELF")
     rt = runtime(
@@ -1461,8 +1459,6 @@ class _BindsAfter(Environment):
         self._log = ""
 
     async def exec(self, command, env=None, **kwargs):
-        from harbor_buzz_orchestra.container_runtime import FORWARDER
-
         self.commands.append((command, env))
         if command.startswith(": >"):
             self._log = ""
@@ -1504,8 +1500,6 @@ async def test_forwarder_gives_up_after_the_attempt_budget(tmp_path, monkeypatch
 
 async def test_forwarder_does_not_retry_other_failures(tmp_path, monkeypatch):
     """Only EADDRINUSE is transient; a silent forwarder is a real fault."""
-    from harbor_buzz_orchestra.container_runtime import FORWARDER
-
     rt = _forwarder_runtime(tmp_path)
     monkeypatch.setattr(type(rt), "readiness_timeout_seconds", 0.0, raising=False)
     rt.readiness_timeout_seconds = 0.0
@@ -1545,8 +1539,6 @@ async def test_forwarder_from_an_earlier_phase_is_adopted(tmp_path):
     without SO_REUSEADDR, so its accepted sockets hold the port in TIME_WAIT
     for 60s after it exits.
     """
-    from harbor_buzz_orchestra.container_runtime import FORWARDER
-
     rt = _forwarder_runtime(tmp_path)
     environment = _HasLiveForwarder(pid=4242)
 

benchmarks/harbor-buzz-orchestra/tests/test_manifest.py

@@ -49,6 +49,17 @@ def test_pinned_effort_is_part_of_the_condition_hash(manifest_data):
     assert pinned_manifest.sha256 != baseline.sha256
 
 
+def test_timing_signal_is_opt_in_and_part_of_the_condition_hash(manifest_data):
+    baseline = ExperimentManifest.load(copy.deepcopy(manifest_data))
+    explicit_off = ExperimentManifest.load({**manifest_data, "timing_signal": False})
+    enabled = ExperimentManifest.load({**manifest_data, "timing_signal": True})
+
+    assert b"timing_signal" not in baseline.canonical_bytes()
+    assert explicit_off.sha256 == baseline.sha256
+    assert b'"timing_signal":true' in enabled.canonical_bytes()
+    assert enabled.sha256 != baseline.sha256
+
+
 MANIFEST_DIR = pathlib.Path(__file__).resolve().parents[1] / "manifests"
 SHIPPED = sorted(MANIFEST_DIR.glob("*.yaml"))
 

benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py

@@ -107,6 +107,7 @@ def test_command_uses_standard_settings_only(args, binaries, agent_binaries):
     assert any(k.startswith("buzz_acp_binary=") for k in kwargs)
     assert any(k.startswith("buzz_agent_binary=") for k in kwargs)
     assert any(k.startswith("buzz_dev_mcp_binary=") for k in kwargs)
+    assert "timeout_multiplier=1.0" in kwargs
 
 
 def test_agent_binaries_must_exist(tmp_path):