mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
moar benchmarks
Co-authored-by: Cursor <cursoragent@cursor.com> Signed-off-by: Atish Patel <atishpatel2012@gmail.com>
This commit is contained in:
Atish Patel
@@ -407,7 +407,7 @@ of every turn and delete most of what the personas currently spend ~600 tokens
|
||||
each rebutting. Worth revisiting if A1n shows `[Base]` is materially hurting,
|
||||
since at that point "production parity" is preserving a known handicap.
|
||||
|
||||
**Endpoint names for Opus 5 are not established.** `databricks-live.json`
|
||||
**Endpoint names for Opus 5 are not established.** `databricks-example.json`
|
||||
currently maps only `databricks-gpt-5-6-sol` and `databricks-gpt-5-6-luna`.
|
||||
Every Tier-2 and Tier-3 condition above assumes an Opus 5 endpoint on the same
|
||||
gateway; A3, B1–B5 and C1–C5 are blocked until it exists and its list prices
|
||||
|
||||
@@ -32,8 +32,9 @@
|
||||
#
|
||||
# THIS RUNS UNDER THE PATCHED HARBOR. LHTB needs continue_until_timeout honored
|
||||
# (patches/apply_continue_until_timeout.py); stock harbor ignores the flag and
|
||||
# agents self-report DONE minutes into hour-long budgets. lhtb46.sh refuses to
|
||||
# start on an unpatched harbor -- do not bypass that check.
|
||||
# agents self-report DONE minutes into hour-long budgets. Gate the run on
|
||||
# patches/gatecheck.py -- an unpatched harbor scores this cell far below the
|
||||
# leaderboard and the run looks merely bad rather than misconfigured.
|
||||
schema_version: "1"
|
||||
condition: lhtb-or-kimi-lead-2deepseek-high
|
||||
roster:
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
# genuine finding about hierarchy, not a failed rewrite.
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names and resolve via
|
||||
# testbed/endpoints/databricks-live.json. buzz-agent's `databricks_v2` provider
|
||||
# testbed/endpoints/databricks-example.json. buzz-agent's `databricks_v2` provider
|
||||
# picks the route per model from the endpoint name; see tb-solo-opus.yaml for the
|
||||
# detail.
|
||||
schema_version: "1"
|
||||
|
||||
@@ -7,11 +7,10 @@
|
||||
# to us through Databricks (`databricks-claude-opus-5`); it is absent from
|
||||
# openai-live.json, and anthropic-live.json carries only Sonnet 4.6 and Haiku
|
||||
# 4.5. Terra is only served direct from OpenAI (`gpt-5.6-terra`); it is absent
|
||||
# from databricks-live.json. So there is no single endpoint config that can serve
|
||||
# both seats, and this manifest resolves against a MERGED config --
|
||||
# testbed/endpoints/mixed-opus-terra.json -- which needs BOTH DATABRICKS_TOKEN
|
||||
# and OPENAI_API_KEY exported. Neither sweep.sh nor sweep-openai.sh does that;
|
||||
# use sweep-mixed.sh.
|
||||
# from the Databricks catalog. So there is no single-provider endpoint config
|
||||
# that can serve both seats, and this manifest resolves against a MERGED config
|
||||
# naming Databricks and OpenAI endpoints in one file -- which needs BOTH
|
||||
# DATABRICKS_TOKEN and OPENAI_API_KEY exported. Pass it with --endpoint-config.
|
||||
#
|
||||
# What that costs interpretively:
|
||||
#
|
||||
@@ -36,7 +35,7 @@
|
||||
# solo opus (0.851), that is worth knowing whatever the routes were, and it is
|
||||
# the last untested corner of the delegation hypothesis.
|
||||
#
|
||||
# Endpoint names resolve via testbed/endpoints/mixed-opus-terra.json.
|
||||
# Endpoint names resolve via the merged endpoint config described above.
|
||||
schema_version: "1"
|
||||
condition: tb-gt-opus-2terra
|
||||
roster:
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
# from its 3-agent sibling in a way the score cannot explain, check `free -m`
|
||||
# and container exit-137 before believing the coordination story.
|
||||
#
|
||||
# Endpoint names resolve via testbed/endpoints/databricks-live.json.
|
||||
# Endpoint names resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-gt-opus-3luna
|
||||
roster:
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# G1sh -- goosetown personas, a gpt-5.6-sol lead over one read-only luna scout
|
||||
# and one luna worker, with every seat pinned to `thinking_effort: high`.
|
||||
# Single provider, single route, direct from OpenAI.
|
||||
#
|
||||
# BUILT FOR LHTB, not Terminal-Bench. Run it with the patched
|
||||
# `continue_until_timeout` (docs/09) at `--timeout-multiplier 3.0` and `-n 8`.
|
||||
#
|
||||
# WHAT THIS ISOLATES. This is tb-gt-sol-2terra-high.yaml with the two worker
|
||||
# seats swapped from terra to luna, and nothing else touched: same lead model,
|
||||
# same personas byte-for-byte, same effort on all three seats, same route, same
|
||||
# container, same clock, same concurrency. Against that cell it is a clean
|
||||
# WORKER-MODEL delta at the top of the effort ramp. Against tb-solo-sol-high.yaml
|
||||
# it is the team-vs-solo question with cheap workers instead of mid-priced ones.
|
||||
#
|
||||
# The high-effort comparison set this belongs to -- all four at n=8, effort high,
|
||||
# OpenAI route, so only the roster moves:
|
||||
#
|
||||
# tb-solo-sol-high 1x sol (A2h)
|
||||
# tb-gt-sol-2luna-high sol lead + 2x luna (this cell)
|
||||
# tb-gt-sol-luna-terra-high sol lead + luna scout + terra worker
|
||||
# tb-gt-sol-2terra-high sol lead + 2x terra (G1sth)
|
||||
#
|
||||
# DO NOT compare this cell against tb-gt-sol-2luna.yaml and call the delta an
|
||||
# effort effect. That manifest is the G-wave cell and pins no effort, but it also
|
||||
# ran at n=4 in the LHTB-46 wave (docs/09 §7.3) -- so the pair moves effort AND
|
||||
# concurrency, and mismatched `-n` compressed between-condition spread by 0.09 in
|
||||
# the TB wave. Compare only within the four cells listed above.
|
||||
#
|
||||
# All three seats are pinned, not just the lead. A lead thinking harder than the
|
||||
# seats it delegates to is a different condition and needs its own manifest.
|
||||
#
|
||||
# Effort support: `gpt-5.6-sol` and `gpt-5.6-luna` both match the `gpt-5.6`
|
||||
# family token in config.rs, whose supported set includes high. A1h ran luna at
|
||||
# `high` and A2x ran sol at `xhigh` on this route -- neither value is rejected
|
||||
# nor silently clamped.
|
||||
#
|
||||
# Endpoint names resolve via testbed/endpoints/openai-live.json.
|
||||
schema_version: "1"
|
||||
condition: tb-gt-sol-2luna-high
|
||||
roster:
|
||||
- id: lead
|
||||
kind: orchestrator
|
||||
# See tb-gt-3luna.yaml: `lead`, `scout` and `worker` are all read verbatim
|
||||
# out of the "Your team" table by the personas. Renaming any of them breaks
|
||||
# addressing silently -- the @mention resolves to nobody, the send still
|
||||
# reports success, and the trial stalls to its timeout.
|
||||
role: lead
|
||||
count: 1
|
||||
endpoint: gpt-5.6-sol
|
||||
model_revision: gpt-5.6-sol
|
||||
prompt:
|
||||
# Byte-identical to the file G0, G1s, G1, G1st, G1sth, G2s and G2 pin, so
|
||||
# the lead's instructions are not a variable anywhere in this wave.
|
||||
path: personas/bench/gt/gt-lead.md
|
||||
sha256: 8a27e833dec0a6d1700c5ea3100f8a81c09e5090ef0a589b9583fac40b24c05c
|
||||
generation:
|
||||
thinking_effort: high
|
||||
|
||||
- id: scout
|
||||
kind: worker
|
||||
role: scout
|
||||
count: 1
|
||||
endpoint: gpt-5.6-luna
|
||||
model_revision: gpt-5.6-luna
|
||||
prompt:
|
||||
path: personas/bench/gt/gt-scout.md
|
||||
sha256: 0359a957428dd09e56e57a7fd3fe455d860264d910705f0e4992337dde25e5a5
|
||||
generation:
|
||||
thinking_effort: high
|
||||
|
||||
- id: worker
|
||||
kind: worker
|
||||
role: worker
|
||||
count: 1
|
||||
endpoint: gpt-5.6-luna
|
||||
model_revision: gpt-5.6-luna
|
||||
prompt:
|
||||
path: personas/bench/gt/gt-worker.md
|
||||
sha256: 7a529ae58f2a2ec635fdf65fb43284b30c09d2c9dbf31e70951fe021c3b2b766
|
||||
generation:
|
||||
thinking_effort: high
|
||||
|
||||
prices:
|
||||
# POST-2026-07-30 SHEET, matching benchmark-runs/tools/rates.py. This differs
|
||||
# from tb-gt-sol-2luna.yaml and tb-gt-sol-2terra-high.yaml, which both carry
|
||||
# the launch sheet (luna 1.00/0.10/6.00, terra 2.50/0.25/15.00) because they
|
||||
# were written before the cut and must not be restated -- editing a completed
|
||||
# cell's prices changes its condition hash. The consequence is that the
|
||||
# harness' own cost_usd for THIS cell is on the current sheet while G1sth's is
|
||||
# on the launch sheet, so do not set the two harness figures side by side.
|
||||
# Reprice both from receipts through rates.py instead; the high-effort wave has
|
||||
# per-phase receipts, so that is exact (verified on G1sth: $462.36 launch ->
|
||||
# $416.49 current, with score and tokens identical).
|
||||
gpt-5.6-sol:
|
||||
# Unchanged by the cut, so this row reads the same on either sheet.
|
||||
input_per_million_usd: 5.0
|
||||
cached_input_per_million_usd: 0.5
|
||||
output_per_million_usd: 30.0
|
||||
cache_read_rate: 0.0
|
||||
gpt-5.6-luna:
|
||||
input_per_million_usd: 0.20
|
||||
cached_input_per_million_usd: 0.02
|
||||
output_per_million_usd: 1.20
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
# Identical to the solo baselines. A team genuinely needs longer than a solo
|
||||
# agent -- every handoff is a round trip -- but giving it a larger budget would
|
||||
# confound the comparison it exists to make, and it does not bind anyway:
|
||||
# Harbor enforces each task's own `[agent] timeout_sec` scaled by 3x.
|
||||
timeout_seconds: 36000
|
||||
|
||||
environment:
|
||||
# Identical in every condition -- see tb-solo-luna.yaml for the full reasoning.
|
||||
# override_cpus 4 with -n 8 is exactly 1:1 on a 32-vCPU m7a.8xlarge.
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,118 @@
|
||||
# G1slth -- goosetown personas, a gpt-5.6-sol lead over a read-only LUNA scout
|
||||
# and a TERRA worker, every seat pinned to `thinking_effort: high`. Single
|
||||
# provider, single route, direct from OpenAI.
|
||||
#
|
||||
# BUILT FOR LHTB, not Terminal-Bench. Run it with the patched
|
||||
# `continue_until_timeout` (docs/09) at `--timeout-multiplier 3.0` and `-n 8`.
|
||||
#
|
||||
# WHAT THIS ISOLATES. The two homogeneous team cells put the same model in both
|
||||
# worker seats; this one splits them, and the split is deliberate rather than
|
||||
# arbitrary. The scout seat is read-only reconnaissance -- it greps, reads and
|
||||
# summarises, so it is the seat whose output is cheapest to be wrong about and
|
||||
# the natural place for the cheap model. The worker seat actually edits and
|
||||
# builds, so it gets the stronger one. If seat-appropriate assignment is worth
|
||||
# anything, this cell is where it shows up: it should land between the all-luna
|
||||
# and all-terra cells on cost while tracking the all-terra cell on score.
|
||||
#
|
||||
# The high-effort comparison set -- all four at n=8, effort high, OpenAI route,
|
||||
# same lead, same personas, so only the roster moves:
|
||||
#
|
||||
# tb-solo-sol-high 1x sol (A2h)
|
||||
# tb-gt-sol-2luna-high sol lead + 2x luna
|
||||
# tb-gt-sol-luna-terra-high sol lead + luna scout + terra worker (this cell)
|
||||
# tb-gt-sol-2terra-high sol lead + 2x terra (G1sth)
|
||||
#
|
||||
# READ THE SCOUT/WORKER ASSIGNMENT BEFORE INTERPRETING ANY DELTA. Against
|
||||
# tb-gt-sol-2luna-high the worker seat is upgraded luna->terra; against
|
||||
# tb-gt-sol-2terra-high the scout seat is downgraded terra->luna. Those are two
|
||||
# different one-seat moves and this cell is the hinge between them, so it can be
|
||||
# read either way -- but only one seat at a time. It is NOT a "mixed vs
|
||||
# homogeneous" cell.
|
||||
#
|
||||
# DO NOT compare against any unpinned-effort or n=4 cell (docs/09 §7.3 ran at
|
||||
# n=4); that moves effort and/or concurrency alongside the roster.
|
||||
#
|
||||
# Effort support: sol, luna and terra all match the `gpt-5.6` family token in
|
||||
# config.rs, whose supported set includes high. A2x ran sol at `xhigh`, A1h ran
|
||||
# luna at `high`, A5h/B3h ran terra at `high` -- no value here is rejected nor
|
||||
# silently clamped on this route.
|
||||
#
|
||||
# Endpoint names resolve via testbed/endpoints/openai-live.json.
|
||||
schema_version: "1"
|
||||
condition: tb-gt-sol-luna-terra-high
|
||||
roster:
|
||||
- id: lead
|
||||
kind: orchestrator
|
||||
# See tb-gt-3luna.yaml: `lead`, `scout` and `worker` are all read verbatim
|
||||
# out of the "Your team" table by the personas. Renaming any of them breaks
|
||||
# addressing silently -- the @mention resolves to nobody, the send still
|
||||
# reports success, and the trial stalls to its timeout.
|
||||
role: lead
|
||||
count: 1
|
||||
endpoint: gpt-5.6-sol
|
||||
model_revision: gpt-5.6-sol
|
||||
prompt:
|
||||
# Byte-identical to the file every other gt cell pins.
|
||||
path: personas/bench/gt/gt-lead.md
|
||||
sha256: 8a27e833dec0a6d1700c5ea3100f8a81c09e5090ef0a589b9583fac40b24c05c
|
||||
generation:
|
||||
thinking_effort: high
|
||||
|
||||
- id: scout
|
||||
kind: worker
|
||||
role: scout
|
||||
count: 1
|
||||
# The CHEAP model in the read-only seat. This is the whole point of the cell.
|
||||
endpoint: gpt-5.6-luna
|
||||
model_revision: gpt-5.6-luna
|
||||
prompt:
|
||||
path: personas/bench/gt/gt-scout.md
|
||||
sha256: 0359a957428dd09e56e57a7fd3fe455d860264d910705f0e4992337dde25e5a5
|
||||
generation:
|
||||
thinking_effort: high
|
||||
|
||||
- id: worker
|
||||
kind: worker
|
||||
role: worker
|
||||
count: 1
|
||||
# The STRONGER model in the seat that edits and builds.
|
||||
endpoint: gpt-5.6-terra
|
||||
model_revision: gpt-5.6-terra
|
||||
prompt:
|
||||
path: personas/bench/gt/gt-worker.md
|
||||
sha256: 7a529ae58f2a2ec635fdf65fb43284b30c09d2c9dbf31e70951fe021c3b2b766
|
||||
generation:
|
||||
thinking_effort: high
|
||||
|
||||
prices:
|
||||
# POST-2026-07-30 SHEET, matching benchmark-runs/tools/rates.py. All three rows
|
||||
# are load-bearing here: this is the only three-model cell in the wave, so the
|
||||
# headline $/reward depends on the split across seats, not just the totals.
|
||||
# tb-gt-sol-2terra-high carries the LAUNCH sheet for terra (2.50/0.25/15.00)
|
||||
# and must not be restated -- reprice from receipts through rates.py to put the
|
||||
# two on one basis.
|
||||
gpt-5.6-sol:
|
||||
# Unchanged by the cut.
|
||||
input_per_million_usd: 5.0
|
||||
cached_input_per_million_usd: 0.5
|
||||
output_per_million_usd: 30.0
|
||||
cache_read_rate: 0.0
|
||||
gpt-5.6-luna:
|
||||
input_per_million_usd: 0.20
|
||||
cached_input_per_million_usd: 0.02
|
||||
output_per_million_usd: 1.20
|
||||
cache_read_rate: 0.0
|
||||
gpt-5.6-terra:
|
||||
input_per_million_usd: 2.00
|
||||
cached_input_per_million_usd: 0.20
|
||||
output_per_million_usd: 12.00
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
# Identical to the solo baselines -- see tb-gt-sol-2terra-high.yaml.
|
||||
timeout_seconds: 36000
|
||||
|
||||
environment:
|
||||
# Identical in every condition. override_cpus 4 with -n 8 is exactly 1:1 on a
|
||||
# 32-vCPU m7a.8xlarge.
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,26 @@
|
||||
schema_version: "1"
|
||||
condition: tb-harness-buzz-agent-luna-high
|
||||
roster:
|
||||
- id: solo
|
||||
kind: orchestrator
|
||||
role: solo
|
||||
count: 1
|
||||
endpoint: gpt-5.6-luna
|
||||
model_revision: gpt-5.6-luna
|
||||
harness: buzz-agent
|
||||
prompt:
|
||||
path: personas/bench/solo-harness-comparison.md
|
||||
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
|
||||
generation:
|
||||
thinking_effort: high
|
||||
prices:
|
||||
gpt-5.6-luna:
|
||||
input_per_million_usd: 0.2
|
||||
cached_input_per_million_usd: 0.02
|
||||
output_per_million_usd: 1.2
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
timeout_seconds: 36000
|
||||
environment:
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,26 @@
|
||||
schema_version: "1"
|
||||
condition: tb-harness-buzz-agent-terra-high
|
||||
roster:
|
||||
- id: solo
|
||||
kind: orchestrator
|
||||
role: solo
|
||||
count: 1
|
||||
endpoint: gpt-5.6-terra
|
||||
model_revision: gpt-5.6-terra
|
||||
harness: buzz-agent
|
||||
prompt:
|
||||
path: personas/bench/solo-harness-comparison.md
|
||||
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
|
||||
generation:
|
||||
thinking_effort: high
|
||||
prices:
|
||||
gpt-5.6-terra:
|
||||
input_per_million_usd: 2.0
|
||||
cached_input_per_million_usd: 0.2
|
||||
output_per_million_usd: 12.0
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
timeout_seconds: 36000
|
||||
environment:
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,26 @@
|
||||
schema_version: "1"
|
||||
condition: tb-harness-codex-luna-high
|
||||
roster:
|
||||
- id: solo
|
||||
kind: orchestrator
|
||||
role: solo
|
||||
count: 1
|
||||
endpoint: gpt-5.6-luna
|
||||
model_revision: gpt-5.6-luna
|
||||
harness: codex
|
||||
prompt:
|
||||
path: personas/bench/solo-harness-comparison.md
|
||||
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
|
||||
generation:
|
||||
thinking_effort: high
|
||||
prices:
|
||||
gpt-5.6-luna:
|
||||
input_per_million_usd: 0.2
|
||||
cached_input_per_million_usd: 0.02
|
||||
output_per_million_usd: 1.2
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
timeout_seconds: 36000
|
||||
environment:
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,26 @@
|
||||
schema_version: "1"
|
||||
condition: tb-harness-codex-terra-high
|
||||
roster:
|
||||
- id: solo
|
||||
kind: orchestrator
|
||||
role: solo
|
||||
count: 1
|
||||
endpoint: gpt-5.6-terra
|
||||
model_revision: gpt-5.6-terra
|
||||
harness: codex
|
||||
prompt:
|
||||
path: personas/bench/solo-harness-comparison.md
|
||||
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
|
||||
generation:
|
||||
thinking_effort: high
|
||||
prices:
|
||||
gpt-5.6-terra:
|
||||
input_per_million_usd: 2.0
|
||||
cached_input_per_million_usd: 0.2
|
||||
output_per_million_usd: 12.0
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
timeout_seconds: 36000
|
||||
environment:
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,26 @@
|
||||
schema_version: "1"
|
||||
condition: tb-harness-goose-luna-high
|
||||
roster:
|
||||
- id: solo
|
||||
kind: orchestrator
|
||||
role: solo
|
||||
count: 1
|
||||
endpoint: gpt-5.6-luna
|
||||
model_revision: gpt-5.6-luna
|
||||
harness: goose
|
||||
prompt:
|
||||
path: personas/bench/solo-harness-comparison.md
|
||||
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
|
||||
generation:
|
||||
thinking_effort: high
|
||||
prices:
|
||||
gpt-5.6-luna:
|
||||
input_per_million_usd: 0.2
|
||||
cached_input_per_million_usd: 0.02
|
||||
output_per_million_usd: 1.2
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
timeout_seconds: 36000
|
||||
environment:
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -0,0 +1,26 @@
|
||||
schema_version: "1"
|
||||
condition: tb-harness-goose-terra-high
|
||||
roster:
|
||||
- id: solo
|
||||
kind: orchestrator
|
||||
role: solo
|
||||
count: 1
|
||||
endpoint: gpt-5.6-terra
|
||||
model_revision: gpt-5.6-terra
|
||||
harness: goose
|
||||
prompt:
|
||||
path: personas/bench/solo-harness-comparison.md
|
||||
sha256: 5a3bea67549937b6590a6d5678261cbfa08f3da67cb891dee9e81d408a93730b
|
||||
generation:
|
||||
thinking_effort: high
|
||||
prices:
|
||||
gpt-5.6-terra:
|
||||
input_per_million_usd: 2.0
|
||||
cached_input_per_million_usd: 0.2
|
||||
output_per_million_usd: 12.0
|
||||
cache_read_rate: 0.0
|
||||
trial_budget:
|
||||
timeout_seconds: 36000
|
||||
environment:
|
||||
override_cpus: 4
|
||||
override_memory_mb: 8192
|
||||
@@ -21,7 +21,7 @@
|
||||
# surface. See docs/08-gemini-provider-fixes.md.
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names; provider/host/key
|
||||
# resolve via testbed/endpoints/databricks-live.json.
|
||||
# resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-peer-2gemini
|
||||
roster:
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
# a defect.
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names; provider/host/key
|
||||
# resolve via testbed/endpoints/databricks-live.json.
|
||||
# resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-peer-2luna
|
||||
roster:
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
# config may name entries from different providers, because benchmark.py's
|
||||
# `write_provisioner_config` resolves each entry's own `api_key_env` and each
|
||||
# entry carries its own `provider` and `env`. It is not needed here -- both
|
||||
# seats are Databricks, so `testbed/endpoints/databricks-live.json` covers them
|
||||
# seats are Databricks, so `testbed/endpoints/databricks-example.json` covers them
|
||||
# -- but it means the OpenAI-routed variant is possible without a harness change.
|
||||
#
|
||||
# WHICH MODEL DRIVES, AND WHY IT IS NOT ARBITRARY. Sol drives, Opus navigates.
|
||||
@@ -51,7 +51,7 @@
|
||||
# untouched. That is bias pointing the wrong way. Do not start this cell while
|
||||
# G1 (`tb-gt-opus-2luna`) is running on another box.
|
||||
#
|
||||
# Endpoint names resolve via testbed/endpoints/databricks-live.json.
|
||||
# Endpoint names resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-peer-sol-opus
|
||||
roster:
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names: the runtime passes
|
||||
# the manifest endpoint name to the gateway as the model. They resolve to
|
||||
# provider/host/key via testbed/endpoints/databricks-live.json, which is
|
||||
# provider/host/key via testbed/endpoints/databricks-example.json, which is
|
||||
# deployment config and deliberately outside this manifest.
|
||||
schema_version: "1"
|
||||
condition: tb-solo-gemini
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names: the runtime
|
||||
# passes the manifest endpoint name to the gateway as the model. They resolve
|
||||
# to provider/host/key via testbed/endpoints/databricks-live.json, which is
|
||||
# to provider/host/key via testbed/endpoints/databricks-example.json, which is
|
||||
# deployment config and deliberately outside this manifest.
|
||||
#
|
||||
# context_window_tokens is 272000, not the model's full window, on purpose.
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
# and the cell would report a null that was really a mislabelled `high` run.
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names; they resolve to
|
||||
# provider/host/key via testbed/endpoints/databricks-live.json.
|
||||
# provider/host/key via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-solo-opus-xhigh
|
||||
roster:
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names: the runtime passes
|
||||
# the manifest endpoint name to the gateway as the model. They resolve to
|
||||
# provider/host/key via testbed/endpoints/databricks-live.json, which is
|
||||
# provider/host/key via testbed/endpoints/databricks-example.json, which is
|
||||
# deployment config and deliberately outside this manifest.
|
||||
schema_version: "1"
|
||||
condition: tb-solo-opus
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names: the runtime passes
|
||||
# the manifest endpoint name to the gateway as the model. They resolve to
|
||||
# provider/host/key via testbed/endpoints/databricks-live.json, which is
|
||||
# provider/host/key via testbed/endpoints/databricks-example.json, which is
|
||||
# deployment config and deliberately outside this manifest.
|
||||
schema_version: "1"
|
||||
condition: tb-solo-sol
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
# anything.
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names; provider/host/key
|
||||
# resolve via testbed/endpoints/databricks-live.json.
|
||||
# resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-team-3luna
|
||||
roster:
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
# mixed setting, keeping Sol only as a solo baseline (tb-solo-sol).
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names; provider/host/key
|
||||
# resolve via testbed/endpoints/databricks-live.json.
|
||||
# resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-team-opus-2luna
|
||||
roster:
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
# this roster may burn up to 5x the tokens of tb-solo-sol and still win on cost.
|
||||
#
|
||||
# Endpoint names are exact Databricks serving-endpoint names; provider/host/key
|
||||
# resolve via testbed/endpoints/databricks-live.json.
|
||||
# resolve via testbed/endpoints/databricks-example.json.
|
||||
schema_version: "1"
|
||||
condition: tb-triad-3luna
|
||||
roster:
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# One high-effort LHTB-46 cell, smoke-gated. One cell per box, so the two cells
|
||||
# of this pair run concurrently on separate hardware instead of serially.
|
||||
#
|
||||
# Usage: lhtb-high.sh <label> <manifest-stem> <jobs-subdir> <required-seat>...
|
||||
#
|
||||
# n=8, NOT the previous LHTB wave's 4. `override_cpus` is 4 per trial container,
|
||||
# so n=8 is exactly the 32 vCPU box, 1:1 -- the ceiling, not oversubscription.
|
||||
# BOTH cells of this pair must run at 8 or the comparison is void: mismatched
|
||||
# -n compressed the between-condition spread by 0.09 in the TB wave.
|
||||
#
|
||||
# The smoke gate is not optional here. Three of the four things it checks
|
||||
# (seat addressing, the phase loop firing, per-phase receipts) fail silently --
|
||||
# they produce a plausible-looking bad number rather than an error, and you find
|
||||
# out two days and several hundred dollars later.
|
||||
set -uo pipefail
|
||||
|
||||
LABEL="${1:?usage: lhtb-high.sh <label> <manifest> <jobs> <seat>...}"
|
||||
MANIFEST="${2:?usage: lhtb-high.sh <label> <manifest> <jobs> <seat>...}"
|
||||
JOBS="${3:?usage: lhtb-high.sh <label> <manifest> <jobs> <seat>...}"
|
||||
shift 3
|
||||
SEATS=("$@")
|
||||
N=8
|
||||
# A fresh smoke needs a fresh jobs dir: gatecheck globs every result.json under
|
||||
# it, so a previous failed attempt would fail the gate a second time.
|
||||
TAG="${SMOKE_TAG:-smoke}"
|
||||
|
||||
# Resolve siblings relative to this script, so it works both from the repo
|
||||
# checkout and from a copy dropped in $HOME on a benchmark box.
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
cd "$HOME/buzz/benchmarks/harbor-buzz-orchestra" || exit 1
|
||||
|
||||
# Refuse to start if a run is already on this box. Two of these racing is not a
|
||||
# slow run, it is a corrupt one: the cleanup below cannot tell a leaked container
|
||||
# from a live trial's, so the second launch force-removes the first launch's task
|
||||
# container and harbor blocks forever in epoll_wait on a container that no longer
|
||||
# exists -- empty trial.log, no error, no timeout. That happened once already,
|
||||
# when a launch command that appeared to hang was in fact still queued and fired
|
||||
# nine minutes later into a box that had since been relaunched by hand.
|
||||
#
|
||||
# argv[0], not a cmdline grep: the shell running this very script carries
|
||||
# "benchmark.py" in its own arguments, so a substring match finds itself and
|
||||
# every run refuses to start.
|
||||
running=$(ps -eo args= | grep -ac "^[^ ]*python3* scripts/benchmark.py")
|
||||
if [ "$running" -gt 0 ]; then
|
||||
echo "=== $(date -u +%H:%M:%SZ) [$LABEL] REFUSING: $running benchmark.py already running here."
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Leaked containers from an earlier run keep holding CPU, which makes the real
|
||||
# concurrency higher than -n claims and quietly breaks the -n match. Safe only
|
||||
# because of the guard above.
|
||||
docker ps --format '{{.Names}}' | grep -v '^buzz-benchmark-' \
|
||||
| xargs -r docker rm -f > /dev/null 2>&1 || true
|
||||
|
||||
echo "=== $(date -u +%H:%M:%SZ) [$LABEL] SMOKE $MANIFEST (great-expectations-audit, ${SMOKE_MULT:-0.15}x)"
|
||||
"$HERE/lhtb-smoke.sh" "$MANIFEST" "${JOBS}-${TAG}" > "$HOME/${JOBS}-${TAG}.log" 2>&1
|
||||
echo "=== $(date -u +%H:%M:%SZ) [$LABEL] smoke exited rc=$?"
|
||||
|
||||
python3 "$HERE/gatecheck.py" "jobs/${JOBS}-${TAG}" "${SEATS[@]}"
|
||||
gate=$?
|
||||
|
||||
if [ "$gate" -ne 0 ]; then
|
||||
echo "=== $(date -u +%H:%M:%SZ) [$LABEL] gate FAILED -- NOT starting the full run."
|
||||
echo "=== inspect ~/${JOBS}-${TAG}.log and"
|
||||
echo "=== jobs/${JOBS}-${TAG}/*/*/agent/buzz/*.stdout.log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== $(date -u +%H:%M:%SZ) [$LABEL] gate PASSED, starting full 46-task run (n=$N)"
|
||||
"$HERE/lhtb46.sh" "$MANIFEST" "$JOBS" "$N" > "$HOME/${JOBS}.log" 2>&1
|
||||
echo "=== $(date -u +%H:%M:%SZ) [$LABEL] full run exited rc=$?"
|
||||
@@ -1,70 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Cheap smoke of one LHTB cell: real manifest, real route, real patched harbor,
|
||||
# but a deliberately tiny --timeout-multiplier so the agent budget is ~9 min
|
||||
# instead of 3h.
|
||||
#
|
||||
# Usage: lhtb-smoke.sh <manifest-stem> <jobs-subdir> [task ...]
|
||||
# defaults to great-expectations-audit, the cheapest continue_until_timeout
|
||||
# task and the one both baseline cells scored at exactly 0.3636.
|
||||
#
|
||||
# SMOKE_MULT (default 0.15) sets the clock. 0.15 is ~9 min of agent budget and
|
||||
# is enough for a default-effort cell -- and for a high-effort *team*, which
|
||||
# turned two phases in that window. It is NOT enough for a high-effort solo:
|
||||
# tb-solo-sol-high timed out inside phase 1 at 540s, so the trial never reached
|
||||
# the verifier and the gate read `phases=0 rewards=[None]` on a cell that was
|
||||
# working fine. Raise it to 0.5 (~30 min) for anything pinned to high effort.
|
||||
# The gate is not part of the measurement, so the two cells of a pair may smoke
|
||||
# at different multipliers; only the full runs have to match.
|
||||
#
|
||||
# The point is not the score -- at 0.15x nothing scores well. The point is that
|
||||
# the plumbing holds: the seats speak, the phase loop fires more than once, the
|
||||
# relay forwarder survives the inter-phase teardown, and receipts land per
|
||||
# phase rather than being overwritten by the last one.
|
||||
set -euo pipefail
|
||||
|
||||
MANIFEST="${1:?usage: lhtb-smoke.sh <manifest-stem> <jobs-subdir> [task ...]}"
|
||||
JOBS="${2:?usage: lhtb-smoke.sh <manifest-stem> <jobs-subdir> [task ...]}"
|
||||
MULT="${SMOKE_MULT:-0.15}"
|
||||
shift 2
|
||||
|
||||
cd "$HOME/buzz/benchmarks/harbor-buzz-orchestra"
|
||||
|
||||
if ! testbed/.venv/bin/python -c \
|
||||
"from harbor.models.task.config import AgentConfig; \
|
||||
raise SystemExit(0 if 'continue_until_timeout' in AgentConfig.model_fields else 1)"
|
||||
then
|
||||
echo "FATAL: harbor is not patched for continue_until_timeout." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PATH="$HOME/buzz/target/debug:$PATH"
|
||||
export BUZZ_BENCHMARK_DOCKER_HOST="$(hostname -I | awk '{print $1}')"
|
||||
export AWS_DEFAULT_REGION=us-east-1
|
||||
export OPENAI_API_KEY="$(aws secretsmanager get-secret-value \
|
||||
--secret-id buzz-oss/staging/benchmark/openai-api-key \
|
||||
--query SecretString --output text)"
|
||||
|
||||
INC=()
|
||||
if [ "$#" -gt 0 ]; then
|
||||
for t in "$@"; do INC+=(--include-task "$t"); done
|
||||
else
|
||||
INC=(--include-task great-expectations-audit)
|
||||
fi
|
||||
|
||||
echo "host $(hostname) ($BUZZ_BENCHMARK_DOCKER_HOST)"
|
||||
echo "commit $(git -C "$HOME/buzz" log --oneline -1)"
|
||||
echo "manifest manifests/${MANIFEST}.yaml"
|
||||
echo "harbor patched: continue_until_timeout honored"
|
||||
echo "jobs jobs/${JOBS} n=1 attempts=1 clock=${MULT}x (SMOKE)"
|
||||
echo "started $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo
|
||||
|
||||
exec uv run --project testbed --no-sync scripts/benchmark.py \
|
||||
--path "$HOME/LHTB/tasks" \
|
||||
"${INC[@]}" \
|
||||
--manifest "manifests/${MANIFEST}.yaml" \
|
||||
--endpoint-config testbed/endpoints/openai-live.json \
|
||||
--jobs-dir "jobs/${JOBS}" \
|
||||
--attempts 1 \
|
||||
--n-concurrent 1 \
|
||||
--timeout-multiplier "$MULT"
|
||||
@@ -1,93 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run one LHTB condition over ALL 46 tasks, with continue_until_timeout honored.
|
||||
#
|
||||
# Usage: lhtb46.sh <manifest-stem> <jobs-subdir> [n-concurrent] [task ...]
|
||||
# with no task list, runs all 46 tasks in ~/LHTB/tasks.
|
||||
#
|
||||
# Differs from lhtb.sh (the unpatched baseline runner) in exactly two ways, both
|
||||
# deliberate and both breaking comparability with LHTB's published leaderboard:
|
||||
#
|
||||
# 1. All 46 tasks run, not 25. The other 21 shipped as allow_internet = false
|
||||
# so the agent could not reach OpenAI from inside the task container. They
|
||||
# have been flipped to true. Several of them (sokoban, 2048, sudoku-recovery,
|
||||
# rush_hour, snake_maze, generals-bot-arena) are puzzle games whose whole
|
||||
# design assumes the agent CANNOT look up a solver, so their scores are now
|
||||
# inflated by an unknown amount. Ours-only numbers.
|
||||
#
|
||||
# One of the 21, `sudoku-recovery`, cannot run here at all and is a
|
||||
# known loss on BOTH cells (so the pairing is unaffected): it sets
|
||||
# `[agent] user = "agent"` to run the agent unprivileged as its anti-cheat
|
||||
# keystone, and the orchestra installs its stack into /opt/buzz, which that
|
||||
# user cannot create -- RuntimeLaunchError before the agent starts. It is
|
||||
# the only task in the set with a user override, and "fix" would mean
|
||||
# giving the agent root, which is precisely what the task is designed to
|
||||
# deny. Exclude it and report the loss; 45 tasks remain.
|
||||
#
|
||||
# A surgical alternative was available and not taken: harbor's
|
||||
# --allow-environment-host promotes a no-network baseline to an allowlist
|
||||
# of exactly the hosts named (see merge_extra_allowlists in
|
||||
# harbor/trial/network_policy.py), which would have let the agent reach its
|
||||
# model and nothing else. Revisit if the game tasks look anomalous.
|
||||
#
|
||||
# 2. continue_until_timeout is honored, via patches/apply_continue_until_timeout.py.
|
||||
# Stock harbor ignores the flag, so 23 of the 25 tasks in the baseline run
|
||||
# went single-shot: agents self-reported DONE at a 3-8 minute mean against
|
||||
# 60+ minute budgets and scored 0 completions.
|
||||
#
|
||||
# Timeout multiplier stays 3.0 and n stays 4, matching the unpatched baseline so
|
||||
# patched-vs-unpatched differs in one variable. That makes this run LONG: with
|
||||
# completions near zero the agents rarely exit early, so budget on roughly
|
||||
# 345 agent-hours per cell (~86h wall at n=4).
|
||||
set -euo pipefail
|
||||
|
||||
MANIFEST="${1:?usage: lhtb46.sh <manifest-stem> <jobs-subdir> [n-concurrent] [task ...]}"
|
||||
JOBS="${2:?usage: lhtb46.sh <manifest-stem> <jobs-subdir> [n-concurrent] [task ...]}"
|
||||
NCONC="${3:-4}"
|
||||
shift 3 2>/dev/null || shift $#
|
||||
|
||||
cd "$HOME/buzz/benchmarks/harbor-buzz-orchestra"
|
||||
|
||||
# Refuse to burn days of model spend on a harbor that silently ignores the flag.
|
||||
if ! testbed/.venv/bin/python -c \
|
||||
"from harbor.models.task.config import AgentConfig; \
|
||||
raise SystemExit(0 if 'continue_until_timeout' in AgentConfig.model_fields else 1)"
|
||||
then
|
||||
echo "FATAL: harbor is not patched for continue_until_timeout." >&2
|
||||
echo " run: testbed/.venv/bin/python patches/apply_continue_until_timeout.py" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PATH="$HOME/buzz/target/debug:$PATH"
|
||||
export BUZZ_BENCHMARK_DOCKER_HOST="$(hostname -I | awk '{print $1}')"
|
||||
export AWS_DEFAULT_REGION=us-east-1
|
||||
export OPENAI_API_KEY="$(aws secretsmanager get-secret-value \
|
||||
--secret-id buzz-oss/staging/benchmark/openai-api-key \
|
||||
--query SecretString --output text)"
|
||||
|
||||
INC=()
|
||||
if [ "$#" -gt 0 ]; then
|
||||
for t in "$@"; do INC+=(--include-task "$t"); done
|
||||
NTASKS="$#"
|
||||
else
|
||||
NTASKS="$(find "$HOME/LHTB/tasks" -mindepth 1 -maxdepth 1 -type d | wc -l)"
|
||||
fi
|
||||
|
||||
echo "host $(hostname) ($BUZZ_BENCHMARK_DOCKER_HOST)"
|
||||
echo "commit $(git -C "$HOME/buzz" log --oneline -1)"
|
||||
echo "manifest manifests/${MANIFEST}.yaml"
|
||||
echo "dataset ~/LHTB/tasks (${NTASKS} tasks, all allow_internet=true)"
|
||||
echo "harbor patched: continue_until_timeout honored"
|
||||
echo "route api.openai.com (direct)"
|
||||
echo "jobs jobs/${JOBS} n=${NCONC} attempts=1 clock=3x"
|
||||
echo "started $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo
|
||||
|
||||
exec uv run --project testbed --no-sync scripts/benchmark.py \
|
||||
--path "$HOME/LHTB/tasks" \
|
||||
"${INC[@]}" \
|
||||
--manifest "manifests/${MANIFEST}.yaml" \
|
||||
--endpoint-config testbed/endpoints/openai-live.json \
|
||||
--jobs-dir "jobs/${JOBS}" \
|
||||
--attempts 1 \
|
||||
--n-concurrent "$NCONC" \
|
||||
--timeout-multiplier 3.0
|
||||
@@ -0,0 +1,9 @@
|
||||
# Solo benchmark agent
|
||||
|
||||
You are the only agent assigned to this task. Work directly in the task
|
||||
container, use the available terminal and file tools, and solve the user's task
|
||||
completely. Inspect and test your work before finishing. Do not delegate.
|
||||
|
||||
When the task is complete, publish a short final message beginning with
|
||||
`DONE:`. If you cannot complete it, publish `DONE:` followed by the blocker and
|
||||
the useful work you did.
|
||||
@@ -97,7 +97,13 @@ MAX_TASK_TIMEOUT_SECONDS = 12000
|
||||
# study actually runs on (doc 04 §6); the Anthropic manifests predate it and
|
||||
# need a key this project does not provision.
|
||||
DEFAULT_MANIFEST = PACKAGE_ROOT / "manifests" / "tb-solo-luna.yaml"
|
||||
DEFAULT_ENDPOINTS = PACKAGE_ROOT / "testbed" / "endpoints" / "databricks-live.json"
|
||||
# An EXAMPLE, not a working config: the workspace host is a placeholder, so a
|
||||
# run that falls through to this default is told to pass --endpoint-config
|
||||
# rather than being allowed to fail later against a host that does not exist.
|
||||
# The endpoint configs a deployment actually runs are operator-specific (host,
|
||||
# credential plumbing, model slate) and live outside this repo.
|
||||
DEFAULT_ENDPOINTS = PACKAGE_ROOT / "testbed" / "endpoints" / "databricks-example.json"
|
||||
PLACEHOLDER_HOSTS = ("example-workspace.cloud.databricks.com",)
|
||||
SCHEMA_SQL = PACKAGE_ROOT / "testbed" / "sql" / "benchmark_schema.sql"
|
||||
|
||||
# Linux builds of the production agent stack, uploaded into each task
|
||||
@@ -414,6 +420,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
"--endpoint-config", type=Path, default=DEFAULT_ENDPOINTS,
|
||||
help=f"Endpoint provider/API-key mapping (default: {DEFAULT_ENDPOINTS.name})",
|
||||
)
|
||||
parser.add_argument("--goose-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-acp-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-code-mode-host-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-runtime-lib-dir", type=Path, default=None)
|
||||
parser.add_argument("--n-concurrent", "-n", type=int, default=4, help="Concurrent trials")
|
||||
parser.add_argument(
|
||||
"--timeout-multiplier", type=float, default=DEFAULT_TIMEOUT_MULTIPLIER,
|
||||
@@ -676,6 +687,26 @@ def resolve_databricks_token(needed_seconds: int) -> None:
|
||||
)
|
||||
|
||||
|
||||
def reject_placeholder_endpoints(endpoint_config: Path) -> None:
|
||||
"""Refuse to launch against an example config's placeholder host.
|
||||
|
||||
The checked-in Databricks config exists to document the file shape, not to
|
||||
be run. Without this the failure surfaces hours later as a DNS error inside
|
||||
a task container, on a run that has already provisioned everything.
|
||||
"""
|
||||
try:
|
||||
raw = endpoint_config.read_text()
|
||||
except OSError:
|
||||
return # write_provisioner_config raises the actionable error.
|
||||
for host in PLACEHOLDER_HOSTS:
|
||||
if host in raw:
|
||||
raise SystemExit(
|
||||
f"{endpoint_config.name} is an example: {host} is a placeholder.\n"
|
||||
f" Copy it, set DATABRICKS_HOST to your own workspace, and pass\n"
|
||||
f" --endpoint-config <path>."
|
||||
)
|
||||
|
||||
|
||||
def required_key_envs(endpoint_config: Path) -> set[str]:
|
||||
"""The API-key environment variable names this endpoint config asks for."""
|
||||
try:
|
||||
@@ -1117,6 +1148,15 @@ def leaderboard_argv(
|
||||
"--n-concurrent", str(args.n_concurrent),
|
||||
"--jobs-dir", str(args.jobs_dir),
|
||||
]
|
||||
for flag, value in (
|
||||
("--goose-binary", args.goose_binary),
|
||||
("--codex-acp-binary", args.codex_acp_binary),
|
||||
("--codex-binary", args.codex_binary),
|
||||
("--codex-code-mode-host-binary", args.codex_code_mode_host_binary),
|
||||
("--codex-runtime-lib-dir", args.codex_runtime_lib_dir),
|
||||
):
|
||||
if value is not None:
|
||||
argv += [flag, str(value)]
|
||||
argv += environment_override_argv(args.manifest)
|
||||
if args.timeout_multiplier != 1.0:
|
||||
# Forwarded only when it changes something: Harbor's validator rejects
|
||||
@@ -1192,6 +1232,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
# Resolve only the credentials this endpoint config actually asks for. A
|
||||
# run on OpenAI must not die because a Databricks OAuth token it will never
|
||||
# send happens to have expired.
|
||||
reject_placeholder_endpoints(args.endpoint_config)
|
||||
needed = required_key_envs(args.endpoint_config)
|
||||
if "DATABRICKS_TOKEN" in needed:
|
||||
resolve_databricks_token(trial_timeout_seconds(args.manifest) * args.attempts)
|
||||
|
||||
@@ -106,6 +106,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
help="Directory with Linux builds of buzz-acp/buzz-agent/buzz-dev-mcp "
|
||||
"to upload into each task container",
|
||||
)
|
||||
parser.add_argument("--goose-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-acp-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-code-mode-host-binary", type=Path, default=None)
|
||||
parser.add_argument("--codex-runtime-lib-dir", type=Path, default=None)
|
||||
parser.add_argument(
|
||||
"--relay-gateway", default="",
|
||||
help="host:port of the benchmark relay as reachable from inside the "
|
||||
@@ -235,6 +240,19 @@ def build_command(
|
||||
"buzz_cli_binary": binaries["buzz"],
|
||||
"run_id": args.job_name,
|
||||
}
|
||||
for name in (
|
||||
"goose_binary",
|
||||
"codex_acp_binary",
|
||||
"codex_binary",
|
||||
"codex_code_mode_host_binary",
|
||||
"codex_runtime_lib_dir",
|
||||
):
|
||||
value = getattr(args, name)
|
||||
if value is not None:
|
||||
exists = value.is_dir() if name == "codex_runtime_lib_dir" else value.is_file()
|
||||
if not exists and not args.dry_run:
|
||||
raise SystemExit(f"{name.replace('_', ' ')} not found: {value}")
|
||||
kwargs[name] = value
|
||||
if args.relay_gateway:
|
||||
kwargs["relay_gateway"] = args.relay_gateway
|
||||
kwargs["forwarder_binary"] = agent_binaries[FORWARDER_BINARY]
|
||||
|
||||
@@ -36,6 +36,11 @@ class BuzzOrchestraAgent(BaseAgent):
|
||||
buzz_acp_binary: str = "buzz-acp",
|
||||
buzz_agent_binary: str = "buzz-agent",
|
||||
buzz_dev_mcp_binary: str = "buzz-dev-mcp",
|
||||
goose_binary: str = "",
|
||||
codex_acp_binary: str = "",
|
||||
codex_binary: str = "",
|
||||
codex_code_mode_host_binary: str = "",
|
||||
codex_runtime_lib_dir: str = "",
|
||||
buzz_cli_binary: str = "buzz",
|
||||
relay_gateway: str = "",
|
||||
forwarder_binary: str = "relay-forwarder",
|
||||
@@ -54,6 +59,11 @@ class BuzzOrchestraAgent(BaseAgent):
|
||||
buzz_acp_binary,
|
||||
buzz_agent_binary,
|
||||
buzz_dev_mcp_binary,
|
||||
goose_binary,
|
||||
codex_acp_binary,
|
||||
codex_binary,
|
||||
codex_code_mode_host_binary,
|
||||
codex_runtime_lib_dir,
|
||||
buzz_cli_binary,
|
||||
relay_gateway,
|
||||
forwarder_binary,
|
||||
@@ -113,6 +123,11 @@ class BuzzOrchestraAgent(BaseAgent):
|
||||
buzz_acp_binary: str,
|
||||
buzz_agent_binary: str,
|
||||
buzz_dev_mcp_binary: str,
|
||||
goose_binary: str,
|
||||
codex_acp_binary: str,
|
||||
codex_binary: str,
|
||||
codex_code_mode_host_binary: str,
|
||||
codex_runtime_lib_dir: str,
|
||||
buzz_cli_binary: str,
|
||||
relay_gateway: str,
|
||||
forwarder_binary: str,
|
||||
@@ -139,6 +154,11 @@ class BuzzOrchestraAgent(BaseAgent):
|
||||
buzz_acp_binary=buzz_acp_binary,
|
||||
buzz_agent_binary=buzz_agent_binary,
|
||||
buzz_dev_mcp_binary=buzz_dev_mcp_binary,
|
||||
goose_binary=goose_binary,
|
||||
codex_acp_binary=codex_acp_binary,
|
||||
codex_binary=codex_binary,
|
||||
codex_code_mode_host_binary=codex_code_mode_host_binary,
|
||||
codex_runtime_lib_dir=codex_runtime_lib_dir,
|
||||
buzz_cli_binary=buzz_cli_binary,
|
||||
relay_gateway=relay_gateway,
|
||||
forwarder_binary=forwarder_binary,
|
||||
@@ -148,6 +168,11 @@ class BuzzOrchestraAgent(BaseAgent):
|
||||
"""Fail fast when the provisioner is configured but its stack is unhealthy."""
|
||||
if self.provisioner is not None:
|
||||
self.provisioner.healthcheck()
|
||||
# Harbor excludes setup from agent_execution timing. Stage the large
|
||||
# third-party runtimes here so the experiment measures their work, not
|
||||
# a 100-300 MB upload. run() checks again for callers that skip setup.
|
||||
if isinstance(self.runtime, BuzzContainerRuntime):
|
||||
await self.runtime.prepare_external_harnesses(environment, self.manifest)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
|
||||
@@ -14,6 +14,7 @@ import asyncio
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -270,6 +271,11 @@ class BuzzContainerRuntime:
|
||||
buzz_acp_binary: str = "buzz-acp",
|
||||
buzz_agent_binary: str = "buzz-agent",
|
||||
buzz_dev_mcp_binary: str = "buzz-dev-mcp",
|
||||
goose_binary: str = "",
|
||||
codex_acp_binary: str = "",
|
||||
codex_binary: str = "",
|
||||
codex_code_mode_host_binary: str = "",
|
||||
codex_runtime_lib_dir: str = "",
|
||||
buzz_cli_binary: str = "buzz",
|
||||
relay_gateway: str = "",
|
||||
forwarder_binary: str = "relay-forwarder",
|
||||
@@ -297,6 +303,11 @@ class BuzzContainerRuntime:
|
||||
self.buzz_acp_binary = buzz_acp_binary
|
||||
self.buzz_agent_binary = buzz_agent_binary
|
||||
self.buzz_dev_mcp_binary = buzz_dev_mcp_binary
|
||||
self.goose_binary = goose_binary
|
||||
self.codex_acp_binary = codex_acp_binary
|
||||
self.codex_binary = codex_binary
|
||||
self.codex_code_mode_host_binary = codex_code_mode_host_binary
|
||||
self.codex_runtime_lib_dir = codex_runtime_lib_dir
|
||||
# Host build used for user/provisioning operations only:
|
||||
self.buzz_cli_binary = buzz_cli_binary
|
||||
# Where the relay actually lives, as seen from inside the task
|
||||
@@ -323,6 +334,10 @@ class BuzzContainerRuntime:
|
||||
manifest: ExperimentManifest,
|
||||
trial: TrialHandle,
|
||||
) -> RuntimeResult:
|
||||
runtime_started = time.monotonic()
|
||||
active_started: float | None = None
|
||||
active_finished: float | None = None
|
||||
usage_settle_seconds_observed = 0.0
|
||||
classes = self._classes_by_agent_id(manifest, trial.credentials)
|
||||
orchestrator = next(c for c in trial.credentials if c.role == "orchestrator")
|
||||
# A zero-worker roster is the single-agent baseline, not an error. The
|
||||
@@ -338,6 +353,7 @@ class BuzzContainerRuntime:
|
||||
verifier_deps = "unknown"
|
||||
try:
|
||||
trust_store = await self._install_stack(environment)
|
||||
await self.prepare_external_harnesses(environment, manifest)
|
||||
forwarder = await self._start_forwarder(environment, trial)
|
||||
if forwarder is not None:
|
||||
infra.append(forwarder)
|
||||
@@ -366,6 +382,7 @@ class BuzzContainerRuntime:
|
||||
# The task arrives exactly as it would in production Buzz: a
|
||||
# user prompt @mentioning the orchestrator. The harness never
|
||||
# speaks as any agent.
|
||||
active_started = time.monotonic()
|
||||
await self._send(
|
||||
trial.user, trial, f"@{orchestrator.agent_id} {instruction}"
|
||||
)
|
||||
@@ -381,8 +398,11 @@ class BuzzContainerRuntime:
|
||||
),
|
||||
timeout=manifest.trial_budget.timeout_seconds,
|
||||
)
|
||||
active_finished = time.monotonic()
|
||||
await self._verify_m1_output(environment, manifest)
|
||||
finally:
|
||||
if active_started is not None and active_finished is None:
|
||||
active_finished = time.monotonic()
|
||||
# `DONE:` is published before the turn's usage notification is, so
|
||||
# teardown has to wait or the trial's tokens are lost.
|
||||
#
|
||||
@@ -400,7 +420,9 @@ class BuzzContainerRuntime:
|
||||
# usage line exists, which after the first round it does. Only a
|
||||
# phase that never completed a single round can spend the full
|
||||
# budget, and that phase has nothing to report anyway.
|
||||
usage_settle_started = time.monotonic()
|
||||
await self._settle_usage(environment, agents)
|
||||
usage_settle_seconds_observed = time.monotonic() - usage_settle_started
|
||||
await self._stop_agents(environment, agents + infra)
|
||||
# Logs first, and ahead of anything that touches the network. The
|
||||
# verifier pre-install below used to run first and, when the proxy
|
||||
@@ -422,6 +444,29 @@ class BuzzContainerRuntime:
|
||||
trial_accounting = self._collect_accounting(
|
||||
trial_dir, manifest, trial, classes
|
||||
)
|
||||
timing = {
|
||||
"schema_version": 1,
|
||||
"definition": (
|
||||
"agent_active_seconds spans task prompt dispatch through "
|
||||
"observed turn completion or timeout; it excludes runtime "
|
||||
"preparation and post-turn settlement/collection"
|
||||
),
|
||||
"runtime_until_bundle_seconds": time.monotonic() - runtime_started,
|
||||
"pre_prompt_seconds": (
|
||||
active_started - runtime_started
|
||||
if active_started is not None
|
||||
else None
|
||||
),
|
||||
"agent_active_seconds": (
|
||||
active_finished - active_started
|
||||
if active_started is not None and active_finished is not None
|
||||
else None
|
||||
),
|
||||
"usage_settle_seconds": usage_settle_seconds_observed,
|
||||
}
|
||||
(trial_dir / "timing.json").write_text(
|
||||
json.dumps(timing, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
bundle.write(
|
||||
trial_dir=trial_dir,
|
||||
manifest=manifest,
|
||||
@@ -448,6 +493,8 @@ class BuzzContainerRuntime:
|
||||
# inferred later from an empty message id.
|
||||
"stopped_without_done": final_message is None,
|
||||
"agent_runtime": "in-container",
|
||||
"agent_active_seconds": timing["agent_active_seconds"],
|
||||
"usage_settle_seconds": timing["usage_settle_seconds"],
|
||||
# present | seeded | failed. `failed` means the container had
|
||||
# no usable trust store and could not be given one, so any
|
||||
# https the agent or the verifier attempted was doomed -- read
|
||||
@@ -549,6 +596,44 @@ class BuzzContainerRuntime:
|
||||
await environment.upload_file(self.ca_bundle, REMOTE_CA_BUNDLE)
|
||||
return await self._seed_system_trust_store(environment)
|
||||
|
||||
async def prepare_external_harnesses(
|
||||
self, environment: BaseEnvironment, manifest: ExperimentManifest
|
||||
) -> None:
|
||||
"""Stage only the non-default ACP runtimes selected by the manifest."""
|
||||
harnesses = {entry.harness for entry in manifest.roster}
|
||||
uploads: dict[str, str] = {}
|
||||
if "goose" in harnesses:
|
||||
uploads[f"{REMOTE_BIN}/goose"] = self.goose_binary
|
||||
if "codex" in harnesses:
|
||||
uploads[f"{REMOTE_BIN}/codex-acp"] = self.codex_acp_binary
|
||||
uploads[f"{REMOTE_BIN}/codex"] = self.codex_binary
|
||||
uploads[f"{REMOTE_BIN}/codex-code-mode-host"] = (
|
||||
self.codex_code_mode_host_binary
|
||||
)
|
||||
runtime_libs = Path(self.codex_runtime_lib_dir)
|
||||
for name in (
|
||||
"ld-musl-x86_64.so.1",
|
||||
"libgcc_s.so.1",
|
||||
"libstdc++.so.6",
|
||||
):
|
||||
uploads[f"{REMOTE_ROOT}/lib/x86_64/{name}"] = str(
|
||||
runtime_libs / name
|
||||
)
|
||||
if not uploads:
|
||||
return
|
||||
missing = [target for target, source in uploads.items() if not Path(source).is_file()]
|
||||
if missing:
|
||||
raise RuntimeLaunchError(
|
||||
"external harness binary not found for: " + ", ".join(missing)
|
||||
)
|
||||
await environment.exec(f"mkdir -p {REMOTE_BIN}")
|
||||
await environment.exec(f"mkdir -p {REMOTE_ROOT}/lib/x86_64")
|
||||
for target, source in uploads.items():
|
||||
present = await environment.exec(f"test -x {shlex.quote(target)}")
|
||||
if present.return_code != 0:
|
||||
await environment.upload_file(source, target)
|
||||
await environment.exec(f"chmod 0755 {shlex.quote(target)}")
|
||||
|
||||
@staticmethod
|
||||
async def _seed_system_trust_store(environment: BaseEnvironment) -> str:
|
||||
"""Give apt, curl and pip the anchors buzz-agent already has.
|
||||
@@ -775,6 +860,54 @@ class BuzzContainerRuntime:
|
||||
turn_timeout_seconds: int = 0,
|
||||
) -> dict[str, str]:
|
||||
"""The desktop-launch environment: real acp/agent/dev-mcp wiring."""
|
||||
effort = agent_class.generation.thinking_effort or THINKING_EFFORT
|
||||
commands = {
|
||||
"buzz-agent": f"{REMOTE_BIN}/buzz-agent",
|
||||
"goose": f"{REMOTE_BIN}/goose",
|
||||
"codex": f"{REMOTE_BIN}/codex-acp",
|
||||
}
|
||||
harness_env: dict[str, str]
|
||||
if agent_class.harness == "goose":
|
||||
harness_env = {
|
||||
"GOOSE_PROVIDER": endpoint.provider,
|
||||
"GOOSE_MODEL": credential.llm_endpoint,
|
||||
"GOOSE_THINKING_EFFORT": effort,
|
||||
"OPENAI_API_KEY": credential.llm_api_key,
|
||||
}
|
||||
elif agent_class.harness == "codex":
|
||||
harness_env = {
|
||||
"CODEX_PATH": f"{REMOTE_BIN}/codex",
|
||||
"CODEX_API_KEY": credential.llm_api_key,
|
||||
"OPENAI_API_KEY": credential.llm_api_key,
|
||||
# Supplying a key only advertises the method; the ACP client
|
||||
# still has to select it. Headless benchmark runs have no UI in
|
||||
# which to answer that auth request.
|
||||
"DEFAULT_AUTH_REQUEST": '{"methodId":"api-key"}',
|
||||
"NO_BROWSER": "1",
|
||||
"INITIAL_AGENT_MODE": "agent-full-access",
|
||||
"LD_LIBRARY_PATH": f"{REMOTE_ROOT}/lib/x86_64",
|
||||
"CODEX_CONFIG": json.dumps(
|
||||
{
|
||||
"model": credential.llm_endpoint,
|
||||
"model_reasoning_effort": effort,
|
||||
"approval_policy": "never",
|
||||
"sandbox_mode": "danger-full-access",
|
||||
},
|
||||
separators=(",", ":"),
|
||||
),
|
||||
}
|
||||
else:
|
||||
harness_env = {
|
||||
"BUZZ_AGENT_PROVIDER": endpoint.provider,
|
||||
"BUZZ_AGENT_MODEL": credential.llm_endpoint,
|
||||
"BUZZ_AGENT_THINKING_EFFORT": effort,
|
||||
**self._window_env(agent_class.generation),
|
||||
**self._compaction_env(agent_class.generation),
|
||||
"BUZZ_AGENT_MAX_ROUNDS": str(
|
||||
agent_class.budget.max_calls or self.max_agent_rounds
|
||||
),
|
||||
"BUZZ_AGENT_NO_HINTS": "1",
|
||||
}
|
||||
return {
|
||||
**self._turn_duration_env(turn_timeout_seconds),
|
||||
**endpoint.env,
|
||||
@@ -792,7 +925,7 @@ class BuzzContainerRuntime:
|
||||
# so buzz-dev-mcp's shim can wire git auth/signing for the agent.
|
||||
"NOSTR_PRIVATE_KEY": credential.nostr_secret_key,
|
||||
"BUZZ_AUTH_TAG": credential.nostr_auth_tag,
|
||||
"BUZZ_ACP_AGENT_COMMAND": f"{REMOTE_BIN}/buzz-agent",
|
||||
"BUZZ_ACP_AGENT_COMMAND": commands[agent_class.harness],
|
||||
"BUZZ_ACP_AGENT_ARGS": "",
|
||||
"BUZZ_ACP_MCP_COMMAND": f"{REMOTE_BIN}/buzz-dev-mcp",
|
||||
"BUZZ_ACP_CHANNELS": trial.channel_id,
|
||||
@@ -801,19 +934,7 @@ class BuzzContainerRuntime:
|
||||
"BUZZ_ACP_NO_MEMORY": "true",
|
||||
"BUZZ_ACP_SYSTEM_PROMPT_FILE": remote_prompt,
|
||||
**self._platform_prompt_env(agent_class),
|
||||
"BUZZ_AGENT_PROVIDER": endpoint.provider,
|
||||
"BUZZ_AGENT_MODEL": credential.llm_endpoint,
|
||||
"BUZZ_AGENT_THINKING_EFFORT": (
|
||||
agent_class.generation.thinking_effort or THINKING_EFFORT
|
||||
),
|
||||
**self._window_env(agent_class.generation),
|
||||
**self._compaction_env(agent_class.generation),
|
||||
"BUZZ_AGENT_MAX_ROUNDS": str(
|
||||
agent_class.budget.max_calls or self.max_agent_rounds
|
||||
),
|
||||
# The pinned persona is the whole prompt: no hint-file or skill
|
||||
# discovery from the task filesystem (metadata reports this).
|
||||
"BUZZ_AGENT_NO_HINTS": "1",
|
||||
**harness_env,
|
||||
endpoint.api_key_env: credential.llm_api_key,
|
||||
}
|
||||
|
||||
|
||||
@@ -180,6 +180,10 @@ class AgentClass(StrictModel):
|
||||
count: int = Field(gt=0)
|
||||
endpoint: str = Field(min_length=1)
|
||||
model_revision: str = Field(min_length=1)
|
||||
# ACP process hosted behind buzz-acp. The default preserves every existing
|
||||
# manifest's behaviour; benchmark-only cells can select Goose or Codex
|
||||
# without smuggling process wiring through an endpoint's environment.
|
||||
harness: Literal["buzz-agent", "goose", "codex"] = "buzz-agent"
|
||||
prompt: ArtifactRef
|
||||
persona: ArtifactRef | None = None
|
||||
skills: tuple[ArtifactRef, ...] = ()
|
||||
@@ -301,6 +305,10 @@ class ExperimentManifest(StrictModel):
|
||||
# also drop the other optional fields that currently serialise as null
|
||||
# and would shift far more hashes than it preserved.
|
||||
for entry in data.get("roster", []):
|
||||
# Adding a selector whose default describes the historical runtime
|
||||
# must not re-identify all historical conditions.
|
||||
if entry.get("harness") == "buzz-agent":
|
||||
entry.pop("harness", None)
|
||||
generation = entry.get("generation")
|
||||
if not isinstance(generation, dict):
|
||||
continue
|
||||
|
||||
@@ -9,65 +9,73 @@ endpoint string remains the join key.
|
||||
Every key in these files must be a manifest endpoint name; the loader treats
|
||||
all entries as endpoint configs (no comment keys).
|
||||
|
||||
**These are examples.** A real run supplies its own file with
|
||||
`--endpoint-config <path>`, because the workspace host, the credential
|
||||
plumbing and the model slate are all operator-specific. Nothing here should
|
||||
name a private host, an internal address, or a secret store entry.
|
||||
|
||||
## m1-local.json
|
||||
|
||||
M1 wiring proof: both placeholder endpoints resolve to one local llama-server
|
||||
(OpenAI-compatible, `http://127.0.0.1:8091/v1`, no cloud keys).
|
||||
Wiring proof: both placeholder endpoints resolve to one local llama-server
|
||||
(OpenAI-compatible, `http://127.0.0.1:8091/v1`, no cloud keys). This is the
|
||||
config to use when you want to exercise the harness without a provider.
|
||||
|
||||
buzz-agent env contract (crates/buzz-agent/src/config.rs, pinned at the M1
|
||||
binary SHA): `provider=openai` reads `OPENAI_COMPAT_API_KEY` +
|
||||
`OPENAI_COMPAT_BASE_URL`; the runtime sets `BUZZ_AGENT_MODEL` from the
|
||||
manifest endpoint name, which overrides `OPENAI_COMPAT_MODEL` — llama-server
|
||||
ignores the model name, so the placeholder value is harmless there.
|
||||
llama-server needs no real key; the provisioner's per-endpoint
|
||||
`llm_api_keys` map supplies a dummy value.
|
||||
|
||||
The Databricks pilot config is the same file shape with real serving
|
||||
endpoint hosts/keys.
|
||||
buzz-agent env contract (`crates/buzz-agent/src/config.rs`):
|
||||
`provider=openai` reads `OPENAI_COMPAT_API_KEY` + `OPENAI_COMPAT_BASE_URL`;
|
||||
the runtime sets `BUZZ_AGENT_MODEL` from the manifest endpoint name, which
|
||||
overrides `OPENAI_COMPAT_MODEL` — llama-server ignores the model name, so the
|
||||
placeholder value is harmless there. llama-server needs no real key; the
|
||||
provisioner's per-endpoint `llm_api_keys` map supplies a dummy value.
|
||||
|
||||
## openai-live.json
|
||||
|
||||
The same models served by the OpenAI API rather than the Databricks gateway.
|
||||
Exists because the Databricks bearer is a short-lived OAuth token that expires
|
||||
roughly hourly, which makes an unattended sweep impossible; a plain OpenAI key
|
||||
does not expire mid-run.
|
||||
Models served by the OpenAI API directly. Endpoint names here are literal
|
||||
OpenAI model ids, because the runtime sets `BUZZ_AGENT_MODEL` from the
|
||||
manifest endpoint name.
|
||||
|
||||
Endpoint names here are literal OpenAI model ids, because the runtime sets
|
||||
`BUZZ_AGENT_MODEL` from the manifest endpoint name. No `OPENAI_COMPAT_BASE_URL`
|
||||
override: the default `https://api.openai.com/v1` is correct, and leaving it
|
||||
unset also lets `OPENAI_COMPAT_API=auto` select the Responses API, which is the
|
||||
route these reasoning models need.
|
||||
No `OPENAI_COMPAT_BASE_URL` override: the default `https://api.openai.com/v1`
|
||||
is correct, and leaving it unset also lets `OPENAI_COMPAT_API=auto` select the
|
||||
Responses API, which is the route these reasoning models need.
|
||||
|
||||
`scripts/benchmark.py` populates `OPENAI_COMPAT_API_KEY` from `OPENAI_API_KEY`
|
||||
when only the latter is exported, and resolves provider credentials only for
|
||||
the endpoints a given config actually names — so an OpenAI run does not require
|
||||
a working Databricks token.
|
||||
|
||||
## databricks-live.json
|
||||
## anthropic-live.json
|
||||
|
||||
`gpt-5.6-sol`, `gpt-5.6-luna` and `claude-opus-5` served by the Databricks AI
|
||||
Gateway (`provider=databricks_v2`, `api_key_env=DATABRICKS_TOKEN`).
|
||||
`DATABRICKS_HOST` rides in each entry's `env` block and Harbor injects it into
|
||||
the agent container — it is not a secret and needs no host-side export.
|
||||
Anthropic's API directly (`provider=anthropic`, `ANTHROPIC_API_KEY`). Same
|
||||
shape; endpoint names are literal Anthropic model ids.
|
||||
|
||||
## openrouter-live.json
|
||||
|
||||
OpenRouter (`provider=openrouter`, `OPENROUTER_API_KEY`).
|
||||
|
||||
**Pin the upstream.** One OpenRouter model id is served by several providers at
|
||||
different prices, quantisations and cache behaviours, so an unpinned run is not
|
||||
a reproducible condition — it silently mixes them. `OPENROUTER_PROVIDER_ORDER`
|
||||
names the upstream (`moonshotai/mxfp4`, `gmicloud/fp8`, …), and ordering alone
|
||||
is not a pin unless fallbacks are disabled.
|
||||
|
||||
## databricks-example.json
|
||||
|
||||
The shape for a Databricks AI Gateway workspace (`provider=databricks_v2`,
|
||||
`api_key_env=DATABRICKS_TOKEN`). Replace `DATABRICKS_HOST` with your own
|
||||
workspace; it rides in each entry's `env` block and Harbor injects it into the
|
||||
agent container, so it needs no host-side export.
|
||||
|
||||
Endpoint names must be the workspace's **serving endpoint names**, which on a
|
||||
Databricks gateway carry a `databricks-` prefix — the console's display label
|
||||
and the model id it echoes back both fail to resolve.
|
||||
|
||||
**Two gateway routes, selected per model by the provider, not by config.**
|
||||
`databricks_v2_route_for_model` in buzz-agent sends `gpt-5`-named models to the
|
||||
OpenAI `/ai-gateway/openai/v1/responses` path and `claude`-named models to
|
||||
`/ai-gateway/anthropic/v1/messages` (Opus 5 does not speak `/responses`). So
|
||||
`databricks-claude-opus-5` works through the same provider with no extra
|
||||
config — the endpoint name is all that picks the route. Verified `200` on the
|
||||
staging box with a well-formed anthropic `usage` block.
|
||||
`databricks_v2_route_for_model` in buzz-agent sends `gpt-5`-named models to
|
||||
`/ai-gateway/openai/v1/responses` and `claude`-named models to
|
||||
`/ai-gateway/anthropic/v1/messages` (Opus 5 does not speak `/responses`).
|
||||
Anything else goes to the shared MLflow chat-completions route. So the endpoint
|
||||
name alone picks the route, with no extra config.
|
||||
|
||||
`DATABRICKS_HOST` points at the **staging** workspace
|
||||
(`block-lakehouse-staging.cloud.databricks.com`), not production. On the
|
||||
buzz-oss staging benchmark runner this is the reachable one: staging's
|
||||
privatelink endpoint (`10.170.107.x`) is routable from the staging VPC both
|
||||
directly and through cloudproxy, whereas production's (`10.172.98.x`) is a
|
||||
blackhole from there. Both serve the same `gpt-5.6-luna`/`sol` endpoints. See
|
||||
`docs/06-ec2-benchmark-runbook.md` §5.5 for the reachability proof and the
|
||||
production caveat.
|
||||
|
||||
Unlike the OAuth bearer the `openai-live.json` note warns about, the token
|
||||
here is a long-lived personal access token (`DATABRICKS_TOKEN`), so an
|
||||
unattended sweep is possible. On the runner it is fetched from the Secrets
|
||||
Manager secret `buzz-oss/staging/benchmark/databricks-token`.
|
||||
Unlike a workspace OAuth bearer, which expires roughly hourly and makes an
|
||||
unattended sweep impossible, a personal access token in `DATABRICKS_TOKEN` is
|
||||
long-lived. Fetch it from wherever your deployment keeps secrets; do not commit
|
||||
one here.
|
||||
|
||||
+4
-4
@@ -3,28 +3,28 @@
|
||||
"provider": "databricks_v2",
|
||||
"api_key_env": "DATABRICKS_TOKEN",
|
||||
"env": {
|
||||
"DATABRICKS_HOST": "https://block-lakehouse-staging.cloud.databricks.com"
|
||||
"DATABRICKS_HOST": "https://example-workspace.cloud.databricks.com"
|
||||
}
|
||||
},
|
||||
"databricks-gpt-5-6-luna": {
|
||||
"provider": "databricks_v2",
|
||||
"api_key_env": "DATABRICKS_TOKEN",
|
||||
"env": {
|
||||
"DATABRICKS_HOST": "https://block-lakehouse-staging.cloud.databricks.com"
|
||||
"DATABRICKS_HOST": "https://example-workspace.cloud.databricks.com"
|
||||
}
|
||||
},
|
||||
"databricks-claude-opus-5": {
|
||||
"provider": "databricks_v2",
|
||||
"api_key_env": "DATABRICKS_TOKEN",
|
||||
"env": {
|
||||
"DATABRICKS_HOST": "https://block-lakehouse-staging.cloud.databricks.com"
|
||||
"DATABRICKS_HOST": "https://example-workspace.cloud.databricks.com"
|
||||
}
|
||||
},
|
||||
"databricks-gemini-3-6-flash": {
|
||||
"provider": "databricks_v2",
|
||||
"api_key_env": "DATABRICKS_TOKEN",
|
||||
"env": {
|
||||
"DATABRICKS_HOST": "https://block-lakehouse-staging.cloud.databricks.com"
|
||||
"DATABRICKS_HOST": "https://example-workspace.cloud.databricks.com"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"databricks-claude-opus-5": {
|
||||
"provider": "databricks_v2",
|
||||
"api_key_env": "DATABRICKS_TOKEN",
|
||||
"env": {
|
||||
"DATABRICKS_HOST": "https://block-lakehouse-staging.cloud.databricks.com"
|
||||
}
|
||||
},
|
||||
"gpt-5.6-terra": {
|
||||
"provider": "openai",
|
||||
"api_key_env": "OPENAI_COMPAT_API_KEY",
|
||||
"env": {}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
# `mixed-opus-terra.json` — why this config exists
|
||||
|
||||
Notes for `mixed-opus-terra.json`. They live in a sibling file rather than a
|
||||
`_comment` key inside the JSON because `write_provisioner_config()` in
|
||||
`scripts/benchmark.py` iterates `endpoints.items()` and indexes
|
||||
`entry["api_key_env"]` **without** an `isinstance` guard (unlike
|
||||
`required_key_envs()`, which has one). Any non-dict value in that top-level
|
||||
object therefore crashes the launch with `TypeError: string indices must be
|
||||
integers` — after container setup, so it reads as a harness bug rather than a
|
||||
config typo. **Do not add descriptive keys to any endpoint config.**
|
||||
|
||||
## What it is for
|
||||
|
||||
`tb-gt-opus-2terra.yaml` (cell **G1ot**) needs Claude Opus 5 as its lead and two
|
||||
`gpt-5.6-terra` subordinates. Those two models are not available from the same
|
||||
vendor:
|
||||
|
||||
| model | `databricks-live.json` | `openai-live.json` | `anthropic-live.json` |
|
||||
|---|---|---|---|
|
||||
| Claude Opus 5 | ✅ `databricks-claude-opus-5` | ✗ | ✗ (Sonnet 4.6 / Haiku 4.5 only) |
|
||||
| GPT-5.6 Terra | ✗ | ✅ `gpt-5.6-terra` | ✗ |
|
||||
|
||||
So no single-vendor config can serve G1ot, and this merged file is the only way
|
||||
to run it at all.
|
||||
|
||||
It is deliberately **minimal** — exactly the two endpoints G1ot names. A
|
||||
manifest typo then fails loudly at resolution instead of quietly binding to some
|
||||
other model that happens to be present.
|
||||
|
||||
## Credentials
|
||||
|
||||
It requires **both** secrets:
|
||||
|
||||
- `DATABRICKS_TOKEN` — for the Opus 5 lead
|
||||
- `OPENAI_API_KEY` — for the Terra seats; `benchmark.py`'s `resolve_openai_key()`
|
||||
copies it into `OPENAI_COMPAT_API_KEY`, which is what the `openai` provider
|
||||
actually reads
|
||||
|
||||
Neither `sweep.sh` (Databricks only) nor `sweep-openai.sh` (OpenAI only) exports
|
||||
both. Use **`sweep-mixed.sh`**.
|
||||
|
||||
## Interpretive caveat
|
||||
|
||||
G1ot is cross-provider by necessity, so `G1 → G1ot` moves seat model *and* seat
|
||||
provider together, and `G1st → G1ot` moves lead model *and* lead provider
|
||||
together. Neither is a one-variable comparison. The clean seat-model read is
|
||||
`G1s → G1st` (both single-route OpenAI). See the header of
|
||||
`manifests/tb-gt-opus-2terra.yaml`.
|
||||
|
||||
Note this is a difference of degree, not kind: every Opus-led cell in the study
|
||||
(C3, G1, G2) is *already* two-route internally, because Opus does not speak
|
||||
`/responses` and takes `/ai-gateway/anthropic/v1/messages` while its seats take
|
||||
the Databricks `/responses` path. G1ot widens that from two paths inside one
|
||||
vendor to two vendors — a second set of rate limits and a second failure mode,
|
||||
not a new class of confound.
|
||||
@@ -627,6 +627,12 @@ async def test_solo_roster_runs_and_is_priced(tmp_path, monkeypatch):
|
||||
assert result.cost_usd == pytest.approx(20.0 + 30.0)
|
||||
assert result.metadata["solo_roster"] is True
|
||||
assert result.metadata["accounting_reconciled"] is True
|
||||
assert result.metadata["agent_active_seconds"] >= 0
|
||||
timing = json.loads((tmp_path / "logs" / "buzz" / "timing.json").read_text())
|
||||
assert timing["agent_active_seconds"] == pytest.approx(
|
||||
result.metadata["agent_active_seconds"]
|
||||
)
|
||||
assert timing["usage_settle_seconds"] >= 0
|
||||
# The bundle landed next to the logs.
|
||||
summary = json.loads((tmp_path / "logs" / "buzz" / "summary.json").read_text())
|
||||
assert summary["solo_roster"] is True
|
||||
@@ -1332,6 +1338,44 @@ async def test_thinking_effort_can_be_raised_per_condition(tmp_path):
|
||||
assert env["BUZZ_AGENT_THINKING_EFFORT"] == "xhigh"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("harness", "command"),
|
||||
[("goose", "goose"), ("codex", "codex-acp")],
|
||||
)
|
||||
async def test_external_harness_gets_same_model_and_effort(
|
||||
tmp_path, harness, command
|
||||
):
|
||||
manifest = write_manifest(tmp_path)
|
||||
entry = manifest.roster[0].model_copy(
|
||||
update={
|
||||
"harness": harness,
|
||||
"generation": manifest.roster[0].generation.model_copy(
|
||||
update={"thinking_effort": "high"}
|
||||
),
|
||||
}
|
||||
)
|
||||
orch = credential("orch-1", "orchestrator", "gpt-5.6-luna")
|
||||
env = runtime(tmp_path)._agent_env(
|
||||
trial=trial_handle((orch,)),
|
||||
credential=orch,
|
||||
agent_class=entry,
|
||||
endpoint=EndpointLaunchConfig("openai", "OPENAI_COMPAT_API_KEY"),
|
||||
remote_prompt="/opt/buzz/prompts/orch-1.system-prompt.md",
|
||||
)
|
||||
assert env["BUZZ_ACP_AGENT_COMMAND"] == f"{REMOTE_BIN}/{command}"
|
||||
assert env["OPENAI_API_KEY"] == orch.llm_api_key
|
||||
assert "BUZZ_AGENT_MODEL" not in env
|
||||
if harness == "goose":
|
||||
assert env["GOOSE_MODEL"] == "gpt-5.6-luna"
|
||||
assert env["GOOSE_THINKING_EFFORT"] == "high"
|
||||
else:
|
||||
assert env["CODEX_PATH"] == f"{REMOTE_BIN}/codex"
|
||||
assert json.loads(env["DEFAULT_AUTH_REQUEST"])["methodId"] == "api-key"
|
||||
assert env["INITIAL_AGENT_MODE"] == "agent-full-access"
|
||||
assert json.loads(env["CODEX_CONFIG"])["model"] == "gpt-5.6-luna"
|
||||
assert json.loads(env["CODEX_CONFIG"])["model_reasoning_effort"] == "high"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("include", "expected"),
|
||||
[(True, None), (False, "1")],
|
||||
|
||||
@@ -49,6 +49,24 @@ def test_pinned_effort_is_part_of_the_condition_hash(manifest_data):
|
||||
assert pinned_manifest.sha256 != baseline.sha256
|
||||
|
||||
|
||||
def test_default_harness_preserves_historical_hash(manifest_data):
|
||||
baseline = ExperimentManifest.load(manifest_data)
|
||||
explicit = copy.deepcopy(manifest_data)
|
||||
explicit["roster"][0]["harness"] = "buzz-agent"
|
||||
assert b'"harness"' not in baseline.canonical_bytes()
|
||||
assert explicit["roster"][0]["harness"] == "buzz-agent"
|
||||
assert ExperimentManifest.load(explicit).sha256 == baseline.sha256
|
||||
|
||||
|
||||
def test_external_harness_is_part_of_the_condition_hash(manifest_data):
|
||||
baseline = ExperimentManifest.load(manifest_data)
|
||||
goose = copy.deepcopy(manifest_data)
|
||||
goose["roster"][0]["harness"] = "goose"
|
||||
goose_manifest = ExperimentManifest.load(goose)
|
||||
assert b'"harness":"goose"' in goose_manifest.canonical_bytes()
|
||||
assert goose_manifest.sha256 != baseline.sha256
|
||||
|
||||
|
||||
MANIFEST_DIR = pathlib.Path(__file__).resolve().parents[1] / "manifests"
|
||||
SHIPPED = sorted(MANIFEST_DIR.glob("*.yaml"))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user