diff --git a/benchmarks/harbor-buzz-orchestra/manifests/lhtb-or-kimi-lead-2deepseek-high.yaml b/benchmarks/harbor-buzz-orchestra/manifests/lhtb-or-kimi-lead-2deepseek-high.yaml new file mode 100644 index 000000000..c9ff7e467 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/manifests/lhtb-or-kimi-lead-2deepseek-high.yaml @@ -0,0 +1,104 @@ +# OR3h -- LHTB team on OpenRouter: kimi-k3 lead, deepseek-v4-flash scout, +# deepseek-v4-flash worker. All three seats at reasoning effort `high`. +# +# Structurally this is tb-gt-sol-luna-terra-high with open-weight models +# substituted seat for seat: an expensive lead directing two cheap seats, one +# read-only scout and one that edits and builds. That cell is the reason to run +# this one -- it scored 0.695 against solo Sol's 0.591 on LHTB-46, and the +# open question is whether the long-horizon team win survives a 22x cheaper +# roster or was really about the specific models. +# +# NOTE THE ASYMMETRY WITH ITS OPENAI ANALOGUE. There, scout and worker were +# DIFFERENT models (luna scout, terra worker), which is what let that cell +# separate "cheap in the read-only seat" from "cheap in the editing seat". +# Here both cheap seats are the same model, so this cell cannot make that +# distinction -- it is the two-model analogue of tb-gt-sol-2terra-high, not of +# tb-gt-sol-luna-terra-high. Reading a seat-placement result out of it would be +# reading something it does not contain. +# +# ROSTER IDS ARE PROTOCOL, NOT LABELS. `lead`, `scout` and `worker` are read +# verbatim out of the "Your team" table by the gt personas. Renaming any of them +# breaks addressing silently: the @mention resolves to nobody, the send still +# reports success, and the trial stalls to its timeout. +# +# BOTH UPSTREAMS ARE PINNED (openrouter-live.json): moonshotai/mxfp4 for the +# lead, gmicloud/fp8 for both cheap seats. On a team cell the pin matters more +# than on a solo one -- unpinned, each seat independently lands on a different +# upstream per request, so "the composition" would not even be stable within a +# single trial. It is also what makes prompt caching work: measured 2026-08-01, +# pinned routes served a repeated prefix from cache on every call while the +# unpinned route managed one in three. A team re-sends more history than a solo +# agent does, so this cell is the one that would have been hurt worst. +# +# THIS RUNS UNDER THE PATCHED HARBOR. LHTB needs continue_until_timeout honored +# (patches/apply_continue_until_timeout.py); stock harbor ignores the flag and +# agents self-report DONE minutes into hour-long budgets. lhtb46.sh refuses to +# start on an unpatched harbor -- do not bypass that check. +schema_version: "1" +condition: lhtb-or-kimi-lead-2deepseek-high +roster: + - id: lead + kind: orchestrator + role: lead + count: 1 + # The EXPENSIVE model in the directing seat -- 22x the input rate of the + # seats it directs. Whether that is worth it is the cell's question. + endpoint: moonshotai/kimi-k3 + model_revision: moonshotai/kimi-k3-20260715 + prompt: + # Byte-identical to the file every other gt cell pins. + path: personas/bench/gt/gt-lead.md + sha256: 8a27e833dec0a6d1700c5ea3100f8a81c09e5090ef0a589b9583fac40b24c05c + generation: + thinking_effort: high + + - id: scout + kind: worker + role: scout + count: 1 + endpoint: deepseek/deepseek-v4-flash-0731 + model_revision: deepseek/deepseek-v4-flash-20260731 + prompt: + path: personas/bench/gt/gt-scout.md + sha256: 0359a957428dd09e56e57a7fd3fe455d860264d910705f0e4992337dde25e5a5 + generation: + thinking_effort: high + + - id: worker + kind: worker + role: worker + count: 1 + endpoint: deepseek/deepseek-v4-flash-0731 + model_revision: deepseek/deepseek-v4-flash-20260731 + prompt: + path: personas/bench/gt/gt-worker.md + sha256: 7a529ae58f2a2ec635fdf65fb43284b30c09d2c9dbf31e70951fe021c3b2b766 + generation: + thinking_effort: high + +prices: + # The rates of the PINNED endpoints, not the model ids' headline rates. Both + # rows are load-bearing: the whole point of the cell is the split between an + # expensive lead and cheap seats, so a wrong row moves the conclusion and not + # just the total. + moonshotai/kimi-k3: + input_per_million_usd: 3.00 + cached_input_per_million_usd: 0.30 + output_per_million_usd: 15.00 + cache_read_rate: 0.0 + deepseek/deepseek-v4-flash-0731: + input_per_million_usd: 0.133 + cached_input_per_million_usd: 0.0266 + output_per_million_usd: 0.266 + cache_read_rate: 0.0 +trial_budget: + # Identical to the LHTB cells it is meant to be read against. Does not bind + # on LHTB anyway -- Harbor enforces each task's own `[agent] timeout_sec` + # scaled by the run's --timeout-multiplier (3.0). + timeout_seconds: 36000 + +environment: + # Identical in every condition. override_cpus 4 with -n 8 is exactly 1:1 on a + # 32-vCPU m7a.8xlarge. + override_cpus: 4 + override_memory_mb: 8192 diff --git a/benchmarks/harbor-buzz-orchestra/manifests/tb-solo-deepseek-v4-flash-high.yaml b/benchmarks/harbor-buzz-orchestra/manifests/tb-solo-deepseek-v4-flash-high.yaml new file mode 100644 index 000000000..30ec5183e --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/manifests/tb-solo-deepseek-v4-flash-high.yaml @@ -0,0 +1,67 @@ +# OR2h -- solo deepseek/deepseek-v4-flash-0731 at reasoning effort `high`, +# via OpenRouter. +# +# The cheap end of the OpenRouter pair. Against tb-solo-kimi-k3-high this is a +# tier comparison at a 22x input price ratio ($0.133 vs $3.00 per Mtok), which +# makes it the natural open-weight analogue of the luna -> sol ladder. +# +# THE UPSTREAM IS PINNED, AND THE CHOICE IS NOT THE OBVIOUS ONE. OpenRouter +# serves this model id from nine upstreams spanning fp4 to fp8 quantization, +# 262K to 1M context, and $0.09-$0.14 per Mtok. Cheapest is deepinfra/fp4 -- +# and it caches well -- but fp4 is a materially more degraded set of weights +# than the fp8 every other endpoint serves, so benchmarking it and labelling +# the row "deepseek-v4-flash-0731" would overstate what ran. This pins +# `gmicloud/fp8`: fp8, the full 1M context, 98.9% uptime, and the cheapest of +# the fp8 endpoints at $0.133/$0.266. +# +# Caching was MEASURED per endpoint, not read off the capability flag, because +# the flag is wrong. `/api/v1/models/.../endpoints` advertises +# `supports_implicit_caching: false` for gmicloud/fp8 -- which caches a repeated +# 21K prefix on every single call -- and `true` only for deepseek/fp8, which is +# not routable on this account at all (404 with allow_fallbacks off). Measured, +# three calls each: gmicloud/fp8 cached 21760 every time at $0.00059/call; +# siliconflow/fp8 cached 0 every time at $0.00306; unpinned cached 0, 0, 21760. +# So the pin buys ~5x on input cost as well as a fixed condition. +# +# EFFORT IS PINNED BUT ITS EFFECT ON THIS MODEL IS SMALL AND WORTH CHECKING IN +# THE RESULTS. deepseek-v4-flash-0731 advertises supported_efforts +# ["max","high","low"] and defaults to "high", so `high` here matches the +# provider default; it is pinned anyway so a later default change cannot +# silently redefine the cell. On the fp4 endpoint, reasoning length did not +# separate low from high at all (medians 1097 / 1067, within-level spread +# wider than between-level). If the effort cells land on top of each other, +# that measurement -- not the benchmark -- is the first thing to re-read. +schema_version: "1" +condition: tb-solo-deepseek-v4-flash-high +roster: + - id: solo + kind: orchestrator + role: solo + count: 1 + endpoint: deepseek/deepseek-v4-flash-0731 + model_revision: deepseek/deepseek-v4-flash-20260731 + prompt: + # Byte-identical to every other solo cell in the study. + path: personas/bench/solo.md + sha256: 61c9ac9dc47cbd3c987a7ee6ceb7397496834df8d3fc284c8fe6980e1d5b4ebe + generation: + thinking_effort: high +prices: + deepseek/deepseek-v4-flash-0731: + # The gmicloud/fp8 endpoint's own rates -- NOT the model id's headline + # $0.14/$0.28, which belongs to a different set of upstreams. Pinning is + # what makes this row the correct one. + input_per_million_usd: 0.133 + cached_input_per_million_usd: 0.0266 + output_per_million_usd: 0.266 + # Fallback only; the pinned route reports the real split and the harness + # prefers it. See tb-solo-kimi-k3-high.yaml. + cache_read_rate: 0.0 +trial_budget: + # Identical to every other TB solo cell. + timeout_seconds: 36000 + +environment: + # Identical in every condition -- see tb-solo-luna.yaml for the full reasoning. + override_cpus: 4 + override_memory_mb: 8192 diff --git a/benchmarks/harbor-buzz-orchestra/manifests/tb-solo-kimi-k3-high.yaml b/benchmarks/harbor-buzz-orchestra/manifests/tb-solo-kimi-k3-high.yaml new file mode 100644 index 000000000..1887b8bc5 --- /dev/null +++ b/benchmarks/harbor-buzz-orchestra/manifests/tb-solo-kimi-k3-high.yaml @@ -0,0 +1,67 @@ +# OR1h -- solo moonshotai/kimi-k3 at reasoning effort `high`, via OpenRouter. +# +# The first cell in the study on a route other than direct-OpenAI or Databricks, +# and the first on open-weight models. Read it against the OpenAI solo cells for +# a tier comparison only -- the ROUTE differs, so this is not a clean +# model-versus-model contrast in the way A5m -> A5h is a clean effort contrast. +# The same caveat that gated G1s versus A2 applies here and is larger. +# +# THE UPSTREAM IS PINNED, AND THAT IS LOAD-BEARING. `moonshotai/kimi-k3` is not +# one deployment: OpenRouter serves it from ten upstreams spanning mxfp4/fp8/ +# unknown quantization, 912K-1048K context, and $3.00-$4.50 per Mtok input. +# Unpinned, consecutive requests land on different ones and the cell measures a +# mixture whose composition moves with provider load. testbed/endpoints/ +# openrouter-live.json pins `moonshotai/mxfp4` -- Moonshot's own deployment, +# 100% uptime over the sampling window -- and buzz-agent sends it with +# `allow_fallbacks: false`, so an unavailable upstream fails loudly instead of +# quietly redefining the condition mid-run. +# +# The pin also decides whether prompt caching happens at all. Measured +# 2026-08-01 over a fixed 21K-token prefix, three calls: unpinned returned +# cached_tokens 0, 0, 20480; pinned to moonshotai/mxfp4 it returned 20480 every +# time, $0.00715 against $0.06245 per call. Nearly 9x on input cost, decided by +# routing luck. Do not remove the pin to "let OpenRouter optimise". +# +# EFFORT REACHES THE MODEL AS OpenRouter's `reasoning.effort`, not OpenAI's +# `reasoning_effort` -- see apply_openrouter_mutations in llm.rs. kimi-k3 +# advertises supported_efforts ["max","high","low"] and defaults to "max", so +# `high` here is a deliberate step DOWN from the provider default and must stay +# pinned: leaving it unset would silently run this cell at max. +# +# PRICES are the moonshotai/mxfp4 endpoint's own rates, which are also the +# headline rates for the model id. They are NOT the rates of the two premium +# kimi endpoints ($4.50/$22.50); the pin is what makes the cheaper row correct, +# which is another reason it cannot be dropped without restating cost. +schema_version: "1" +condition: tb-solo-kimi-k3-high +roster: + - id: solo + kind: orchestrator + role: solo + count: 1 + endpoint: moonshotai/kimi-k3 + model_revision: moonshotai/kimi-k3-20260715 + prompt: + # Byte-identical to every other solo cell in the study. + path: personas/bench/solo.md + sha256: 61c9ac9dc47cbd3c987a7ee6ceb7397496834df8d3fc284c8fe6980e1d5b4ebe + generation: + thinking_effort: high +prices: + moonshotai/kimi-k3: + input_per_million_usd: 3.00 + cached_input_per_million_usd: 0.30 + output_per_million_usd: 15.00 + # Fallback only. OpenRouter reports the split at + # usage.prompt_tokens_details.cached_tokens, which buzz-agent already reads + # (openai_chat_cached_tokens, llm.rs) -- verified live on this exact pinned + # route, not assumed. 0.0 keeps an unmeasured run conservative. + cache_read_rate: 0.0 +trial_budget: + # Identical to every other TB solo cell. + timeout_seconds: 36000 + +environment: + # Identical in every condition -- see tb-solo-luna.yaml for the full reasoning. + override_cpus: 4 + override_memory_mb: 8192