Files
buzz/desktop/test-loader-hooks.mjs
1b7e5ac1be feat(model-capabilities): drive model capabilities and labels from one manifest (#5597)
## Summary

Centralizes model capability knowledge — thinking mode, supported effort
levels, wire routes, and human-readable labels — into a single manifest,
`scripts/model-capabilities.json`. Rust and TypeScript each get a small
interpreter that reads the same manifest, replacing hand-maintained
tables scattered across both languages that had already drifted apart. A
capability change is now a data edit, not parallel edits to two code
paths. Supersedes the codegen approach explored in #3603.

A cross-language contract keeps the two interpreters honest:
`scripts/normative-corpus.json` is a golden snapshot generated from the
Rust resolver (103 vectors covering all six capability axes) and
replayed natively in TS. CI fails if either language disagrees with the
corpus or the corpus drifts from the resolver. Regenerate with `just
regen-model-corpus`.

## Behavior changes

- **Effort dropdown for `openai-compat` providers** no longer offers
`max`. The request path always clamped `max` to `xhigh` on the wire, so
the UI stops offering a value that was silently rewritten. UI-only,
wire-identical.
- **Databricks v2 routing (wire-visible):** uncurated endpoint names
carrying a bare Claude code-name segment (e.g. `goose-opus-5`) now route
to the MLflow chat wire instead of Anthropic Messages — they lose
Anthropic prompt caching but still succeed on a valid OpenAI-compatible
wire. Curated `databricks-claude-*` records and any name starting with
`claude` are unchanged. A handful of other uncurated/adversarial name
shapes similarly fall back to MLflow chat instead of pattern-matched
routes; every curated model resolves identically to before, all axes.
- **Curated model labels on the real discovery path.** The Databricks
API returns no display name, so discovery emits the raw endpoint id as
the model `name` (`{id, name: id}`) on every path. `ModelEntry.name` is
now curated at all four construction seams in `buzz-agent` — v2
discovery, v1 parse, the auth-empty default catalog, and the
configured-model fallback — via a read-only `databricks_registry_label`
lookup over the manifest's `databricks_v2` exact records; `id` stays the
raw wire/config value. A known id renders its curated label
(`databricks-gpt-5-5` → `GPT-5.5`), an unknown id passes through
unchanged, and the default-catalog row reads `GPT-5.5 (default
catalog)`. As a defense against older `buzz-agent` binaries and any
harness that echoes ids, `resolveModelLabel` treats a discovered name
equal to the trimmed id as absent and falls through to the registry
tier; a genuinely distinct name (including the suffixed default-catalog
label) still wins.

## Cleanup

Deletes the duplicated capability tables and their tests: the
`config.rs` gpt5 matchers, effort tables, and clamp logic; the legacy
segment-based Databricks v2 route classifier in `llm.rs`; and the TS
hand tables plus `effortTable.fixture.json`. All are replaced by
manifest lookups through the shared resolver — no line of capability
data exists in two places.

---------

Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
Signed-off-by: Duncan <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@buzz.block.builderlab.xyz>
Co-authored-by: Duncan <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@buzz.block.builderlab.xyz>
2026-08-17 11:23:28 -04:00

191 lines
6.3 KiB
JavaScript

import { fileURLToPath, pathToFileURL } from "node:url";
import fs from "node:fs";
import path from "node:path";
import ts from "typescript";
const srcRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"src",
);
const repoRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"..",
);
// `nextResolve` requires specifiers to be URLs or relative paths. Passing an
// absolute filesystem path happens to work on POSIX (node coerces it), but on
// Windows an absolute path like `C:\...` parses as a URL with protocol `c:`
// and every test run dies with ERR_UNSUPPORTED_ESM_URL_SCHEME. Hand absolute
// paths to node as proper file:// URLs on all platforms.
function toFileSpecifier(candidatePath) {
return path.isAbsolute(candidatePath)
? pathToFileURL(candidatePath).href
: candidatePath;
}
function resolveSourcePath(basePath) {
// Existence decides, not path.extname — a dotted basename like
// `ProfileAvatarEditor.utils` (→ .utils.ts on disk) looks like an
// extension but still needs resolving.
if (fs.existsSync(basePath) && fs.statSync(basePath).isFile()) {
return basePath;
}
for (const extension of [".ts", ".tsx", ".js", ".jsx", ".mjs"]) {
const candidate = `${basePath}${extension}`;
if (fs.existsSync(candidate)) {
return candidate;
}
}
for (const extension of [".ts", ".tsx", ".js", ".jsx", ".mjs"]) {
const candidate = path.join(basePath, `index${extension}`);
if (fs.existsSync(candidate)) {
return candidate;
}
}
return null;
}
// emoji-mart ships a bundled CJS main that node's cjs-module-lexer cannot
// extract named exports from (`import { init } from "emoji-mart"` throws
// under node ESM even though the bundler handles it). Tests never exercise
// the picker, so serve inert stubs for the emoji-mart entrypoints.
const stubModules = new Map([
[
"emoji-mart",
"export const init = () => {};\n" +
"export const SearchIndex = { search: async () => [] };\n" +
"export default {};\n",
],
["@emoji-mart/react", "export default function Picker() { return null; }\n"],
]);
const STUB_URL_PREFIX = "buzz-test-stub:";
// Vite resolves asset imports (`./logo.png`, `./logo.png?inline`) to a URL or
// base64 string at bundle time; node's ESM resolver has no such loader and
// throws on the query suffix. Serve an inert string so components that embed
// assets stay unit-testable.
const ASSET_SPECIFIER = /\.(?:png|jpe?g|gif|svg|webp|avif|ico)(?:\?[^/]*)?$/;
const ASSET_URL_PREFIX = "buzz-test-asset:";
export function resolve(specifier, context, nextResolve) {
if (ASSET_SPECIFIER.test(specifier)) {
return {
shortCircuit: true,
url: `${ASSET_URL_PREFIX}${specifier}`,
};
}
if (stubModules.has(specifier)) {
return {
shortCircuit: true,
url: `${STUB_URL_PREFIX}${specifier}`,
};
}
if (specifier === "@features-manifest") {
const resolved = path.join(repoRoot, "preview-features.json");
return nextResolve(toFileSpecifier(resolved), context);
}
if (specifier === "@model-capabilities-manifest") {
const resolved = path.join(repoRoot, "scripts", "model-capabilities.json");
return nextResolve(toFileSpecifier(resolved), context);
}
if (specifier.startsWith("@/")) {
const stripped = specifier.slice(2);
// Preserve explicit extensions (.mjs, .js, .json, .ts, etc.). The bundler
// tolerates extensionless `@/` imports for source files; node's ESM
// resolver does not, so resolve against the extensions the app uses.
// Otherwise paths like `@/.../foo.mjs` would be coerced into `foo.mjs.ts`
// and fail to resolve.
const resolved = resolveSourcePath(`${srcRoot}/${stripped}`);
return nextResolve(
toFileSpecifier(resolved ?? `${srcRoot}/${stripped}`),
context,
);
}
// Resolve extensionless relative TS imports (e.g. `./parseImeta`) — the app's
// bundler adds the extension, but node's ESM resolver does not. Without this,
// any .ts that relative-imports a sibling .ts can't be imported from a test,
// which previously forced stale inlined copies of the source under test.
// Dotted basenames (`./ProfileAvatarEditor.utils`) look like extensions to
// path.extname, so resolveSourcePath existence-checks instead.
if (
(specifier.startsWith("./") || specifier.startsWith("../")) &&
context.parentURL?.startsWith("file:")
) {
const parentPath = fileURLToPath(context.parentURL);
const resolved = resolveSourcePath(
path.resolve(path.dirname(parentPath), specifier),
);
if (resolved) {
return nextResolve(toFileSpecifier(resolved), context);
}
return nextResolve(specifier, context);
}
return nextResolve(specifier, context);
}
export async function load(url, context, nextLoad) {
if (url.startsWith(ASSET_URL_PREFIX)) {
return {
format: "module",
shortCircuit: true,
source: 'export default "test-asset";\n',
};
}
if (url.startsWith(STUB_URL_PREFIX)) {
return {
format: "module",
shortCircuit: true,
source: stubModules.get(url.slice(STUB_URL_PREFIX.length)) ?? "",
};
}
// The app bundler loads .json imports without attributes (e.g. the bare
// `@emoji-mart/data` entrypoint); node's ESM resolver requires
// `with { type: "json" }` on every hop. Serve json here so transitive
// imports from source under test don't need bundler-only semantics.
if (url.endsWith(".json")) {
return {
format: "json",
shortCircuit: true,
source: fs.readFileSync(fileURLToPath(url), "utf8"),
};
}
// Vite handles side-effect CSS imports (e.g. `import "./card-texture.css"`
// in shared/ui) at bundle time; node's ESM loader has no CSS support. Serve
// them as empty modules so components with style imports stay unit-testable.
if (url.endsWith(".css")) {
return {
format: "module",
shortCircuit: true,
source: "",
};
}
if (url.endsWith(".tsx")) {
const source = fs.readFileSync(fileURLToPath(url), "utf8");
const transpiled = ts.transpileModule(source, {
compilerOptions: {
jsx: ts.JsxEmit.ReactJSX,
module: ts.ModuleKind.ESNext,
target: ts.ScriptTarget.ES2020,
},
fileName: fileURLToPath(url),
});
return {
format: "module",
shortCircuit: true,
source: transpiled.outputText,
};
}
return nextLoad(url, context);
}