External/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-08-18 06:50:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(models): round-3 corrective pass — boundary strip, gpt-version-segment, 6-axis corpus, validation symmetry, prototype-leak

Browse Source 
- stripCatalogPrefix boundary-aligned: customgpt/sgpt/mygpt no longer false-strip
  to gpt-* forms; boundary check requires position 0 or non-alphanumeric precursor
- New match_kind gpt-version-segment: gpt-neox-20b routes mlflow-chat (was residual
  collision); gpt segment match requires next segment to start with digit or be
  dashless gpt5 form
- PROVIDER_FALLBACKS Record→Map: constructor/__proto__ prototype-key leak closed;
  resolveModelCapabilities/getProviderEffortConfig return complete records for all
  provider strings
- Corpus 69→77 vectors, all 6-axis mandatory; JS and Rust runners hard-fail on missing
  axes; prototype-key vectors added; boundary-negative vectors added (sgpt-5-5,
  mygpt-5, customgpt-5-5-endpoint all route mlflow-chat); gpt-neox-20b pinned as
  mlflow-chat (corrected behavior, not residual collision)
- Manifest validator 33→42 tests: family match_priority integer guard, lowercase
  duplicate-key detection, post-inheritance materialized default∈supported check,
  assertEfforts shared helper covering family/fallback/exact
- Clippy: map_or(false)→is_some_and in emitter template; fmt clean
- Desktop regression tests: constructor/__proto__ prototype-key stability

Co-authored-by: Will Pfleger <pfleger.will@gmail.com>
Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
This commit is contained in:
npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7
2026-08-04 15:30:06 -04:00
co-authored by Will Pfleger
parent 6301594dc6
commit 60fc24b0c6
9 changed files with 1163 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/buzz-agent/src/generated_model_capabilities.rs
+69 -14
View File
@@ -258,18 +258,48 @@ pub fn lookup_exact(provider: &str, raw_model_id: &str) -> Option<CapabilityResu
// ---------------------------------------------------------------------------
/// Strip any catalog-naming prefix to get the normalized model alias for family matching.
/// Finds the first occurrence of a known family token (claude-, gpt-) and returns from there.
/// Finds the first boundary-aligned occurrence of a known family token and returns from there.
/// Boundary-aligned: the token must start at position 0 or be preceded by a non-alphanumeric char.
/// This prevents "customgpt-5-5-endpoint" from stripping to "gpt-5-5-endpoint" via "gpt-" inside
/// the "customgpt-" prefix.
///
/// Examples:
/// "goose-claude-fable-5" → "claude-fable-5"
/// "databricks-gpt-5.5" → "gpt-5.5"
/// "team-x-claude-opus-4-7" → "claude-opus-4-7"
/// "claude-opus-4-7" → "claude-opus-4-7" (no prefix)
/// "goose-claude-fable-5" → "claude-fable-5" (boundary: preceded by "-")
/// "databricks-gpt-5.5" → "gpt-5.5" (boundary: preceded by "-")
/// "team-x-claude-opus-4-7" → "claude-opus-4-7" (boundary: preceded by "-")
/// "claude-opus-4-7" → "claude-opus-4-7" (no prefix, already boundary)
/// "customgpt-5-5-ep" → "customgpt-5-5-ep" (no boundary match for "gpt-")
/// "llama-3" → "llama-3" (no family token)
pub fn strip_catalog_prefix(model: &str) -> &str {
const FAMILY_TOKENS: &[&str] = &["claude-", "gpt-"];
let lower = model.to_ascii_lowercase();
let first_idx = FAMILY_TOKENS.iter().filter_map(|tok| lower.find(tok)).min();
let mut first_idx: Option<usize> = None;
for tok in FAMILY_TOKENS {
let tok_bytes = tok.as_bytes();
let lower_bytes = lower.as_bytes();
let mut start = 0usize;
loop {
match lower_bytes[start..]
.windows(tok_bytes.len())
.position(|w| w == tok_bytes)
{
None => break,
Some(rel) => {
let idx = start + rel;
// Boundary check: position 0 or preceded by a non-alphanumeric byte
let at_boundary = idx == 0 || {
let prev = lower_bytes[idx - 1];
!prev.is_ascii_alphanumeric()
};
if at_boundary {
first_idx = Some(first_idx.map_or(idx, |f| f.min(idx)));
break;
}
start = idx + 1;
}
}
}
}
match first_idx {
Some(idx) => &model[idx..],
None => model,
@@ -1004,12 +1034,8 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option<Capabi
}
// rule: dbv2-gpt-code-names-segment, provider: databricks_v2, priority: 6
if provider == "databricks_v2"
&& (lower
.split(|c: char| !c.is_ascii_alphanumeric())
.any(|s| s == "gpt")
|| lower
.split(|c: char| !c.is_ascii_alphanumeric())
.any(|s| s == "gpt5"))
&& (gpt_version_segment_matches_rs(lower, "gpt")
|| gpt_version_segment_matches_rs(lower, "gpt5"))
{
return Some(CapabilityResult {
registry_label: None,
@@ -1448,8 +1474,7 @@ fn gpt5_base_matches_rs(model: &str, token: &str) -> bool {
let first_non_digit = dash_rest
.find(|c: char| !c.is_ascii_digit())
.unwrap_or(dash_rest.len());
let is_short_version = first_non_digit >= 1
&& first_non_digit <= 3
let is_short_version = (1..=3).contains(&first_non_digit)
&& (first_non_digit == dash_rest.len()
|| !dash_rest.as_bytes()[first_non_digit].is_ascii_alphanumeric());
if is_short_version {
@@ -1462,6 +1487,36 @@ fn gpt5_base_matches_rs(model: &str, token: &str) -> bool {
}
}
// ---------------------------------------------------------------------------
// gpt-version-segment helper (used by generated family resolver)
// ---------------------------------------------------------------------------
/// gpt-version-segment match: token is an exact segment AND (token itself starts with a digit,
/// OR the next segment after it starts with a digit). Prevents "gpt-neox-20b" from matching
/// "gpt" because "neox" starts with a letter, while "gpt-5.5" matches because next seg "5" is digit.
fn gpt_version_segment_matches_rs(model: &str, token: &str) -> bool {
let segs: Vec<&str> = model.split(|c: char| !c.is_ascii_alphanumeric()).collect();
for (i, seg) in segs.iter().enumerate() {
if *seg == token {
// Dashless numeric form (e.g. "gpt5"): token length > 3 or starts with digit
if token.len() > 3 || token.as_bytes().first().is_some_and(|b| b.is_ascii_digit()) {
return true;
}
// For short alpha tokens like "gpt": require the next segment to start with a digit
if let Some(next_seg) = segs.get(i + 1) {
if next_seg
.as_bytes()
.first()
.is_some_and(|b| b.is_ascii_digit())
{
return true;
}
}
}
}
false
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
crates/buzz-agent/src/generated_model_capabilities_tests.rs
+34 -29
View File
@@ -40,14 +40,16 @@ mod shared_corpus_tests {
expect: Option<CorpusExpect>,
}
/// All six axes are required on every corpus vector.
/// A missing axis is a schema error that hard-fails the test.
#[derive(Deserialize)]
struct CorpusExpect {
thinking_mode: Option<String>,
supported_efforts: Option<Vec<String>>,
default_effort: Option<serde_json::Value>, // string or null
databricks_v2_wire_route: Option<String>,
normalization_policy: Option<String>,
registry_label: Option<serde_json::Value>, // string or null
thinking_mode: String,
supported_efforts: Vec<String>,
default_effort: serde_json::Value, // string or null
databricks_v2_wire_route: String,
normalization_policy: String,
registry_label: serde_json::Value, // string or null
}
// ---------------------------------------------------------------------------
@@ -162,9 +164,10 @@ mod shared_corpus_tests {
let result = resolve_model_capabilities(canonical_provider, raw_model_id);
ran += 1;
// Check thinking_mode if present in expect
if let Some(expected_mode) = &expect.thinking_mode {
let expected = parse_thinking_mode(expected_mode);
// All six axes are required — no optional field checks.
// thinking_mode
{
let expected = parse_thinking_mode(&expect.thinking_mode);
if result.thinking_mode != expected {
failures.push(format!(
"[{id}] thinking_mode: got {:?}, expected {:?}",
@@ -173,10 +176,13 @@ mod shared_corpus_tests {
}
}
// Check supported_efforts if present
if let Some(expected_efforts) = &expect.supported_efforts {
let expected: Vec<ThinkingEffort> =
expected_efforts.iter().map(|s| parse_effort(s)).collect();
// supported_efforts
{
let expected: Vec<ThinkingEffort> = expect
.supported_efforts
.iter()
.map(|s| parse_effort(s))
.collect();
let actual: Vec<ThinkingEffort> =
result.supported_efforts.iter().cloned().collect();
if actual != expected {
@@ -186,9 +192,9 @@ mod shared_corpus_tests {
}
}
// Check default_effort if present
if let Some(expected_de) = &expect.default_effort {
let expected_parsed = match expected_de {
// default_effort (string or null)
{
let expected_parsed: Option<ThinkingEffort> = match &expect.default_effort {
serde_json::Value::Null => None,
serde_json::Value::String(s) => Some(parse_effort(s)),
other => {
@@ -203,9 +209,9 @@ mod shared_corpus_tests {
}
}
// Check databricks_v2_wire_route if present
if let Some(expected_route) = &expect.databricks_v2_wire_route {
let expected = parse_route(expected_route);
// databricks_v2_wire_route
{
let expected = parse_route(&expect.databricks_v2_wire_route);
if result.databricks_v2_wire_route != expected {
failures.push(format!(
"[{id}] databricks_v2_wire_route: got {:?}, expected {:?}",
@@ -214,9 +220,9 @@ mod shared_corpus_tests {
}
}
// Check normalization_policy if present
if let Some(expected_policy) = &expect.normalization_policy {
let expected = parse_normalization_policy(expected_policy);
// normalization_policy
{
let expected = parse_normalization_policy(&expect.normalization_policy);
if result.normalization_policy != expected {
failures.push(format!(
"[{id}] normalization_policy: got {:?}, expected {:?}",
@@ -225,14 +231,14 @@ mod shared_corpus_tests {
}
}
// Check registry_label if present (JSON string or null)
if let Some(expected_rl) = &expect.registry_label {
let expected_opt: Option<&str> = match expected_rl {
// registry_label (string or null)
{
let expected_opt: Option<&str> = match &expect.registry_label {
serde_json::Value::Null => None,
serde_json::Value::String(s) => Some(s.as_str()),
other => panic!(
"unexpected registry_label value in corpus vector {id}: {other:?}"
),
other => {
panic!("unexpected registry_label value in corpus vector {id}: {other:?}")
}
};
if result.registry_label != expected_opt {
failures.push(format!(
@@ -241,7 +247,6 @@ mod shared_corpus_tests {
));
}
}
}
if !failures.is_empty() {
desktop/src/features/agents/ui/buzzAgentConfig.test.mjs
+37
View File
@@ -703,3 +703,40 @@ test("openai-compat alias handles mixed case + whitespace: ' OpenAI-Compat ' c
assert.deepEqual([...messy.validValues], [...canonical.validValues]);
assert.equal(messy.defaultValue, canonical.defaultValue);
});
// ---------------------------------------------------------------------------
// PROVIDER_FALLBACKS prototype-key safety regression
// ---------------------------------------------------------------------------
test("prototype-key safety: provider='constructor' returns a complete record (Map prevents prototype leak)", () => {
const result = getProviderEffortConfig("constructor", "some-model");
// Must not crash and must return a non-empty validValues (complete record)
assert.ok(Array.isArray(result.validValues) && result.validValues.length > 0,
"constructor provider must return a complete record (not an empty/broken result from prototype chain)");
});
test("prototype-key safety: provider='__proto__' returns a complete record", () => {
const result = getProviderEffortConfig("__proto__", "some-model");
assert.ok(Array.isArray(result.validValues) && result.validValues.length > 0,
"__proto__ provider must return a complete record");
});
test("prototype-key safety: provider='hasOwnProperty' returns a complete record", () => {
const result = getProviderEffortConfig("hasOwnProperty", "some-model");
assert.ok(Array.isArray(result.validValues) && result.validValues.length > 0,
"hasOwnProperty provider must return a complete record");
});
test("prototype-key safety: provider='toString' returns a complete record", () => {
const result = getProviderEffortConfig("toString", "some-model");
assert.ok(Array.isArray(result.validValues) && result.validValues.length > 0,
"toString provider must return a complete record");
});
test("prototype-key safety: validValues.includes() does not throw for prototype-key provider", () => {
// This exercises the crash path: EffortSelectField calls validValues.includes()
const result = getProviderEffortConfig("constructor", "some-model");
assert.doesNotThrow(() => result.validValues.includes("low"),
"validValues.includes() must not throw for prototype-key providers");
});
desktop/src/features/agents/ui/modelCapabilities.ts
+46 -17
View File
@@ -122,6 +122,25 @@ function gpt5BaseMatchesGenerated(m: string, token: string): boolean {
}
}
/**
* gpt-version-segment match: token is an exact segment AND (token itself starts with a digit,
* OR the next segment after it starts with a digit). Prevents "gpt-neox-20b" from matching
* "gpt" because "neox" starts with a letter, while "gpt-5.5" matches because next seg "5" is digit.
*/
function gptVersionSegmentMatchesGenerated(m: string, token: string): boolean {
const segs = m.split(/[^a-z0-9]+/);
for (let i = 0; i < segs.length; i++) {
if (segs[i] === token) {
// Dashless numeric form (e.g. "gpt5") or token itself starts with digit
if (token.length > 3 || /^\d/.test(token)) return true;
// For "gpt": require the next segment to start with a digit
const nextSeg = segs[i + 1];
if (nextSeg !== undefined && /^\d/.test(nextSeg)) return true;
}
}
return false;
}
// ---------------------------------------------------------------------------
// Exact records — provider-qualified, pre-prefix-stripping
// ---------------------------------------------------------------------------
@@ -189,8 +208,8 @@ const EXACT_RECORDS = new Map<string, CapabilityResult>([
// Provider fallbacks
// ---------------------------------------------------------------------------
const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnknown: CapabilityResult }> = {
"anthropic": {
const PROVIDER_FALLBACKS = new Map<string, { blank: CapabilityResult; concreteUnknown: CapabilityResult }>([
["anthropic", {
blank: {
registryLabel: null,
thinkingMode: "adaptive",
@@ -207,8 +226,8 @@ const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnkn
databricksV2WireRoute: "not-applicable",
normalizationPolicy: "none",
},
},
"openai": {
}],
["openai", {
blank: {
registryLabel: null,
thinkingMode: "none",
@@ -225,8 +244,8 @@ const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnkn
databricksV2WireRoute: "not-applicable",
normalizationPolicy: "openai-clamp-max-to-xhigh",
},
},
"databricks_v2": {
}],
["databricks_v2", {
blank: {
registryLabel: null,
thinkingMode: "none",
@@ -243,8 +262,8 @@ const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnkn
databricksV2WireRoute: "mlflow-chat",
normalizationPolicy: "openai-clamp-max-to-xhigh",
},
},
"databricks": {
}],
["databricks", {
blank: {
registryLabel: null,
thinkingMode: "none",
@@ -261,8 +280,8 @@ const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnkn
databricksV2WireRoute: "not-applicable",
normalizationPolicy: "openai-clamp-max-to-xhigh",
},
},
"openrouter": {
}],
["openrouter", {
blank: {
registryLabel: null,
thinkingMode: "none",
@@ -279,8 +298,8 @@ const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnkn
databricksV2WireRoute: "not-applicable",
normalizationPolicy: "none",
},
},
};
}],
]);
const DEFAULT_FALLBACK = {
blank: {
@@ -302,15 +321,25 @@ const DEFAULT_FALLBACK = {
};
// ---------------------------------------------------------------------------
// Strip catalog prefix — finds first family token occurrence
// Strip catalog prefix — boundary-aware, finds first boundary-aligned family token
// ---------------------------------------------------------------------------
export function stripCatalogPrefix(model: string): string {
const FAMILY_TOKENS = ["claude-", "gpt-"] as const;
const lower = model.toLowerCase();
let firstIdx = Infinity;
for (const tok of FAMILY_TOKENS) {
const idx = model.toLowerCase().indexOf(tok);
if (idx !== -1 && idx < firstIdx) firstIdx = idx;
let start = 0;
while (true) {
const idx = lower.indexOf(tok, start);
if (idx === -1) break;
// Boundary check: position 0 or preceded by a non-alphanumeric character
if (idx === 0 || !/[a-z0-9]/.test(lower[idx - 1])) {
if (idx < firstIdx) firstIdx = idx;
break;
}
start = idx + 1;
}
}
return firstIdx === Infinity ? model : model.slice(firstIdx);
}
@@ -762,7 +791,7 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe
};
}
// rule: dbv2-gpt-code-names-segment, provider: databricks_v2, priority: 6
if (provider === "databricks_v2" && (lower.split(/[^a-z0-9]+/).includes("gpt") || lower.split(/[^a-z0-9]+/).includes("gpt5"))) {
if (provider === "databricks_v2" && (gptVersionSegmentMatchesGenerated(lower, "gpt") || gptVersionSegmentMatchesGenerated(lower, "gpt5"))) {
return {
registryLabel: null,
thinkingMode: "none",
@@ -828,6 +857,6 @@ export function resolveModelCapabilities(
// Step 3: provider fallback
const isBlank = rawModelId.trim() === "";
const fb = PROVIDER_FALLBACKS[provider] ?? DEFAULT_FALLBACK;
const fb = PROVIDER_FALLBACKS.get(provider) ?? DEFAULT_FALLBACK;
return isBlank ? { ...fb.blank, registryLabel: null } : { ...fb.concreteUnknown, registryLabel: null };
}
scripts/generate-model-capabilities.mjs
+197 -52
View File
@@ -290,6 +290,7 @@ const VALID_MATCH_KINDS = [
"gpt5-base",
"segment",
"segment-prefix",
"gpt-version-segment",
];
function assertEnum(value, valid, label) {
@@ -304,13 +305,32 @@ function assertNonEmpty(arr, label) {
}
}
/**
* Shared effort-list validator: non-empty, all valid enum values, no duplicates,
* canonical sort order (Rust clamp logic depends on sorted order).
*/
function assertEfforts(efforts, label) {
assertNonEmpty(efforts, label);
const seen = new Set();
for (const e of efforts) {
assertEnum(e, VALID_EFFORTS, `${label}[]`);
if (seen.has(e)) throw new Error(`${label}: duplicate effort "${e}"`);
seen.add(e);
}
const indices = efforts.map((e) => VALID_EFFORTS.indexOf(e));
for (let i = 1; i < indices.length; i++) {
if (indices[i] <= indices[i - 1]) {
throw new Error(
`${label}: must follow canonical order [${VALID_EFFORTS.join(", ")}]; got [${efforts.join(", ")}]`,
);
}
}
}
function validateFallbackRecord(rec, label) {
assertEnum(rec.databricks_v2_wire_route, VALID_DBV2_ROUTES, `${label}.databricks_v2_wire_route`);
assertEnum(rec.thinking_mode, VALID_THINKING_MODES, `${label}.thinking_mode`);
assertNonEmpty(rec.supported_efforts, `${label}.supported_efforts`);
for (const e of rec.supported_efforts) {
assertEnum(e, VALID_EFFORTS, `${label}.supported_efforts[]`);
}
assertEfforts(rec.supported_efforts, `${label}.supported_efforts`);
if (rec.default_effort !== null) {
assertEnum(rec.default_effort, VALID_EFFORTS, `${label}.default_effort`);
if (!rec.supported_efforts.includes(rec.default_effort)) {
@@ -334,10 +354,11 @@ for (const rule of manifest.family_rules) {
seenRuleIds.add(rule.id);
assertEnum(rule.match_kind, VALID_MATCH_KINDS, `rule ${rule.id} match_kind`);
assertEnum(rule.thinking_mode, VALID_THINKING_MODES, `rule ${rule.id} thinking_mode`);
assertNonEmpty(rule.supported_efforts, `rule ${rule.id} supported_efforts`);
for (const e of rule.supported_efforts) {
assertEnum(e, VALID_EFFORTS, `rule ${rule.id} supported_efforts[]`);
// match_priority must be a non-negative integer (interpolated into Rust comments and TS code)
if (!Number.isInteger(rule.match_priority) || rule.match_priority < 0) {
throw new Error(`rule ${rule.id}: match_priority must be a non-negative integer, got ${JSON.stringify(rule.match_priority)}`);
}
assertEfforts(rule.supported_efforts, `rule ${rule.id} supported_efforts`);
if (rule.default_effort !== null) {
assertEnum(rule.default_effort, VALID_EFFORTS, `rule ${rule.id} default_effort`);
if (!rule.supported_efforts.includes(rule.default_effort)) {
@@ -368,39 +389,25 @@ for (const [provider, fb] of Object.entries(manifest.provider_fallbacks)) {
}
// Validate exact_records — full invariant checks on every override axis
// Keys are normalized to lowercase before duplicate detection to match build-time key lowercasing.
const seenExactKeys = new Set();
for (const rec of manifest.exact_records ?? []) {
if (!rec.provider || !rec.raw_model_id)
throw new Error("exact_record missing provider or raw_model_id");
const key = `${rec.provider}::${rec.raw_model_id}`;
// Normalize to lowercase — both emitters lowercase provider+raw_model_id at build time
const key = `${rec.provider.toLowerCase()}::${rec.raw_model_id.toLowerCase()}`;
if (seenExactKeys.has(key)) throw new Error(`duplicate exact_record key: ${key}`);
seenExactKeys.add(key);
// Validate match_priority: must be a non-negative integer if present (injected into comments).
// Validate match_priority: must be a non-negative integer if present (interpolated into source).
if (rec.match_priority !== undefined) {
if (!Number.isInteger(rec.match_priority) || rec.match_priority < 0)
throw new Error(`exact_record ${key}: match_priority must be a non-negative integer`);
}
// Validate supported_efforts_override if present.
// Validate supported_efforts_override if present (shared validator: non-empty, no dupes, canonical order).
if (rec.supported_efforts_override !== undefined) {
assertNonEmpty(rec.supported_efforts_override, `exact_record ${key} supported_efforts_override`);
const seenEfforts = new Set();
for (const e of rec.supported_efforts_override) {
assertEnum(e, VALID_EFFORTS, `exact_record ${key} supported_efforts_override[]`);
if (seenEfforts.has(e))
throw new Error(`exact_record ${key}: duplicate effort "${e}" in supported_efforts_override`);
seenEfforts.add(e);
}
// Validate ordering matches canonical effort order (Rust clamp assumes sorted).
const canonicalIndices = rec.supported_efforts_override.map((e) => VALID_EFFORTS.indexOf(e));
for (let i = 1; i < canonicalIndices.length; i++) {
if (canonicalIndices[i] <= canonicalIndices[i - 1]) {
throw new Error(
`exact_record ${key}: supported_efforts_override must follow canonical order [${VALID_EFFORTS.join(", ")}]; got [${rec.supported_efforts_override.join(", ")}]`,
);
}
}
assertEfforts(rec.supported_efforts_override, `exact_record ${key} supported_efforts_override`);
// Validate default_effort is in the override if present.
if (rec.default_effort !== undefined && rec.default_effort !== null) {
assertEnum(rec.default_effort, VALID_EFFORTS, `exact_record ${key} default_effort`);
@@ -431,23 +438,53 @@ for (const rec of manifest.exact_records ?? []) {
}
}
// Post-inheritance materialized validation for exact_records.
// An exact_record with no supported_efforts_override inherits the family default — validate
// that the final materialized default_effort is within the final materialized supported_efforts.
for (const rec of manifest.exact_records ?? []) {
const key = `${rec.provider.toLowerCase()}::${rec.raw_model_id.toLowerCase()}`;
const materializedResult = resolve(rec.provider, rec.raw_model_id);
const matEfforts = materializedResult.supported_efforts;
const matDefault = materializedResult.default_effort;
if (matDefault !== undefined && matDefault !== null) {
if (!matEfforts.includes(matDefault)) {
throw new Error(
`exact_record ${key}: materialized default_effort "${matDefault}" not in materialized supported_efforts [${matEfforts.join(", ")}] (check family default inheritance)`,
);
}
}
}
// ---------------------------------------------------------------------------
// Resolution engine (mirrors plan resolver contract)
// ---------------------------------------------------------------------------
/**
* Strip catalog prefix to get the normalized alias for family-rule matching.
* Finds the first occurrence of a known family token and returns from there.
* e.g. "goose-claude-fable-5" → "claude-fable-5"
* "databricks-gpt-5.5" → "gpt-5.5"
* "claude-opus-4-7" → "claude-opus-4-7" (no prefix)
* Finds the first boundary-aligned occurrence of a known family token and returns from there.
* Boundary-aligned: the token must start at position 0 or be preceded by a non-alphanumeric char.
* This prevents "customgpt-5-5-endpoint" from stripping to "gpt-5-5-endpoint" via "gpt-" inside
* the "customgpt-" prefix.
* e.g. "goose-claude-fable-5" → "claude-fable-5" (boundary: preceded by "-")
* "databricks-gpt-5.5" → "gpt-5.5" (boundary: preceded by "-")
* "claude-opus-4-7" → "claude-opus-4-7" (no prefix, already at boundary)
* "customgpt-5-5-ep" → "customgpt-5-5-ep" (no boundary match: 'g' preceded by 'm')
*/
function stripCatalogPrefix(model) {
const lower = model.toLowerCase();
let firstIdx = Infinity;
for (const tok of manifest.family_tokens) {
const idx = lower.indexOf(tok);
if (idx !== -1 && idx < firstIdx) firstIdx = idx;
let start = 0;
while (true) {
const idx = lower.indexOf(tok, start);
if (idx === -1) break;
// Boundary check: position 0 or preceded by a non-alphanumeric character
if (idx === 0 || !/[a-z0-9]/.test(lower[idx - 1])) {
if (idx < firstIdx) firstIdx = idx;
break;
}
start = idx + 1;
}
}
return firstIdx === Infinity ? model : model.slice(firstIdx);
}
@@ -516,6 +553,30 @@ function gpt5BaseMatches(model, token) {
}
}
/**
* gpt-version-segment match: the token appears as an exact segment AND the next segment
* (the one immediately after the token) starts with a digit.
* This matches "gpt" in "gpt-5.5" (next seg "5") and "gpt" in "gpt-5-4-mini" (next seg "5")
* but NOT "gpt" in "gpt-neox-20b" (next seg "neox" starts with a letter).
* Also matches the dashless exact-segment alias (e.g. "gpt5") directly via segment equality.
*/
function gptVersionSegmentMatches(model, token) {
const lower = model.toLowerCase();
const tok = token.toLowerCase();
const segs = lower.split(/[^a-z0-9]+/);
for (let i = 0; i < segs.length; i++) {
if (segs[i] === tok) {
// Exact segment match (e.g. "gpt5" matches "gpt5-custom")
if (tok.length > 3 || /^\d/.test(tok)) return true; // dashless numeric form like "gpt5"
// For "gpt": require the next segment to start with a digit
const nextSeg = segs[i + 1];
if (nextSeg !== undefined && /^\d/.test(nextSeg)) return true;
// No valid next segment — this "gpt" segment alone does not match
}
}
return false;
}
/**
* Test if a family rule matches the given (normalized) model string for a provider.
*/
@@ -540,6 +601,8 @@ function ruleMatchesModel(rule, normalizedModel, provider) {
const segs = lower.split(/[^a-z0-9]+/);
return allTokens.some((t) => segs.some((s) => s.startsWith(t.toLowerCase())));
}
case "gpt-version-segment":
return allTokens.some((t) => gptVersionSegmentMatches(lower, t));
default:
throw new Error(`unknown match_kind: ${rule.match_kind}`);
}
@@ -876,21 +939,45 @@ ${emitRustCapabilityResult(clean, " ")}
// ---------------------------------------------------------------------------
/// Strip any catalog-naming prefix to get the normalized model alias for family matching.
/// Finds the first occurrence of a known family token (claude-, gpt-) and returns from there.
/// Finds the first boundary-aligned occurrence of a known family token and returns from there.
/// Boundary-aligned: the token must start at position 0 or be preceded by a non-alphanumeric char.
/// This prevents "customgpt-5-5-endpoint" from stripping to "gpt-5-5-endpoint" via "gpt-" inside
/// the "customgpt-" prefix.
///
/// Examples:
/// "goose-claude-fable-5" → "claude-fable-5"
/// "databricks-gpt-5.5" → "gpt-5.5"
/// "team-x-claude-opus-4-7" → "claude-opus-4-7"
/// "claude-opus-4-7" → "claude-opus-4-7" (no prefix)
/// "goose-claude-fable-5" → "claude-fable-5" (boundary: preceded by "-")
/// "databricks-gpt-5.5" → "gpt-5.5" (boundary: preceded by "-")
/// "team-x-claude-opus-4-7" → "claude-opus-4-7" (boundary: preceded by "-")
/// "claude-opus-4-7" → "claude-opus-4-7" (no prefix, already boundary)
/// "customgpt-5-5-ep" → "customgpt-5-5-ep" (no boundary match for "gpt-")
/// "llama-3" → "llama-3" (no family token)
pub fn strip_catalog_prefix(model: &str) -> &str {
const FAMILY_TOKENS: &[&str] = &[${manifest.family_tokens.map((t) => `"${t}"`).join(", ")}];
let lower = model.to_ascii_lowercase();
let first_idx = FAMILY_TOKENS
.iter()
.filter_map(|tok| lower.find(tok))
.min();
let mut first_idx: Option<usize> = None;
for tok in FAMILY_TOKENS {
let tok_bytes = tok.as_bytes();
let lower_bytes = lower.as_bytes();
let mut start = 0usize;
loop {
match lower_bytes[start..].windows(tok_bytes.len()).position(|w| w == tok_bytes) {
None => break,
Some(rel) => {
let idx = start + rel;
// Boundary check: position 0 or preceded by a non-alphanumeric byte
let at_boundary = idx == 0 || {
let prev = lower_bytes[idx - 1];
!prev.is_ascii_alphanumeric()
};
if at_boundary {
first_idx = Some(first_idx.map_or(idx, |f| f.min(idx)));
break;
}
start = idx + 1;
}
}
}
}
match first_idx {
Some(idx) => &model[idx..],
None => model,
@@ -1082,6 +1169,8 @@ function buildRustMatchExpr(rule, provider) {
)
.join(" || ");
}
case "gpt-version-segment":
return allTokens.map((t) => `gpt_version_segment_matches_rs(lower, "${t.toLowerCase()}")`).join(" || ");
default:
throw new Error(`unknown match_kind: ${rule.match_kind}`);
}
@@ -1157,8 +1246,7 @@ fn gpt5_base_matches_rs(model: &str, token: &str) -> bool {
let dash_rest = &suffix[1..];
// Reject -<1-3 digits> followed by non-alphanumeric or end (mirrors TS /^\\d{1,3}(?:[^a-z\\d]|$)/i).
let first_non_digit = dash_rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(dash_rest.len());
let is_short_version = first_non_digit >= 1
&& first_non_digit <= 3
let is_short_version = (1..=3).contains(&first_non_digit)
&& (first_non_digit == dash_rest.len()
|| !dash_rest.as_bytes()[first_non_digit].is_ascii_alphanumeric());
if is_short_version {
@@ -1171,6 +1259,32 @@ fn gpt5_base_matches_rs(model: &str, token: &str) -> bool {
}
}
// ---------------------------------------------------------------------------
// gpt-version-segment helper (used by generated family resolver)
// ---------------------------------------------------------------------------
/// gpt-version-segment match: token is an exact segment AND (token itself starts with a digit,
/// OR the next segment after it starts with a digit). Prevents "gpt-neox-20b" from matching
/// "gpt" because "neox" starts with a letter, while "gpt-5.5" matches because next seg "5" is digit.
fn gpt_version_segment_matches_rs(model: &str, token: &str) -> bool {
let segs: Vec<&str> = model.split(|c: char| !c.is_ascii_alphanumeric()).collect();
for (i, seg) in segs.iter().enumerate() {
if *seg == token {
// Dashless numeric form (e.g. "gpt5"): token length > 3 or starts with digit
if token.len() > 3 || token.as_bytes().first().is_some_and(|b| b.is_ascii_digit()) {
return true;
}
// For short alpha tokens like "gpt": require the next segment to start with a digit
if let Some(next_seg) = segs.get(i + 1) {
if next_seg.as_bytes().first().is_some_and(|b| b.is_ascii_digit()) {
return true;
}
}
}
}
false
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
@@ -1286,6 +1400,8 @@ function buildTsMatchExpr(rule, provider) {
`lower.split(/[^a-z0-9]+/).some(s => s.startsWith("${t.toLowerCase()}"))`,
)
.join(" || ");
case "gpt-version-segment":
return allTokens.map((t) => `gptVersionSegmentMatchesGenerated(lower, "${t.toLowerCase()}")`).join(" || ");
default:
throw new Error(`unknown match_kind: ${rule.match_kind}`);
}
@@ -1298,10 +1414,10 @@ const tsProviderFallbacks = providerFallbackKeys
const concClean = { ...fb.concrete_unknown, registry_label: null };
delete blankClean._provenance;
delete concClean._provenance;
return ` "${provider}": {
return ` ["${provider}", {
blank: ${emitTsCapabilityResult(blankClean, " ")},
concreteUnknown: ${emitTsCapabilityResult(concClean, " ")},
},`;
}],`;
})
.join("\n");
@@ -1411,6 +1527,25 @@ function gpt5BaseMatchesGenerated(m: string, token: string): boolean {
}
}
/**
* gpt-version-segment match: token is an exact segment AND (token itself starts with a digit,
* OR the next segment after it starts with a digit). Prevents "gpt-neox-20b" from matching
* "gpt" because "neox" starts with a letter, while "gpt-5.5" matches because next seg "5" is digit.
*/
function gptVersionSegmentMatchesGenerated(m: string, token: string): boolean {
const segs = m.split(/[^a-z0-9]+/);
for (let i = 0; i < segs.length; i++) {
if (segs[i] === token) {
// Dashless numeric form (e.g. "gpt5") or token itself starts with digit
if (token.length > 3 || /^\\d/.test(token)) return true;
// For "gpt": require the next segment to start with a digit
const nextSeg = segs[i + 1];
if (nextSeg !== undefined && /^\\d/.test(nextSeg)) return true;
}
}
return false;
}
// ---------------------------------------------------------------------------
// Exact records — provider-qualified, pre-prefix-stripping
// ---------------------------------------------------------------------------
@@ -1428,9 +1563,9 @@ ${tsExactEntries
// Provider fallbacks
// ---------------------------------------------------------------------------
const PROVIDER_FALLBACKS: Record<string, { blank: CapabilityResult; concreteUnknown: CapabilityResult }> = {
const PROVIDER_FALLBACKS = new Map<string, { blank: CapabilityResult; concreteUnknown: CapabilityResult }>([
${tsProviderFallbacks}
};
]);
const DEFAULT_FALLBACK = {
blank: ${emitTsCapabilityResult(defaultFbBlank, " ")},
@@ -1438,15 +1573,25 @@ const DEFAULT_FALLBACK = {
};
// ---------------------------------------------------------------------------
// Strip catalog prefix — finds first family token occurrence
// Strip catalog prefix — boundary-aware, finds first boundary-aligned family token
// ---------------------------------------------------------------------------
export function stripCatalogPrefix(model: string): string {
const FAMILY_TOKENS = [${manifest.family_tokens.map((t) => `"${t}"`).join(", ")}] as const;
const lower = model.toLowerCase();
let firstIdx = Infinity;
for (const tok of FAMILY_TOKENS) {
const idx = model.toLowerCase().indexOf(tok);
if (idx !== -1 && idx < firstIdx) firstIdx = idx;
let start = 0;
while (true) {
const idx = lower.indexOf(tok, start);
if (idx === -1) break;
// Boundary check: position 0 or preceded by a non-alphanumeric character
if (idx === 0 || !/[a-z0-9]/.test(lower[idx - 1])) {
if (idx < firstIdx) firstIdx = idx;
break;
}
start = idx + 1;
}
}
return firstIdx === Infinity ? model : model.slice(firstIdx);
}
@@ -1492,7 +1637,7 @@ export function resolveModelCapabilities(
// Step 3: provider fallback
const isBlank = rawModelId.trim() === "";
const fb = PROVIDER_FALLBACKS[provider] ?? DEFAULT_FALLBACK;
const fb = PROVIDER_FALLBACKS.get(provider) ?? DEFAULT_FALLBACK;
return isBlank ? { ...fb.blank, registryLabel: null } : { ...fb.concreteUnknown, registryLabel: null };
}
`;
scripts/model-capabilities.json
+2 -2
View File
@@ -436,8 +436,8 @@
},
{
"id": "dbv2-gpt-code-names-segment",
"_comment": "DBv2-only rule: endpoint names with exact GPT segment route via OpenAI Responses. Handles segment-exact matches of \"gpt\" or \"gpt5\" (e.g. databricks-gpt-5.5 \u2192 segments include \"gpt\"). Priority > dbv2-claude (6 vs 5) restores old-contract: OpenAI checked before Claude for dual-marker names. segment match (not segment-prefix) prevents gptoss/gptj false-positives. Note: gpt-neox is a residual collision — ‘gpt-neox’ segments to [‘gpt’,‘neox’], so ‘gpt’ matches and it routes openai-responses. This is pinned as known behavior in the normative corpus, not an endorsement.",
"match_kind": "segment",
"_comment": "DBv2-only rule: models whose normalized alias has 'gpt' as a segment followed by a numeric segment (e.g. databricks-gpt-5.5 \u2192 'gpt' seg + '5' seg) route via OpenAI Responses. 'gpt5' dashless alias catches gpt5-custom forms. match_kind=gpt-version-segment: token must be an exact segment AND the next segment must start with a digit \u2014 prevents gptoss/gptj (no 'gpt' segment) AND gpt-neox-20b ('gpt' segment but next segment 'neox' is not numeric). Priority 6 > dbv2-claude's 5.",
"match_kind": "gpt-version-segment",
"match_value": "gpt",
"providers": [
"databricks_v2"
scripts/normative-corpus.json
+635 -110
View File
File diff suppressed because it is too large Load Diff
scripts/run-corpus.mjs
+18
View File
@@ -48,11 +48,29 @@ function canonicalizeProvider(provider) {
let passed = 0;
let failed = 0;
const REQUIRED_AXES = [
"thinking_mode",
"supported_efforts",
"default_effort",
"databricks_v2_wire_route",
"normalization_policy",
"registry_label",
];
for (const entry of corpus) {
// Skip group header entries
if (entry._group) continue;
if (!entry.expect) continue;
// Require all 6 axes on every executable vector — sparse vectors hide divergences.
const missingAxes = REQUIRED_AXES.filter((ax) => !(ax in entry.expect));
if (missingAxes.length > 0) {
failed++;
console.error(` FAIL ${entry.id}: sparse vector — missing axes: ${missingAxes.join(", ")}`);
console.error(` All six axes are required: ${REQUIRED_AXES.join(", ")}`);
continue;
}
// resolveModelCapabilities returns camelCase keys (registryLabel, thinkingMode, etc.)
const result = resolveModelCapabilities(canonicalizeProvider(entry.provider), entry.raw_model_id);
const expect = entry.expect;
scripts/test-manifest-validator.mjs
+125
View File
@@ -538,4 +538,129 @@ test("schema-negative: exact_record invalid normalization_policy is rejected", (
);
});
// ---------------------------------------------------------------------------
// Rule: family_rule match_priority must be a non-negative integer
// ---------------------------------------------------------------------------
test("schema-negative: family_rule match_priority string (injection vector) is rejected", () => {
assertRejects(
"family_rule string match_priority",
mutate((m) => {
// Inject a string that contains a compile_error! macro — must be rejected before emission
m.family_rules[0].match_priority = 'compile_error!("THUFIR_INJECTED")';
}),
"match_priority",
);
});
test("schema-negative: family_rule match_priority negative integer is rejected", () => {
assertRejects(
"family_rule negative match_priority",
mutate((m) => {
m.family_rules[0].match_priority = -1;
}),
"match_priority",
);
});
test("schema-negative: family_rule match_priority float is rejected", () => {
assertRejects(
"family_rule float match_priority",
mutate((m) => {
m.family_rules[0].match_priority = 1.5;
}),
"match_priority",
);
});
// ---------------------------------------------------------------------------
// Rule: exact_record uppercase duplicate key is rejected (after lowercasing)
// ---------------------------------------------------------------------------
test("schema-negative: exact_record uppercase duplicate key is rejected", () => {
assertRejects(
"exact_record uppercase duplicate key",
mutate((m) => {
// Add an uppercase copy of an existing exact record key
const existing = m.exact_records[0];
m.exact_records.push({
...existing,
provider: existing.provider.toUpperCase(),
raw_model_id: existing.raw_model_id.toUpperCase(),
});
}),
"duplicate exact_record key",
);
});
// ---------------------------------------------------------------------------
// Rule: exact_record inherited default_effort not in inherited supported_efforts
// ---------------------------------------------------------------------------
test("schema-negative: exact_record inherited default_effort outside materialized supported_efforts is rejected", () => {
assertRejects(
"exact_record inherited default out of materialized efforts",
mutate((m) => {
// Override supported_efforts_override to a single value that excludes the family default.
// For any exact record that inherits family default_effort, override efforts to exclude it.
const rec = m.exact_records.find((r) => r.raw_model_id === "databricks-gpt-5-4-mini");
// Family default for the gpt5-4 rule is "none". Override to only ["low"] to force mismatch.
rec.supported_efforts_override = ["low"];
// No explicit default_effort — inherits "none" from family, but "none" is not in ["low"]
}),
"materialized default_effort",
);
});
// ---------------------------------------------------------------------------
// Rule: family_rule supported_efforts must have no duplicates
// ---------------------------------------------------------------------------
test("schema-negative: family_rule supported_efforts with duplicate is rejected", () => {
assertRejects(
"family_rule duplicate effort",
mutate((m) => {
m.family_rules[0].supported_efforts = ["low", "low", "medium"];
}),
"duplicate effort",
);
});
// ---------------------------------------------------------------------------
// Rule: family_rule supported_efforts must follow canonical order
// ---------------------------------------------------------------------------
test("schema-negative: family_rule supported_efforts out of canonical order is rejected", () => {
assertRejects(
"family_rule efforts out of order",
mutate((m) => {
m.family_rules[0].supported_efforts = ["high", "low", "medium"];
}),
"canonical order",
);
});
// ---------------------------------------------------------------------------
// Rule: provider_fallback supported_efforts must have no duplicates
// ---------------------------------------------------------------------------
test("schema-negative: provider_fallback supported_efforts with duplicate is rejected", () => {
assertRejects(
"provider_fallback duplicate effort",
mutate((m) => {
const provider = Object.keys(m.provider_fallbacks)[0];
m.provider_fallbacks[provider].blank.supported_efforts = ["low", "low", "medium"];
}),
"duplicate effort",
);
});
// ---------------------------------------------------------------------------
// Rule: provider_fallback supported_efforts must follow canonical order
// ---------------------------------------------------------------------------
test("schema-negative: provider_fallback supported_efforts out of canonical order is rejected", () => {
assertRejects(
"provider_fallback efforts out of order",
mutate((m) => {
const provider = Object.keys(m.provider_fallbacks)[0];
m.provider_fallbacks[provider].blank.supported_efforts = ["high", "low", "medium"];
}),
"canonical order",
);
});
console.log("\nSchema-negative validator tests complete.");