diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8f077806..936767483 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,21 +149,15 @@ jobs: node-version: '22' package-manager-cache: false - - name: Regenerate artifacts - run: node scripts/generate-model-capabilities.mjs - - - name: Diff check — fail if generated files are stale + - name: Check generated artifacts are up to date run: | - if ! git diff --exit-code \ - crates/buzz-agent/src/generated_model_capabilities.rs \ - desktop/src/features/agents/ui/modelCapabilities.ts; then + if ! node scripts/generate-model-capabilities.mjs --check; then echo "" echo "ERROR: Generated model-capability files are stale." echo "Run: node scripts/generate-model-capabilities.mjs" echo "Then commit the regenerated files." exit 1 fi - echo "✓ All generated files are up to date." - name: Run corpus (TS interpreter via --experimental-strip-types) run: node --experimental-strip-types scripts/run-corpus.mjs diff --git a/crates/buzz-agent/src/generated_model_capabilities.rs b/crates/buzz-agent/src/generated_model_capabilities.rs index 219c22e35..962f8253f 100644 --- a/crates/buzz-agent/src/generated_model_capabilities.rs +++ b/crates/buzz-agent/src/generated_model_capabilities.rs @@ -1402,3 +1402,231 @@ fn gpt5_base_matches_rs(model: &str, token: &str) -> bool { #[cfg(test)] #[path = "generated_model_capabilities_tests.rs"] mod tests; + +// --------------------------------------------------------------------------- +// Pricing axis — exact (authority, model) lookup +// --------------------------------------------------------------------------- + +/// Per-model token pricing in USD per million tokens. +/// All fields are USD / 1,000,000 tokens. +/// +/// `None` fields mean the provider has not published a price for that token +/// category — never treat them as zero. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ModelPricing { + pub input_usd_per_mtok: f64, + pub output_usd_per_mtok: f64, + /// Cache-read price, if published. + pub cache_read_usd_per_mtok: Option, + /// Cache-write (cache creation) price, if published. + /// Scoped to the default ephemeral cache class. + pub cache_write_usd_per_mtok: Option, +} + +/// Look up pricing for an exact (billing authority, model) pair. +/// +/// * `authority` — the registered billing-authority token, e.g. +/// `"api.anthropic.com"`, `"api.openai.com"`. This is NOT the +/// runtime transport `Provider` enum; it is the proven billing namespace. +/// Match is exact — the caller must supply the exact registered token. +/// * `model` — the exact model string returned by the provider API +/// in its response body. Match is exact — no normalization applied. +/// +/// Returns `None` when the (authority, model) pair has no pricing record +/// — callers MUST treat `None` as "price unknown", never as "price zero". +/// Unknown authorities and unknown models both return `None`; there is no +/// family, prefix, or case-folding fallback. +pub fn lookup_pricing(authority: &str, model: &str) -> Option { + match authority { + // authority: api.anthropic.com + "api.anthropic.com" => match model { + // (api.anthropic.com, claude-fable-5) + "claude-fable-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 10.0, + output_usd_per_mtok: 50.0, + cache_read_usd_per_mtok: Some(1.0_f64), + cache_write_usd_per_mtok: Some(12.5_f64), + }) + } + // (api.anthropic.com, claude-sonnet-5) + "claude-sonnet-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 2.0, + output_usd_per_mtok: 10.0, + cache_read_usd_per_mtok: Some(0.2_f64), + cache_write_usd_per_mtok: Some(2.5_f64), + }) + } + // (api.anthropic.com, claude-opus-5) + "claude-opus-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 25.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.anthropic.com, claude-opus-4-8) + "claude-opus-4-8" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 25.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.anthropic.com, claude-opus-4-7) + "claude-opus-4-7" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 25.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.anthropic.com, claude-opus-4-6) + "claude-opus-4-6" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 25.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.anthropic.com, claude-opus-4-5) + "claude-opus-4-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 25.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.anthropic.com, claude-sonnet-4-6) + "claude-sonnet-4-6" => { + return Some(ModelPricing { + input_usd_per_mtok: 3.0, + output_usd_per_mtok: 15.0, + cache_read_usd_per_mtok: Some(0.3_f64), + cache_write_usd_per_mtok: Some(3.75_f64), + }) + } + // (api.anthropic.com, claude-sonnet-4-5) + "claude-sonnet-4-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 3.0, + output_usd_per_mtok: 15.0, + cache_read_usd_per_mtok: Some(0.3_f64), + cache_write_usd_per_mtok: Some(3.75_f64), + }) + } + // (api.anthropic.com, claude-haiku-4-5) + "claude-haiku-4-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 1.0, + output_usd_per_mtok: 5.0, + cache_read_usd_per_mtok: Some(0.1_f64), + cache_write_usd_per_mtok: Some(1.25_f64), + }) + } + // (api.anthropic.com, claude-opus-4-1) + "claude-opus-4-1" => { + return Some(ModelPricing { + input_usd_per_mtok: 15.0, + output_usd_per_mtok: 75.0, + cache_read_usd_per_mtok: Some(1.5_f64), + cache_write_usd_per_mtok: Some(18.75_f64), + }) + } + _ => {} + }, + // authority: api.openai.com + "api.openai.com" => match model { + // (api.openai.com, gpt-5-pro) + "gpt-5-pro" => { + return Some(ModelPricing { + input_usd_per_mtok: 15.0, + output_usd_per_mtok: 120.0, + cache_read_usd_per_mtok: None, + cache_write_usd_per_mtok: None, + }) + } + // (api.openai.com, gpt-5.6) + "gpt-5.6" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 30.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.openai.com, gpt-5.6-sol) + "gpt-5.6-sol" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 30.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: Some(6.25_f64), + }) + } + // (api.openai.com, gpt-5.6-luna) + "gpt-5.6-luna" => { + return Some(ModelPricing { + input_usd_per_mtok: 0.2, + output_usd_per_mtok: 1.2, + cache_read_usd_per_mtok: Some(0.02_f64), + cache_write_usd_per_mtok: Some(0.25_f64), + }) + } + // (api.openai.com, gpt-5.6-terra) + "gpt-5.6-terra" => { + return Some(ModelPricing { + input_usd_per_mtok: 2.0, + output_usd_per_mtok: 12.0, + cache_read_usd_per_mtok: Some(0.2_f64), + cache_write_usd_per_mtok: Some(2.5_f64), + }) + } + // (api.openai.com, gpt-5.5) + "gpt-5.5" => { + return Some(ModelPricing { + input_usd_per_mtok: 5.0, + output_usd_per_mtok: 30.0, + cache_read_usd_per_mtok: Some(0.5_f64), + cache_write_usd_per_mtok: None, + }) + } + // (api.openai.com, gpt-5.4) + "gpt-5.4" => { + return Some(ModelPricing { + input_usd_per_mtok: 2.5, + output_usd_per_mtok: 15.0, + cache_read_usd_per_mtok: Some(0.25_f64), + cache_write_usd_per_mtok: None, + }) + } + // (api.openai.com, gpt-5.1) + "gpt-5.1" => { + return Some(ModelPricing { + input_usd_per_mtok: 1.25, + output_usd_per_mtok: 10.0, + cache_read_usd_per_mtok: Some(0.125_f64), + cache_write_usd_per_mtok: None, + }) + } + // (api.openai.com, gpt-5) + "gpt-5" => { + return Some(ModelPricing { + input_usd_per_mtok: 1.25, + output_usd_per_mtok: 10.0, + cache_read_usd_per_mtok: Some(0.125_f64), + cache_write_usd_per_mtok: None, + }) + } + _ => {} + }, + _ => {} + } + None +} diff --git a/crates/buzz-agent/src/generated_model_capabilities_tests.rs b/crates/buzz-agent/src/generated_model_capabilities_tests.rs index 4f956af8d..4a22e7888 100644 --- a/crates/buzz-agent/src/generated_model_capabilities_tests.rs +++ b/crates/buzz-agent/src/generated_model_capabilities_tests.rs @@ -747,3 +747,246 @@ mod corpus_tests { } } } + +// --------------------------------------------------------------------------- +// Pricing lookup tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod pricing_tests { + use crate::generated_model_capabilities::lookup_pricing; + + // ----------------------------------------------------------------------- + // Anthropic models — authority: api.anthropic.com + // ----------------------------------------------------------------------- + + #[test] + fn test_lookup_pricing_anthropic_claude_fable5_returns_correct_rates() { + let p = lookup_pricing("api.anthropic.com", "claude-fable-5") + .expect("(api.anthropic.com, claude-fable-5) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 10.0); + assert_eq!(p.output_usd_per_mtok, 50.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(1.0)); + assert_eq!(p.cache_write_usd_per_mtok, Some(12.5)); + } + + #[test] + fn test_lookup_pricing_anthropic_claude_opus5_returns_correct_rates() { + let p = lookup_pricing("api.anthropic.com", "claude-opus-5") + .expect("(api.anthropic.com, claude-opus-5) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 5.0); + assert_eq!(p.output_usd_per_mtok, 25.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(0.5)); + assert_eq!(p.cache_write_usd_per_mtok, Some(6.25)); + } + + #[test] + fn test_lookup_pricing_anthropic_claude_sonnet5_returns_correct_rates() { + let p = lookup_pricing("api.anthropic.com", "claude-sonnet-5") + .expect("(api.anthropic.com, claude-sonnet-5) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 2.0); + assert_eq!(p.output_usd_per_mtok, 10.0); + } + + #[test] + fn test_lookup_pricing_anthropic_claude_opus48_exact_match() { + // Exact match on the model string as returned by the Anthropic API + let p = lookup_pricing("api.anthropic.com", "claude-opus-4-8") + .expect("(api.anthropic.com, claude-opus-4-8) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 5.0); + assert_eq!(p.output_usd_per_mtok, 25.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(0.5)); + assert_eq!(p.cache_write_usd_per_mtok, Some(6.25)); + } + + #[test] + fn test_lookup_pricing_anthropic_claude_sonnet46_returns_correct_rates() { + let p = lookup_pricing("api.anthropic.com", "claude-sonnet-4-6") + .expect("(api.anthropic.com, claude-sonnet-4-6) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 3.0); + assert_eq!(p.output_usd_per_mtok, 15.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(0.3)); + assert_eq!(p.cache_write_usd_per_mtok, Some(3.75)); + } + + #[test] + fn test_lookup_pricing_anthropic_claude_haiku45_returns_correct_rates() { + let p = lookup_pricing("api.anthropic.com", "claude-haiku-4-5") + .expect("(api.anthropic.com, claude-haiku-4-5) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 1.0); + assert_eq!(p.output_usd_per_mtok, 5.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(0.1)); + assert_eq!(p.cache_write_usd_per_mtok, Some(1.25)); + } + + // ----------------------------------------------------------------------- + // OpenAI models — authority: api.openai.com + // ----------------------------------------------------------------------- + + #[test] + fn test_lookup_pricing_openai_gpt56_returns_correct_rates() { + let p = lookup_pricing("api.openai.com", "gpt-5.6") + .expect("(api.openai.com, gpt-5.6) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 5.0); + assert_eq!(p.output_usd_per_mtok, 30.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(0.5)); + assert_eq!(p.cache_write_usd_per_mtok, Some(6.25)); + } + + #[test] + fn test_lookup_pricing_openai_gpt56_sol_returns_correct_rates() { + // gpt-5.6-sol is a separate exact record at the same price tier as gpt-5.6 + let p = lookup_pricing("api.openai.com", "gpt-5.6-sol") + .expect("(api.openai.com, gpt-5.6-sol) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 5.0); + assert_eq!(p.output_usd_per_mtok, 30.0); + } + + #[test] + fn test_lookup_pricing_openai_gpt56_luna_different_tier() { + // gpt-5.6-luna is a lower-cost tier within the gpt-5.6 family + let p = lookup_pricing("api.openai.com", "gpt-5.6-luna") + .expect("(api.openai.com, gpt-5.6-luna) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 0.2); + assert_eq!(p.output_usd_per_mtok, 1.2); + } + + #[test] + fn test_lookup_pricing_openai_gpt55_cache_write_is_none() { + // gpt-5.5 has cache_read but no cache_write published + let p = lookup_pricing("api.openai.com", "gpt-5.5") + .expect("(api.openai.com, gpt-5.5) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 5.0); + assert_eq!(p.output_usd_per_mtok, 30.0); + assert_eq!(p.cache_read_usd_per_mtok, Some(0.5)); + assert_eq!(p.cache_write_usd_per_mtok, None); + } + + #[test] + fn test_lookup_pricing_openai_gpt5_pro_no_cache_fields() { + let p = lookup_pricing("api.openai.com", "gpt-5-pro") + .expect("(api.openai.com, gpt-5-pro) must have pricing"); + assert_eq!(p.input_usd_per_mtok, 15.0); + assert_eq!(p.output_usd_per_mtok, 120.0); + assert_eq!(p.cache_read_usd_per_mtok, None); + assert_eq!(p.cache_write_usd_per_mtok, None); + } + + // ----------------------------------------------------------------------- + // Null guards — unknown authority, unknown model, wrong authority, + // empty strings, custom base URL (not an allowlisted billing authority) + // ----------------------------------------------------------------------- + + #[test] + fn test_lookup_pricing_unknown_authority_returns_none() { + // Custom base URL — not in the billing-authority allowlist + assert!( + lookup_pricing("custom.openai-compat.example.com", "gpt-5.6").is_none(), + "custom/unknown authority must return None" + ); + } + + #[test] + fn test_lookup_pricing_official_authority_custom_base_url_same_model_returns_none() { + // Same model string but via a gateway/custom endpoint — no billing proof + assert!( + lookup_pricing("gateway.internal.example.com", "claude-sonnet-4-6").is_none(), + "unknown authority with known model string must still return None" + ); + } + + #[test] + fn test_lookup_pricing_unknown_model_on_known_authority_returns_none() { + assert!( + lookup_pricing("api.anthropic.com", "claude-ultra-9000").is_none(), + "unknown model must return None, never Some(0)" + ); + } + + #[test] + fn test_lookup_pricing_empty_authority_returns_none() { + assert!( + lookup_pricing("", "claude-fable-5").is_none(), + "empty authority must return None" + ); + } + + #[test] + fn test_lookup_pricing_empty_model_returns_none() { + assert!( + lookup_pricing("api.anthropic.com", "").is_none(), + "empty model must return None" + ); + } + + #[test] + fn test_lookup_pricing_both_empty_returns_none() { + assert!( + lookup_pricing("", "").is_none(), + "both empty must return None" + ); + } + + #[test] + fn test_lookup_pricing_wrong_authority_for_known_model_returns_none() { + // claude-sonnet-4-6 is an Anthropic model; looking it up under openai authority returns None + assert!( + lookup_pricing("api.openai.com", "claude-sonnet-4-6").is_none(), + "Anthropic model under OpenAI authority must return None" + ); + } + + // ----------------------------------------------------------------------- + // Exact-match guards — non-canonical casing returns None + // ----------------------------------------------------------------------- + + #[test] + fn test_lookup_pricing_uppercase_authority_returns_none() { + // Authority must be the exact registered lowercase token; uppercase is not canonicalized + assert!( + lookup_pricing("API.ANTHROPIC.COM", "claude-sonnet-4-6").is_none(), + "uppercase authority must return None — no case folding" + ); + } + + #[test] + fn test_lookup_pricing_mixed_case_authority_returns_none() { + assert!( + lookup_pricing("Api.Openai.Com", "gpt-5.6").is_none(), + "mixed-case authority must return None — no case folding" + ); + } + + #[test] + fn test_lookup_pricing_mixed_case_model_returns_none() { + // Model must match exactly as the provider API returns it + assert!( + lookup_pricing("api.anthropic.com", "Claude-Sonnet-4-6").is_none(), + "mixed-case model must return None — no normalization" + ); + } + + // ----------------------------------------------------------------------- + // Databricks guard — Databricks routes are never priced by manifest + // ----------------------------------------------------------------------- + + #[test] + fn test_lookup_pricing_databricks_authority_returns_none() { + // Databricks is a corporate-internal route; its workspace URL is not + // a registered billing-authority token and must always return None. + assert!( + lookup_pricing("adb-1234567890.azuredatabricks.net", "claude-sonnet-4-6").is_none(), + "Databricks workspace URL must return None" + ); + } + + #[test] + fn test_lookup_pricing_databricks_route_returns_none() { + // Even if someone passed a known Databricks model string with an + // unknown authority, it must return None. + assert!( + lookup_pricing("databricks", "databricks-claude-sonnet-5").is_none(), + "Databricks route authority must return None" + ); + } +} diff --git a/desktop/src/features/agents/ui/modelPricing.ts b/desktop/src/features/agents/ui/modelPricing.ts new file mode 100644 index 000000000..16df135f2 --- /dev/null +++ b/desktop/src/features/agents/ui/modelPricing.ts @@ -0,0 +1,172 @@ +// biome-ignore-all format: generated — do not edit by hand. +// Regenerate with: node scripts/generate-model-capabilities.mjs +// Source: scripts/model-capabilities.json +// +// Pricing axis: exact (authority, model) lookup for cost estimation. +// Consumers: agent-usage aggregation (Phase 4 of Usage v2 plan). + +// --------------------------------------------------------------------------- +// Pricing axis — exact (authority, model) lookup +// --------------------------------------------------------------------------- + +/** Per-model token pricing, USD per million tokens. null = price not published. */ +export type ModelPricing = { + /** Input (fresh, non-cache) token price, USD/MTok. */ + readonly inputUsdPerMtok: number; + /** Output token price, USD/MTok. */ + readonly outputUsdPerMtok: number; + /** Cache-read token price, USD/MTok. null when provider has not published it. */ + readonly cacheReadUsdPerMtok: number | null; + /** + * Cache-write (cache creation) token price, USD/MTok. null when not published. + * Scoped to the default ephemeral cache class. + */ + readonly cacheWriteUsdPerMtok: number | null; +}; + +/** + * Composite key for pricing lookup: `${authority}\0${model}` (exact strings, no transforms). + * + * authority — registered billing-authority token (e.g. "api.anthropic.com"). + * model — exact model string from the provider API response body. + */ +const PRICING_TABLE: Map = new Map([ + ["api.anthropic.com\u0000claude-fable-5", { + inputUsdPerMtok: 10, + outputUsdPerMtok: 50, + cacheReadUsdPerMtok: 1, + cacheWriteUsdPerMtok: 12.5, + }], + ["api.anthropic.com\u0000claude-sonnet-5", { + inputUsdPerMtok: 2, + outputUsdPerMtok: 10, + cacheReadUsdPerMtok: 0.2, + cacheWriteUsdPerMtok: 2.5, + }], + ["api.anthropic.com\u0000claude-opus-5", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 25, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.anthropic.com\u0000claude-opus-4-8", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 25, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.anthropic.com\u0000claude-opus-4-7", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 25, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.anthropic.com\u0000claude-opus-4-6", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 25, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.anthropic.com\u0000claude-opus-4-5", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 25, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.anthropic.com\u0000claude-sonnet-4-6", { + inputUsdPerMtok: 3, + outputUsdPerMtok: 15, + cacheReadUsdPerMtok: 0.3, + cacheWriteUsdPerMtok: 3.75, + }], + ["api.anthropic.com\u0000claude-sonnet-4-5", { + inputUsdPerMtok: 3, + outputUsdPerMtok: 15, + cacheReadUsdPerMtok: 0.3, + cacheWriteUsdPerMtok: 3.75, + }], + ["api.anthropic.com\u0000claude-haiku-4-5", { + inputUsdPerMtok: 1, + outputUsdPerMtok: 5, + cacheReadUsdPerMtok: 0.1, + cacheWriteUsdPerMtok: 1.25, + }], + ["api.anthropic.com\u0000claude-opus-4-1", { + inputUsdPerMtok: 15, + outputUsdPerMtok: 75, + cacheReadUsdPerMtok: 1.5, + cacheWriteUsdPerMtok: 18.75, + }], + ["api.openai.com\u0000gpt-5-pro", { + inputUsdPerMtok: 15, + outputUsdPerMtok: 120, + cacheReadUsdPerMtok: null, + cacheWriteUsdPerMtok: null, + }], + ["api.openai.com\u0000gpt-5.6", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 30, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.openai.com\u0000gpt-5.6-sol", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 30, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: 6.25, + }], + ["api.openai.com\u0000gpt-5.6-luna", { + inputUsdPerMtok: 0.2, + outputUsdPerMtok: 1.2, + cacheReadUsdPerMtok: 0.02, + cacheWriteUsdPerMtok: 0.25, + }], + ["api.openai.com\u0000gpt-5.6-terra", { + inputUsdPerMtok: 2, + outputUsdPerMtok: 12, + cacheReadUsdPerMtok: 0.2, + cacheWriteUsdPerMtok: 2.5, + }], + ["api.openai.com\u0000gpt-5.5", { + inputUsdPerMtok: 5, + outputUsdPerMtok: 30, + cacheReadUsdPerMtok: 0.5, + cacheWriteUsdPerMtok: null, + }], + ["api.openai.com\u0000gpt-5.4", { + inputUsdPerMtok: 2.5, + outputUsdPerMtok: 15, + cacheReadUsdPerMtok: 0.25, + cacheWriteUsdPerMtok: null, + }], + ["api.openai.com\u0000gpt-5.1", { + inputUsdPerMtok: 1.25, + outputUsdPerMtok: 10, + cacheReadUsdPerMtok: 0.125, + cacheWriteUsdPerMtok: null, + }], + ["api.openai.com\u0000gpt-5", { + inputUsdPerMtok: 1.25, + outputUsdPerMtok: 10, + cacheReadUsdPerMtok: 0.125, + cacheWriteUsdPerMtok: null, + }], +]); + +/** + * Look up pricing for an exact (billing authority, model) pair. + * + * Both arguments are matched exactly as supplied — no case normalization. + * The caller must supply the exact registered authority token and the exact + * provider-API model string. Returns `null` for unrecognised pairs — callers + * MUST treat null as "price unknown", never as zero cost. There is no family, + * prefix, or case-folding fallback. + * + * @param authority - Registered billing-authority token, e.g. "api.anthropic.com". + * This is NOT the runtime provider name; it is the proven billing namespace. + * @param model - Exact model string returned by the provider API response body. + */ +export function lookupModelPricing(authority: string, model: string): ModelPricing | null { + const key = `${authority}\0${model}`; + return PRICING_TABLE.get(key) ?? null; +} diff --git a/scripts/generate-model-capabilities.mjs b/scripts/generate-model-capabilities.mjs index 1ac36cba0..c992b6936 100644 --- a/scripts/generate-model-capabilities.mjs +++ b/scripts/generate-model-capabilities.mjs @@ -1397,44 +1397,337 @@ export function resolveModelCapabilities( // Write or check files // --------------------------------------------------------------------------- -const outputs = [ +// Outputs are written after the pricing axis is built below — see outputsWithPricing. +// (The outputs array is kept here as a reference; actual writing uses outputsWithPricing.) +const _capabilityOnlyOutputs = [finalRustContent, tsContent]; // referenced below +let checkFailed = false; // used by the pricing write loop below + +// --------------------------------------------------------------------------- +// Pricing axis — exact (authority, model) lookup, no inference +// --------------------------------------------------------------------------- +// +// Pricing is keyed on (billing-authority domain, exact model string) — both +// as returned by the provider API. No prefix matching, no normalization, +// no catalog-prefix stripping. "No applicable price" is a complete result. +// +// Databricks routes are corporate-internal and are not represented here. +// The publisher (P2 scope) must omit PricingIdentity for those routes. +// --------------------------------------------------------------------------- + +const pricingRecords = manifest.pricing_records ?? []; + +// --------------------------------------------------------------------------- +// Pricing-record validation — every entry must be structurally complete. +// No silent skips; any non-conforming entry is a hard error. +// --------------------------------------------------------------------------- + +/** Registered billing-namespace identifiers (closed set; extends only by NIP amendment). */ +const REGISTERED_AUTHORITIES = new Set([ + "api.anthropic.com", + "api.openai.com", + "openrouter.ai", +]); + +/** + * Return true if the string contains any character that is unsafe in a + * generated Rust string literal or a TS composite key: control characters + * (U+0000–U+001F, U+007F), double-quotes, backslashes, or NUL. + */ +function hasUnsafeKeyChars(s) { + return /[\x00-\x1f\x7f"\\]/.test(s); +} + +for (let i = 0; i < pricingRecords.length; i++) { + const rec = pricingRecords[i]; + const loc = `pricing_records[${i}]`; + + // Every entry must have authority and model — no comment-only entries allowed + // in the records array (top-level _comment_pricing is a separate key). + if (typeof rec.authority !== "string" || rec.authority.length === 0) { + throw new Error(`${loc}: missing or empty "authority" — every pricing record must have a registered authority`); + } + if (!REGISTERED_AUTHORITIES.has(rec.authority)) { + throw new Error( + `${loc}: unregistered authority "${rec.authority}". ` + + `Allowed values: ${[...REGISTERED_AUTHORITIES].join(", ")}. ` + + `The set extends only by NIP amendment.`, + ); + } + if (typeof rec.model !== "string" || rec.model.length === 0) { + throw new Error(`${loc} (${rec.authority}): missing or empty "model"`); + } + if (hasUnsafeKeyChars(rec.authority)) { + throw new Error(`${loc}: authority "${rec.authority}" contains unsafe characters (control chars, quotes, backslashes)`); + } + if (hasUnsafeKeyChars(rec.model)) { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): model contains unsafe characters (control chars, quotes, backslashes)`); + } + + // usd_per_mtok must be present with finite nonnegative input and output + const p = rec.usd_per_mtok; + if (!p || typeof p !== "object") { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): missing usd_per_mtok`); + } + if (typeof p.input !== "number" || !isFinite(p.input) || p.input < 0) { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): usd_per_mtok.input must be a finite nonnegative number`); + } + if (typeof p.output !== "number" || !isFinite(p.output) || p.output < 0) { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): usd_per_mtok.output must be a finite nonnegative number`); + } + for (const cacheField of ["cache_read", "cache_write"]) { + if (!Object.hasOwn(p, cacheField)) { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): usd_per_mtok.${cacheField} is required (use null for unknown)`); + } + const v = p[cacheField]; + if (v !== null) { + if (typeof v !== "number" || !isFinite(v) || v < 0) { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): usd_per_mtok.${cacheField} must be null or a finite nonnegative number`); + } + } + } + + // Provenance: _source must be present and non-empty + if (typeof rec._source !== "string" || rec._source.length === 0) { + throw new Error(`${loc} (${rec.authority}, ${rec.model}): missing "_source" provenance field`); + } +} + +// Duplicate detection on the exact emitted (authority, model) identity — no transforms applied. +const pricingKeysSeen = new Set(); +for (const rec of pricingRecords) { + const key = `${rec.authority}\0${rec.model}`; + if (pricingKeysSeen.has(key)) { + throw new Error(`duplicate pricing record for (${rec.authority}, ${rec.model})`); + } + pricingKeysSeen.add(key); +} + +// --------------------------------------------------------------------------- +// Rust pricing generation +// --------------------------------------------------------------------------- + +function rustOptionFloat(v) { + if (v === null || v === undefined) return "None"; + const f = Number(v); + const s = Number.isInteger(f) ? `${f}.0_f64` : `${f}_f64`; + return `Some(${s})`; +} + +function rustFloat(v) { + const f = Number(v); + return Number.isInteger(f) ? `${f}.0` : `${f}`; +} + +function emitRustPricingResult(p, indent = " ") { + const i = indent; + return [ + `${i}ModelPricing {`, + `${i} input_usd_per_mtok: ${rustFloat(p.input)},`, + `${i} output_usd_per_mtok: ${rustFloat(p.output)},`, + `${i} cache_read_usd_per_mtok: ${rustOptionFloat(p.cache_read)},`, + `${i} cache_write_usd_per_mtok: ${rustOptionFloat(p.cache_write)},`, + `${i}}`, + ].join("\n"); +} + +// Each record becomes one arm in a nested match: authority → model → prices +// We group records by authority first for a readable nested match. +/** @type {Map} */ +const byAuthority = new Map(); +for (const rec of pricingRecords) { + const auth = rec.authority; + if (!byAuthority.has(auth)) byAuthority.set(auth, []); + byAuthority.get(auth).push(rec); +} + +function buildRustPricingBody() { + const authArms = []; + for (const [auth, recs] of byAuthority) { + const modelArms = recs.map((rec) => { + const m = rec.model; + return ( + ` // (${auth}, ${m})\n` + + ` "${m}" => return Some(\n${emitRustPricingResult(rec.usd_per_mtok, " ")}\n ),` + ); + }); + authArms.push( + ` // authority: ${auth}\n` + + ` "${auth}" => match model {\n${modelArms.join("\n")}\n _ => {}\n },` + ); + } + return authArms.join("\n"); +} + +const rustPricingContent = ` +// --------------------------------------------------------------------------- +// Pricing axis — exact (authority, model) lookup +// --------------------------------------------------------------------------- + +/// Per-model token pricing in USD per million tokens. +/// All fields are USD / 1,000,000 tokens. +/// +/// \`None\` fields mean the provider has not published a price for that token +/// category — never treat them as zero. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ModelPricing { + pub input_usd_per_mtok: f64, + pub output_usd_per_mtok: f64, + /// Cache-read price, if published. + pub cache_read_usd_per_mtok: Option, + /// Cache-write (cache creation) price, if published. + /// Scoped to the default ephemeral cache class. + pub cache_write_usd_per_mtok: Option, +} + +/// Look up pricing for an exact (billing authority, model) pair. +/// +/// * \`authority\` — the registered billing-authority token, e.g. +/// \`"api.anthropic.com"\`, \`"api.openai.com"\`. This is NOT the +/// runtime transport \`Provider\` enum; it is the proven billing namespace. +/// Match is exact — the caller must supply the exact registered token. +/// * \`model\` — the exact model string returned by the provider API +/// in its response body. Match is exact — no normalization applied. +/// +/// Returns \`None\` when the (authority, model) pair has no pricing record +/// — callers MUST treat \`None\` as "price unknown", never as "price zero". +/// Unknown authorities and unknown models both return \`None\`; there is no +/// family, prefix, or case-folding fallback. +pub fn lookup_pricing(authority: &str, model: &str) -> Option { + match authority { +${buildRustPricingBody()} + _ => {} + } + None +} +`; + +const finalRustPricingContent = rustfmt(rustContent + rustGpt5Helpers + rustPricingContent); + +// --------------------------------------------------------------------------- +// TypeScript pricing generation +// --------------------------------------------------------------------------- + +function tsPricingField(v) { + if (v === null || v === undefined) return "null"; + return `${v}`; +} + +function emitTsPricingResult(p, indent = " ") { + const i = indent; + return [ + `{`, + `${i} inputUsdPerMtok: ${p.input},`, + `${i} outputUsdPerMtok: ${p.output},`, + `${i} cacheReadUsdPerMtok: ${tsPricingField(p.cache_read)},`, + `${i} cacheWriteUsdPerMtok: ${tsPricingField(p.cache_write)},`, + `${i}}`, + ].join(`\n${i}`); +} + +// Build TS pricing map entries: key is `${authority}\0${model}` (exact, no transforms) +const tsPricingMapEntries = pricingRecords + .map((rec) => { + const key = `${rec.authority}\0${rec.model}`; + return ` [${JSON.stringify(key)}, ${emitTsPricingResult(rec.usd_per_mtok, " ")}],`; + }) + .join("\n"); + +const tsPricingContent = ` +// --------------------------------------------------------------------------- +// Pricing axis — exact (authority, model) lookup +// --------------------------------------------------------------------------- + +/** Per-model token pricing, USD per million tokens. null = price not published. */ +export type ModelPricing = { + /** Input (fresh, non-cache) token price, USD/MTok. */ + readonly inputUsdPerMtok: number; + /** Output token price, USD/MTok. */ + readonly outputUsdPerMtok: number; + /** Cache-read token price, USD/MTok. null when provider has not published it. */ + readonly cacheReadUsdPerMtok: number | null; + /** + * Cache-write (cache creation) token price, USD/MTok. null when not published. + * Scoped to the default ephemeral cache class. + */ + readonly cacheWriteUsdPerMtok: number | null; +}; + +/** + * Composite key for pricing lookup: \`\${authority}\\0\${model}\` (exact strings, no transforms). + * + * authority — registered billing-authority token (e.g. "api.anthropic.com"). + * model — exact model string from the provider API response body. + */ +const PRICING_TABLE: Map = new Map([ +${tsPricingMapEntries} +]); + +/** + * Look up pricing for an exact (billing authority, model) pair. + * + * Both arguments are matched exactly as supplied — no case normalization. + * The caller must supply the exact registered authority token and the exact + * provider-API model string. Returns \`null\` for unrecognised pairs — callers + * MUST treat null as "price unknown", never as zero cost. There is no family, + * prefix, or case-folding fallback. + * + * @param authority - Registered billing-authority token, e.g. "api.anthropic.com". + * This is NOT the runtime provider name; it is the proven billing namespace. + * @param model - Exact model string returned by the provider API response body. + */ +export function lookupModelPricing(authority: string, model: string): ModelPricing | null { + const key = \`\${authority}\\0\${model}\`; + return PRICING_TABLE.get(key) ?? null; +} +`; + +// Standalone pricing TS file that does NOT import from modelCapabilities +// (no normalization or catalog-prefix logic needed) +const tsPricingFileContent = `// biome-ignore-all format: generated — do not edit by hand. +// Regenerate with: node scripts/generate-model-capabilities.mjs +// Source: scripts/model-capabilities.json +// +// Pricing axis: exact (authority, model) lookup for cost estimation. +// Consumers: agent-usage aggregation (Phase 4 of Usage v2 plan). +${tsPricingContent}`; + +const outputsWithPricing = [ { path: outputDirOverride ? join(outputDirOverride, "generated_model_capabilities.rs") : join(repoRoot, "crates", "buzz-agent", "src", "generated_model_capabilities.rs"), - content: finalRustContent, - label: "Rust", + content: finalRustPricingContent, + label: "Rust (with pricing)", }, { path: outputDirOverride ? join(outputDirOverride, "modelCapabilities.ts") - : join( - repoRoot, - "desktop", - "src", - "features", - "agents", - "ui", - "modelCapabilities.ts", - ), + : join(repoRoot, "desktop", "src", "features", "agents", "ui", "modelCapabilities.ts"), content: tsContent, label: "TypeScript", }, + { + path: outputDirOverride + ? join(outputDirOverride, "modelPricing.ts") + : join(repoRoot, "desktop", "src", "features", "agents", "ui", "modelPricing.ts"), + content: tsPricingFileContent, + label: "TypeScript Pricing", + }, ]; -let checkFailed = false; -for (const { path, content, label } of outputs) { +let checkFailedPricing = false; +for (const { path, content, label } of outputsWithPricing) { if (CHECK_MODE) { if (!existsSync(path)) { console.error(`CHECK FAILED: ${label} file does not exist: ${path}`); - checkFailed = true; + checkFailedPricing = true; continue; } const existing = readFileSync(path, "utf8"); if (existing !== content) { console.error(`CHECK FAILED: ${label} file is stale: ${path}`); console.error("Run: node scripts/generate-model-capabilities.mjs to regenerate."); - checkFailed = true; + checkFailedPricing = true; } else { console.log(`OK: ${label}`); } @@ -1444,9 +1737,9 @@ for (const { path, content, label } of outputs) { } } -if (CHECK_MODE && checkFailed) { +if ((CHECK_MODE && checkFailed) || (CHECK_MODE && checkFailedPricing)) { process.exit(1); } if (!CHECK_MODE) { - console.log("Done. Generated 3 files."); + console.log("Done. Generated 3 files (Rust capabilities + TS capabilities + TS pricing)."); } diff --git a/scripts/model-capabilities.json b/scripts/model-capabilities.json index baac0a5d7..6217b0668 100644 --- a/scripts/model-capabilities.json +++ b/scripts/model-capabilities.json @@ -7,7 +7,8 @@ "anthropic_thinking": "https://platform.claude.com/docs/en/build-with-claude/extended-thinking (July 2025)", "anthropic_effort": "https://platform.claude.com/docs/en/build-with-claude/effort (July 2025)", "openai_reasoning": "https://platform.openai.com/docs/guides/reasoning (July 2025)", - "goose_known_models": "goose revision 6789d4af (crates/goose-providers/src/databricks_v2.rs:41-42) \u2014 two IDs: databricks-gpt-5-5, databricks-claude-opus-4-7" + "goose_known_models": "goose revision 6789d4af (crates/goose-providers/src/databricks_v2.rs:41-42) — two IDs: databricks-gpt-5-5, databricks-claude-opus-4-7", + "models_dev_pricing": "https://models.dev/api.json (retrieved 2026-08-03, SHA-256 6fd293aec9fc4274cf2ff1d7bdbe180156b8dde9b80dc9cc0c7e72757931c494) — providers.anthropic and providers.openai cost fields, USD per million tokens" }, "family_tokens": [ "claude-", @@ -622,7 +623,7 @@ "medium", "high" ], - "source": "models.dev reasoning_options: low|medium|high (family rule adds none+xhigh \u2014 adopt provider-advertised)", + "source": "models.dev reasoning_options: low|medium|high (family rule adds none+xhigh — adopt provider-advertised)", "_reconciliation": "adopt", "_reconciliation_note": "models.dev advertises low|medium|high. Family rule (gpt5-4) adds none+xhigh. Provider-advertised wins per plan F1 policy.", "_reconciliation_doc": "https://models.dev/api.json (retrieved 2026-07-31, SHA-256 d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0): providers.databricks.models[\"databricks-gpt-5-4-mini\"].reasoning_options=[{\"type\":\"effort\",\"values\":[\"low\",\"medium\",\"high\"]}]" @@ -651,7 +652,7 @@ "high", "max" ], - "source": "models.dev reasoning_options: low|medium|high|max (family rule adds none+xhigh \u2014 provider-advertised wins per plan F1)", + "source": "models.dev reasoning_options: low|medium|high|max (family rule adds none+xhigh — provider-advertised wins per plan F1)", "_reconciliation": "adopt", "_reconciliation_note": "models.dev advertises [low, medium, high, max]. Family rule (gpt5-6) has none+xhigh+max; sol endpoint does not expose none or xhigh. Provider-advertised wins.", "_reconciliation_doc": "https://models.dev/api.json (retrieved 2026-07-31, SHA-256 d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0): providers.databricks.models[\"databricks-gpt-5-6-sol\"].reasoning_options=[{\"type\":\"effort\",\"values\":[\"low\",\"medium\",\"high\",\"max\"]}]" @@ -660,7 +661,7 @@ "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-5", "registry_label": "GPT-5.5", - "source": "models.dev reasoning_options: low|medium|high (family rule adds none+xhigh \u2014 provider-advertised wins per plan F1)", + "source": "models.dev reasoning_options: low|medium|high (family rule adds none+xhigh — provider-advertised wins per plan F1)", "_reconciliation": "adopt", "_reconciliation_note": "models.dev (pinned payload) advertises [low, medium, high]. Family rule (gpt5-5) has none+xhigh; this Databricks endpoint does not expose none or xhigh. Provider-advertised wins.", "supported_efforts_override": [ @@ -676,7 +677,7 @@ "registry_label": "Claude Opus 4.7", "source": "DATABRICKS_V2_KNOWN_MODELS; family rule anthropic-adaptive-xhigh-opus-4-7 applies", "_reconciliation": "no-effort-divergence", - "_reconciliation_note": "models.dev advertises reasoning_options=[{\"type\":\"budget_tokens\",\"min\":1024}]. This is a different capability axis (extended thinking token budget), not an effort-level selector. No effort divergence to reconcile \u2014 efforts for this model come from the anthropic family rule (anthropic-adaptive-xhigh-opus-4-7).", + "_reconciliation_note": "models.dev advertises reasoning_options=[{\"type\":\"budget_tokens\",\"min\":1024}]. This is a different capability axis (extended thinking token budget), not an effort-level selector. No effort divergence to reconcile — efforts for this model come from the anthropic family rule (anthropic-adaptive-xhigh-opus-4-7).", "_reconciliation_doc": "https://models.dev/api.json (retrieved 2026-07-31, SHA-256 d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0): providers.databricks.models[\"databricks-claude-opus-4-7\"].reasoning_options=[{\"type\":\"budget_tokens\",\"min\":1024}]" } ], @@ -864,5 +865,230 @@ "normalization_policy": "none" } } - } + }, + "_comment_pricing": "Pricing manifest: USD per million tokens (usd_per_mtok). Lookup is exact (authority, model) — no prefix inference, no normalization. authority is the canonical billing-authority domain; model is the exact string the provider API returns in its response body. null fields mean the price is not published — never treat as zero. Databricks routes are corporate-internal and are not represented here; the publisher must omit PricingIdentity for those routes. Source: models.dev/api.json (retrieved 2026-08-03, SHA-256 6fd293ae). Semantics: current list prices, labeled estimated at current prices (ccusage-aligned). cache_write prices are scoped to the default ephemeral cache class; other classes/TTLs are not priced here.", + "pricing_records": [ + { + "_comment": "Anthropic — authority: api.anthropic.com. model strings are the canonical IDs returned by the Anthropic API.", + "authority": "api.anthropic.com", + "model": "claude-fable-5", + "usd_per_mtok": { + "input": 10.0, + "output": 50.0, + "cache_read": 1.0, + "cache_write": 12.5 + }, + "_source": "models.dev anthropic/claude-fable-5" + }, + { + "authority": "api.anthropic.com", + "model": "claude-sonnet-5", + "usd_per_mtok": { + "input": 2.0, + "output": 10.0, + "cache_read": 0.2, + "cache_write": 2.5 + }, + "_source": "models.dev anthropic/claude-sonnet-5" + }, + { + "authority": "api.anthropic.com", + "model": "claude-opus-5", + "usd_per_mtok": { + "input": 5.0, + "output": 25.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev anthropic/claude-opus-5" + }, + { + "authority": "api.anthropic.com", + "model": "claude-opus-4-8", + "usd_per_mtok": { + "input": 5.0, + "output": 25.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev anthropic/claude-opus-4-8" + }, + { + "authority": "api.anthropic.com", + "model": "claude-opus-4-7", + "usd_per_mtok": { + "input": 5.0, + "output": 25.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev anthropic/claude-opus-4-7" + }, + { + "authority": "api.anthropic.com", + "model": "claude-opus-4-6", + "usd_per_mtok": { + "input": 5.0, + "output": 25.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev anthropic/claude-opus-4-6" + }, + { + "authority": "api.anthropic.com", + "model": "claude-opus-4-5", + "usd_per_mtok": { + "input": 5.0, + "output": 25.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev anthropic/claude-opus-4-5" + }, + { + "authority": "api.anthropic.com", + "model": "claude-sonnet-4-6", + "usd_per_mtok": { + "input": 3.0, + "output": 15.0, + "cache_read": 0.3, + "cache_write": 3.75 + }, + "_source": "models.dev anthropic/claude-sonnet-4-6" + }, + { + "authority": "api.anthropic.com", + "model": "claude-sonnet-4-5", + "usd_per_mtok": { + "input": 3.0, + "output": 15.0, + "cache_read": 0.3, + "cache_write": 3.75 + }, + "_source": "models.dev anthropic/claude-sonnet-4-5" + }, + { + "authority": "api.anthropic.com", + "model": "claude-haiku-4-5", + "usd_per_mtok": { + "input": 1.0, + "output": 5.0, + "cache_read": 0.1, + "cache_write": 1.25 + }, + "_source": "models.dev anthropic/claude-haiku-4-5" + }, + { + "authority": "api.anthropic.com", + "model": "claude-opus-4-1", + "usd_per_mtok": { + "input": 15.0, + "output": 75.0, + "cache_read": 1.5, + "cache_write": 18.75 + }, + "_source": "models.dev anthropic/claude-opus-4-1 (listed as claude-opus-4 legacy tier)" + }, + { + "_comment": "OpenAI — authority: api.openai.com. model strings are the canonical IDs returned by the OpenAI API.", + "authority": "api.openai.com", + "model": "gpt-5-pro", + "usd_per_mtok": { + "input": 15.0, + "output": 120.0, + "cache_read": null, + "cache_write": null + }, + "_source": "models.dev openai/gpt-5-pro" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.6", + "usd_per_mtok": { + "input": 5.0, + "output": 30.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev openai/gpt-5.6" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.6-sol", + "usd_per_mtok": { + "input": 5.0, + "output": 30.0, + "cache_read": 0.5, + "cache_write": 6.25 + }, + "_source": "models.dev openai/gpt-5.6-sol (same price tier as gpt-5.6)" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.6-luna", + "usd_per_mtok": { + "input": 0.2, + "output": 1.2, + "cache_read": 0.02, + "cache_write": 0.25 + }, + "_source": "models.dev openai/gpt-5.6-luna (different price tier from gpt-5.6-sol/base)" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.6-terra", + "usd_per_mtok": { + "input": 2.0, + "output": 12.0, + "cache_read": 0.2, + "cache_write": 2.5 + }, + "_source": "models.dev openai/gpt-5.6-terra" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.5", + "usd_per_mtok": { + "input": 5.0, + "output": 30.0, + "cache_read": 0.5, + "cache_write": null + }, + "_source": "models.dev openai/gpt-5.5" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.4", + "usd_per_mtok": { + "input": 2.5, + "output": 15.0, + "cache_read": 0.25, + "cache_write": null + }, + "_source": "models.dev openai/gpt-5.4" + }, + { + "authority": "api.openai.com", + "model": "gpt-5.1", + "usd_per_mtok": { + "input": 1.25, + "output": 10.0, + "cache_read": 0.125, + "cache_write": null + }, + "_source": "models.dev openai/gpt-5.1" + }, + { + "authority": "api.openai.com", + "model": "gpt-5", + "usd_per_mtok": { + "input": 1.25, + "output": 10.0, + "cache_read": 0.125, + "cache_write": null + }, + "_source": "models.dev openai/gpt-5" + } + ] } diff --git a/scripts/test-manifest-validator.mjs b/scripts/test-manifest-validator.mjs index cbe596a76..4aade7084 100644 --- a/scripts/test-manifest-validator.mjs +++ b/scripts/test-manifest-validator.mjs @@ -410,4 +410,261 @@ test("schema-negative: exact_record registry_label with unsafe chars is rejected ); }); +// =========================================================================== +// Pricing record validation rules +// =========================================================================== + +// Helper: insert a bad pricing record (replaces the first real record) +function mutatePricing(fn) { + return mutate((m) => { + const rec = JSON.parse(JSON.stringify(m.pricing_records[0])); + fn(rec, m); + m.pricing_records[0] = rec; + }); +} + +// --------------------------------------------------------------------------- +// Rule: pricing record missing authority +// --------------------------------------------------------------------------- +test("schema-negative: pricing record missing authority is rejected", () => { + assertRejects( + "pricing record missing authority", + mutate((m) => { + m.pricing_records.push({ + model: "some-model", + usd_per_mtok: { input: 1, output: 5, cache_read: null, cache_write: null }, + _source: "test", + }); + }), + "authority", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: pricing record with unregistered authority is rejected +// --------------------------------------------------------------------------- +test("schema-negative: pricing record with path-bearing authority is rejected", () => { + assertRejects( + "path-bearing authority", + mutatePricing((rec) => { + rec.authority = "api.openai.com/v1"; + }), + "unregistered authority", + ); +}); + +test("schema-negative: pricing record with scheme in authority is rejected", () => { + assertRejects( + "scheme in authority", + mutatePricing((rec) => { + rec.authority = "https://api.anthropic.com"; + }), + "unregistered authority", + ); +}); + +test("schema-negative: pricing record with custom/unknown authority is rejected", () => { + assertRejects( + "unknown authority", + mutatePricing((rec) => { + rec.authority = "custom.openai-compat.example.com"; + }), + "unregistered authority", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: pricing record with empty model is rejected +// --------------------------------------------------------------------------- +test("schema-negative: pricing record with empty model is rejected", () => { + assertRejects( + "empty model", + mutatePricing((rec) => { + rec.model = ""; + }), + "model", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: unsafe characters in key fields are rejected +// --------------------------------------------------------------------------- +test("schema-negative: pricing record model with double-quote is rejected", () => { + assertRejects( + "model with double-quote", + mutatePricing((rec) => { + rec.model = 'bad"model'; + }), + "unsafe", + ); +}); + +test("schema-negative: pricing record model with backslash is rejected", () => { + assertRejects( + "model with backslash", + mutatePricing((rec) => { + rec.model = "bad\\model"; + }), + "unsafe", + ); +}); + +test("schema-negative: pricing record model with NUL character is rejected", () => { + assertRejects( + "model with NUL", + mutatePricing((rec) => { + rec.model = "bad\x00model"; + }), + "unsafe", + ); +}); + +test("schema-negative: pricing record model with control character is rejected", () => { + assertRejects( + "model with control char", + mutatePricing((rec) => { + rec.model = "bad\x01model"; + }), + "unsafe", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: malformed rates are rejected +// --------------------------------------------------------------------------- +test("schema-negative: pricing record missing output rate is rejected", () => { + assertRejects( + "missing output", + mutatePricing((rec) => { + delete rec.usd_per_mtok.output; + }), + "output", + ); +}); + +test("schema-negative: pricing record with negative input rate is rejected", () => { + assertRejects( + "negative input", + mutatePricing((rec) => { + rec.usd_per_mtok.input = -1; + }), + "input", + ); +}); + +test("schema-negative: pricing record with non-finite input rate is rejected", () => { + assertRejects( + "Infinity input", + mutatePricing((rec) => { + rec.usd_per_mtok.input = Infinity; + }), + "input", + ); +}); + +test("schema-negative: pricing record with NaN output rate is rejected", () => { + assertRejects( + "NaN output", + mutatePricing((rec) => { + rec.usd_per_mtok.output = NaN; + }), + "output", + ); +}); + +test("schema-negative: pricing record with negative cache_read rate is rejected", () => { + assertRejects( + "negative cache_read", + mutatePricing((rec) => { + rec.usd_per_mtok.cache_read = -0.5; + }), + "cache_read", + ); +}); + +test("schema-negative: pricing record with string cache_write rate is rejected", () => { + // Note: Infinity/NaN cannot be tested via JSON mutation (they serialize to null, + // which is valid for cache fields meaning "not published"). Use a string value, + // which is also invalid and does survive JSON serialization. + assertRejects( + "string cache_write", + mutatePricing((rec) => { + rec.usd_per_mtok.cache_write = "not-a-number"; + }), + "cache_write", + ); +}); + +test("schema-negative: pricing record with deleted cache_read is rejected", () => { + assertRejects( + "deleted cache_read", + mutatePricing((rec) => { + delete rec.usd_per_mtok.cache_read; + }), + "cache_read", + ); +}); + +test("schema-negative: pricing record with deleted cache_write is rejected", () => { + assertRejects( + "deleted cache_write", + mutatePricing((rec) => { + delete rec.usd_per_mtok.cache_write; + }), + "cache_write", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: missing provenance is rejected +// --------------------------------------------------------------------------- +test("schema-negative: pricing record missing _source provenance is rejected", () => { + assertRejects( + "missing _source", + mutatePricing((rec) => { + delete rec._source; + }), + "_source", + ); +}); + +test("schema-negative: pricing record with empty _source provenance is rejected", () => { + assertRejects( + "empty _source", + mutatePricing((rec) => { + rec._source = ""; + }), + "_source", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: duplicate (authority, model) pairs are rejected +// --------------------------------------------------------------------------- +test("schema-negative: duplicate pricing record (authority, model) is rejected", () => { + assertRejects( + "duplicate pricing record", + mutate((m) => { + // Add a record identical to the first one + m.pricing_records.push({ ...m.pricing_records[0] }); + }), + "duplicate", + ); +}); + +// --------------------------------------------------------------------------- +// Rule: case-colliding pairs must not silently generate (they are rejected +// at validation because all authorities are already lowercase registered +// tokens; a record with uppercase authority is caught before emit) +// --------------------------------------------------------------------------- +test("schema-negative: pricing record with uppercase authority is rejected before emit", () => { + assertRejects( + "uppercase authority", + mutatePricing((rec) => { + rec.authority = "API.ANTHROPIC.COM"; + }), + "unregistered authority", + ); +}); + console.log("\nSchema-negative validator tests complete.");