diff --git a/desktop/src-tauri/src/commands/mesh_experimental_catalog.rs b/desktop/src-tauri/src/commands/mesh_experimental_catalog.rs new file mode 100644 index 000000000..bd418d2d5 --- /dev/null +++ b/desktop/src-tauri/src/commands/mesh_experimental_catalog.rs @@ -0,0 +1,21 @@ +//! Split-capable model catalog for the experimental tier of Share compute. +//! +//! Its own file because `commands/mesh_llm.rs` sits at 993 of the 1000-line +//! ceiling, and this is a separable concern: the community catalog, not the node +//! lifecycle. + +use crate::commands::CmdResult; +use crate::mesh_llm; + +/// Models that can be served across several machines. +/// +/// Deliberately node-independent — you choose what to share before you start +/// sharing it, so this must work with the runtime off. Reads mesh-llm's own +/// catalog cache from disk, and only downloads when nothing is cached; both are +/// blocking I/O, hence `spawn_blocking` (matching `mesh_model_catalog`). +#[tauri::command] +pub async fn mesh_experimental_catalog() -> CmdResult { + tokio::task::spawn_blocking(mesh_llm::experimental_catalog) + .await + .map_err(|error| format!("mesh experimental catalog task failed: {error}")) +} diff --git a/desktop/src-tauri/src/commands/mod.rs b/desktop/src-tauri/src/commands/mod.rs index 947504854..6bac9b90c 100644 --- a/desktop/src-tauri/src/commands/mod.rs +++ b/desktop/src-tauri/src/commands/mod.rs @@ -33,6 +33,8 @@ mod media_snapshot_png; mod media_transcode; mod media_upload_progress; #[cfg(feature = "mesh-llm")] +mod mesh_experimental_catalog; +#[cfg(feature = "mesh-llm")] mod mesh_live_view; #[cfg(feature = "mesh-llm")] pub(crate) mod mesh_llm; @@ -93,6 +95,8 @@ pub use media::*; pub use media_download::*; pub use media_raw::*; #[cfg(feature = "mesh-llm")] +pub use mesh_experimental_catalog::*; +#[cfg(feature = "mesh-llm")] pub use mesh_live_view::*; #[cfg(feature = "mesh-llm")] pub use mesh_llm::*; diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 7674b1a99..b1f5dd995 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -776,6 +776,7 @@ pub fn run() { mesh_serving_usage, mesh_installed_models, mesh_model_catalog, + mesh_experimental_catalog, mesh_snapshot, mesh_live_view, update_managed_agent, diff --git a/desktop/src-tauri/src/mesh_llm/catalog.rs b/desktop/src-tauri/src/mesh_llm/catalog.rs index f4939e984..7b0eb9902 100644 --- a/desktop/src-tauri/src/mesh_llm/catalog.rs +++ b/desktop/src-tauri/src/mesh_llm/catalog.rs @@ -85,7 +85,7 @@ pub enum ModelFit { TooLarge, } -fn fit_code(model_gb: f64, vram_gb: f64) -> ModelFit { +pub(super) fn fit_code(model_gb: f64, vram_gb: f64) -> ModelFit { if model_gb <= vram_gb * 0.6 { ModelFit::Comfortable } else if model_gb <= vram_gb * 0.9 { diff --git a/desktop/src-tauri/src/mesh_llm/experimental_catalog.rs b/desktop/src-tauri/src/mesh_llm/experimental_catalog.rs new file mode 100644 index 000000000..3a21364d7 --- /dev/null +++ b/desktop/src-tauri/src/mesh_llm/experimental_catalog.rs @@ -0,0 +1,294 @@ +//! Split-capable models — the experimental tier of Share compute. +//! +//! ## What makes a model splittable +//! +//! Not its size, and not its name. A plain GGUF is one opaque blob; splitting +//! needs a *repackaged* model, produced by `mesh-llm models package`, which +//! reads the source tensors, groups them by `layer_index`, and publishes +//! per-layer artifacts alongside a `model-package.json` manifest. mesh-llm +//! detects one by probing for that manifest and explicitly ignores repo naming. +//! +//! The community catalog records the result as a `layer-package` entry, and that +//! flag — never a size heuristic — is what this module reads. +//! +//! ## Why the catalog comes from mesh-llm, not from us +//! +//! `mesh_llm_host_runtime::models::remote_catalog` already loads, caches, and +//! parses . Buzz calls that +//! loader instead of fetching and parsing the JSON itself, so there is exactly +//! one implementation of the catalog format. (Hand-parsing it is also a trap: +//! the package discriminator is `type` on the wire but `package_type` in Rust, +//! via `#[serde(rename)]` — a detail a second parser gets wrong silently.) +//! +//! No running node is required, which is the point: you have to choose what to +//! share *before* you start sharing it. +//! +//! ## Why "too large to fit" is the partition, not "multi-machine" +//! +//! A model can carry a layer package and still fit on one machine — +//! `Qwen3-8B-Q4_K_M` is 5 GB and split-capable. mesh-llm's own precedence says +//! the same: `evaluate_model_target_capacity` returns `SingleNodeFit` first and +//! only falls back to `SplitCandidate` when no single node can hold the model. +//! +//! So splittability alone would tell someone a 5 GB model needs a cluster. The +//! experimental tier is the intersection: split-capable **and** beyond this +//! machine's memory. `fits_locally` is reported per entry so that rule lives in +//! one tested place rather than being re-derived per surface. + +use serde::Serialize; + +use mesh_llm_host_runtime::models::remote_catalog::CatalogEntry; +use mesh_llm_system::hardware; + +use super::catalog::{fit_code, ModelFit}; + +/// A model that can be served across several machines. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MeshSplitEntry { + /// Model reference, valid as-is in the model field. + pub name: String, + /// Catalog size label. Some entries publish `"30 layers"` instead of bytes. + pub size_label: Option, + /// Parsed size in GB, or `None` when the catalog publishes no byte size. + /// + /// Two upstream entries (gemma-4-26B, Kimi-K2) carry a layer count in the + /// size field, which parses to nothing. That is also why mesh-llm reports + /// `unknown_model_size` for them — the size hint is never inserted. + pub size_gb: Option, + /// Layers in the published package: the unit distributed across machines. + pub layer_count: Option, + /// The `-layers` repo holding the package. + pub package_repo: String, + /// Whether this machine alone could hold it. See the module note: a + /// split-capable model that fits locally is not an experimental choice. + pub fits_locally: bool, + pub description: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MeshExperimentalCatalog { + /// Split-capable models, smallest reach first. + pub entries: Vec, + /// This machine's usable AI memory, the figure `fits_locally` is against. + pub vram_gb: f64, + /// The on-disk catalog cache is older than 24h (or absent and unreachable). + /// + /// Surfaced rather than hidden: an empty list means something different when + /// the catalog could not be read than when it genuinely holds no packages. + pub stale: bool, +} + +/// Load the community catalog and project its split-capable models. +/// +/// Blocking: reads the HF cache from disk, and downloads only when nothing is +/// cached. Callers must use `spawn_blocking`. +pub fn experimental_catalog() -> MeshExperimentalCatalog { + use mesh_llm_host_runtime::models::remote_catalog; + + let survey = hardware::survey(); + let vram_gb = survey.vram_bytes as f64 / 1e9; + + // Disk first: instant, and enough to render. A stale cache still describes + // real published packages, so it beats blocking the settings screen on a + // network round trip. + let _ = remote_catalog::load_catalog_from_disk(); + if remote_catalog::catalog_entries().is_none() { + // Nothing cached at all — this is the one case worth waiting for. + let _ = remote_catalog::ensure_catalog(); + } + let stale = remote_catalog::is_catalog_stale(); + let Some(entries) = remote_catalog::catalog_entries() else { + return MeshExperimentalCatalog { + entries: Vec::new(), + vram_gb, + stale: true, + }; + }; + + MeshExperimentalCatalog { + entries: project_split_entries(&entries, vram_gb), + vram_gb, + stale, + } +} + +/// Project split-capable models out of catalog entries. +/// +/// Split from the I/O above so the selection rules are testable without a +/// hardware survey, a network fetch, or mesh-llm's process-global catalog +/// override (which a parallel test run would race on). +fn project_split_entries(entries: &[CatalogEntry], vram_gb: f64) -> Vec { + let mut projected = Vec::new(); + for entry in entries { + for (variant_name, variant) in &entry.variants { + // The layer package is the whole qualification. Never a size + // heuristic, and never the repo name — mesh-llm ignores naming and + // probes for the package manifest, so this must agree. + let Some(package) = variant + .packages + .iter() + .find(|package| package.package_type == "layer-package") + else { + continue; + }; + // `curated.name` is the reference mesh-llm registers as an alias; + // fall back to the variant key, which it also registers. + let name = if variant.curated.name.trim().is_empty() { + variant_name.clone() + } else { + variant.curated.name.clone() + }; + let size_label = variant.curated.size.clone(); + let size_gb = size_label + .as_deref() + .map(mesh_llm_client::models::catalog::parse_size_gb) + .filter(|gb| *gb > 0.0); + projected.push(MeshSplitEntry { + name, + size_label, + size_gb, + layer_count: package.layer_count, + package_repo: package.repo.clone(), + // Unknown size cannot be claimed to fit. Erring toward "needs + // several machines" is the honest direction: it offers the split + // path rather than promising a local start that then fails. + fits_locally: size_gb + .map(|gb| fit_code(gb, vram_gb) != ModelFit::TooLarge) + .unwrap_or(false), + description: variant.curated.description.clone(), + }); + } + } + + projected.sort_by(|left, right| { + // Smallest reach first: the least ambitious split is the likeliest to + // actually form. Unknown sizes sort last — they promise nothing. + let key = |entry: &MeshSplitEntry| entry.size_gb.unwrap_or(f64::MAX); + key(left) + .partial_cmp(&key(right)) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| left.name.cmp(&right.name)) + }); + projected.dedup_by(|left, right| left.name == right.name); + projected +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Fixtures are JSON, not constructed structs. + /// + /// Partly forced — mesh-llm re-exports `CatalogVariant` and friends only + /// under its own `#[cfg(test)]`, so a downstream crate cannot build them — + /// but better regardless: the wire format is where this projection can + /// actually go wrong. The package discriminator is `type` in JSON and + /// `package_type` in Rust (via `#[serde(rename)]`), and an earlier revision + /// of this work read the Rust name off the wire and concluded that *zero* + /// models were split-capable. A struct-built fixture would have passed that + /// bug straight through; this one cannot. + fn entries(json: &str) -> Vec { + vec![serde_json::from_str(json).expect("catalog entry fixture")] + } + + fn one_variant(name: &str, size: &str, packages: &str) -> Vec { + entries(&format!( + r#"{{"schema_version":1,"source_repo":"vendor/Model-GGUF","variants":{{ + "{name}":{{ + "source":{{"repo":"vendor/Model-GGUF"}}, + "curated":{{"name":"{name}","size":"{size}"}}, + "packages":{packages} + }}}}}}"# + )) + } + + const LAYER_PKG: &str = + r#"[{"type":"layer-package","repo":"meshllm/Model-layers","layer_count":64}]"#; + + #[test] + fn only_layer_packaged_variants_qualify() { + // A plain GGUF is one opaque blob; splitting needs the repackaged form, + // and size has no bearing on the question. mesh-llm probes for the + // package manifest and ignores repo naming, so this must agree. + let json = format!( + r#"{{"schema_version":1,"source_repo":"vendor/M-GGUF","variants":{{ + "Plain-Q4":{{"source":{{"repo":"vendor/M-GGUF"}}, + "curated":{{"name":"Plain-Q4","size":"400GB"}},"packages":[]}}, + "Split-Q4":{{"source":{{"repo":"vendor/M-GGUF"}}, + "curated":{{"name":"Split-Q4","size":"20GB"}},"packages":{LAYER_PKG}}} + }}}}"# + ); + let projected = project_split_entries(&entries(&json), 100.0); + assert_eq!(projected.len(), 1); + assert_eq!(projected[0].name, "Split-Q4"); + assert_eq!(projected[0].layer_count, Some(64)); + assert_eq!(projected[0].package_repo, "meshllm/Model-layers"); + } + + #[test] + fn a_split_capable_model_can_still_fit_locally() { + // Qwen3-8B is 5GB and split-capable. Calling it "needs several machines" + // would be false, and mesh-llm agrees: `evaluate_model_target_capacity` + // returns SingleNodeFit before it ever considers a split. This is why + // the experimental tier gates on fit, not on splittability. + let fixture = one_variant("Small-Q4", "5.0GB", LAYER_PKG); + assert!(project_split_entries(&fixture, 100.0)[0].fits_locally); + assert!(!project_split_entries(&fixture, 4.0)[0].fits_locally); + } + + #[test] + fn a_layer_count_in_the_size_field_is_not_a_size() { + // Two upstream entries publish "30 layers" where bytes belong. It parses + // to nothing -- which is also why mesh-llm reports `unknown_model_size` + // for the very model this machine serves. + let projected = project_split_entries(&one_variant("L-Q4", "30 layers", LAYER_PKG), 512.0); + assert_eq!(projected[0].size_gb, None); + assert_eq!(projected[0].size_label.as_deref(), Some("30 layers")); + // Unknown size is never claimed to fit, even on a huge machine: offering + // the split path is recoverable, promising a local start is not. + assert!(!projected[0].fits_locally); + } + + #[test] + fn smallest_reach_sorts_first_and_unknown_sizes_last() { + let json = format!( + r#"{{"schema_version":1,"source_repo":"v/M","variants":{{ + "Big":{{"source":{{"repo":"v/M"}},"curated":{{"name":"Big","size":"300GB"}},"packages":{LAYER_PKG}}}, + "Small":{{"source":{{"repo":"v/M"}},"curated":{{"name":"Small","size":"20GB"}},"packages":{LAYER_PKG}}}, + "Unsized":{{"source":{{"repo":"v/M"}},"curated":{{"name":"Unsized","size":"61 layers"}},"packages":{LAYER_PKG}}} + }}}}"# + ); + let names: Vec<_> = project_split_entries(&entries(&json), 100.0) + .into_iter() + .map(|entry| entry.name) + .collect(); + assert_eq!(names, vec!["Small", "Big", "Unsized"]); + } + + #[test] + fn the_wire_discriminator_is_type_not_package_type() { + // Regression guard for the mistake that produced "0 split-capable + // models": reading Rust's field name (`package_type`) off a wire that + // spells it `type`. + // + // The guard turned out stronger than expected. A wrong key does not + // project to an empty list -- serde refuses the entry outright, because + // `type` is required and unknown fields are ignored. So a format drift + // upstream surfaces as a parse error, never as a silently empty + // experimental tier, which is the failure mode that would have been + // impossible to notice. + let wrong = r#"{"schema_version":1,"source_repo":"v/M","variants":{ + "M-Q4":{"source":{"repo":"v/M"},"curated":{"name":"M-Q4","size":"20GB"}, + "packages":[{"package_type":"layer-package","repo":"meshllm/M-layers"}]}}}"#; + let parsed = serde_json::from_str::(wrong); + assert!(parsed.is_err(), "wrong discriminator must not parse"); + + // And the correct spelling projects one entry. + assert_eq!( + project_split_entries(&one_variant("M-Q4", "20GB", LAYER_PKG), 100.0).len(), + 1 + ); + } +} diff --git a/desktop/src-tauri/src/mesh_llm/mod.rs b/desktop/src-tauri/src/mesh_llm/mod.rs index 87b3ac347..8b142538b 100644 --- a/desktop/src-tauri/src/mesh_llm/mod.rs +++ b/desktop/src-tauri/src/mesh_llm/mod.rs @@ -17,8 +17,10 @@ mod snapshot; pub use snapshot::{snapshot_from_events, MeshSnapshot}; mod catalog; +mod experimental_catalog; pub(crate) use catalog::canonical_curated_model_id; pub use catalog::{model_catalog, MeshModelCatalog}; +pub use experimental_catalog::{experimental_catalog, MeshExperimentalCatalog}; mod identity; pub use identity::ensure_owner_identity; diff --git a/desktop/src-tauri/src/mesh_llm_stubs.rs b/desktop/src-tauri/src/mesh_llm_stubs.rs index f9fc52460..b10913bf4 100644 --- a/desktop/src-tauri/src/mesh_llm_stubs.rs +++ b/desktop/src-tauri/src/mesh_llm_stubs.rs @@ -52,3 +52,8 @@ pub async fn mesh_snapshot(_state: State<'_, AppState>) -> CmdResult) -> CmdResult<()> { Err("mesh-llm feature is not enabled".to_string()) } + +#[tauri::command] +pub async fn mesh_experimental_catalog() -> CmdResult<()> { + Err("mesh-llm feature is not enabled in this build".into()) +} diff --git a/desktop/src/features/mesh-compute/experimentalModel.test.mjs b/desktop/src/features/mesh-compute/experimentalModel.test.mjs new file mode 100644 index 000000000..76cdcd478 --- /dev/null +++ b/desktop/src/features/mesh-compute/experimentalModel.test.mjs @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + describeExperimentalEmpty, + describeSplitEntry, + deriveMeshExperimentalModel, +} from "./experimentalModel.ts"; + +function entry(overrides = {}) { + return { + name: "Model-Q4", + sizeLabel: "20GB", + sizeGb: 20, + layerCount: 64, + packageRepo: "meshllm/Model-layers", + fitsLocally: false, + description: null, + ...overrides, + }; +} + +function catalog(overrides = {}) { + return { entries: [], vramGb: 96, stale: false, ...overrides }; +} + +test("only models beyond this machine are experimental", () => { + // A split-capable model that fits locally is not an experimental choice -- + // Qwen3-8B is 5GB and split-capable, and mesh-llm returns SingleNodeFit for + // it before ever considering a split. Offering it here would tell someone a + // 5GB model needs a cluster. + const model = deriveMeshExperimentalModel( + catalog({ + entries: [ + entry({ name: "Fits", sizeGb: 5, fitsLocally: true }), + entry({ name: "Beyond", sizeGb: 300, fitsLocally: false }), + ], + }), + ); + assert.equal(model.options.length, 1); + assert.equal(model.options[0].entry.name, "Beyond"); +}); + +test("a machine that fits everything is told so, not shown a failure", () => { + // The good outcome must not read like an error. + const model = deriveMeshExperimentalModel( + catalog({ entries: [entry({ fitsLocally: true })] }), + ); + assert.equal(model.options.length, 0); + assert.equal(model.emptyReason, "machineFitsEverything"); + assert.match(describeExperimentalEmpty(model.emptyReason), /on its own/); +}); + +test("empty has distinct causes that never read alike", () => { + // "Could not read the catalog", "the catalog has no packages", and "this + // machine is big enough" are three different facts. + assert.equal(deriveMeshExperimentalModel(undefined).emptyReason, "loading"); + assert.equal( + deriveMeshExperimentalModel(null).emptyReason, + "catalogUnavailable", + ); + assert.equal( + deriveMeshExperimentalModel(catalog()).emptyReason, + "noPackages", + ); + assert.equal( + deriveMeshExperimentalModel(catalog({ stale: true })).emptyReason, + "catalogUnavailable", + ); + + const messages = new Set( + [ + "loading", + "catalogUnavailable", + "noPackages", + "machineFitsEverything", + ].map(describeExperimentalEmpty), + ); + assert.equal(messages.size, 4, "each cause needs its own sentence"); +}); + +test("detail states size and layers, and never invents either", () => { + assert.equal(describeSplitEntry(entry()), "20 GB · 64 layers"); + // Two upstream entries publish "30 layers" where bytes belong. Layers are the + // unit actually distributed, so they stand alone; a size is never synthesized + // from a layer count, because they are not convertible. + assert.equal( + describeSplitEntry( + entry({ sizeGb: null, sizeLabel: "30 layers", layerCount: 30 }), + ), + "30 layers", + ); + // Neither figure published: fall back to the raw label rather than inventing. + assert.equal( + describeSplitEntry( + entry({ sizeGb: null, sizeLabel: "unknown", layerCount: null }), + ), + "unknown", + ); +}); + +test("a large model with no published size still reaches the list", () => { + // Unknown size is reported as not fitting, which is the honest direction: + // offering the split path is recoverable, promising a local start is not. + const model = deriveMeshExperimentalModel( + catalog({ + entries: [ + entry({ + sizeGb: null, + sizeLabel: "61 layers", + layerCount: 61, + fitsLocally: false, + }), + ], + }), + ); + assert.equal(model.options.length, 1); + assert.equal(model.options[0].detail, "61 layers"); +}); diff --git a/desktop/src/features/mesh-compute/experimentalModel.ts b/desktop/src/features/mesh-compute/experimentalModel.ts new file mode 100644 index 000000000..4b8e34b60 --- /dev/null +++ b/desktop/src/features/mesh-compute/experimentalModel.ts @@ -0,0 +1,133 @@ +import type { + MeshExperimentalCatalog, + MeshSplitEntry, +} from "@/shared/api/tauriMesh"; + +/** + * Pure projection for the experimental (split) tier of Advanced settings. + * + * ## Why "beyond this machine" is the partition + * + * Splittability alone is not the interesting fact. A model can carry a layer + * package and still fit on one machine — `Qwen3-8B-Q4_K_M` is 5 GB and + * split-capable — and mesh-llm agrees about precedence: + * `evaluate_model_target_capacity` returns `SingleNodeFit` first and only falls + * back to `SplitCandidate` when no single node can hold the model. + * + * So offering every split-capable model here would tell someone a 5 GB model + * needs a cluster. The experimental tier is the intersection: split-capable + * **and** beyond local memory. Anything that fits belongs in the ordinary + * picker, where it starts immediately. + * + * ## Why the consequence is stated before selection, not after + * + * A collective model does not load on demand. mesh-llm gathers a cohort over + * gossip and holds a settle barrier — at least two participants, membership + * unchanged for a dwell period — then elects a coordinator deterministically by + * VRAM. Until that forms, a node named for a collective model sits waiting. + * + * Nothing about that looks different from a slow download, so a person who was + * not told would reasonably read it as a hang and file a bug. The disclosure is + * therefore part of the option, not a footnote after the fact. + */ + +export type MeshExperimentalOption = { + entry: MeshSplitEntry; + /** e.g. "134 GB · 80 layers", or "61 layers" when no byte size is published. */ + detail: string; +}; + +export type MeshExperimentalModel = { + /** Split-capable models that exceed this machine — the experimental offer. */ + options: MeshExperimentalOption[]; + /** + * Why the list is empty, or null when it has entries. + * + * An empty list has three quite different causes and they must not read + * alike: the catalog could not be read, it holds no packages at all, or this + * machine is simply large enough that nothing needs splitting. The third is + * good news and must not look like a failure. + */ + emptyReason: + | "loading" + | "catalogUnavailable" + | "noPackages" + | "machineFitsEverything" + | null; +}; + +/** Bytes → GB label, matching the rest of the mesh surfaces. */ +function formatGb(gb: number): string { + return `${gb >= 10 ? Math.round(gb) : Math.round(gb * 10) / 10} GB`; +} + +/** + * One line describing reach: size where known, layers always. + * + * Layers are the unit actually distributed, so they are the honest figure when + * the catalog publishes a layer count in place of bytes (two upstream entries + * do). Never synthesizes a size from a layer count — they are not convertible. + */ +export function describeSplitEntry(entry: MeshSplitEntry): string { + const parts: string[] = []; + if (entry.sizeGb !== null) { + parts.push(formatGb(entry.sizeGb)); + } + if (entry.layerCount !== null) { + parts.push(`${entry.layerCount} layers`); + } + if (parts.length === 0) { + // Neither figure published. Say nothing rather than invent a number. + return entry.sizeLabel ?? ""; + } + return parts.join(" · "); +} + +export function deriveMeshExperimentalModel( + catalog: MeshExperimentalCatalog | null | undefined, +): MeshExperimentalModel { + if (catalog === undefined) { + return { options: [], emptyReason: "loading" }; + } + if (catalog === null) { + return { options: [], emptyReason: "catalogUnavailable" }; + } + if (catalog.entries.length === 0) { + return { + options: [], + emptyReason: catalog.stale ? "catalogUnavailable" : "noPackages", + }; + } + const beyondThisMachine = catalog.entries.filter( + (entry) => !entry.fitsLocally, + ); + if (beyondThisMachine.length === 0) { + // Every split-capable model fits here. Not a failure — the machine is big + // enough that splitting buys nothing. + return { options: [], emptyReason: "machineFitsEverything" }; + } + return { + options: beyondThisMachine.map((entry) => ({ + entry, + detail: describeSplitEntry(entry), + })), + emptyReason: null, + }; +} + +/** Copy for an empty experimental tier, keyed on why. */ +export function describeExperimentalEmpty( + reason: NonNullable, +): string { + switch (reason) { + case "loading": + return "Checking for models that can run across machines…"; + case "catalogUnavailable": + return "Can't reach the model catalog right now."; + case "noPackages": + return "No models in the catalog can run across machines yet."; + case "machineFitsEverything": + // Deliberately not phrased as an absence: this is the good outcome. + return "This computer can run every model in the catalog on its own."; + } +} diff --git a/desktop/src/features/mesh-compute/hooks/useMeshExperimentalCatalog.ts b/desktop/src/features/mesh-compute/hooks/useMeshExperimentalCatalog.ts new file mode 100644 index 000000000..fe1384f9a --- /dev/null +++ b/desktop/src/features/mesh-compute/hooks/useMeshExperimentalCatalog.ts @@ -0,0 +1,62 @@ +import * as React from "react"; + +import { meshExperimentalCatalog } from "@/shared/api/tauriMesh"; +import type { MeshExperimentalCatalog } from "@/shared/api/tauriMesh"; + +/** + * Load the split-capable model catalog. + * + * Fetched once rather than polled: the community catalog is a published dataset + * that changes when someone packages a model, not live state. mesh-llm caches it + * on disk for 24h, so re-asking on a timer would add nothing. + * + * `undefined` while in flight and `null` on failure are deliberately distinct — + * the experimental tier has to tell "still looking" from "couldn't read the + * catalog" from "the catalog genuinely has no packages", and all three would + * otherwise render as an unexplained empty list. + * + * Requires no running node: you have to choose what to share before you start + * sharing it. + */ +export function useMeshExperimentalCatalog(enabled: boolean): { + catalog: MeshExperimentalCatalog | null | undefined; + error: string | null; +} { + const [catalog, setCatalog] = React.useState< + MeshExperimentalCatalog | null | undefined + >(undefined); + const [error, setError] = React.useState(null); + const loaded = React.useRef(false); + + React.useEffect(() => { + // Only when the section is actually opened: the first call can download the + // catalog, which is not work to do for a screen nobody expanded. + if (!enabled || loaded.current) return; + loaded.current = true; + let cancelled = false; + void (async () => { + try { + const next = await meshExperimentalCatalog(); + if (!cancelled) { + setCatalog(next); + setError(null); + } + } catch (err) { + if (!cancelled) { + // null, not an empty catalog: "could not read" must not masquerade as + // "nothing published". + setCatalog(null); + setError(err instanceof Error ? err.message : String(err)); + // Allow a retry on reopen — a transient network failure should not + // permanently empty the section. + loaded.current = false; + } + } + })(); + return () => { + cancelled = true; + }; + }, [enabled]); + + return { catalog, error }; +} diff --git a/desktop/src/features/mesh-compute/ui/MeshComputeSettingsCard.tsx b/desktop/src/features/mesh-compute/ui/MeshComputeSettingsCard.tsx index 6fc728125..940ffc45b 100644 --- a/desktop/src/features/mesh-compute/ui/MeshComputeSettingsCard.tsx +++ b/desktop/src/features/mesh-compute/ui/MeshComputeSettingsCard.tsx @@ -27,6 +27,7 @@ import type { MeshModelOption, MeshNodeStatus, } from "@/shared/api/tauriMesh"; +import { MeshExperimentalSection } from "./MeshExperimentalSection"; import { SettingsSectionHeader } from "@/features/settings/ui/SettingsSectionHeader"; import { classifyModelRef } from "../classifyModelRef"; import { @@ -204,7 +205,7 @@ export function MeshComputeSettingsCard() {
{error ? ( @@ -301,6 +302,18 @@ export function MeshComputeSettingsCard() { usePersonaInputStyle value={maxVramGb} /> + + {/* + Nested one level deeper than the memory cap: capping memory is a + normal advanced adjustment, whereas a collective model changes + whether sharing starts at all. Two clicks to reach it is the + point. + */} + ) : null} diff --git a/desktop/src/features/mesh-compute/ui/MeshExperimentalSection.tsx b/desktop/src/features/mesh-compute/ui/MeshExperimentalSection.tsx new file mode 100644 index 000000000..842839e65 --- /dev/null +++ b/desktop/src/features/mesh-compute/ui/MeshExperimentalSection.tsx @@ -0,0 +1,131 @@ +import { ChevronDown, FlaskConical } from "lucide-react"; +import * as React from "react"; + +import { cn } from "@/shared/lib/cn"; + +import { useMeshExperimentalCatalog } from "../hooks/useMeshExperimentalCatalog"; +import { + describeExperimentalEmpty, + deriveMeshExperimentalModel, +} from "../experimentalModel"; + +/** + * The experimental tier of Advanced settings: models served across machines. + * + * ## Why this is a separate, collapsed tier + * + * Every other model in the picker starts when you flip the switch. These do not. + * mesh-llm gathers a cohort over gossip and holds a settle barrier — at least + * two participants, membership unchanged for a dwell period — then elects a + * coordinator deterministically by VRAM. Until that forms, a node named for a + * collective model waits. + * + * That waiting is indistinguishable from a slow download, so someone who was not + * warned would read it as a hang. The disclosure therefore sits above the list, + * before a choice is made, rather than as a toast afterwards. + * + * ## Why it can be empty for good reasons + * + * Three different facts produce an empty list — the catalog could not be read, + * it holds no layer packages, or this machine is large enough that nothing needs + * splitting — and the third is good news. `deriveMeshExperimentalModel` keeps + * them distinct so this never renders an unexplained blank. + */ +export function MeshExperimentalSection({ + disabled, + onModelChange, + selectedModel, +}: { + disabled: boolean; + onModelChange: (model: string) => void; + selectedModel: string; +}) { + const [open, setOpen] = React.useState(false); + // Fetch only once opened: the first call can download the catalog, which is + // not work to do for a section nobody expanded. + const { catalog } = useMeshExperimentalCatalog(open); + const model = React.useMemo( + () => deriveMeshExperimentalModel(catalog), + [catalog], + ); + const selected = selectedModel.trim(); + + return ( +
+ + + {open ? ( +
+ {/* + Stated before the list, not after a selection. The wait is the whole + character of these models, and it looks exactly like a hang to + anyone who was not told. + */} +

+ These models are too large for this computer alone and run across + several machines at once.{" "} + + Sharing will not start until enough well-connected computers are + sharing the same model + {" "} + — it waits instead of loading. For advanced setups. +

+ + {model.emptyReason ? ( +

+ {describeExperimentalEmpty(model.emptyReason)} +

+ ) : ( +
    + {model.options.map((option) => { + const isSelected = selected === option.entry.name; + return ( +
  • + +
    • + ); + })} +
+ )} +
+ ) : null} +
+ ); +} diff --git a/desktop/src/shared/api/tauriMesh.ts b/desktop/src/shared/api/tauriMesh.ts index 4a66d5b67..2206912ac 100644 --- a/desktop/src/shared/api/tauriMesh.ts +++ b/desktop/src/shared/api/tauriMesh.ts @@ -226,3 +226,58 @@ export type MeshLiveView = { export async function meshLiveView(): Promise { return await invokeTauri("mesh_live_view"); } + +/** + * A model that can be served across several machines. + * + * Qualification is the presence of a published *layer package* — never a size + * heuristic and never the repo name. A plain GGUF is one opaque blob; splitting + * needs the repackaged form produced by `mesh-llm models package`, and mesh-llm + * detects it by probing for the package manifest. + */ +export type MeshSplitEntry = { + /** Model reference, valid as-is in the model field. */ + name: string; + /** Catalog size label. Some entries publish "30 layers" instead of bytes. */ + sizeLabel: string | null; + /** Parsed size, or null when the catalog publishes no byte size. */ + sizeGb: number | null; + /** Layers in the package: the unit distributed across machines. */ + layerCount: number | null; + /** The `-layers` repo holding the package. */ + packageRepo: string; + /** + * Whether this machine alone could hold it. + * + * Load-bearing: a split-capable model that fits locally is not an + * experimental choice. Qwen3-8B is 5 GB and split-capable, and mesh-llm + * returns `SingleNodeFit` for it before ever considering a split. + */ + fitsLocally: boolean; + description: string | null; +}; + +export type MeshExperimentalCatalog = { + /** Split-capable models, smallest reach first. */ + entries: MeshSplitEntry[]; + /** This machine's usable AI memory — the figure `fitsLocally` is against. */ + vramGb: number; + /** + * The catalog cache is stale or unreadable. Surfaced rather than hidden: an + * empty list means something different when the catalog could not be read + * than when it genuinely holds no packages. + */ + stale: boolean; +}; + +/** + * Models that can be served across several machines. + * + * Deliberately node-independent — you choose what to share before you start + * sharing it, so this works with the runtime off. + */ +export async function meshExperimentalCatalog(): Promise { + return await invokeTauri( + "mesh_experimental_catalog", + ); +}