From b6c538fb33e82573c0891061132d2efd88e6f59a Mon Sep 17 00:00:00 2001 From: Nystik <236107-Nystik@users.noreply.gitlab.com> Date: Tue, 16 Jun 2026 23:14:26 +0200 Subject: [PATCH] hold Obsidian boot until the priority cache slice loads --- apps/ignis-server/server/assets/index.html | 76 ++++++-- packages/shim/src/fs/indexer-prefetch.js | 164 +++++++++++++----- packages/shim/src/fs/indexer-prefetch.test.js | 147 ++++++++++++++++ packages/shim/src/init.js | 30 +++- 4 files changed, 353 insertions(+), 64 deletions(-) create mode 100644 packages/shim/src/fs/indexer-prefetch.test.js diff --git a/apps/ignis-server/server/assets/index.html b/apps/ignis-server/server/assets/index.html index c3e7578..f9a60f2 100644 --- a/apps/ignis-server/server/assets/index.html +++ b/apps/ignis-server/server/assets/index.html @@ -65,25 +65,67 @@ }, 250); } - update(); + function appendScripts() { + // No Obsidian scripts to load (markup or scrape mismatch); clear the splash instead of pulsing forever. + if (scripts.length === 0) { + done(); + return; + } - for (var i = 0; i < scripts.length; i++) { - var s = document.createElement("script"); - s.type = "text/javascript"; - s.src = scripts[i]; - s.async = false; - s.onload = function () { - loaded++; - update(); - if (loaded === scripts.length) done(); - }; - s.onerror = function () { - loaded++; - update(); - if (loaded === scripts.length) done(); - }; - document.body.appendChild(s); + update(); + + for (var i = 0; i < scripts.length; i++) { + var s = document.createElement("script"); + s.type = "text/javascript"; + s.src = scripts[i]; + s.async = false; + s.onload = function () { + loaded++; + update(); + if (loaded === scripts.length) done(); + }; + s.onerror = function () { + loaded++; + update(); + if (loaded === scripts.length) done(); + }; + document.body.appendChild(s); + } } + + // Hold Obsidian's scripts until the shim signals the priority cache slice has landed (window.__ignisBootReady), so Obsidian's early config and plugin reads hit the warm cache. + // A timeout proceeds anyway, so a missing or never-resolving promise degrades to loading immediately instead of blocking boot. + var ready = window.__ignisBootReady; + if (!ready || typeof ready.then !== "function") { + appendScripts(); + return; + } + + var started = false; + + function start() { + if (started) { + return; + } + + started = true; + // Tell the shim's progress writer to stop touching the splash label now that we own it. + window.__ignisBootStarted = true; + appendScripts(); + } + + var timer = setTimeout(start, 3000); + + ready.then( + function () { + clearTimeout(timer); + start(); + }, + function () { + clearTimeout(timer); + start(); + }, + ); })(); diff --git a/packages/shim/src/fs/indexer-prefetch.js b/packages/shim/src/fs/indexer-prefetch.js index 1ec1ccd..e4c5132 100644 --- a/packages/shim/src/fs/indexer-prefetch.js +++ b/packages/shim/src/fs/indexer-prefetch.js @@ -1,10 +1,7 @@ -// Eager batch pre-fetch of vault content into ContentCache. -// -// Fired once after the metadata cache is populated. Iterates the tree in -// directory-traversal order and pulls text file contents in batches via -// /api/fs/batch-read. Caps at MAX_BYTES so it doesn't thrash the LRU. -// Drops content directly into ContentCache; the indexer hits the cache -// instead of fetching each file individually. +// Batch pre-fetch of vault content into ContentCache. +// Pulls text file contents in batches via /api/fs/batch-read and drops them into ContentCache so Obsidian's startup reads hit the cache instead of fetching each file individually. +// The priority slice (.obsidian configs and plugin entry files) is fetched first and its promise resolves once it lands, so boot can wait for those reads to be warm. +// The bulk slice (everything else) streams afterward without blocking boot. const TEXT_EXTENSIONS = new Set([ ".md", ".markdown", ".txt", ".json", ".csv", @@ -13,8 +10,12 @@ const TEXT_EXTENSIONS = new Set([ ".svg", ]); -const MAX_BYTES = 30 * 1024 * 1024; // 30 MB -const MAX_FILE_BYTES = 512 * 1024; // skip files larger than 512 KB +const MAX_BYTES = 30 * 1024 * 1024; // 30 MB total across both slices +const MAX_FILE_BYTES = 512 * 1024; // skip bulk files larger than 512 KB +// Plugin main.js bundles can run a few MB and Obsidian needs them at boot, so the priority slice accepts larger files than the bulk slice. +const PRIORITY_MAX_FILE_BYTES = 4 * 1024 * 1024; // 4 MB +// Cap the priority slice's share of the total so a heavy config or plugin set cannot starve the bulk slice. +const PRIORITY_MAX_BYTES = 10 * 1024 * 1024; // 10 MB const BATCH_SIZE = 50; function isTextPath(path) { @@ -27,36 +28,68 @@ function isTextPath(path) { return TEXT_EXTENSIONS.has(path.slice(dot).toLowerCase()); } -function selectPrefetchTargets(tree) { - const paths = []; +// Boot-critical files: root-level .obsidian configs and each plugin's entry files. +// Plugin data.json and other nested config fall to the bulk slice so a large blob does not inflate the awaited slice. +function isPriorityPath(path) { + if (!path.startsWith(".obsidian/")) { + return false; + } + + // Root-level configs only (app.json, appearance.json, core-plugins.json, workspace.json, etc.). + if (/^\.obsidian\/[^/]+\.json$/.test(path)) { + return true; + } + + return /^\.obsidian\/plugins\/[^/]+\/(main\.js|manifest\.json|styles\.css)$/.test( + path, + ); +} + +function collectSlice(entries, predicate, perFileCap, budget) { + const files = []; let bytes = 0; - // Iterate in tree key order, which already matches directory traversal - // because the server's walk emits parent-before-children. - for (const [path, entry] of Object.entries(tree)) { - if (entry.type !== "file") { - continue; - } - - if (!isTextPath(path)) { + for (const [path, entry] of entries) { + if (entry.type !== "file" || !isTextPath(path) || !predicate(path)) { continue; } const size = entry.size || 0; - if (size === 0 || size > MAX_FILE_BYTES) { + if (size === 0 || size > perFileCap) { continue; } - if (bytes + size > MAX_BYTES) { - break; + if (bytes + size > budget) { + continue; } - paths.push(path); + files.push({ path, size }); bytes += size; } - return { paths, bytes }; + return { files, bytes }; +} + +function selectPrefetchTargets(tree) { + // Tree key order matches directory traversal (the server walk emits parent before children). + const entries = Object.entries(tree); + const priority = collectSlice( + entries, + isPriorityPath, + PRIORITY_MAX_FILE_BYTES, + PRIORITY_MAX_BYTES, + ); + + // Bulk fills whatever byte budget the priority slice left. + const bulk = collectSlice( + entries, + (path) => !isPriorityPath(path), + MAX_FILE_BYTES, + MAX_BYTES - priority.bytes, + ); + + return { priority, bulk }; } async function fetchBatch(vaultId, paths) { @@ -73,41 +106,84 @@ async function fetchBatch(vaultId, paths) { return res.json(); } -export async function prefetchVaultContent(vaultId, tree, contentCache) { - if (!vaultId || !tree) { - return; - } - - const { paths, bytes } = selectPrefetchTargets(tree); - - if (paths.length === 0) { +async function runBatches(vaultId, slice, contentCache, label, onProgress) { + if (slice.files.length === 0) { return; } const t0 = Date.now(); let cached = 0; + let received = 0; - for (let i = 0; i < paths.length; i += BATCH_SIZE) { - const batch = paths.slice(i, i + BATCH_SIZE); + // Report the total up front so the splash shows the target before the first batch lands. + if (onProgress) { + onProgress(0, slice.bytes); + } + + for (let i = 0; i < slice.files.length; i += BATCH_SIZE) { + const batch = slice.files.slice(i, i + BATCH_SIZE); + + let result; try { - const result = await fetchBatch(vaultId, batch); - - for (const [path, content] of Object.entries(result.files || {})) { - if (typeof content === "string") { - contentCache.set(path, content); - cached++; - } - } + result = await fetchBatch( + vaultId, + batch.map((f) => f.path), + ); } catch (e) { - console.warn("[ignis] Prefetch batch failed:", e.message); + // Abandon the rest of this slice; the returned promise still resolves so boot is never blocked on a failed batch. + console.warn(`[ignis] Prefetch ${label} batch failed:`, e.message); return; } + + for (const [path, content] of Object.entries(result.files || {})) { + if (typeof content === "string") { + contentCache.set(path, content); + cached++; + } + } + + if (onProgress) { + for (const f of batch) { + received += f.size; + } + + onProgress(received, slice.bytes); + } } const ms = Date.now() - t0; console.log( - `[ignis] Prefetched ${cached}/${paths.length} files (${(bytes / 1024).toFixed(0)} KB) in ${ms}ms`, + `[ignis] Prefetched ${label} ${cached}/${slice.files.length} files (${(slice.bytes / 1024).toFixed(0)} KB) in ${ms}ms`, ); } + +// Returns { priority, bulk }: a promise for each slice. +// The priority promise resolves once the boot-critical files have landed (or were abandoned on a batch failure), so it is always safe to await. +export function prefetchVaultContent(vaultId, tree, contentCache, options = {}) { + if (!vaultId || !tree) { + return { priority: Promise.resolve(), bulk: Promise.resolve() }; + } + + const { priority, bulk } = selectPrefetchTargets(tree); + + const priorityDone = runBatches( + vaultId, + priority, + contentCache, + "priority", + options.onProgress, + ); + + // Bulk streams after the priority slice so it does not contend for the connection pool while boot is waiting on priority. + // It runs regardless of how priority settled and swallows its own rejection, since init.js discards this promise. + const bulkDone = priorityDone + .catch(() => {}) + .then(() => runBatches(vaultId, bulk, contentCache, "bulk")) + .catch((e) => { + console.warn("[ignis] Prefetch bulk failed:", e && e.message); + }); + + return { priority: priorityDone, bulk: bulkDone }; +} diff --git a/packages/shim/src/fs/indexer-prefetch.test.js b/packages/shim/src/fs/indexer-prefetch.test.js new file mode 100644 index 0000000..2bc23a0 --- /dev/null +++ b/packages/shim/src/fs/indexer-prefetch.test.js @@ -0,0 +1,147 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { prefetchVaultContent } from "./indexer-prefetch.js"; + +const MB = 1024 * 1024; + +// Every selection rule. +const tree = { + ".obsidian/app.json": { type: "file", size: 100 }, + ".obsidian/community-plugins.json": { type: "file", size: 50 }, + ".obsidian/plugins/big/main.js": { type: "file", size: 2 * MB }, + ".obsidian/plugins/big/manifest.json": { type: "file", size: 80 }, + ".obsidian/plugins/big/styles.css": { type: "file", size: 200 }, + ".obsidian/plugins/big/data.json": { type: "file", size: 300 * 1024 }, + "Note.md": { type: "file", size: 100 }, + "Big.md": { type: "file", size: 600 * 1024 }, + "plugins/fake/main.js": { type: "file", size: 100 }, + somedir: { type: "directory" }, +}; + +const PRIORITY_BYTES = 100 + 50 + 2 * MB + 80 + 200; + +let fetchCalls; + +function makeCache() { + const store = new Map(); + return { store, set: (path, content) => store.set(path, content) }; +} + +beforeEach(() => { + fetchCalls = []; + globalThis.fetch = vi.fn(async (url, init) => { + const paths = JSON.parse(init.body).paths; + fetchCalls.push(paths); + + const files = {}; + + for (const p of paths) { + files[p] = "content:" + p; + } + + return { ok: true, json: async () => ({ files }) }; + }); +}); + +afterEach(() => { + delete globalThis.fetch; + vi.restoreAllMocks(); +}); + +describe("prefetchVaultContent slicing", () => { + it("fetches the priority slice before the bulk slice", async () => { + const result = prefetchVaultContent("v", tree, makeCache()); + await result.bulk; + + expect(fetchCalls.length).toBe(2); + const [priorityPaths, bulkPaths] = fetchCalls; + + expect(priorityPaths).toEqual( + expect.arrayContaining([ + ".obsidian/app.json", + ".obsidian/community-plugins.json", + ".obsidian/plugins/big/main.js", + ".obsidian/plugins/big/manifest.json", + ".obsidian/plugins/big/styles.css", + ]), + ); + expect(priorityPaths).not.toContain("Note.md"); + + expect(bulkPaths).toEqual( + expect.arrayContaining(["Note.md", "plugins/fake/main.js"]), + ); + }); + + it("anchors the plugin predicate to .obsidian, so a bare plugins/ path is bulk", async () => { + const result = prefetchVaultContent("v", tree, makeCache()); + await result.bulk; + + expect(fetchCalls[0]).not.toContain("plugins/fake/main.js"); + expect(fetchCalls[1]).toContain("plugins/fake/main.js"); + }); + + it("leaves plugin data.json to the bulk slice, not priority", async () => { + const result = prefetchVaultContent("v", tree, makeCache()); + await result.bulk; + + expect(fetchCalls[0]).not.toContain(".obsidian/plugins/big/data.json"); + expect(fetchCalls[1]).toContain(".obsidian/plugins/big/data.json"); + }); + + it("caps the priority slice at its own byte budget", async () => { + // Three 4MB plugin entry files: two fit the 10MB priority budget, the third is dropped. + const bigTree = { + ".obsidian/plugins/a/main.js": { type: "file", size: 4 * MB }, + ".obsidian/plugins/b/main.js": { type: "file", size: 4 * MB }, + ".obsidian/plugins/c/main.js": { type: "file", size: 4 * MB }, + }; + + const result = prefetchVaultContent("v", bigTree, makeCache()); + await result.bulk; + + expect(fetchCalls[0].length).toBe(2); + }); + + it("drops a bulk file over the 512KB per-file cap", async () => { + const result = prefetchVaultContent("v", tree, makeCache()); + await result.bulk; + + expect(fetchCalls.flat()).not.toContain("Big.md"); + }); + + it("reports priority byte progress from zero up to the slice total", async () => { + const onProgress = vi.fn(); + const result = prefetchVaultContent("v", tree, makeCache(), { onProgress }); + await result.priority; + + expect(onProgress).toHaveBeenCalledWith(0, PRIORITY_BYTES); + expect(onProgress).toHaveBeenLastCalledWith(PRIORITY_BYTES, PRIORITY_BYTES); + }); + + it("caches returned content under its path", async () => { + const cache = makeCache(); + const result = prefetchVaultContent("v", tree, cache); + await result.bulk; + + expect(cache.store.get(".obsidian/app.json")).toBe( + "content:.obsidian/app.json", + ); + expect(cache.store.get("Note.md")).toBe("content:Note.md"); + }); + + it("resolves both promises without fetching when there is no vault", async () => { + const result = prefetchVaultContent("", tree, makeCache()); + + await result.priority; + await result.bulk; + + expect(fetchCalls.length).toBe(0); + }); + + it("resolves the priority promise even when a batch fails", async () => { + globalThis.fetch = vi.fn(async () => ({ ok: false, status: 500 })); + + const result = prefetchVaultContent("v", tree, makeCache()); + + await expect(result.priority).resolves.toBeUndefined(); + }); +}); diff --git a/packages/shim/src/init.js b/packages/shim/src/init.js index c9154b1..907c628 100644 --- a/packages/shim/src/init.js +++ b/packages/shim/src/init.js @@ -208,8 +208,27 @@ function initCoreSyncGuardFallback() { } } +// Reflect the priority prefetch's byte progress on the boot splash so the awaited slice reads as active rather than hung. +// The splash logo keeps pulsing through a transit stall, when the byte count would otherwise freeze. +function updateBootProgress(received, total) { + // Once the injector starts appending Obsidian's scripts it owns the splash label, so stop writing progress over it. + if (window.__ignisBootStarted) { + return; + } + + const label = document.getElementById("ignis-status-label"); + + if (!label || !total) { + return; + } + + const mb = (n) => (n / (1024 * 1024)).toFixed(1); + label.textContent = `Loading plugins... ${mb(received)}/${mb(total)} MB`; +} + export function initialize() { if (maybeProvisionDemoVault()) { + window.__ignisBootReady = Promise.resolve(); return; } @@ -229,14 +248,19 @@ export function initialize() { bootstrapVirtualPlugins = bootstrap.virtualPlugins || []; applyServerSettings(bootstrap.settings); - // Race the indexer: batch-fetch text content into ContentCache so - // Obsidian's startup indexing reads hit the cache instead of the network. - prefetchVaultContent( + // Warm the caches before Obsidian boots. + // The priority slice (configs and plugin entry files) resolves window.__ignisBootReady, which the index.html injector waits on before appending Obsidian's scripts, so Obsidian's early reads hit the cache. + // The bulk slice streams afterward without blocking boot. + const { priority } = prefetchVaultContent( window.__currentVaultId, bootstrap.tree, fsShim._contentCache, + { onProgress: updateBootProgress }, ); + + window.__ignisBootReady = priority; } else { + window.__ignisBootReady = Promise.resolve(); initVaultConfigFallback(); initVaultListFallback(); initMetadataCacheFallback();