hold Obsidian boot until the priority cache slice loads

2026-06-17 04:35:53 +00:00 · 2026-06-16 23:14:26 +02:00
parent 201607dbea
commit b6c538fb33
4 changed files with 353 additions and 64 deletions
--- a/apps/ignis-server/server/assets/index.html
+++ b/apps/ignis-server/server/assets/index.html
@@ -65,25 +65,67 @@
    }, 250);
  }

-  update();
+  function appendScripts() {
+    // No Obsidian scripts to load (markup or scrape mismatch); clear the splash instead of pulsing forever.
+    if (scripts.length === 0) {
+      done();
+      return;
+    }

-  for (var i = 0; i < scripts.length; i++) {
-    var s = document.createElement("script");
-    s.type = "text/javascript";
-    s.src = scripts[i];
-    s.async = false;
-    s.onload = function () {
-      loaded++;
-      update();
-      if (loaded === scripts.length) done();
-    };
-    s.onerror = function () {
-      loaded++;
-      update();
-      if (loaded === scripts.length) done();
-    };
-    document.body.appendChild(s);
+    update();
+
+    for (var i = 0; i < scripts.length; i++) {
+      var s = document.createElement("script");
+      s.type = "text/javascript";
+      s.src = scripts[i];
+      s.async = false;
+      s.onload = function () {
+        loaded++;
+        update();
+        if (loaded === scripts.length) done();
+      };
+      s.onerror = function () {
+        loaded++;
+        update();
+        if (loaded === scripts.length) done();
+      };
+      document.body.appendChild(s);
+    }
  }
+
+  // Hold Obsidian's scripts until the shim signals the priority cache slice has landed (window.__ignisBootReady), so Obsidian's early config and plugin reads hit the warm cache.
+  // A timeout proceeds anyway, so a missing or never-resolving promise degrades to loading immediately instead of blocking boot.
+  var ready = window.__ignisBootReady;
+  if (!ready || typeof ready.then !== "function") {
+    appendScripts();
+    return;
+  }
+
+  var started = false;
+
+  function start() {
+    if (started) {
+      return;
+    }
+
+    started = true;
+    // Tell the shim's progress writer to stop touching the splash label now that we own it.
+    window.__ignisBootStarted = true;
+    appendScripts();
+  }
+
+  var timer = setTimeout(start, 3000);
+
+  ready.then(
+    function () {
+      clearTimeout(timer);
+      start();
+    },
+    function () {
+      clearTimeout(timer);
+      start();
+    },
+  );
 })();
 </script>
 </body>
--- a/packages/shim/src/fs/indexer-prefetch.js
+++ b/packages/shim/src/fs/indexer-prefetch.js
@@ -1,10 +1,7 @@
-// Eager batch pre-fetch of vault content into ContentCache.
-//
-// Fired once after the metadata cache is populated. Iterates the tree in
-// directory-traversal order and pulls text file contents in batches via
-// /api/fs/batch-read. Caps at MAX_BYTES so it doesn't thrash the LRU.
-// Drops content directly into ContentCache; the indexer hits the cache
-// instead of fetching each file individually.
+// Batch pre-fetch of vault content into ContentCache.
+// Pulls text file contents in batches via /api/fs/batch-read and drops them into ContentCache so Obsidian's startup reads hit the cache instead of fetching each file individually.
+// The priority slice (.obsidian configs and plugin entry files) is fetched first and its promise resolves once it lands, so boot can wait for those reads to be warm.
+// The bulk slice (everything else) streams afterward without blocking boot.

 const TEXT_EXTENSIONS = new Set([
  ".md", ".markdown", ".txt", ".json", ".csv",
@@ -13,8 +10,12 @@ const TEXT_EXTENSIONS = new Set([
  ".svg",
 ]);

-const MAX_BYTES = 30 * 1024 * 1024; // 30 MB
-const MAX_FILE_BYTES = 512 * 1024; // skip files larger than 512 KB
+const MAX_BYTES = 30 * 1024 * 1024; // 30 MB total across both slices
+const MAX_FILE_BYTES = 512 * 1024; // skip bulk files larger than 512 KB
+// Plugin main.js bundles can run a few MB and Obsidian needs them at boot, so the priority slice accepts larger files than the bulk slice.
+const PRIORITY_MAX_FILE_BYTES = 4 * 1024 * 1024; // 4 MB
+// Cap the priority slice's share of the total so a heavy config or plugin set cannot starve the bulk slice.
+const PRIORITY_MAX_BYTES = 10 * 1024 * 1024; // 10 MB
 const BATCH_SIZE = 50;

 function isTextPath(path) {
@@ -27,36 +28,68 @@ function isTextPath(path) {
  return TEXT_EXTENSIONS.has(path.slice(dot).toLowerCase());
 }

-function selectPrefetchTargets(tree) {
-  const paths = [];
+// Boot-critical files: root-level .obsidian configs and each plugin's entry files.
+// Plugin data.json and other nested config fall to the bulk slice so a large blob does not inflate the awaited slice.
+function isPriorityPath(path) {
+  if (!path.startsWith(".obsidian/")) {
+    return false;
+  }
+
+  // Root-level configs only (app.json, appearance.json, core-plugins.json, workspace.json, etc.).
+  if (/^\.obsidian\/[^/]+\.json$/.test(path)) {
+    return true;
+  }
+
+  return /^\.obsidian\/plugins\/[^/]+\/(main\.js|manifest\.json|styles\.css)$/.test(
+    path,
+  );
+}
+
+function collectSlice(entries, predicate, perFileCap, budget) {
+  const files = [];
  let bytes = 0;

-  // Iterate in tree key order, which already matches directory traversal
-  // because the server's walk emits parent-before-children.
-  for (const [path, entry] of Object.entries(tree)) {
-    if (entry.type !== "file") {
-      continue;
-    }
-
-    if (!isTextPath(path)) {
+  for (const [path, entry] of entries) {
+    if (entry.type !== "file" || !isTextPath(path) || !predicate(path)) {
      continue;
    }

    const size = entry.size || 0;

-    if (size === 0 || size > MAX_FILE_BYTES) {
+    if (size === 0 || size > perFileCap) {
      continue;
    }

-    if (bytes + size > MAX_BYTES) {
-      break;
+    if (bytes + size > budget) {
+      continue;
    }

-    paths.push(path);
+    files.push({ path, size });
    bytes += size;
  }

-  return { paths, bytes };
+  return { files, bytes };
+}
+
+function selectPrefetchTargets(tree) {
+  // Tree key order matches directory traversal (the server walk emits parent before children).
+  const entries = Object.entries(tree);
+  const priority = collectSlice(
+    entries,
+    isPriorityPath,
+    PRIORITY_MAX_FILE_BYTES,
+    PRIORITY_MAX_BYTES,
+  );
+
+  // Bulk fills whatever byte budget the priority slice left.
+  const bulk = collectSlice(
+    entries,
+    (path) => !isPriorityPath(path),
+    MAX_FILE_BYTES,
+    MAX_BYTES - priority.bytes,
+  );
+
+  return { priority, bulk };
 }

 async function fetchBatch(vaultId, paths) {
@@ -73,41 +106,84 @@ async function fetchBatch(vaultId, paths) {
  return res.json();
 }

-export async function prefetchVaultContent(vaultId, tree, contentCache) {
-  if (!vaultId || !tree) {
-    return;
-  }
-
-  const { paths, bytes } = selectPrefetchTargets(tree);
-
-  if (paths.length === 0) {
+async function runBatches(vaultId, slice, contentCache, label, onProgress) {
+  if (slice.files.length === 0) {
    return;
  }

  const t0 = Date.now();
  let cached = 0;
+  let received = 0;

-  for (let i = 0; i < paths.length; i += BATCH_SIZE) {
-    const batch = paths.slice(i, i + BATCH_SIZE);
+  // Report the total up front so the splash shows the target before the first batch lands.
+  if (onProgress) {
+    onProgress(0, slice.bytes);
+  }
+
+  for (let i = 0; i < slice.files.length; i += BATCH_SIZE) {
+    const batch = slice.files.slice(i, i + BATCH_SIZE);
+
+    let result;

    try {
-      const result = await fetchBatch(vaultId, batch);
-
-      for (const [path, content] of Object.entries(result.files || {})) {
-        if (typeof content === "string") {
-          contentCache.set(path, content);
-          cached++;
-        }
-      }
+      result = await fetchBatch(
+        vaultId,
+        batch.map((f) => f.path),
+      );
    } catch (e) {
-      console.warn("[ignis] Prefetch batch failed:", e.message);
+      // Abandon the rest of this slice; the returned promise still resolves so boot is never blocked on a failed batch.
+      console.warn(`[ignis] Prefetch ${label} batch failed:`, e.message);
      return;
    }
+
+    for (const [path, content] of Object.entries(result.files || {})) {
+      if (typeof content === "string") {
+        contentCache.set(path, content);
+        cached++;
+      }
+    }
+
+    if (onProgress) {
+      for (const f of batch) {
+        received += f.size;
+      }
+
+      onProgress(received, slice.bytes);
+    }
  }

  const ms = Date.now() - t0;

  console.log(
-    `[ignis] Prefetched ${cached}/${paths.length} files (${(bytes / 1024).toFixed(0)} KB) in ${ms}ms`,
+    `[ignis] Prefetched ${label} ${cached}/${slice.files.length} files (${(slice.bytes / 1024).toFixed(0)} KB) in ${ms}ms`,
  );
 }
+
+// Returns { priority, bulk }: a promise for each slice.
+// The priority promise resolves once the boot-critical files have landed (or were abandoned on a batch failure), so it is always safe to await.
+export function prefetchVaultContent(vaultId, tree, contentCache, options = {}) {
+  if (!vaultId || !tree) {
+    return { priority: Promise.resolve(), bulk: Promise.resolve() };
+  }
+
+  const { priority, bulk } = selectPrefetchTargets(tree);
+
+  const priorityDone = runBatches(
+    vaultId,
+    priority,
+    contentCache,
+    "priority",
+    options.onProgress,
+  );
+
+  // Bulk streams after the priority slice so it does not contend for the connection pool while boot is waiting on priority.
+  // It runs regardless of how priority settled and swallows its own rejection, since init.js discards this promise.
+  const bulkDone = priorityDone
+    .catch(() => {})
+    .then(() => runBatches(vaultId, bulk, contentCache, "bulk"))
+    .catch((e) => {
+      console.warn("[ignis] Prefetch bulk failed:", e && e.message);
+    });
+
+  return { priority: priorityDone, bulk: bulkDone };
+}
--- a/packages/shim/src/fs/indexer-prefetch.test.js
+++ b/packages/shim/src/fs/indexer-prefetch.test.js
@@ -0,0 +1,147 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { prefetchVaultContent } from "./indexer-prefetch.js";
+
+const MB = 1024 * 1024;
+
+// Every selection rule.
+const tree = {
+  ".obsidian/app.json": { type: "file", size: 100 },
+  ".obsidian/community-plugins.json": { type: "file", size: 50 },
+  ".obsidian/plugins/big/main.js": { type: "file", size: 2 * MB },
+  ".obsidian/plugins/big/manifest.json": { type: "file", size: 80 },
+  ".obsidian/plugins/big/styles.css": { type: "file", size: 200 },
+  ".obsidian/plugins/big/data.json": { type: "file", size: 300 * 1024 },
+  "Note.md": { type: "file", size: 100 },
+  "Big.md": { type: "file", size: 600 * 1024 },
+  "plugins/fake/main.js": { type: "file", size: 100 },
+  somedir: { type: "directory" },
+};
+
+const PRIORITY_BYTES = 100 + 50 + 2 * MB + 80 + 200;
+
+let fetchCalls;
+
+function makeCache() {
+  const store = new Map();
+  return { store, set: (path, content) => store.set(path, content) };
+}
+
+beforeEach(() => {
+  fetchCalls = [];
+  globalThis.fetch = vi.fn(async (url, init) => {
+    const paths = JSON.parse(init.body).paths;
+    fetchCalls.push(paths);
+
+    const files = {};
+
+    for (const p of paths) {
+      files[p] = "content:" + p;
+    }
+
+    return { ok: true, json: async () => ({ files }) };
+  });
+});
+
+afterEach(() => {
+  delete globalThis.fetch;
+  vi.restoreAllMocks();
+});
+
+describe("prefetchVaultContent slicing", () => {
+  it("fetches the priority slice before the bulk slice", async () => {
+    const result = prefetchVaultContent("v", tree, makeCache());
+    await result.bulk;
+
+    expect(fetchCalls.length).toBe(2);
+    const [priorityPaths, bulkPaths] = fetchCalls;
+
+    expect(priorityPaths).toEqual(
+      expect.arrayContaining([
+        ".obsidian/app.json",
+        ".obsidian/community-plugins.json",
+        ".obsidian/plugins/big/main.js",
+        ".obsidian/plugins/big/manifest.json",
+        ".obsidian/plugins/big/styles.css",
+      ]),
+    );
+    expect(priorityPaths).not.toContain("Note.md");
+
+    expect(bulkPaths).toEqual(
+      expect.arrayContaining(["Note.md", "plugins/fake/main.js"]),
+    );
+  });
+
+  it("anchors the plugin predicate to .obsidian, so a bare plugins/ path is bulk", async () => {
+    const result = prefetchVaultContent("v", tree, makeCache());
+    await result.bulk;
+
+    expect(fetchCalls[0]).not.toContain("plugins/fake/main.js");
+    expect(fetchCalls[1]).toContain("plugins/fake/main.js");
+  });
+
+  it("leaves plugin data.json to the bulk slice, not priority", async () => {
+    const result = prefetchVaultContent("v", tree, makeCache());
+    await result.bulk;
+
+    expect(fetchCalls[0]).not.toContain(".obsidian/plugins/big/data.json");
+    expect(fetchCalls[1]).toContain(".obsidian/plugins/big/data.json");
+  });
+
+  it("caps the priority slice at its own byte budget", async () => {
+    // Three 4MB plugin entry files: two fit the 10MB priority budget, the third is dropped.
+    const bigTree = {
+      ".obsidian/plugins/a/main.js": { type: "file", size: 4 * MB },
+      ".obsidian/plugins/b/main.js": { type: "file", size: 4 * MB },
+      ".obsidian/plugins/c/main.js": { type: "file", size: 4 * MB },
+    };
+
+    const result = prefetchVaultContent("v", bigTree, makeCache());
+    await result.bulk;
+
+    expect(fetchCalls[0].length).toBe(2);
+  });
+
+  it("drops a bulk file over the 512KB per-file cap", async () => {
+    const result = prefetchVaultContent("v", tree, makeCache());
+    await result.bulk;
+
+    expect(fetchCalls.flat()).not.toContain("Big.md");
+  });
+
+  it("reports priority byte progress from zero up to the slice total", async () => {
+    const onProgress = vi.fn();
+    const result = prefetchVaultContent("v", tree, makeCache(), { onProgress });
+    await result.priority;
+
+    expect(onProgress).toHaveBeenCalledWith(0, PRIORITY_BYTES);
+    expect(onProgress).toHaveBeenLastCalledWith(PRIORITY_BYTES, PRIORITY_BYTES);
+  });
+
+  it("caches returned content under its path", async () => {
+    const cache = makeCache();
+    const result = prefetchVaultContent("v", tree, cache);
+    await result.bulk;
+
+    expect(cache.store.get(".obsidian/app.json")).toBe(
+      "content:.obsidian/app.json",
+    );
+    expect(cache.store.get("Note.md")).toBe("content:Note.md");
+  });
+
+  it("resolves both promises without fetching when there is no vault", async () => {
+    const result = prefetchVaultContent("", tree, makeCache());
+
+    await result.priority;
+    await result.bulk;
+
+    expect(fetchCalls.length).toBe(0);
+  });
+
+  it("resolves the priority promise even when a batch fails", async () => {
+    globalThis.fetch = vi.fn(async () => ({ ok: false, status: 500 }));
+
+    const result = prefetchVaultContent("v", tree, makeCache());
+
+    await expect(result.priority).resolves.toBeUndefined();
+  });
+});
--- a/packages/shim/src/init.js
+++ b/packages/shim/src/init.js
@@ -208,8 +208,27 @@ function initCoreSyncGuardFallback() {
  }
 }

+// Reflect the priority prefetch's byte progress on the boot splash so the awaited slice reads as active rather than hung.
+// The splash logo keeps pulsing through a transit stall, when the byte count would otherwise freeze.
+function updateBootProgress(received, total) {
+  // Once the injector starts appending Obsidian's scripts it owns the splash label, so stop writing progress over it.
+  if (window.__ignisBootStarted) {
+    return;
+  }
+
+  const label = document.getElementById("ignis-status-label");
+
+  if (!label || !total) {
+    return;
+  }
+
+  const mb = (n) => (n / (1024 * 1024)).toFixed(1);
+  label.textContent = `Loading plugins... ${mb(received)}/${mb(total)} MB`;
+}
+
 export function initialize() {
  if (maybeProvisionDemoVault()) {
+    window.__ignisBootReady = Promise.resolve();
    return;
  }

@@ -229,14 +248,19 @@ export function initialize() {
    bootstrapVirtualPlugins = bootstrap.virtualPlugins || [];
    applyServerSettings(bootstrap.settings);

-    // Race the indexer: batch-fetch text content into ContentCache so
-    // Obsidian's startup indexing reads hit the cache instead of the network.
-    prefetchVaultContent(
+    // Warm the caches before Obsidian boots.
+    // The priority slice (configs and plugin entry files) resolves window.__ignisBootReady, which the index.html injector waits on before appending Obsidian's scripts, so Obsidian's early reads hit the cache.
+    // The bulk slice streams afterward without blocking boot.
+    const { priority } = prefetchVaultContent(
      window.__currentVaultId,
      bootstrap.tree,
      fsShim._contentCache,
+      { onProgress: updateBootProgress },
    );
+
+    window.__ignisBootReady = priority;
  } else {
+    window.__ignisBootReady = Promise.resolve();
    initVaultConfigFallback();
    initVaultListFallback();
    initMetadataCacheFallback();