add write coalescing

README.md

@@ -116,6 +116,7 @@ volumes:
 | `OBSIDIAN_VERSION` | Obsidian version to download | `1.12.4` |
 | `PUID` | User ID for file ownership | `1000` |
 | `PGID` | Group ID for file ownership | `1000` |
+| `WRITE_COALESCE_MS` | Debounce window (ms) for rapid writes. Useful for slow filesystems (rclone, NFS, SMB). Set to `0` to disable. | `5000` |
 
 ## Contributing
 

docs/ARCHITECTURE.md

@@ -24,7 +24,9 @@ The shim layer makes Obsidian think it's running in Electron. The bridge plugin
 
 ### Loading
 
-Obsidian's index file is intercepted during serving and the shim loader is loaded ahead of Obsidian's own scripts. It replaces the module system and makes a blocking HTTP request to fetch the vault's directory tree into memory. The request has to be blocking because Obsidian makes synchronous filesystem calls during page load, before the event loop is running, so the cache has to already be populated.
+The server serves its own `index.html` (in `server/assets/`) rather than Obsidian's. At startup it reads Obsidian's `index.html` once to discover which scripts Obsidian expects, then embeds that list in our HTML as a JSON array. The client-side HTML loads the shim loader and UI bundle first (non-deferred), then a small inline script dynamically injects Obsidian's scripts in order. Obsidian's files are never modified, read into responses, or transformed in transit.
+
+The shim loader replaces the module system and makes a blocking HTTP request to fetch the vault's directory tree into memory. The request has to be blocking because Obsidian makes synchronous filesystem calls during page load, before the event loop is running, so the cache has to already be populated.
 
 ### Modules
 
@@ -43,7 +45,9 @@ Unknown modules return an empty proxy and log a warning. The shim exposes two co
 
 ### Filesystem
 
-On page load the server returns the full directory tree, which gets cached in memory with paths, sizes, and modification times. Sync filesystem calls hit the cache rather than the network. File contents are cached in an LRU cache after first read and written through immediately on writes.
+On page load the server returns the full directory tree, which gets cached in memory with paths, sizes, and modification times. Sync filesystem calls hit the cache rather than the network. File contents are cached in an LRU cache after first read.
+
+Writes go through a server-side write coalescer (`server/write-coalescer.js`) designed for slow filesystems like rclone FUSE mounts. The first write to a file goes to disk immediately. Subsequent writes within a configurable window (default 5 seconds, `WRITE_COALESCE_MS`) are buffered in memory; the timer resets on each write. After the window elapses with no new writes, the buffered data is flushed to disk. Reads for pending paths serve the buffered content so clients never see stale data. All pending writes are flushed on graceful shutdown.
 
 Sync calls use synchronous XHR to ensure blocking behavior. Async calls use fetch. Everything goes through a transport layer that handles vault ID injection, base64 encoding for binary files, and mapping HTTP error codes back to Node errno values.
 

server/config.js

@@ -75,6 +75,8 @@ module.exports = {
     vaults = discoverVaults();
     return vaults;
   },
+  writeCoalesceMs: parseInt(process.env.WRITE_COALESCE_MS) || 5000,
+
   obsidianAssetsPath:
     process.env.OBSIDIAN_ASSETS_PATH ||
     path.join(__dirname, "..", "investigation", "obsidian_1.12.4_unpacked"),

server/index.js

@@ -9,6 +9,7 @@ const watcher = require("./watcher");
 const { updateBridgePluginInAllVaults } = require("./bridge-plugin");
 const { initPlugins, shutdownPlugins } = require("./plugin-system/manager");
 const pluginRoutes = require("./routes/plugins");
+const { flushAll } = require("./write-coalescer");
 
 const ANSI_RED = "\x1b[31m";
 const ANSI_YELLOW = "\x1b[33m";
@@ -159,6 +160,7 @@ const wss = setupWebSocket(server);
 async function gracefulShutdown(signal) {
   console.log(`\n[ignis] Received ${signal}, shutting down gracefully...`);
 
+  await flushAll();
   await shutdownPlugins();
 
   server.close(() => {

server/routes/fs.js

@@ -3,6 +3,7 @@ const fs = require("fs");
 const path = require("path");
 const archiver = require("archiver");
 const config = require("../config");
+const { writeCoalesced, getPending } = require("../write-coalescer");
 
 const router = express.Router();
 
@@ -115,6 +116,25 @@ router.get("/stat", async (req, res) => {
   }
 
   try {
+    // If a coalesced write is pending, report its size instead of stale disk data
+    const buffered = getPending(resolved);
+
+    if (buffered) {
+      const diskStat = await fs.promises.stat(resolved).catch(() => null);
+      const size = Buffer.isBuffer(buffered.data)
+        ? buffered.data.length
+        : Buffer.byteLength(buffered.data, buffered.encoding || "utf-8");
+
+      res.json({
+        type: "file",
+        size,
+        mtime: Date.now(),
+        ctime: diskStat ? diskStat.ctimeMs : Date.now(),
+      });
+
+      return;
+    }
+
     const stat = await fs.promises.stat(resolved);
 
     res.json({
@@ -183,6 +203,21 @@ router.get("/readFile", async (req, res) => {
       });
     }
 
+    // Serve buffered content if a coalesced write is pending for this path
+    const buffered = getPending(resolved);
+
+    if (buffered) {
+      const encoding = req.query.encoding;
+
+      if (encoding === "utf8" || encoding === "utf-8") {
+        res.type("text/plain").send(buffered.data);
+      } else {
+        res.type("application/octet-stream").send(buffered.data);
+      }
+
+      return;
+    }
+
     const encoding = req.query.encoding;
 
     if (encoding === "utf8" || encoding === "utf-8") {
@@ -221,15 +256,9 @@ router.post("/writeFile", async (req, res) => {
       data = Buffer.from(req.body.content, "base64");
     }
 
-    await fs.promises.writeFile(
-      resolved,
-      data,
-      encoding === "binary" ? undefined : encoding,
-    );
+    const result = await writeCoalesced(resolved, data, encoding);
 
-    const stat = await fs.promises.stat(resolved);
-
-    res.json({ ok: true, mtime: stat.mtimeMs, size: stat.size });
+    res.json({ ok: true, mtime: result.mtime, size: result.size });
   } catch (e) {
     res.status(500).json({ error: e.message, code: e.code });
   }

server/write-coalescer.js

@@ -0,0 +1,176 @@
+// Write coalescer for slow filesystems (rclone, FUSE, NFS, SMB).
+//
+// First write to a path goes to disk immediately. Subsequent writes within
+// the coalesce window are buffered; the timer resets on each write. After
+// the window elapses with no new writes, the buffered data is flushed.
+//
+// This prevents rapid-fire writes (e.g. workspace.json saved 20x/min)
+// from overwhelming network-mounted filesystems.
+
+const fs = require("fs");
+const config = require("./config");
+
+const FLUSH_TIMEOUT_MS = 10000;
+
+// absPath -> timestamp of last completed write
+const lastWriteTime = new Map();
+
+// absPath -> { data, encoding, timer, resolvers: [{ resolve, reject }] }
+const pending = new Map();
+
+async function writeToDisk(absPath, data, encoding) {
+  await fs.promises.writeFile(
+    absPath,
+    data,
+    encoding === "binary" ? undefined : encoding,
+  );
+
+  lastWriteTime.set(absPath, Date.now());
+  const stat = await fs.promises.stat(absPath);
+
+  return { mtime: stat.mtimeMs, size: stat.size };
+}
+
+function flushEntry(absPath) {
+  const entry = pending.get(absPath);
+
+  if (!entry) {
+    return;
+  }
+
+  clearTimeout(entry.timer);
+  pending.delete(absPath);
+
+  writeToDisk(absPath, entry.data, entry.encoding).then(
+    (result) => {
+      for (const r of entry.resolvers) {
+        r.resolve(result);
+      }
+    },
+    (err) => {
+      for (const r of entry.resolvers) {
+        r.reject(err);
+      }
+    },
+  );
+}
+
+function scheduleFlush(absPath) {
+  const entry = pending.get(absPath);
+
+  if (!entry) {
+    return;
+  }
+
+  clearTimeout(entry.timer);
+  entry.timer = setTimeout(() => flushEntry(absPath), config.writeCoalesceMs);
+}
+
+/**
+ * Write file content, coalescing rapid writes.
+ * The returned promise resolves with { mtime, size } once data hits disk.
+ */
+async function writeCoalesced(absPath, data, encoding) {
+  const windowMs = config.writeCoalesceMs;
+
+  // Coalescing disabled or first write to this path
+  const last = lastWriteTime.get(absPath);
+
+  if (windowMs <= 0 || !last || Date.now() - last >= windowMs) {
+    // Resolve any pending write for this path first
+    if (pending.has(absPath)) {
+      clearTimeout(pending.get(absPath).timer);
+      pending.delete(absPath);
+    }
+
+    return writeToDisk(absPath, data, encoding);
+  }
+
+  // Within the coalesce window: buffer the write
+  return new Promise((resolve, reject) => {
+    const existing = pending.get(absPath);
+
+    if (existing) {
+      // Update data and add another resolver
+      existing.data = data;
+      existing.encoding = encoding;
+      existing.resolvers.push({ resolve, reject });
+      scheduleFlush(absPath);
+    } else {
+      const entry = {
+        data,
+        encoding,
+        timer: null,
+        resolvers: [{ resolve, reject }],
+      };
+
+      pending.set(absPath, entry);
+      scheduleFlush(absPath);
+    }
+  });
+}
+
+/**
+ * Get pending (not yet flushed) data for a path, or null.
+ * Used by readFile to serve buffered content instead of stale disk data.
+ */
+function getPending(absPath) {
+  const entry = pending.get(absPath);
+
+  if (entry) {
+    return { data: entry.data, encoding: entry.encoding };
+  }
+
+  return null;
+}
+
+/**
+ * Flush all pending writes to disk. Called on graceful shutdown.
+ */
+async function flushAll() {
+  const paths = [...pending.keys()];
+
+  if (paths.length === 0) {
+    return;
+  }
+
+  console.log(`[write-coalesce] Flushing ${paths.length} pending write(s)...`);
+
+  // Clear all timers
+  for (const entry of pending.values()) {
+    clearTimeout(entry.timer);
+  }
+
+  const writes = paths.map((absPath) => {
+    const entry = pending.get(absPath);
+    pending.delete(absPath);
+
+    return writeToDisk(absPath, entry.data, entry.encoding).then(
+      (result) => {
+        for (const r of entry.resolvers) {
+          r.resolve(result);
+        }
+      },
+      (err) => {
+        console.error(`[write-coalesce] Failed to flush ${absPath}:`, err);
+
+        for (const r of entry.resolvers) {
+          r.reject(err);
+        }
+      },
+    );
+  });
+
+  const timeout = new Promise((resolve) => {
+    setTimeout(() => {
+      console.warn(
+        "[write-coalesce] Flush timeout -- some writes may be lost",
+      );
+      resolve();
+    }, FLUSH_TIMEOUT_MS);
+  });
+
+  await Promise.race([Promise.allSettled(writes), timeout]);
+}
+
+module.exports = { writeCoalesced, getPending, flushAll };