implement fs shims

shims/fs/constants.js

@@ -0,0 +1,20 @@
+// Node.js fs.constants equivalents
+
+export const constants = {
+  F_OK: 0,
+  R_OK: 4,
+  W_OK: 2,
+  X_OK: 1,
+
+  COPYFILE_EXCL: 1,
+  COPYFILE_FICLONE: 2,
+  COPYFILE_FICLONE_FORCE: 4,
+
+  O_RDONLY: 0,
+  O_WRONLY: 1,
+  O_RDWR: 2,
+  O_CREAT: 64,
+  O_EXCL: 128,
+  O_TRUNC: 512,
+  O_APPEND: 1024,
+};

shims/fs/content-cache.js

@@ -0,0 +1,91 @@
+// In-memory content cache with simple LRU eviction
+// Stores file content fetched from the server.
+
+const DEFAULT_MAX_SIZE = 50 * 1024 * 1024; // 50 MB
+
+export class ContentCache {
+  constructor(maxSize = DEFAULT_MAX_SIZE) {
+    this._cache = new Map(); // path -> { data, size, accessedAt }
+    this._currentSize = 0;
+    this._maxSize = maxSize;
+  }
+
+  has(path) {
+    return this._cache.has(this._normalize(path));
+  }
+
+  get(path) {
+    const entry = this._cache.get(this._normalize(path));
+    if (entry) {
+      entry.accessedAt = Date.now();
+      return entry.data;
+    }
+    return null;
+  }
+
+  set(path, data) {
+    const norm = this._normalize(path);
+    const size = data ? data.length || data.byteLength || 0 : 0;
+
+    // Remove old entry if replacing
+    if (this._cache.has(norm)) {
+      this._currentSize -= this._cache.get(norm).size;
+    }
+
+    // Evict LRU entries if needed
+    while (this._currentSize + size > this._maxSize && this._cache.size > 0) {
+      this._evictOne();
+    }
+
+    this._cache.set(norm, { data, size, accessedAt: Date.now() });
+    this._currentSize += size;
+  }
+
+  delete(path) {
+    const norm = this._normalize(path);
+    const entry = this._cache.get(norm);
+    if (entry) {
+      this._currentSize -= entry.size;
+      this._cache.delete(norm);
+    }
+  }
+
+  // Invalidate a path (remove from cache so next read fetches fresh)
+  invalidate(path) {
+    this.delete(path);
+  }
+
+  clear() {
+    this._cache.clear();
+    this._currentSize = 0;
+  }
+
+  get size() {
+    return this._cache.size;
+  }
+
+  get currentBytes() {
+    return this._currentSize;
+  }
+
+  _evictOne() {
+    let oldest = null;
+    let oldestTime = Infinity;
+    for (const [key, entry] of this._cache) {
+      if (entry.accessedAt < oldestTime) {
+        oldest = key;
+        oldestTime = entry.accessedAt;
+      }
+    }
+    if (oldest) {
+      this.delete(oldest);
+    }
+  }
+
+  _normalize(p) {
+    return (p || "")
+      .replace(/\\/g, "/")
+      .replace(/^\/+/, "")
+      .replace(/\/+$/, "");
+  }
+}

shims/fs/index.js

@@ -0,0 +1,52 @@
+// Filesystem shim  -  the core piece
+// Returned for both require('original-fs') and require('fs')
+//
+// Strategy: metadata cache + on-demand content fetch + write-through
+// Server sync mechanism (REST vs WebSocket) is TBD  -  abstracted behind
+// the transport layer in ./transport.js
+
+import { MetadataCache } from './metadata-cache.js';
+import { ContentCache } from './content-cache.js';
+import { transport } from './transport.js';
+import { createFsPromises } from './promises.js';
+import { createFsSync } from './sync.js';
+import { createFsWatch } from './watch.js';
+import { constants } from './constants.js';
+
+const metadataCache = new MetadataCache();
+const contentCache = new ContentCache();
+
+const fsPromises = createFsPromises(metadataCache, contentCache, transport);
+const fsSync = createFsSync(metadataCache, contentCache, transport);
+const fsWatch = createFsWatch(transport);
+
+export const fsShim = {
+  // Async promise-based API (this.fsPromises = this.fs.promises)
+  promises: fsPromises,
+
+  // Sync methods
+  existsSync: fsSync.existsSync,
+  readFileSync: fsSync.readFileSync,
+  writeFileSync: fsSync.writeFileSync,
+  unlinkSync: fsSync.unlinkSync,
+  accessSync: fsSync.accessSync,
+  statSync: fsSync.statSync,
+  readdirSync: fsSync.readdirSync,
+
+  // Watch
+  watch: fsWatch.watch,
+
+  // Constants
+  constants,
+
+  // Internal: for initialization
+  _metadataCache: metadataCache,
+  _contentCache: contentCache,
+
+  // Initialize the caches by fetching the full tree from server
+  async _init(basePath) {
+    const tree = await transport.fetchTree(basePath);
+    metadataCache.populate(tree);
+    console.log(`[shim:fs] Initialized with ${metadataCache.size} entries`);
+  },
+};

shims/fs/metadata-cache.js

@@ -0,0 +1,116 @@
+// In-memory metadata cache
+// Populated from /api/fs/tree on startup, kept in sync via transport events.
+// All stat/exists/readdir calls are served from this cache (zero latency).
+
+export class MetadataCache {
+  constructor() {
+    // Map<string, { type: 'file'|'directory', size: number, mtime: number, ctime: number }>
+    this._entries = new Map();
+  }
+
+  // Populate from a server-provided tree object
+  // tree shape: { "relative/path": { type, size, mtime, ctime }, ... }
+  populate(tree) {
+    this._entries.clear();
+    for (const [path, meta] of Object.entries(tree)) {
+      this._entries.set(this._normalize(path), meta);
+    }
+  }
+
+  has(path) {
+    return this._entries.has(this._normalize(path));
+  }
+
+  get(path) {
+    return this._entries.get(this._normalize(path)) || null;
+  }
+
+  set(path, meta) {
+    this._entries.set(this._normalize(path), meta);
+  }
+
+  delete(path) {
+    this._entries.delete(this._normalize(path));
+  }
+
+  // Rename: move metadata from old path to new path (and children if directory)
+  rename(oldPath, newPath) {
+    const oldNorm = this._normalize(oldPath);
+    const newNorm = this._normalize(newPath);
+    const meta = this._entries.get(oldNorm);
+    if (meta) {
+      this._entries.delete(oldNorm);
+      this._entries.set(newNorm, meta);
+    }
+    // Move children
+    const prefix = oldNorm + "/";
+    for (const [key, val] of this._entries) {
+      if (key.startsWith(prefix)) {
+        const newKey = newNorm + "/" + key.slice(prefix.length);
+        this._entries.delete(key);
+        this._entries.set(newKey, val);
+      }
+    }
+  }
+
+  // List direct children of a directory path
+  readdir(dirPath) {
+    const norm = this._normalize(dirPath);
+    const prefix = norm === "" ? "" : norm + "/";
+    const results = [];
+    const seen = new Set();
+
+    for (const [key, meta] of this._entries) {
+      if (prefix === "" || key.startsWith(prefix)) {
+        const rest = key.slice(prefix.length);
+        const slashIdx = rest.indexOf("/");
+        const childName = slashIdx >= 0 ? rest.slice(0, slashIdx) : rest;
+        if (childName && !seen.has(childName)) {
+          seen.add(childName);
+          const childMeta = this._entries.get(prefix + childName);
+          results.push({
+            name: childName,
+            type: childMeta?.type || (slashIdx >= 0 ? "directory" : "file"),
+          });
+        }
+      }
+    }
+    return results;
+  }
+
+  get size() {
+    return this._entries.size;
+  }
+
+  // Build a stat-like object from metadata
+  toStat(path) {
+    const meta = this.get(path);
+    if (!meta) return null;
+    return {
+      size: meta.size || 0,
+      mtimeMs: meta.mtime || 0,
+      ctimeMs: meta.ctime || 0,
+      atimeMs: meta.mtime || 0,
+      birthtimeMs: meta.ctime || 0,
+      mtime: new Date(meta.mtime || 0),
+      ctime: new Date(meta.ctime || 0),
+      atime: new Date(meta.mtime || 0),
+      birthtime: new Date(meta.ctime || 0),
+      isFile: () => meta.type === "file",
+      isDirectory: () => meta.type === "directory",
+      isSymbolicLink: () => false,
+      isBlockDevice: () => false,
+      isCharacterDevice: () => false,
+      isFIFO: () => false,
+      isSocket: () => false,
+    };
+  }
+
+  _normalize(p) {
+    // Normalize slashes, remove leading and trailing slashes
+    return (p || "")
+      .replace(/\\/g, "/")
+      .replace(/^\/+/, "")
+      .replace(/\/+$/, "");
+  }
+}

shims/fs/promises.js

@@ -0,0 +1,163 @@
+// Async fs.promises implementation
+// Maps to transport layer (REST/WebSocket/hybrid  -  TBD)
+
+export function createFsPromises(metadataCache, contentCache, transport) {
+  return {
+    async stat(path) {
+      // Try cache first, fall back to server
+      const cached = metadataCache.toStat(path);
+      if (cached) return cached;
+
+      const meta = await transport.stat(path);
+      metadataCache.set(path, meta);
+      return metadataCache.toStat(path);
+    },
+
+    async lstat(path) {
+      // No symlinks in our context  -  same as stat
+      return this.stat(path);
+    },
+
+    async readdir(path) {
+      // If metadata cache knows this is a file, return empty (ENOTDIR)
+      const meta = metadataCache.get(path);
+      if (meta && meta.type === "file") {
+        return [];
+      }
+      // Serve from metadata cache
+      const entries = metadataCache.readdir(path);
+      if (entries.length > 0) {
+        return entries.map((e) => e.name);
+      }
+      // Fallback to server
+      const serverEntries = await transport.readdir(path);
+      return serverEntries.map((e) => e.name);
+    },
+
+    async readFile(path, encoding) {
+      if (typeof encoding === "object") encoding = encoding?.encoding;
+      const wantText = encoding === "utf8" || encoding === "utf-8";
+
+      // Check content cache
+      const cached = contentCache.get(path);
+      if (cached !== null) {
+        if (wantText) {
+          return typeof cached === "string"
+            ? cached
+            : new TextDecoder().decode(cached);
+        }
+        // Binary mode: ensure we return a proper Uint8Array with .buffer
+        if (typeof cached === "string") {
+          return new TextEncoder().encode(cached);
+        }
+        return cached;
+      }
+
+      // Fetch from server
+      const data = await transport.readFile(path, encoding);
+      contentCache.set(path, data);
+      return data;
+    },
+
+    async writeFile(path, data, encoding) {
+      if (typeof encoding === "object") encoding = encoding?.encoding;
+
+      // Update caches optimistically
+      contentCache.set(path, data);
+      const size =
+        typeof data === "string" ? data.length : data.byteLength || 0;
+      metadataCache.set(path, {
+        type: "file",
+        size,
+        mtime: Date.now(),
+        ctime: metadataCache.get(path)?.ctime || Date.now(),
+      });
+
+      // Send to server
+      const result = await transport.writeFile(path, data, encoding);
+      // Update metadata with server-confirmed values
+      if (result.mtime) {
+        metadataCache.set(path, {
+          type: "file",
+          size: result.size || size,
+          mtime: result.mtime,
+          ctime: metadataCache.get(path)?.ctime || Date.now(),
+        });
+      }
+    },
+
+    async appendFile(path, data, encoding) {
+      contentCache.invalidate(path);
+      await transport.appendFile(path, data);
+      // Refresh metadata
+      const meta = await transport.stat(path);
+      metadataCache.set(path, meta);
+    },
+
+    async unlink(path) {
+      contentCache.delete(path);
+      metadataCache.delete(path);
+      await transport.unlink(path);
+    },
+
+    async rename(oldPath, newPath) {
+      // Move content cache entry
+      const content = contentCache.get(oldPath);
+      if (content !== null) {
+        contentCache.set(newPath, content);
+        contentCache.delete(oldPath);
+      }
+      // Move metadata
+      metadataCache.rename(oldPath, newPath);
+
+      await transport.rename(oldPath, newPath);
+    },
+
+    async mkdir(path, options) {
+      const recursive =
+        typeof options === "object" ? !!options.recursive : !!options;
+      metadataCache.set(path, { type: "directory" });
+      await transport.mkdir(path, recursive);
+    },
+
+    async rmdir(path) {
+      metadataCache.delete(path);
+      await transport.rmdir(path);
+    },
+
+    async rm(path, options) {
+      const recursive =
+        typeof options === "object" ? !!options.recursive : false;
+      metadataCache.delete(path);
+      contentCache.delete(path);
+      await transport.rm(path, recursive);
+    },
+
+    async copyFile(src, dest) {
+      await transport.copyFile(src, dest);
+      // Refresh metadata for dest
+      const meta = await transport.stat(dest);
+      metadataCache.set(dest, meta);
+    },
+
+    async access(path) {
+      if (metadataCache.has(path)) return;
+      await transport.access(path);
+    },
+
+    async realpath(path) {
+      // Empty path = vault root, return the vault base path
+      if (!path || path === "/" || path === ".") return "/";
+      return transport.realpath(path);
+    },
+
+    async utimes(path, atime, mtime) {
+      await transport.utimes(path, atime, mtime);
+      const meta = metadataCache.get(path);
+      if (meta) {
+        meta.mtime = typeof mtime === "number" ? mtime : mtime.getTime();
+        metadataCache.set(path, meta);
+      }
+    },
+  };
+}

shims/fs/sync.js

@@ -0,0 +1,81 @@
+// Synchronous fs method implementations
+// Served from caches where possible, sync XHR fallback for uncached content.
+
+export function createFsSync(metadataCache, contentCache, transport) {
+  return {
+    existsSync(path) {
+      return metadataCache.has(path);
+    },
+
+    statSync(path) {
+      const stat = metadataCache.toStat(path);
+      if (!stat) {
+        const err = new Error(`ENOENT: no such file or directory, stat '${path}'`);
+        err.code = 'ENOENT';
+        throw err;
+      }
+      return stat;
+    },
+
+    accessSync(path, mode) {
+      if (!metadataCache.has(path)) {
+        const err = new Error(`ENOENT: no such file or directory, access '${path}'`);
+        err.code = 'ENOENT';
+        throw err;
+      }
+    },
+
+    readFileSync(path, encoding) {
+      if (typeof encoding === 'object') encoding = encoding?.encoding;
+
+      // Try content cache first
+      const cached = contentCache.get(path);
+      if (cached !== null) {
+        if (encoding === 'utf8' || encoding === 'utf-8') {
+          return typeof cached === 'string' ? cached : new TextDecoder().decode(cached);
+        }
+        return cached;
+      }
+
+      // Fallback: synchronous XHR
+      console.warn('[shim:fs] readFileSync cache miss, using sync XHR:', path);
+      const data = transport.readFileSync(path, encoding);
+      contentCache.set(path, data);
+      return data;
+    },
+
+    writeFileSync(path, data, encoding) {
+      if (typeof encoding === 'object') encoding = encoding?.encoding;
+
+      // Write to cache immediately (sync return)
+      contentCache.set(path, data);
+      const size = typeof data === 'string' ? data.length : (data.byteLength || 0);
+      metadataCache.set(path, {
+        type: 'file',
+        size,
+        mtime: Date.now(),
+        ctime: metadataCache.get(path)?.ctime || Date.now(),
+      });
+
+      // Fire-and-forget async send to server
+      transport.writeFile(path, data, encoding).catch((e) => {
+        console.error('[shim:fs] writeFileSync background save failed:', path, e);
+      });
+    },
+
+    unlinkSync(path) {
+      contentCache.delete(path);
+      metadataCache.delete(path);
+
+      // Fire-and-forget
+      transport.unlink(path).catch((e) => {
+        console.error('[shim:fs] unlinkSync background delete failed:', path, e);
+      });
+    },
+
+    readdirSync(path) {
+      const entries = metadataCache.readdir(path);
+      return entries.map(e => e.name);
+    },
+  };
+}

shims/fs/transport.js

@@ -0,0 +1,205 @@
+// Transport abstraction layer
+// Decouples the fs shim from the sync mechanism (REST, WebSocket, or hybrid).
+// Currently implements a REST-based transport. This can be swapped or extended
+// once the sync strategy is finalized.
+
+const API_BASE = "/api/fs";
+
+// Strip leading slashes from paths before sending to server
+function normPath(p) {
+  return (p || "").replace(/^\/+/, "");
+}
+
+async function request(method, endpoint, params = {}) {
+  const url = new URL(API_BASE + endpoint, window.location.origin);
+
+  const options = { method };
+
+  if (method === "GET" || method === "DELETE") {
+    for (const [key, val] of Object.entries(params)) {
+      url.searchParams.set(key, val);
+    }
+  } else {
+    options.headers = { "Content-Type": "application/json" };
+    options.body = JSON.stringify(params);
+  }
+
+  const res = await fetch(url.toString(), options);
+  if (!res.ok) {
+    const err = await res
+      .json()
+      .catch(() => ({ error: res.statusText, code: "UNKNOWN" }));
+    const e = new Error(err.error || res.statusText);
+    e.code = err.code || "UNKNOWN";
+    throw e;
+  }
+  return res;
+}
+
+async function requestJson(method, endpoint, params = {}) {
+  const res = await request(method, endpoint, params);
+  return res.json();
+}
+
+// Synchronous XHR  -  used only as fallback for sync fs calls on uncached content.
+// Blocking but functional. Should be rare after pre-warming.
+function requestSync(method, endpoint, params = {}) {
+  const url = new URL(API_BASE + endpoint, window.location.origin);
+
+  if (method === "GET") {
+    for (const [key, val] of Object.entries(params)) {
+      url.searchParams.set(key, val);
+    }
+  }
+
+  const xhr = new XMLHttpRequest();
+  xhr.open(method, url.toString(), false); // synchronous
+
+  if (method !== "GET") {
+    xhr.setRequestHeader("Content-Type", "application/json");
+    xhr.send(JSON.stringify(params));
+  } else {
+    xhr.send();
+  }
+
+  if (xhr.status >= 400) {
+    let err;
+    try {
+      const body = JSON.parse(xhr.responseText);
+      err = new Error(body.error || "Request failed");
+      err.code = body.code || "UNKNOWN";
+    } catch {
+      err = new Error("Request failed: " + xhr.status);
+      err.code = "UNKNOWN";
+    }
+    throw err;
+  }
+
+  return xhr;
+}
+
+export const transport = {
+  // --- Async methods (used by fs.promises) ---
+
+  async fetchTree(basePath) {
+    return requestJson("GET", "/tree", basePath ? { path: basePath } : {});
+  },
+
+  async stat(path) {
+    return requestJson("GET", "/stat", { path: normPath(path) });
+  },
+
+  async readdir(path) {
+    return requestJson("GET", "/readdir", { path: normPath(path) });
+  },
+
+  async readFile(path, encoding) {
+    const res = await request("GET", "/readFile", {
+      path: normPath(path),
+      encoding: encoding || "",
+    });
+    if (encoding === "utf8" || encoding === "utf-8") {
+      return res.text();
+    }
+    const buf = await res.arrayBuffer();
+    return new Uint8Array(buf);
+  },
+
+  async writeFile(path, content, encoding) {
+    const isText = typeof content === "string";
+    return requestJson("POST", "/writeFile", {
+      path: normPath(path),
+      content: isText ? content : btoa(String.fromCharCode(...content)),
+      encoding: encoding || (isText ? "utf-8" : "binary"),
+      base64: !isText,
+    });
+  },
+
+  async appendFile(path, content) {
+    return requestJson("POST", "/appendFile", {
+      path: normPath(path),
+      content,
+    });
+  },
+
+  async mkdir(path, recursive) {
+    return requestJson("POST", "/mkdir", { path: normPath(path), recursive });
+  },
+
+  async rename(oldPath, newPath) {
+    return requestJson("POST", "/rename", {
+      oldPath: normPath(oldPath),
+      newPath: normPath(newPath),
+    });
+  },
+
+  async copyFile(src, dest) {
+    return requestJson("POST", "/copyFile", {
+      src: normPath(src),
+      dest: normPath(dest),
+    });
+  },
+
+  async unlink(path) {
+    return requestJson("DELETE", "/unlink", { path: normPath(path) });
+  },
+
+  async rmdir(path) {
+    return requestJson("DELETE", "/rmdir", { path: normPath(path) });
+  },
+
+  async rm(path, recursive) {
+    return requestJson("DELETE", "/rm", {
+      path: normPath(path),
+      recursive: recursive ? "true" : "false",
+    });
+  },
+
+  async access(path) {
+    return requestJson("GET", "/access", { path: normPath(path) });
+  },
+
+  async realpath(path) {
+    const result = await requestJson("GET", "/realpath", {
+      path: normPath(path),
+    });
+    return result.path;
+  },
+
+  async utimes(path, atime, mtime) {
+    return requestJson("POST", "/utimes", {
+      path: normPath(path),
+      atime,
+      mtime,
+    });
+  },
+
+  // --- Sync methods (fallback) ---
+
+  readFileSync(path, encoding) {
+    const xhr = requestSync("GET", "/readFile", {
+      path: normPath(path),
+      encoding: encoding || "",
+    });
+    if (encoding === "utf8" || encoding === "utf-8") {
+      return xhr.responseText;
+    }
+    // Binary: return as Uint8Array
+    const binary = xhr.responseText;
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+      bytes[i] = binary.charCodeAt(i);
+    }
+    return bytes;
+  },
+
+  writeFileSync(path, content, encoding) {
+    const isText = typeof content === "string";
+    requestSync("POST", "/writeFile", {
+      path: normPath(path),
+      content: isText ? content : btoa(String.fromCharCode(...content)),
+      encoding: encoding || (isText ? "utf-8" : "binary"),
+      base64: !isText,
+    });
+  },
+};

shims/fs/watch.js

@@ -0,0 +1,57 @@
+// File watching shim
+// Translates fs.watch() calls into WebSocket subscriptions.
+// The server pushes file-change events; this module dispatches them
+// to registered watch listeners.
+
+export function createFsWatch(transport) {
+  const watchers = new Map(); // path -> Set<listener>
+
+  return {
+    watch(path, options, listener) {
+      if (typeof options === 'function') {
+        listener = options;
+        options = {};
+      }
+
+      if (!watchers.has(path)) {
+        watchers.set(path, new Set());
+      }
+      watchers.get(path).add(listener);
+
+      // TODO: send watch subscription to server via transport
+
+      // Return a watcher-like object
+      return {
+        close() {
+          const set = watchers.get(path);
+          if (set) {
+            set.delete(listener);
+            if (set.size === 0) {
+              watchers.delete(path);
+              // TODO: send unwatch to server
+            }
+          }
+        },
+        on() { return this; },
+        once() { return this; },
+        removeListener() { return this; },
+      };
+    },
+
+    // Internal: called when transport receives a file-change event
+    _dispatch(eventType, filePath) {
+      for (const [watchPath, listeners] of watchers) {
+        if (filePath === watchPath || filePath.startsWith(watchPath + '/')) {
+          const relativeName = filePath.slice(watchPath.length + 1) || filePath;
+          for (const fn of listeners) {
+            try {
+              fn(eventType, relativeName);
+            } catch (e) {
+              console.error('[shim:fs:watch] Listener error:', e);
+            }
+          }
+        }
+      }
+    },
+  };
+}