From 192c5fb093feb1a0bf311b4e5436750adb01a034 Mon Sep 17 00:00:00 2001 From: Nystik <236107-Nystik@users.noreply.gitlab.com> Date: Sat, 7 Mar 2026 15:42:19 +0100 Subject: [PATCH] implement fs shims --- shims/fs/constants.js | 20 ++++ shims/fs/content-cache.js | 91 ++++++++++++++++ shims/fs/index.js | 52 ++++++++++ shims/fs/metadata-cache.js | 116 +++++++++++++++++++++ shims/fs/promises.js | 163 +++++++++++++++++++++++++++++ shims/fs/sync.js | 81 +++++++++++++++ shims/fs/transport.js | 205 +++++++++++++++++++++++++++++++++++++ shims/fs/watch.js | 57 +++++++++++ 8 files changed, 785 insertions(+) create mode 100644 shims/fs/constants.js create mode 100644 shims/fs/content-cache.js create mode 100644 shims/fs/index.js create mode 100644 shims/fs/metadata-cache.js create mode 100644 shims/fs/promises.js create mode 100644 shims/fs/sync.js create mode 100644 shims/fs/transport.js create mode 100644 shims/fs/watch.js diff --git a/shims/fs/constants.js b/shims/fs/constants.js new file mode 100644 index 0000000..61f97fc --- /dev/null +++ b/shims/fs/constants.js @@ -0,0 +1,20 @@ +// Node.js fs.constants equivalents + +export const constants = { + F_OK: 0, + R_OK: 4, + W_OK: 2, + X_OK: 1, + + COPYFILE_EXCL: 1, + COPYFILE_FICLONE: 2, + COPYFILE_FICLONE_FORCE: 4, + + O_RDONLY: 0, + O_WRONLY: 1, + O_RDWR: 2, + O_CREAT: 64, + O_EXCL: 128, + O_TRUNC: 512, + O_APPEND: 1024, +}; diff --git a/shims/fs/content-cache.js b/shims/fs/content-cache.js new file mode 100644 index 0000000..86a883b --- /dev/null +++ b/shims/fs/content-cache.js @@ -0,0 +1,91 @@ +// In-memory content cache with simple LRU eviction +// Stores file content fetched from the server. + +const DEFAULT_MAX_SIZE = 50 * 1024 * 1024; // 50 MB + +export class ContentCache { + constructor(maxSize = DEFAULT_MAX_SIZE) { + this._cache = new Map(); // path -> { data, size, accessedAt } + this._currentSize = 0; + this._maxSize = maxSize; + } + + has(path) { + return this._cache.has(this._normalize(path)); + } + + get(path) { + const entry = this._cache.get(this._normalize(path)); + if (entry) { + entry.accessedAt = Date.now(); + return entry.data; + } + return null; + } + + set(path, data) { + const norm = this._normalize(path); + const size = data ? data.length || data.byteLength || 0 : 0; + + // Remove old entry if replacing + if (this._cache.has(norm)) { + this._currentSize -= this._cache.get(norm).size; + } + + // Evict LRU entries if needed + while (this._currentSize + size > this._maxSize && this._cache.size > 0) { + this._evictOne(); + } + + this._cache.set(norm, { data, size, accessedAt: Date.now() }); + this._currentSize += size; + } + + delete(path) { + const norm = this._normalize(path); + const entry = this._cache.get(norm); + if (entry) { + this._currentSize -= entry.size; + this._cache.delete(norm); + } + } + + // Invalidate a path (remove from cache so next read fetches fresh) + invalidate(path) { + this.delete(path); + } + + clear() { + this._cache.clear(); + this._currentSize = 0; + } + + get size() { + return this._cache.size; + } + + get currentBytes() { + return this._currentSize; + } + + _evictOne() { + let oldest = null; + let oldestTime = Infinity; + for (const [key, entry] of this._cache) { + if (entry.accessedAt < oldestTime) { + oldest = key; + oldestTime = entry.accessedAt; + } + } + if (oldest) { + this.delete(oldest); + } + } + + _normalize(p) { + return (p || "") + .replace(/\\/g, "/") + .replace(/^\/+/, "") + .replace(/\/+$/, ""); + } +} diff --git a/shims/fs/index.js b/shims/fs/index.js new file mode 100644 index 0000000..4bad27d --- /dev/null +++ b/shims/fs/index.js @@ -0,0 +1,52 @@ +// Filesystem shim - the core piece +// Returned for both require('original-fs') and require('fs') +// +// Strategy: metadata cache + on-demand content fetch + write-through +// Server sync mechanism (REST vs WebSocket) is TBD - abstracted behind +// the transport layer in ./transport.js + +import { MetadataCache } from './metadata-cache.js'; +import { ContentCache } from './content-cache.js'; +import { transport } from './transport.js'; +import { createFsPromises } from './promises.js'; +import { createFsSync } from './sync.js'; +import { createFsWatch } from './watch.js'; +import { constants } from './constants.js'; + +const metadataCache = new MetadataCache(); +const contentCache = new ContentCache(); + +const fsPromises = createFsPromises(metadataCache, contentCache, transport); +const fsSync = createFsSync(metadataCache, contentCache, transport); +const fsWatch = createFsWatch(transport); + +export const fsShim = { + // Async promise-based API (this.fsPromises = this.fs.promises) + promises: fsPromises, + + // Sync methods + existsSync: fsSync.existsSync, + readFileSync: fsSync.readFileSync, + writeFileSync: fsSync.writeFileSync, + unlinkSync: fsSync.unlinkSync, + accessSync: fsSync.accessSync, + statSync: fsSync.statSync, + readdirSync: fsSync.readdirSync, + + // Watch + watch: fsWatch.watch, + + // Constants + constants, + + // Internal: for initialization + _metadataCache: metadataCache, + _contentCache: contentCache, + + // Initialize the caches by fetching the full tree from server + async _init(basePath) { + const tree = await transport.fetchTree(basePath); + metadataCache.populate(tree); + console.log(`[shim:fs] Initialized with ${metadataCache.size} entries`); + }, +}; diff --git a/shims/fs/metadata-cache.js b/shims/fs/metadata-cache.js new file mode 100644 index 0000000..b421972 --- /dev/null +++ b/shims/fs/metadata-cache.js @@ -0,0 +1,116 @@ +// In-memory metadata cache +// Populated from /api/fs/tree on startup, kept in sync via transport events. +// All stat/exists/readdir calls are served from this cache (zero latency). + +export class MetadataCache { + constructor() { + // Map + this._entries = new Map(); + } + + // Populate from a server-provided tree object + // tree shape: { "relative/path": { type, size, mtime, ctime }, ... } + populate(tree) { + this._entries.clear(); + for (const [path, meta] of Object.entries(tree)) { + this._entries.set(this._normalize(path), meta); + } + } + + has(path) { + return this._entries.has(this._normalize(path)); + } + + get(path) { + return this._entries.get(this._normalize(path)) || null; + } + + set(path, meta) { + this._entries.set(this._normalize(path), meta); + } + + delete(path) { + this._entries.delete(this._normalize(path)); + } + + // Rename: move metadata from old path to new path (and children if directory) + rename(oldPath, newPath) { + const oldNorm = this._normalize(oldPath); + const newNorm = this._normalize(newPath); + const meta = this._entries.get(oldNorm); + if (meta) { + this._entries.delete(oldNorm); + this._entries.set(newNorm, meta); + } + // Move children + const prefix = oldNorm + "/"; + for (const [key, val] of this._entries) { + if (key.startsWith(prefix)) { + const newKey = newNorm + "/" + key.slice(prefix.length); + this._entries.delete(key); + this._entries.set(newKey, val); + } + } + } + + // List direct children of a directory path + readdir(dirPath) { + const norm = this._normalize(dirPath); + const prefix = norm === "" ? "" : norm + "/"; + const results = []; + const seen = new Set(); + + for (const [key, meta] of this._entries) { + if (prefix === "" || key.startsWith(prefix)) { + const rest = key.slice(prefix.length); + const slashIdx = rest.indexOf("/"); + const childName = slashIdx >= 0 ? rest.slice(0, slashIdx) : rest; + if (childName && !seen.has(childName)) { + seen.add(childName); + const childMeta = this._entries.get(prefix + childName); + results.push({ + name: childName, + type: childMeta?.type || (slashIdx >= 0 ? "directory" : "file"), + }); + } + } + } + return results; + } + + get size() { + return this._entries.size; + } + + // Build a stat-like object from metadata + toStat(path) { + const meta = this.get(path); + if (!meta) return null; + return { + size: meta.size || 0, + mtimeMs: meta.mtime || 0, + ctimeMs: meta.ctime || 0, + atimeMs: meta.mtime || 0, + birthtimeMs: meta.ctime || 0, + mtime: new Date(meta.mtime || 0), + ctime: new Date(meta.ctime || 0), + atime: new Date(meta.mtime || 0), + birthtime: new Date(meta.ctime || 0), + isFile: () => meta.type === "file", + isDirectory: () => meta.type === "directory", + isSymbolicLink: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isFIFO: () => false, + isSocket: () => false, + }; + } + + _normalize(p) { + // Normalize slashes, remove leading and trailing slashes + return (p || "") + .replace(/\\/g, "/") + .replace(/^\/+/, "") + .replace(/\/+$/, ""); + } +} diff --git a/shims/fs/promises.js b/shims/fs/promises.js new file mode 100644 index 0000000..a9be4ff --- /dev/null +++ b/shims/fs/promises.js @@ -0,0 +1,163 @@ +// Async fs.promises implementation +// Maps to transport layer (REST/WebSocket/hybrid - TBD) + +export function createFsPromises(metadataCache, contentCache, transport) { + return { + async stat(path) { + // Try cache first, fall back to server + const cached = metadataCache.toStat(path); + if (cached) return cached; + + const meta = await transport.stat(path); + metadataCache.set(path, meta); + return metadataCache.toStat(path); + }, + + async lstat(path) { + // No symlinks in our context - same as stat + return this.stat(path); + }, + + async readdir(path) { + // If metadata cache knows this is a file, return empty (ENOTDIR) + const meta = metadataCache.get(path); + if (meta && meta.type === "file") { + return []; + } + // Serve from metadata cache + const entries = metadataCache.readdir(path); + if (entries.length > 0) { + return entries.map((e) => e.name); + } + // Fallback to server + const serverEntries = await transport.readdir(path); + return serverEntries.map((e) => e.name); + }, + + async readFile(path, encoding) { + if (typeof encoding === "object") encoding = encoding?.encoding; + const wantText = encoding === "utf8" || encoding === "utf-8"; + + // Check content cache + const cached = contentCache.get(path); + if (cached !== null) { + if (wantText) { + return typeof cached === "string" + ? cached + : new TextDecoder().decode(cached); + } + // Binary mode: ensure we return a proper Uint8Array with .buffer + if (typeof cached === "string") { + return new TextEncoder().encode(cached); + } + return cached; + } + + // Fetch from server + const data = await transport.readFile(path, encoding); + contentCache.set(path, data); + return data; + }, + + async writeFile(path, data, encoding) { + if (typeof encoding === "object") encoding = encoding?.encoding; + + // Update caches optimistically + contentCache.set(path, data); + const size = + typeof data === "string" ? data.length : data.byteLength || 0; + metadataCache.set(path, { + type: "file", + size, + mtime: Date.now(), + ctime: metadataCache.get(path)?.ctime || Date.now(), + }); + + // Send to server + const result = await transport.writeFile(path, data, encoding); + // Update metadata with server-confirmed values + if (result.mtime) { + metadataCache.set(path, { + type: "file", + size: result.size || size, + mtime: result.mtime, + ctime: metadataCache.get(path)?.ctime || Date.now(), + }); + } + }, + + async appendFile(path, data, encoding) { + contentCache.invalidate(path); + await transport.appendFile(path, data); + // Refresh metadata + const meta = await transport.stat(path); + metadataCache.set(path, meta); + }, + + async unlink(path) { + contentCache.delete(path); + metadataCache.delete(path); + await transport.unlink(path); + }, + + async rename(oldPath, newPath) { + // Move content cache entry + const content = contentCache.get(oldPath); + if (content !== null) { + contentCache.set(newPath, content); + contentCache.delete(oldPath); + } + // Move metadata + metadataCache.rename(oldPath, newPath); + + await transport.rename(oldPath, newPath); + }, + + async mkdir(path, options) { + const recursive = + typeof options === "object" ? !!options.recursive : !!options; + metadataCache.set(path, { type: "directory" }); + await transport.mkdir(path, recursive); + }, + + async rmdir(path) { + metadataCache.delete(path); + await transport.rmdir(path); + }, + + async rm(path, options) { + const recursive = + typeof options === "object" ? !!options.recursive : false; + metadataCache.delete(path); + contentCache.delete(path); + await transport.rm(path, recursive); + }, + + async copyFile(src, dest) { + await transport.copyFile(src, dest); + // Refresh metadata for dest + const meta = await transport.stat(dest); + metadataCache.set(dest, meta); + }, + + async access(path) { + if (metadataCache.has(path)) return; + await transport.access(path); + }, + + async realpath(path) { + // Empty path = vault root, return the vault base path + if (!path || path === "/" || path === ".") return "/"; + return transport.realpath(path); + }, + + async utimes(path, atime, mtime) { + await transport.utimes(path, atime, mtime); + const meta = metadataCache.get(path); + if (meta) { + meta.mtime = typeof mtime === "number" ? mtime : mtime.getTime(); + metadataCache.set(path, meta); + } + }, + }; +} diff --git a/shims/fs/sync.js b/shims/fs/sync.js new file mode 100644 index 0000000..ceb6dc3 --- /dev/null +++ b/shims/fs/sync.js @@ -0,0 +1,81 @@ +// Synchronous fs method implementations +// Served from caches where possible, sync XHR fallback for uncached content. + +export function createFsSync(metadataCache, contentCache, transport) { + return { + existsSync(path) { + return metadataCache.has(path); + }, + + statSync(path) { + const stat = metadataCache.toStat(path); + if (!stat) { + const err = new Error(`ENOENT: no such file or directory, stat '${path}'`); + err.code = 'ENOENT'; + throw err; + } + return stat; + }, + + accessSync(path, mode) { + if (!metadataCache.has(path)) { + const err = new Error(`ENOENT: no such file or directory, access '${path}'`); + err.code = 'ENOENT'; + throw err; + } + }, + + readFileSync(path, encoding) { + if (typeof encoding === 'object') encoding = encoding?.encoding; + + // Try content cache first + const cached = contentCache.get(path); + if (cached !== null) { + if (encoding === 'utf8' || encoding === 'utf-8') { + return typeof cached === 'string' ? cached : new TextDecoder().decode(cached); + } + return cached; + } + + // Fallback: synchronous XHR + console.warn('[shim:fs] readFileSync cache miss, using sync XHR:', path); + const data = transport.readFileSync(path, encoding); + contentCache.set(path, data); + return data; + }, + + writeFileSync(path, data, encoding) { + if (typeof encoding === 'object') encoding = encoding?.encoding; + + // Write to cache immediately (sync return) + contentCache.set(path, data); + const size = typeof data === 'string' ? data.length : (data.byteLength || 0); + metadataCache.set(path, { + type: 'file', + size, + mtime: Date.now(), + ctime: metadataCache.get(path)?.ctime || Date.now(), + }); + + // Fire-and-forget async send to server + transport.writeFile(path, data, encoding).catch((e) => { + console.error('[shim:fs] writeFileSync background save failed:', path, e); + }); + }, + + unlinkSync(path) { + contentCache.delete(path); + metadataCache.delete(path); + + // Fire-and-forget + transport.unlink(path).catch((e) => { + console.error('[shim:fs] unlinkSync background delete failed:', path, e); + }); + }, + + readdirSync(path) { + const entries = metadataCache.readdir(path); + return entries.map(e => e.name); + }, + }; +} diff --git a/shims/fs/transport.js b/shims/fs/transport.js new file mode 100644 index 0000000..a425260 --- /dev/null +++ b/shims/fs/transport.js @@ -0,0 +1,205 @@ +// Transport abstraction layer +// Decouples the fs shim from the sync mechanism (REST, WebSocket, or hybrid). +// Currently implements a REST-based transport. This can be swapped or extended +// once the sync strategy is finalized. + +const API_BASE = "/api/fs"; + +// Strip leading slashes from paths before sending to server +function normPath(p) { + return (p || "").replace(/^\/+/, ""); +} + +async function request(method, endpoint, params = {}) { + const url = new URL(API_BASE + endpoint, window.location.origin); + + const options = { method }; + + if (method === "GET" || method === "DELETE") { + for (const [key, val] of Object.entries(params)) { + url.searchParams.set(key, val); + } + } else { + options.headers = { "Content-Type": "application/json" }; + options.body = JSON.stringify(params); + } + + const res = await fetch(url.toString(), options); + if (!res.ok) { + const err = await res + .json() + .catch(() => ({ error: res.statusText, code: "UNKNOWN" })); + const e = new Error(err.error || res.statusText); + e.code = err.code || "UNKNOWN"; + throw e; + } + return res; +} + +async function requestJson(method, endpoint, params = {}) { + const res = await request(method, endpoint, params); + return res.json(); +} + +// Synchronous XHR - used only as fallback for sync fs calls on uncached content. +// Blocking but functional. Should be rare after pre-warming. +function requestSync(method, endpoint, params = {}) { + const url = new URL(API_BASE + endpoint, window.location.origin); + + if (method === "GET") { + for (const [key, val] of Object.entries(params)) { + url.searchParams.set(key, val); + } + } + + const xhr = new XMLHttpRequest(); + xhr.open(method, url.toString(), false); // synchronous + + if (method !== "GET") { + xhr.setRequestHeader("Content-Type", "application/json"); + xhr.send(JSON.stringify(params)); + } else { + xhr.send(); + } + + if (xhr.status >= 400) { + let err; + try { + const body = JSON.parse(xhr.responseText); + err = new Error(body.error || "Request failed"); + err.code = body.code || "UNKNOWN"; + } catch { + err = new Error("Request failed: " + xhr.status); + err.code = "UNKNOWN"; + } + throw err; + } + + return xhr; +} + +export const transport = { + // --- Async methods (used by fs.promises) --- + + async fetchTree(basePath) { + return requestJson("GET", "/tree", basePath ? { path: basePath } : {}); + }, + + async stat(path) { + return requestJson("GET", "/stat", { path: normPath(path) }); + }, + + async readdir(path) { + return requestJson("GET", "/readdir", { path: normPath(path) }); + }, + + async readFile(path, encoding) { + const res = await request("GET", "/readFile", { + path: normPath(path), + encoding: encoding || "", + }); + if (encoding === "utf8" || encoding === "utf-8") { + return res.text(); + } + const buf = await res.arrayBuffer(); + return new Uint8Array(buf); + }, + + async writeFile(path, content, encoding) { + const isText = typeof content === "string"; + return requestJson("POST", "/writeFile", { + path: normPath(path), + content: isText ? content : btoa(String.fromCharCode(...content)), + encoding: encoding || (isText ? "utf-8" : "binary"), + base64: !isText, + }); + }, + + async appendFile(path, content) { + return requestJson("POST", "/appendFile", { + path: normPath(path), + content, + }); + }, + + async mkdir(path, recursive) { + return requestJson("POST", "/mkdir", { path: normPath(path), recursive }); + }, + + async rename(oldPath, newPath) { + return requestJson("POST", "/rename", { + oldPath: normPath(oldPath), + newPath: normPath(newPath), + }); + }, + + async copyFile(src, dest) { + return requestJson("POST", "/copyFile", { + src: normPath(src), + dest: normPath(dest), + }); + }, + + async unlink(path) { + return requestJson("DELETE", "/unlink", { path: normPath(path) }); + }, + + async rmdir(path) { + return requestJson("DELETE", "/rmdir", { path: normPath(path) }); + }, + + async rm(path, recursive) { + return requestJson("DELETE", "/rm", { + path: normPath(path), + recursive: recursive ? "true" : "false", + }); + }, + + async access(path) { + return requestJson("GET", "/access", { path: normPath(path) }); + }, + + async realpath(path) { + const result = await requestJson("GET", "/realpath", { + path: normPath(path), + }); + return result.path; + }, + + async utimes(path, atime, mtime) { + return requestJson("POST", "/utimes", { + path: normPath(path), + atime, + mtime, + }); + }, + + // --- Sync methods (fallback) --- + + readFileSync(path, encoding) { + const xhr = requestSync("GET", "/readFile", { + path: normPath(path), + encoding: encoding || "", + }); + if (encoding === "utf8" || encoding === "utf-8") { + return xhr.responseText; + } + // Binary: return as Uint8Array + const binary = xhr.responseText; + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; + }, + + writeFileSync(path, content, encoding) { + const isText = typeof content === "string"; + requestSync("POST", "/writeFile", { + path: normPath(path), + content: isText ? content : btoa(String.fromCharCode(...content)), + encoding: encoding || (isText ? "utf-8" : "binary"), + base64: !isText, + }); + }, +}; diff --git a/shims/fs/watch.js b/shims/fs/watch.js new file mode 100644 index 0000000..5e23798 --- /dev/null +++ b/shims/fs/watch.js @@ -0,0 +1,57 @@ +// File watching shim +// Translates fs.watch() calls into WebSocket subscriptions. +// The server pushes file-change events; this module dispatches them +// to registered watch listeners. + +export function createFsWatch(transport) { + const watchers = new Map(); // path -> Set + + return { + watch(path, options, listener) { + if (typeof options === 'function') { + listener = options; + options = {}; + } + + if (!watchers.has(path)) { + watchers.set(path, new Set()); + } + watchers.get(path).add(listener); + + // TODO: send watch subscription to server via transport + + // Return a watcher-like object + return { + close() { + const set = watchers.get(path); + if (set) { + set.delete(listener); + if (set.size === 0) { + watchers.delete(path); + // TODO: send unwatch to server + } + } + }, + on() { return this; }, + once() { return this; }, + removeListener() { return this; }, + }; + }, + + // Internal: called when transport receives a file-change event + _dispatch(eventType, filePath) { + for (const [watchPath, listeners] of watchers) { + if (filePath === watchPath || filePath.startsWith(watchPath + '/')) { + const relativeName = filePath.slice(watchPath.length + 1) || filePath; + for (const fn of listeners) { + try { + fn(eventType, relativeName); + } catch (e) { + console.error('[shim:fs:watch] Listener error:', e); + } + } + } + } + }, + }; +}