mirror of
https://github.com/only-cli/oc.git
synced 2026-09-15 10:40:56 +02:00
only-cli
6a399cf448
nodejs.org has no search results page: the site's search box is a JavaScript modal asking a third-party service, so the search verb went through DuckDuckGo. But the API docs publish their entire reference as one static JSON file, all.json, much the way a Sphinx site publishes its search index, so a new nodedoc backend ranks that file locally: every module, class, method, property, and event heading becomes a result linking to its own anchor, and oc node search prints them as a normal numbered page. The file is ~8MB (~1MB over the wire) and static, so the day cache the Sphinx backend used moves to a shared cache module both backends call: one directory per backend, one file per host, parsed before written so a block page never poisons it. A typical result list costs under 100 tokens and answers from disk in under 100ms once cached.
44 lines
1.8 KiB
JavaScript
44 lines
1.8 KiB
JavaScript
/**
|
|
* Day cache for the big static files a local search ranks: a Sphinx site's
|
|
* searchindex.js, the Node.js docs' all.json. Each is megabytes over the
|
|
* wire but rebuilds at most a few times a day, and a stale result list still
|
|
* links to live pages, so a day-old copy is a fair trade against moving the
|
|
* file again on every search.
|
|
*/
|
|
|
|
import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
import { homedir } from 'node:os';
|
|
import { extname, join } from 'node:path';
|
|
import { fetchPage } from './fetch.js';
|
|
|
|
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
|
|
/**
|
|
* The file at `url`, parsed, from the disk cache while it is fresh and from
|
|
* the network otherwise. One directory per backend, one file per host, kept
|
|
* under the URL's own extension so the cache directory reads plainly. The
|
|
* file is parsed before it is written, so a block page or an error never
|
|
* poisons the cache, and a cache that cannot be written costs nothing but
|
|
* the refetch, the same policy session state follows.
|
|
* @param {string} kind - cache subdirectory, one per backend ('sphinx')
|
|
* @param {string} url
|
|
* @param {(text: string) => any} parse - throws on anything but the real file
|
|
* @returns {Promise<{data: any, via: 'cache'|'network'}>}
|
|
*/
|
|
export async function cachedFile(kind, url, parse) {
|
|
const dir = join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), kind);
|
|
const file = join(dir, `${new URL(url).host}${extname(new URL(url).pathname)}`);
|
|
try {
|
|
if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) {
|
|
return { data: parse(readFileSync(file, 'utf8')), via: 'cache' };
|
|
}
|
|
} catch {}
|
|
const { html } = await fetchPage(url);
|
|
const data = parse(html);
|
|
try {
|
|
mkdirSync(dir, { recursive: true });
|
|
writeFileSync(file, html);
|
|
} catch {}
|
|
return { data, via: 'network' };
|
|
}
|