feat: rank Ruby docs search locally via the RDoc index

RDoc publishes a site's whole search index as one static JS file, the same way Sphinx and the Node.js docs do, so 'oc ruby search' fetches docs.ruby-lang.org's index (en/3.4, where the file exists), caches it under the shared day cache, and ranks it with the same scorer the Node backend uses. Class methods print as Class.name(params), instance methods as Class#name(params), and every entry links straight to its method anchor. 'oc ruby class Array' opens a class page directly. cli.js now dispatches the local search backends through one map, and the site-resolution guard keys on the query field, which only a search shape ever carries.
2026-09-15 10:40:56 +02:00 · 2026-08-24 12:27:05 -04:00
parent 6a399cf448
commit fa41abbb2e
5 changed files with 211 additions and 22 deletions
@@ -0,0 +1,7 @@
+{
+  "domain": "docs.ruby-lang.org",
+  "commands": {
+    "class": { "open": "https://docs.ruby-lang.org/en/3.4/{class}.html", "args": ["class"] },
+    "search": { "rdoc": "https://docs.ruby-lang.org/en/3.4/", "args": ["query"] }
+  }
+}
@@ -6,6 +6,7 @@ import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } fr
 import { resolveSite, listSites } from './sites.js';
 import { sphinxSearch } from './sphinx.js';
 import { nodeSearch } from './nodedocs.js';
+import { rdocSearch } from './rdoc.js';
 import { apiSearch } from './apisearch.js';
 import * as act from './act.js';
 import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
@@ -118,7 +119,8 @@ async function main() {
  if (!COMMANDS.has(command)) {
    const site = resolveSite(command, args);
    if (!site) throw new Error(`unknown command '${command}', run oc --help`);
-    if (site.sphinx || site.nodedoc || site.api) {
+    // Only a search shape resolves with a query; a URL shape never has one.
+    if (site.query != null) {
      search = site;
      command = 'search';
    } else {
@@ -212,18 +214,18 @@ async function main() {
      return;
    }
    case 'search': {
-      // A search oc runs itself: a Sphinx site's index or the Node.js docs
-      // corpus is fetched (or read back from its day cache) and ranked here,
-      // a JSON search API is asked directly. Either way the result list rides
-      // the exact `open` path: distilled, rendered, remembered, so `do <n>`
+      // A search oc runs itself: a site's static index or docs corpus is
+      // fetched (or read back from its day cache) and ranked here, a JSON
+      // search API is asked directly. Either way the result list rides the
+      // exact `open` path: distilled, rendered, remembered, so `do <n>`
      // follows a result. Only the list is ever printed; the index, corpus,
      // and response stay out of context.
      const t0 = performance.now();
-      const { url, html, via } = search.sphinx
-        ? await sphinxSearch(search.sphinx, search.query)
-        : search.nodedoc
-          ? await nodeSearch(search.nodedoc, search.query)
-          : await apiSearch(search.api, search.query);
+      const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch };
+      const kind = Object.keys(local).find((k) => search[k]);
+      const { url, html, via } = kind
+        ? await local[kind](search[kind], search.query)
+        : await apiSearch(search.api, search.query);
      const page = distill(html, url);
      if (values.json) {
        remember(page, sessionName);
@@ -0,0 +1,98 @@
+/**
+ * RDoc search backend. The Ruby docs (docs.ruby-lang.org) are built with
+ * RDoc, which like Sphinx has no search server: the generated site ships its
+ * whole search index as one static file, js/search_index.js, and matches in
+ * the visitor's browser. So `search` ranks that file locally: every class,
+ * module, method, and guide page in the index becomes a result linking to
+ * its own anchor. The file is ~3.4MB (~560KB over the wire) and static, so
+ * it lives in the same day cache the other local backends use, and what
+ * reaches the agent is the ranked result list only.
+ */
+
+import { cachedFile } from './cache.js';
+import { escapeHTML } from './sphinx.js';
+import { searchEntries } from './nodedocs.js';
+
+const MAX_RESULTS = 20;
+
+/**
+ * The index file is `var search_data = {...}`: JSON behind one assignment
+ * for the browser's benefit. Anything that does not parse that way, or that
+ * lacks the info rows, is not an RDoc index, which on a wrong or moved URL
+ * is the honest error, and it keeps a block page out of the cache too.
+ * @param {string} js
+ * @returns {any}
+ */
+export function parseRdocIndex(js) {
+  const start = js.indexOf('=');
+  if (start >= 0) {
+    try {
+      const data = JSON.parse(js.slice(start + 1));
+      if (Array.isArray(data?.index?.info)) return data;
+    } catch {}
+  }
+  throw new Error('not an RDoc search index');
+}
+
+/**
+ * Flatten the index's info rows into the entries the shared ranker scores.
+ * A row is [name, namespace, path, params, snippet]; the path's own anchor
+ * says what the row is, so 'dig' in 'Array' with anchor method-i-dig reads
+ * back as the heading a rubyist expects, Array#dig(*args). Snippets stay
+ * behind: matching on them would rank prose over the symbol asked for.
+ * @param {any} data - parsed search_index.js
+ * @returns {{text: string, name: string, kind: string, path: string}[]}
+ */
+export function buildRdocEntries(data) {
+  return data.index.info.map(([name, namespace, path, params]) => {
+    const p = String(path ?? '');
+    const kind = p.includes('#method-c-') ? 'class method'
+      : p.includes('#method-i-') ? 'method'
+        : /^[A-Z]/.test(String(name)) ? 'class' : 'page';
+    const text = kind === 'class method' ? `${namespace}.${name}${params}`
+      : kind === 'method' ? `${namespace}#${name}${params}`
+        : namespace ? `${namespace}::${name}` : String(name ?? '');
+    return { text, name: String(name ?? ''), kind, path: p };
+  }).filter((e) => e.text && e.path);
+}
+
+/**
+ * The result list becomes the same small synthetic page the other search
+ * backends emit, so it distills, renders, numbers, and remembers like any
+ * fetched page and `do <n>` follows a result.
+ * @param {string} base
+ * @param {string} query
+ * @param {ReturnType<typeof searchEntries>} found
+ * @returns {string}
+ */
+export function resultsToHTML(base, query, found) {
+  const host = new URL(base).host;
+  const items = found.hits.map((e) =>
+    `<li><a href="${escapeHTML(new URL(e.path, base).href)}">${escapeHTML(e.text)}</a>`
+    + ` ${escapeHTML(e.kind)}</li>`);
+  const partial = found.partial ? '; no entry matches every word, so these match some' : '';
+  const summary = items.length
+    ? `${found.total} entr${found.total === 1 ? 'y matches' : 'ies match'} in the docs' own index,`
+      + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
+    : `nothing in the docs' own index matches; try fewer or different words`;
+  return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
+    + `<p>${summary}</p>`
+    + (items.length ? `<ol>${items.join('')}</ol>` : '')
+    + `</main></body></html>`;
+}
+
+/**
+ * Search one RDoc site. The site has no search URL of its own to remember,
+ * so the session keeps the docs root, the page a reader would start from.
+ * @param {string} base - docs root ending in '/', e.g. https://docs.ruby-lang.org/en/3.4/
+ * @param {string} query
+ */
+export async function rdocSearch(base, query) {
+  if (!query.trim()) throw new Error('usage: search <query>');
+  const { data, via } = await cachedFile('rdoc', new URL('js/search_index.js', base).href, parseRdocIndex);
+  return {
+    url: new URL('index.html', base).href,
+    html: resultsToHTML(base, query, searchEntries(buildRdocEntries(data), query)),
+    via,
+  };
+}
@@ -5,9 +5,10 @@
 * resolves to one and hands off to the same fetch and render path `oc open`
 * uses, so nothing here can change what a page costs or how it reads. The
 * other shapes are searches cli.js runs itself and renders like any other
- * page: `sphinx`, for a docs site whose search only exists as a static index
- * file, `nodedoc`, for the Node.js API docs, which ship their reference the
- * same way, and `api`, for a site whose search answers as JSON.
+ * page: `sphinx` and `rdoc`, for docs sites whose search only exists as a
+ * static index file, `nodedoc`, for the Node.js API docs, which ship their
+ * reference the same way, and `api`, for a site whose search answers as
+ * JSON.
 */

 import { readdirSync, readFileSync } from 'node:fs';
@@ -29,12 +30,17 @@ const ALIASES = {
  py: 'docs.python.org',
  mdn: 'developer.mozilla.org',
  node: 'nodejs.org',
+  rust: 'doc.rust-lang.org',
+  java: 'docs.oracle.com',
+  ruby: 'docs.ruby-lang.org',
+  cpp: 'en.cppreference.com',
+  ts: 'typescriptlang.org',
  gcp: 'cloud.google.com',
  learn: 'learn.microsoft.com',
  wiki: 'wikipedia.org',
 };

-/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
+/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
 /** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */

 /** @type {Map<string, Site>|null} */
@@ -91,7 +97,7 @@ const verbs = (site) =>
 * instead, since the agent has the right site and only needs the verb list.
 * @param {string} name
 * @param {string[]} args
- * @returns {{url?: string, sphinx?: string, nodedoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
+ * @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
 */
 export function resolveSite(name, args) {
  const site = sites().get(name.toLowerCase());
@@ -109,13 +115,14 @@ export function resolveSite(name, args) {
  // separate words ('oc ddg search claude code cli') works unquoted.
  const values = need.map((_, i) =>
    i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]);
-  // A sphinx, nodedoc, or API search has no page URL to build: the query is
-  // handed back whole for cli.js to run against the site's own search.
-  if (def.sphinx) {
-    return { sphinx: def.sphinx, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
-  }
-  if (def.nodedoc) {
-    return { nodedoc: def.nodedoc, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
+  // A search oc runs itself has no page URL to build: the query is handed
+  // back whole for cli.js to run against the site's own search. The local
+  // backends need only their docs root; the API shape needs its whole
+  // definition, since it names the endpoint and the response fields.
+  for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
+    if (def[kind]) {
+      return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
+    }
  }
  if (def.api) {
    return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
@@ -0,0 +1,75 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+const { parseRdocIndex, buildRdocEntries, resultsToHTML } = await import('../src/rdoc.js');
+const { searchEntries } = await import('../src/nodedocs.js');
+
+const BASE = 'https://docs.ruby-lang.org/en/3.4/';
+
+// A miniature search_index.js: a class page, instance and class methods, a
+// name shared across classes, a guide page, and a snippet that would be
+// markup if it were ever trusted. Rows are [name, namespace, path, params,
+// snippet], the shape RDoc generates.
+const INDEX = `var search_data = ${JSON.stringify({
+  index: {
+    searchIndex: ['array', 'dig', 'dig', 'new', 'each_slice', 'contributing', 'evil'],
+    longSearchIndex: ['array', 'array::dig', 'hash::dig', 'array::new', 'array::each_slice', 'contributing', 'evil'],
+    info: [
+      ['Array', '', 'Array.html', '', '<p>An Array is an ordered collection.'],
+      ['dig', 'Array', 'Array.html#method-i-dig', '(*args)', '<p>Finds the object in nested objects.'],
+      ['dig', 'Hash', 'Hash.html#method-i-dig', '(*args)', '<p>Finds the object in nested objects.'],
+      ['new', 'Array', 'Array.html#method-c-new', '(size, default)', '<p>Returns a new array.'],
+      ['each_slice', 'Array', 'Array.html#method-i-each_slice', '(n)', '<p>Iterates in slices.'],
+      ['contributing', '', 'contributing_md.html', '', '<p>How to contribute.'],
+      ['evil<script>alert(1)</script>', 'Array', 'Array.html#method-i-evil', '', ''],
+    ],
+  },
+})}`;
+
+const entries = () => buildRdocEntries(parseRdocIndex(INDEX));
+
+test('only an RDoc index parses, so an error page never enters the cache', () => {
+  assert.ok(parseRdocIndex(INDEX).index.info.length > 0);
+  assert.throws(() => parseRdocIndex('<html>blocked</html>'), /not an RDoc search index/);
+  assert.throws(() => parseRdocIndex('var search_data = {"unrelated": true}'), /not an RDoc search index/);
+});
+
+test('rows read back as the headings a rubyist expects', () => {
+  const all = entries();
+  assert.equal(all.find((e) => e.path.includes('method-i-dig') && e.text.startsWith('Array')).text, 'Array#dig(*args)');
+  assert.equal(all.find((e) => e.path.includes('method-c-new')).text, 'Array.new(size, default)');
+  assert.equal(all.find((e) => e.path === 'Array.html').text, 'Array');
+  assert.equal(all.find((e) => e.path === 'contributing_md.html').kind, 'page');
+});
+
+test('a symbol query finds its methods across classes, exact name first', () => {
+  const found = searchEntries(entries(), 'dig');
+  assert.equal(found.total, 2);
+  assert.deepEqual(found.hits.map((e) => e.text).sort(), ['Array#dig(*args)', 'Hash#dig(*args)']);
+  assert.equal(found.partial, false);
+});
+
+test('a class and method pair narrows to the one entry matching both words', () => {
+  const found = searchEntries(entries(), 'array each_slice');
+  assert.equal(found.total, 1);
+  assert.equal(found.hits[0].text, 'Array#each_slice(n)');
+});
+
+test('results link into the live docs, anchors intact, and name their kind', () => {
+  const html = resultsToHTML(BASE, 'dig', searchEntries(entries(), 'dig'));
+  assert.match(html, /href="https:\/\/docs\.ruby-lang\.org\/en\/3\.4\/Array\.html#method-i-dig"/);
+  assert.match(html, /Array#dig\(\*args\)<\/a> method/);
+  assert.match(html, /2 entries match in the docs' own index, ranked locally:/);
+});
+
+test('an index row is data, never markup on the results page', () => {
+  const html = resultsToHTML(BASE, 'evil', searchEntries(entries(), 'evil'));
+  assert.doesNotMatch(html, /<script/);
+  assert.match(html, /evil&lt;script&gt;/);
+});
+
+test('nothing matching renders an honest empty page, not an error', () => {
+  const html = resultsToHTML(BASE, 'zzqqxx', searchEntries(entries(), 'zzqqxx'));
+  assert.match(html, /nothing in the docs' own index matches/);
+  assert.doesNotMatch(html, /<ol>/);
+});