diff --git a/clis/docs.ruby-lang.org.json b/clis/docs.ruby-lang.org.json new file mode 100644 index 0000000..caab335 --- /dev/null +++ b/clis/docs.ruby-lang.org.json @@ -0,0 +1,7 @@ +{ + "domain": "docs.ruby-lang.org", + "commands": { + "class": { "open": "https://docs.ruby-lang.org/en/3.4/{class}.html", "args": ["class"] }, + "search": { "rdoc": "https://docs.ruby-lang.org/en/3.4/", "args": ["query"] } + } +} diff --git a/src/cli.js b/src/cli.js index 57eb25d..4e9eec8 100755 --- a/src/cli.js +++ b/src/cli.js @@ -6,6 +6,7 @@ import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } fr import { resolveSite, listSites } from './sites.js'; import { sphinxSearch } from './sphinx.js'; import { nodeSearch } from './nodedocs.js'; +import { rdocSearch } from './rdoc.js'; import { apiSearch } from './apisearch.js'; import * as act from './act.js'; import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js'; @@ -118,7 +119,8 @@ async function main() { if (!COMMANDS.has(command)) { const site = resolveSite(command, args); if (!site) throw new Error(`unknown command '${command}', run oc --help`); - if (site.sphinx || site.nodedoc || site.api) { + // Only a search shape resolves with a query; a URL shape never has one. + if (site.query != null) { search = site; command = 'search'; } else { @@ -212,18 +214,18 @@ async function main() { return; } case 'search': { - // A search oc runs itself: a Sphinx site's index or the Node.js docs - // corpus is fetched (or read back from its day cache) and ranked here, - // a JSON search API is asked directly. Either way the result list rides - // the exact `open` path: distilled, rendered, remembered, so `do ` + // A search oc runs itself: a site's static index or docs corpus is + // fetched (or read back from its day cache) and ranked here, a JSON + // search API is asked directly. Either way the result list rides the + // exact `open` path: distilled, rendered, remembered, so `do ` // follows a result. Only the list is ever printed; the index, corpus, // and response stay out of context. const t0 = performance.now(); - const { url, html, via } = search.sphinx - ? await sphinxSearch(search.sphinx, search.query) - : search.nodedoc - ? await nodeSearch(search.nodedoc, search.query) - : await apiSearch(search.api, search.query); + const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch }; + const kind = Object.keys(local).find((k) => search[k]); + const { url, html, via } = kind + ? await local[kind](search[kind], search.query) + : await apiSearch(search.api, search.query); const page = distill(html, url); if (values.json) { remember(page, sessionName); diff --git a/src/rdoc.js b/src/rdoc.js new file mode 100644 index 0000000..588ab59 --- /dev/null +++ b/src/rdoc.js @@ -0,0 +1,98 @@ +/** + * RDoc search backend. The Ruby docs (docs.ruby-lang.org) are built with + * RDoc, which like Sphinx has no search server: the generated site ships its + * whole search index as one static file, js/search_index.js, and matches in + * the visitor's browser. So `search` ranks that file locally: every class, + * module, method, and guide page in the index becomes a result linking to + * its own anchor. The file is ~3.4MB (~560KB over the wire) and static, so + * it lives in the same day cache the other local backends use, and what + * reaches the agent is the ranked result list only. + */ + +import { cachedFile } from './cache.js'; +import { escapeHTML } from './sphinx.js'; +import { searchEntries } from './nodedocs.js'; + +const MAX_RESULTS = 20; + +/** + * The index file is `var search_data = {...}`: JSON behind one assignment + * for the browser's benefit. Anything that does not parse that way, or that + * lacks the info rows, is not an RDoc index, which on a wrong or moved URL + * is the honest error, and it keeps a block page out of the cache too. + * @param {string} js + * @returns {any} + */ +export function parseRdocIndex(js) { + const start = js.indexOf('='); + if (start >= 0) { + try { + const data = JSON.parse(js.slice(start + 1)); + if (Array.isArray(data?.index?.info)) return data; + } catch {} + } + throw new Error('not an RDoc search index'); +} + +/** + * Flatten the index's info rows into the entries the shared ranker scores. + * A row is [name, namespace, path, params, snippet]; the path's own anchor + * says what the row is, so 'dig' in 'Array' with anchor method-i-dig reads + * back as the heading a rubyist expects, Array#dig(*args). Snippets stay + * behind: matching on them would rank prose over the symbol asked for. + * @param {any} data - parsed search_index.js + * @returns {{text: string, name: string, kind: string, path: string}[]} + */ +export function buildRdocEntries(data) { + return data.index.info.map(([name, namespace, path, params]) => { + const p = String(path ?? ''); + const kind = p.includes('#method-c-') ? 'class method' + : p.includes('#method-i-') ? 'method' + : /^[A-Z]/.test(String(name)) ? 'class' : 'page'; + const text = kind === 'class method' ? `${namespace}.${name}${params}` + : kind === 'method' ? `${namespace}#${name}${params}` + : namespace ? `${namespace}::${name}` : String(name ?? ''); + return { text, name: String(name ?? ''), kind, path: p }; + }).filter((e) => e.text && e.path); +} + +/** + * The result list becomes the same small synthetic page the other search + * backends emit, so it distills, renders, numbers, and remembers like any + * fetched page and `do ` follows a result. + * @param {string} base + * @param {string} query + * @param {ReturnType} found + * @returns {string} + */ +export function resultsToHTML(base, query, found) { + const host = new URL(base).host; + const items = found.hits.map((e) => + `
  • ${escapeHTML(e.text)}` + + ` ${escapeHTML(e.kind)}
    • `); + const partial = found.partial ? '; no entry matches every word, so these match some' : ''; + const summary = items.length + ? `${found.total} entr${found.total === 1 ? 'y matches' : 'ies match'} in the docs' own index,` + + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:` + : `nothing in the docs' own index matches; try fewer or different words`; + return `${escapeHTML(host)} search: ${escapeHTML(query)}
    ` + + `

    ${summary}

    ` + + (items.length ? `
      ${items.join('')}
    ` : '') + + `
    `; +} + +/** + * Search one RDoc site. The site has no search URL of its own to remember, + * so the session keeps the docs root, the page a reader would start from. + * @param {string} base - docs root ending in '/', e.g. https://docs.ruby-lang.org/en/3.4/ + * @param {string} query + */ +export async function rdocSearch(base, query) { + if (!query.trim()) throw new Error('usage: search '); + const { data, via } = await cachedFile('rdoc', new URL('js/search_index.js', base).href, parseRdocIndex); + return { + url: new URL('index.html', base).href, + html: resultsToHTML(base, query, searchEntries(buildRdocEntries(data), query)), + via, + }; +} diff --git a/src/sites.js b/src/sites.js index c8a8a81..2eb4217 100644 --- a/src/sites.js +++ b/src/sites.js @@ -5,9 +5,10 @@ * resolves to one and hands off to the same fetch and render path `oc open` * uses, so nothing here can change what a page costs or how it reads. The * other shapes are searches cli.js runs itself and renders like any other - * page: `sphinx`, for a docs site whose search only exists as a static index - * file, `nodedoc`, for the Node.js API docs, which ship their reference the - * same way, and `api`, for a site whose search answers as JSON. + * page: `sphinx` and `rdoc`, for docs sites whose search only exists as a + * static index file, `nodedoc`, for the Node.js API docs, which ship their + * reference the same way, and `api`, for a site whose search answers as + * JSON. */ import { readdirSync, readFileSync } from 'node:fs'; @@ -29,12 +30,17 @@ const ALIASES = { py: 'docs.python.org', mdn: 'developer.mozilla.org', node: 'nodejs.org', + rust: 'doc.rust-lang.org', + java: 'docs.oracle.com', + ruby: 'docs.ruby-lang.org', + cpp: 'en.cppreference.com', + ts: 'typescriptlang.org', gcp: 'cloud.google.com', learn: 'learn.microsoft.com', wiki: 'wikipedia.org', }; -/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, api?: string, page?: string, results?: string, fields?: Record, total?: string, args?: string[]}} Shortcut */ +/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record, total?: string, args?: string[]}} Shortcut */ /** @typedef {{domain: string, commands: Record}} Site */ /** @type {Map|null} */ @@ -91,7 +97,7 @@ const verbs = (site) => * instead, since the agent has the right site and only needs the verb list. * @param {string} name * @param {string[]} args - * @returns {{url?: string, sphinx?: string, nodedoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null} + * @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null} */ export function resolveSite(name, args) { const site = sites().get(name.toLowerCase()); @@ -109,13 +115,14 @@ export function resolveSite(name, args) { // separate words ('oc ddg search claude code cli') works unquoted. const values = need.map((_, i) => i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]); - // A sphinx, nodedoc, or API search has no page URL to build: the query is - // handed back whole for cli.js to run against the site's own search. - if (def.sphinx) { - return { sphinx: def.sphinx, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; - } - if (def.nodedoc) { - return { nodedoc: def.nodedoc, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; + // A search oc runs itself has no page URL to build: the query is handed + // back whole for cli.js to run against the site's own search. The local + // backends need only their docs root; the API shape needs its whole + // definition, since it names the endpoint and the response fields. + for (const kind of ['sphinx', 'nodedoc', 'rdoc']) { + if (def[kind]) { + return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; + } } if (def.api) { return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; diff --git a/tests/rdoc.test.js b/tests/rdoc.test.js new file mode 100644 index 0000000..37b2357 --- /dev/null +++ b/tests/rdoc.test.js @@ -0,0 +1,75 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +const { parseRdocIndex, buildRdocEntries, resultsToHTML } = await import('../src/rdoc.js'); +const { searchEntries } = await import('../src/nodedocs.js'); + +const BASE = 'https://docs.ruby-lang.org/en/3.4/'; + +// A miniature search_index.js: a class page, instance and class methods, a +// name shared across classes, a guide page, and a snippet that would be +// markup if it were ever trusted. Rows are [name, namespace, path, params, +// snippet], the shape RDoc generates. +const INDEX = `var search_data = ${JSON.stringify({ + index: { + searchIndex: ['array', 'dig', 'dig', 'new', 'each_slice', 'contributing', 'evil'], + longSearchIndex: ['array', 'array::dig', 'hash::dig', 'array::new', 'array::each_slice', 'contributing', 'evil'], + info: [ + ['Array', '', 'Array.html', '', '

    An Array is an ordered collection.'], + ['dig', 'Array', 'Array.html#method-i-dig', '(*args)', '

    Finds the object in nested objects.'], + ['dig', 'Hash', 'Hash.html#method-i-dig', '(*args)', '

    Finds the object in nested objects.'], + ['new', 'Array', 'Array.html#method-c-new', '(size, default)', '

    Returns a new array.'], + ['each_slice', 'Array', 'Array.html#method-i-each_slice', '(n)', '

    Iterates in slices.'], + ['contributing', '', 'contributing_md.html', '', '

    How to contribute.'], + ['evil', 'Array', 'Array.html#method-i-evil', '', ''], + ], + }, +})}`; + +const entries = () => buildRdocEntries(parseRdocIndex(INDEX)); + +test('only an RDoc index parses, so an error page never enters the cache', () => { + assert.ok(parseRdocIndex(INDEX).index.info.length > 0); + assert.throws(() => parseRdocIndex('blocked'), /not an RDoc search index/); + assert.throws(() => parseRdocIndex('var search_data = {"unrelated": true}'), /not an RDoc search index/); +}); + +test('rows read back as the headings a rubyist expects', () => { + const all = entries(); + assert.equal(all.find((e) => e.path.includes('method-i-dig') && e.text.startsWith('Array')).text, 'Array#dig(*args)'); + assert.equal(all.find((e) => e.path.includes('method-c-new')).text, 'Array.new(size, default)'); + assert.equal(all.find((e) => e.path === 'Array.html').text, 'Array'); + assert.equal(all.find((e) => e.path === 'contributing_md.html').kind, 'page'); +}); + +test('a symbol query finds its methods across classes, exact name first', () => { + const found = searchEntries(entries(), 'dig'); + assert.equal(found.total, 2); + assert.deepEqual(found.hits.map((e) => e.text).sort(), ['Array#dig(*args)', 'Hash#dig(*args)']); + assert.equal(found.partial, false); +}); + +test('a class and method pair narrows to the one entry matching both words', () => { + const found = searchEntries(entries(), 'array each_slice'); + assert.equal(found.total, 1); + assert.equal(found.hits[0].text, 'Array#each_slice(n)'); +}); + +test('results link into the live docs, anchors intact, and name their kind', () => { + const html = resultsToHTML(BASE, 'dig', searchEntries(entries(), 'dig')); + assert.match(html, /href="https:\/\/docs\.ruby-lang\.org\/en\/3\.4\/Array\.html#method-i-dig"/); + assert.match(html, /Array#dig\(\*args\)<\/a> method/); + assert.match(html, /2 entries match in the docs' own index, ranked locally:/); +}); + +test('an index row is data, never markup on the results page', () => { + const html = resultsToHTML(BASE, 'evil', searchEntries(entries(), 'evil')); + assert.doesNotMatch(html, /