From e834309363116ae8c15f95b250ac19f2cfb3a4e2 Mon Sep 17 00:00:00 2001 From: only-cli Date: Mon, 24 Aug 2026 09:59:34 -0400 Subject: [PATCH] prototype a Sphinx search backend for the Python docs docs.python.org has no search server: its search page downloads the whole Sphinx index (searchindex.js, ~900KB gzipped) and ranks matches in the browser. oc now does the same ranking itself, so 'oc py search' answers from the site's own index instead of DuckDuckGo. The index is cached on disk for a day and never printed; what renders is a numbered result list that rides the normal open path, so 'do ' follows a result. A query that names a symbol exactly (json.dumps) links straight to its anchor. A site definition opts in with a 'sphinx' field naming the docs root, so any Sphinx site can use the backend later. --- README.md | 4 +- clis/docs.python.org.json | 2 +- skills/web-browsing-cli/SKILL.md | 2 +- src/cli.js | 32 ++++- src/sites.js | 18 ++- src/sphinx.js | 230 +++++++++++++++++++++++++++++++ tests/sites.test.js | 7 +- tests/sphinx.test.js | 66 +++++++++ 8 files changed, 349 insertions(+), 12 deletions(-) create mode 100644 src/sphinx.js create mode 100644 tests/sphinx.test.js diff --git a/README.md b/README.md index 5dbb713..63eb59f 100644 --- a/README.md +++ b/README.md @@ -125,13 +125,13 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum | AWS docs | `oc aws` (search via DuckDuckGo) | `guide `, `page `, `cli `, `search ` | | Google Cloud docs | `oc gcp` (via docs.cloud.google.com, search via DuckDuckGo) | `docs `, `page `, `gcloud `, `search ` | | Microsoft Learn | `oc learn` (search via its RSS API) | `azure `, `doc `, `cli `, `search ` | -| Python docs | `oc py` (search via DuckDuckGo) | `library `, `doc `, `search ` | +| Python docs | `oc py` (search via the docs' own index) | `library `, `doc `, `search ` | | MDN | `oc mdn` (search via DuckDuckGo) | `js `, `css `, `doc `, `search ` | | Node.js docs | `oc node` (search via DuckDuckGo) | `api `, `search ` | A shortcut only ever resolves to a URL and then takes the same path `oc open` does, so it changes nothing about what a page costs or how it reads. The last argument takes every word after it, so `oc ddg search claude code cli` and `oc aws search s3 lifecycle rules` need no quoting, and a path argument keeps its slashes, so `oc learn doc azure/aks/what-is-aks` reaches that page. -A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, Python, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. +A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. Want a website on that list? Open a pull request, or an issue naming the site; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/clis/docs.python.org.json b/clis/docs.python.org.json index 64904c4..9e44c1e 100644 --- a/clis/docs.python.org.json +++ b/clis/docs.python.org.json @@ -3,6 +3,6 @@ "commands": { "library": { "open": "https://docs.python.org/3/library/{module}.html", "args": ["module"] }, "doc": { "open": "https://docs.python.org/3/{path}.html", "args": ["path"] }, - "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Adocs.python.org+{query}", "args": ["query"] } + "search": { "sphinx": "https://docs.python.org/3/", "args": ["query"] } } } diff --git a/skills/web-browsing-cli/SKILL.md b/skills/web-browsing-cli/SKILL.md index 2508a34..b2f8df2 100644 --- a/skills/web-browsing-cli/SKILL.md +++ b/skills/web-browsing-cli/SKILL.md @@ -21,7 +21,7 @@ None of these except `open`/`do`/`raw ` fetch anything; they replay the pag ## Site shortcuts -`oc [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. +`oc [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. One verb is not a URL: `oc py search ` ranks the Python docs' own search index locally and prints results as a normal numbered page. ``` oc hn top oc reddit sub ClaudeAI oc gh repo only-cli oc diff --git a/src/cli.js b/src/cli.js index ddc4c75..991d7ee 100755 --- a/src/cli.js +++ b/src/cli.js @@ -4,6 +4,7 @@ import { fetchPage } from './fetch.js'; import { distill, toMarkdown, toHTML } from './distill.js'; import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js'; import { resolveSite, listSites } from './sites.js'; +import { sphinxSearch } from './sphinx.js'; import * as act from './act.js'; import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js'; @@ -111,11 +112,17 @@ async function main() { // A first word that is not a command may still be a site oc ships a // definition for, and a shortcut is only ever a URL, so it resolves to one // here and the rest of this function never learns it was not typed. + let sphinx = null; if (!COMMANDS.has(command)) { const site = resolveSite(command, args); if (!site) throw new Error(`unknown command '${command}', run oc --help`); - args = [site.url]; - command = 'open'; + if (site.sphinx) { + sphinx = site; + command = 'sphinx'; + } else { + args = [site.url]; + command = 'open'; + } } const sessionName = values.session || DEFAULT_SESSION; @@ -202,6 +209,27 @@ async function main() { if (failure) noContent(finalUrl, failure); return; } + case 'sphinx': { + // A Sphinx site's search index is fetched (or read back from its day + // cache) and ranked here, then the result list rides the exact `open` + // path: distilled, rendered, remembered, so `do ` follows a result. + // Only the list is ever printed; the index itself stays out of context. + const t0 = performance.now(); + const { url, html, via } = await sphinxSearch(sphinx.sphinx, sphinx.query); + const page = distill(html, url); + if (values.json) { + remember(page, sessionName); + console.log(JSON.stringify({ ...page, empty: false })); + return; + } + const { text, stats } = render(page, { budget: asked || 500 }); + remember(page, sessionName, stats.next); + console.log(text); + if (verbose) { + console.error(`~${stats.tokens} tokens, search index via ${via}, ${Math.round(performance.now() - t0)}ms`); + } + return; + } case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 })); case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 })); case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 })); diff --git a/src/sites.js b/src/sites.js index b5be2b7..1b01473 100644 --- a/src/sites.js +++ b/src/sites.js @@ -1,9 +1,12 @@ /** * Site shortcuts. `clis/*.json` names the URLs on a site worth reaching * directly, so `oc hn item 4711` gets there without the agent knowing that - * Hacker News spells it /item?id=. A shortcut is only ever a URL: it resolves - * to one and hands off to the same fetch and render path `oc open` uses, so - * nothing here can change what a page costs or how it reads. + * Hacker News spells it /item?id=. A shortcut is almost always a URL: it + * resolves to one and hands off to the same fetch and render path `oc open` + * uses, so nothing here can change what a page costs or how it reads. The + * one other shape is `sphinx`, for a docs site whose search only exists as a + * static index file; cli.js runs that search and renders the results like + * any other page. */ import { readdirSync, readFileSync } from 'node:fs'; @@ -30,7 +33,7 @@ const ALIASES = { wiki: 'wikipedia.org', }; -/** @typedef {{open: string, args?: string[]}} Shortcut */ +/** @typedef {{open?: string, sphinx?: string, args?: string[]}} Shortcut */ /** @typedef {{domain: string, commands: Record}} Site */ /** @type {Map|null} */ @@ -87,7 +90,7 @@ const verbs = (site) => * instead, since the agent has the right site and only needs the verb list. * @param {string} name * @param {string[]} args - * @returns {{url: string, domain: string, command: string}|null} + * @returns {{url?: string, sphinx?: string, query?: string, domain: string, command: string}|null} */ export function resolveSite(name, args) { const site = sites().get(name.toLowerCase()); @@ -105,6 +108,11 @@ export function resolveSite(name, args) { // separate words ('oc ddg search claude code cli') works unquoted. const values = need.map((_, i) => i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]); + // A sphinx search has no URL to build: the query is ranked against the + // site's index locally, so it is handed back whole for cli.js to run. + if (def.sphinx) { + return { sphinx: def.sphinx, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; + } const url = need.reduce( (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)), def.open); diff --git a/src/sphinx.js b/src/sphinx.js new file mode 100644 index 0000000..c333a66 --- /dev/null +++ b/src/sphinx.js @@ -0,0 +1,230 @@ +/** + * Sphinx search backend. A Sphinx-built documentation site (docs.python.org, + * most Read the Docs projects) has no search server: its search page ships + * the site's entire full-text index as one static file, searchindex.js, and + * ranks matches in the visitor's browser. oc can run the same ranking here, + * so `search` on such a site answers from the site's own index instead of a + * third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB + * over the wire) but static, so it is cached on disk for a day and never + * printed: what reaches the agent is only the ranked result list. + */ + +import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { fetchPage } from './fetch.js'; + +// A documentation set rebuilds at most a few times a day, and a stale result +// list still links to live pages, so a day-old index is a fair trade against +// moving ~4MB per search. +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const MAX_RESULTS = 20; + +const cacheDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sphinx'); + +/** + * The index file is `Search.setIndex({...})`: JSON wrapped in one function + * call for the browser's benefit. Anything that does not parse that way is + * not a Sphinx index, which on a wrong or moved URL is the honest error. + * @param {string} js + * @returns {any} + */ +export function parseIndex(js) { + const start = js.indexOf('('); + const end = js.lastIndexOf(')'); + if (start >= 0 && end > start) { + try { + return JSON.parse(js.slice(start + 1, end)); + } catch {} + } + throw new Error('not a Sphinx search index'); +} + +// terms and titleterms store a bare number when a word appears in one +// document and an array when it appears in several. +const docsFor = (table, word) => { + const hit = table?.[word]; + return hit == null ? null : Array.isArray(hit) ? hit : [hit]; +}; + +/** + * Sphinx stems words before indexing ('threading' is stored as 'thread'), so + * an exact lookup misses common query spellings. Rather than shipping the + * Porter stemmer, try the word with common suffixes stripped, and only then + * a prefix scan: an index key that extends the word, or that the word + * extends, counts at reduced weight. + * @param {Record} table + * @param {string} word + * @returns {{docs: number[], exact: boolean}|null} + */ +function lookup(table, word) { + const exact = docsFor(table, word); + if (exact) return { docs: exact, exact: true }; + for (const suffix of ['ing', 'ed', 'es', 's', 'e']) { + if (word.length - suffix.length >= 3 && word.endsWith(suffix)) { + const hit = docsFor(table, word.slice(0, -suffix.length)); + if (hit) return { docs: hit, exact: false }; + } + } + if (word.length >= 4) { + const docs = new Set(); + for (const key of Object.keys(table ?? {})) { + if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) { + for (const d of docsFor(table, key)) docs.add(d); + } + } + if (docs.size) return { docs: [...docs], exact: false }; + } + return null; +} + +/** + * An exact object hit ('json.dumps', or just 'dumps') beats any full-text + * rank: the index maps the symbol straight to its anchor on the page, so it + * goes at the top as a direct link. Only single-word queries can be symbols. + * @param {any} index + * @param {string} query + */ +function objectHits(index, query) { + const q = query.trim().toLowerCase(); + if (!q || q.includes(' ')) return []; + const hits = []; + for (const [prefix, entries] of Object.entries(index.objects ?? {})) { + for (const [doc, typeIdx, priority, anchor, name] of entries) { + const full = prefix ? `${prefix}.${name}` : name; + if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue; + hits.push({ + name: full, + type: index.objnames?.[typeIdx]?.[2] ?? '', + doc, + anchor: anchor === '' ? full : anchor, + priority, + }); + } + } + return hits.sort((a, b) => a.priority - b.priority).slice(0, 5); +} + +/** + * Rank the index against a query the way the site's own search page would: + * a document must match every word, a title hit weighs far more than a body + * hit, and only when nothing matches every word does any-word matching kick + * in, and then the result page says so. + * @param {any} index + * @param {string} query + */ +export function searchIndex(index, query) { + const words = [...new Set( + query.toLowerCase().split(/\s+/) + .map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, '')) + .filter(Boolean))]; + const scores = new Map(); + const matched = new Map(); + for (const word of words) { + const perDoc = new Map(); + const body = lookup(index.terms, word); + if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2); + const title = lookup(index.titleterms, word); + if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5)); + for (const [d, score] of perDoc) { + scores.set(d, (scores.get(d) ?? 0) + score); + matched.set(d, (matched.get(d) ?? 0) + 1); + } + } + let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length); + const partial = !docs.length && words.length > 1 && scores.size > 0; + if (partial) docs = [...scores.keys()]; + docs.sort((a, b) => + scores.get(b) - scores.get(a) + || String(index.titles[a]).localeCompare(String(index.titles[b]))); + return { + words, + partial, + total: docs.length, + objects: objectHits(index, query), + docs: docs.slice(0, MAX_RESULTS).map((d) => ({ + doc: d, + title: plainTitle(index.titles[d]) || index.docnames[d], + })), + }; +} + +// Titles in the index arrive as the HTML of the page's

, markup and all +// (docs.python.org wraps module names in spans), so they are flattened +// to text before they are placed on the results page. +const plainTitle = (t) => String(t).replace(/<[^>]*>/g, ''); + +const escapeHTML = (s) => String(s) + .replaceAll('&', '&').replaceAll('<', '<') + .replaceAll('>', '>').replaceAll('"', '"'); + +/** + * The result list becomes a small HTML page and rides the same distill and + * render path a fetched page does. That is what makes results numbered, + * followable with `do `, and saved as session state, with nothing new for + * an agent to learn. + * @param {string} base + * @param {string} query + * @param {ReturnType} found + * @param {any} index + * @returns {string} + */ +export function resultsToHTML(base, query, found, index) { + const host = new URL(base).host; + const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href; + const items = found.objects.map((o) => + `
  • ${escapeHTML(o.name)}` + + ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}
    • `); + for (const r of found.docs) { + items.push(`
  • ${escapeHTML(r.title)}
    • `); + } + const partial = found.partial ? '; no page matches every word, so these match some' : ''; + const summary = items.length + ? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,` + + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:` + : `nothing in the site's own search index matches; try fewer or different words`; + return `${escapeHTML(host)} search: ${escapeHTML(query)}
    ` + + `

    ${summary}

    ` + + (items.length ? `
      ${items.join('')}
    ` : '') + + `
    `; +} + +/** + * The index for one site, from the disk cache while it is fresh and from the + * network otherwise. A cache that cannot be written costs nothing but the + * refetch, the same policy session state follows. + * @param {string} base + */ +async function loadIndex(base) { + const file = join(cacheDir(), `${new URL(base).host}.js`); + try { + if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) { + return { index: parseIndex(readFileSync(file, 'utf8')), via: 'cache' }; + } + } catch {} + const { html } = await fetchPage(new URL('searchindex.js', base).href); + // Parse before caching, so a block page or an error never poisons the cache. + const index = parseIndex(html); + try { + mkdirSync(cacheDir(), { recursive: true }); + writeFileSync(file, html); + } catch {} + return { index, via: 'network' }; +} + +/** + * Search one Sphinx site. Returns the synthetic results page plus the URL the + * session should remember: the site's human search URL, so the state reads + * sensibly in `oc session` listings and error messages. + * @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/ + * @param {string} query + */ +export async function sphinxSearch(base, query) { + if (!query.trim()) throw new Error('usage: search '); + const { index, via } = await loadIndex(base); + return { + url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href, + html: resultsToHTML(base, query, searchIndex(index, query), index), + via, + }; +} diff --git a/tests/sites.test.js b/tests/sites.test.js index ec82ec4..979b392 100644 --- a/tests/sites.test.js +++ b/tests/sites.test.js @@ -52,7 +52,9 @@ test('every shipped definition is reachable and every url template is filled', ( for (const [name, site] of sites()) { for (const [verb, def] of Object.entries(site.commands)) { const args = (def.args ?? []).map((a) => `test-${a}`); - const { url } = resolveSite(name, [verb, ...args]); + const resolved = resolveSite(name, [verb, ...args]); + // A sphinx-backed verb resolves to a site root to search, not a URL. + const url = resolved.url ?? resolved.sphinx; assert.doesNotMatch(url, /[{}]/, `oc ${name} ${verb} left a template var in ${url}`); assert.equal(new URL(url).protocol, 'https:', `oc ${name} ${verb} is not https`); } @@ -87,4 +89,7 @@ test('language docs shortcuts resolve, and a doc path keeps its slashes', () => assert.equal( resolveSite('nodejs.org', ['search', 'readFile options']).url, 'https://html.duckduckgo.com/html/?q=site%3Anodejs.org+readFile%20options'); + const py = resolveSite('py', ['search', 'json', 'dumps']); + assert.equal(py.sphinx, 'https://docs.python.org/3/'); + assert.equal(py.query, 'json dumps'); }); diff --git a/tests/sphinx.test.js b/tests/sphinx.test.js new file mode 100644 index 0000000..4037540 --- /dev/null +++ b/tests/sphinx.test.js @@ -0,0 +1,66 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +const { parseIndex, searchIndex, resultsToHTML } = await import('../src/sphinx.js'); + +// A miniature docs.python.org: enough shape to exercise every lookup path +// (single-number terms, title boosts, stemmed words, object anchors) without +// a fixture file to drift out of date. +const INDEX = { + docnames: ['library/json', 'library/threading', 'tutorial/appendix'], + titles: [ + 'json - JSON encoder and decoder', + 'threading - Thread-based parallelism', + 'Appendix', + ], + terms: { json: 0, thread: [1, 2], socket: [2] }, + titleterms: { json: [0], thread: [1] }, + objects: { json: [[0, 3, 1, '', 'dumps']] }, + objnames: { 3: ['py', 'function', 'Python function'] }, +}; +const BASE = 'https://docs.python.org/3/'; + +test('parseIndex unwraps Search.setIndex() and refuses anything else', () => { + assert.equal(parseIndex('Search.setIndex({"a": 1})').a, 1); + assert.throws(() => parseIndex('a block page'), /not a Sphinx search index/); + assert.throws(() => parseIndex('Search.setIndex(undefined)'), /not a Sphinx search index/); +}); + +test('a title hit outranks body hits, and one-doc terms stored as a bare number work', () => { + const found = searchIndex(INDEX, 'thread'); + assert.deepEqual(found.docs.map((d) => d.doc), [1, 2]); + assert.equal(searchIndex(INDEX, 'json').docs[0].doc, 0); +}); + +test('a word Sphinx stemmed away still matches through its stem', () => { + // The index stores 'thread'; a query typed as English says 'threading'. + const found = searchIndex(INDEX, 'threading'); + assert.equal(found.docs[0].doc, 1); +}); + +test('titles are flattened to text before they reach the results page', () => { + const found = searchIndex(INDEX, 'json'); + assert.equal(found.docs[0].title, 'json - JSON encoder and decoder'); +}); + +test('every word must match, and when none can, any-word results say so', () => { + // 'json' hits doc 0, 'socket' hits doc 2, nothing hits both. + const found = searchIndex(INDEX, 'json socket'); + assert.equal(found.partial, true); + const html = resultsToHTML(BASE, 'json socket', found, INDEX); + assert.match(html, /no page matches every word/); +}); + +test('an exact symbol query becomes a direct link to its anchor', () => { + const found = searchIndex(INDEX, 'json.dumps'); + assert.equal(found.objects.length, 1); + const html = resultsToHTML(BASE, 'json.dumps', found, INDEX); + assert.match(html, /href="https:\/\/docs\.python\.org\/3\/library\/json\.html#json\.dumps"/); + assert.match(html, /Python function/); +}); + +test('no matches renders an honest empty page, not an error', () => { + const html = resultsToHTML(BASE, 'zzqqxx', searchIndex(INDEX, 'zzqqxx'), INDEX); + assert.match(html, /nothing in the site's own search index matches/); + assert.doesNotMatch(html, /
      /); +});