diff --git a/README.md b/README.md index 5dbb713..63eb59f 100644 --- a/README.md +++ b/README.md @@ -125,13 +125,13 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum | AWS docs | `oc aws` (search via DuckDuckGo) | `guide `, `page `, `cli `, `search ` | | Google Cloud docs | `oc gcp` (via docs.cloud.google.com, search via DuckDuckGo) | `docs `, `page `, `gcloud `, `search ` | | Microsoft Learn | `oc learn` (search via its RSS API) | `azure `, `doc `, `cli `, `search ` | -| Python docs | `oc py` (search via DuckDuckGo) | `library `, `doc `, `search ` | +| Python docs | `oc py` (search via the docs' own index) | `library `, `doc `, `search ` | | MDN | `oc mdn` (search via DuckDuckGo) | `js `, `css `, `doc `, `search ` | | Node.js docs | `oc node` (search via DuckDuckGo) | `api `, `search ` | A shortcut only ever resolves to a URL and then takes the same path `oc open` does, so it changes nothing about what a page costs or how it reads. The last argument takes every word after it, so `oc ddg search claude code cli` and `oc aws search s3 lifecycle rules` need no quoting, and a path argument keeps its slashes, so `oc learn doc azure/aks/what-is-aks` reaches that page. -A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, Python, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. +A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. Want a website on that list? Open a pull request, or an issue naming the site; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/clis/docs.python.org.json b/clis/docs.python.org.json index 64904c4..9e44c1e 100644 --- a/clis/docs.python.org.json +++ b/clis/docs.python.org.json @@ -3,6 +3,6 @@ "commands": { "library": { "open": "https://docs.python.org/3/library/{module}.html", "args": ["module"] }, "doc": { "open": "https://docs.python.org/3/{path}.html", "args": ["path"] }, - "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Adocs.python.org+{query}", "args": ["query"] } + "search": { "sphinx": "https://docs.python.org/3/", "args": ["query"] } } } diff --git a/skills/web-browsing-cli/SKILL.md b/skills/web-browsing-cli/SKILL.md index 2508a34..b2f8df2 100644 --- a/skills/web-browsing-cli/SKILL.md +++ b/skills/web-browsing-cli/SKILL.md @@ -21,7 +21,7 @@ None of these except `open`/`do`/`raw ` fetch anything; they replay the pag ## Site shortcuts -`oc [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. +`oc [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. One verb is not a URL: `oc py search ` ranks the Python docs' own search index locally and prints results as a normal numbered page. ``` oc hn top oc reddit sub ClaudeAI oc gh repo only-cli oc diff --git a/src/cli.js b/src/cli.js index ddc4c75..991d7ee 100755 --- a/src/cli.js +++ b/src/cli.js @@ -4,6 +4,7 @@ import { fetchPage } from './fetch.js'; import { distill, toMarkdown, toHTML } from './distill.js'; import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js'; import { resolveSite, listSites } from './sites.js'; +import { sphinxSearch } from './sphinx.js'; import * as act from './act.js'; import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js'; @@ -111,11 +112,17 @@ async function main() { // A first word that is not a command may still be a site oc ships a // definition for, and a shortcut is only ever a URL, so it resolves to one // here and the rest of this function never learns it was not typed. + let sphinx = null; if (!COMMANDS.has(command)) { const site = resolveSite(command, args); if (!site) throw new Error(`unknown command '${command}', run oc --help`); - args = [site.url]; - command = 'open'; + if (site.sphinx) { + sphinx = site; + command = 'sphinx'; + } else { + args = [site.url]; + command = 'open'; + } } const sessionName = values.session || DEFAULT_SESSION; @@ -202,6 +209,27 @@ async function main() { if (failure) noContent(finalUrl, failure); return; } + case 'sphinx': { + // A Sphinx site's search index is fetched (or read back from its day + // cache) and ranked here, then the result list rides the exact `open` + // path: distilled, rendered, remembered, so `do ` follows a result. + // Only the list is ever printed; the index itself stays out of context. + const t0 = performance.now(); + const { url, html, via } = await sphinxSearch(sphinx.sphinx, sphinx.query); + const page = distill(html, url); + if (values.json) { + remember(page, sessionName); + console.log(JSON.stringify({ ...page, empty: false })); + return; + } + const { text, stats } = render(page, { budget: asked || 500 }); + remember(page, sessionName, stats.next); + console.log(text); + if (verbose) { + console.error(`~${stats.tokens} tokens, search index via ${via}, ${Math.round(performance.now() - t0)}ms`); + } + return; + } case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 })); case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 })); case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 })); diff --git a/src/sites.js b/src/sites.js index b5be2b7..1b01473 100644 --- a/src/sites.js +++ b/src/sites.js @@ -1,9 +1,12 @@ /** * Site shortcuts. `clis/*.json` names the URLs on a site worth reaching * directly, so `oc hn item 4711` gets there without the agent knowing that - * Hacker News spells it /item?id=. A shortcut is only ever a URL: it resolves - * to one and hands off to the same fetch and render path `oc open` uses, so - * nothing here can change what a page costs or how it reads. + * Hacker News spells it /item?id=. A shortcut is almost always a URL: it + * resolves to one and hands off to the same fetch and render path `oc open` + * uses, so nothing here can change what a page costs or how it reads. The + * one other shape is `sphinx`, for a docs site whose search only exists as a + * static index file; cli.js runs that search and renders the results like + * any other page. */ import { readdirSync, readFileSync } from 'node:fs'; @@ -30,7 +33,7 @@ const ALIASES = { wiki: 'wikipedia.org', }; -/** @typedef {{open: string, args?: string[]}} Shortcut */ +/** @typedef {{open?: string, sphinx?: string, args?: string[]}} Shortcut */ /** @typedef {{domain: string, commands: Record}} Site */ /** @type {Map|null} */ @@ -87,7 +90,7 @@ const verbs = (site) => * instead, since the agent has the right site and only needs the verb list. * @param {string} name * @param {string[]} args - * @returns {{url: string, domain: string, command: string}|null} + * @returns {{url?: string, sphinx?: string, query?: string, domain: string, command: string}|null} */ export function resolveSite(name, args) { const site = sites().get(name.toLowerCase()); @@ -105,6 +108,11 @@ export function resolveSite(name, args) { // separate words ('oc ddg search claude code cli') works unquoted. const values = need.map((_, i) => i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]); + // A sphinx search has no URL to build: the query is ranked against the + // site's index locally, so it is handed back whole for cli.js to run. + if (def.sphinx) { + return { sphinx: def.sphinx, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; + } const url = need.reduce( (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)), def.open); diff --git a/src/sphinx.js b/src/sphinx.js new file mode 100644 index 0000000..c333a66 --- /dev/null +++ b/src/sphinx.js @@ -0,0 +1,230 @@ +/** + * Sphinx search backend. A Sphinx-built documentation site (docs.python.org, + * most Read the Docs projects) has no search server: its search page ships + * the site's entire full-text index as one static file, searchindex.js, and + * ranks matches in the visitor's browser. oc can run the same ranking here, + * so `search` on such a site answers from the site's own index instead of a + * third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB + * over the wire) but static, so it is cached on disk for a day and never + * printed: what reaches the agent is only the ranked result list. + */ + +import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { fetchPage } from './fetch.js'; + +// A documentation set rebuilds at most a few times a day, and a stale result +// list still links to live pages, so a day-old index is a fair trade against +// moving ~4MB per search. +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const MAX_RESULTS = 20; + +const cacheDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sphinx'); + +/** + * The index file is `Search.setIndex({...})`: JSON wrapped in one function + * call for the browser's benefit. Anything that does not parse that way is + * not a Sphinx index, which on a wrong or moved URL is the honest error. + * @param {string} js + * @returns {any} + */ +export function parseIndex(js) { + const start = js.indexOf('('); + const end = js.lastIndexOf(')'); + if (start >= 0 && end > start) { + try { + return JSON.parse(js.slice(start + 1, end)); + } catch {} + } + throw new Error('not a Sphinx search index'); +} + +// terms and titleterms store a bare number when a word appears in one +// document and an array when it appears in several. +const docsFor = (table, word) => { + const hit = table?.[word]; + return hit == null ? null : Array.isArray(hit) ? hit : [hit]; +}; + +/** + * Sphinx stems words before indexing ('threading' is stored as 'thread'), so + * an exact lookup misses common query spellings. Rather than shipping the + * Porter stemmer, try the word with common suffixes stripped, and only then + * a prefix scan: an index key that extends the word, or that the word + * extends, counts at reduced weight. + * @param {Record} table + * @param {string} word + * @returns {{docs: number[], exact: boolean}|null} + */ +function lookup(table, word) { + const exact = docsFor(table, word); + if (exact) return { docs: exact, exact: true }; + for (const suffix of ['ing', 'ed', 'es', 's', 'e']) { + if (word.length - suffix.length >= 3 && word.endsWith(suffix)) { + const hit = docsFor(table, word.slice(0, -suffix.length)); + if (hit) return { docs: hit, exact: false }; + } + } + if (word.length >= 4) { + const docs = new Set(); + for (const key of Object.keys(table ?? {})) { + if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) { + for (const d of docsFor(table, key)) docs.add(d); + } + } + if (docs.size) return { docs: [...docs], exact: false }; + } + return null; +} + +/** + * An exact object hit ('json.dumps', or just 'dumps') beats any full-text + * rank: the index maps the symbol straight to its anchor on the page, so it + * goes at the top as a direct link. Only single-word queries can be symbols. + * @param {any} index + * @param {string} query + */ +function objectHits(index, query) { + const q = query.trim().toLowerCase(); + if (!q || q.includes(' ')) return []; + const hits = []; + for (const [prefix, entries] of Object.entries(index.objects ?? {})) { + for (const [doc, typeIdx, priority, anchor, name] of entries) { + const full = prefix ? `${prefix}.${name}` : name; + if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue; + hits.push({ + name: full, + type: index.objnames?.[typeIdx]?.[2] ?? '', + doc, + anchor: anchor === '' ? full : anchor, + priority, + }); + } + } + return hits.sort((a, b) => a.priority - b.priority).slice(0, 5); +} + +/** + * Rank the index against a query the way the site's own search page would: + * a document must match every word, a title hit weighs far more than a body + * hit, and only when nothing matches every word does any-word matching kick + * in, and then the result page says so. + * @param {any} index + * @param {string} query + */ +export function searchIndex(index, query) { + const words = [...new Set( + query.toLowerCase().split(/\s+/) + .map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, '')) + .filter(Boolean))]; + const scores = new Map(); + const matched = new Map(); + for (const word of words) { + const perDoc = new Map(); + const body = lookup(index.terms, word); + if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2); + const title = lookup(index.titleterms, word); + if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5)); + for (const [d, score] of perDoc) { + scores.set(d, (scores.get(d) ?? 0) + score); + matched.set(d, (matched.get(d) ?? 0) + 1); + } + } + let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length); + const partial = !docs.length && words.length > 1 && scores.size > 0; + if (partial) docs = [...scores.keys()]; + docs.sort((a, b) => + scores.get(b) - scores.get(a) + || String(index.titles[a]).localeCompare(String(index.titles[b]))); + return { + words, + partial, + total: docs.length, + objects: objectHits(index, query), + docs: docs.slice(0, MAX_RESULTS).map((d) => ({ + doc: d, + title: plainTitle(index.titles[d]) || index.docnames[d], + })), + }; +} + +// Titles in the index arrive as the HTML of the page's

, markup and all +// (docs.python.org wraps module names in spans), so they are flattened +// to text before they are placed on the results page. +const plainTitle = (t) => String(t).replace(/<[^>]*>/g, ''); + +const escapeHTML = (s) => String(s) + .replaceAll('&', '&').replaceAll('<', '<') + .replaceAll('>', '>').replaceAll('"', '"'); + +/** + * The result list becomes a small HTML page and rides the same distill and + * render path a fetched page does. That is what makes results numbered, + * followable with `do `, and saved as session state, with nothing new for + * an agent to learn. + * @param {string} base + * @param {string} query + * @param {ReturnType} found + * @param {any} index + * @returns {string} + */ +export function resultsToHTML(base, query, found, index) { + const host = new URL(base).host; + const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href; + const items = found.objects.map((o) => + `
  • ${escapeHTML(o.name)}` + + ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}
    • `); + for (const r of found.docs) { + items.push(`
  • ${escapeHTML(r.title)}
    • `); + } + const partial = found.partial ? '; no page matches every word, so these match some' : ''; + const summary = items.length + ? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,` + + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:` + : `nothing in the site's own search index matches; try fewer or different words`; + return `${escapeHTML(host)} search: ${escapeHTML(query)}
    ` + + `

    ${summary}

    ` + + (items.length ? `
      ${items.join('')}
    ` : '') + + `
    `; +} + +/** + * The index for one site, from the disk cache while it is fresh and from the + * network otherwise. A cache that cannot be written costs nothing but the + * refetch, the same policy session state follows. + * @param {string} base + */ +async function loadIndex(base) { + const file = join(cacheDir(), `${new URL(base).host}.js`); + try { + if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) { + return { index: parseIndex(readFileSync(file, 'utf8')), via: 'cache' }; + } + } catch {} + const { html } = await fetchPage(new URL('searchindex.js', base).href); + // Parse before caching, so a block page or an error never poisons the cache. + const index = parseIndex(html); + try { + mkdirSync(cacheDir(), { recursive: true }); + writeFileSync(file, html); + } catch {} + return { index, via: 'network' }; +} + +/** + * Search one Sphinx site. Returns the synthetic results page plus the URL the + * session should remember: the site's human search URL, so the state reads + * sensibly in `oc session` listings and error messages. + * @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/ + * @param {string} query + */ +export async function sphinxSearch(base, query) { + if (!query.trim()) throw new Error('usage: search '); + const { index, via } = await loadIndex(base); + return { + url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href, + html: resultsToHTML(base, query, searchIndex(index, query), index), + via, + }; +} diff --git a/tests/sites.test.js b/tests/sites.test.js index ec82ec4..979b392 100644 --- a/tests/sites.test.js +++ b/tests/sites.test.js @@ -52,7 +52,9 @@ test('every shipped definition is reachable and every url template is filled', ( for (const [name, site] of sites()) { for (const [verb, def] of Object.entries(site.commands)) { const args = (def.args ?? []).map((a) => `test-${a}`); - const { url } = resolveSite(name, [verb, ...args]); + const resolved = resolveSite(name, [verb, ...args]); + // A sphinx-backed verb resolves to a site root to search, not a URL. + const url = resolved.url ?? resolved.sphinx; assert.doesNotMatch(url, /[{}]/, `oc ${name} ${verb} left a template var in ${url}`); assert.equal(new URL(url).protocol, 'https:', `oc ${name} ${verb} is not https`); } @@ -87,4 +89,7 @@ test('language docs shortcuts resolve, and a doc path keeps its slashes', () => assert.equal( resolveSite('nodejs.org', ['search', 'readFile options']).url, 'https://html.duckduckgo.com/html/?q=site%3Anodejs.org+readFile%20options'); + const py = resolveSite('py', ['search', 'json', 'dumps']); + assert.equal(py.sphinx, 'https://docs.python.org/3/'); + assert.equal(py.query, 'json dumps'); }); diff --git a/tests/sphinx.test.js b/tests/sphinx.test.js new file mode 100644 index 0000000..4037540 --- /dev/null +++ b/tests/sphinx.test.js @@ -0,0 +1,66 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +const { parseIndex, searchIndex, resultsToHTML } = await import('../src/sphinx.js'); + +// A miniature docs.python.org: enough shape to exercise every lookup path +// (single-number terms, title boosts, stemmed words, object anchors) without +// a fixture file to drift out of date. +const INDEX = { + docnames: ['library/json', 'library/threading', 'tutorial/appendix'], + titles: [ + 'json - JSON encoder and decoder', + 'threading - Thread-based parallelism', + 'Appendix', + ], + terms: { json: 0, thread: [1, 2], socket: [2] }, + titleterms: { json: [0], thread: [1] }, + objects: { json: [[0, 3, 1, '', 'dumps']] }, + objnames: { 3: ['py', 'function', 'Python function'] }, +}; +const BASE = 'https://docs.python.org/3/'; + +test('parseIndex unwraps Search.setIndex() and refuses anything else', () => { + assert.equal(parseIndex('Search.setIndex({"a": 1})').a, 1); + assert.throws(() => parseIndex('a block page'), /not a Sphinx search index/); + assert.throws(() => parseIndex('Search.setIndex(undefined)'), /not a Sphinx search index/); +}); + +test('a title hit outranks body hits, and one-doc terms stored as a bare number work', () => { + const found = searchIndex(INDEX, 'thread'); + assert.deepEqual(found.docs.map((d) => d.doc), [1, 2]); + assert.equal(searchIndex(INDEX, 'json').docs[0].doc, 0); +}); + +test('a word Sphinx stemmed away still matches through its stem', () => { + // The index stores 'thread'; a query typed as English says 'threading'. + const found = searchIndex(INDEX, 'threading'); + assert.equal(found.docs[0].doc, 1); +}); + +test('titles are flattened to text before they reach the results page', () => { + const found = searchIndex(INDEX, 'json'); + assert.equal(found.docs[0].title, 'json - JSON encoder and decoder'); +}); + +test('every word must match, and when none can, any-word results say so', () => { + // 'json' hits doc 0, 'socket' hits doc 2, nothing hits both. + const found = searchIndex(INDEX, 'json socket'); + assert.equal(found.partial, true); + const html = resultsToHTML(BASE, 'json socket', found, INDEX); + assert.match(html, /no page matches every word/); +}); + +test('an exact symbol query becomes a direct link to its anchor', () => { + const found = searchIndex(INDEX, 'json.dumps'); + assert.equal(found.objects.length, 1); + const html = resultsToHTML(BASE, 'json.dumps', found, INDEX); + assert.match(html, /href="https:\/\/docs\.python\.org\/3\/library\/json\.html#json\.dumps"/); + assert.match(html, /Python function/); +}); + +test('no matches renders an honest empty page, not an error', () => { + const html = resultsToHTML(BASE, 'zzqqxx', searchIndex(INDEX, 'zzqqxx'), INDEX); + assert.match(html, /nothing in the site's own search index matches/); + assert.doesNotMatch(html, /
      /); +});