From f3f5a93466046da37acfdccb46a55e216430b4c1 Mon Sep 17 00:00:00 2001 From: only-cli Date: Mon, 24 Aug 2026 09:26:41 -0400 Subject: [PATCH 01/11] add Python, MDN, and Node.js documentation shortcuts First three languages from #25. Each is one definition in clis/: - oc py library json, oc py doc reference/datamodel (the .html is appended, matching how the aws shortcuts spell pages) - oc mdn js Array/map, oc mdn css grid-template-columns, and a generic oc mdn doc for everything else under /en-US/docs - oc node api fs Python renders search client-side (Sphinx), MDN serves its search page as a client-rendered shell with only the nav in the HTML, and nodejs.org has no search, so all three get the DuckDuckGo site: filter fallback the aws and gcp shortcuts already use. Short names py, mdn, and node join the alias table; python, mozilla, nodejs, and the full domains resolve as before through the generic rules. --- README.md | 5 ++++- clis/developer.mozilla.org.json | 9 +++++++++ clis/docs.python.org.json | 8 ++++++++ clis/nodejs.org.json | 7 +++++++ llms.txt | 2 +- skills/web-browsing-cli/SKILL.md | 3 ++- src/sites.js | 3 +++ tests/sites.test.js | 21 +++++++++++++++++++++ 8 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 clis/developer.mozilla.org.json create mode 100644 clis/docs.python.org.json create mode 100644 clis/nodejs.org.json diff --git a/README.md b/README.md index ae9aa4e..c78d00d 100644 --- a/README.md +++ b/README.md @@ -125,10 +125,13 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum | AWS docs | `oc aws` (search via DuckDuckGo) | `guide `, `page `, `cli `, `search ` | | Google Cloud docs | `oc gcp` (via docs.cloud.google.com, search via DuckDuckGo) | `docs `, `page `, `gcloud `, `search ` | | Microsoft Learn | `oc learn` (search via its RSS API) | `azure `, `doc `, `cli `, `search ` | +| Python docs | `oc py` (search via DuckDuckGo) | `library `, `doc `, `search ` | +| MDN | `oc mdn` (search via DuckDuckGo) | `js `, `css `, `doc `, `search ` | +| Node.js docs | `oc node` (search via DuckDuckGo) | `api `, `search ` | A shortcut only ever resolves to a URL and then takes the same path `oc open` does, so it changes nothing about what a page costs or how it reads. The last argument takes every word after it, so `oc ddg search claude code cli` and `oc aws search s3 lifecycle rules` need no quoting, and a path argument keeps its slashes, so `oc learn doc azure/aks/what-is-aks` reaches that page. -A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS and Google Cloud render docs search purely client-side with no feed, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. +A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, Python, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. Want a website on that list? Open a pull request, or an issue naming the site; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/clis/developer.mozilla.org.json b/clis/developer.mozilla.org.json new file mode 100644 index 0000000..46c63d5 --- /dev/null +++ b/clis/developer.mozilla.org.json @@ -0,0 +1,9 @@ +{ + "domain": "developer.mozilla.org", + "commands": { + "js": { "open": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/{page}", "args": ["page"] }, + "css": { "open": "https://developer.mozilla.org/en-US/docs/Web/CSS/{page}", "args": ["page"] }, + "doc": { "open": "https://developer.mozilla.org/en-US/docs/{path}", "args": ["path"] }, + "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Adeveloper.mozilla.org+{query}", "args": ["query"] } + } +} diff --git a/clis/docs.python.org.json b/clis/docs.python.org.json new file mode 100644 index 0000000..64904c4 --- /dev/null +++ b/clis/docs.python.org.json @@ -0,0 +1,8 @@ +{ + "domain": "docs.python.org", + "commands": { + "library": { "open": "https://docs.python.org/3/library/{module}.html", "args": ["module"] }, + "doc": { "open": "https://docs.python.org/3/{path}.html", "args": ["path"] }, + "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Adocs.python.org+{query}", "args": ["query"] } + } +} diff --git a/clis/nodejs.org.json b/clis/nodejs.org.json new file mode 100644 index 0000000..180b12f --- /dev/null +++ b/clis/nodejs.org.json @@ -0,0 +1,7 @@ +{ + "domain": "nodejs.org", + "commands": { + "api": { "open": "https://nodejs.org/api/{module}.html", "args": ["module"] }, + "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Anodejs.org+{query}", "args": ["query"] } + } +} diff --git a/llms.txt b/llms.txt index d984407..2d884af 100644 --- a/llms.txt +++ b/llms.txt @@ -10,7 +10,7 @@ Key facts: - The budget is a target rather than a hard cap: a page that would finish within about four times it is printed whole, because a second command costs the agent far more than the lines the cut would have saved - The render leads with the page's main content and puts navigation, sidebar, and footer after it, so the budget is spent on what was asked for rather than on menus - Benchmarked at roughly 45x fewer tokens than reading raw HTML, with per-task numbers at https://github.com/only-cli/benchmarks -- Works on any mostly-static website; tuned shortcuts ship for Hacker News, Reddit, GitHub, X, LinkedIn (public guest views), DuckDuckGo, Bing, Stack Overflow (via its Atom feeds and the Stack Exchange API), Yahoo Finance (quotes, history, markets), Wikipedia (articles, search, and other language editions), and the AWS, Google Cloud, and Microsoft Learn documentation sites (guides, CLI reference, and search) +- Works on any mostly-static website; tuned shortcuts ship for Hacker News, Reddit, GitHub, X, LinkedIn (public guest views), DuckDuckGo, Bing, Stack Overflow (via its Atom feeds and the Stack Exchange API), Yahoo Finance (quotes, history, markets), Wikipedia (articles, search, and other language editions), the AWS, Google Cloud, and Microsoft Learn documentation sites (guides, CLI reference, and search), and the Python, MDN, and Node.js language references - JSON APIs render like pages: an endpoint that answers with JSON becomes one numbered item per record, with the fields that differ between items kept and the ones every item shares stated once, so a search endpoint reads like a results page for a few hundred tokens - A page that comes back with no readable text (JavaScript-only, a consent wall, a bot challenge) prints one line on stderr and exits 2, rather than reporting an empty render as a success. `--json` carries the same verdict as an `empty` field, so a caller can tell "nothing on this page" from "oc could not read this page" and fall back to a browser only when it is worth it - A shortcut is `oc [args]`: `oc hn top`, `oc reddit sub ClaudeAI`, `oc gh repo only-cli oc`, `oc ddg search claude code cli`, `oc learn doc azure/aks/what-is-aks`. Name the site by its short name, bare name, or domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`), the last argument takes every word after it so a query needs no quoting, and `oc sites` lists every site with its verbs. A shortcut resolves to a URL and then behaves exactly like `oc open ` diff --git a/skills/web-browsing-cli/SKILL.md b/skills/web-browsing-cli/SKILL.md index c467f5d..2508a34 100644 --- a/skills/web-browsing-cli/SKILL.md +++ b/skills/web-browsing-cli/SKILL.md @@ -27,9 +27,10 @@ None of these except `open`/`do`/`raw ` fetch anything; they replay the pag oc hn top oc reddit sub ClaudeAI oc gh repo only-cli oc oc wiki article Eiffel Tower oc wiki search anthropic oc wiki lang de Berlin oc ddg search claude code oc so question 231767 oc learn doc azure/aks/what-is-aks +oc py library json oc mdn js Array/map oc node api fs ``` -Sites: `hn`, `reddit`, `gh`, `x`, `linkedin`, `ddg`, `bing`, `so`, `finance`, `yt`, `aws`, `gcp`, `learn`, `wiki`. Name one by short name, bare name, or domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`). The last argument takes every word after it, so a query or title needs no quoting. `oc sites` lists every site with its verbs, which is cheaper than guessing one. +Sites: `hn`, `reddit`, `gh`, `x`, `linkedin`, `ddg`, `bing`, `so`, `finance`, `yt`, `aws`, `gcp`, `learn`, `wiki`, `py`, `mdn`, `node`. Name one by short name, bare name, or domain (`oc hn`, `oc ycombinator`, `oc news.ycombinator.com`). The last argument takes every word after it, so a query or title needs no quoting. `oc sites` lists every site with its verbs, which is cheaper than guessing one. Prefer a shortcut over a hand-built URL when one exists for the site, and prefer `oc wiki article ` over a search when you already know the article's name. diff --git a/src/sites.js b/src/sites.js index eade056..b5be2b7 100644 --- a/src/sites.js +++ b/src/sites.js @@ -22,6 +22,9 @@ const ALIASES = { finance: 'finance.yahoo.com', twitter: 'x.com', aws: 'docs.aws.amazon.com', + py: 'docs.python.org', + mdn: 'developer.mozilla.org', + node: 'nodejs.org', gcp: 'cloud.google.com', learn: 'learn.microsoft.com', wiki: 'wikipedia.org', diff --git a/tests/sites.test.js b/tests/sites.test.js index d0ea516..ec82ec4 100644 --- a/tests/sites.test.js +++ b/tests/sites.test.js @@ -67,3 +67,24 @@ test('oc sites lists every site once, with a verb line an agent can copy', () => } assert.match(text, /^oc hn <verb> \(ycombinator, news\.ycombinator\.com\): top \| new \| item <id>/m); }); + +test('language docs shortcuts resolve, and a doc path keeps its slashes', () => { + assert.equal( + resolveSite('py', ['library', 'json']).url, + 'https://docs.python.org/3/library/json.html'); + assert.equal( + resolveSite('python', ['doc', 'reference/datamodel']).url, + 'https://docs.python.org/3/reference/datamodel.html'); + assert.equal( + resolveSite('mdn', ['js', 'Array/map']).url, + 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/map'); + assert.equal( + resolveSite('mozilla', ['css', 'grid-template-columns']).url, + 'https://developer.mozilla.org/en-US/docs/Web/CSS/grid-template-columns'); + assert.equal( + resolveSite('node', ['api', 'fs']).url, + 'https://nodejs.org/api/fs.html'); + assert.equal( + resolveSite('nodejs.org', ['search', 'readFile options']).url, + 'https://html.duckduckgo.com/html/?q=site%3Anodejs.org+readFile%20options'); +}); From 4ae8cffdd2b8f813e0547cc105f759e54284650b Mon Sep 17 00:00:00 2001 From: only-cli <only-cli@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:44:56 -0400 Subject: [PATCH 02/11] readme: benchmark table covers the three new documentation pages --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c78d00d..5dbb713 100644 --- a/README.md +++ b/README.md @@ -137,13 +137,13 @@ Want a website on that list? Open a pull request, or an issue naming the site; s ## Benchmarks -Full methodology, per-task numbers, and other agents/models live in [only-cli/benchmarks](https://github.com/only-cli/benchmarks). The short version, measured with oc 0.4.0 on 2026-08-24 against live sites across a news front page, a Reddit discussion, a search results page, a stock quote, three cloud CLI reference pages, and more: +Full methodology, per-task numbers, and other agents/models live in [only-cli/benchmarks](https://github.com/only-cli/benchmarks). The short version, measured with oc 0.4.0 on 2026-08-24 against live sites across a news front page, a Reddit discussion, a search results page, a stock quote, three cloud CLI reference pages, the Python, MDN, and Node.js references, and more: -| method | tokens for 12 real pages | notes | +| method | tokens for 15 real pages | notes | | --- | ---: | --- | -| `oc open` | 9,487 | only method that returned real content on every page | -| Jina Reader | 90,929 | blocked on both Reddit pages, failed the stock quote page | -| raw HTML fetch | 1,183,149 | the stock quote page alone is 371,597 tokens | +| `oc open` | 10,973 | only method that returned real content on every page | +| Jina Reader | 170,505 | blocked on both Reddit pages, failed the LinkedIn page | +| raw HTML fetch | 1,535,791 | the stock quote page alone is 375,721 tokens | Read cost is one thing, but what an agent actually spends is another, so a second suite runs whole tasks end to end in Claude Code and compares `oc` From e834309363116ae8c15f95b250ac19f2cfb3a4e2 Mon Sep 17 00:00:00 2001 From: only-cli <only-cli@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:59:34 -0400 Subject: [PATCH 03/11] prototype a Sphinx search backend for the Python docs docs.python.org has no search server: its search page downloads the whole Sphinx index (searchindex.js, ~900KB gzipped) and ranks matches in the browser. oc now does the same ranking itself, so 'oc py search' answers from the site's own index instead of DuckDuckGo. The index is cached on disk for a day and never printed; what renders is a numbered result list that rides the normal open path, so 'do <n>' follows a result. A query that names a symbol exactly (json.dumps) links straight to its anchor. A site definition opts in with a 'sphinx' field naming the docs root, so any Sphinx site can use the backend later. --- README.md | 4 +- clis/docs.python.org.json | 2 +- skills/web-browsing-cli/SKILL.md | 2 +- src/cli.js | 32 ++++- src/sites.js | 18 ++- src/sphinx.js | 230 +++++++++++++++++++++++++++++++ tests/sites.test.js | 7 +- tests/sphinx.test.js | 66 +++++++++ 8 files changed, 349 insertions(+), 12 deletions(-) create mode 100644 src/sphinx.js create mode 100644 tests/sphinx.test.js diff --git a/README.md b/README.md index 5dbb713..63eb59f 100644 --- a/README.md +++ b/README.md @@ -125,13 +125,13 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum | AWS docs | `oc aws` (search via DuckDuckGo) | `guide <service> <page>`, `page <service> <guide> <page>`, `cli <command>`, `search <query>` | | Google Cloud docs | `oc gcp` (via docs.cloud.google.com, search via DuckDuckGo) | `docs <product>`, `page <product> <page>`, `gcloud <command>`, `search <query>` | | Microsoft Learn | `oc learn` (search via its RSS API) | `azure <page>`, `doc <path>`, `cli <command>`, `search <query>` | -| Python docs | `oc py` (search via DuckDuckGo) | `library <module>`, `doc <path>`, `search <query>` | +| Python docs | `oc py` (search via the docs' own index) | `library <module>`, `doc <path>`, `search <query>` | | MDN | `oc mdn` (search via DuckDuckGo) | `js <page>`, `css <page>`, `doc <path>`, `search <query>` | | Node.js docs | `oc node` (search via DuckDuckGo) | `api <module>`, `search <query>` | A shortcut only ever resolves to a URL and then takes the same path `oc open` does, so it changes nothing about what a page costs or how it reads. The last argument takes every word after it, so `oc ddg search claude code cli` and `oc aws search s3 lifecycle rules` need no quoting, and a path argument keeps its slashes, so `oc learn doc azure/aks/what-is-aks` reaches that page. -A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question <id>` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, Python, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. +A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question <id>` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. Want a website on that list? Open a pull request, or an issue naming the site; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/clis/docs.python.org.json b/clis/docs.python.org.json index 64904c4..9e44c1e 100644 --- a/clis/docs.python.org.json +++ b/clis/docs.python.org.json @@ -3,6 +3,6 @@ "commands": { "library": { "open": "https://docs.python.org/3/library/{module}.html", "args": ["module"] }, "doc": { "open": "https://docs.python.org/3/{path}.html", "args": ["path"] }, - "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Adocs.python.org+{query}", "args": ["query"] } + "search": { "sphinx": "https://docs.python.org/3/", "args": ["query"] } } } diff --git a/skills/web-browsing-cli/SKILL.md b/skills/web-browsing-cli/SKILL.md index 2508a34..b2f8df2 100644 --- a/skills/web-browsing-cli/SKILL.md +++ b/skills/web-browsing-cli/SKILL.md @@ -21,7 +21,7 @@ None of these except `open`/`do`/`raw <url>` fetch anything; they replay the pag ## Site shortcuts -`oc <site> <verb> [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. +`oc <site> <verb> [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. One verb is not a URL: `oc py search <query>` ranks the Python docs' own search index locally and prints results as a normal numbered page. ``` oc hn top oc reddit sub ClaudeAI oc gh repo only-cli oc diff --git a/src/cli.js b/src/cli.js index ddc4c75..991d7ee 100755 --- a/src/cli.js +++ b/src/cli.js @@ -4,6 +4,7 @@ import { fetchPage } from './fetch.js'; import { distill, toMarkdown, toHTML } from './distill.js'; import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js'; import { resolveSite, listSites } from './sites.js'; +import { sphinxSearch } from './sphinx.js'; import * as act from './act.js'; import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js'; @@ -111,11 +112,17 @@ async function main() { // A first word that is not a command may still be a site oc ships a // definition for, and a shortcut is only ever a URL, so it resolves to one // here and the rest of this function never learns it was not typed. + let sphinx = null; if (!COMMANDS.has(command)) { const site = resolveSite(command, args); if (!site) throw new Error(`unknown command '${command}', run oc --help`); - args = [site.url]; - command = 'open'; + if (site.sphinx) { + sphinx = site; + command = 'sphinx'; + } else { + args = [site.url]; + command = 'open'; + } } const sessionName = values.session || DEFAULT_SESSION; @@ -202,6 +209,27 @@ async function main() { if (failure) noContent(finalUrl, failure); return; } + case 'sphinx': { + // A Sphinx site's search index is fetched (or read back from its day + // cache) and ranked here, then the result list rides the exact `open` + // path: distilled, rendered, remembered, so `do <n>` follows a result. + // Only the list is ever printed; the index itself stays out of context. + const t0 = performance.now(); + const { url, html, via } = await sphinxSearch(sphinx.sphinx, sphinx.query); + const page = distill(html, url); + if (values.json) { + remember(page, sessionName); + console.log(JSON.stringify({ ...page, empty: false })); + return; + } + const { text, stats } = render(page, { budget: asked || 500 }); + remember(page, sessionName, stats.next); + console.log(text); + if (verbose) { + console.error(`~${stats.tokens} tokens, search index via ${via}, ${Math.round(performance.now() - t0)}ms`); + } + return; + } case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 })); case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 })); case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 })); diff --git a/src/sites.js b/src/sites.js index b5be2b7..1b01473 100644 --- a/src/sites.js +++ b/src/sites.js @@ -1,9 +1,12 @@ /** * Site shortcuts. `clis/*.json` names the URLs on a site worth reaching * directly, so `oc hn item 4711` gets there without the agent knowing that - * Hacker News spells it /item?id=. A shortcut is only ever a URL: it resolves - * to one and hands off to the same fetch and render path `oc open` uses, so - * nothing here can change what a page costs or how it reads. + * Hacker News spells it /item?id=. A shortcut is almost always a URL: it + * resolves to one and hands off to the same fetch and render path `oc open` + * uses, so nothing here can change what a page costs or how it reads. The + * one other shape is `sphinx`, for a docs site whose search only exists as a + * static index file; cli.js runs that search and renders the results like + * any other page. */ import { readdirSync, readFileSync } from 'node:fs'; @@ -30,7 +33,7 @@ const ALIASES = { wiki: 'wikipedia.org', }; -/** @typedef {{open: string, args?: string[]}} Shortcut */ +/** @typedef {{open?: string, sphinx?: string, args?: string[]}} Shortcut */ /** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */ /** @type {Map<string, Site>|null} */ @@ -87,7 +90,7 @@ const verbs = (site) => * instead, since the agent has the right site and only needs the verb list. * @param {string} name * @param {string[]} args - * @returns {{url: string, domain: string, command: string}|null} + * @returns {{url?: string, sphinx?: string, query?: string, domain: string, command: string}|null} */ export function resolveSite(name, args) { const site = sites().get(name.toLowerCase()); @@ -105,6 +108,11 @@ export function resolveSite(name, args) { // separate words ('oc ddg search claude code cli') works unquoted. const values = need.map((_, i) => i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]); + // A sphinx search has no URL to build: the query is ranked against the + // site's index locally, so it is handed back whole for cli.js to run. + if (def.sphinx) { + return { sphinx: def.sphinx, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; + } const url = need.reduce( (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)), def.open); diff --git a/src/sphinx.js b/src/sphinx.js new file mode 100644 index 0000000..c333a66 --- /dev/null +++ b/src/sphinx.js @@ -0,0 +1,230 @@ +/** + * Sphinx search backend. A Sphinx-built documentation site (docs.python.org, + * most Read the Docs projects) has no search server: its search page ships + * the site's entire full-text index as one static file, searchindex.js, and + * ranks matches in the visitor's browser. oc can run the same ranking here, + * so `search` on such a site answers from the site's own index instead of a + * third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB + * over the wire) but static, so it is cached on disk for a day and never + * printed: what reaches the agent is only the ranked result list. + */ + +import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { fetchPage } from './fetch.js'; + +// A documentation set rebuilds at most a few times a day, and a stale result +// list still links to live pages, so a day-old index is a fair trade against +// moving ~4MB per search. +const CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const MAX_RESULTS = 20; + +const cacheDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sphinx'); + +/** + * The index file is `Search.setIndex({...})`: JSON wrapped in one function + * call for the browser's benefit. Anything that does not parse that way is + * not a Sphinx index, which on a wrong or moved URL is the honest error. + * @param {string} js + * @returns {any} + */ +export function parseIndex(js) { + const start = js.indexOf('('); + const end = js.lastIndexOf(')'); + if (start >= 0 && end > start) { + try { + return JSON.parse(js.slice(start + 1, end)); + } catch {} + } + throw new Error('not a Sphinx search index'); +} + +// terms and titleterms store a bare number when a word appears in one +// document and an array when it appears in several. +const docsFor = (table, word) => { + const hit = table?.[word]; + return hit == null ? null : Array.isArray(hit) ? hit : [hit]; +}; + +/** + * Sphinx stems words before indexing ('threading' is stored as 'thread'), so + * an exact lookup misses common query spellings. Rather than shipping the + * Porter stemmer, try the word with common suffixes stripped, and only then + * a prefix scan: an index key that extends the word, or that the word + * extends, counts at reduced weight. + * @param {Record<string, number|number[]>} table + * @param {string} word + * @returns {{docs: number[], exact: boolean}|null} + */ +function lookup(table, word) { + const exact = docsFor(table, word); + if (exact) return { docs: exact, exact: true }; + for (const suffix of ['ing', 'ed', 'es', 's', 'e']) { + if (word.length - suffix.length >= 3 && word.endsWith(suffix)) { + const hit = docsFor(table, word.slice(0, -suffix.length)); + if (hit) return { docs: hit, exact: false }; + } + } + if (word.length >= 4) { + const docs = new Set(); + for (const key of Object.keys(table ?? {})) { + if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) { + for (const d of docsFor(table, key)) docs.add(d); + } + } + if (docs.size) return { docs: [...docs], exact: false }; + } + return null; +} + +/** + * An exact object hit ('json.dumps', or just 'dumps') beats any full-text + * rank: the index maps the symbol straight to its anchor on the page, so it + * goes at the top as a direct link. Only single-word queries can be symbols. + * @param {any} index + * @param {string} query + */ +function objectHits(index, query) { + const q = query.trim().toLowerCase(); + if (!q || q.includes(' ')) return []; + const hits = []; + for (const [prefix, entries] of Object.entries(index.objects ?? {})) { + for (const [doc, typeIdx, priority, anchor, name] of entries) { + const full = prefix ? `${prefix}.${name}` : name; + if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue; + hits.push({ + name: full, + type: index.objnames?.[typeIdx]?.[2] ?? '', + doc, + anchor: anchor === '' ? full : anchor, + priority, + }); + } + } + return hits.sort((a, b) => a.priority - b.priority).slice(0, 5); +} + +/** + * Rank the index against a query the way the site's own search page would: + * a document must match every word, a title hit weighs far more than a body + * hit, and only when nothing matches every word does any-word matching kick + * in, and then the result page says so. + * @param {any} index + * @param {string} query + */ +export function searchIndex(index, query) { + const words = [...new Set( + query.toLowerCase().split(/\s+/) + .map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, '')) + .filter(Boolean))]; + const scores = new Map(); + const matched = new Map(); + for (const word of words) { + const perDoc = new Map(); + const body = lookup(index.terms, word); + if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2); + const title = lookup(index.titleterms, word); + if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5)); + for (const [d, score] of perDoc) { + scores.set(d, (scores.get(d) ?? 0) + score); + matched.set(d, (matched.get(d) ?? 0) + 1); + } + } + let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length); + const partial = !docs.length && words.length > 1 && scores.size > 0; + if (partial) docs = [...scores.keys()]; + docs.sort((a, b) => + scores.get(b) - scores.get(a) + || String(index.titles[a]).localeCompare(String(index.titles[b]))); + return { + words, + partial, + total: docs.length, + objects: objectHits(index, query), + docs: docs.slice(0, MAX_RESULTS).map((d) => ({ + doc: d, + title: plainTitle(index.titles[d]) || index.docnames[d], + })), + }; +} + +// Titles in the index arrive as the HTML of the page's <h1>, markup and all +// (docs.python.org wraps module names in <code> spans), so they are flattened +// to text before they are placed on the results page. +const plainTitle = (t) => String(t).replace(/<[^>]*>/g, ''); + +const escapeHTML = (s) => String(s) + .replaceAll('&', '&').replaceAll('<', '<') + .replaceAll('>', '>').replaceAll('"', '"'); + +/** + * The result list becomes a small HTML page and rides the same distill and + * render path a fetched page does. That is what makes results numbered, + * followable with `do <n>`, and saved as session state, with nothing new for + * an agent to learn. + * @param {string} base + * @param {string} query + * @param {ReturnType<typeof searchIndex>} found + * @param {any} index + * @returns {string} + */ +export function resultsToHTML(base, query, found, index) { + const host = new URL(base).host; + const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href; + const items = found.objects.map((o) => + `<li><a href="${escapeHTML(`${pageURL(o.doc)}#${o.anchor}`)}">${escapeHTML(o.name)}</a>` + + ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}</li>`); + for (const r of found.docs) { + items.push(`<li><a href="${escapeHTML(pageURL(r.doc))}">${escapeHTML(r.title)}</a></li>`); + } + const partial = found.partial ? '; no page matches every word, so these match some' : ''; + const summary = items.length + ? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,` + + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:` + : `nothing in the site's own search index matches; try fewer or different words`; + return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}
` + + `

${summary}

` + + (items.length ? `
    ${items.join('')}
` : '') + + `
`; +} + +/** + * The index for one site, from the disk cache while it is fresh and from the + * network otherwise. A cache that cannot be written costs nothing but the + * refetch, the same policy session state follows. + * @param {string} base + */ +async function loadIndex(base) { + const file = join(cacheDir(), `${new URL(base).host}.js`); + try { + if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) { + return { index: parseIndex(readFileSync(file, 'utf8')), via: 'cache' }; + } + } catch {} + const { html } = await fetchPage(new URL('searchindex.js', base).href); + // Parse before caching, so a block page or an error never poisons the cache. + const index = parseIndex(html); + try { + mkdirSync(cacheDir(), { recursive: true }); + writeFileSync(file, html); + } catch {} + return { index, via: 'network' }; +} + +/** + * Search one Sphinx site. Returns the synthetic results page plus the URL the + * session should remember: the site's human search URL, so the state reads + * sensibly in `oc session` listings and error messages. + * @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/ + * @param {string} query + */ +export async function sphinxSearch(base, query) { + if (!query.trim()) throw new Error('usage: search '); + const { index, via } = await loadIndex(base); + return { + url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href, + html: resultsToHTML(base, query, searchIndex(index, query), index), + via, + }; +} diff --git a/tests/sites.test.js b/tests/sites.test.js index ec82ec4..979b392 100644 --- a/tests/sites.test.js +++ b/tests/sites.test.js @@ -52,7 +52,9 @@ test('every shipped definition is reachable and every url template is filled', ( for (const [name, site] of sites()) { for (const [verb, def] of Object.entries(site.commands)) { const args = (def.args ?? []).map((a) => `test-${a}`); - const { url } = resolveSite(name, [verb, ...args]); + const resolved = resolveSite(name, [verb, ...args]); + // A sphinx-backed verb resolves to a site root to search, not a URL. + const url = resolved.url ?? resolved.sphinx; assert.doesNotMatch(url, /[{}]/, `oc ${name} ${verb} left a template var in ${url}`); assert.equal(new URL(url).protocol, 'https:', `oc ${name} ${verb} is not https`); } @@ -87,4 +89,7 @@ test('language docs shortcuts resolve, and a doc path keeps its slashes', () => assert.equal( resolveSite('nodejs.org', ['search', 'readFile options']).url, 'https://html.duckduckgo.com/html/?q=site%3Anodejs.org+readFile%20options'); + const py = resolveSite('py', ['search', 'json', 'dumps']); + assert.equal(py.sphinx, 'https://docs.python.org/3/'); + assert.equal(py.query, 'json dumps'); }); diff --git a/tests/sphinx.test.js b/tests/sphinx.test.js new file mode 100644 index 0000000..4037540 --- /dev/null +++ b/tests/sphinx.test.js @@ -0,0 +1,66 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +const { parseIndex, searchIndex, resultsToHTML } = await import('../src/sphinx.js'); + +// A miniature docs.python.org: enough shape to exercise every lookup path +// (single-number terms, title boosts, stemmed words, object anchors) without +// a fixture file to drift out of date. +const INDEX = { + docnames: ['library/json', 'library/threading', 'tutorial/appendix'], + titles: [ + 'json - JSON encoder and decoder', + 'threading - Thread-based parallelism', + 'Appendix', + ], + terms: { json: 0, thread: [1, 2], socket: [2] }, + titleterms: { json: [0], thread: [1] }, + objects: { json: [[0, 3, 1, '', 'dumps']] }, + objnames: { 3: ['py', 'function', 'Python function'] }, +}; +const BASE = 'https://docs.python.org/3/'; + +test('parseIndex unwraps Search.setIndex() and refuses anything else', () => { + assert.equal(parseIndex('Search.setIndex({"a": 1})').a, 1); + assert.throws(() => parseIndex('a block page'), /not a Sphinx search index/); + assert.throws(() => parseIndex('Search.setIndex(undefined)'), /not a Sphinx search index/); +}); + +test('a title hit outranks body hits, and one-doc terms stored as a bare number work', () => { + const found = searchIndex(INDEX, 'thread'); + assert.deepEqual(found.docs.map((d) => d.doc), [1, 2]); + assert.equal(searchIndex(INDEX, 'json').docs[0].doc, 0); +}); + +test('a word Sphinx stemmed away still matches through its stem', () => { + // The index stores 'thread'; a query typed as English says 'threading'. + const found = searchIndex(INDEX, 'threading'); + assert.equal(found.docs[0].doc, 1); +}); + +test('titles are flattened to text before they reach the results page', () => { + const found = searchIndex(INDEX, 'json'); + assert.equal(found.docs[0].title, 'json - JSON encoder and decoder'); +}); + +test('every word must match, and when none can, any-word results say so', () => { + // 'json' hits doc 0, 'socket' hits doc 2, nothing hits both. + const found = searchIndex(INDEX, 'json socket'); + assert.equal(found.partial, true); + const html = resultsToHTML(BASE, 'json socket', found, INDEX); + assert.match(html, /no page matches every word/); +}); + +test('an exact symbol query becomes a direct link to its anchor', () => { + const found = searchIndex(INDEX, 'json.dumps'); + assert.equal(found.objects.length, 1); + const html = resultsToHTML(BASE, 'json.dumps', found, INDEX); + assert.match(html, /href="https:\/\/docs\.python\.org\/3\/library\/json\.html#json\.dumps"/); + assert.match(html, /Python function/); +}); + +test('no matches renders an honest empty page, not an error', () => { + const html = resultsToHTML(BASE, 'zzqqxx', searchIndex(INDEX, 'zzqqxx'), INDEX); + assert.match(html, /nothing in the site's own search index matches/); + assert.doesNotMatch(html, /
    /); +}); From d56ddb30a03421e9c7b09786d4b073059d7cb287 Mon Sep 17 00:00:00 2001 From: only-cli Date: Mon, 24 Aug 2026 10:18:05 -0400 Subject: [PATCH 04/11] fix: never let a '<' survive title flattening CodeQL flagged the tag strip in plainTitle: requiring a closing '>' means a title whose markup never closes keeps its '' is optional now, so every '<' starts a strip and none reaches the results page, and the output is trimmed. Both call sites already HTML-escape the result; this hardens the flattening itself. Covered by a new test. --- src/sphinx.js | 8 ++++++-- tests/sphinx.test.js | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/sphinx.js b/src/sphinx.js index c333a66..e8a0be8 100644 --- a/src/sphinx.js +++ b/src/sphinx.js @@ -151,8 +151,12 @@ export function searchIndex(index, query) { // Titles in the index arrive as the HTML of the page's

    , markup and all // (docs.python.org wraps module names in spans), so they are flattened -// to text before they are placed on the results page. -const plainTitle = (t) => String(t).replace(/<[^>]*>/g, ''); +// to text before they are placed on the results page. The '>' is optional so +// a tag the title never closes is dropped too, not left standing as ' String(t).replace(/<[^>]*>?/g, '').trim(); const escapeHTML = (s) => String(s) .replaceAll('&', '&').replaceAll('<', '<') diff --git a/tests/sphinx.test.js b/tests/sphinx.test.js index 4037540..7964784 100644 --- a/tests/sphinx.test.js +++ b/tests/sphinx.test.js @@ -43,6 +43,15 @@ test('titles are flattened to text before they reach the results page', () => { assert.equal(found.docs[0].title, 'json - JSON encoder and decoder'); }); +test('a tag the title never closes is stripped, not left standing', () => { + // 'json encoder ', summary: 'Holds key-value pairs.' }, + ], + metadata: { total: { value: 2696 } }, +}; + +test('results map through the named fields and resolve against the endpoint', () => { + const html = resultsToHTML(DEF, 'map', DATA, API); + assert.match(html, /href="https:\/\/developer\.mozilla\.org\/en-US\/docs\/Web\/JavaScript\/Reference\/Global_Objects\/Array\/map"/); + assert.match(html, /Array\.prototype\.map\(\)/); + assert.match(html, /Creates a new array/); +}); + +test('the site total is reported, and result count is what the page shows', () => { + assert.match(resultsToHTML(DEF, 'map', DATA, API), /2696 pages match, ranked by the site's own search, top 2 shown:/); +}); + +test('a title is response data, never markup on the results page', () => { + const html = resultsToHTML(DEF, 'map', DATA, API); + assert.doesNotMatch(html, /', 'Array', 'Array.html#method-i-evil', '', ''], + ], + }, +})}`; + +const entries = () => buildRdocEntries(parseRdocIndex(INDEX)); + +test('only an RDoc index parses, so an error page never enters the cache', () => { + assert.ok(parseRdocIndex(INDEX).index.info.length > 0); + assert.throws(() => parseRdocIndex('blocked'), /not an RDoc search index/); + assert.throws(() => parseRdocIndex('var search_data = {"unrelated": true}'), /not an RDoc search index/); +}); + +test('rows read back as the headings a rubyist expects', () => { + const all = entries(); + assert.equal(all.find((e) => e.path.includes('method-i-dig') && e.text.startsWith('Array')).text, 'Array#dig(*args)'); + assert.equal(all.find((e) => e.path.includes('method-c-new')).text, 'Array.new(size, default)'); + assert.equal(all.find((e) => e.path === 'Array.html').text, 'Array'); + assert.equal(all.find((e) => e.path === 'contributing_md.html').kind, 'page'); +}); + +test('a symbol query finds its methods across classes, exact name first', () => { + const found = searchEntries(entries(), 'dig'); + assert.equal(found.total, 2); + assert.deepEqual(found.hits.map((e) => e.text).sort(), ['Array#dig(*args)', 'Hash#dig(*args)']); + assert.equal(found.partial, false); +}); + +test('a class and method pair narrows to the one entry matching both words', () => { + const found = searchEntries(entries(), 'array each_slice'); + assert.equal(found.total, 1); + assert.equal(found.hits[0].text, 'Array#each_slice(n)'); +}); + +test('results link into the live docs, anchors intact, and name their kind', () => { + const html = resultsToHTML(BASE, 'dig', searchEntries(entries(), 'dig')); + assert.match(html, /href="https:\/\/docs\.ruby-lang\.org\/en\/3\.4\/Array\.html#method-i-dig"/); + assert.match(html, /Array#dig\(\*args\)<\/a> method/); + assert.match(html, /2 entries match in the docs' own index, ranked locally:/); +}); + +test('an index row is data, never markup on the results page', () => { + const html = resultsToHTML(BASE, 'evil', searchEntries(entries(), 'evil')); + assert.doesNotMatch(html, /