From 5706aab2a18c413f6cbcf6b0ade2dac5c52d3098 Mon Sep 17 00:00:00 2001 From: only-cli Date: Mon, 24 Aug 2026 10:29:28 -0400 Subject: [PATCH] let MDN search ask MDN MDN renders its search results client-side, so 'oc mdn search' has gone through DuckDuckGo with a site: filter. But the page it renders gets its results from a public JSON endpoint, /api/v1/search, which returns the site's own ranking with a summary per result in one ~11KB response. A new 'api' shortcut shape names such an endpoint and which response fields hold the list, title, URL, and snippet; the response becomes the same synthetic results page a Sphinx search produces and rides the normal render path, so 'do ' follows a result. First view is ~520 tokens and works for any site whose search answers as JSON. --- README.md | 4 +- clis/developer.mozilla.org.json | 2 +- skills/web-browsing-cli/SKILL.md | 2 +- src/apisearch.js | 80 ++++++++++++++++++++++++++++++++ src/cli.js | 26 ++++++----- src/sites.js | 17 ++++--- src/sphinx.js | 2 +- tests/apisearch.test.js | 55 ++++++++++++++++++++++ tests/sites.test.js | 10 +++- 9 files changed, 173 insertions(+), 25 deletions(-) create mode 100644 src/apisearch.js create mode 100644 tests/apisearch.test.js diff --git a/README.md b/README.md index 63eb59f..6365cda 100644 --- a/README.md +++ b/README.md @@ -126,12 +126,12 @@ Works on any mostly-static site with no per-site setup: news sites, blogs, docum | Google Cloud docs | `oc gcp` (via docs.cloud.google.com, search via DuckDuckGo) | `docs `, `page `, `gcloud `, `search ` | | Microsoft Learn | `oc learn` (search via its RSS API) | `azure `, `doc `, `cli `, `search ` | | Python docs | `oc py` (search via the docs' own index) | `library `, `doc `, `search ` | -| MDN | `oc mdn` (search via DuckDuckGo) | `js `, `css `, `doc `, `search ` | +| MDN | `oc mdn` (search via the site's own API) | `js `, `css `, `doc `, `search ` | | Node.js docs | `oc node` (search via DuckDuckGo) | `api `, `search ` | A shortcut only ever resolves to a URL and then takes the same path `oc open` does, so it changes nothing about what a page costs or how it reads. The last argument takes every word after it, so `oc ddg search claude code cli` and `oc aws search s3 lifecycle rules` need no quoting, and a path argument keeps its slashes, so `oc learn doc azure/aks/what-is-aks` reaches that page. -A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, MDN, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. +A few of these (X, Stack Overflow, YouTube, Microsoft Learn search) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, inline data, or public API the page already ships without a login. Stack Overflow search goes through the Stack Exchange API, and each result prints its `question_id`: read one with the `question ` feed rather than following its link, since the question page itself answers a bot challenge instead of the question. AWS, Google Cloud, and Node.js render docs search client-side or ship none at all, so their `search` goes through DuckDuckGo with a baked-in `site:` filter instead. Python's docs are built with Sphinx, which publishes the site's full-text search index as one static file, so `oc py search` fetches that index (cached on disk for a day), ranks it locally, and prints a numbered result list; a query that names a symbol exactly, like `json.dumps`, links straight to its anchor. The same backend will work for any Sphinx site, including most Read the Docs projects. MDN also renders its search client-side, but the page gets its results from a public JSON endpoint, so `oc mdn search` asks that endpoint directly and prints the site's own ranking; that `api` shape in a site definition works for any site whose search answers as JSON. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed. Want a website on that list? Open a pull request, or an issue naming the site; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/clis/developer.mozilla.org.json b/clis/developer.mozilla.org.json index 46c63d5..50ecaf1 100644 --- a/clis/developer.mozilla.org.json +++ b/clis/developer.mozilla.org.json @@ -4,6 +4,6 @@ "js": { "open": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/{page}", "args": ["page"] }, "css": { "open": "https://developer.mozilla.org/en-US/docs/Web/CSS/{page}", "args": ["page"] }, "doc": { "open": "https://developer.mozilla.org/en-US/docs/{path}", "args": ["path"] }, - "search": { "open": "https://html.duckduckgo.com/html/?q=site%3Adeveloper.mozilla.org+{query}", "args": ["query"] } + "search": { "api": "https://developer.mozilla.org/api/v1/search?q={query}&locale=en-US", "page": "https://developer.mozilla.org/en-US/search?q={query}", "results": "documents", "fields": { "title": "title", "url": "mdn_url", "text": "summary" }, "total": "metadata.total.value", "args": ["query"] } } } diff --git a/skills/web-browsing-cli/SKILL.md b/skills/web-browsing-cli/SKILL.md index b2f8df2..7ca0cfc 100644 --- a/skills/web-browsing-cli/SKILL.md +++ b/skills/web-browsing-cli/SKILL.md @@ -21,7 +21,7 @@ None of these except `open`/`do`/`raw ` fetch anything; they replay the pag ## Site shortcuts -`oc [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. One verb is not a URL: `oc py search ` ranks the Python docs' own search index locally and prints results as a normal numbered page. +`oc [args]` resolves to a URL and then behaves exactly like `open` on it, so it costs the same and reads the same. It saves guessing a URL shape and, on a few sites, points at the feed or public API that answers without a login. Two verbs are not URLs: `oc py search ` ranks the Python docs' own search index locally, and `oc mdn search ` asks MDN's own search API; both print results as a normal numbered page. ``` oc hn top oc reddit sub ClaudeAI oc gh repo only-cli oc diff --git a/src/apisearch.js b/src/apisearch.js new file mode 100644 index 0000000..ae517fd --- /dev/null +++ b/src/apisearch.js @@ -0,0 +1,80 @@ +/** + * JSON search API backend. Some sites render search results only in the + * browser but run their search behind a public JSON endpoint the results + * page calls (MDN's /api/v1/search). The site definition names that endpoint + * and which fields of the response hold the result list, each result's + * title, URL, and snippet; the ranked answer the site already computed + * becomes the same synthetic results page a Sphinx search produces, riding + * the normal distill and render path, so `do ` follows a result and + * nothing else is new. Unlike the Sphinx backend nothing is fetched but the + * one response, so there is no cache to keep. + */ + +import { fetchPage } from './fetch.js'; +import { escapeHTML } from './sphinx.js'; + +const MAX_RESULTS = 20; + +// Response fields are named by dot path ('metadata.total.value'), so a +// definition can reach into whatever shape a site's API answers with. +const pick = (obj, path) => + String(path).split('.').reduce((o, key) => (o == null ? undefined : o[key]), obj); + +/** + * The result list becomes a small HTML page, the same move sphinx.js makes + * and for the same reason: numbered results, followable with `do `, + * saved as session state. Result URLs are often paths ('/en-US/docs/...'), + * so they resolve against the endpoint they came from. + * @param {{results?: string, fields?: Record, total?: string}} def + * @param {string} query + * @param {any} data - the endpoint's parsed JSON response + * @param {string} apiURL - the URL the response came from + * @returns {string} + */ +export function resultsToHTML(def, query, data, apiURL) { + const host = new URL(apiURL).host; + const fields = def.fields ?? {}; + const list = pick(data, def.results ?? 'results'); + const items = (Array.isArray(list) ? list : []).slice(0, MAX_RESULTS).map((item) => { + const href = new URL(String(pick(item, fields.url ?? 'url') ?? ''), apiURL).href; + const title = String(pick(item, fields.title ?? 'title') ?? href); + const text = fields.text ? String(pick(item, fields.text) ?? '').trim() : ''; + return `
  • ${escapeHTML(title)}` + + `${text ? ` ${escapeHTML(text)}` : ''}
    • `; + }); + const total = Number(def.total ? pick(data, def.total) : NaN); + const count = Number.isFinite(total) && total >= items.length ? total : items.length; + const summary = items.length + ? `${count} page${count === 1 ? '' : 's'} match, ranked by the site's own search` + + `${count > items.length ? `, top ${items.length} shown` : ''}:` + : `nothing in the site's own search matches; try fewer or different words`; + return `${escapeHTML(host)} search: ${escapeHTML(query)}
    ` + + `

    ${summary}

    ` + + (items.length ? `
      ${items.join('')}
    ` : '') + + `
    `; +} + +/** + * Search one site through its JSON endpoint. Returns the synthetic results + * page plus the URL the session should remember: the site's human search + * page when the definition names one, so session listings read sensibly. + * @param {{api: string, page?: string} & Parameters[0]} def + * @param {string} query + */ +export async function apiSearch(def, query) { + if (!query.trim()) throw new Error('usage: search '); + const q = encodeURIComponent(query); + const apiURL = def.api.replaceAll('{query}', q); + const { url: finalURL, html: body } = await fetchPage(apiURL); + let data; + try { + data = JSON.parse(body); + } catch { + throw new Error(`the search API at ${apiURL} did not answer with JSON`); + } + return { + url: (def.page ?? def.api).replaceAll('{query}', q), + html: resultsToHTML(def, query, data, finalURL), + via: 'api', + }; +} diff --git a/src/cli.js b/src/cli.js index 991d7ee..451927a 100755 --- a/src/cli.js +++ b/src/cli.js @@ -5,6 +5,7 @@ import { distill, toMarkdown, toHTML } from './distill.js'; import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js'; import { resolveSite, listSites } from './sites.js'; import { sphinxSearch } from './sphinx.js'; +import { apiSearch } from './apisearch.js'; import * as act from './act.js'; import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js'; @@ -112,13 +113,13 @@ async function main() { // A first word that is not a command may still be a site oc ships a // definition for, and a shortcut is only ever a URL, so it resolves to one // here and the rest of this function never learns it was not typed. - let sphinx = null; + let search = null; if (!COMMANDS.has(command)) { const site = resolveSite(command, args); if (!site) throw new Error(`unknown command '${command}', run oc --help`); - if (site.sphinx) { - sphinx = site; - command = 'sphinx'; + if (site.sphinx || site.api) { + search = site; + command = 'search'; } else { args = [site.url]; command = 'open'; @@ -209,13 +210,16 @@ async function main() { if (failure) noContent(finalUrl, failure); return; } - case 'sphinx': { - // A Sphinx site's search index is fetched (or read back from its day - // cache) and ranked here, then the result list rides the exact `open` - // path: distilled, rendered, remembered, so `do ` follows a result. - // Only the list is ever printed; the index itself stays out of context. + case 'search': { + // A search oc runs itself: a Sphinx site's index is fetched (or read + // back from its day cache) and ranked here, a JSON search API is asked + // directly. Either way the result list rides the exact `open` path: + // distilled, rendered, remembered, so `do ` follows a result. Only + // the list is ever printed; index and response stay out of context. const t0 = performance.now(); - const { url, html, via } = await sphinxSearch(sphinx.sphinx, sphinx.query); + const { url, html, via } = search.sphinx + ? await sphinxSearch(search.sphinx, search.query) + : await apiSearch(search.api, search.query); const page = distill(html, url); if (values.json) { remember(page, sessionName); @@ -226,7 +230,7 @@ async function main() { remember(page, sessionName, stats.next); console.log(text); if (verbose) { - console.error(`~${stats.tokens} tokens, search index via ${via}, ${Math.round(performance.now() - t0)}ms`); + console.error(`~${stats.tokens} tokens, results via ${via}, ${Math.round(performance.now() - t0)}ms`); } return; } diff --git a/src/sites.js b/src/sites.js index 1b01473..861d991 100644 --- a/src/sites.js +++ b/src/sites.js @@ -4,9 +4,9 @@ * Hacker News spells it /item?id=. A shortcut is almost always a URL: it * resolves to one and hands off to the same fetch and render path `oc open` * uses, so nothing here can change what a page costs or how it reads. The - * one other shape is `sphinx`, for a docs site whose search only exists as a - * static index file; cli.js runs that search and renders the results like - * any other page. + * two other shapes are searches cli.js runs itself and renders like any + * other page: `sphinx`, for a docs site whose search only exists as a + * static index file, and `api`, for a site whose search answers as JSON. */ import { readdirSync, readFileSync } from 'node:fs'; @@ -33,7 +33,7 @@ const ALIASES = { wiki: 'wikipedia.org', }; -/** @typedef {{open?: string, sphinx?: string, args?: string[]}} Shortcut */ +/** @typedef {{open?: string, sphinx?: string, api?: string, page?: string, results?: string, fields?: Record, total?: string, args?: string[]}} Shortcut */ /** @typedef {{domain: string, commands: Record}} Site */ /** @type {Map|null} */ @@ -90,7 +90,7 @@ const verbs = (site) => * instead, since the agent has the right site and only needs the verb list. * @param {string} name * @param {string[]} args - * @returns {{url?: string, sphinx?: string, query?: string, domain: string, command: string}|null} + * @returns {{url?: string, sphinx?: string, api?: Shortcut, query?: string, domain: string, command: string}|null} */ export function resolveSite(name, args) { const site = sites().get(name.toLowerCase()); @@ -108,11 +108,14 @@ export function resolveSite(name, args) { // separate words ('oc ddg search claude code cli') works unquoted. const values = need.map((_, i) => i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]); - // A sphinx search has no URL to build: the query is ranked against the - // site's index locally, so it is handed back whole for cli.js to run. + // A sphinx or API search has no page URL to build: the query is handed + // back whole for cli.js to run against the site's own search. if (def.sphinx) { return { sphinx: def.sphinx, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; } + if (def.api) { + return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb }; + } const url = need.reduce( (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)), def.open); diff --git a/src/sphinx.js b/src/sphinx.js index a5f428c..04c84c0 100644 --- a/src/sphinx.js +++ b/src/sphinx.js @@ -167,7 +167,7 @@ const plainTitle = (t) => { return text.trim(); }; -const escapeHTML = (s) => String(s) +export const escapeHTML = (s) => String(s) .replaceAll('&', '&').replaceAll('<', '<') .replaceAll('>', '>').replaceAll('"', '"'); diff --git a/tests/apisearch.test.js b/tests/apisearch.test.js new file mode 100644 index 0000000..52c2a44 --- /dev/null +++ b/tests/apisearch.test.js @@ -0,0 +1,55 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +const { resultsToHTML } = await import('../src/apisearch.js'); + +// The MDN-shaped definition oc ships, minus the endpoint itself: which +// response fields hold the list, and what each result calls its parts. +const DEF = { + results: 'documents', + fields: { title: 'title', url: 'mdn_url', text: 'summary' }, + total: 'metadata.total.value', +}; +const API = 'https://developer.mozilla.org/api/v1/search?q=map'; + +// A miniature /api/v1/search answer: a relative URL, a nested total, and a +// title that would be markup if it were ever trusted. +const DATA = { + documents: [ + { mdn_url: '/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/map', title: 'Array.prototype.map()', summary: 'Creates a new array from results of a callback.' }, + { mdn_url: '/en-US/docs/Web/API/Map', title: 'Map ', summary: 'Holds key-value pairs.' }, + ], + metadata: { total: { value: 2696 } }, +}; + +test('results map through the named fields and resolve against the endpoint', () => { + const html = resultsToHTML(DEF, 'map', DATA, API); + assert.match(html, /href="https:\/\/developer\.mozilla\.org\/en-US\/docs\/Web\/JavaScript\/Reference\/Global_Objects\/Array\/map"/); + assert.match(html, /Array\.prototype\.map\(\)/); + assert.match(html, /Creates a new array/); +}); + +test('the site total is reported, and result count is what the page shows', () => { + assert.match(resultsToHTML(DEF, 'map', DATA, API), /2696 pages match, ranked by the site's own search, top 2 shown:/); +}); + +test('a title is response data, never markup on the results page', () => { + const html = resultsToHTML(DEF, 'map', DATA, API); + assert.doesNotMatch(html, /