diff --git a/clis/stackoverflow.com.json b/clis/stackoverflow.com.json new file mode 100644 index 0000000..d8052d2 --- /dev/null +++ b/clis/stackoverflow.com.json @@ -0,0 +1,9 @@ +{ + "domain": "stackoverflow.com", + "commands": { + "question": { "open": "https://stackoverflow.com/feeds/question/{id}", "args": ["id"] }, + "tag": { "open": "https://stackoverflow.com/feeds/tag?tagnames={name}&sort=newest", "args": ["name"] }, + "user": { "open": "https://stackoverflow.com/feeds/user/{id}", "args": ["id"] }, + "recent": { "open": "https://stackoverflow.com/feeds" } + } +} diff --git a/src/distill.js b/src/distill.js index d5b4ffd..17134db 100644 --- a/src/distill.js +++ b/src/distill.js @@ -35,7 +35,7 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim(); * @returns {Page} */ export function distill(html, url = '') { - const { document } = parseHTML(html); + const { document } = parseHTML(feedToHTML(html) ?? html); const title = clean(document.querySelector('title')?.textContent ?? ''); /** @type {Block[]} */ const blocks = []; @@ -107,7 +107,7 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document * @param {string} html */ function cleanDocument(html) { - const { document } = parseHTML(html); + const { document } = parseHTML(feedToHTML(html) ?? html); for (const tag of DROP) { for (const el of [...document.querySelectorAll(tag)]) el.remove(); } @@ -145,6 +145,56 @@ export function toHTML(html) { return el ? el.innerHTML.trim() : ''; } +/** + * Sites behind hard bot challenges often leave their Atom or RSS feeds open: + * Stack Overflow challenges every HTML page but publishes full question and + * answer bodies under /feeds. A feed is XML with the real content escaped + * inside each entry, so this converts one into a plain HTML document and + * everything downstream stays unchanged. Returns null for non-feed input. + * @param {string} text + * @returns {string | null} + */ +export function feedToHTML(text) { + const head = text.slice(0, 2000); + if (!/<(feed|rss)[\s>]/i.test(head) || /<(html|body)[\s>]/i.test(head)) return null; + // RSS wraps bodies in CDATA, which an HTML parser reads as a comment and + // drops. Escaping the section turns it into ordinary text, the same shape + // Atom feeds already use. + const xml = text.replace(//g, (_, inner) => + inner.replace(/&/g, '&').replace(//g, '>')); + const { document } = parseHTML(xml); + const root = document.querySelector('feed, rss'); + if (!root) return null; + const esc = (s) => s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + // Feed elements are unknown to the HTML parser, so self-closed ones like + // stay open and swallow their siblings. Descendant queries + // still land, because a closing or pops the whole pile. + const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? ''); + const feedTitle = field(root, 'title'); + const parts = []; + for (const entry of root.querySelectorAll('entry, item')) { + const title = field(entry, 'title'); + const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href') + ?? entry.querySelector('link[href]')?.getAttribute('href') + ?? field(entry, 'guid'); + const author = field(entry, 'author name') || field(entry, 'author'); + const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10); + const byline = [author && `by ${author}`, date].filter(Boolean).join(', '); + // Atom escapes the entry body, so textContent of content/summary is the + // HTML itself, ready to be embedded and parsed like any page. + const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? ''; + parts.push('
'); + if (title) parts.push(`

${esc(title)}

`); + if (byline || href) { + parts.push(`

${esc(byline)}${href ? ` open` : ''}

`); + } + parts.push(body, '
'); + } + // A full skeleton, because linkedom treats the first element of a bare + // multi-rooted fragment as the whole document and drops its siblings. + return `${esc(feedTitle)}\n${parts.join('\n')}\n`; +} + /** * Adjacent text nodes arrive fragmented (one per inline element boundary). * Merging them is what turns DOM noise into readable lines. diff --git a/src/fetch.js b/src/fetch.js index 6262e3c..9bc84ed 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -61,7 +61,7 @@ async function viaFetch(target) { throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${target}`); } const type = res.headers.get('content-type') ?? ''; - if (type && !type.includes('html')) { + if (type && !type.includes('html') && !type.includes('xml')) { throw new Error(`not an HTML page (${type.split(';')[0]}), nothing to distill`); } return { url: res.url, html: await res.text(), status: res.status, via: 'fetch' };