${esc(title)}
`); + if (byline || href) { + parts.push(`${esc(byline)}${href ? ` open` : ''}
`); + } + parts.push(body, 'diff --git a/clis/stackoverflow.com.json b/clis/stackoverflow.com.json
new file mode 100644
index 0000000..d8052d2
--- /dev/null
+++ b/clis/stackoverflow.com.json
@@ -0,0 +1,9 @@
+{
+ "domain": "stackoverflow.com",
+ "commands": {
+ "question": { "open": "https://stackoverflow.com/feeds/question/{id}", "args": ["id"] },
+ "tag": { "open": "https://stackoverflow.com/feeds/tag?tagnames={name}&sort=newest", "args": ["name"] },
+ "user": { "open": "https://stackoverflow.com/feeds/user/{id}", "args": ["id"] },
+ "recent": { "open": "https://stackoverflow.com/feeds" }
+ }
+}
diff --git a/src/distill.js b/src/distill.js
index d5b4ffd..17134db 100644
--- a/src/distill.js
+++ b/src/distill.js
@@ -35,7 +35,7 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
* @returns {Page}
*/
export function distill(html, url = '') {
- const { document } = parseHTML(html);
+ const { document } = parseHTML(feedToHTML(html) ?? html);
const title = clean(document.querySelector('title')?.textContent ?? '');
/** @type {Block[]} */
const blocks = [];
@@ -107,7 +107,7 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
* @param {string} html
*/
function cleanDocument(html) {
- const { document } = parseHTML(html);
+ const { document } = parseHTML(feedToHTML(html) ?? html);
for (const tag of DROP) {
for (const el of [...document.querySelectorAll(tag)]) el.remove();
}
@@ -145,6 +145,56 @@ export function toHTML(html) {
return el ? el.innerHTML.trim() : '';
}
+/**
+ * Sites behind hard bot challenges often leave their Atom or RSS feeds open:
+ * Stack Overflow challenges every HTML page but publishes full question and
+ * answer bodies under /feeds. A feed is XML with the real content escaped
+ * inside each entry, so this converts one into a plain HTML document and
+ * everything downstream stays unchanged. Returns null for non-feed input.
+ * @param {string} text
+ * @returns {string | null}
+ */
+export function feedToHTML(text) {
+ const head = text.slice(0, 2000);
+ if (!/<(feed|rss)[\s>]/i.test(head) || /<(html|body)[\s>]/i.test(head)) return null;
+ // RSS wraps bodies in CDATA, which an HTML parser reads as a comment and
+ // drops. Escaping the section turns it into ordinary text, the same shape
+ // Atom feeds already use.
+ const xml = text.replace(//g, (_, inner) =>
+ inner.replace(/&/g, '&').replace(//g, '>'));
+ const { document } = parseHTML(xml);
+ const root = document.querySelector('feed, rss');
+ if (!root) return null;
+ const esc = (s) => s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"');
+ // Feed elements are unknown to the HTML parser, so self-closed ones like
+ // ${esc(byline)}${href ? ` open` : ''}${esc(title)}
`);
+ if (byline || href) {
+ parts.push(`