From 853bbca853c575c74a9d98cb2321f759d7b7f688 Mon Sep 17 00:00:00 2001 From: only-cli Date: Tue, 18 Aug 2026 10:40:08 -0400 Subject: [PATCH] feed reading: offline tests, readme row, and a raw-mode title fix The feed fixture covers Atom entries (escaped bodies, self-closed categories, bylines) and an RSS item with a CDATA body. Raw markdown of a feed exposed an old quirk: cleanDocument removed the head before toMarkdown read the title, so any page whose body lacked a matching h1 lost its title in raw mode. cleanDocument now captures the title first and returns both. --- README.md | 5 ++++- src/distill.js | 9 +++++---- tests/distill.test.js | 37 ++++++++++++++++++++++++++++++++++++- tests/pages/feed.xml | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 tests/pages/feed.xml diff --git a/README.md b/README.md index 6d725fe..ae00cfd 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,11 @@ only-cli works on any mostly-static website with no per-site setup: news sites, | LinkedIn | linkedin.com | `profile `, `company `, `jobs ` (public guest views) | | DuckDuckGo | duckduckgo.com | `search `, `lite ` | | Bing | bing.com | `search `, `news ` | +| Stack Overflow | stackoverflow.com (via Atom feeds) | `question `, `tag `, `user `, `recent` | -Not supported yet: pages that only render with JavaScript (a headless fallback is planned for v0.3), sites behind logins (sessions land in v0.2), and sites with hard bot challenges. Adding a site shortcut is a small JSON file; see [CONTRIBUTING.md](CONTRIBUTING.md). +The engine also renders Atom and RSS feeds as regular pages. That is how Stack Overflow works: the site serves every HTML page a Cloudflare challenge, but publishes full question and answer bodies under `/feeds`, so `oc open stackoverflow.com/feeds/question/11227809` returns the question and its top answers in about 500 tokens. The same trick applies to any site that gates its pages but leaves its feeds open. + +Not supported yet: pages that only render with JavaScript (a headless fallback is planned for v0.3), sites behind logins (sessions land in v0.2), and sites with hard bot challenges that do not expose feeds. Adding a site shortcut is a small JSON file; see [CONTRIBUTING.md](CONTRIBUTING.md). ## Benchmarks diff --git a/src/distill.js b/src/distill.js index 17134db..6e6c955 100644 --- a/src/distill.js +++ b/src/distill.js @@ -108,6 +108,8 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document */ function cleanDocument(html) { const { document } = parseHTML(feedToHTML(html) ?? html); + // Read the title before the sweep below removes the head with it. + const title = clean(document.querySelector('title')?.textContent ?? ''); for (const tag of DROP) { for (const el of [...document.querySelectorAll(tag)]) el.remove(); } @@ -115,7 +117,7 @@ function cleanDocument(html) { for (const el of [...document.querySelectorAll('[style]')]) { if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove(); } - return document; + return { document, title }; } /** @@ -125,8 +127,7 @@ function cleanDocument(html) { * @returns {string} */ export function toMarkdown(html) { - const document = cleanDocument(html); - const title = clean(document.querySelector('title')?.textContent ?? ''); + const { document, title } = cleanDocument(html); const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' }); const el = bodyOf(document); const body = el ? turndown.turndown(el.innerHTML).trim() : ''; @@ -140,7 +141,7 @@ export function toMarkdown(html) { * @returns {string} */ export function toHTML(html) { - const document = cleanDocument(html); + const { document } = cleanDocument(html); const el = bodyOf(document); return el ? el.innerHTML.trim() : ''; } diff --git a/tests/distill.test.js b/tests/distill.test.js index 7e1ebf2..b97aac8 100644 --- a/tests/distill.test.js +++ b/tests/distill.test.js @@ -1,11 +1,12 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; -import { distill, toMarkdown, toHTML } from '../src/distill.js'; +import { distill, toMarkdown, toHTML, feedToHTML } from '../src/distill.js'; import { render, estimateTokens } from '../src/render.js'; const html = readFileSync(new URL('./pages/news.html', import.meta.url), 'utf8'); const page = () => distill(html, 'https://example.test/news'); +const feed = readFileSync(new URL('./pages/feed.xml', import.meta.url), 'utf8'); test('noise never reaches the output, compact or raw', () => { for (const out of [render(page(), { budget: 5000 }).text, toMarkdown(html), toHTML(html)]) { @@ -69,6 +70,40 @@ test('token estimate is stable and roughly chars over four', () => { assert.equal(estimateTokens('abcdefgh'), 2); }); +test('atom feeds render as pages: entries become headings, bodies unescape', () => { + const p = distill(feed, 'https://example.test/feeds/question/42'); + assert.equal(p.title, 'Why is the sky blue? - Fixture Overflow'); + const headings = p.blocks.filter((b) => b.type === 'heading'); + assert.equal(headings[0].text, 'Why is the sky blue?'); + assert.equal(headings[1].text, 'Answer by Tyndall for Why is the sky blue?'); + const open = p.blocks.find((b) => b.type === 'link' && b.text === 'open'); + assert.equal(open.href, 'https://example.test/questions/42/why-is-the-sky-blue'); + const text = p.blocks.map((b) => b.text).join(' '); + assert.ok(text.includes('Rayleigh scattering'), 'entry body missing'); + assert.ok(text.includes('by Ray Leigh, 2026-04-08'), 'byline missing'); + assert.ok(!text.includes('<'), 'entry body left escaped'); +}); + +test('feed entry code blocks survive raw markdown', () => { + const md = toMarkdown(feed); + assert.ok(md.startsWith('# Why is the sky blue? - Fixture Overflow')); + assert.ok(md.includes('wavelength < 450nm'), 'code content missing'); + assert.ok(md.includes('[the derivation](https://example.test/scattering)'), 'link inside entry body missing'); +}); + +test('rss with cdata bodies converts too, ordinary html does not', () => { + const rss = `Fixture Blog + Post onehttps://example.test/p/1 + Mon, 17 Aug 2026 00:00:00 GMT + A cdata body with markup.

]]> + `; + const p = distill(rss, 'https://example.test/rss'); + assert.equal(p.title, 'Fixture Blog'); + const text = p.blocks.map((b) => b.text).join(' '); + assert.ok(text.includes('A cdata body with markup.'), 'cdata body missing'); + assert.equal(feedToHTML(html), null, 'ordinary html misread as a feed'); +}); + test('long runs of short links collapse into a range marker', () => { const nav = Array.from({ length: 15 }, (_, i) => `sub${i}`).join(' '); const navHtml = `T${nav}

actual content

`; diff --git a/tests/pages/feed.xml b/tests/pages/feed.xml new file mode 100644 index 0000000..0d89853 --- /dev/null +++ b/tests/pages/feed.xml @@ -0,0 +1,41 @@ + + + Why is the sky blue? - Fixture Overflow + + + most recent 30 from example.test + 2026-08-18T00:00:00Z + https://example.test/feeds/question/42 + + https://example.test/q/42 + 7 + Why is the sky blue? + + + + Ray Leigh + https://example.test/users/1 + + + 2012-06-27T13:51:36Z + 2026-04-08T05:35:32Z + + <p>Looking up on a clear day the sky is blue, yet sunlight is white.</p> <pre><code>wavelength &lt; 450nm </code></pre> <p>What scatters the shorter wavelengths?</p> + + + + https://example.test/a/43 + 12 + Answer by Tyndall for Why is the sky blue? + + Tyndall + https://example.test/users/2 + + + 2012-06-27T14:00:00Z + 2012-06-27T14:00:00Z + + <p>Rayleigh scattering. Shorter wavelengths scatter far more strongly, see <a href="https://example.test/scattering">the derivation</a>.</p> + + +