External/oc
1
0
Fork 0
mirror of https://github.com/only-cli/oc.git synced 2026-09-15 10:40:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

feed reading: offline tests, readme row, and a raw-mode title fix

Browse Source 
The feed fixture covers Atom entries (escaped bodies, self-closed
categories, bylines) and an RSS item with a CDATA body. Raw markdown of
a feed exposed an old quirk: cleanDocument removed the head before
toMarkdown read the title, so any page whose body lacked a matching h1
lost its title in raw mode. cleanDocument now captures the title first
and returns both.
This commit is contained in:
only-cli
2026-08-18 10:40:08 -04:00
parent bb214dcf59
commit 853bbca853
4 changed files with 86 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -1
View File
@@ -63,8 +63,11 @@ only-cli works on any mostly-static website with no per-site setup: news sites,
| LinkedIn | linkedin.com | `profile <name>`, `company <name>`, `jobs <query>` (public guest views) |
| DuckDuckGo | duckduckgo.com | `search <query>`, `lite <query>` |
| Bing | bing.com | `search <query>`, `news <query>` |
| Stack Overflow | stackoverflow.com (via Atom feeds) | `question <id>`, `tag <name>`, `user <id>`, `recent` |
Not supported yet: pages that only render with JavaScript (a headless fallback is planned for v0.3), sites behind logins (sessions land in v0.2), and sites with hard bot challenges. Adding a site shortcut is a small JSON file; see [CONTRIBUTING.md](CONTRIBUTING.md).
The engine also renders Atom and RSS feeds as regular pages. That is how Stack Overflow works: the site serves every HTML page a Cloudflare challenge, but publishes full question and answer bodies under `/feeds`, so `oc open stackoverflow.com/feeds/question/11227809` returns the question and its top answers in about 500 tokens. The same trick applies to any site that gates its pages but leaves its feeds open.
Not supported yet: pages that only render with JavaScript (a headless fallback is planned for v0.3), sites behind logins (sessions land in v0.2), and sites with hard bot challenges that do not expose feeds. Adding a site shortcut is a small JSON file; see [CONTRIBUTING.md](CONTRIBUTING.md).
## Benchmarks
src/distill.js
+5 -4
View File
@@ -108,6 +108,8 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
*/
function cleanDocument(html) {
const { document } = parseHTML(feedToHTML(html) ?? html);
// Read the title before the sweep below removes the head with it.
const title = clean(document.querySelector('title')?.textContent ?? '');
for (const tag of DROP) {
for (const el of [...document.querySelectorAll(tag)]) el.remove();
}
@@ -115,7 +117,7 @@ function cleanDocument(html) {
for (const el of [...document.querySelectorAll('[style]')]) {
if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
}
return document;
return { document, title };
}
/**
@@ -125,8 +127,7 @@ function cleanDocument(html) {
* @returns {string}
*/
export function toMarkdown(html) {
const document = cleanDocument(html);
const title = clean(document.querySelector('title')?.textContent ?? '');
const { document, title } = cleanDocument(html);
const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
const el = bodyOf(document);
const body = el ? turndown.turndown(el.innerHTML).trim() : '';
@@ -140,7 +141,7 @@ export function toMarkdown(html) {
* @returns {string}
*/
export function toHTML(html) {
const document = cleanDocument(html);
const { document } = cleanDocument(html);
const el = bodyOf(document);
return el ? el.innerHTML.trim() : '';
}
tests/distill.test.js
+36 -1
View File
@@ -1,11 +1,12 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { distill, toMarkdown, toHTML } from '../src/distill.js';
import { distill, toMarkdown, toHTML, feedToHTML } from '../src/distill.js';
import { render, estimateTokens } from '../src/render.js';
const html = readFileSync(new URL('./pages/news.html', import.meta.url), 'utf8');
const page = () => distill(html, 'https://example.test/news');
const feed = readFileSync(new URL('./pages/feed.xml', import.meta.url), 'utf8');
test('noise never reaches the output, compact or raw', () => {
for (const out of [render(page(), { budget: 5000 }).text, toMarkdown(html), toHTML(html)]) {
@@ -69,6 +70,40 @@ test('token estimate is stable and roughly chars over four', () => {
assert.equal(estimateTokens('abcdefgh'), 2);
});
test('atom feeds render as pages: entries become headings, bodies unescape', () => {
const p = distill(feed, 'https://example.test/feeds/question/42');
assert.equal(p.title, 'Why is the sky blue? - Fixture Overflow');
const headings = p.blocks.filter((b) => b.type === 'heading');
assert.equal(headings[0].text, 'Why is the sky blue?');
assert.equal(headings[1].text, 'Answer by Tyndall for Why is the sky blue?');
const open = p.blocks.find((b) => b.type === 'link' && b.text === 'open');
assert.equal(open.href, 'https://example.test/questions/42/why-is-the-sky-blue');
const text = p.blocks.map((b) => b.text).join(' ');
assert.ok(text.includes('Rayleigh scattering'), 'entry body missing');
assert.ok(text.includes('by Ray Leigh, 2026-04-08'), 'byline missing');
assert.ok(!text.includes('&lt;'), 'entry body left escaped');
});
test('feed entry code blocks survive raw markdown', () => {
const md = toMarkdown(feed);
assert.ok(md.startsWith('# Why is the sky blue? - Fixture Overflow'));
assert.ok(md.includes('wavelength < 450nm'), 'code content missing');
assert.ok(md.includes('[the derivation](https://example.test/scattering)'), 'link inside entry body missing');
});
test('rss with cdata bodies converts too, ordinary html does not', () => {
const rss = `<?xml version="1.0"?><rss version="2.0"><channel><title>Fixture Blog</title>
<item><title>Post one</title><guid>https://example.test/p/1</guid>
<pubDate>Mon, 17 Aug 2026 00:00:00 GMT</pubDate>
<description><![CDATA[<p>A <em>cdata</em> body with markup.</p>]]></description></item>
</channel></rss>`;
const p = distill(rss, 'https://example.test/rss');
assert.equal(p.title, 'Fixture Blog');
const text = p.blocks.map((b) => b.text).join(' ');
assert.ok(text.includes('A cdata body with markup.'), 'cdata body missing');
assert.equal(feedToHTML(html), null, 'ordinary html misread as a feed');
});
test('long runs of short links collapse into a range marker', () => {
const nav = Array.from({ length: 15 }, (_, i) => `<a href="/s/${i}">sub${i}</a>`).join(' ');
const navHtml = `<html><head><title>T</title></head><body>${nav}<p>actual content</p></body></html>`;
tests/pages/feed.xml
+41
View File
@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:re="http://purl.org/atompub/rank/1.0">
<title type="text">Why is the sky blue? - Fixture Overflow</title>
<link rel="self" href="https://example.test/feeds/question/42" type="application/atom+xml" />
<link rel="alternate" href="https://example.test/q/42" type="text/html" />
<subtitle>most recent 30 from example.test</subtitle>
<updated>2026-08-18T00:00:00Z</updated>
<id>https://example.test/feeds/question/42</id>
<entry>
<id>https://example.test/q/42</id>
<re:rank scheme="https://example.test">7</re:rank>
<title type="text">Why is the sky blue?</title>
<category scheme="https://example.test/tags" term="physics" />
<category scheme="https://example.test/tags" term="optics" />
<author>
<name>Ray Leigh</name>
<uri>https://example.test/users/1</uri>
</author>
<link rel="alternate" href="https://example.test/questions/42/why-is-the-sky-blue" />
<published>2012-06-27T13:51:36Z</published>
<updated>2026-04-08T05:35:32Z</updated>
<summary type="html">
&lt;p&gt;Looking up on a clear day the sky is blue, yet sunlight is white.&lt;/p&gt;&#xA;&lt;pre&gt;&lt;code&gt;wavelength &amp;lt; 450nm&#xA;&lt;/code&gt;&lt;/pre&gt;&#xA;&lt;p&gt;What scatters the shorter wavelengths?&lt;/p&gt;
</summary>
</entry>
<entry>
<id>https://example.test/a/43</id>
<re:rank scheme="https://example.test">12</re:rank>
<title type="text">Answer by Tyndall for Why is the sky blue?</title>
<author>
<name>Tyndall</name>
<uri>https://example.test/users/2</uri>
</author>
<link rel="alternate" href="https://example.test/questions/42/why-is-the-sky-blue/43#43" />
<published>2012-06-27T14:00:00Z</published>
<updated>2012-06-27T14:00:00Z</updated>
<summary type="html">
&lt;p&gt;Rayleigh scattering. Shorter wavelengths scatter far more strongly, see &lt;a href=&quot;https://example.test/scattering&quot;&gt;the derivation&lt;/a&gt;.&lt;/p&gt;
</summary>
</entry>
</feed>