import test from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync, readdirSync } from 'node:fs';
import { distill, toMarkdown, toHTML, feedToHTML, jsonToHTML, youtubeToHTML, transcriptToHTML, TEXT_CAP } from '../src/distill.js';
import { render, estimateTokens, contentTokens, contentFailure } from '../src/render.js';
const PAGES = new URL('./pages/', import.meta.url).pathname;
const html = readFileSync(new URL('./pages/news.html', import.meta.url), 'utf8');
const page = () => distill(html, 'https://example.test/news');
const feed = readFileSync(new URL('./pages/feed.xml', import.meta.url), 'utf8');
const forum = readFileSync(new URL('./pages/forum.html', import.meta.url), 'utf8');
const thread = () => distill(forum, 'https://example.test/t/1');
const timeline = readFileSync(new URL('./pages/social.html', import.meta.url), 'utf8');
const social = () => distill(timeline, 'https://social.test/fixture');
const api = readFileSync(new URL('./pages/api.json', import.meta.url), 'utf8');
const API_URL = 'https://api.example.test/2.3/search/advanced?site=fixture&q=sky+blue';
const results = () => distill(api, API_URL);
// Turndown escapes the underscores in field names, which is correct markdown
// and only noise to assert against.
const rawApi = () => toMarkdown(api, API_URL).replace(/\\_/g, '_');
const searchHTML = readFileSync(new URL('./pages/search.html', import.meta.url), 'utf8');
const search = () => distill(searchHTML, 'https://fixture.test/html/?q=s3+cp+recursive');
const docsHTML = readFileSync(new URL('./pages/docs.html', import.meta.url), 'utf8');
const docs = () => distill(docsHTML, 'https://docs.fixture.test/s3/cp.html');
const codeBlock = (match) => docs().blocks.find((b) => (b.text ?? '').includes(match))?.text ?? '';
test('noise never reaches the output, compact or raw', () => {
for (const out of [render(page(), { budget: 5000 }).text, toMarkdown(html), toHTML(html)]) {
assert.ok(!out.includes('tracker'), 'script content leaked');
assert.ok(!out.includes('font-family'), 'style content leaked');
assert.ok(!out.includes('cookies'), 'display:none content leaked');
assert.ok(!out.includes('hidden drawer'), 'hidden attribute content leaked');
assert.ok(!out.includes('csrf'), 'hidden input leaked');
}
});
test('raw mode emits real markdown with hrefs an agent can follow', () => {
const md = toMarkdown(html);
assert.ok(md.startsWith('# Fixture News'));
assert.ok(md.includes('[Show HN: I built a tiny CSV toolkit](/item?id=1)'), 'link markdown missing');
});
test('raw html mode keeps markup', () => {
const out = toHTML(html);
assert.ok(out.includes(''), 'anchor tag missing');
assert.ok(!out.includes('`;
const page = distill(`
To continue, accept cookies. We and our partners store and access ' + 'information on your device to personalise the content you see here.
'; assert.match(verdict(gate, 60_000), /tokens of text out of ~\d+ of HTML/); // The same page without that weight behind it is a short page, not a failed // render, so it has to pass. assert.equal(verdict(gate, 0), null); }); test('a link-list page counts as content even with no prose on it', () => { // Hacker News and search results are links and nothing else, so a rule that // counted only prose would call the tool's best pages empty. const links = Array.from({ length: 12 }, (_, i) => `A headline long enough to be a headline, number ${i}`).join(''); const page = distill(`${links}`, 'https://fixture.test/list'); assert.equal(contentFailure(contentTokens(page), 30_000), null); }); test('every fixture page reads as content, none as a failed render', () => { // Feeds, a JSON API, and a YouTube watch page are all thin by design, which // is exactly where this check must not cry wolf. for (const name of readdirSync(PAGES)) { const raw = readFileSync(PAGES + name, 'utf8'); const page = distill(raw, `https://api.example.test/2.3/search/advanced?site=fixture&f=${name}`); assert.equal(contentFailure(contentTokens(page), estimateTokens(raw)), null, name); } });