diff --git a/src/cli.js b/src/cli.js index ddc4c75..b02ea09 100755 --- a/src/cli.js +++ b/src/cli.js @@ -182,8 +182,12 @@ async function main() { // Only the blank case here. `raw` is the fallback the compact view's // failure line names, so it must not fail on the same pages: a page // whose only text is its menu still has markup, and printing it is the - // whole point of `raw`. - if (outTokens < MIN_CONTENT) noContent(finalUrl, `~${outTokens} tokens of markdown`, ''); + // whole point of `raw`. And a short page that arrived short is not + // blank, so the verdict needs the same evidence the compact view asks + // for: near-nothing distilled out of markup that promised more. + if (outTokens < MIN_CONTENT && contentFailure(outTokens, htmlTokens)) { + noContent(finalUrl, `~${outTokens} tokens of markdown`, ''); + } return; } const page = distill(html, finalUrl); diff --git a/src/render.js b/src/render.js index 54f17d2..ee417a0 100644 --- a/src/render.js +++ b/src/render.js @@ -35,8 +35,9 @@ const num = (v) => v.toLocaleString('en-US'); // which is what tells a link-list page (Hacker News, search results) from a // page whose only links are its own menu. const CONTENT_LABEL = 25; -// Below this there is nothing to read whatever the page is, so how much markup -// it arrived in does not matter. +// Below this a render is suspiciously thin, but thin is only a verdict when +// the page's own size says there should have been more. A terse page that +// arrived terse (a status endpoint, a one-line answer) distilled fine. export const MIN_CONTENT = 25; // Below this, with markup that large behind it, the fetch worked and the render // did not: a real page of that weight always distills to more. A genuinely @@ -65,7 +66,11 @@ export const contentTokens = (page) => * @returns {string|null} */ export function contentFailure(content, htmlTokens) { - if (content < MIN_CONTENT) return `~${content} tokens of text on the whole page`; + // Nothing extracted is empty whatever the page weighed. Anything more is + // only a failure with evidence: a small page that renders small is not + // gated, it is small, and exit 2 on it would send an agent to a browser + // for a page it was already holding. + if (content === 0) return 'no text on the whole page'; if (content < THIN_CONTENT && htmlTokens > THIN_HTML) { return `~${content} tokens of text out of ~${htmlTokens} of HTML`; } diff --git a/tests/distill.test.js b/tests/distill.test.js index d8f99df..b69301c 100644 --- a/tests/distill.test.js +++ b/tests/distill.test.js @@ -490,13 +490,13 @@ test('a page that arrives with no readable text is reported as a failure', () => }; // Nothing at all, whatever the page weighed. - assert.match(verdict('
', 0), /~0 tokens of text on the whole page/); + assert.match(verdict('
', 0), /no text on the whole page/); // Menu links only: short labels are furniture, so this page has no content // either, however much markup came with it. const chrome = ['Help', 'Log in', 'Content Policy', 'About', 'Careers', 'Press'] .map((t) => `${t}`).join(''); - assert.match(verdict(chrome, 60_000), /~0 tokens of text on the whole page/); + assert.match(verdict(chrome, 60_000), /no text on the whole page/); // A consent wall or a login gate: a sentence or two of real text, out of // markup far too big to have carried only that. @@ -508,6 +508,17 @@ test('a page that arrives with no readable text is reported as a failure', () => assert.equal(verdict(gate, 0), null); }); +test('a terse page that arrived terse is content, not a failed render', () => { + // A status endpoint or a one-line answer distills fine and has to exit 0: + // calling it gated would send an agent to a browser for a page it was + // already holding. Only weight it never rendered is evidence of a gate. + const html = 'status

All systems operational.

'; + const page = distill(html, 'https://fixture.test/status'); + assert.equal(contentFailure(contentTokens(page), estimateTokens(html)), null); + const json = distill('{"status":"ok"}', 'https://fixture.test/health'); + assert.equal(contentFailure(contentTokens(json), 4), null); +}); + test('a link-list page counts as content even with no prose on it', () => { // Hacker News and search results are links and nothing else, so a rule that // counted only prose would call the tool's best pages empty.