diff --git a/src/distill.js b/src/distill.js index 96ef24e..f2134be 100644 --- a/src/distill.js +++ b/src/distill.js @@ -513,6 +513,15 @@ const TITLE_KEYS = [ 'full_name', 'summary', 'question', 'message', ]; +// Keys an API is likely to put its list of results under. A response using one +// of these is a list whatever else it carries, so the name settles it before +// shape does: a sideloaded `included` array can outnumber the `items` the +// request was for without being what the request was for. +const CONTAINER_KEYS = [ + 'items', 'data', 'results', 'hits', 'records', 'rows', 'entries', + 'nodes', 'edges', 'docs', 'list', 'children', 'values', +]; + // Keys holding the item's own page. A URL under any other name is still found, // by looking at values rather than names, but these win when several qualify. const LINK_KEYS = ['link', 'url', 'html_url', 'web_url', 'permalink', 'href']; @@ -540,6 +549,14 @@ const NESTED_PENALTY = 0.25; // How many names the footer lists when it says which fields it left out. const DROPPED_LISTED = 5; +// Characters of markup a field may carry before the compact view stops +// treating it as a document and starts treating it as somewhere to go. A +// question body arrives well under this and is worth rendering in place, links +// and code and all. A package readme arrives at four figures and distils into +// more blocks than the resource it is attached to has fields, which buries the +// resource the response was fetched for. `oc raw` renders either one in full. +const BODY_CAP = 4000; + // Seconds and milliseconds since the epoch, bounded either side so an ordinary // count (a score, a byte size) is never mistaken for a date. const EPOCH_S = [1e9, 4e9]; @@ -572,6 +589,10 @@ const decodeEntities = (s) => const isPlain = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); const isURL = (v) => typeof v === 'string' && /^https?:\/\/\S+$/.test(v); const looksHTML = (v) => typeof v === 'string' && /<\/?(p|div|pre|code|br|ul|ol|li|h[1-6]|blockquote|table|img|a|em|strong)\b[^>]*>/i.test(v); +// Markup as one line of prose, for a body the compact view is pointing at +// rather than rendering. The distiller is what reads markup properly; this +// only has to make a line an agent can tell one body from another by. +const stripTags = (v) => clean(decodeEntities(String(v).replace(/<[^>]*>/g, ' '))); /** * One level of flattening, so `owner: {display_name}` becomes an @@ -662,32 +683,40 @@ function chooseFields(rows, skip) { } /** - * Pick the array the response is actually about: the root when it is one, - * otherwise the longest array of objects at the top level, which is where - * `items`, `data`, `results`, and `hits` all live. Everything beside it is - * metadata about the request rather than content. + * Pick what the response is actually about: the root when it is an array or a + * single resource, otherwise the array of objects at the top level that holds + * the results. Everything beside it is metadata about the request rather than + * content. * @param {any} data * @returns {{items: any[], meta: Record}} */ function mainArray(data) { if (Array.isArray(data)) return { items: data, meta: {} }; - let key = ''; - /** @type {any[] | null} */ - let items = null; + /** @type {Map} */ + const arrays = new Map(); for (const [k, v] of Object.entries(data)) { if (!Array.isArray(v) || !v.length) continue; if (!v.some(isPlain)) continue; - if (!items || v.length > items.length) { - items = v; - key = k; + arrays.set(k, v); + } + let key = CONTAINER_KEYS.find((k) => arrays.has(k)) ?? ''; + // A root carrying its own name is the resource, and an array hanging off it + // describes that resource rather than being the subject in its place. Taking + // the longest array regardless titled the npm registry's package endpoint + // after its two maintainers and demoted the package to the metadata line, + // where a 9KB readme then cost more than the rest of the page put together. + const named = TITLE_KEYS.some((k) => typeof data[k] === 'string' && data[k].trim() !== ''); + if (!key && !named) { + for (const [k, v] of arrays) { + if (!key || v.length > (arrays.get(key)?.length ?? 0)) key = k; } } - // A response with no array is a single resource, which renders as one item - // rather than as a special case. - if (!items) return { items: [data], meta: {} }; + // A response with no results array of its own is a single resource, which + // renders as one item rather than as a special case. + if (!key) return { items: [data], meta: {} }; const meta = { ...data }; delete meta[key]; - return { items, meta }; + return { items: arrays.get(key) ?? [data], meta }; } /** @@ -784,9 +813,16 @@ export function jsonToHTML(text, url = '', { full = false } = {}) { if (kept && !kept.has(key) && !looksHTML(value)) continue; // A field carrying HTML is a page in itself: `filter=withbody` on the // Stack Exchange API puts a whole question in one. It goes through the - // distiller like any other markup instead of into a cell. + // distiller like any other markup instead of into a cell, unless it is + // longer than the compact view can afford, in which case it becomes one + // numbered line rather than a dozen blocks that bury the item it hangs + // off. `oc read ` opens it at a budget that fits it, `oc raw` always. if (looksHTML(value)) { - bodies.push(String(value)); + if (full || String(value).length <= BODY_CAP) { + bodies.push(String(value)); + continue; + } + long.push(`

${escHTML(`${key}: ${stripTags(value)}`)}

`); continue; } const rendered = row.get(key); @@ -824,12 +860,21 @@ export function jsonToHTML(text, url = '', { full = false } = {}) { } const metaBits = []; + const metaLong = []; for (const [key, value] of Object.entries(meta)) { if (value === null || typeof value === 'object') continue; const rendered = renderValue(key, value); - if (rendered) metaBits.push(`${key}=${rendered}`); + if (!rendered) continue; + // A summary line has to stay a line. One long scalar at the root, a + // package readme or an endpoint description, would otherwise spend the + // whole page budget here, so it becomes a block of its own instead. The + // block is numbered, so `oc read ` opens it when it fits that budget + // and `oc raw` has it whatever its size. + if (rendered.length > TEXT_CAP) metaLong.push(`

${escHTML(`${key}: ${rendered}`)}

`); + else metaBits.push(`${key}=${rendered}`); } if (metaBits.length) parts.push(`

response: ${escHTML(metaBits.join(', '))}

`); + parts.push(...metaLong); const count = `${items.length} ${items.length === 1 ? 'item' : 'items'}`; return `${escHTML(jsonTitle(url, count))}\n${parts.join('\n')}\n`; diff --git a/tests/distill.test.js b/tests/distill.test.js index e896d7d..b35ab4d 100644 --- a/tests/distill.test.js +++ b/tests/distill.test.js @@ -322,6 +322,51 @@ test('json shapes other than a wrapped array still render', () => { assert.ok(single.blocks.some((b) => b.text?.includes('score: 3')), 'a single resource lost its fields'); }); +test('a resource with its own name is the subject, not the array hanging off it', () => { + // The npm registry shape: a named package carrying a short array of + // maintainers. Picking the longest array made the maintainers the subject + // and pushed the package into the metadata line. + const pkg = JSON.stringify({ + name: 'turnstile', license: 'MIT', description: 'does a thing', + maintainers: [{ name: 'ada', email: 'ada@example.test' }, { name: 'grace', email: 'grace@example.test' }], + }); + const p = distill(pkg, 'https://registry.example.test/turnstile'); + assert.ok(p.title.includes('(1 item)'), `the package was not the subject:\n${p.title}`); + assert.ok(p.blocks.some((b) => b.text === 'turnstile'), 'the resource lost its name'); + assert.ok(!p.blocks.some((b) => b.text?.startsWith('response:')), 'the resource was demoted to metadata'); + // A conventional container key still wins over the root's own name, so a + // named collection is still read as the collection it is. + const coll = distill(JSON.stringify({ name: 'a collection', items: [{ title: 'one' }, { title: 'two' }] }), 'https://x.test/c.json'); + assert.ok(coll.title.includes('(2 items)'), `a named collection lost its items:\n${coll.title}`); +}); + +test('one long field at the root cannot spend the whole page budget', () => { + const long = 'sentence about the package. '.repeat(400); + const body = JSON.stringify({ readme: long, total: 2, items: [{ title: 'one' }, { title: 'two' }] }); + const p = distill(body, 'https://x.test/list.json'); + const meta = p.blocks.find((b) => b.text?.startsWith('response:')); + assert.ok(meta, 'the request metadata went missing'); + assert.ok(meta.text.includes('total=2'), 'a short metadata field was lost with the long one'); + assert.ok(!meta.text.includes(long.slice(0, 200)), 'a long field stayed on the summary line'); + assert.ok(meta.text.length < TEXT_CAP * 2, `the summary line is not a line:\n${meta.text.slice(0, 300)}`); + // Off the line, not out of the page: it is its own block, and raw has it. + assert.ok(p.blocks.some((b) => b.text?.startsWith('readme:')), 'the long field vanished instead of moving'); + assert.ok(toMarkdown(body, 'https://x.test/list.json').includes('sentence about the package'), 'raw lost the long field'); +}); + +test('a body too long for the compact view becomes a line, not a dozen blocks', () => { + const short = '

A short body with a link.

'; + const huge = `

${'A paragraph that goes on. '.repeat(400)}

code()
`; + const withShort = distill(JSON.stringify({ items: [{ title: 'q', body: short }] }), 'https://x.test/a.json'); + assert.ok(withShort.blocks.some((b) => b.type === 'link' && b.text === 'a link'), 'a body that fits lost its links'); + const withHuge = distill(JSON.stringify({ items: [{ title: 'q', body: huge }] }), 'https://x.test/b.json'); + const line = withHuge.blocks.find((b) => b.text?.startsWith('body:')); + assert.ok(line, 'an oversized body left nothing behind'); + assert.ok(!line.text.includes('

'), 'markup reached the line unstripped'); + // raw is still the escape hatch, and still reads the markup as markup. + assert.ok(toMarkdown(JSON.stringify({ items: [{ title: 'q', body: huge }] }), 'https://x.test/b.json').includes('code()'), 'raw lost the oversized body'); +}); + test('only json is read as json', () => { assert.equal(jsonToHTML(html), null, 'an html page was parsed as json'); assert.equal(jsonToHTML(feed), null, 'a feed was parsed as json');