diff --git a/src/act.js b/src/act.js
index 4e23665..b61a0a7 100644
--- a/src/act.js
+++ b/src/act.js
@@ -282,7 +282,11 @@ function search(blocks, terms) {
if (out.length && out.at(-1).n === n) continue;
const start = Math.max(0, Math.min(...found) - BEFORE);
const end = Math.min(block.text.length, start + SNIPPET);
- const snippet = `${start > 0 ? '... ' : ''}${block.text.slice(start, end)}${end < block.text.length ? ' ...' : ''}`;
+ // One line per match is the promise the list makes, and a code block is the
+ // one kind of block that carries lines of its own. They survive where they
+ // are read rather than indexed: the whole-match mode above, and `read`.
+ const window = block.text.slice(start, end).replace(/\n/g, ' ');
+ const snippet = `${start > 0 ? '... ' : ''}${window}${end < block.text.length ? ' ...' : ''}`;
out.push({ n, type: block.type, snippet, text: block.text });
}
return out;
diff --git a/src/distill.js b/src/distill.js
index 38bd552..bed20c7 100644
--- a/src/distill.js
+++ b/src/distill.js
@@ -32,6 +32,32 @@ const DROP = new Set([
// them, so the search below refuses to descend into them.
const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
+// Controls a page puts inside its own code samples, and the selector that
+// finds one. Node's API docs give every code block a copy button, a module
+// toggle and a language label, so the text of the sample ends in
+// `javascriptcopy` unless the toolbar holding them is left out of it.
+const CODE_CONTROLS = new Set(['button', 'input', 'select', 'textarea', 'label']);
+const CONTROL = [...CODE_CONTROLS].join(', ');
+
+// Lines in a code block are kept, where every other kind of text has its
+// whitespace collapsed. Joining them would put `// comment` in front of the
+// statements that followed it, so a sample an agent could run would arrive
+// commented out, and the shape of the wreck is invisible on one line. Blank
+// runs and the indentation the whole block shares are the parts that carry no
+// meaning, so those go.
+const codeText = (raw) => {
+ const lines = raw.replace(/\r\n?/g, '\n').replace(/[^\S\n]+$/gm, '').split('\n');
+ while (lines.length && !lines[0].trim()) lines.shift();
+ while (lines.length && !lines[lines.length - 1].trim()) lines.pop();
+ const indent = lines
+ .filter((l) => l.trim())
+ .reduce((least, l) => Math.min(least, l.length - l.trimStart().length), Infinity);
+ return lines
+ .map((l) => (Number.isFinite(indent) ? l.slice(indent) : l).trimEnd())
+ .join('\n')
+ .replace(/\n{3,}/g, '\n\n');
+};
+
// Elements that end a line on the page and so must end one here. Without this
// the text of six separate posts merges into a single block, because nothing
// between them survives distillation to keep them apart.
@@ -128,6 +154,45 @@ export function distill(html, url = '') {
/** Subtree already emitted, skipped when the rest of the page is walked. */
let done = null;
+ /**
+ * The text of a code subtree, read as one string.
+ *
+ * A syntax highlighter gives every token its own element, so `s3://bucket/`
+ * reaches the walk as `s3`, `:`, `//`, `bucket`, `/`, and the rule that puts
+ * fragments back together only glues the ones that share a parent. The rest
+ * are space-joined, which turned an AWS example into
+ * `aws s3 cp s3 : // bucket / -- recursive`, a command an agent cannot run.
+ * Reading the subtree whole is what fixes that, and it discards nothing: over
+ * 172 pre elements on the AWS CLI reference, the Rust book, the Node API docs
+ * and the Python library docs, 159 are split this way and not one of them
+ * contains a link.
+ *
+ * textContent would be enough if pages put only code in their code blocks.
+ * A control inside one is chrome, and so is the block-level element holding
+ * it: that is how the toolbar's `javascript` label leaves with the copy
+ * button it sits beside. The test stays on block-level wrappers because a
+ * highlighter's own elements are inline, so a stray control can never take a
+ * line of code out with it.
+ * @param {any} node
+ * @returns {string}
+ */
+ const verbatim = (node) => {
+ let out = '';
+ const gather = (n) => {
+ if (n.nodeType === 3) {
+ out += n.textContent ?? '';
+ return;
+ }
+ if (n.nodeType !== 1) return;
+ const tag = n.localName;
+ if (DROP.has(tag) || CODE_CONTROLS.has(tag) || hidden(n)) return;
+ if (n !== node && BLOCKY.has(tag) && n.querySelector(CONTROL)) return;
+ for (const child of n.childNodes) gather(child);
+ };
+ gather(node);
+ return out;
+ };
+
const walk = (node) => {
if (node.nodeType === 3) {
const raw = node.textContent ?? '';
@@ -183,6 +248,21 @@ export function distill(html, url = '') {
if (text) blocks.push({ type: 'button', text });
return;
}
+ if (tag === 'pre' || tag === 'code') {
+ const raw = verbatim(node);
+ const text = tag === 'pre' ? codeText(raw) : clean(raw);
+ // A pre is its own line, which is what BLOCKY gave it before this branch
+ // started claiming it first. Inline code belongs to the sentence around
+ // it, so it carries the parent and the edge spacing a text node would and
+ // merges back into that sentence the same way.
+ const own = tag === 'pre';
+ if (own) blocks.push({ type: 'break' });
+ if (text) {
+ blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
+ }
+ if (own) blocks.push({ type: 'break' });
+ return;
+ }
if (BLOCKY.has(tag)) {
blocks.push({ type: 'break' });
for (const child of node.childNodes) walk(child);
diff --git a/src/render.js b/src/render.js
index 65dd564..8c50d03 100644
--- a/src/render.js
+++ b/src/render.js
@@ -174,6 +174,12 @@ const SENTENCE_FLOOR = 0.7;
const truncate = (s) => {
if (s.length <= TEXT_CAP) return s;
+ // A line is to code what a sentence is to prose, and a code block is the only
+ // text that keeps its newlines, so the same courtesy applies: cut where a
+ // line ended. A period in code ends nothing, which is why this returns
+ // instead of falling through to the sentence rule below.
+ const line = s.slice(0, TEXT_CAP).lastIndexOf('\n');
+ if (line >= TEXT_CAP * SENTENCE_FLOOR) return `${s.slice(0, line)} ... +${num(s.length - line)} chars`;
let cut = TEXT_CAP;
for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
const end = (m.index ?? 0) + m[0].length;
diff --git a/tests/act.test.js b/tests/act.test.js
index 3bbf0a1..c755536 100644
--- a/tests/act.test.js
+++ b/tests/act.test.js
@@ -212,3 +212,22 @@ test('history grows with each page and stays bounded', () => {
assert.equal(state.history.length, 20);
assert.equal(state.history.at(-1), 'https://example.test/p24');
});
+
+test('a snippet stays one line even when the block it came from is code', () => {
+ // Code blocks keep their newlines. An index that prints one match per line
+ // cannot, or the header's count stops matching what is on screen. Long
+ // filler beside it is what keeps find on the snippet path.
+ const filler = 'x'.repeat(600);
+ saveSession('code', {
+ url: 'https://fixture.test/c',
+ blocks: [
+ { n: 1, type: 'text', text: ['first();', 'needle();', 'third();'].join('\n') },
+ { n: 2, type: 'text', text: `${filler} needle ${filler}` },
+ ],
+ cursor: null,
+ });
+ const out = find('needle', { session: 'code', budget: 20 });
+ const lines = out.split('\n');
+ assert.match(lines[0], /^2 matches for "needle"/);
+ assert.equal(lines[1], '[1] first(); needle(); third();');
+});
diff --git a/tests/distill.test.js b/tests/distill.test.js
index d6f261b..8e553a3 100644
--- a/tests/distill.test.js
+++ b/tests/distill.test.js
@@ -19,6 +19,9 @@ const results = () => distill(api, API_URL);
const rawApi = () => toMarkdown(api, API_URL).replace(/\\_/g, '_');
const searchHTML = readFileSync(new URL('./pages/search.html', import.meta.url), 'utf8');
const search = () => distill(searchHTML, 'https://fixture.test/html/?q=s3+cp+recursive');
+const docsHTML = readFileSync(new URL('./pages/docs.html', import.meta.url), 'utf8');
+const docs = () => distill(docsHTML, 'https://docs.fixture.test/s3/cp.html');
+const codeBlock = (match) => docs().blocks.find((b) => (b.text ?? '').includes(match))?.text ?? '';
test('noise never reaches the output, compact or raw', () => {
for (const out of [render(page(), { budget: 5000 }).text, toMarkdown(html), toHTML(html)]) {
@@ -427,3 +430,49 @@ test('a truncated block ends on a sentence, so what is shown can be trusted', ()
).text;
assert.ok(unbroken.length > TEXT_CAP, `an early sentence end must not shrink the view:\n${unbroken}`);
});
+
+test('a highlighted command comes out runnable', () => {
+ // Every token of this command is its own element on the page. Space-joining
+ // them gave `aws s3 cp s3 : // bucket / -- recursive`, which is not a command
+ // an agent can run, and the agent has no way to see that from the output.
+ assert.equal(codeBlock('aws s3 cp test.txt'), 'aws s3 cp test.txt s3://amzn-demo/ --recursive');
+});
+
+test('a code sample keeps its lines, so a comment cannot eat the rest', () => {
+ assert.equal(
+ codeBlock('readFileSync'),
+ "const fs = require('node:fs');\n// read it back\nfs.readFileSync('out.txt');",
+ );
+ // A shell continuation is only a continuation while the break is still there.
+ assert.equal(codeBlock('--expires'), 'aws s3 cp test.txt s3://amzn-demo/ \\\n --expires 2014-10-01T20:30:00Z');
+});
+
+test('a code block loses the indentation it all shares and keeps the rest', () => {
+ assert.equal(codeBlock('def load'), 'def load(path):\n with open(path) as fh:\n return json.load(fh)');
+});
+
+test('a toolbar inside a code block is not part of the sample', () => {
+ const block = codeBlock("import fs from 'node:fs'");
+ assert.equal(block, "import fs from 'node:fs';");
+ const out = render(docs(), { budget: 5000 }).text;
+ assert.ok(!out.includes('javascript'), 'the language label leaked into the page');
+ // The controls go with it: a copy button is not something oc can press, and
+ // `do` on it would be a turn spent on nothing.
+ assert.equal(docs().blocks.filter((b) => b.type === 'button' || b.type === 'input').length, 0);
+});
+
+test('inline code joins the sentence it sits in', () => {
+ assert.ok(docs().blocks.some((b) => b.text === 'Pass the --recursive flag to copy a directory.'));
+ assert.ok(docs().blocks.some((b) => b.text === 'A period (.) means the working directory.'));
+});
+
+test('a truncated code block ends on a line, not mid-statement', () => {
+ const code = ['first();', 'second();', ...Array.from({ length: 20 }, (_, i) => `line${i}('${'x'.repeat(20)}');`)].join('\n');
+ const page = { url: 'https://fixture.test/c', title: 'c', blocks: [{ type: 'text', n: 1, text: code }] };
+ const shown = render(page, { budget: 10 }).text.split('\n');
+ const marker = shown.findIndex((l) => l.includes('... +'));
+ assert.ok(marker > 0, 'nothing was truncated');
+ // The kept part stops where a statement did, so every line shown is whole.
+ assert.ok(shown[marker].startsWith('line'), `cut mid-line: ${shown[marker]}`);
+ assert.ok(shown[marker].includes(');'), `cut mid-statement: ${shown[marker]}`);
+});
diff --git a/tests/pages/docs.html b/tests/pages/docs.html
new file mode 100644
index 0000000..e25dbdc
--- /dev/null
+++ b/tests/pages/docs.html
@@ -0,0 +1,42 @@
+
+