fix: hand over code an agent can actually run

A syntax highlighter gives every token of a command its own element, so
`s3://bucket/` reached the walk as `s3`, `:`, `//`, `bucket`, `/`, and the
rule that reassembles text fragments only glues the ones sharing a parent.
The rest were space-joined. The AWS CLI reference handed over

  aws s3 cp test . txt s3 : // amzn - s3 - demo - bucket / test2 . txt

and Node's fs docs handed over `console . log` and `fd ?. close ()`. Any
command or snippet an agent took from a docs page was wrong, and nothing in
the output said so. Highlighting is not an edge case: of 172 pre elements on
the AWS CLI reference, the Rust book, the Node API docs and the Python
library docs, 159 are split this way.

A pre or code subtree is now read as one string. That costs nothing to
follow, because not one of those 172 blocks contains a link, and it also
fixes inline code, which was inserting a space into `Byte ( u8 only)`.

Two things fall out of it.

Page furniture had to stop riding along. Node puts a language label beside a
copy button inside every code block, so the subtree's text ended in
`javascriptcopy`. A control is chrome, and so is the block-level element
holding it, which is how the label leaves with the button it sits beside.
The test stays on block-level wrappers because a highlighter's own elements
are inline, so a stray control can never take a line of code out with it.

Code blocks keep their lines. Collapsing them was survivable while the code
was already wrecked; once it reads correctly, `// read it back` in front of
the statements that followed it is worse, because the output now looks
trustworthy. Blank runs and the indentation the whole block shares carry no
meaning and still go. A cut lands on a line end for the same reason it
already lands on a sentence end, and a snippet stays one line, because that
is what find's index promises.

Measured over five real pages, the compact view moves by -15, -7, +208, -45
and 0 characters, about 35 tokens in total, all of the growth being Python's
pretty-printed output getting its indentation back.

src/act.js

@@ -282,7 +282,11 @@ function search(blocks, terms) {
     if (out.length && out.at(-1).n === n) continue;
     const start = Math.max(0, Math.min(...found) - BEFORE);
     const end = Math.min(block.text.length, start + SNIPPET);
-    const snippet = `${start > 0 ? '... ' : ''}${block.text.slice(start, end)}${end < block.text.length ? ' ...' : ''}`;
+    // One line per match is the promise the list makes, and a code block is the
+    // one kind of block that carries lines of its own. They survive where they
+    // are read rather than indexed: the whole-match mode above, and `read`.
+    const window = block.text.slice(start, end).replace(/\n/g, ' ');
+    const snippet = `${start > 0 ? '... ' : ''}${window}${end < block.text.length ? ' ...' : ''}`;
     out.push({ n, type: block.type, snippet, text: block.text });
   }
   return out;

src/distill.js

@@ -32,6 +32,32 @@ const DROP = new Set([
 // them, so the search below refuses to descend into them.
 const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
 
+// Controls a page puts inside its own code samples, and the selector that
+// finds one. Node's API docs give every code block a copy button, a module
+// toggle and a language label, so the text of the sample ends in
+// `javascriptcopy` unless the toolbar holding them is left out of it.
+const CODE_CONTROLS = new Set(['button', 'input', 'select', 'textarea', 'label']);
+const CONTROL = [...CODE_CONTROLS].join(', ');
+
+// Lines in a code block are kept, where every other kind of text has its
+// whitespace collapsed. Joining them would put `// comment` in front of the
+// statements that followed it, so a sample an agent could run would arrive
+// commented out, and the shape of the wreck is invisible on one line. Blank
+// runs and the indentation the whole block shares are the parts that carry no
+// meaning, so those go.
+const codeText = (raw) => {
+  const lines = raw.replace(/\r\n?/g, '\n').replace(/[^\S\n]+$/gm, '').split('\n');
+  while (lines.length && !lines[0].trim()) lines.shift();
+  while (lines.length && !lines[lines.length - 1].trim()) lines.pop();
+  const indent = lines
+    .filter((l) => l.trim())
+    .reduce((least, l) => Math.min(least, l.length - l.trimStart().length), Infinity);
+  return lines
+    .map((l) => (Number.isFinite(indent) ? l.slice(indent) : l).trimEnd())
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n');
+};
+
 // Elements that end a line on the page and so must end one here. Without this
 // the text of six separate posts merges into a single block, because nothing
 // between them survives distillation to keep them apart.
@@ -128,6 +154,45 @@ export function distill(html, url = '') {
   /** Subtree already emitted, skipped when the rest of the page is walked. */
   let done = null;
 
+  /**
+   * The text of a code subtree, read as one string.
+   *
+   * A syntax highlighter gives every token its own element, so `s3://bucket/`
+   * reaches the walk as `s3`, `:`, `//`, `bucket`, `/`, and the rule that puts
+   * fragments back together only glues the ones that share a parent. The rest
+   * are space-joined, which turned an AWS example into
+   * `aws s3 cp s3 : // bucket / -- recursive`, a command an agent cannot run.
+   * Reading the subtree whole is what fixes that, and it discards nothing: over
+   * 172 pre elements on the AWS CLI reference, the Rust book, the Node API docs
+   * and the Python library docs, 159 are split this way and not one of them
+   * contains a link.
+   *
+   * textContent would be enough if pages put only code in their code blocks.
+   * A control inside one is chrome, and so is the block-level element holding
+   * it: that is how the toolbar's `javascript` label leaves with the copy
+   * button it sits beside. The test stays on block-level wrappers because a
+   * highlighter's own elements are inline, so a stray control can never take a
+   * line of code out with it.
+   * @param {any} node
+   * @returns {string}
+   */
+  const verbatim = (node) => {
+    let out = '';
+    const gather = (n) => {
+      if (n.nodeType === 3) {
+        out += n.textContent ?? '';
+        return;
+      }
+      if (n.nodeType !== 1) return;
+      const tag = n.localName;
+      if (DROP.has(tag) || CODE_CONTROLS.has(tag) || hidden(n)) return;
+      if (n !== node && BLOCKY.has(tag) && n.querySelector(CONTROL)) return;
+      for (const child of n.childNodes) gather(child);
+    };
+    gather(node);
+    return out;
+  };
+
   const walk = (node) => {
     if (node.nodeType === 3) {
       const raw = node.textContent ?? '';
@@ -183,6 +248,21 @@ export function distill(html, url = '') {
       if (text) blocks.push({ type: 'button', text });
       return;
     }
+    if (tag === 'pre' || tag === 'code') {
+      const raw = verbatim(node);
+      const text = tag === 'pre' ? codeText(raw) : clean(raw);
+      // A pre is its own line, which is what BLOCKY gave it before this branch
+      // started claiming it first. Inline code belongs to the sentence around
+      // it, so it carries the parent and the edge spacing a text node would and
+      // merges back into that sentence the same way.
+      const own = tag === 'pre';
+      if (own) blocks.push({ type: 'break' });
+      if (text) {
+        blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
+      }
+      if (own) blocks.push({ type: 'break' });
+      return;
+    }
     if (BLOCKY.has(tag)) {
       blocks.push({ type: 'break' });
       for (const child of node.childNodes) walk(child);

src/render.js

@@ -174,6 +174,12 @@ const SENTENCE_FLOOR = 0.7;
 
 const truncate = (s) => {
   if (s.length <= TEXT_CAP) return s;
+  // A line is to code what a sentence is to prose, and a code block is the only
+  // text that keeps its newlines, so the same courtesy applies: cut where a
+  // line ended. A period in code ends nothing, which is why this returns
+  // instead of falling through to the sentence rule below.
+  const line = s.slice(0, TEXT_CAP).lastIndexOf('\n');
+  if (line >= TEXT_CAP * SENTENCE_FLOOR) return `${s.slice(0, line)} ... +${num(s.length - line)} chars`;
   let cut = TEXT_CAP;
   for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
     const end = (m.index ?? 0) + m[0].length;

tests/act.test.js

@@ -212,3 +212,22 @@ test('history grows with each page and stays bounded', () => {
   assert.equal(state.history.length, 20);
   assert.equal(state.history.at(-1), 'https://example.test/p24');
 });
+
+test('a snippet stays one line even when the block it came from is code', () => {
+  // Code blocks keep their newlines. An index that prints one match per line
+  // cannot, or the header's count stops matching what is on screen. Long
+  // filler beside it is what keeps find on the snippet path.
+  const filler = 'x'.repeat(600);
+  saveSession('code', {
+    url: 'https://fixture.test/c',
+    blocks: [
+      { n: 1, type: 'text', text: ['first();', 'needle();', 'third();'].join('\n') },
+      { n: 2, type: 'text', text: `${filler} needle ${filler}` },
+    ],
+    cursor: null,
+  });
+  const out = find('needle', { session: 'code', budget: 20 });
+  const lines = out.split('\n');
+  assert.match(lines[0], /^2 matches for "needle"/);
+  assert.equal(lines[1], '[1] first(); needle(); third();');
+});

tests/distill.test.js

@@ -19,6 +19,9 @@ const results = () => distill(api, API_URL);
 const rawApi = () => toMarkdown(api, API_URL).replace(/\\_/g, '_');
 const searchHTML = readFileSync(new URL('./pages/search.html', import.meta.url), 'utf8');
 const search = () => distill(searchHTML, 'https://fixture.test/html/?q=s3+cp+recursive');
+const docsHTML = readFileSync(new URL('./pages/docs.html', import.meta.url), 'utf8');
+const docs = () => distill(docsHTML, 'https://docs.fixture.test/s3/cp.html');
+const codeBlock = (match) => docs().blocks.find((b) => (b.text ?? '').includes(match))?.text ?? '';
 
 test('noise never reaches the output, compact or raw', () => {
   for (const out of [render(page(), { budget: 5000 }).text, toMarkdown(html), toHTML(html)]) {
@@ -427,3 +430,49 @@ test('a truncated block ends on a sentence, so what is shown can be trusted', ()
   ).text;
   assert.ok(unbroken.length > TEXT_CAP, `an early sentence end must not shrink the view:\n${unbroken}`);
 });
+
+test('a highlighted command comes out runnable', () => {
+  // Every token of this command is its own element on the page. Space-joining
+  // them gave `aws s3 cp s3 : // bucket / -- recursive`, which is not a command
+  // an agent can run, and the agent has no way to see that from the output.
+  assert.equal(codeBlock('aws s3 cp test.txt'), 'aws s3 cp test.txt s3://amzn-demo/ --recursive');
+});
+
+test('a code sample keeps its lines, so a comment cannot eat the rest', () => {
+  assert.equal(
+    codeBlock('readFileSync'),
+    "const fs = require('node:fs');\n// read it back\nfs.readFileSync('out.txt');",
+  );
+  // A shell continuation is only a continuation while the break is still there.
+  assert.equal(codeBlock('--expires'), 'aws s3 cp test.txt s3://amzn-demo/ \\\n    --expires 2014-10-01T20:30:00Z');
+});
+
+test('a code block loses the indentation it all shares and keeps the rest', () => {
+  assert.equal(codeBlock('def load'), 'def load(path):\n    with open(path) as fh:\n        return json.load(fh)');
+});
+
+test('a toolbar inside a code block is not part of the sample', () => {
+  const block = codeBlock("import fs from 'node:fs'");
+  assert.equal(block, "import fs from 'node:fs';");
+  const out = render(docs(), { budget: 5000 }).text;
+  assert.ok(!out.includes('javascript'), 'the language label leaked into the page');
+  // The controls go with it: a copy button is not something oc can press, and
+  // `do` on it would be a turn spent on nothing.
+  assert.equal(docs().blocks.filter((b) => b.type === 'button' || b.type === 'input').length, 0);
+});
+
+test('inline code joins the sentence it sits in', () => {
+  assert.ok(docs().blocks.some((b) => b.text === 'Pass the --recursive flag to copy a directory.'));
+  assert.ok(docs().blocks.some((b) => b.text === 'A period (.) means the working directory.'));
+});
+
+test('a truncated code block ends on a line, not mid-statement', () => {
+  const code = ['first();', 'second();', ...Array.from({ length: 20 }, (_, i) => `line${i}('${'x'.repeat(20)}');`)].join('\n');
+  const page = { url: 'https://fixture.test/c', title: 'c', blocks: [{ type: 'text', n: 1, text: code }] };
+  const shown = render(page, { budget: 10 }).text.split('\n');
+  const marker = shown.findIndex((l) => l.includes('... +'));
+  assert.ok(marker > 0, 'nothing was truncated');
+  // The kept part stops where a statement did, so every line shown is whole.
+  assert.ok(shown[marker].startsWith('line'), `cut mid-line: ${shown[marker]}`);
+  assert.ok(shown[marker].includes(');'), `cut mid-statement: ${shown[marker]}`);
+});

tests/pages/docs.html

@@ -0,0 +1,42 @@
+<!doctype html>
+<html><head><title>cp - Fixture CLI Command Reference</title></head>
+<body>
+<main>
+
+  <h1>cp</h1>
+
+  <!-- What a syntax highlighter does to a command: one element per token, so
+       the walk sees `s3`, `:`, `//`, `bucket` as separate fragments with
+       different parents. Every real highlighter emits this shape. -->
+  <p>Copy a file to the bucket:</p>
+  <pre class="highlight"><code><span class="nb">aws</span> <span class="n">s3</span> <span class="n">cp</span> <span class="n">test</span><span class="p">.</span><span class="n">txt</span> <span class="n">s3</span><span class="p">:</span><span class="p">//</span><span class="n">amzn</span><span class="p">-</span><span class="n">demo</span><span class="p">/</span> <span class="p">--</span><span class="n">recursive</span></code></pre>
+
+  <!-- A sample the page wrote across lines, with a shell continuation. Joining
+       these would hide the break; joining the next one would comment out the
+       call that follows the comment. -->
+  <pre><code>aws s3 cp test.txt s3://amzn-demo/ \
+    --expires 2014-10-01T20:30:00Z</code></pre>
+
+  <pre><code>const fs = require('node:fs');
+// read it back
+fs.readFileSync('out.txt');</code></pre>
+
+  <!-- Node's docs put a toolbar inside the block itself: a language label
+       sitting beside a copy button, plus a flavour toggle. None of it is part
+       of the sample. -->
+  <pre class="shiki"><input class="js-flavor-toggle" type="checkbox"><div class="code-toolbar"><span class="code-language">javascript</span><button class="copy-button">copy</button></div><code><span>import</span> <span>fs</span> <span>from</span> <span>'node:fs'</span><span>;</span></code></pre>
+
+  <!-- Indentation the whole block shares says nothing; indentation inside it
+       is the program. -->
+  <pre><code>    def load(path):
+        with open(path) as fh:
+            return json.load(fh)</code></pre>
+
+  <!-- Inline code belongs to the sentence it sits in, and it gets split into
+       tokens the same way. -->
+  <p>Pass the <code><span class="p">--</span><span class="n">recursive</span></code> flag to copy a directory.</p>
+
+  <p>A period (<code>.</code>) means the working directory.</p>
+
+</main>
+</body></html>