support youtube watch pages and transcripts

Watch pages need client JS to become interactive, but the title, description, view count, and caption tracks already ship inline in the initial HTML as ytInitialPlayerResponse, so this reads that directly instead of waiting on the v0.3 headless fallback. Each caption track becomes a numbered link, and oc do on it fetches the timedtext transcript, collapsed into one block so it pages through oc next/oc read like any other long document instead of costing one block per caption line. Adds youtubeToHTML and transcriptToHTML alongside feedToHTML in the distiller, a youtube.com.json shortcut, and offline tests against a fixture watch page. Fixes #6
2026-09-15 10:40:56 +02:00 · 2026-08-19 13:27:52 -04:00
parent ea3afeb25a
commit 01b607ac81
5 changed files with 156 additions and 3 deletions
@@ -138,11 +138,14 @@ only-cli works on any mostly-static website with no per-site setup: news sites,
 | Bing | bing.com | `search <query>`, `news <query>` |
 | Stack Overflow | stackoverflow.com (via Atom feeds) | `question <id>`, `tag <name>`, `user <id>`, `recent` |
 | Yahoo Finance | finance.yahoo.com | `quote <symbol>`, `news <symbol>`, `history <symbol>`, `lookup <query>`, `markets`, `gainers`, `losers`, `trending` |
+| YouTube | youtube.com | `video <id>`, `channel <name>` |

 X is worth a note because it is usually written off as unreadable without a login. Two of its pages are not: a profile (`x.com/jack`) and a post with its replies (`x.com/jack/status/20`) both arrive as server-rendered HTML, so `oc open` reads them without an account, a token, or a third-party mirror. A profile comes to about 390 tokens including the visible timeline, a post with four replies about 260. The rest of the site does need a login: search, explore, hashtag pages, and the replies, media, and highlights tabs all answer with "Something went wrong", and oc says so rather than pretending.

 The engine also renders Atom and RSS feeds as regular pages. That is how Stack Overflow works: the site serves every HTML page a Cloudflare challenge, but publishes full question and answer bodies under `/feeds`, so `oc open stackoverflow.com/feeds/question/11227809` returns the question and its top answers in about 500 tokens. The same trick applies to any site that gates its pages but leaves its feeds open.

+YouTube watch pages get a similar trick for a different reason: the page needs client JS to become interactive, but the title, description, view count, and caption tracks already ship inline in the initial HTML response as `ytInitialPlayerResponse`, so `oc open` on a watch page reads that directly instead of waiting on the v0.3 headless fallback. Each caption track becomes a numbered link, and `oc do` on it fetches the transcript as one block of plain text, paged like any other long document with `oc next`/`oc read`. Channel pages and everything else on the site that still needs real JS rendering remain in the "not supported yet" list below.
+
 Not supported yet: pages that only render with JavaScript (a headless fallback is planned for v0.3), sites behind logins (page state is saved, cookies are not, so logins land in v0.2), and sites with hard bot challenges that do not expose feeds.

 Want a website on that list? Open a pull request, or open an issue naming the site and the commands it should have and leave it for someone else to pick up. Either is welcome, and the issue is genuinely useful on its own: knowing which sites people want is the part that is hard to guess.
@@ -0,0 +1,7 @@
+{
+  "domain": "youtube.com",
+  "commands": {
+    "video": { "open": "https://www.youtube.com/watch?v={id}", "args": ["id"] },
+    "channel": { "open": "https://www.youtube.com/@{name}", "args": ["name"] }
+  }
+}
@@ -80,7 +80,7 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
 * @returns {Page}
 */
 export function distill(html, url = '') {
-  const { document } = parseHTML(feedToHTML(html) ?? html);
+  const { document } = parseHTML(youtubeToHTML(html) ?? transcriptToHTML(html) ?? feedToHTML(html) ?? html);
  const title = clean(document.querySelector('title')?.textContent ?? '');
  /** @type {Block[]} */
  const blocks = [];
@@ -303,7 +303,7 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
 * @param {string} html
 */
 function cleanDocument(html) {
-  const { document } = parseHTML(feedToHTML(html) ?? html);
+  const { document } = parseHTML(youtubeToHTML(html) ?? transcriptToHTML(html) ?? feedToHTML(html) ?? html);
  // Read the title before the sweep below removes the head with it.
  const title = clean(document.querySelector('title')?.textContent ?? '');
  for (const tag of DROP) {
@@ -392,6 +392,103 @@ export function feedToHTML(text) {
  return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
 }

+const escHTML = (s) => String(s ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+
+/**
+ * Pull one embedded JSON object out of a page by its variable name, scanning
+ * forward from the first `{` and tracking string/escape state so a brace
+ * inside a quoted value never ends the object early. A regex can't do this
+ * safely because the JSON itself contains braces.
+ * @param {string} text
+ * @param {string} marker
+ * @returns {any}
+ */
+function extractJSON(text, marker) {
+  const at = text.indexOf(marker);
+  if (at === -1) return null;
+  const start = text.indexOf('{', at);
+  if (start === -1) return null;
+  let depth = 0, inString = false, escaped = false;
+  for (let i = start; i < text.length; i++) {
+    const c = text[i];
+    if (inString) {
+      if (escaped) escaped = false;
+      else if (c === '\\') escaped = true;
+      else if (c === '"') inString = false;
+      continue;
+    }
+    if (c === '"') inString = true;
+    else if (c === '{') depth++;
+    else if (c === '}' && --depth === 0) {
+      try {
+        return JSON.parse(text.slice(start, i + 1));
+      } catch {
+        return null;
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * A YouTube watch page needs client JS to become interactive, but the initial
+ * HTML response already carries the title, description, view count, and
+ * caption tracks inline as `ytInitialPlayerResponse`, so none of that has to
+ * wait on the v0.3 headless fallback. Each caption track becomes a link to
+ * its timedtext URL, which `oc do` follows and transcriptToHTML turns into
+ * readable text. Returns null for anything that isn't a watch page.
+ * @param {string} text
+ * @returns {string | null}
+ */
+export function youtubeToHTML(text) {
+  const data = extractJSON(text, 'ytInitialPlayerResponse');
+  const details = data?.videoDetails;
+  if (!details?.title) return null;
+  const micro = data.microformat?.playerMicroformatRenderer;
+  const tracks = data.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
+  const views = details.viewCount ? `${Number(details.viewCount).toLocaleString('en-US')} views` : '';
+  const byline = [details.author && `by ${details.author}`, views, micro?.publishDate].filter(Boolean).join(', ');
+  const channelHref = details.channelId ? `https://www.youtube.com/channel/${details.channelId}` : '';
+  const parts = [`<h1>${escHTML(details.title)}</h1>`];
+  if (byline || channelHref) {
+    parts.push(`<p>${escHTML(byline)}${channelHref ? ` <a href="${escHTML(channelHref)}">channel</a>` : ''}</p>`);
+  }
+  const description = details.shortDescription ?? micro?.description?.simpleText ?? '';
+  for (const para of description.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean)) {
+    parts.push(`<p>${escHTML(para)}</p>`);
+  }
+  for (const track of tracks) {
+    const label = track.name?.simpleText || track.languageCode || 'transcript';
+    const kind = track.kind === 'asr' ? ' (auto-generated)' : '';
+    parts.push(`<p><a href="${escHTML(track.baseUrl)}">transcript: ${escHTML(label)}${kind}</a></p>`);
+  }
+  return `<html><head><title>${escHTML(details.title)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
+}
+
+/**
+ * YouTube's timedtext endpoint answers plain XML, one `<text>` cue per
+ * caption line. Concatenated into a single block rather than one paragraph
+ * per cue, so a whole transcript pages through `oc next`/`oc read` like any
+ * other long document instead of costing one numbered block per few words.
+ * Consecutive duplicate cues (auto captions repeat words across overlapping
+ * segments) are dropped. Returns null for anything that isn't a transcript.
+ * @param {string} text
+ * @returns {string | null}
+ */
+export function transcriptToHTML(text) {
+  if (!/<transcript[\s>]/i.test(text.slice(0, 200))) return null;
+  const { document } = parseHTML(text);
+  const cues = [...document.querySelectorAll('text')];
+  if (!cues.length) return null;
+  const lines = [];
+  for (const cue of cues) {
+    const line = clean(cue.textContent ?? '');
+    if (line && line !== lines[lines.length - 1]) lines.push(line);
+  }
+  if (!lines.length) return null;
+  return `<html><head><title>Transcript</title></head><body>\n<p>${escHTML(lines.join(' '))}</p>\n</body></html>`;
+}
+
 /**
 * Adjacent text nodes arrive fragmented (one per inline element boundary).
 * Merging them is what turns DOM noise into readable lines.
@@ -1,7 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { readFileSync } from 'node:fs';
-import { distill, toMarkdown, toHTML, feedToHTML, TEXT_CAP } from '../src/distill.js';
+import { distill, toMarkdown, toHTML, feedToHTML, youtubeToHTML, transcriptToHTML, TEXT_CAP } from '../src/distill.js';
 import { render, estimateTokens } from '../src/render.js';

 const html = readFileSync(new URL('./pages/news.html', import.meta.url), 'utf8');
@@ -153,6 +153,40 @@ test('rss with cdata bodies converts too, ordinary html does not', () => {
  assert.equal(feedToHTML(html), null, 'ordinary html misread as a feed');
 });

+test('a youtube watch page renders as title, byline, description, and transcript links', () => {
+  const watch = readFileSync(new URL('./pages/youtube_watch.html', import.meta.url), 'utf8');
+  const p = distill(watch, 'https://www.youtube.com/watch?v=fixture123');
+  assert.equal(p.title, 'Rust for busy people: the borrow checker in five minutes');
+  const heading = p.blocks.find((b) => b.type === 'heading');
+  assert.equal(heading.text, 'Rust for busy people: the borrow checker in five minutes');
+  const text = p.blocks.map((b) => b.text).join(' | ');
+  assert.ok(text.includes('by Fixture Channel'), 'author missing from byline');
+  assert.ok(text.includes('128,453 views'), 'view count missing');
+  assert.ok(text.includes('2026-03-04'), 'publish date missing');
+  assert.ok(text.includes('A quick walkthrough of ownership and borrowing.'), 'description missing');
+  const channel = p.blocks.find((b) => b.type === 'link' && b.text === 'channel');
+  assert.equal(channel.href, 'https://www.youtube.com/channel/UCexampleChannelId000001');
+  const transcripts = p.blocks.filter((b) => b.type === 'link' && b.text.startsWith('transcript:'));
+  assert.equal(transcripts.length, 2);
+  assert.ok(transcripts[0].text.includes('auto-generated'), 'asr track not labeled auto-generated');
+  assert.equal(transcripts[0].href, 'https://www.youtube.com/api/timedtext?v=fixture123&lang=en');
+  assert.ok(!transcripts[1].text.includes('auto-generated'), 'manual track mislabeled auto-generated');
+  assert.equal(youtubeToHTML(html), null, 'ordinary html misread as a watch page');
+});
+
+test('a youtube timedtext response becomes one readable block, duplicates dropped', () => {
+  const xml = '<?xml version="1.0" encoding="utf-8" ?><transcript>'
+    + '<text start="0.0" dur="2.5">We&#39;re no strangers to love</text>'
+    + '<text start="2.5" dur="2.5">We&#39;re no strangers to love</text>'
+    + '<text start="5.0" dur="3.0">You know the rules and so do I</text>'
+    + '</transcript>';
+  const p = distill(xml, 'https://www.youtube.com/api/timedtext?v=fixture123&lang=en');
+  const text = p.blocks.map((b) => b.text).join(' ');
+  assert.equal((text.match(/We're no strangers to love/g) ?? []).length, 1, 'duplicate cue was not dropped');
+  assert.ok(text.includes('You know the rules and so do I'), 'second cue missing');
+  assert.equal(transcriptToHTML(html), null, 'ordinary html misread as a transcript');
+});
+
 test('the content leads the page and the chrome follows it', () => {
  const { text } = render(thread(), { budget: 500 });
  const lines = text.split('\n');
@@ -0,0 +1,12 @@
+<!doctype html>
+<html>
+<head><title>Rust for busy people: the borrow checker in five minutes - YouTube</title></head>
+<body>
+<div id="content">
+  <nav>skip to content</nav>
+  <div id="player-shell">loading player...</div>
+</div>
+<script nonce="abc123">var ytInitialPlayerResponse = {"videoDetails":{"videoId":"fixture123","title":"Rust for busy people: the borrow checker in five minutes","lengthSeconds":"312","channelId":"UCexampleChannelId000001","shortDescription":"A quick walkthrough of ownership and borrowing.\n\nTimestamps:\n0:00 intro\n1:20 ownership\n\n#rust #programming","viewCount":"128453","author":"Fixture Channel"},"microformat":{"playerMicroformatRenderer":{"publishDate":"2026-03-04","description":{"simpleText":"A quick walkthrough of ownership and borrowing."}}},"captions":{"playerCaptionsTracklistRenderer":{"captionTracks":[{"baseUrl":"https://www.youtube.com/api/timedtext?v=fixture123&lang=en","name":{"simpleText":"English"},"languageCode":"en","kind":"asr"},{"baseUrl":"https://www.youtube.com/api/timedtext?v=fixture123&lang=en&kind=manual","name":{"simpleText":"English"},"languageCode":"en"}]}}};</script>
+<script nonce="abc123">var ytcfg = {"INNERTUBE_CONTEXT": {}};</script>
+</body>
+</html>