support youtube watch pages and transcripts

Watch pages need client JS to become interactive, but the title, description, view count, and caption tracks already ship inline in the initial HTML as ytInitialPlayerResponse, so this reads that directly instead of waiting on the v0.3 headless fallback. Each caption track becomes a numbered link, and oc do on it fetches the timedtext transcript, collapsed into one block so it pages through oc next/oc read like any other long document instead of costing one block per caption line. Adds youtubeToHTML and transcriptToHTML alongside feedToHTML in the distiller, a youtube.com.json shortcut, and offline tests against a fixture watch page. Fixes #6
2026-09-15 10:40:56 +02:00 · 2026-08-19 13:27:52 -04:00
parent ea3afeb25a
commit 01b607ac81
5 changed files with 156 additions and 3 deletions
@@ -1,7 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { readFileSync } from 'node:fs';
-import { distill, toMarkdown, toHTML, feedToHTML, TEXT_CAP } from '../src/distill.js';
+import { distill, toMarkdown, toHTML, feedToHTML, youtubeToHTML, transcriptToHTML, TEXT_CAP } from '../src/distill.js';
 import { render, estimateTokens } from '../src/render.js';

 const html = readFileSync(new URL('./pages/news.html', import.meta.url), 'utf8');
@@ -153,6 +153,40 @@ test('rss with cdata bodies converts too, ordinary html does not', () => {
  assert.equal(feedToHTML(html), null, 'ordinary html misread as a feed');
 });

+test('a youtube watch page renders as title, byline, description, and transcript links', () => {
+  const watch = readFileSync(new URL('./pages/youtube_watch.html', import.meta.url), 'utf8');
+  const p = distill(watch, 'https://www.youtube.com/watch?v=fixture123');
+  assert.equal(p.title, 'Rust for busy people: the borrow checker in five minutes');
+  const heading = p.blocks.find((b) => b.type === 'heading');
+  assert.equal(heading.text, 'Rust for busy people: the borrow checker in five minutes');
+  const text = p.blocks.map((b) => b.text).join(' | ');
+  assert.ok(text.includes('by Fixture Channel'), 'author missing from byline');
+  assert.ok(text.includes('128,453 views'), 'view count missing');
+  assert.ok(text.includes('2026-03-04'), 'publish date missing');
+  assert.ok(text.includes('A quick walkthrough of ownership and borrowing.'), 'description missing');
+  const channel = p.blocks.find((b) => b.type === 'link' && b.text === 'channel');
+  assert.equal(channel.href, 'https://www.youtube.com/channel/UCexampleChannelId000001');
+  const transcripts = p.blocks.filter((b) => b.type === 'link' && b.text.startsWith('transcript:'));
+  assert.equal(transcripts.length, 2);
+  assert.ok(transcripts[0].text.includes('auto-generated'), 'asr track not labeled auto-generated');
+  assert.equal(transcripts[0].href, 'https://www.youtube.com/api/timedtext?v=fixture123&lang=en');
+  assert.ok(!transcripts[1].text.includes('auto-generated'), 'manual track mislabeled auto-generated');
+  assert.equal(youtubeToHTML(html), null, 'ordinary html misread as a watch page');
+});
+
+test('a youtube timedtext response becomes one readable block, duplicates dropped', () => {
+  const xml = '<?xml version="1.0" encoding="utf-8" ?><transcript>'
+    + '<text start="0.0" dur="2.5">We&#39;re no strangers to love</text>'
+    + '<text start="2.5" dur="2.5">We&#39;re no strangers to love</text>'
+    + '<text start="5.0" dur="3.0">You know the rules and so do I</text>'
+    + '</transcript>';
+  const p = distill(xml, 'https://www.youtube.com/api/timedtext?v=fixture123&lang=en');
+  const text = p.blocks.map((b) => b.text).join(' ');
+  assert.equal((text.match(/We're no strangers to love/g) ?? []).length, 1, 'duplicate cue was not dropped');
+  assert.ok(text.includes('You know the rules and so do I'), 'second cue missing');
+  assert.equal(transcriptToHTML(html), null, 'ordinary html misread as a transcript');
+});
+
 test('the content leads the page and the chrome follows it', () => {
  const { text } = render(thread(), { budget: 500 });
  const lines = text.split('\n');
@@ -0,0 +1,12 @@
+<!doctype html>
+<html>
+<head><title>Rust for busy people: the borrow checker in five minutes - YouTube</title></head>
+<body>
+<div id="content">
+  <nav>skip to content</nav>
+  <div id="player-shell">loading player...</div>
+</div>
+<script nonce="abc123">var ytInitialPlayerResponse = {"videoDetails":{"videoId":"fixture123","title":"Rust for busy people: the borrow checker in five minutes","lengthSeconds":"312","channelId":"UCexampleChannelId000001","shortDescription":"A quick walkthrough of ownership and borrowing.\n\nTimestamps:\n0:00 intro\n1:20 ownership\n\n#rust #programming","viewCount":"128453","author":"Fixture Channel"},"microformat":{"playerMicroformatRenderer":{"publishDate":"2026-03-04","description":{"simpleText":"A quick walkthrough of ownership and borrowing."}}},"captions":{"playerCaptionsTracklistRenderer":{"captionTracks":[{"baseUrl":"https://www.youtube.com/api/timedtext?v=fixture123&lang=en","name":{"simpleText":"English"},"languageCode":"en","kind":"asr"},{"baseUrl":"https://www.youtube.com/api/timedtext?v=fixture123&lang=en&kind=manual","name":{"simpleText":"English"},"languageCode":"en"}]}}};</script>
+<script nonce="abc123">var ytcfg = {"INNERTUBE_CONTEXT": {}};</script>
+</body>
+</html>