feat(markdown): add /g/ [code] tag syntax highlighting

Parse [code] blocks on /g/ into Prettify-style highlighted code boxes while leaving the tags as literal text on other boards.
This commit is contained in:
Tommaso Casaburi
2026-06-05 17:02:44 +07:00
parent bdbe44aa90
commit ae16ae3dd3
8 changed files with 508 additions and 27 deletions
@@ -0,0 +1,47 @@
/* Matches 4chan's computed prettyprint styling: bare monospace at the surrounding post-text
size, 5px padding, no border on light themes (white box on the post bg), translucent box
with a border on dark themes. box-sizing/max-width keep it from overflowing on mobile. */
.code {
display: inline-block;
box-sizing: border-box;
/* Center the box on the line so a single-line block sits level with adjacent text;
has no effect on a block that sits alone on its own line (the common 4chan case). */
vertical-align: middle;
max-width: min(600px, 100%);
max-height: 400px;
overflow: auto;
margin: 0;
padding: 5px;
background-color: var(--code-block-background-color);
border: var(--code-block-border, none);
color: var(--code-token-pln-color);
font-family: monospace;
line-height: normal;
white-space: pre;
tab-size: 8;
-moz-tab-size: 8;
}
.pln {
color: var(--code-token-pln-color);
}
.str {
color: var(--code-token-str-color);
}
.kwd {
color: var(--code-token-kwd-color);
}
.com {
color: var(--code-token-com-color);
}
.lit {
color: var(--code-token-lit-color);
}
.pun {
color: var(--code-token-pun-color);
}
+25
View File
@@ -0,0 +1,25 @@
import { useMemo } from 'react';
import { highlightCode } from '../../lib/utils/code-highlight';
import styles from './code-block.module.css';
/**
* Renders a [code] block the way 4chan does on /g/: a flat, square, monospaced box
* that preserves whitespace and syntax-highlights tokens using a Prettify-style palette.
* Uses a block-level <code> element (phrasing content) so it nests validly inside the
* inline markdown <span>.
*/
const CodeBlock = ({ source }: { source: string }) => {
const tokens = useMemo(() => highlightCode(source), [source]);
return (
<code className={styles.code}>
{tokens.map((token) => (
<span key={token.start} className={styles[token.cls]}>
{token.value}
</span>
))}
</code>
);
};
export default CodeBlock;
+1
View File
@@ -0,0 +1 @@
export { default } from './code-block';
@@ -285,6 +285,34 @@ describe('Markdown', () => {
expect(container.textContent).toContain('[spoiler]spoiled text[/spoiler]'); expect(container.textContent).toContain('[spoiler]spoiled text[/spoiler]');
}); });
it('renders [code] blocks as syntax-highlighted code on /g/', async () => {
await renderMarkdown(
{
content: 'before\n[code]>not a quote\nconst x = 1;[/code]\nafter',
communityAddress: 'technology-posting.bso',
},
'/g/thread/post-1',
);
const code = container.querySelector('code');
expect(code).not.toBeNull();
expect(code?.textContent).toBe('>not a quote\nconst x = 1;');
// Code contents are tokenized into spans and never parsed as greentext.
expect(code?.querySelectorAll('span').length).toBeGreaterThan(0);
expect(container.querySelector('.greentext')).toBeNull();
expect(container.textContent).toContain('before');
expect(container.textContent).toContain('after');
});
it('renders [code] as literal text off /g/', async () => {
await renderMarkdown({
content: '[code]const x = 1;[/code]',
});
expect(container.querySelector('code')).toBeNull();
expect(container.textContent).toBe('[code]const x = 1;[/code]');
});
it('renders greentext for any leading marker run while preserving quote links', async () => { it('renders greentext for any leading marker run while preserving quote links', async () => {
await renderMarkdown({ await renderMarkdown({
content: '>green line\n>>test\n>>>>>>>test\n>>42\n>>>/fit/', content: '>green line\n>>test\n>>>>>>>test\n>>42\n>>>/fit/',
+81 -27
View File
@@ -6,6 +6,7 @@ import { getLinkMediaInfo, getHasThumbnail } from '../../lib/utils/media-utils';
import { isCatalogView } from '../../lib/utils/view-utils'; import { isCatalogView } from '../../lib/utils/view-utils';
import useIsMobile from '../../hooks/use-is-mobile'; import useIsMobile from '../../hooks/use-is-mobile';
import CommentMedia from '../comment-media'; import CommentMedia from '../comment-media';
import CodeBlock from '../code-block';
import styles from './markdown.module.css'; import styles from './markdown.module.css';
import { Link, useLocation, useParams } from 'react-router-dom'; import { Link, useLocation, useParams } from 'react-router-dom';
import { canEmbed } from '../embed'; import { canEmbed } from '../embed';
@@ -708,6 +709,69 @@ const renderLineContent = (line: string, context: RenderContext): React.ReactNod
return elements; return elements;
}; };
// [code]...[/code] blocks (4chan rule 4 of /g/): rendered literally, never parsed for
// greentext/quotelinks/spoilers, and syntax-highlighted via <CodeBlock>.
const CODE_TAG_REGEX = /\[code\]([\s\S]*?)\[\/code\]/gi;
const HAS_CODE_TAG_REGEX = /\[code\]/i;
const CODE_DIRECTORY_CODE = 'g';
type ContentSegment = { type: 'text' | 'code'; value: string; start: number };
const splitCodeSegments = (raw: string): ContentSegment[] => {
const segments: ContentSegment[] = [];
const regex = new RegExp(CODE_TAG_REGEX.source, 'gi');
let lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = regex.exec(raw)) !== null) {
if (match.index > lastIndex) {
segments.push({ type: 'text', value: raw.slice(lastIndex, match.index), start: lastIndex });
}
// Drop one leading/trailing newline so [code]\n...\n[/code] has no blank first/last line.
segments.push({ type: 'code', value: match[1].replace(/^\n/, '').replace(/\n$/, ''), start: match.index });
lastIndex = regex.lastIndex;
}
if (lastIndex < raw.length) {
segments.push({ type: 'text', value: raw.slice(lastIndex), start: lastIndex });
}
return segments;
};
const renderTextLines = (normalized: string, context: RenderContext, keyPrefix: string): React.ReactNode[] => {
const lines = normalized.split('\n');
const elements: React.ReactNode[] = [];
let lineOffset = 0;
lines.forEach((line, lineIndex) => {
const lineKey = `${keyPrefix}line-${lineOffset}`;
lineOffset += line.length + 1;
if (lineIndex > 0) {
elements.push(<br key={`br-${lineKey}`} />);
}
if (line.length === 0) return;
const isGreentext = isGreentextLine(line);
const lineElements = renderLineContent(line, context);
if (isGreentext) {
elements.push(
<span key={lineKey} className='greentext'>
{lineElements}
</span>,
);
} else {
elements.push(<React.Fragment key={lineKey}>{lineElements}</React.Fragment>);
}
});
return elements;
};
const Markdown = ({ content, title, postCid, communityAddress, parseSpoilers = true }: MarkdownProps) => { const Markdown = ({ content, title, postCid, communityAddress, parseSpoilers = true }: MarkdownProps) => {
const location = useLocation(); const location = useLocation();
const params = useParams(); const params = useParams();
@@ -716,42 +780,32 @@ const Markdown = ({ content, title, postCid, communityAddress, parseSpoilers = t
const enableQstBbcode = location.pathname.split('/').filter(Boolean)[0] === 'qst'; const enableQstBbcode = location.pathname.split('/').filter(Boolean)[0] === 'qst';
const activeDirectoryCode = getActiveDirectoryCode(location.pathname, communityAddress, directories); const activeDirectoryCode = getActiveDirectoryCode(location.pathname, communityAddress, directories);
const enableFortuneMarkup = isFortuneDirectoryCode(activeDirectoryCode); const enableFortuneMarkup = isFortuneDirectoryCode(activeDirectoryCode);
// [code] tags are a /g/ feature (4chan rule 4): enabled when the post's board or the current
// route resolves to /g/, and rendered as literal text everywhere else.
const enableCodeTags =
getDirectoryCodeForIdentifier(getRouteBoardIdentifier(location.pathname), directories) === CODE_DIRECTORY_CODE ||
getDirectoryCodeForIdentifier(communityAddress, directories) === CODE_DIRECTORY_CODE;
const rendered = useMemo(() => { const rendered = useMemo(() => {
const normalized = normalizeContent(content || '');
const lines = normalized.split('\n');
const elements: React.ReactNode[] = [];
let lineOffset = 0;
const context = { isInCatalogView, postCid, communityAddress, enableFortuneMarkup, enableQstBbcode, parseSpoilers }; const context = { isInCatalogView, postCid, communityAddress, enableFortuneMarkup, enableQstBbcode, parseSpoilers };
const raw = content || '';
lines.forEach((line, lineIndex) => { if (!enableCodeTags || !HAS_CODE_TAG_REGEX.test(raw)) {
const lineKey = `line-${lineOffset}`; return renderTextLines(normalizeContent(raw), context, '');
lineOffset += line.length + 1; }
if (lineIndex > 0) { const elements: React.ReactNode[] = [];
elements.push(<br key={`br-${lineKey}`} />); splitCodeSegments(raw).forEach((segment) => {
} if (segment.type === 'code') {
elements.push(<CodeBlock key={`code-${segment.start}`} source={segment.value} />);
if (line.length === 0) return; return;
const isGreentext = isGreentextLine(line);
const lineElements = renderLineContent(line, context);
if (isGreentext) {
elements.push(
<span key={lineKey} className='greentext'>
{lineElements}
</span>,
);
} else {
elements.push(<React.Fragment key={lineKey}>{lineElements}</React.Fragment>);
} }
if (!segment.value) return;
elements.push(<React.Fragment key={`text-${segment.start}`}>{renderTextLines(normalizeContent(segment.value), context, `${segment.start}:`)}</React.Fragment>);
}); });
return elements; return elements;
}, [content, isInCatalogView, postCid, communityAddress, enableFortuneMarkup, enableQstBbcode, parseSpoilers]); }, [content, isInCatalogView, postCid, communityAddress, enableFortuneMarkup, enableQstBbcode, parseSpoilers, enableCodeTags]);
return ( return (
<span className={styles.markdown}> <span className={styles.markdown}>
@@ -0,0 +1,62 @@
import { describe, it, expect } from 'vitest';
import { highlightCode, type CodeToken } from '../code-highlight';
// Adjacent plain tokens (including whitespace) are merged, like Prettify, so match by trimmed value.
const classOf = (tokens: CodeToken[], value: string): string | undefined => tokens.find((token) => token.value.trim() === value)?.cls;
const reassemble = (tokens: CodeToken[]): string => tokens.map((token) => token.value).join('');
describe('highlightCode', () => {
it('preserves the source exactly when tokens are concatenated', () => {
const source = '.element {\n animation-delay: calc((sibling-index() - 1) * 100ms);\n}';
expect(reassemble(highlightCode(source))).toBe(source);
});
it('classifies the 4chan CSS example like Prettify (punctuation, plain, literals)', () => {
const tokens = highlightCode('.element {\n animation-delay: calc((sibling-index() - 1) * 100ms);\n}');
// Numbers (incl. CSS units) are literals.
expect(classOf(tokens, '1')).toBe('lit');
expect(classOf(tokens, '100ms')).toBe('lit');
// Hyphenated CSS identifiers split on punctuation, exactly as Prettify renders them.
expect(classOf(tokens, 'animation')).toBe('pln');
expect(classOf(tokens, 'delay')).toBe('pln');
expect(tokens.some((token) => token.cls === 'pun' && token.value.includes('{'))).toBe(true);
});
it('highlights keywords from the union of languages (like Prettify with no language)', () => {
const tokens = highlightCode('select string body');
expect(classOf(tokens, 'select')).toBe('kwd');
expect(classOf(tokens, 'string')).toBe('kwd');
expect(classOf(tokens, 'body')).toBe('pln');
});
it('highlights strings, including unterminated ones', () => {
expect(classOf(highlightCode('font-family: "Consolas";'), '"Consolas"')).toBe('str');
expect(classOf(highlightCode("x = 'hi'"), "'hi'")).toBe('str');
expect(highlightCode('s = "unterminated').some((token) => token.cls === 'str' && token.value === '"unterminated')).toBe(true);
});
it('highlights line, block, and hash comments', () => {
expect(classOf(highlightCode('x = 1 // note'), '// note')).toBe('com');
expect(classOf(highlightCode('a /* mid */ b'), '/* mid */')).toBe('com');
expect(classOf(highlightCode('x = 1 # python'), '# python')).toBe('com');
});
it('does not treat CSS hex colors as comments', () => {
const tokens = highlightCode('color: #fff;');
expect(tokens.some((token) => token.cls === 'com')).toBe(false);
expect(classOf(tokens, '#')).toBe('pun');
expect(classOf(tokens, 'fff')).toBe('pln');
});
it('treats keywords individually and merges adjacent plain runs', () => {
const tokens = highlightCode('const x');
expect(classOf(tokens, 'const')).toBe('kwd');
// " x" (leading space + identifier) merges into a single plain token.
expect(tokens.find((token) => token.value === ' x')?.cls).toBe('pln');
});
it('returns no tokens for empty input', () => {
expect(highlightCode('')).toEqual([]);
});
});
+238
View File
@@ -0,0 +1,238 @@
/**
* Self-contained, language-agnostic syntax highlighter that mirrors the token
* classes 4chan emits for [code] blocks via Google Code Prettify
* (https://github.com/googlearchive/code-prettify): a generic lexer that splits
* source into plain text, strings, comments, numbers, keywords and punctuation.
*
* We replicate Prettify's class names (`pln`/`str`/`kwd`/`com`/`lit`/`pun`) so the
* theme CSS can colour tokens the same way the default `prettify.css` palette does,
* without pulling in the archived library or running DOM mutation / dangerouslySetInnerHTML.
*
* Like Prettify's no-language lexer, keyword matching uses the union of common
* language keywords, which is why words such as `select` or `string` highlight as
* keywords regardless of the snippet's actual language.
*/
export type CodeTokenClass = 'pln' | 'str' | 'kwd' | 'com' | 'lit' | 'pun';
export interface CodeToken {
cls: CodeTokenClass;
value: string;
/** Start offset of the token in the source, usable as a stable React key. */
start: number;
}
// Union of keywords across common languages seen on /g/ (C/C++, Java, C#, JS/TS,
// Python, Ruby, Go, Rust, PHP, shell, SQL). Matches Prettify's "all keywords" behaviour.
const KEYWORDS = new Set<string>([
// control flow
'if',
'else',
'elif',
'for',
'while',
'do',
'switch',
'case',
'default',
'break',
'continue',
'return',
'goto',
'yield',
'await',
'async',
// declarations / structure
'var',
'let',
'const',
'function',
'func',
'fn',
'def',
'lambda',
'class',
'struct',
'enum',
'interface',
'trait',
'impl',
'type',
'typedef',
'union',
'template',
'typename',
'namespace',
'module',
'package',
'mod',
'crate',
'import',
'export',
'from',
'require',
'include',
'using',
'use',
'extends',
'implements',
'public',
'private',
'protected',
'static',
'final',
'abstract',
'virtual',
'override',
'readonly',
'volatile',
'register',
'inline',
'extern',
'explicit',
'friend',
'mutable',
'constexpr',
'decltype',
'operator',
'pub',
// values / types
'true',
'false',
'null',
'nil',
'none',
'undefined',
'void',
'int',
'long',
'short',
'char',
'float',
'double',
'bool',
'boolean',
'string',
'str',
'byte',
'unsigned',
'signed',
'auto',
'new',
'delete',
'this',
'self',
'super',
'sizeof',
'typeof',
'instanceof',
'in',
'is',
'as',
'of',
// exceptions
'try',
'catch',
'finally',
'throw',
'throws',
'raise',
'except',
'ensure',
'rescue',
// python / ruby
'and',
'or',
'not',
'pass',
'with',
'global',
'nonlocal',
'del',
'then',
'end',
'begin',
'unless',
'until',
'when',
'redo',
'retry',
// go / rust / misc
'go',
'defer',
'chan',
'select',
'map',
'range',
'make',
'mut',
'match',
'where',
'move',
'ref',
'loop',
'unsafe',
'nullptr',
]);
const WHITESPACE_RE = /^\s+/;
const BLOCK_COMMENT_RE = /^\/\*[\s\S]*?(?:\*\/|$)/;
const LINE_COMMENT_RE = /^\/\/[^\n]*/;
// Hash comment only when followed by whitespace, so CSS hex colours (#fff) and
// preprocessor/anchors (#include, #foo) are not swallowed as comments.
const HASH_COMMENT_RE = /^#[ \t][^\n]*/;
const DQUOTE_STRING_RE = /^"(?:\\[\s\S]|[^"\\\n])*(?:"|$)/;
const SQUOTE_STRING_RE = /^'(?:\\[\s\S]|[^'\\\n])*(?:'|$)/;
const TEMPLATE_STRING_RE = /^`(?:\\[\s\S]|[^`\\])*(?:`|$)/;
// Numbers with optional CSS unit / numeric suffix (100ms, 20px, 0xFF, 1.5e3, 100%).
const NUMBER_RE = /^(?:0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)[a-zA-Z%]*/;
const IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z0-9_$]*/;
// Runs of punctuation, excluding the chars that start the rules above (/ " ' ` # \ $).
const PUNCTUATION_RE = /^[-!%&()*+,.:;<=>?@^{|}~[\]]+/;
const FALLBACK_PUNCTUATION_RE = /[/#\\]/;
/**
* Tokenize source code into Prettify-style classified tokens. Adjacent tokens of the
* same class are merged (matching Prettify's plain-run grouping) to keep the DOM small.
*/
export const highlightCode = (source: string): CodeToken[] => {
const tokens: CodeToken[] = [];
const push = (cls: CodeTokenClass, value: string, start: number) => {
if (!value) return;
const last = tokens[tokens.length - 1];
if (last && last.cls === cls) {
last.value += value;
} else {
tokens.push({ cls, value, start });
}
};
let index = 0;
while (index < source.length) {
const start = index;
const rest = source.slice(index);
let match: RegExpExecArray | null;
if ((match = WHITESPACE_RE.exec(rest))) {
push('pln', match[0], start);
} else if ((match = BLOCK_COMMENT_RE.exec(rest)) || (match = LINE_COMMENT_RE.exec(rest)) || (match = HASH_COMMENT_RE.exec(rest))) {
push('com', match[0], start);
} else if ((match = DQUOTE_STRING_RE.exec(rest)) || (match = SQUOTE_STRING_RE.exec(rest)) || (match = TEMPLATE_STRING_RE.exec(rest))) {
push('str', match[0], start);
} else if ((match = NUMBER_RE.exec(rest))) {
push('lit', match[0], start);
} else if ((match = IDENTIFIER_RE.exec(rest))) {
push(KEYWORDS.has(match[0]) ? 'kwd' : 'pln', match[0], start);
} else if ((match = PUNCTUATION_RE.exec(rest))) {
push('pun', match[0], start);
} else {
const char = rest[0];
push(FALLBACK_PUNCTUATION_RE.test(char) ? 'pun' : 'pln', char, start);
}
index += match ? match[0].length : 1;
}
return tokens;
};
+26
View File
@@ -1,3 +1,29 @@
/* Code block ([code] tags on /g/) — Prettify-style token palette.
Light themes use the default google-code-prettify colours; dark themes
(.tomorrow, .spooky) use 4chan's "Tomorrow" code palette. */
:root {
--code-block-background-color: #ffffff;
--code-block-border: none;
--code-token-pln-color: #000000;
--code-token-str-color: #008000;
--code-token-kwd-color: #000088;
--code-token-com-color: #880000;
--code-token-lit-color: #006666;
--code-token-pun-color: #666600;
}
:root .tomorrow,
:root .spooky {
--code-block-background-color: rgba(255, 255, 255, 0.1);
--code-block-border: 1px solid rgba(0, 0, 0, 0.5);
--code-token-pln-color: #c5c8c6;
--code-token-str-color: #8ea062;
--code-token-kwd-color: #81a2be;
--code-token-com-color: #bb6793;
--code-token-lit-color: #6e9b89;
--code-token-pun-color: #72814d;
}
:root .yotsuba { :root .yotsuba {
--color-scheme: light; --color-scheme: light;