mirror of
https://github.com/bitsocialnet/5chan.git
synced 2026-08-03 07:41:04 +02:00
feat(markdown): add /g/ [code] tag syntax highlighting
Parse [code] blocks on /g/ into Prettify-style highlighted code boxes while leaving the tags as literal text on other boards.
This commit is contained in:
Tommaso Casaburi
@@ -0,0 +1,62 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { highlightCode, type CodeToken } from '../code-highlight';
|
||||
|
||||
// Adjacent plain tokens (including whitespace) are merged, like Prettify, so match by trimmed value.
|
||||
const classOf = (tokens: CodeToken[], value: string): string | undefined => tokens.find((token) => token.value.trim() === value)?.cls;
|
||||
|
||||
const reassemble = (tokens: CodeToken[]): string => tokens.map((token) => token.value).join('');
|
||||
|
||||
describe('highlightCode', () => {
|
||||
it('preserves the source exactly when tokens are concatenated', () => {
|
||||
const source = '.element {\n animation-delay: calc((sibling-index() - 1) * 100ms);\n}';
|
||||
expect(reassemble(highlightCode(source))).toBe(source);
|
||||
});
|
||||
|
||||
it('classifies the 4chan CSS example like Prettify (punctuation, plain, literals)', () => {
|
||||
const tokens = highlightCode('.element {\n animation-delay: calc((sibling-index() - 1) * 100ms);\n}');
|
||||
// Numbers (incl. CSS units) are literals.
|
||||
expect(classOf(tokens, '1')).toBe('lit');
|
||||
expect(classOf(tokens, '100ms')).toBe('lit');
|
||||
// Hyphenated CSS identifiers split on punctuation, exactly as Prettify renders them.
|
||||
expect(classOf(tokens, 'animation')).toBe('pln');
|
||||
expect(classOf(tokens, 'delay')).toBe('pln');
|
||||
expect(tokens.some((token) => token.cls === 'pun' && token.value.includes('{'))).toBe(true);
|
||||
});
|
||||
|
||||
it('highlights keywords from the union of languages (like Prettify with no language)', () => {
|
||||
const tokens = highlightCode('select string body');
|
||||
expect(classOf(tokens, 'select')).toBe('kwd');
|
||||
expect(classOf(tokens, 'string')).toBe('kwd');
|
||||
expect(classOf(tokens, 'body')).toBe('pln');
|
||||
});
|
||||
|
||||
it('highlights strings, including unterminated ones', () => {
|
||||
expect(classOf(highlightCode('font-family: "Consolas";'), '"Consolas"')).toBe('str');
|
||||
expect(classOf(highlightCode("x = 'hi'"), "'hi'")).toBe('str');
|
||||
expect(highlightCode('s = "unterminated').some((token) => token.cls === 'str' && token.value === '"unterminated')).toBe(true);
|
||||
});
|
||||
|
||||
it('highlights line, block, and hash comments', () => {
|
||||
expect(classOf(highlightCode('x = 1 // note'), '// note')).toBe('com');
|
||||
expect(classOf(highlightCode('a /* mid */ b'), '/* mid */')).toBe('com');
|
||||
expect(classOf(highlightCode('x = 1 # python'), '# python')).toBe('com');
|
||||
});
|
||||
|
||||
it('does not treat CSS hex colors as comments', () => {
|
||||
const tokens = highlightCode('color: #fff;');
|
||||
expect(tokens.some((token) => token.cls === 'com')).toBe(false);
|
||||
expect(classOf(tokens, '#')).toBe('pun');
|
||||
expect(classOf(tokens, 'fff')).toBe('pln');
|
||||
});
|
||||
|
||||
it('treats keywords individually and merges adjacent plain runs', () => {
|
||||
const tokens = highlightCode('const x');
|
||||
expect(classOf(tokens, 'const')).toBe('kwd');
|
||||
// " x" (leading space + identifier) merges into a single plain token.
|
||||
expect(tokens.find((token) => token.value === ' x')?.cls).toBe('pln');
|
||||
});
|
||||
|
||||
it('returns no tokens for empty input', () => {
|
||||
expect(highlightCode('')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* Self-contained, language-agnostic syntax highlighter that mirrors the token
|
||||
* classes 4chan emits for [code] blocks via Google Code Prettify
|
||||
* (https://github.com/googlearchive/code-prettify): a generic lexer that splits
|
||||
* source into plain text, strings, comments, numbers, keywords and punctuation.
|
||||
*
|
||||
* We replicate Prettify's class names (`pln`/`str`/`kwd`/`com`/`lit`/`pun`) so the
|
||||
* theme CSS can colour tokens the same way the default `prettify.css` palette does,
|
||||
* without pulling in the archived library or running DOM mutation / dangerouslySetInnerHTML.
|
||||
*
|
||||
* Like Prettify's no-language lexer, keyword matching uses the union of common
|
||||
* language keywords, which is why words such as `select` or `string` highlight as
|
||||
* keywords regardless of the snippet's actual language.
|
||||
*/
|
||||
|
||||
export type CodeTokenClass = 'pln' | 'str' | 'kwd' | 'com' | 'lit' | 'pun';
|
||||
|
||||
export interface CodeToken {
|
||||
cls: CodeTokenClass;
|
||||
value: string;
|
||||
/** Start offset of the token in the source, usable as a stable React key. */
|
||||
start: number;
|
||||
}
|
||||
|
||||
// Union of keywords across common languages seen on /g/ (C/C++, Java, C#, JS/TS,
|
||||
// Python, Ruby, Go, Rust, PHP, shell, SQL). Matches Prettify's "all keywords" behaviour.
|
||||
const KEYWORDS = new Set<string>([
|
||||
// control flow
|
||||
'if',
|
||||
'else',
|
||||
'elif',
|
||||
'for',
|
||||
'while',
|
||||
'do',
|
||||
'switch',
|
||||
'case',
|
||||
'default',
|
||||
'break',
|
||||
'continue',
|
||||
'return',
|
||||
'goto',
|
||||
'yield',
|
||||
'await',
|
||||
'async',
|
||||
// declarations / structure
|
||||
'var',
|
||||
'let',
|
||||
'const',
|
||||
'function',
|
||||
'func',
|
||||
'fn',
|
||||
'def',
|
||||
'lambda',
|
||||
'class',
|
||||
'struct',
|
||||
'enum',
|
||||
'interface',
|
||||
'trait',
|
||||
'impl',
|
||||
'type',
|
||||
'typedef',
|
||||
'union',
|
||||
'template',
|
||||
'typename',
|
||||
'namespace',
|
||||
'module',
|
||||
'package',
|
||||
'mod',
|
||||
'crate',
|
||||
'import',
|
||||
'export',
|
||||
'from',
|
||||
'require',
|
||||
'include',
|
||||
'using',
|
||||
'use',
|
||||
'extends',
|
||||
'implements',
|
||||
'public',
|
||||
'private',
|
||||
'protected',
|
||||
'static',
|
||||
'final',
|
||||
'abstract',
|
||||
'virtual',
|
||||
'override',
|
||||
'readonly',
|
||||
'volatile',
|
||||
'register',
|
||||
'inline',
|
||||
'extern',
|
||||
'explicit',
|
||||
'friend',
|
||||
'mutable',
|
||||
'constexpr',
|
||||
'decltype',
|
||||
'operator',
|
||||
'pub',
|
||||
// values / types
|
||||
'true',
|
||||
'false',
|
||||
'null',
|
||||
'nil',
|
||||
'none',
|
||||
'undefined',
|
||||
'void',
|
||||
'int',
|
||||
'long',
|
||||
'short',
|
||||
'char',
|
||||
'float',
|
||||
'double',
|
||||
'bool',
|
||||
'boolean',
|
||||
'string',
|
||||
'str',
|
||||
'byte',
|
||||
'unsigned',
|
||||
'signed',
|
||||
'auto',
|
||||
'new',
|
||||
'delete',
|
||||
'this',
|
||||
'self',
|
||||
'super',
|
||||
'sizeof',
|
||||
'typeof',
|
||||
'instanceof',
|
||||
'in',
|
||||
'is',
|
||||
'as',
|
||||
'of',
|
||||
// exceptions
|
||||
'try',
|
||||
'catch',
|
||||
'finally',
|
||||
'throw',
|
||||
'throws',
|
||||
'raise',
|
||||
'except',
|
||||
'ensure',
|
||||
'rescue',
|
||||
// python / ruby
|
||||
'and',
|
||||
'or',
|
||||
'not',
|
||||
'pass',
|
||||
'with',
|
||||
'global',
|
||||
'nonlocal',
|
||||
'del',
|
||||
'then',
|
||||
'end',
|
||||
'begin',
|
||||
'unless',
|
||||
'until',
|
||||
'when',
|
||||
'redo',
|
||||
'retry',
|
||||
// go / rust / misc
|
||||
'go',
|
||||
'defer',
|
||||
'chan',
|
||||
'select',
|
||||
'map',
|
||||
'range',
|
||||
'make',
|
||||
'mut',
|
||||
'match',
|
||||
'where',
|
||||
'move',
|
||||
'ref',
|
||||
'loop',
|
||||
'unsafe',
|
||||
'nullptr',
|
||||
]);
|
||||
|
||||
const WHITESPACE_RE = /^\s+/;
|
||||
const BLOCK_COMMENT_RE = /^\/\*[\s\S]*?(?:\*\/|$)/;
|
||||
const LINE_COMMENT_RE = /^\/\/[^\n]*/;
|
||||
// Hash comment only when followed by whitespace, so CSS hex colours (#fff) and
|
||||
// preprocessor/anchors (#include, #foo) are not swallowed as comments.
|
||||
const HASH_COMMENT_RE = /^#[ \t][^\n]*/;
|
||||
const DQUOTE_STRING_RE = /^"(?:\\[\s\S]|[^"\\\n])*(?:"|$)/;
|
||||
const SQUOTE_STRING_RE = /^'(?:\\[\s\S]|[^'\\\n])*(?:'|$)/;
|
||||
const TEMPLATE_STRING_RE = /^`(?:\\[\s\S]|[^`\\])*(?:`|$)/;
|
||||
// Numbers with optional CSS unit / numeric suffix (100ms, 20px, 0xFF, 1.5e3, 100%).
|
||||
const NUMBER_RE = /^(?:0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)[a-zA-Z%]*/;
|
||||
const IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z0-9_$]*/;
|
||||
// Runs of punctuation, excluding the chars that start the rules above (/ " ' ` # \ $).
|
||||
const PUNCTUATION_RE = /^[-!%&()*+,.:;<=>?@^{|}~[\]]+/;
|
||||
const FALLBACK_PUNCTUATION_RE = /[/#\\]/;
|
||||
|
||||
/**
|
||||
* Tokenize source code into Prettify-style classified tokens. Adjacent tokens of the
|
||||
* same class are merged (matching Prettify's plain-run grouping) to keep the DOM small.
|
||||
*/
|
||||
export const highlightCode = (source: string): CodeToken[] => {
|
||||
const tokens: CodeToken[] = [];
|
||||
|
||||
const push = (cls: CodeTokenClass, value: string, start: number) => {
|
||||
if (!value) return;
|
||||
const last = tokens[tokens.length - 1];
|
||||
if (last && last.cls === cls) {
|
||||
last.value += value;
|
||||
} else {
|
||||
tokens.push({ cls, value, start });
|
||||
}
|
||||
};
|
||||
|
||||
let index = 0;
|
||||
while (index < source.length) {
|
||||
const start = index;
|
||||
const rest = source.slice(index);
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
if ((match = WHITESPACE_RE.exec(rest))) {
|
||||
push('pln', match[0], start);
|
||||
} else if ((match = BLOCK_COMMENT_RE.exec(rest)) || (match = LINE_COMMENT_RE.exec(rest)) || (match = HASH_COMMENT_RE.exec(rest))) {
|
||||
push('com', match[0], start);
|
||||
} else if ((match = DQUOTE_STRING_RE.exec(rest)) || (match = SQUOTE_STRING_RE.exec(rest)) || (match = TEMPLATE_STRING_RE.exec(rest))) {
|
||||
push('str', match[0], start);
|
||||
} else if ((match = NUMBER_RE.exec(rest))) {
|
||||
push('lit', match[0], start);
|
||||
} else if ((match = IDENTIFIER_RE.exec(rest))) {
|
||||
push(KEYWORDS.has(match[0]) ? 'kwd' : 'pln', match[0], start);
|
||||
} else if ((match = PUNCTUATION_RE.exec(rest))) {
|
||||
push('pun', match[0], start);
|
||||
} else {
|
||||
const char = rest[0];
|
||||
push(FALLBACK_PUNCTUATION_RE.test(char) ? 'pun' : 'pln', char, start);
|
||||
}
|
||||
|
||||
index += match ? match[0].length : 1;
|
||||
}
|
||||
|
||||
return tokens;
|
||||
};
|
||||
Reference in New Issue
Block a user