mirror of
https://github.com/bitsocialnet/5chan.git
synced 2026-08-03 07:41:04 +02:00
feat(directories): add schema adapters and preserve v2 metadata
Multi-adapter boundary in normalizeDirectoriesData() supports both communities[] and directories[] upstream schemas. Sync script mirrors logic and writes canonical format. Preserves directoryCode and features for future use.
This commit is contained in:
+157
-21
@@ -12,36 +12,172 @@ const __dirname = dirname(__filename);
|
|||||||
const GITHUB_URL = 'https://raw.githubusercontent.com/bitsocialhq/lists/master/5chan-directories.json';
|
const GITHUB_URL = 'https://raw.githubusercontent.com/bitsocialhq/lists/master/5chan-directories.json';
|
||||||
const OUTPUT_PATH = join(__dirname, '..', 'src', 'data', '5chan-directories.json');
|
const OUTPUT_PATH = join(__dirname, '..', 'src', 'data', '5chan-directories.json');
|
||||||
const TIMEOUT_MS = 5000;
|
const TIMEOUT_MS = 5000;
|
||||||
|
const DEFAULT_METADATA = {
|
||||||
|
title: '5chan directories',
|
||||||
|
description: '',
|
||||||
|
createdAt: 0,
|
||||||
|
updatedAt: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const isRecord = (value) => typeof value === 'object' && value !== null;
|
||||||
|
|
||||||
|
const normalizeFeatures = (value) => {
|
||||||
|
if (!isRecord(value)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedFeatures = Object.entries(value).reduce((acc, [key, featureValue]) => {
|
||||||
|
if (typeof featureValue === 'string' || typeof featureValue === 'boolean' || typeof featureValue === 'number') {
|
||||||
|
acc[key] = featureValue;
|
||||||
|
}
|
||||||
|
return acc;
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
return Object.keys(normalizedFeatures).length > 0 ? normalizedFeatures : undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
const toCanonicalCommunity = ({ address, title, nsfw, directoryCode, features }) => {
|
||||||
|
if (typeof address !== 'string') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedFeatures = normalizeFeatures(features);
|
||||||
|
const topLevelNsfw = typeof nsfw === 'boolean' ? nsfw : undefined;
|
||||||
|
const featuresNsfw = typeof normalizedFeatures?.nsfw === 'boolean' ? normalizedFeatures.nsfw : undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
address,
|
||||||
|
...(typeof title === 'string' ? { title } : {}),
|
||||||
|
...(typeof directoryCode === 'string' ? { directoryCode } : {}),
|
||||||
|
...(normalizedFeatures ? { features: normalizedFeatures } : {}),
|
||||||
|
...((topLevelNsfw ?? featuresNsfw) !== undefined ? { nsfw: topLevelNsfw ?? featuresNsfw } : {}),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const dedupeCommunities = (entries) => {
|
||||||
|
const seenAddresses = new Set();
|
||||||
|
const normalized = [];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (seenAddresses.has(entry.address)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seenAddresses.add(entry.address);
|
||||||
|
normalized.push(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized;
|
||||||
|
};
|
||||||
|
|
||||||
|
const adaptV2Directories = (value) => {
|
||||||
|
if (!Array.isArray(value.directories)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const communities = value.directories
|
||||||
|
.map((directory) => {
|
||||||
|
if (!isRecord(directory)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const features = isRecord(directory.features) ? directory.features : null;
|
||||||
|
return toCanonicalCommunity({
|
||||||
|
address: directory.communityAddress,
|
||||||
|
title: directory.title,
|
||||||
|
nsfw: features?.nsfw,
|
||||||
|
directoryCode: directory.directoryCode,
|
||||||
|
features,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
return dedupeCommunities(communities);
|
||||||
|
};
|
||||||
|
|
||||||
|
const adaptV1Communities = (value) => {
|
||||||
|
if (!Array.isArray(value.communities)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const communities = value.communities
|
||||||
|
.map((community) => {
|
||||||
|
if (!isRecord(community)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return toCanonicalCommunity({
|
||||||
|
address: community.address,
|
||||||
|
title: community.title,
|
||||||
|
nsfw: community.nsfw,
|
||||||
|
directoryCode: community.directoryCode,
|
||||||
|
features: community.features,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
return dedupeCommunities(communities);
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalizeDirectoriesData = (value, fallbackMetadata = DEFAULT_METADATA) => {
|
||||||
|
if (!isRecord(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const adapters = [adaptV2Directories, adaptV1Communities];
|
||||||
|
const communities = adapters.map((adapter) => adapter(value)).find((normalized) => normalized.length > 0) || [];
|
||||||
|
if (communities.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: typeof value.title === 'string' ? value.title : fallbackMetadata.title,
|
||||||
|
description: typeof value.description === 'string' ? value.description : fallbackMetadata.description,
|
||||||
|
createdAt: typeof value.createdAt === 'number' ? value.createdAt : fallbackMetadata.createdAt,
|
||||||
|
updatedAt: typeof value.updatedAt === 'number' ? value.updatedAt : fallbackMetadata.updatedAt,
|
||||||
|
communities,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const getErrorMessage = (error) => (error instanceof Error ? error.message : String(error));
|
||||||
|
|
||||||
const sync = async () => {
|
const sync = async () => {
|
||||||
try {
|
try {
|
||||||
const controller = new AbortController();
|
let existing = '';
|
||||||
const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
let fallbackMetadata = DEFAULT_METADATA;
|
||||||
|
try {
|
||||||
const response = await fetch(GITHUB_URL, { signal: controller.signal });
|
existing = readFileSync(OUTPUT_PATH, 'utf8');
|
||||||
clearTimeout(timeout);
|
const parsedExisting = JSON.parse(existing);
|
||||||
|
const normalizedExisting = normalizeDirectoriesData(parsedExisting);
|
||||||
if (!response.ok) {
|
if (normalizedExisting) {
|
||||||
throw new Error(`HTTP ${response.status}`);
|
fallbackMetadata = {
|
||||||
|
title: normalizedExisting.title,
|
||||||
|
description: normalizedExisting.description,
|
||||||
|
createdAt: normalizedExisting.createdAt,
|
||||||
|
updatedAt: normalizedExisting.updatedAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// file doesn't exist yet or is invalid JSON
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = await response.json();
|
const controller = new AbortController();
|
||||||
|
const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
||||||
|
let response;
|
||||||
|
try {
|
||||||
|
response = await fetch(GITHUB_URL, { signal: controller.signal });
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
}
|
||||||
|
|
||||||
// Basic sanity check — must have a communities array
|
if (!response || !response.ok) {
|
||||||
if (!Array.isArray(data?.communities) || data.communities.length === 0) {
|
throw new Error(`HTTP ${response?.status ?? 'unknown'}`);
|
||||||
throw new Error('Invalid data: missing or empty communities array');
|
}
|
||||||
|
|
||||||
|
const data = normalizeDirectoriesData(await response.json(), fallbackMetadata);
|
||||||
|
if (!data) {
|
||||||
|
throw new Error('Invalid directories payload');
|
||||||
}
|
}
|
||||||
|
|
||||||
const formatted = JSON.stringify(data, null, 2) + '\n';
|
const formatted = JSON.stringify(data, null, 2) + '\n';
|
||||||
|
|
||||||
// Only write if content actually changed
|
|
||||||
let existing = '';
|
|
||||||
try {
|
|
||||||
existing = readFileSync(OUTPUT_PATH, 'utf8');
|
|
||||||
} catch {
|
|
||||||
// file doesn't exist yet
|
|
||||||
}
|
|
||||||
|
|
||||||
if (formatted === existing) {
|
if (formatted === existing) {
|
||||||
console.log('✅ Vendored directories already up to date');
|
console.log('✅ Vendored directories already up to date');
|
||||||
return;
|
return;
|
||||||
@@ -50,7 +186,7 @@ const sync = async () => {
|
|||||||
writeFileSync(OUTPUT_PATH, formatted, 'utf8');
|
writeFileSync(OUTPUT_PATH, formatted, 'utf8');
|
||||||
console.log(`✅ Synced vendored directories (${data.communities.length} communities)`);
|
console.log(`✅ Synced vendored directories (${data.communities.length} communities)`);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.warn(`⚠️ Could not sync directories from GitHub (keeping existing file): ${e.message}`);
|
console.warn(`⚠️ Could not sync directories from GitHub (keeping existing file): ${getErrorMessage(e)}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
+172
-20
@@ -2,101 +2,253 @@
|
|||||||
"title": "/all/ - All 5chan Directories",
|
"title": "/all/ - All 5chan Directories",
|
||||||
"description": "Each 5chan directory is assigned to a board handpicked by the team, until DAO curation is implemented.\n\nhttps://github.com/bitsocialhq/lists/blob/master/5chan-directories.json",
|
"description": "Each 5chan directory is assigned to a board handpicked by the team, until DAO curation is implemented.\n\nhttps://github.com/bitsocialhq/lists/blob/master/5chan-directories.json",
|
||||||
"createdAt": 1762179811,
|
"createdAt": 1762179811,
|
||||||
"updatedAt": 1762179811,
|
"updatedAt": 1771140295,
|
||||||
"communities": [
|
"communities": [
|
||||||
{
|
{
|
||||||
"title": "/pol/ - Politically Incorrect",
|
|
||||||
"address": "politically-incorrect.eth",
|
"address": "politically-incorrect.eth",
|
||||||
|
"title": "/pol/ - Politically Incorrect",
|
||||||
|
"directoryCode": "pol",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": true,
|
||||||
|
"hasFlags": true,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": true
|
"nsfw": true
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/biz/ - Business & Finance",
|
|
||||||
"address": "business-and-finance.eth",
|
"address": "business-and-finance.eth",
|
||||||
|
"title": "/biz/ - Business & Finance",
|
||||||
|
"directoryCode": "biz",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-post",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/sci/ - Science & Math",
|
|
||||||
"address": "science-and-math.eth",
|
"address": "science-and-math.eth",
|
||||||
|
"title": "/sci/ - Science & Math",
|
||||||
|
"directoryCode": "sci",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/g/ - Technology",
|
|
||||||
"address": "technology-posting.eth",
|
"address": "technology-posting.eth",
|
||||||
|
"title": "/g/ - Technology",
|
||||||
|
"directoryCode": "g",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/vg/ - Video Game Generals",
|
|
||||||
"address": "videogame-generals.eth",
|
"address": "videogame-generals.eth",
|
||||||
|
"title": "/vg/ - Video Game Generals",
|
||||||
|
"directoryCode": "vg",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/fit/ - Fitness",
|
|
||||||
"address": "fitness-posting.eth",
|
"address": "fitness-posting.eth",
|
||||||
|
"title": "/fit/ - Fitness",
|
||||||
|
"directoryCode": "fit",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/adv/ - Advice",
|
|
||||||
"address": "advice-posting.eth",
|
"address": "advice-posting.eth",
|
||||||
|
"title": "/adv/ - Advice",
|
||||||
|
"directoryCode": "adv",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/wsg/ - Worksafe GIF",
|
|
||||||
"address": "worksafe-gif.eth",
|
"address": "worksafe-gif.eth",
|
||||||
|
"title": "/wsg/ - Worksafe GIF",
|
||||||
|
"directoryCode": "wsg",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/diy/ - Do It Yourself",
|
|
||||||
"address": "do-it-yourself.eth",
|
"address": "do-it-yourself.eth",
|
||||||
|
"title": "/diy/ - Do It Yourself",
|
||||||
|
"directoryCode": "diy",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/out/ - Outdoors",
|
|
||||||
"address": "outdoors-posting.eth",
|
"address": "outdoors-posting.eth",
|
||||||
|
"title": "/out/ - Outdoors",
|
||||||
|
"directoryCode": "out",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/ic/ - Artwork/Critique",
|
|
||||||
"address": "artwork-critique.eth",
|
"address": "artwork-critique.eth",
|
||||||
|
"title": "/ic/ - Artwork/Critique",
|
||||||
|
"directoryCode": "ic",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/mu/ - Music",
|
|
||||||
"address": "music-posting.eth",
|
"address": "music-posting.eth",
|
||||||
|
"title": "/mu/ - Music",
|
||||||
|
"directoryCode": "mu",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/int/ - International",
|
|
||||||
"address": "international-sfw.eth",
|
"address": "international-sfw.eth",
|
||||||
|
"title": "/int/ - International",
|
||||||
|
"directoryCode": "int",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": true,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/lit/ - Literature",
|
|
||||||
"address": "literature-posting.eth",
|
"address": "literature-posting.eth",
|
||||||
|
"title": "/lit/ - Literature",
|
||||||
|
"directoryCode": "lit",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/tv/ - Television & Film",
|
|
||||||
"address": "television-and-film.eth",
|
"address": "television-and-film.eth",
|
||||||
|
"title": "/tv/ - Television & Film",
|
||||||
|
"directoryCode": "tv",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/vip/ - Very Important Posts",
|
|
||||||
"address": "very-important-posts.eth",
|
"address": "very-important-posts.eth",
|
||||||
|
"title": "/vip/ - Very Important Posts",
|
||||||
|
"directoryCode": "vip",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": false,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": false
|
"nsfw": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/gif/ - Adult GIF",
|
|
||||||
"address": "adult-gif.eth",
|
"address": "adult-gif.eth",
|
||||||
|
"title": "/gif/ - Adult GIF",
|
||||||
|
"directoryCode": "gif",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": true,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": true
|
"nsfw": true
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/bant/ - International/Random",
|
|
||||||
"address": "international-nsfw.eth",
|
"address": "international-nsfw.eth",
|
||||||
|
"title": "/bant/ - International/Random",
|
||||||
|
"directoryCode": "bant",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-post",
|
||||||
|
"nsfw": true,
|
||||||
|
"hasFlags": true,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": true
|
"nsfw": true
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "/b/ - Random",
|
|
||||||
"address": "random-nsfw.eth",
|
"address": "random-nsfw.eth",
|
||||||
|
"title": "/b/ - Random",
|
||||||
|
"directoryCode": "b",
|
||||||
|
"features": {
|
||||||
|
"pseudonymityMode": "per-reply",
|
||||||
|
"nsfw": true,
|
||||||
|
"hasFlags": false,
|
||||||
|
"requirePostLink": true,
|
||||||
|
"requirePostLinkIsMedia": true
|
||||||
|
},
|
||||||
"nsfw": true
|
"nsfw": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -8,10 +8,21 @@ export interface DirectoriesMetadata {
|
|||||||
updatedAt: number;
|
updatedAt: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DirectoryFeatures {
|
||||||
|
pseudonymityMode?: string;
|
||||||
|
nsfw?: boolean;
|
||||||
|
hasFlags?: boolean;
|
||||||
|
requirePostLink?: boolean;
|
||||||
|
requirePostLinkIsMedia?: boolean;
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DirectoryCommunity {
|
export interface DirectoryCommunity {
|
||||||
title?: string;
|
title?: string;
|
||||||
address: string;
|
address: string;
|
||||||
nsfw?: boolean;
|
nsfw?: boolean;
|
||||||
|
directoryCode?: string;
|
||||||
|
features?: DirectoryFeatures;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DirectoriesData {
|
export interface DirectoriesData {
|
||||||
@@ -32,11 +43,130 @@ const GITHUB_URL = 'https://raw.githubusercontent.com/bitsocialhq/lists/master/5
|
|||||||
const LOCALSTORAGE_KEY = '5chan-directories-cache';
|
const LOCALSTORAGE_KEY = '5chan-directories-cache';
|
||||||
const LOCALSTORAGE_TIMESTAMP_KEY = '5chan-directories-cache-timestamp';
|
const LOCALSTORAGE_TIMESTAMP_KEY = '5chan-directories-cache-timestamp';
|
||||||
const CACHE_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
const CACHE_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
||||||
|
const FALLBACK_DIRECTORIES_DATA = directoriesData as DirectoriesData;
|
||||||
|
|
||||||
let cacheCommunities: DirectoryCommunity[] | null = null;
|
let cacheCommunities: DirectoryCommunity[] | null = null;
|
||||||
let cacheMetadata: DirectoriesMetadata | null = null;
|
let cacheMetadata: DirectoriesMetadata | null = null;
|
||||||
let inFlightGitHubFetch: Promise<DirectoriesData> | null = null;
|
let inFlightGitHubFetch: Promise<DirectoriesData> | null = null;
|
||||||
|
|
||||||
|
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null;
|
||||||
|
|
||||||
|
const normalizeFeatures = (value: unknown): DirectoryFeatures | undefined => {
|
||||||
|
if (!isRecord(value)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedFeatures = Object.entries(value).reduce<DirectoryFeatures>((acc, [key, featureValue]) => {
|
||||||
|
if (typeof featureValue === 'string' || typeof featureValue === 'boolean' || typeof featureValue === 'number') {
|
||||||
|
acc[key] = featureValue;
|
||||||
|
}
|
||||||
|
return acc;
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
return Object.keys(normalizedFeatures).length > 0 ? normalizedFeatures : undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
const toCanonicalCommunity = (value: { address: unknown; title: unknown; nsfw: unknown; directoryCode?: unknown; features?: unknown }): DirectoryCommunity | null => {
|
||||||
|
if (typeof value.address !== 'string') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const features = normalizeFeatures(value.features);
|
||||||
|
const topLevelNsfw = typeof value.nsfw === 'boolean' ? value.nsfw : undefined;
|
||||||
|
const featuresNsfw = typeof features?.nsfw === 'boolean' ? features.nsfw : undefined;
|
||||||
|
|
||||||
|
return {
|
||||||
|
address: value.address,
|
||||||
|
...(typeof value.title === 'string' ? { title: value.title } : {}),
|
||||||
|
...(typeof value.directoryCode === 'string' ? { directoryCode: value.directoryCode } : {}),
|
||||||
|
...(features ? { features } : {}),
|
||||||
|
...((topLevelNsfw ?? featuresNsfw) !== undefined ? { nsfw: topLevelNsfw ?? featuresNsfw } : {}),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const dedupeCommunities = (entries: DirectoryCommunity[]): DirectoryCommunity[] => {
|
||||||
|
const seenAddresses = new Set<string>();
|
||||||
|
const normalizedEntries: DirectoryCommunity[] = [];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (seenAddresses.has(entry.address)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seenAddresses.add(entry.address);
|
||||||
|
normalizedEntries.push(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalizedEntries;
|
||||||
|
};
|
||||||
|
|
||||||
|
const adaptV2Directories = (value: Record<string, unknown>): DirectoryCommunity[] => {
|
||||||
|
if (!Array.isArray(value.directories)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const communities = value.directories
|
||||||
|
.map((directory) => {
|
||||||
|
if (!isRecord(directory)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const features = isRecord(directory.features) ? directory.features : null;
|
||||||
|
return toCanonicalCommunity({
|
||||||
|
address: directory.communityAddress,
|
||||||
|
title: directory.title,
|
||||||
|
nsfw: features?.nsfw,
|
||||||
|
directoryCode: directory.directoryCode,
|
||||||
|
features,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.filter((community): community is DirectoryCommunity => community !== null);
|
||||||
|
|
||||||
|
return dedupeCommunities(communities);
|
||||||
|
};
|
||||||
|
|
||||||
|
const adaptV1Communities = (value: Record<string, unknown>): DirectoryCommunity[] => {
|
||||||
|
if (!Array.isArray(value.communities)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const communities = value.communities
|
||||||
|
.map((community) => {
|
||||||
|
if (!isRecord(community)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return toCanonicalCommunity({
|
||||||
|
address: community.address,
|
||||||
|
title: community.title,
|
||||||
|
nsfw: community.nsfw,
|
||||||
|
directoryCode: community.directoryCode,
|
||||||
|
features: community.features,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.filter((community): community is DirectoryCommunity => community !== null);
|
||||||
|
|
||||||
|
return dedupeCommunities(communities);
|
||||||
|
};
|
||||||
|
|
||||||
|
const normalizeDirectoriesData = (value: unknown): DirectoriesData | null => {
|
||||||
|
if (!isRecord(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const adapters: Array<(raw: Record<string, unknown>) => DirectoryCommunity[]> = [adaptV2Directories, adaptV1Communities];
|
||||||
|
const communities = adapters.map((adapter) => adapter(value)).find((normalized) => normalized.length > 0) ?? [];
|
||||||
|
|
||||||
|
if (communities.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: typeof value.title === 'string' ? value.title : FALLBACK_DIRECTORIES_DATA.title,
|
||||||
|
description: typeof value.description === 'string' ? value.description : FALLBACK_DIRECTORIES_DATA.description,
|
||||||
|
createdAt: typeof value.createdAt === 'number' ? value.createdAt : FALLBACK_DIRECTORIES_DATA.createdAt,
|
||||||
|
updatedAt: typeof value.updatedAt === 'number' ? value.updatedAt : FALLBACK_DIRECTORIES_DATA.updatedAt,
|
||||||
|
communities,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
const getFromLocalStorage = (): DirectoriesData | null => {
|
const getFromLocalStorage = (): DirectoriesData | null => {
|
||||||
try {
|
try {
|
||||||
const cached = localStorage.getItem(LOCALSTORAGE_KEY);
|
const cached = localStorage.getItem(LOCALSTORAGE_KEY);
|
||||||
@@ -44,7 +174,14 @@ const getFromLocalStorage = (): DirectoriesData | null => {
|
|||||||
if (cached && timestamp) {
|
if (cached && timestamp) {
|
||||||
const age = Date.now() - parseInt(timestamp, 10);
|
const age = Date.now() - parseInt(timestamp, 10);
|
||||||
if (age < CACHE_MAX_AGE_MS) {
|
if (age < CACHE_MAX_AGE_MS) {
|
||||||
return JSON.parse(cached);
|
const parsed = JSON.parse(cached);
|
||||||
|
const normalized = normalizeDirectoriesData(parsed);
|
||||||
|
if (normalized) {
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
console.warn('Invalid directories cache format, clearing stale cache');
|
||||||
|
localStorage.removeItem(LOCALSTORAGE_KEY);
|
||||||
|
localStorage.removeItem(LOCALSTORAGE_TIMESTAMP_KEY);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -67,7 +204,10 @@ const fetchDirectoriesFromGitHub = async (): Promise<DirectoriesData> => {
|
|||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`HTTP error! status: ${response.status}`);
|
throw new Error(`HTTP error! status: ${response.status}`);
|
||||||
}
|
}
|
||||||
const data = await response.json();
|
const data = normalizeDirectoriesData(await response.json());
|
||||||
|
if (!data) {
|
||||||
|
throw new Error('Invalid directories payload');
|
||||||
|
}
|
||||||
// Save successful fetch to localStorage
|
// Save successful fetch to localStorage
|
||||||
saveToLocalStorage(data);
|
saveToLocalStorage(data);
|
||||||
return data;
|
return data;
|
||||||
@@ -86,7 +226,7 @@ export const useDirectories = () => {
|
|||||||
// Use vendored data as initial state to prevent theme flash on first load
|
// Use vendored data as initial state to prevent theme flash on first load
|
||||||
// This ensures NSFW status is known synchronously before first render
|
// This ensures NSFW status is known synchronously before first render
|
||||||
const [state, setState] = useState<DirectoriesState>({
|
const [state, setState] = useState<DirectoriesState>({
|
||||||
communities: (directoriesData as DirectoriesData).communities,
|
communities: FALLBACK_DIRECTORIES_DATA.communities,
|
||||||
loading: true,
|
loading: true,
|
||||||
error: null,
|
error: null,
|
||||||
});
|
});
|
||||||
@@ -127,7 +267,7 @@ export const useDirectories = () => {
|
|||||||
console.warn('Failed to fetch directories from GitHub:', e);
|
console.warn('Failed to fetch directories from GitHub:', e);
|
||||||
// Only fall back if we don't already have memory/localStorage data
|
// Only fall back if we don't already have memory/localStorage data
|
||||||
if (!cacheCommunities) {
|
if (!cacheCommunities) {
|
||||||
hydrateCommunities(directoriesData as DirectoriesData);
|
hydrateCommunities(FALLBACK_DIRECTORIES_DATA);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
@@ -140,13 +280,13 @@ export const useDirectories = () => {
|
|||||||
// Always prefer cacheCommunities (module-level, stable reference) when available
|
// Always prefer cacheCommunities (module-level, stable reference) when available
|
||||||
// Only use state.communities during initial load before cache is populated
|
// Only use state.communities during initial load before cache is populated
|
||||||
// This ensures a stable reference for memoization in consuming hooks
|
// This ensures a stable reference for memoization in consuming hooks
|
||||||
return cacheCommunities || state.communities;
|
return cacheCommunities || state.communities || FALLBACK_DIRECTORIES_DATA.communities;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const useDirectoriesState = () => {
|
export const useDirectoriesState = () => {
|
||||||
// Use vendored data as fallback to prevent theme flash on first load
|
// Use vendored data as fallback to prevent theme flash on first load
|
||||||
const [state, setState] = useState<DirectoriesState>({
|
const [state, setState] = useState<DirectoriesState>({
|
||||||
communities: cacheCommunities || (directoriesData as DirectoriesData).communities,
|
communities: cacheCommunities || FALLBACK_DIRECTORIES_DATA.communities,
|
||||||
loading: !cacheCommunities,
|
loading: !cacheCommunities,
|
||||||
error: null,
|
error: null,
|
||||||
});
|
});
|
||||||
@@ -187,7 +327,7 @@ export const useDirectoriesState = () => {
|
|||||||
console.warn('Failed to fetch directories from GitHub:', e);
|
console.warn('Failed to fetch directories from GitHub:', e);
|
||||||
// Only fall back if we don't already have memory/localStorage data
|
// Only fall back if we don't already have memory/localStorage data
|
||||||
if (!cacheCommunities) {
|
if (!cacheCommunities) {
|
||||||
hydrateCommunities(directoriesData as DirectoriesData);
|
hydrateCommunities(FALLBACK_DIRECTORIES_DATA);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
@@ -202,7 +342,7 @@ export const useDirectoriesState = () => {
|
|||||||
|
|
||||||
export const useDirectoryAddresses = () => {
|
export const useDirectoryAddresses = () => {
|
||||||
const directories = useDirectories();
|
const directories = useDirectories();
|
||||||
return useMemo(() => directories.map((community) => community.address), [directories]);
|
return useMemo(() => (Array.isArray(directories) ? directories.map((community) => community.address) : []), [directories]);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const useDirectoriesMetadata = () => {
|
export const useDirectoriesMetadata = () => {
|
||||||
@@ -242,7 +382,7 @@ export const useDirectoriesMetadata = () => {
|
|||||||
console.warn('Failed to fetch directory metadata from GitHub:', e);
|
console.warn('Failed to fetch directory metadata from GitHub:', e);
|
||||||
// Only fall back if we don't already have memory/localStorage data
|
// Only fall back if we don't already have memory/localStorage data
|
||||||
if (!cacheMetadata) {
|
if (!cacheMetadata) {
|
||||||
hydrateMetadata(directoriesData as DirectoriesData);
|
hydrateMetadata(FALLBACK_DIRECTORIES_DATA);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|||||||
Reference in New Issue
Block a user