feat(post): add client-side thumbnail fetching for websites with CORS access

src/components/post-desktop/post-desktop.tsx

@@ -1,10 +1,10 @@
-import { useState } from 'react';
+import { useEffect, useMemo, useState } from 'react';
 import { Trans, useTranslation } from 'react-i18next';
 import { Link, useLocation, useParams } from 'react-router-dom';
 import { Comment, useAuthorAvatar, useComment, useEditedComment, useSubplebbit } from '@plebbit/plebbit-react-hooks';
 import Plebbit from '@plebbit/plebbit-js/dist/browser/index.js';
 import styles from '../../views/post/post.module.css';
-import { getCommentMediaInfo, getDisplayMediaInfoType, getHasThumbnail, getMediaDimensions } from '../../lib/utils/media-utils';
+import { fetchWebpageThumbnailIfNeeded, getCommentMediaInfo, getDisplayMediaInfoType, getHasThumbnail, getMediaDimensions } from '../../lib/utils/media-utils';
 import { hashStringToColor, getTextColorForBackground } from '../../lib/utils/post-utils';
 import { getFormattedDate, getFormattedTimeAgo } from '../../lib/utils/time-utils';
 import { isValidURL } from '../../lib/utils/url-utils';
@@ -208,7 +208,17 @@ const PostInfo = ({ openReplyModal, post, postReplyCount = 0, roles, isHidden }:
 const PostMedia = ({ post }: PostProps) => {
   const { t } = useTranslation();
   const { link, spoiler } = post || {};
-  const commentMediaInfo = getCommentMediaInfo(post);
+
+  // some sites have CORS access, so the thumbnail can be fetched client-side, which is helpful if subplebbit.settings.fetchThumbnailUrls is false
+  const initialCommentMediaInfo = useMemo(() => getCommentMediaInfo(post), [post]);
+  const [commentMediaInfo, setCommentMediaInfo] = useState(initialCommentMediaInfo);
+  useEffect(() => {
+    if (initialCommentMediaInfo?.type === 'webpage' && !initialCommentMediaInfo.thumbnail) {
+      fetchWebpageThumbnailIfNeeded(initialCommentMediaInfo).then(setCommentMediaInfo);
+    } else {
+      setCommentMediaInfo(initialCommentMediaInfo);
+    }
+  }, [initialCommentMediaInfo]);
 
   const { url } = commentMediaInfo || {};
   let type = commentMediaInfo?.type;

src/components/post-mobile/post-mobile.tsx

@@ -1,10 +1,10 @@
-import { useState } from 'react';
+import { useEffect, useMemo, useState } from 'react';
 import { Trans, useTranslation } from 'react-i18next';
 import { Link, useLocation, useParams } from 'react-router-dom';
 import { Comment, useAuthorAvatar, useComment, useEditedComment, useSubplebbit } from '@plebbit/plebbit-react-hooks';
 import Plebbit from '@plebbit/plebbit-js/dist/browser/index.js';
 import styles from '../../views/post/post.module.css';
-import { getCommentMediaInfo, getHasThumbnail } from '../../lib/utils/media-utils';
+import { fetchWebpageThumbnailIfNeeded, getCommentMediaInfo, getHasThumbnail } from '../../lib/utils/media-utils';
 import { getTextColorForBackground, hashStringToColor } from '../../lib/utils/post-utils';
 import { getFormattedDate, getFormattedTimeAgo } from '../../lib/utils/time-utils';
 import { isAllView, isPendingPostView, isPostPageView, isSubscriptionsView } from '../../lib/utils/view-utils';
@@ -40,7 +40,17 @@ const PostInfoAndMedia = ({ openReplyModal, post, postReplyCount = 0, roles }: P
   const isInPostPageView = isPostPageView(location.pathname, params);
   const isInSubscriptionsView = isSubscriptionsView(location.pathname, params);
 
-  const commentMediaInfo = getCommentMediaInfo(post);
+  // some sites have CORS access, so the thumbnail can be fetched client-side, which is helpful if subplebbit.settings.fetchThumbnailUrls is false
+  const initialCommentMediaInfo = useMemo(() => getCommentMediaInfo(post), [post]);
+  const [commentMediaInfo, setCommentMediaInfo] = useState(initialCommentMediaInfo);
+  useEffect(() => {
+    if (initialCommentMediaInfo?.type === 'webpage' && !initialCommentMediaInfo.thumbnail) {
+      fetchWebpageThumbnailIfNeeded(initialCommentMediaInfo).then(setCommentMediaInfo);
+    } else {
+      setCommentMediaInfo(initialCommentMediaInfo);
+    }
+  }, [initialCommentMediaInfo]);
+
   const hasThumbnail = getHasThumbnail(commentMediaInfo, link);
   const [showThumbnail, setShowThumbnail] = useState(true);
 

src/lib/utils/media-utils.ts

@@ -105,6 +105,41 @@ export const getLinkMediaInfo = memoize(
   { max: 1000 },
 );
 
+// some sites have CORS access, so the thumbnail can be fetched client-side, which is helpful if subplebbit.settings.fetchThumbnailUrls is false
+const fetchWebpageThumbnail = async (url: string): Promise<string | undefined> => {
+  try {
+    const response = await fetch(url);
+    const html = await response.text();
+    const parser = new DOMParser();
+    const doc = parser.parseFromString(html, 'text/html');
+
+    // Try to find Open Graph image
+    const ogImage = doc.querySelector('meta[property="og:image"]');
+    if (ogImage && ogImage.getAttribute('content')) {
+      return ogImage.getAttribute('content')!;
+    }
+
+    // If no Open Graph image, try to find the first image
+    const firstImage = doc.querySelector('img');
+    if (firstImage && firstImage.getAttribute('src')) {
+      return new URL(firstImage.getAttribute('src')!, url).href;
+    }
+
+    return undefined;
+  } catch (error) {
+    console.error('Error fetching webpage thumbnail:', error);
+    return undefined;
+  }
+};
+
+export const fetchWebpageThumbnailIfNeeded = async (commentMediaInfo: CommentMediaInfo): Promise<CommentMediaInfo> => {
+  if (commentMediaInfo.type === 'webpage' && !commentMediaInfo.thumbnail) {
+    const thumbnail = await fetchWebpageThumbnail(commentMediaInfo.url);
+    return { ...commentMediaInfo, thumbnail };
+  }
+  return commentMediaInfo;
+};
+
 export const getCommentMediaInfo = (comment: Comment): CommentMediaInfo | undefined => {
   if (!comment?.thumbnailUrl && !comment?.link) {
     return;