improving bot detection prevention

* upgrading dependencies, fixing image placeholder

* improving processing times label and hide when screen width is too low

* aligning run now button

* renaming settings -> general settings

* smaller security and memory improvements

* improving footer

lib/provider/regionalimmobilien24.js

@@ -8,7 +8,7 @@ function normalize(o) {
   const title = o.title || 'No title available';
   const link = o.link != null ? decodeURIComponent(o.link) : config.url;
 
-  var urlReg = new RegExp(/url\((.*?)\)/gim);
+  const urlReg = new RegExp(/url\((.*?)\)/gim);
   const image = o.image != null ? urlReg.exec(o.image)[1] : null;
   return Object.assign(o, { id, address, title, link, image });
 }

lib/services/extractor/puppeteerExtractor.js

@@ -5,6 +5,7 @@ import logger from '../logger.js';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
+import { URL } from 'url';
 
 puppeteer.use(StealthPlugin());
 
@@ -27,23 +28,97 @@ export default async function execute(url, waitForSelector, options) {
       removeUserDataDir = true;
     }
 
+    const launchArgs = [
+      '--no-sandbox',
+      '--disable-gpu',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage',
+      '--disable-crash-reporter',
+      '--no-first-run',
+      '--no-default-browser-check',
+    ];
+    if (options?.proxyUrl) {
+      launchArgs.push(`--proxy-server=${options.proxyUrl}`);
+    }
+
     browser = await puppeteer.launch({
-      headless: options.puppeteerHeadless ?? true,
-      args: [
-        '--no-sandbox',
-        '--disable-gpu',
-        '--disable-setuid-sandbox',
-        '--disable-dev-shm-usage',
-        '--disable-crash-reporter',
-      ],
-      timeout: options.puppeteerTimeout || 30_000,
+      headless: options?.puppeteerHeadless ?? true,
+      args: launchArgs,
+      timeout: options?.puppeteerTimeout || 30_000,
       userDataDir,
+      executablePath: options?.executablePath, // allow using system Chrome
     });
+
     page = await browser.newPage();
-    await page.setExtraHTTPHeaders(DEFAULT_HEADER);
-    const response = await page.goto(url, {
-      waitUntil: 'domcontentloaded',
+
+    // Derive domain-specific defaults
+    const { hostname } = new URL(url);
+
+    // Set a realistic modern user agent unless provided
+    const userAgent =
+      options?.userAgent ||
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+    await page.setUserAgent(userAgent);
+
+    // Viewport and device scale for typical desktop
+    await page.setViewport({ width: 1366, height: 768, deviceScaleFactor: 1 });
+
+    // Extra HTTP headers with localized Accept-Language
+    const acceptLanguage = options?.acceptLanguage || 'de-DE,de;q=0.9,en-US;q=0.7,en;q=0.5';
+    const headers = {
+      ...DEFAULT_HEADER,
+      'Accept-Language': acceptLanguage,
+      'User-Agent': userAgent,
+      Referer: options?.referer || `https://${hostname}/`,
+      Connection: 'keep-alive',
+      DNT: '1',
+    };
+    await page.setExtraHTTPHeaders(headers);
+
+    // Timezone and locale tweaks to look German when needed
+    try {
+      const tz = options?.timezone || 'Europe/Berlin';
+      if (tz) await page.emulateTimezone(tz);
+    } catch {
+      //noop
+    }
+
+    // Harden navigator properties (stealth already covers many, but we ensure critical ones)
+    await page.evaluateOnNewDocument(() => {
+      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+      // Plugins and mimeTypes
+      // @ts-ignore
+      Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
+      // @ts-ignore
+      Object.defineProperty(navigator, 'languages', {
+        get: () => (window.localStorage.getItem('__LANGS__') || 'de-DE,de').split(','),
+      });
     });
+    // Provide languages value before navigation
+    await page.evaluateOnNewDocument((langs) => {
+      try {
+        window.localStorage.setItem('__LANGS__', langs);
+      } catch {
+        //noop
+      }
+    }, acceptLanguage.split(';')[0]);
+
+    // Optional cookies
+    if (Array.isArray(options?.cookies) && options.cookies.length > 0) {
+      await page.setCookie(...options.cookies);
+    }
+
+    // Navigation
+    const response = await page.goto(url, {
+      waitUntil: options?.waitUntil || 'domcontentloaded',
+    });
+
+    // Optionally wait a random small delay to mimic human rendering time
+    if (options?.humanDelay !== false) {
+      const delay = 200 + Math.floor(Math.random() * 400);
+      await new Promise((res) => setTimeout(res, delay));
+    }
+
     let pageSource;
     // if we're extracting data from a SPA, we must wait for the selector
     if (waitForSelector != null) {
@@ -57,7 +132,7 @@ export default async function execute(url, waitForSelector, options) {
       pageSource = await page.content();
     }
 
-    const statusCode = response.status();
+    const statusCode = response?.status?.() ?? 200;
 
     if (botDetected(pageSource, statusCode)) {
       logger.warn('We have been detected as a bot :-/ Tried url: => ', url);

lib/services/storage/listingsStorage.js

@@ -152,8 +152,9 @@ export const storeListings = (jobId, providerId, listings) => {
    */
   function extractNumber(str) {
     if (!str) return null;
-    const match = str.replace(/[.,]/g, '').match(/\d+/);
-    return match ? +match[0] : null;
+    const cleaned = str.replace(/\./g, '').replace(',', '.');
+    const num = parseFloat(cleaned);
+    return isNaN(num) ? null : num;
   }
 
   /**

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "14.3.1",
+  "version": "14.3.4",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -56,13 +56,13 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-icons": "^2.86.0",
-    "@douyinfe/semi-ui": "2.86.0",
+    "@douyinfe/semi-icons": "^2.88.0",
+    "@douyinfe/semi-ui": "2.88.0",
     "@sendgrid/mail": "8.1.6",
-    "@visactor/react-vchart": "^2.0.5",
-    "@visactor/vchart": "^2.0.5",
+    "@visactor/react-vchart": "^2.0.8",
+    "@visactor/vchart": "^2.0.8",
     "@visactor/vchart-semi-theme": "^1.12.2",
-    "@vitejs/plugin-react": "5.0.4",
+    "@vitejs/plugin-react": "5.1.1",
     "better-sqlite3": "^12.4.1",
     "body-parser": "2.2.0",
     "cheerio": "^1.1.2",
@@ -72,41 +72,41 @@
     "nanoid": "5.1.6",
     "node-cron": "^4.2.1",
     "node-fetch": "3.3.2",
-    "node-mailjet": "6.0.9",
-    "p-throttle": "^8.0.0",
+    "node-mailjet": "6.0.11",
+    "p-throttle": "^8.1.0",
     "package-up": "^5.0.0",
-    "puppeteer": "^24.24.0",
+    "puppeteer": "^24.30.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "query-string": "9.3.1",
     "react": "18.3.1",
     "react-dom": "18.3.1",
-    "react-router": "7.9.4",
-    "react-router-dom": "7.9.4",
+    "react-router": "7.9.6",
+    "react-router-dom": "7.9.6",
     "restana": "5.1.0",
     "semver": "^7.7.3",
     "serve-static": "2.2.0",
     "slack": "11.0.2",
-    "vite": "7.1.9",
+    "vite": "7.2.2",
     "x-var": "^3.0.1",
     "zustand": "^5.0.8"
   },
   "devDependencies": {
-    "@babel/core": "7.28.4",
-    "@babel/eslint-parser": "7.28.4",
-    "@babel/preset-env": "7.28.3",
-    "@babel/preset-react": "7.27.1",
-    "chai": "6.2.0",
-    "eslint": "9.37.0",
+    "@babel/core": "7.28.5",
+    "@babel/eslint-parser": "7.28.5",
+    "@babel/preset-env": "7.28.5",
+    "@babel/preset-react": "7.28.5",
+    "chai": "6.2.1",
+    "eslint": "9.39.1",
     "eslint-config-prettier": "10.1.8",
     "eslint-plugin-react": "7.37.5",
     "esmock": "2.7.3",
     "history": "5.3.0",
     "husky": "9.1.7",
     "less": "4.4.2",
-    "lint-staged": "16.2.4",
-    "mocha": "11.7.4",
-    "nodemon": "^3.1.10",
+    "lint-staged": "16.2.6",
+    "mocha": "11.7.5",
+    "nodemon": "^3.1.11",
     "prettier": "3.6.2"
   }
 }

ui/src/components/footer/FredyFooter.less

@@ -5,6 +5,8 @@
   justify-content: space-between;
   align-items: center;
   height: 1.7rem;
+  border-radius: .3rem;
+  border-top: 1px solid #45464b;
 
   &__version {
     padding-left: .5rem;

ui/src/components/navigation/Navigation.jsx

@@ -22,7 +22,7 @@ export default function Navigation({ isAdmin }) {
 
   if (isAdmin) {
     items.push({ itemKey: '/users', text: 'User Management', icon: <IconUser /> });
-    items.push({ itemKey: '/generalSettings', text: 'Settings', icon: <IconSetting /> });
+    items.push({ itemKey: '/generalSettings', text: 'General Settings', icon: <IconSetting /> });
   }
 
   function parsePathName(name) {

ui/src/components/table/listings/ListingsTable.jsx

@@ -14,8 +14,8 @@ import ListingsFilter from './ListingsFilter.jsx';
 
 const columns = [
   {
-    title: '#',
-    width: 100,
+    title: 'Watchlist',
+    width: 110,
     dataIndex: 'isWatched',
     sorter: true,
     render: (id, row) => {
@@ -180,6 +180,7 @@ export default function ListingsTable() {
   const [activityFilter, setActivityFilter] = useState(null);
   const [providerFilter, setProviderFilter] = useState(null);
 
+  const [imageWidth, setImageWidth] = useState('100%');
   const handlePageChange = (_page) => {
     setPage(_page);
   };
@@ -208,14 +209,29 @@ export default function ListingsTable() {
 
   const handleFilterChange = useMemo(() => debounce((value) => setFreeTextFilter(value), 500), []);
 
+  useEffect(() => {
+    return () => {
+      // cleanup debounced handler to avoid memory leaks
+      handleFilterChange.cancel && handleFilterChange.cancel();
+    };
+  }, [handleFilterChange]);
+
   const expandRowRender = (record) => {
     return (
       <div className="listingsTable__expanded">
         <div>
           {record.image_url == null ? (
-            <Image height={200} src={no_image} />
+            <Image height={200} width={180} src={no_image} />
           ) : (
-            <Image height={200} src={record.image_url} />
+            <Image
+              height={200}
+              width={imageWidth}
+              src={record.image_url}
+              onError={() => {
+                setImageWidth('180px');
+              }}
+              fallback={<Image height={200} src={no_image} />}
+            />
           )}
         </div>
         <div>
@@ -226,7 +242,7 @@ export default function ListingsTable() {
               </Tag>
             </Descriptions.Item>
             <Descriptions.Item itemKey="Link">
-              <a href={record.link} target="_blank" rel="noreferrer">
+              <a href={record.link} target="_blank" rel="noopener noreferrer">
                 Link to Listing
               </a>
             </Descriptions.Item>

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,16 +1,32 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
 import { Button, Card, Col, Row, Toast } from '@douyinfe/semi-ui';
-import { IconPlayCircle } from '@douyinfe/semi-icons';
+import {
+  IconClock,
+  IconDoubleChevronLeft,
+  IconDoubleChevronRight,
+  IconPlayCircle,
+  IconSearch,
+} from '@douyinfe/semi-icons';
 import { xhrPost } from '../../services/xhr.js';
 
 import './ProsessingTimes.less';
+import { useScreenWidth } from '../../hooks/screenWidth.js';
 
-function InfoCard({ title, value }) {
+function InfoCard({ title, value, icon }) {
+  const { Meta } = Card;
   return (
-    <Card style={{ maxWidth: '13rem', margin: '1rem', background: 'rgb(53, 54, 60)' }} title={title}>
-      {value}
-    </Card>
+    <div
+      style={{
+        margin: '1rem',
+        background: 'rgb(53, 54, 60)',
+        borderRadius: '.3rem',
+        padding: '1rem',
+        minHeight: '3rem',
+      }}
+    >
+      <Meta title={title} description={value} avatar={icon} />
+    </div>
   );
 }
 
@@ -18,32 +34,57 @@ export default function ProcessingTimes({ processingTimes = {} }) {
   if (Object.keys(processingTimes).length === 0) {
     return null;
   }
+  const width = useScreenWidth();
+  const invisible = width <= 1180;
+
+  if (invisible) {
+    return null;
+  }
+
   return (
     <Row>
       <Col span={6}>
-        <InfoCard title="Processing Interval" value={`${processingTimes.interval} min`} />
+        <InfoCard
+          title="Search Interval"
+          value={`${processingTimes.interval} min`}
+          icon={<IconClock style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
+        />
       </Col>
       {processingTimes.lastRun && (
         <>
           <Col span={6}>
-            <InfoCard title="Last run" value={format(processingTimes.lastRun)} />
+            <InfoCard
+              title="Last search"
+              icon={<IconDoubleChevronLeft style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
+              value={format(processingTimes.lastRun)}
+            />
           </Col>
           <Col span={6}>
-            <InfoCard title="Next run" value={format(processingTimes.lastRun + processingTimes.interval * 60000)} />
+            <InfoCard
+              title="Next search"
+              icon={<IconDoubleChevronRight style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
+              value={format(processingTimes.lastRun + processingTimes.interval * 60000)}
+            />
           </Col>
         </>
       )}
       <Col span={6}>
         <InfoCard
-          title="Find Listings Now"
+          title="Search Now"
+          icon={<IconSearch style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
           value={
             <Button
               size="small"
+              style={{ marginTop: '.2rem' }}
               icon={<IconPlayCircle />}
               aria-label="Start now"
               onClick={async () => {
-                await xhrPost('/api/jobs/startAll', null);
-                Toast.success('Successfully triggered Fredy search.');
+                try {
+                  await xhrPost('/api/jobs/startAll', null);
+                  Toast.success('Successfully triggered Fredy search.');
+                } catch {
+                  Toast.error('Failed to trigger search');
+                }
               }}
             >
               Search now