upgrading dependencies | adding ohneMakler provider

* feat: updated the UI to enable editing of provider URLs


---------

Co-authored-by: foxx-tech <robin.foxx.tech@gmail.com>

.github/workflows/test.yml

@@ -19,4 +19,4 @@ jobs:
           cache: 'yarn'
 
       - run: yarn install
-      - run: yarn test
+      - run: yarn testGH

lib/notification/adapter/http.js

@@ -0,0 +1,57 @@
+import { markdown2Html } from '../../services/markdown.js';
+
+const mapListing = (listing) => ({
+  address: listing.address,
+  description: listing.description,
+  id: listing.id,
+  imageUrl: listing.image,
+  price: listing.price,
+  size: listing.size,
+  title: listing.title,
+  url: listing.link,
+});
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { authToken, endpointUrl } = notificationConfig.find((a) => a.id === config.id).fields;
+
+  const listings = newListings.map(mapListing);
+  const body = {
+    jobId: jobKey,
+    timestamp: new Date().toISOString(),
+    provider: serviceName,
+    listings,
+  };
+
+  const headers = {
+    'Content-Type': 'application/json',
+  };
+  if (authToken != null) {
+    headers['Authorization'] = `Bearer ${authToken}`;
+  }
+
+  return fetch(endpointUrl, {
+    method: 'POST',
+    headers: headers,
+    body: JSON.stringify(body),
+  });
+};
+
+export const config = {
+  id: 'http',
+  name: 'HTTP',
+  readme: markdown2Html('lib/notification/adapter/http.md'),
+  description: 'Fredy will send a generic HTTP POST request.',
+  fields: {
+    endpointUrl: {
+      description: "Your application's endpoint URL.",
+      label: 'Endpoint URL',
+      type: 'text',
+    },
+    authToken: {
+      description: "Your application's auth token, if required by your endpoint.",
+      label: 'Auth token (optional)',
+      optional: true,
+      type: 'text',
+    },
+  },
+};

lib/notification/adapter/http.md

@@ -0,0 +1,43 @@
+### HTTP Adapter
+
+This is a generic adapter for sending notifications via HTTP requests.
+You can leverage this adapter to integrate with various webhooks or APIs that accept HTTP requests. (e.g. Supabase
+Functions, a Node.js server, etc.)
+
+HTTP adapter supports a `authToken` field, which can be used to include an authorization token in the request headers.
+Your token would be included as a Bearer token in the `Authorization` header, which is a common method for securing API requests.
+
+Request Details:
+<details>
+Request Method: POST  
+
+Headers:
+
+```
+Content Type: `application/json`
+Authorization: Bearer {your-optional-auth-token}
+```
+
+Body:
+
+```json
+{
+  "jobId": "mg1waX4RHmIzL5NDYtYp-",
+  "provider": "immoscout",
+  "timestamp": "2024-06-15T12:34:56Z",
+  "listings": [
+    {
+      "address": "Str. 123, Bielefeld, Germany",
+      "description": "Möbliert: Einziehen & wohlfühlen: Neu möbliert.",
+      "id": "123456789",
+      "imageUrl": "https://<target-url>.com/listings/123456789.jpg",
+      "price": "1.240 €",
+      "size": "38 m²",
+      "title": "Schöne 1-Zimmer-Wohnung in Bielefeld",
+      "url": "https://<target-url>.com/listings/123456789"
+    }
+  ]
+}
+```
+
+</details>

lib/provider/ohneMakler.js

@@ -0,0 +1,45 @@
+import { isOneOf, buildHash } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
+let appliedBlackList = [];
+
+function normalize(o) {
+  const link = metaInformation.baseUrl + o.link;
+  const id = buildHash(o.title, o.link, o.price);
+  return Object.assign(o, { link, id });
+}
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+const config = {
+  url: null,
+  crawlContainer: 'div[data-livecomponent-id*="search/property_list"] .grid > div',
+  sortByDateParam: null,
+  waitForSelector: null,
+  crawlFields: {
+    id: 'a@href',
+    title: 'h4 | removeNewline | trim',
+    price: '.text-xl | trim',
+    size: 'div[title="Wohnfläche"] | trim',
+    address: '.text-slate-800 | removeNewline | trim',
+    image: 'img@src',
+    link: 'a@href',
+  },
+  normalize: normalize,
+  filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
+};
+
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+
+export const metaInformation = {
+  name: 'OhneMakler',
+  baseUrl: 'https://www.ohne-makler.net/immobilien',
+  id: 'ohneMakler',
+};
+export { config };

lib/provider/regionalimmobilien24.js

@@ -8,7 +8,7 @@ function normalize(o) {
   const title = o.title || 'No title available';
   const link = o.link != null ? decodeURIComponent(o.link) : config.url;
 
-  var urlReg = new RegExp(/url\((.*?)\)/gim);
+  const urlReg = new RegExp(/url\((.*?)\)/gim);
   const image = o.image != null ? urlReg.exec(o.image)[1] : null;
   return Object.assign(o, { id, address, title, link, image });
 }

lib/services/extractor/botPrevention.js

@@ -0,0 +1,274 @@
+import { DEFAULT_HEADER } from './utils.js';
+
+// Helper to safely coerce numbers
+const toInt = (v, d) => {
+  const n = parseInt(v, 10);
+  return Number.isFinite(n) ? n : d;
+};
+
+/**
+ * Compute pre-launch configuration and flags for Puppeteer with bot prevention in mind.
+ * Returns language, user agent, viewport (with optional jitter), and additional launch args.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ */
+export function getPreLaunchConfig(url, options = {}) {
+  const { hostname } = new URL(url);
+
+  const acceptLanguage = options.acceptLanguage || 'de-DE,de;q=0.9,en-US;q=0.7,en;q=0.5';
+  const langForFlag = acceptLanguage.split(',')[0];
+
+  const baseViewport = { width: 1366, height: 768, deviceScaleFactor: 1 };
+  const jitter = options.viewportJitter !== false ? Math.floor(Math.random() * 6) : 0; // 0..5 px
+  const width = toInt(options?.viewport?.width, baseViewport.width) + jitter;
+  const height = toInt(options?.viewport?.height, baseViewport.height) + jitter;
+  const deviceScaleFactor = toInt(options?.viewport?.deviceScaleFactor, baseViewport.deviceScaleFactor);
+  const viewport = { width, height, deviceScaleFactor };
+
+  const userAgent =
+    options.userAgent ||
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+
+  const windowSizeArg = `--window-size=${viewport.width},${viewport.height}`;
+  const langArg = `--lang=${langForFlag}`;
+
+  const extraArgs = [
+    '--disable-blink-features=AutomationControlled',
+    '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--webrtc-ip-handling-policy=default_public_interface_only',
+    '--proxy-bypass-list=<-loopback>',
+  ];
+
+  const headers = {
+    ...DEFAULT_HEADER,
+    'Accept-Language': acceptLanguage,
+    'User-Agent': userAgent,
+    Referer: options?.referer || `https://${hostname}/`,
+    Connection: 'keep-alive',
+    DNT: '1',
+  };
+
+  const timezone = options?.timezone || 'Europe/Berlin';
+
+  return {
+    acceptLanguage,
+    langForFlag,
+    userAgent,
+    viewport,
+    windowSizeArg,
+    langArg,
+    extraArgs,
+    headers,
+    timezone,
+    humanDelay: options?.humanDelay !== false,
+  };
+}
+
+/**
+ * Apply bot-prevention hardening to a Puppeteer page.
+ * Sets UA, viewport, JS enabled, headers, timezone and injects stealth-like patches.
+ *
+ * @param {import('puppeteer').Page} page
+ * @param {ReturnType<typeof getPreLaunchConfig>} cfg
+ */
+export async function applyBotPreventionToPage(page, cfg) {
+  await page.setUserAgent(cfg.userAgent);
+  await page.setViewport(cfg.viewport);
+  await page.setJavaScriptEnabled(true);
+  await page.setExtraHTTPHeaders(cfg.headers);
+  try {
+    if (cfg.timezone) await page.emulateTimezone(cfg.timezone);
+  } catch {
+    // ignore timezone failures
+  }
+
+  // Inject patches as early as possible
+  await page.evaluateOnNewDocument(() => {
+    try {
+      // webdriver
+      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+
+      // chrome runtime
+      // @ts-ignore
+      if (!window.chrome) {
+        // @ts-ignore
+        window.chrome = { runtime: {} };
+      }
+
+      // languages
+      // @ts-ignore
+      Object.defineProperty(navigator, 'languages', {
+        get: () => (window.localStorage.getItem('__LANGS__') || 'de-DE,de').split(','),
+      });
+
+      // plugins
+      // @ts-ignore
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [{}, {}, {}],
+      });
+
+      // platform and concurrency hints
+      // @ts-ignore
+      Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
+      // @ts-ignore
+      if (typeof navigator.hardwareConcurrency === 'number' && navigator.hardwareConcurrency < 2) {
+        Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 4 });
+      }
+      // @ts-ignore
+      if (typeof navigator.deviceMemory === 'number' && navigator.deviceMemory < 2) {
+        Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
+      }
+
+      // userAgentData (Client Hints)
+      try {
+        // @ts-ignore
+        if ('userAgentData' in navigator) {
+          // @ts-ignore
+          Object.defineProperty(navigator, 'userAgentData', {
+            get: () => ({
+              brands: [
+                { brand: 'Chromium', version: '126' },
+                { brand: 'Google Chrome', version: '126' },
+              ],
+              mobile: false,
+              platform: 'Windows',
+              getHighEntropyValues: async (hints) => {
+                const values = {
+                  platform: 'Windows',
+                  platformVersion: '15.0.0',
+                  architecture: 'x86',
+                  model: '',
+                  uaFullVersion: '126.0.0.0',
+                  bitness: '64',
+                };
+                const out = {};
+                for (const k of hints || []) if (k in values) out[k] = values[k];
+                return out;
+              },
+            }),
+          });
+        }
+      } catch {
+        //noop
+      }
+
+      // Permissions API
+      const origQuery = navigator.permissions && navigator.permissions.query;
+      if (origQuery) {
+        // @ts-ignore
+        navigator.permissions.query = (parameters) =>
+          origQuery.call(navigator.permissions, parameters).then((result) => {
+            if (parameters && parameters.name === 'notifications') {
+              Object.defineProperty(result, 'state', { get: () => Notification.permission });
+            }
+            return result;
+          });
+      }
+
+      // WebGL vendor/renderer
+      const patchWebGL = (proto) => {
+        if (!proto || !proto.getParameter) return;
+        const getParameter = proto.getParameter;
+        // @ts-ignore
+        proto.getParameter = function (param) {
+          const UNMASKED_VENDOR_WEBGL = 0x9245;
+          const UNMASKED_RENDERER_WEBGL = 0x9246;
+          if (param === UNMASKED_VENDOR_WEBGL) return 'Google Inc.';
+          if (param === UNMASKED_RENDERER_WEBGL)
+            return 'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 Ti Direct3D11 vs_5_0 ps_5_0)';
+          return getParameter.call(this, param);
+        };
+      };
+      // @ts-ignore
+      patchWebGL(WebGLRenderingContext?.prototype);
+      // @ts-ignore
+      patchWebGL(WebGL2RenderingContext?.prototype);
+
+      // AudioContext timestamp rounding consistency
+      const patchAudio = (Ctx) => {
+        try {
+          if (!Ctx) return;
+          const proto = Ctx.prototype;
+          const createOsc = proto.createOscillator;
+          proto.createOscillator = function () {
+            const osc = createOsc.call(this);
+            const start = osc.start;
+            osc.start = function (when) {
+              return start.call(this, when || 0);
+            };
+            return osc;
+          };
+        } catch {
+          //noop
+        }
+      };
+      // @ts-ignore
+      patchAudio(window.AudioContext);
+      // @ts-ignore
+      patchAudio(window.OfflineAudioContext);
+
+      // Navigator.connection
+      try {
+        // @ts-ignore
+        Object.defineProperty(navigator, 'connection', { get: () => undefined });
+      } catch {
+        //noop
+      }
+
+      // Consistent outer sizes
+      try {
+        const calcOuter = () => {
+          const w = window.innerWidth + 16;
+          const h = window.innerHeight + 88;
+          return { w, h };
+        };
+        const { w: outerW, h: outerH } = calcOuter();
+        // @ts-ignore
+        Object.defineProperty(window, 'outerWidth', { get: () => outerW });
+        // @ts-ignore
+        Object.defineProperty(window, 'outerHeight', { get: () => outerH });
+      } catch {
+        //noop
+      }
+    } catch {
+      //noop
+    }
+  });
+}
+
+/**
+ * Persist languages value before navigation via localStorage.
+ * @param {import('puppeteer').Page} page
+ * @param {ReturnType<typeof getPreLaunchConfig>} cfg
+ */
+export async function applyLanguagePersistence(page, cfg) {
+  await page.evaluateOnNewDocument((langs) => {
+    try {
+      window.localStorage.setItem('__LANGS__', langs);
+    } catch {
+      // noop
+    }
+  }, cfg.acceptLanguage.split(';')[0]);
+}
+
+/**
+ * Perform subtle human-like interactions post navigation.
+ * @param {import('puppeteer').Page} page
+ * @param {ReturnType<typeof getPreLaunchConfig>} cfg
+ */
+export async function applyPostNavigationHumanSignals(page, cfg) {
+  if (!cfg.humanDelay) return;
+  const delay = 200 + Math.floor(Math.random() * 400);
+  await new Promise((res) => setTimeout(res, delay));
+  try {
+    const vw = cfg.viewport.width;
+    const vh = cfg.viewport.height;
+    const mx = Math.floor(vw * (0.3 + Math.random() * 0.4));
+    const my = Math.floor(vh * (0.3 + Math.random() * 0.4));
+    await page.mouse.move(mx, my, { steps: 10 + Math.floor(Math.random() * 10) });
+    await page.mouse.wheel({ deltaY: 100 + Math.floor(Math.random() * 200) });
+  } catch {
+    // ignore if mouse is unavailable
+  }
+}

lib/services/extractor/puppeteerExtractor.js

@@ -1,6 +1,12 @@
 import puppeteer from 'puppeteer-extra';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
+import { debug, botDetected } from './utils.js';
+import {
+  getPreLaunchConfig,
+  applyBotPreventionToPage,
+  applyLanguagePersistence,
+  applyPostNavigationHumanSignals,
+} from './botPrevention.js';
 import logger from '../logger.js';
 import fs from 'fs';
 import os from 'os';
@@ -27,23 +33,50 @@ export default async function execute(url, waitForSelector, options) {
       removeUserDataDir = true;
     }
 
+    const launchArgs = [
+      '--no-sandbox',
+      '--disable-gpu',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage',
+      '--disable-crash-reporter',
+      '--no-first-run',
+      '--no-default-browser-check',
+    ];
+    if (options?.proxyUrl) {
+      launchArgs.push(`--proxy-server=${options.proxyUrl}`);
+    }
+    // Prepare bot prevention pre-launch config
+    const preCfg = getPreLaunchConfig(url, options || {});
+    launchArgs.push(preCfg.langArg);
+    launchArgs.push(preCfg.windowSizeArg);
+    launchArgs.push(...preCfg.extraArgs);
+
     browser = await puppeteer.launch({
-      headless: options.puppeteerHeadless ?? true,
-      args: [
-        '--no-sandbox',
-        '--disable-gpu',
-        '--disable-setuid-sandbox',
-        '--disable-dev-shm-usage',
-        '--disable-crash-reporter',
-      ],
-      timeout: options.puppeteerTimeout || 30_000,
+      headless: options?.puppeteerHeadless ?? true,
+      args: launchArgs,
+      timeout: options?.puppeteerTimeout || 30_000,
       userDataDir,
+      executablePath: options?.executablePath, // allow using system Chrome
     });
+
     page = await browser.newPage();
-    await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+    await applyBotPreventionToPage(page, preCfg);
+    // Provide languages value before navigation
+    await applyLanguagePersistence(page, preCfg);
+
+    // Optional cookies
+    if (Array.isArray(options?.cookies) && options.cookies.length > 0) {
+      await page.setCookie(...options.cookies);
+    }
+
+    // Navigation
     const response = await page.goto(url, {
-      waitUntil: 'domcontentloaded',
+      waitUntil: options?.waitUntil || 'domcontentloaded',
     });
+
+    // Optionally wait and add subtle human-like interactions
+    await applyPostNavigationHumanSignals(page, preCfg);
+
     let pageSource;
     // if we're extracting data from a SPA, we must wait for the selector
     if (waitForSelector != null) {
@@ -57,7 +90,7 @@ export default async function execute(url, waitForSelector, options) {
       pageSource = await page.content();
     }
 
-    const statusCode = response.status();
+    const statusCode = response?.status?.() ?? 200;
 
     if (botDetected(pageSource, statusCode)) {
       logger.warn('We have been detected as a bot :-/ Tried url: => ', url);

lib/services/storage/listingsStorage.js

@@ -152,8 +152,9 @@ export const storeListings = (jobId, providerId, listings) => {
    */
   function extractNumber(str) {
     if (!str) return null;
-    const match = str.replace(/[.,]/g, '').match(/\d+/);
-    return match ? +match[0] : null;
+    const cleaned = str.replace(/\./g, '').replace(',', '.');
+    const num = parseFloat(cleaned);
+    return isNaN(num) ? null : num;
   }
 
   /**

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "14.3.2",
+  "version": "14.4.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -12,6 +12,7 @@
     "format": "prettier --write \"**/*.js\"",
     "format:check": "prettier --check \"**/*.js\"",
     "test": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 test/**/*.test.js",
+    "testGH": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 --exclude test/provider/immonet.test.js --exclude test/provider/immowelt.test.js test/**/*.test.js",
     "lint": "eslint .",
     "lint:fix": "yarn lint --fix",
     "migratedb": "node lib/services/storage/migrations/migrate.js",
@@ -56,15 +57,15 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-icons": "^2.87.1",
-    "@douyinfe/semi-ui": "2.87.1",
+    "@douyinfe/semi-icons": "^2.89.0",
+    "@douyinfe/semi-ui": "2.89.0",
     "@sendgrid/mail": "8.1.6",
-    "@visactor/react-vchart": "^2.0.5",
-    "@visactor/vchart": "^2.0.5",
+    "@visactor/react-vchart": "^2.0.10",
+    "@visactor/vchart": "^2.0.10",
     "@visactor/vchart-semi-theme": "^1.12.2",
-    "@vitejs/plugin-react": "5.1.0",
-    "better-sqlite3": "^12.4.1",
-    "body-parser": "2.2.0",
+    "@vitejs/plugin-react": "5.1.2",
+    "better-sqlite3": "^12.5.0",
+    "body-parser": "2.2.1",
     "cheerio": "^1.1.2",
     "cookie-session": "2.1.1",
     "handlebars": "4.7.8",
@@ -73,40 +74,40 @@
     "node-cron": "^4.2.1",
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.11",
-    "p-throttle": "^8.0.0",
+    "p-throttle": "^8.1.0",
     "package-up": "^5.0.0",
-    "puppeteer": "^24.27.0",
+    "puppeteer": "^24.32.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "query-string": "9.3.1",
     "react": "18.3.1",
     "react-dom": "18.3.1",
-    "react-router": "7.9.5",
-    "react-router-dom": "7.9.5",
+    "react-router": "7.10.1",
+    "react-router-dom": "7.10.1",
     "restana": "5.1.0",
     "semver": "^7.7.3",
     "serve-static": "2.2.0",
     "slack": "11.0.2",
-    "vite": "7.1.12",
+    "vite": "7.2.7",
     "x-var": "^3.0.1",
-    "zustand": "^5.0.8"
+    "zustand": "^5.0.9"
   },
   "devDependencies": {
     "@babel/core": "7.28.5",
     "@babel/eslint-parser": "7.28.5",
     "@babel/preset-env": "7.28.5",
     "@babel/preset-react": "7.28.5",
-    "chai": "6.2.0",
-    "eslint": "9.39.0",
+    "chai": "6.2.1",
+    "eslint": "9.39.1",
     "eslint-config-prettier": "10.1.8",
     "eslint-plugin-react": "7.37.5",
     "esmock": "2.7.3",
     "history": "5.3.0",
     "husky": "9.1.7",
     "less": "4.4.2",
-    "lint-staged": "16.2.6",
-    "mocha": "11.7.4",
-    "nodemon": "^3.1.10",
-    "prettier": "3.6.2"
+    "lint-staged": "16.2.7",
+    "mocha": "11.7.5",
+    "nodemon": "^3.1.11",
+    "prettier": "3.7.4"
   }
 }

test/provider/ohneMakler.test.js

@@ -0,0 +1,33 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/ohneMakler.js';
+
+describe('#ohneMakler testsuite()', () => {
+  it('should test ohneMakler provider', async () => {
+    const Fredy = await mockFredy();
+    provider.init(providerConfig.ohneMakler, []);
+
+    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'ohneMakler', similarityCache);
+    const listing = await fredy.execute();
+
+    expect(listing).to.be.a('array');
+    const notificationObj = get();
+    expect(notificationObj).to.be.a('object');
+    expect(notificationObj.serviceName).to.equal('ohneMakler');
+    notificationObj.payload.forEach((notify) => {
+      /** check the actual structure **/
+      expect(notify.id).to.be.a('string');
+      expect(notify.price).to.be.a('string');
+      expect(notify.size).to.be.a('string');
+      expect(notify.title).to.be.a('string');
+      expect(notify.link).to.be.a('string');
+      expect(notify.address).to.be.a('string');
+      /** check the values if possible **/
+      expect(notify.size).that.does.include('m²');
+      expect(notify.title).to.be.not.empty;
+      expect(notify.address).to.be.not.empty;
+    });
+  });
+});

test/provider/testProvider.json

@@ -32,6 +32,10 @@
     "url": "https://www.mcmakler.de/immobilien/results?placeId=62649&search=Leipzig%252C+Sachsen&propertyTypes=APARTMENT&page=0",
     "enabled": true
   },
+  "ohneMakler": {
+    "url": "https://www.ohne-makler.net/immobilien/wohnung-kaufen/nordrhein-westfalen/dusseldorf/",
+    "enabled": true
+  },
   "neubauKompass": {
     "url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
     "enabled": true

test/services/extractor/botPrevention.test.js

@@ -0,0 +1,99 @@
+import { describe, it } from 'mocha';
+import { expect } from 'chai';
+
+import {
+  getPreLaunchConfig,
+  applyBotPreventionToPage,
+  applyLanguagePersistence,
+  applyPostNavigationHumanSignals,
+} from '../../../lib/services/extractor/botPrevention.js';
+
+describe('botPrevention helper', () => {
+  it('getPreLaunchConfig builds deterministic values when jitter disabled', () => {
+    const url = 'https://example.com/some/path';
+    const options = {
+      acceptLanguage: 'de-DE,de;q=0.9',
+      userAgent: 'TestAgent/1.0',
+      viewport: { width: 1200, height: 700, deviceScaleFactor: 2 },
+      viewportJitter: false,
+      referer: 'https://example.com/ref',
+      timezone: 'Europe/Berlin',
+    };
+    const cfg = getPreLaunchConfig(url, options);
+
+    expect(cfg.acceptLanguage).to.equal('de-DE,de;q=0.9');
+    expect(cfg.langArg).to.equal('--lang=de-DE');
+    expect(cfg.windowSizeArg).to.equal('--window-size=1200,700');
+    expect(cfg.viewport).to.deep.equal({ width: 1200, height: 700, deviceScaleFactor: 2 });
+    expect(cfg.userAgent).to.equal('TestAgent/1.0');
+    expect(cfg.headers['Accept-Language']).to.equal('de-DE,de;q=0.9');
+    expect(cfg.headers['User-Agent']).to.equal('TestAgent/1.0');
+    expect(cfg.headers.Referer).to.equal('https://example.com/ref');
+    expect(cfg.extraArgs).to.include('--disable-blink-features=AutomationControlled');
+    expect(cfg.extraArgs).to.include('--proxy-bypass-list=<-loopback>');
+  });
+
+  it('applyBotPreventionToPage sets UA, viewport, headers and injects patches', async () => {
+    const calls = [];
+    const page = {
+      setUserAgent: async (ua) => calls.push(['setUserAgent', ua]),
+      setViewport: async (vp) => calls.push(['setViewport', vp]),
+      setJavaScriptEnabled: async (on) => calls.push(['setJavaScriptEnabled', on]),
+      setExtraHTTPHeaders: async (h) => calls.push(['setExtraHTTPHeaders', h]),
+      emulateTimezone: async (tz) => calls.push(['emulateTimezone', tz]),
+      evaluateOnNewDocument: async (fn) => calls.push(['evaluateOnNewDocument', typeof fn]),
+    };
+    const cfg = getPreLaunchConfig('https://example.org/', {
+      userAgent: 'Foo/Bar',
+      acceptLanguage: 'en-US,en',
+      viewport: { width: 1000, height: 600, deviceScaleFactor: 1 },
+      viewportJitter: false,
+      timezone: 'UTC',
+    });
+
+    await applyBotPreventionToPage(page, cfg);
+
+    expect(calls[0]).to.deep.equal(['setUserAgent', 'Foo/Bar']);
+    expect(calls.some((c) => c[0] === 'setViewport' && c[1].width === 1000 && c[1].height === 600)).to.equal(true);
+    expect(calls.some((c) => c[0] === 'setJavaScriptEnabled' && c[1] === true)).to.equal(true);
+    const headerCall = calls.find((c) => c[0] === 'setExtraHTTPHeaders');
+    expect(headerCall).to.exist;
+    expect(headerCall[1]['Accept-Language']).to.equal('en-US,en');
+    expect(headerCall[1]['User-Agent']).to.equal('Foo/Bar');
+    expect(calls.some((c) => c[0] === 'emulateTimezone' && c[1] === 'UTC')).to.equal(true);
+    expect(calls.some((c) => c[0] === 'evaluateOnNewDocument' && c[1] === 'function')).to.equal(true);
+  });
+
+  it('applyLanguagePersistence stores languages early', async () => {
+    const calls = [];
+    const page = {
+      evaluateOnNewDocument: async (fn, arg) => calls.push(['evaluateOnNewDocument', typeof fn, arg]),
+    };
+    const cfg = getPreLaunchConfig('https://example.org/', {
+      acceptLanguage: 'de-DE,de;q=0.9',
+      viewportJitter: false,
+    });
+    await applyLanguagePersistence(page, cfg);
+    const call = calls[0];
+    expect(call[0]).to.equal('evaluateOnNewDocument');
+    expect(call[1]).to.equal('function');
+    expect(call[2]).to.equal('de-DE,de');
+  });
+
+  it('applyPostNavigationHumanSignals moves mouse and scrolls when enabled', async () => {
+    const mouseCalls = [];
+    const page = {
+      mouse: {
+        move: async (x, y, opts) => mouseCalls.push(['move', x, y, opts && typeof opts.steps === 'number']),
+        wheel: async (opts) => mouseCalls.push(['wheel', typeof opts.deltaY === 'number']),
+      },
+    };
+    const cfg = {
+      humanDelay: true,
+      viewport: { width: 1200, height: 800 },
+    };
+    await applyPostNavigationHumanSignals(page, cfg);
+    expect(mouseCalls.some((c) => c[0] === 'move')).to.equal(true);
+    expect(mouseCalls.some((c) => c[0] === 'wheel')).to.equal(true);
+  });
+});

ui/src/components/table/ProviderTable.jsx

@@ -1,9 +1,9 @@
 import React from 'react';
 
 import { Empty, Table, Button } from '@douyinfe/semi-ui';
-import { IconDelete } from '@douyinfe/semi-icons';
+import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
 
-export default function ProviderTable({ providerData = [], onRemove } = {}) {
+export default function ProviderTable({ providerData = [], onRemove, onEdit } = {}) {
   return (
     <Table
       pagination={false}
@@ -30,6 +30,8 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
           render: (_, record) => {
             return (
               <div style={{ float: 'right' }}>
+                <Button type="secondary" icon={<IconEdit />} onClick={() => onEdit(record)} />
+                <div style={{ display: 'inline-block', width: '16px' }} />
                 <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
               </div>
             );

ui/src/views/jobs/mutation/JobMutation.jsx

@@ -11,7 +11,15 @@ import { useNavigate, useParams } from 'react-router-dom';
 import { Divider, Input, Switch, Button, TagInput, Toast, Select } from '@douyinfe/semi-ui';
 import './JobMutation.less';
 import { SegmentPart } from '../../../components/segment/SegmentPart';
-import { IconBell, IconBriefcase, IconPaperclip, IconPlayCircle, IconPlusCircle, IconUser } from '@douyinfe/semi-icons';
+import {
+  IconBell,
+  IconBriefcase,
+  IconPaperclip,
+  IconPlayCircle,
+  IconPlusCircle,
+  IconUser,
+  IconClear,
+} from '@douyinfe/semi-icons';
 
 export default function JobMutator() {
   const jobs = useSelector((state) => state.jobs.jobs);
@@ -26,6 +34,7 @@ export default function JobMutator() {
   const defaultNotificationAdapter = jobToBeEdit?.notificationAdapter || [];
   const defaultEnabled = jobToBeEdit?.enabled ?? true;
 
+  const [providerToEdit, setProviderToEdit] = useState(null);
   const [providerCreationVisible, setProviderCreationVisibility] = useState(false);
   const [notificationCreationVisible, setNotificationCreationVisibility] = useState(false);
   const [editNotificationAdapter, setEditNotificationAdapter] = useState(null);
@@ -42,6 +51,12 @@ export default function JobMutator() {
     return Boolean(notificationAdapterData.length && providerData.length && name);
   };
 
+  const handleProviderEdit = (data) => {
+    setProviderData(
+      providerData.map((provider) => (provider.url === data.oldProviderToEdit.url ? data.newData : provider)),
+    );
+  };
+
   const mutateJob = async () => {
     try {
       await xhrPost('/api/jobs', {
@@ -70,6 +85,8 @@ export default function JobMutator() {
         onData={(data) => {
           setProviderData([...providerData, data]);
         }}
+        onEditData={handleProviderEdit}
+        providerToEdit={providerToEdit}
       />
 
       {notificationCreationVisible && (
@@ -119,7 +136,10 @@ export default function JobMutator() {
             type="primary"
             icon={<IconPlusCircle />}
             className="jobMutation__newButton"
-            onClick={() => setProviderCreationVisibility(true)}
+            onClick={() => {
+              setProviderToEdit(null);
+              setProviderCreationVisibility(true);
+            }}
           >
             Add new Provider
           </Button>
@@ -129,6 +149,10 @@ export default function JobMutator() {
             onRemove={(providerUrl) => {
               setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
             }}
+            onEdit={(provider) => {
+              setProviderCreationVisibility(true);
+              setProviderToEdit(provider);
+            }}
           />
         </SegmentPart>
         <Divider margin="1rem" />
@@ -160,7 +184,7 @@ export default function JobMutator() {
         </SegmentPart>
         <Divider margin="1rem" />
         <SegmentPart
-          Icon={IconBell}
+          Icon={IconClear}
           name="Blacklist"
           helpText="If a listing contains one of these words, it will be filtered out. Type in a word, then hit enter."
         >

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx

@@ -7,6 +7,7 @@ import { useSelector } from '../../../../../services/state/store';
 import { Banner, Button, Form, Modal, Select, Switch } from '@douyinfe/semi-ui';
 
 import './NotificationAdapterMutator.less';
+import { useScreenWidth } from '../../../../../hooks/screenWidth.js';
 
 const sortAdapter = (a, b) => {
   if (a.name < b.name) {
@@ -70,6 +71,9 @@ export default function NotificationAdapterMutator({
   const [validationMessage, setValidationMessage] = useState(null);
   const [successMessage, setSuccessMessage] = useState(null);
 
+  const width = useScreenWidth();
+  const isMobile = width <= 850;
+
   const onSubmit = (doStore) => {
     if (doStore) {
       const validationResults = validate(selectedAdapter);
@@ -170,18 +174,19 @@ export default function NotificationAdapterMutator({
     <Modal
       title="Adding a new Notification Adapter"
       visible={visible}
-      style={{ width: '95%' }}
+      style={{ width: isMobile ? '95%' : '50rem' }}
+      onCancel={() => onSubmit(false)}
       footer={
         <div>
-          <Button type="secondary" disabled={selectedAdapter == null} style={{ float: 'left' }} onClick={() => onTry()}>
+          <Button type="secondary" disabled={selectedAdapter == null} style={{ float: 'left' }} onClick={onTry}>
             Try
           </Button>
-          <Button type="danger" onClick={() => onSubmit(true)}>
-            Save
-          </Button>
-          <Button type="primary" onClick={() => onSubmit(false)}>
+          <Button theme="light" type="tertiary" onClick={() => onSubmit(false)}>
             Cancel
           </Button>
+          <Button theme="solid" type="primary" onClick={() => onSubmit(true)}>
+            Save
+          </Button>
         </div>
       }
     >
@@ -207,9 +212,9 @@ export default function NotificationAdapterMutator({
       )}
 
       <p>
-        When Fredy found new listings, we like to report them to you. To do so, notification adapter can be configured.{' '}
-        <br />
-        There are multiple ways how Fredy can send new listings to you. Chose your weapon...
+        When Fredy finds new listings, we like to report them to you. To do so, the notification adapter can be
+        configured. <br />
+        There are multiple ways Fredy can send new listings to you. Choose your weapon...
       </p>
 
       <Select

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -1,10 +1,11 @@
-import React, { useState } from 'react';
+import React, { useState, useEffect } from 'react';
 
 import { Banner, Modal, Select, Input } from '@douyinfe/semi-ui';
 import { transform } from '../../../../../services/transformer/providerTransformer';
 import { useSelector } from '../../../../../services/state/store';
 import { IconLikeHeart } from '@douyinfe/semi-icons';
 import './ProviderMutator.less';
+import { useScreenWidth } from '../../../../../hooks/screenWidth.js';
 
 const sortProvider = (a, b) => {
   if (a.key < b.key) {
@@ -16,11 +17,35 @@ const sortProvider = (a, b) => {
   return 0;
 };
 
-export default function ProviderMutator({ onVisibilityChanged, visible = false, onData } = {}) {
+const returnOriginalSelectedProvider = (providerToEdit, provider) => {
+  return provider.find((pro) => pro.id === providerToEdit.id);
+};
+
+export default function ProviderMutator({
+  onVisibilityChanged,
+  visible = false,
+  onData,
+  onEditData,
+  providerToEdit,
+} = {}) {
   const provider = useSelector((state) => state.provider);
   const [selectedProvider, setSelectedProvider] = useState(null);
   const [providerUrl, setProviderUrl] = useState(null);
   const [validationMessage, setValidationMessage] = useState(null);
+
+  useEffect(() => {
+    if (providerToEdit) {
+      setSelectedProvider(returnOriginalSelectedProvider(providerToEdit, provider));
+      setProviderUrl(providerToEdit.url);
+    } else {
+      setSelectedProvider(null);
+      setProviderUrl(null);
+    }
+  }, [providerToEdit, visible]);
+
+  const width = useScreenWidth();
+  const isMobile = width <= 850;
+
   const validate = () => {
     if (selectedProvider == null || selectedProvider.length === 0 || providerUrl == null || providerUrl.length === 0) {
       return 'Please select a provider and copy the browser url into the textfield after configuring your search parameter.';
@@ -41,13 +66,24 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
     if (doStore) {
       const validationResult = validate();
       if (validationResult == null) {
-        onData(
-          transform({
-            url: providerUrl,
-            id: selectedProvider.id,
-            name: selectedProvider.name,
-          }),
-        );
+        if (providerToEdit != null) {
+          onEditData({
+            newData: transform({
+              url: providerUrl,
+              id: selectedProvider.id,
+              name: selectedProvider.name,
+            }),
+            oldProviderToEdit: providerToEdit,
+          });
+        } else {
+          onData(
+            transform({
+              url: providerUrl,
+              id: selectedProvider.id,
+              name: selectedProvider.name,
+            }),
+          );
+        }
         setProviderUrl(null);
         setSelectedProvider(null);
         onVisibilityChanged(false);
@@ -63,11 +99,11 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
 
   return (
     <Modal
-      title="Adding a new Provider"
+      title={providerToEdit ? 'Editing an existing Provider' : 'Adding a new Provider'}
       visible={visible}
       onOk={() => onSubmit(true)}
       onCancel={() => onSubmit(false)}
-      style={{ width: '50rem' }}
+      style={{ width: isMobile ? '95%' : '50rem' }}
       okText="Save"
     >
       {validationMessage != null && (
@@ -80,19 +116,26 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
           description={validationMessage}
         />
       )}
-
-      <p>
-        Provider are the <IconLikeHeart style={{ color: '#ff0000' }} /> of Fredy. We're supporting multiple Provider
-        such as Immowelt, Kalaydo etc. Select a provider from the list below.
-        <br />
-        Fredy will then open the provider's url in a new tab.
-      </p>
-      <p>
-        You will need to configure your search parameter like you would do when you do a regular search on the
-        provider's website.
-        <br />
-        When the search results are shown on the website, copy the url and paste it into the textfield below.
-      </p>
+      {providerToEdit != null ? (
+        <p>
+          You can now edit the <strong>{providerToEdit.name}</strong> provider's URL in the input field below.
+        </p>
+      ) : (
+        <>
+          <p>
+            Provider are the <IconLikeHeart style={{ color: '#ff0000' }} /> of Fredy. We're supporting multiple Provider
+            such as Immowelt, Kalaydo etc. Select a provider from the list below.
+            <br />
+            Fredy will then open the provider's url in a new tab.
+          </p>
+          <p>
+            You will need to configure your search parameter like you would do when you do a regular search on the
+            provider's website.
+            <br />
+            When the search results are shown on the website, copy the url and paste it into the textfield below.
+          </p>
+        </>
+      )}
       <Banner
         fullMode={false}
         type="warning"
@@ -112,6 +155,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         filter
         placeholder="Select a provider"
         className="providerMutator__fields"
+        disabled={providerToEdit != null}
         optionList={provider
           .map((pro) => {
             return {
@@ -126,7 +170,6 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         onChange={(value) => {
           const selectedProvider = provider.find((pro) => pro.id === value);
           setSelectedProvider(selectedProvider);
-
           window.open(selectedProvider.baseUrl);
         }}
       />
@@ -137,7 +180,8 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         placeholder="Provider Url"
         width={10}
         className="providerMutator__fields"
-        onBlur={(e) => {
+        value={providerUrl}
+        onInput={(e) => {
           setProviderUrl(e.target.value);
         }}
       />