fixing immoscout

Fixes https://github.com/orangecoding/fredy/issues/87

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 16
+          node-version: 18
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3
-FROM node:16-alpine AS builder
+FROM node:18-alpine AS builder
 COPY --chown=1000:1000 . /fredy
 WORKDIR  /fredy
 USER 1000
@@ -10,6 +10,7 @@ FROM node:16-alpine
 COPY --from=builder --chown=1000:1000 /fredy /fredy
 RUN mkdir /db /conf && \
   chown 1000:1000 /db /conf && \
+  chmod 777 -R /db/ && \
   ln -s /db /fredy/db && ln -s /conf /fredy/conf
 EXPOSE 9998
 USER 1000

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Christian Kellner
+Copyright (c) 2024 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use Node.js 16 or above
+- Make sure to use Node.js 18 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -33,9 +33,6 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
          
   <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
 </p>
-<p align="center">
-
-</p>
 
 ## Understanding the fundamentals
 There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
@@ -81,10 +78,10 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-### Immoscout
-I have added **experimental** support for Immoscout. Immoscout is somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+### Immoscout / Immonet
+I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
-To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
+To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
 The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
 
 If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).

lib/FredyRuntime.js

@@ -45,15 +45,15 @@ class FredyRuntime {
   _getListings(url) {
     return new Promise((resolve, reject) => {
       const id = this._providerId;
-      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
-        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
+      if (scrapingAnt.needScrapingAnt(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
+        const error = 'Immoscout or Immonet can only be used with if you have set an apikey for scrapingAnt.';
         /* eslint-disable no-console */
         console.log(error);
         /* eslint-enable no-console */
         reject(error);
         return;
       }
-      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
+      const u = scrapingAnt.needScrapingAnt(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
       try {
         if (this._providerConfig.paginate != null) {
           xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])

lib/api/api.js

@@ -13,7 +13,7 @@ import files from 'serve-static';
 import path from 'path';
 import { getDirName } from '../utils.js';
 const service = restana();
-const staticService = files(path.join(getDirName(), '../../ui/public'));
+const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
 
 service.use(bodyParser.json());

lib/api/routes/jobRouter.js

@@ -28,7 +28,7 @@ jobRouter.get('/processingTimes', async (req, res) => {
   let scrapingAntData = null;
   if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
     try {
-      const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
+      const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
       scrapingAntData = await response.json();
     } catch (Exception) {
       console.error('Could not query plan data from scraping ant.', Exception);

lib/notification/adapter/mailJet.js

@@ -9,7 +9,7 @@ const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTe
 const emailTemplate = Handlebars.compile(template);
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailJet'
+    (adapter) => adapter.id === config.id,
   ).fields;
   const to = receiver
     .trim()
@@ -18,7 +18,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
       Email: r.trim(),
     }));
   return mailjet
-    .connect(apiPublicKey, apiPrivateKey)
+    .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [

lib/notification/adapter/mattermost.js

@@ -2,13 +2,13 @@ import { markdown2Html } from '../../services/markdown.js';
 import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
   message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
   message += newListings.map(
-    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
   );
   return fetch(webhook, {
     method: 'POST',

lib/notification/adapter/ntfy.js

@@ -0,0 +1,51 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const message = `Address: ${newListing.address} Size: ${newListing.size.replace(/2m/g, '$m^2$')} Price: ${
+      newListing.price
+    }`;
+    return fetch(server, {
+      method: 'POST',
+      body: JSON.stringify({
+        topic: topic,
+        message: message,
+        title: newListing.title,
+        tags: [serviceName, jobName],
+        priority: parseInt(priority),
+        click: newListing.link,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'ntfy',
+  name: 'ntfy',
+  readme: markdown2Html('lib/notification/adapter/ntfy.md'),
+  description: 'Fredy will send new listings to your ntfy.',
+  fields: {
+    priority: {
+      type: 'number',
+      label: 'Priority',
+      description: 'The priority of the send notification.',
+    },
+    server: {
+      type: 'text',
+      label: 'Server-URL',
+      description: 'The server url to the send the notification to.',
+    },
+    topic: {
+      type: 'text',
+      label: 'topic',
+      description:
+        'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
+    },
+  },
+};

lib/notification/adapter/ntfy.md

@@ -0,0 +1,5 @@
+### ntfy Adapter
+
+For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the  [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+
+As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.

lib/notification/adapter/sendGrid.js

@@ -1,7 +1,7 @@
 import sgMail from '@sendgrid/mail';
 import { markdown2Html } from '../../services/markdown.js';
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,

lib/notification/adapter/slack.js

@@ -2,7 +2,7 @@ import Slack from 'slack';
 import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   return newListings.map((payload) =>
     msg({
       token,
@@ -35,7 +35,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
           ts: new Date().getTime() / 1000,
         },
       ],
-    })
+    }),
   );
 };
 export const config = {

lib/notification/adapter/sqlite.js

@@ -1,25 +0,0 @@
-import { markdown2Html } from '../../services/markdown.js';
-import Database from 'better-sqlite3';
-export const send = ({ serviceName, newListings, jobKey }) => {
-  const db = new Database('db/listings.db');
-  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
-  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
-  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
-  newListings.map((listing) => {
-    let insertListing = {};
-    fields.map((field) => {
-      insertListing[field] = listing[field];
-    });
-    insertListing.serviceName = serviceName;
-    insertListing.jobKey = jobKey;
-    insert.run(insertListing);
-  });
-  return Promise.resolve();
-};
-export const config = {
-  id: 'sqlite',
-  name: 'Sqlite',
-  description: 'This adapter stores listings in a local sqlite3 database.',
-  config: {},
-  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
-};

lib/notification/adapter/sqlite.md

@@ -1,3 +0,0 @@
-### Sqlite Adapter
-
-This adapter stores search results in an sqlite database in db/listings.db

lib/notification/adapter/telegram.js

@@ -19,7 +19,7 @@ function shorten(str, len = 30) {
   return str.length > len ? str.substring(0, len) + '...' : str;
 }
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
@@ -30,7 +30,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
       (o) =>
         `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
         [o.address, o.price, o.size].join(' | ') +
-        '\n\n'
+        '\n\n',
     );
     /**
      * This is to not break the rate limit. It is to only send 1 message per second

lib/provider/immonet.js

@@ -1,14 +1,12 @@
 import utils from '../utils.js';
 let appliedBlackList = [];
 function normalize(o) {
-  const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
+  const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
-  //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
-  //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
-  const link = `https://www.immonet.de/angebot/${id}`;
+  const link = o.id;
   return Object.assign(o, { id, address, price, size, title, link });
 }
 function applyBlacklist(o) {
@@ -18,14 +16,14 @@ function applyBlacklist(o) {
 }
 const config = {
   url: null,
-  crawlContainer: '#result-list-stage .item',
+  crawlContainer: '.content-wrapper-tiles .ng-star-inserted',
   sortByDateParam: 'sortby=19',
   crawlFields: {
-    id: '@id',
-    price: 'div[id*="selPrice_"] | trim',
-    size: 'div[id*="selArea_"] | trim',
-    title: '.item a img@title',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
+    id: '.card a@href',
+    title: '.card h3 |trim',
+    price: '.card .has-font-300 .is-bold | trim',
+    size: '.card .has-font-300 .ml-100 | trim',
+    address: '.card span:nth-child(2) | trim',
   },
   paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,

lib/provider/immoscout.js

@@ -6,7 +6,7 @@ function nullOrEmpty(val) {
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
   return Object.assign(o, { title, address, link });
 }
 function applyBlacklist(o) {
@@ -20,7 +20,7 @@ const config = {
     id: '.result-list-entry@data-obid | int',
     price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
     size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
+    title: '.result-list-entry .result-list-entry__brand-title-container h2 | removeNewline | trim',
     link: '.result-list-entry .result-list-entry__brand-title-container@href',
     address: '.result-list-entry .result-list-entry__map-link',
   },

lib/provider/kleinanzeigen.js

@@ -19,7 +19,7 @@ const config = {
   sortByDateParam: null,
   crawlFields: {
     id: '.aditem@data-adid | int',
-    price: '.aditem-main--middle--price | removeNewline | trim',
+    price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
     size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
     title: '.aditem-main .text-module-begin a | removeNewline | trim',
     link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
@@ -32,7 +32,7 @@ const config = {
 };
 export const metaInformation = {
   name: 'Ebay Kleinanzeigen',
-  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
+  baseUrl: 'https://www.kleinanzeigen.de/',
   id: 'kleinanzeigen',
 };
 export const init = (sourceConfig, blacklist, blacklistedDistricts) => {

lib/services/requestDriver.js

@@ -1,11 +1,16 @@
 import fetch from 'node-fetch';
 import { config } from '../utils.js';
 import { makeUrlResidential } from './scrapingAnt.js';
+import https from 'https';
 //if ScrapingAnt got blocked, this http status is returned
 const BLOCKED_HTTP_STATUS = 423;
 const NOT_FOUND_HTTP_STATUS = 404;
 const MAX_RETRIES_SCRAPING_ANT = 10;
 const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
+const agent = new https.Agent({
+  rejectUnauthorized: false,
+});
+
 function makeDriver(headers = {}) {
   let cookies = '';
   async function scrapingAntDriver(context, callback, retryCounter = 0) {
@@ -41,9 +46,10 @@ function makeDriver(headers = {}) {
       /* eslint-enable no-console */
     }
   }
+
   /**
    * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
-   * everything != Immoscout as of writing this)
+   * everything != Immoscout & Immonet as of writing this)
    */
   return async function driver(context, callback) {
     if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
@@ -55,6 +61,7 @@ function makeDriver(headers = {}) {
           ...headers,
           Cookie: cookies,
         },
+        agent,
       });
       const result = await response.text();
       callback(null, result);

lib/services/scrapingAnt.js

@@ -1,12 +1,22 @@
-import { metaInformation } from '../provider/immoscout.js';
+import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
+import { metaInformation as immoNetInfo } from '../provider/immonet.js';
 import { config } from '../utils.js';
-const isImmoscout = (id) => {
-  return id.toLowerCase() === metaInformation.id;
+
+const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
+  'window.scrollTo(0,document.body.scrollHeight);'
+).toString('base64')}`;
+
+const needScrapingAnt = (id) => {
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
 };
 export const transformUrlForScrapingAnt = (url, id) => {
-  if (isImmoscout(id)) {
-    //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
+  let urlParams = '';
+  if (needScrapingAnt(id)) {
+    if (id.toLowerCase() === immoNetInfo.id) {
+      urlParams = additionalImmonetUrlParams;
+    }
+    //only do calls to scrapingAnt when dealing with Immoscout/Immonet
+    url = `https://api.scrapingant.com/v2/general?url=${encodeURIComponent(url)}&proxy_type=datacenter${urlParams}`;
   }
   return url;
 };
@@ -16,4 +26,4 @@ export const isScrapingAntApiKeySet = () => {
 export const makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');
 };
-export { isImmoscout };
+export { needScrapingAnt };

lib/services/storage/LowDashAdapter.js

@@ -1,8 +1,8 @@
 import lodash from 'lodash';
 import { LowSync } from 'lowdb';
 export default class LowdashAdapter extends LowSync {
-  constructor(adapter) {
-    super(adapter);
+  constructor(adapter, defaultData = {}) {
+    super(adapter, defaultData);
     this.chain = lodash.chain(this).get('data');
   }
 }

lib/services/storage/jobStorage.js

@@ -7,11 +7,10 @@ import LowdashAdapter from './LowDashAdapter.js';
 
 const file = path.join(getDirName(), '../', 'db/jobs.json');
 const adapter = new JSONFileSync(file);
-const db = new LowdashAdapter(adapter);
+const db = new LowdashAdapter(adapter, { jobs: [] });
 
 db.read();
 
-db.data ||= { jobs: [] };
 
 export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =

lib/services/storage/listingsStorage.js

@@ -5,12 +5,10 @@ import LowdashAdapter from './LowDashAdapter.js';
 
 const file = path.join(getDirName(), '../', 'db/jobListingData.json');
 const adapter = new JSONFileSync(file);
-const db = new LowdashAdapter(adapter);
+const db = new LowdashAdapter(adapter, {});
 
 db.read();
 
-db.data ||= {};
-
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
   if (jobKey == null && endpoint == null) {

lib/services/storage/userStorage.js

@@ -6,24 +6,24 @@ import * as jobStorage from './jobStorage.js';
 import path from 'path';
 import LowdashAdapter from './LowDashAdapter.js';
 
+const defaultData = {
+    user: [
+        //you probably want to change the default password ;)
+        {
+            id: nanoid(),
+            lastLogin: Date.now(),
+            username: 'admin',
+            password: hasher.hash('admin'),
+            isAdmin: true,
+        },
+    ],
+};
+
 const file = path.join(getDirName(), '../', 'db/users.json');
 const adapter = new JSONFileSync(file);
-const db = new LowdashAdapter(adapter);
+const db = new LowdashAdapter(adapter, defaultData);
 
 db.read();
-db.data ||= {
-  user: [
-    //you probably want to change the default password ;)
-    {
-      id: nanoid(),
-      lastLogin: Date.now(),
-      username: 'admin',
-      password: hasher.hash('admin'),
-      isAdmin: true,
-      isDemo: false,
-    },
-  ],
-};
 
 export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "7.1.0",
+  "version": "8.0.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -55,54 +55,54 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.31.0",
+    "@douyinfe/semi-ui": "2.52.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.7.0",
-    "@vitejs/plugin-react": "3.1.0",
-    "better-sqlite3": "8.2.0",
+    "@sendgrid/mail": "8.1.0",
+    "@vitejs/plugin-react": "4.2.1",
+    "better-sqlite3": "8.6.0",
     "body-parser": "1.20.2",
-    "cookie-session": "2.0.0",
-    "handlebars": "4.7.7",
-    "highcharts": "10.3.3",
-    "highcharts-react-official": "3.2.0",
+    "cookie-session": "2.1.0",
+    "handlebars": "4.7.8",
+    "highcharts": "11.3.0",
+    "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
-    "lowdb": "5.1.0",
+    "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "4.0.1",
-    "node-fetch": "3.3.1",
-    "node-mailjet": "6.0.2",
-    "query-string": "8.1.0",
+    "nanoid": "5.0.5",
+    "node-fetch": "3.3.2",
+    "node-mailjet": "6.0.5",
+    "query-string": "8.2.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
-    "react-redux": "8.0.5",
+    "react-redux": "9.1.0",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "redux": "4.2.1",
-    "redux-thunk": "2.4.2",
+    "redux": "5.0.1",
+    "redux-thunk": "3.1.0",
     "restana": "4.9.7",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "4.2.0",
+    "vite": "5.0.12",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.21.3",
-    "@babel/eslint-parser": "7.21.3",
-    "@babel/preset-env": "7.20.2",
-    "@babel/preset-react": "7.18.6",
-    "chai": "4.3.7",
-    "eslint": "8.36.0",
-    "eslint-config-prettier": "8.7.0",
-    "eslint-plugin-react": "7.32.2",
-    "esmock": "2.1.0",
+    "@babel/core": "7.23.9",
+    "@babel/eslint-parser": "7.23.10",
+    "@babel/preset-env": "7.23.9",
+    "@babel/preset-react": "7.23.3",
+    "chai": "5.0.3",
+    "eslint": "8.56.0",
+    "eslint-config-prettier": "8.8.0",
+    "eslint-plugin-react": "7.33.2",
+    "esmock": "2.6.3",
     "history": "5.3.0",
     "husky": "4.3.8",
-    "less": "4.1.3",
-    "lint-staged": "13.2.0",
+    "less": "4.2.0",
+    "lint-staged": "13.2.2",
     "mocha": "10.2.0",
-    "prettier": "2.8.5",
+    "prettier": "3.2.5",
     "redux-logger": "3.0.6"
   }
 }

test/provider/einsAImmobilien.test.js

@@ -1,11 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { providerConfig, mockFredy } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/einsAImmobilien.js';
 
-const expect = chai.expect;
-
 describe('#einsAImmobilien testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immobilienDe.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { providerConfig, mockFredy } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immobilienDe.js';
-const expect = chai.expect;
+
 describe('#immobilien.de testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immonet.test.js

@@ -1,9 +1,10 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immonet.js';
-const expect = chai.expect;
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
+
 describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -12,6 +13,13 @@ describe('#immonet testsuite()', () => {
   it('should test immonet provider', async () => {
     const Fredy = await mockFredy();
     return await new Promise((resolve) => {
+      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+        /* eslint-disable no-console */
+        console.info('Skipping Immonet test as ScrapingAnt Api Key is not set.');
+        /* eslint-enable no-console */
+        resolve();
+        return;
+      }
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
@@ -20,17 +28,17 @@ describe('#immonet testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('immonet');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
+
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immonet.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/immoscout.test.js

@@ -1,10 +1,10 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoscout.js';
 import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const expect = chai.expect;
+
 describe('#immoscout testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immoswp.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoswp.js';
-const expect = chai.expect;
+
 describe('#immoswp testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immowelt.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immowelt.js';
-const expect = chai.expect;
+
 describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/kleinanzeigen.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/kleinanzeigen.js';
-const expect = chai.expect;
+
 describe('#kleinanzeigen testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -26,7 +26,7 @@ describe('#kleinanzeigen testsuite()', () => {
           expect(notify.address).to.be.a('string');
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
+          expect(notify.link).that.does.include('https://www.kleinanzeigen.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/neubauKompass.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
-const expect = chai.expect;
+
 describe('#neubauKompass testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/testProvider.json

@@ -9,7 +9,7 @@
     "enabled": true
   },
   "immonet": {
-    "url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
+    "url": "https://www.immonet.de/immobiliensuche/beta?pageoffset=1&listsize=100&objecttype=1&locationname=D%C3%BCsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
     "enabled": true
   },
   "immowelt": {
@@ -29,7 +29,7 @@
     "enabled": true
   },
   "kleinanzeigen": {
-    "url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
+    "url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
     "enabled": true
   },
   "neubauKompass": {

test/provider/utils.test.js

@@ -1,7 +1,7 @@
 import utils from '../../lib/utils.js';
 import assert from 'assert';
-import chai from 'chai';
-const expect = chai.expect;
+import { expect } from 'chai';
+
 const fakeWorkingHoursConfig = (from, to) => ({
   workingHours: {
     to,

test/provider/wgGesucht.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/wgGesucht.js';
-const expect = chai.expect;
+
 describe('#wgGesucht testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -23,7 +23,6 @@ describe('#wgGesucht testsuite()', () => {
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.details).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.link).to.be.a('string');
         });

test/queryStringMutator/queryStringMutator.test.js

@@ -1,16 +1,15 @@
 import fs from 'fs';
-import chai from 'chai';
+import { expect } from 'chai';
 import { readFile } from 'fs/promises';
 import mutator from '../../lib/services/queryStringMutator.js';
 import queryString from 'query-string';
-const expect = chai.expect;
 
 const data = await readFile(new URL('./testData.json', import.meta.url));
 
 const testData = JSON.parse(data);
 
 let _provider = await Promise.all(
-  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
+  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`)),
 );
 
 /**

test/similarity/similarity.test.js

@@ -1,6 +1,6 @@
 import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
-import chai from 'chai';
-const expect = chai.expect;
+import { expect } from 'chai';
+
 describe('similarityCheck', () => {
   describe('#similarityCheck()', () => {
     it('should be false', () => {
@@ -29,10 +29,10 @@ describe('similarityCheck', () => {
     it('should be false', () => {
       const check = new SimilarityCacheEntry(0);
       check.setCacheEntry(
-        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.'
+        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.',
       );
       check.setCacheEntry(
-        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
+        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
     });
   });

ui/src/views/jobs/ProcessingTimes.jsx

@@ -47,7 +47,7 @@ export default function ProcessingTimes({ processingTimes }) {
               Credits: {processingTimes.scrapingAntData.remained_credits}/
               {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
             </p>
-            If you want to scrape Immoscout more often, you have to purchase a premium account of{' '}
+            If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
             <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
               ScrapingAnt
             </a>

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx

@@ -25,9 +25,12 @@ const validate = (selectedAdapter) => {
       results.push('All fields are mandatory and must be set.');
       continue;
     }
-    if (uiElement.type === 'number' && (typeof uiElement.value !== 'number' || uiElement.value < 0)) {
-      results.push('A number field cannot contain anything else and must be > 0.');
-      continue;
+    if (uiElement.type === 'number') {
+      const numberValue = parseFloat(uiElement.value);
+      if(isNaN(numberValue) || numberValue < 0) {
+        results.push('A number field cannot contain anything else and must be > 0.');
+        continue;
+      }
     }
     if (uiElement.type === 'boolean' && typeof uiElement.value !== 'boolean') {
       results.push('A boolean field cannot be of a different type.');

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              If you chose Immoscout as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
               (See readme)
             </p>
             <p>