upgrading dependencies | adding sqlite for later analysis

* building multi platform docker images

* upgrading dependencies | using scraping ant for neubaukompass

.github/workflows/docker.yml

@@ -44,3 +44,4 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

Dockerfile

@@ -1,18 +1,20 @@
-# syntax=docker/dockerfile:1.3
+FROM node:20
-FROM node:16-alpine AS builder
+
-COPY --chown=1000:1000 . /fredy
 WORKDIR  /fredy
-USER 1000
+
+COPY . /fredy
+
 RUN yarn install
+
+RUN yarn global add pm2
+
 RUN yarn run prod
 
-FROM node:16-alpine
-COPY --from=builder --chown=1000:1000 /fredy /fredy
 RUN mkdir /db /conf && \
   chown 1000:1000 /db /conf && \
+  chmod 777 -R /db/ && \
   ln -s /db /fredy/db && ln -s /conf /fredy/conf
+
 EXPOSE 9998
-USER 1000
+
-VOLUME [ "/conf", "/db" ]
+CMD pm2-runtime index.js
-WORKDIR  /fredy
-CMD node index.js --no-daemon

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Christian Kellner
+Copyright (c) 2024 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use Node.js 16 or above
+- Make sure to use Node.js 20 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -33,9 +33,6 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
          
   <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
 </p>
-<p align="center">
-
-</p>
 
 ## Understanding the fundamentals
 There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
@@ -81,15 +78,20 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-### Immoscout / Immonet
+### Immoscout / Immonet / NeubauKompass
-I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+I have added **experimental** support for Immoscout, Immonet and NeubauKompass. They all are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
 To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
 The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
 
 If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
 
-### Contribution guidelines  
+### 👐 Contributing
+Thanks to all the people who already contributed!
+
+<a href="https://github.com/orangecoding/fredy/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
+</a>
 
 See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.3'
+version: '3.8'
 services:
   fredy:
     container_name: fredy
@@ -13,3 +13,4 @@ services:
       - ./db:/db
     ports:
         - 9998:9998
+    restart: unless-stopped

lib/FredyRuntime.js

@@ -87,7 +87,10 @@ class FredyRuntime {
     return listings.map(this._providerConfig.normalize);
   }
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);

lib/notification/adapter/apprise.js

@@ -0,0 +1,36 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        body: message,
+        title: title,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'apprise',
+  name: 'Apprise',
+  readme: markdown2Html('lib/notification/adapter/apprise.md'),
+  description: 'Fredy will send new listings to your Apprise instance.',
+  fields: {
+    server: {
+      type: 'text',
+      label: 'Server',
+      description: 'The server URL to send the notification to.',
+    },
+  },
+};

lib/notification/adapter/apprise.md

@@ -0,0 +1,3 @@
+### Apprise Adapter
+
+Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.

lib/notification/adapter/mailJet.js

@@ -9,7 +9,7 @@ const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTe
 const emailTemplate = Handlebars.compile(template);
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailjet'
+    (adapter) => adapter.id === config.id,
   ).fields;
   const to = receiver
     .trim()

lib/notification/adapter/mattermost.js

@@ -2,13 +2,13 @@ import { markdown2Html } from '../../services/markdown.js';
 import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
   message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
   message += newListings.map(
-    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
   );
   return fetch(webhook, {
     method: 'POST',

lib/notification/adapter/ntfy.js

@@ -3,7 +3,7 @@ import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields;
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   const promises = newListings.map((newListing) => {

lib/notification/adapter/pushover.js

@@ -0,0 +1,50 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+    return fetch('https://api.pushover.net/1/messages.json', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        token: token,
+        user: user,
+        message: message,
+        device: device,
+        title: title,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+
+export const config = {
+  id: 'pushover',
+  name: 'Pushover',
+  readme: markdown2Html('lib/notification/adapter/pushover.md'),
+  description: 'Fredy will send new listings to your mobile using Pushover.',
+  fields: {
+    token: {
+      type: 'text',
+      label: 'API token',
+      description: 'Your application\'s API token.',
+    },
+    user: {
+      type: 'text',
+      label: 'User key',
+      description: 'Your user/group key.',
+    },
+    device: {
+      type: 'text',
+      label: 'Device name',
+      description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
+    },
+  },
+};

lib/notification/adapter/pushover.md

@@ -0,0 +1,5 @@
+### Pushover Adapter
+
+Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+
+After setting up the application, please enter both your newly created User key and API token.

lib/notification/adapter/sendGrid.js

@@ -1,7 +1,7 @@
 import sgMail from '@sendgrid/mail';
 import { markdown2Html } from '../../services/markdown.js';
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,

lib/notification/adapter/slack.js

@@ -2,7 +2,7 @@ import Slack from 'slack';
 import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   return newListings.map((payload) =>
     msg({
       token,
@@ -35,7 +35,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
           ts: new Date().getTime() / 1000,
         },
       ],
-    })
+    }),
   );
 };
 export const config = {

lib/notification/adapter/sqlite.md

@@ -1,3 +1,7 @@
 ### Sqlite Adapter
+This adapter stores search results in a sqlite database located in db/listings.db. This file can be used for further analysis later on.   
 
-This adapter stores search results in an sqlite database in db/listings.db
+Fields are:
+```
+['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description']
+```

lib/notification/adapter/telegram.js

@@ -19,7 +19,7 @@ function shorten(str, len = 30) {
   return str.length > len ? str.substring(0, len) + '...' : str;
 }
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
@@ -30,7 +30,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
       (o) =>
         `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
         [o.address, o.price, o.size].join(' | ') +
-        '\n\n'
+        '\n\n',
     );
     /**
      * This is to not break the rate limit. It is to only send 1 message per second

lib/provider/einsAImmobilien.js

@@ -1,18 +1,40 @@
-import utils from '../utils.js';
+import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
+
 function normalize(o) {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
   if (o.rooms != null) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-  return Object.assign(o, { size, link });
+  const price = normalizePrice(o.price);
+  const id = buildHash(o.id, price);
+  return Object.assign(o, { id, price, size, link });
+}
+
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
 }
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
   url: null,
   crawlContainer: '.tabelle',
@@ -23,7 +45,6 @@ const config = {
     size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
     rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
     title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immobilienDe.js

@@ -1,4 +1,4 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
@@ -7,12 +7,12 @@ function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
 function normalize(o) {
-  const id = parseId(shortenLink(o.link));
   const size = o.size || 'N/A m²';
   const price = o.price || 'N/A €';
   const title = o.title || 'No title available';
   const address = o.address || 'No address available';
   const link = shortenLink(o.link);
+  const id = buildHash(parseId(shortenLink(o.link)), o.price);
   return Object.assign(o, { id, price, size, title, address, link });
 }
 function applyBlacklist(o) {

lib/provider/immonet.js

@@ -1,12 +1,12 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function normalize(o) {
-  const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
   const link = o.id;
+  const id = buildHash(o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length), price);
   return Object.assign(o, { id, address, price, size, title, link });
 }
 function applyBlacklist(o) {

lib/provider/immoscout.js

@@ -1,4 +1,4 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
@@ -6,8 +6,9 @@ function nullOrEmpty(val) {
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
-  return Object.assign(o, { title, address, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, title, address, link });
 }
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
@@ -20,7 +21,7 @@ const config = {
     id: '.result-list-entry@data-obid | int',
     price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
     size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
+    title: '.result-list-entry .result-list-entry__brand-title-container h2 | removeNewline | trim',
     link: '.result-list-entry .result-list-entry__brand-title-container@href',
     address: '.result-list-entry .result-list-entry__map-link',
   },

lib/provider/immoswp.js

@@ -1,44 +1,48 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+    const size = o.size || 'N/A m²';
-  const size = o.size || 'N/A m²';
+    const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
-  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+    const title = o.title || 'No title available';
-  const address = o.address || 'No address available';
+    const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
-  const title = o.title || 'No title available';
+    const link = `https://immo.swp.de/immobilien/${immoId}`;
-  const link = `https://immo.swp.de/immobilien/${id}`;
+    const description = o.description;
-  const description = o.description;
+    const id = buildHash(immoId, price);
-  return Object.assign(o, { id, address, price, size, title, link, description });
+    return Object.assign(o, {id, price, size, title, link, description});
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
+    return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.js-serp-item',
+    crawlContainer: '.js-serp-item',
-  sortByDateParam: 's=most_recently_updated_first',
+    sortByDateParam: 's=most_recently_updated_first',
-  crawlFields: {
+    crawlFields: {
-    id: '@id',
+        id: '.js-bookmark-btn@data-id',
-    price: 'div.item__spec.item-spec-price | trim',
+        price: 'div.align-items-start div:first-child | trim',
-    size: 'div.item__spec.item-spec-area | trim',
+        size: 'div.align-items-start div:nth-child(3) | trim',
-    title: 'a.js-item-title-link@title',
+        title: '.card-title h2 | trim',
-    address: 'div.item__locality | removeNewline | trim',
+        link: '.ci-search-result__link@href',
-    description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
+        description: '.js-show-more-item-sm | removeNewline | trim',
-  },
+    },
-  paginate: 'li.page-item.pagination__item a.page-link@href',
+    paginate: 'li.page-item.pagination__item a.page-link@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Immo Südwest Presse',
+    name: 'Immo Südwest Presse',
-  baseUrl: 'https://immo.swp.de/',
+    baseUrl: 'https://immo.swp.de/',
-  id: 'immoswp',
+    id: 'immoswp',
 };
-export { config };
+export {config};

lib/provider/immowelt.js

@@ -1,24 +1,30 @@
-import utils from '../utils.js';
+import utils, { buildHash } from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  return o;
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id });
 }
+
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
   url: null,
-  crawlContainer: "div[class^='EstateItem-']",
+  crawlContainer:
-  sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
+    'div[data-testid="serp-card-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"])',
+  sortByDateParam: 'order=DateDesc',
   crawlFields: {
     id: 'a@id',
-    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
+    price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
-    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: "div[class^='FactsMain-'] h2",
+    title: '.css-1cbj9xw',
     link: 'a@href',
-    address: "div[class^='estateFacts-'] span | removeNewline | trim",
+    address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },
   paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,

lib/provider/kleinanzeigen.js

@@ -1,44 +1,49 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
+
 function normalize(o) {
-  const size = o.size || '--- m²';
+    const size = o.size || '--- m²';
-  return Object.assign(o, { size });
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id, size});
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  const isBlacklistedDistrict =
+    const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
+        appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
+    return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '#srchrslt-adtable .ad-listitem ',
+    crawlContainer: '#srchrslt-adtable .ad-listitem ',
-  //sort by date is standard oO
+    //sort by date is standard oO
-  sortByDateParam: null,
+    sortByDateParam: null,
-  crawlFields: {
+    crawlFields: {
-    id: '.aditem@data-adid | int',
+        id: '.aditem@data-adid | int',
-    price: '.aditem-main--middle--price | removeNewline | trim',
+        price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
-    size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
+        size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
-    title: '.aditem-main .text-module-begin a | removeNewline | trim',
+        title: '.aditem-main .text-module-begin a | removeNewline | trim',
-    link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
+        link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
-    description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
+        description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
-    address: '.aditem-main--top--left | trim | removeNewline',
+        address: '.aditem-main--top--left | trim | removeNewline',
-  },
+    },
-  paginate: '#srchrslt-pagination .pagination-next@href',
+    paginate: '#srchrslt-pagination .pagination-next@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const metaInformation = {
-  name: 'Ebay Kleinanzeigen',
+    name: 'Ebay Kleinanzeigen',
-  baseUrl: 'https://www.kleinanzeigen.de/',
+    baseUrl: 'https://www.kleinanzeigen.de/',
-  id: 'kleinanzeigen',
+    id: 'kleinanzeigen',
 };
 export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlacklistedDistricts = blacklistedDistricts || [];
+    appliedBlacklistedDistricts = blacklistedDistricts || [];
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
-export { config };
+export {config};

lib/provider/neubauKompass.js

@@ -1,34 +1,44 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
+function nullOrEmpty(val) {
+    return val == null || val.length === 0;
+}
+
 function normalize(o) {
-  return o;
+    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id, link});
 }
+
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
+    return !utils.isOneOf(o.title, appliedBlackList);
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.nbk-container >div article',
+    crawlContainer: '.nbk-container >div article',
-  sortByDateParam: 'Sortierung=Id&Richtung=DESC',
+    sortByDateParam: 'Sortierung=Id&Richtung=DESC',
-  crawlFields: {
+    crawlFields: {
-    id: '@id',
+        id: '@id',
-    title: 'a.nbk-truncate@title | removeNewline | trim',
+        title: 'a.nbk-truncate@title | removeNewline | trim',
-    link: 'a.nbk-truncate@href',
+        link: 'a.nbk-truncate@href',
-    address: 'p.nbk-truncate | removeNewline | trim',
+        address: 'p.nbk-truncate | removeNewline | trim',
-    price: 'p.nbk-mb-0 | removeNewline | trim',
+        price: 'p.nbk-mb-0 | removeNewline | trim',
-  },
+    },
-  paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
+    paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Neubau Kompass',
+    name: 'Neubau Kompass',
-  baseUrl: 'https://www.neubaukompass.de/',
+    baseUrl: 'https://www.neubaukompass.de/',
-  id: 'neubauKompass',
+    id: 'neubauKompass',
 };
-export { config };
+export {config};

lib/provider/wgGesucht.js

@@ -1,36 +1,41 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  return o;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id});
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  return o.id != null && titleNotBlacklisted && descNotBlacklisted;
+    return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '#main_column .wgg_card',
+    crawlContainer: '#main_column .wgg_card',
-  sortByDateParam: 'sort_column=0&sort_order=0',
+    sortByDateParam: 'sort_column=0&sort_order=0',
-  crawlFields: {
+    crawlFields: {
-    id: '@data-id',
+        id: '@data-id',
-    details: '.row .noprint .col-xs-11 |removeNewline |trim',
+        details: '.row .noprint .col-xs-11 |removeNewline |trim',
-    price: '.middle .col-xs-3 |removeNewline |trim',
+        price: '.middle .col-xs-3 |removeNewline |trim',
-    size: '.middle .text-right |removeNewline |trim',
+        size: '.middle .text-right |removeNewline |trim',
-    title: '.truncate_title a |removeNewline |trim',
+        title: '.truncate_title a |removeNewline |trim',
-    link: '.truncate_title a@href',
+        link: '.truncate_title a@href',
-  },
+    },
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Wg gesucht',
+    name: 'Wg gesucht',
-  baseUrl: 'https://www.wg-gesucht.de/',
+    baseUrl: 'https://www.wg-gesucht.de/',
-  id: 'wgGesucht',
+    id: 'wgGesucht',
 };
-export { config };
+export {config};

lib/services/requestDriver.js

@@ -24,13 +24,16 @@ function makeDriver(headers = {}) {
         },
       });
       const result = await response.text();
+      if (EXPECTED_STATUS_CODES.includes(response.status)) {
+        throw new Error(`${response.status}`);
+      }
       if (cookies.length === 0) {
         cookies = response.headers.raw()['set-cookie'] || [];
       }
       callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
-      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status) && !EXPECTED_STATUS_CODES.includes(Number(exception.message))) {
         console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
         callback(null, []);
         return;

lib/services/scrapingAnt.js

@@ -1,5 +1,6 @@
 import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
 import { metaInformation as immoNetInfo } from '../provider/immonet.js';
+import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
 import { config } from '../utils.js';
 
 const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
@@ -7,7 +8,7 @@ const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js
 ).toString('base64')}`;
 
 const needScrapingAnt = (id) => {
-  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
 };
 export const transformUrlForScrapingAnt = (url, id) => {
   let urlParams = '';

lib/utils.js

@@ -1,51 +1,69 @@
-import { dirname } from 'node:path';
+import {dirname} from 'node:path';
-import { fileURLToPath } from 'node:url';
+import {fileURLToPath} from 'node:url';
-import { readFile } from 'fs/promises';
+import {readFile} from 'fs/promises';
+import {createHash} from 'crypto';
 
 function isOneOf(word, arr) {
-  if (arr == null || arr.length === 0) {
+    if (arr == null || arr.length === 0) {
-    return false;
+        return false;
-  }
+    }
-  const expression = String.raw`\b(${arr.join('|')})\b`;
+    const expression = String.raw`\b(${arr.join('|')})\b`;
-  const blacklist = new RegExp(expression, 'ig');
+    const blacklist = new RegExp(expression, 'ig');
-  return blacklist.test(word);
+    return blacklist.test(word);
 }
+
 function nullOrEmpty(val) {
-  return val == null || val.length === 0;
+    return val == null || val.length === 0;
 }
+
 function timeStringToMs(timeString, now) {
-  const d = new Date(now);
+    const d = new Date(now);
-  const parts = timeString.split(':');
+    const parts = timeString.split(':');
-  d.setHours(parts[0]);
+    d.setHours(parts[0]);
-  d.setMinutes(parts[1]);
+    d.setMinutes(parts[1]);
-  d.setSeconds(0);
+    d.setSeconds(0);
-  return d.getTime();
+    return d.getTime();
 }
+
 function duringWorkingHoursOrNotSet(config, now) {
-  const { workingHours } = config;
+    const {workingHours} = config;
-  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+    if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
-    return true;
+        return true;
-  }
+    }
-  const toDate = timeStringToMs(workingHours.to, now);
+    const toDate = timeStringToMs(workingHours.to, now);
-  const fromDate = timeStringToMs(workingHours.from, now);
+    const fromDate = timeStringToMs(workingHours.from, now);
-  return fromDate <= now && toDate >= now;
+    return fromDate <= now && toDate >= now;
 }
 
 function getDirName() {
-  return dirname(fileURLToPath(import.meta.url));
+    return dirname(fileURLToPath(import.meta.url));
+}
+
+function buildHash(...inputs) {
+    if (inputs == null) {
+        return null;
+    }
+    const cleaned = inputs.filter(i => i != null && i.length > 0);
+    if (cleaned.length === 0) {
+        return null;
+    }
+    return createHash('sha256')
+        .update(cleaned.join(','))
+        .digest('hex');
 }
 
 const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
 
-export { isOneOf };
+export {isOneOf};
-export { nullOrEmpty };
+export {nullOrEmpty};
-export { duringWorkingHoursOrNotSet };
+export {duringWorkingHoursOrNotSet};
-export { getDirName };
+export {getDirName};
-export { config };
+export {config};
+export {buildHash};
 export default {
-  isOneOf,
+    isOneOf,
-  nullOrEmpty,
+    nullOrEmpty,
-  duringWorkingHoursOrNotSet,
+    duringWorkingHoursOrNotSet,
-  getDirName,
+    getDirName,
-  config,
+    config,
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "7.3.2",
+  "version": "10.2.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -45,7 +45,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=20.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -55,54 +55,54 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.42.4",
+    "@douyinfe/semi-ui": "2.68.3",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.7.0",
+    "@sendgrid/mail": "8.1.4",
-    "@vitejs/plugin-react": "4.0.4",
+    "@vitejs/plugin-react": "4.3.3",
-    "better-sqlite3": "8.6.0",
+    "better-sqlite3": "^11.5.0",
-    "body-parser": "1.20.2",
+    "body-parser": "1.20.3",
-    "cookie-session": "2.0.0",
+    "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "11.1.0",
+    "highcharts": "11.4.8",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "4.0.2",
+    "nanoid": "5.0.8",
     "node-fetch": "3.3.2",
-    "node-mailjet": "6.0.4",
+    "node-mailjet": "6.0.6",
-    "query-string": "8.1.0",
+    "query-string": "9.1.1",
-    "react": "18.2.0",
+    "react": "18.3.1",
-    "react-dom": "18.2.0",
+    "react-dom": "18.3.1",
-    "react-redux": "8.1.2",
+    "react-redux": "9.1.2",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "redux": "4.2.1",
+    "redux": "5.0.1",
-    "redux-thunk": "2.4.2",
+    "redux-thunk": "3.1.0",
-    "restana": "4.9.7",
+    "restana": "4.9.9",
-    "serve-static": "1.15.0",
+    "serve-static": "1.16.2",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "4.4.9",
+    "vite": "5.4.10",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.22.15",
+    "@babel/core": "7.26.0",
-    "@babel/eslint-parser": "7.22.15",
+    "@babel/eslint-parser": "7.25.9",
-    "@babel/preset-env": "7.22.15",
+    "@babel/preset-env": "7.26.0",
-    "@babel/preset-react": "7.22.15",
+    "@babel/preset-react": "7.25.9",
-    "chai": "4.3.8",
+    "chai": "5.1.2",
-    "eslint": "8.48.0",
+    "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.33.2",
+    "eslint-plugin-react": "7.37.2",
-    "esmock": "2.4.0",
+    "esmock": "2.6.9",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.2.0",
     "lint-staged": "13.2.2",
-    "mocha": "10.2.0",
+    "mocha": "10.8.2",
-    "prettier": "2.8.8",
+    "prettier": "3.3.3",
     "redux-logger": "3.0.6"
   }
 }

test/provider/einsAImmobilien.test.js

@@ -1,11 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { providerConfig, mockFredy } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/einsAImmobilien.js';
 
-const expect = chai.expect;
-
 describe('#einsAImmobilien testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -22,7 +20,7 @@ describe('#einsAImmobilien testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');

test/provider/immobilienDe.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { providerConfig, mockFredy } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immobilienDe.js';
-const expect = chai.expect;
+
 describe('#immobilien.de testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immonet.test.js

@@ -1,10 +1,10 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immonet.js';
 import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const expect = chai.expect;
+
 describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immoscout.test.js

@@ -1,10 +1,10 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoscout.js';
 import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const expect = chai.expect;
+
 describe('#immoscout testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immoswp.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoswp.js';
-const expect = chai.expect;
+
 describe('#immoswp testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -25,12 +25,10 @@ describe('#immoswp testsuite()', () => {
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://immo.swp.de');
-          expect(notify.address).to.be.not.empty;
         });
         resolve();
       });

test/provider/immowelt.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immowelt.js';
-const expect = chai.expect;
+
 describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/kleinanzeigen.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/kleinanzeigen.js';
-const expect = chai.expect;
+
 describe('#kleinanzeigen testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -20,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');

test/provider/neubauKompass.test.js

@@ -1,36 +1,44 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
+import {get} from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
+import {mockFredy, providerConfig} from '../utils.js';
-import chai from 'chai';
+import {expect} from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
-const expect = chai.expect;
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
+
 describe('#neubauKompass testsuite()', () => {
-  after(() => {
+    after(() => {
-    similarityCache.stopCacheCleanup();
+        similarityCache.stopCacheCleanup();
-  });
+    });
-  provider.init(providerConfig.neubauKompass, [], []);
+    provider.init(providerConfig.neubauKompass, [], []);
-  it('should test neubauKompass provider', async () => {
+    it('should test neubauKompass provider', async () => {
-    const Fredy = await mockFredy();
+        const Fredy = await mockFredy();
-    return await new Promise((resolve) => {
+        return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
+            if (!scrapingAnt.isScrapingAntApiKeySet()) {
-      fredy.execute().then((listing) => {
+                /* eslint-disable no-console */
-        expect(listing).to.be.a('array');
+                console.info('Skipping Neubaukompass test as ScrapingAnt Api Key is not set.');
-        const notificationObj = get();
+                /* eslint-enable no-console */
-        expect(notificationObj.serviceName).to.equal('neubauKompass');
+                resolve();
-        notificationObj.payload.forEach((notify) => {
+                return;
-          expect(notify).to.be.a('object');
+            }
-          /** check the actual structure **/
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
-          expect(notify.id).to.be.a('string');
+            fredy.execute().then((listing) => {
-          expect(notify.title).to.be.a('string');
+                expect(listing).to.be.a('array');
-          expect(notify.link).to.be.a('string');
+                const notificationObj = get();
-          expect(notify.address).to.be.a('string');
+                expect(notificationObj.serviceName).to.equal('neubauKompass');
-          /** check the values if possible **/
+                notificationObj.payload.forEach((notify) => {
-          expect(notify.title).to.be.not.empty;
+                    expect(notify).to.be.a('object');
-          expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    /** check the actual structure **/
-          expect(notify.address).to.be.not.empty;
+                    expect(notify.id).to.be.a('string');
-        });
+                    expect(notify.title).to.be.a('string');
-        resolve();
+                    expect(notify.link).to.be.a('string');
-      });
+                    expect(notify.address).to.be.a('string');
+                    /** check the values if possible **/
+                    expect(notify.title).to.be.not.empty;
+                    expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    expect(notify.address).to.be.not.empty;
+                });
+                resolve();
+            });
+        });
     });
-  });
 });

test/provider/testProvider.json

@@ -13,7 +13,7 @@
     "enabled": true
   },
   "immowelt": {
-    "url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?d=true&rmi=3&sd=DESC&sf=TIMESTAMP&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
     "enabled": true
   },
   "immoscout": {

test/provider/utils.test.js

@@ -1,7 +1,7 @@
 import utils from '../../lib/utils.js';
 import assert from 'assert';
-import chai from 'chai';
+import { expect } from 'chai';
-const expect = chai.expect;
+
 const fakeWorkingHoursConfig = (from, to) => ({
   workingHours: {
     to,

test/provider/wgGesucht.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/wgGesucht.js';
-const expect = chai.expect;
+
 describe('#wgGesucht testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/queryStringMutator/queryStringMutator.test.js

@@ -1,16 +1,15 @@
 import fs from 'fs';
-import chai from 'chai';
+import { expect } from 'chai';
 import { readFile } from 'fs/promises';
 import mutator from '../../lib/services/queryStringMutator.js';
 import queryString from 'query-string';
-const expect = chai.expect;
 
 const data = await readFile(new URL('./testData.json', import.meta.url));
 
 const testData = JSON.parse(data);
 
 let _provider = await Promise.all(
-  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
+  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`)),
 );
 
 /**

test/queryStringMutator/testData.json

@@ -1,7 +1,7 @@
 [
   {
-    "url": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=PRIMARY_PRICE_AMOUNT&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
-    "shouldBecome": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=TIMESTAMP&sp=1",
+    "shouldBecome": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350&order=DateDesc",
     "id": "immowelt"
   },
   {

test/similarity/similarity.test.js

@@ -1,6 +1,6 @@
 import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
-import chai from 'chai';
+import { expect } from 'chai';
-const expect = chai.expect;
+
 describe('similarityCheck', () => {
   describe('#similarityCheck()', () => {
     it('should be false', () => {
@@ -29,10 +29,10 @@ describe('similarityCheck', () => {
     it('should be false', () => {
       const check = new SimilarityCacheEntry(0);
       check.setCacheEntry(
-        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.'
+        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.',
       );
       check.setCacheEntry(
-        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
+        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
     });
   });

test/utils/utils.test.js

@@ -0,0 +1,16 @@
+import { expect } from 'chai';
+import {buildHash} from '../../lib/utils.js';
+
+describe('utilsCheck', () => {
+  describe('#utilsCheck()', () => {
+      it('should be null when null input', () => {
+          expect(buildHash(null)).to.be.null;
+      });
+      it('should be null when null empty', () => {
+          expect(buildHash('')).to.be.null;
+      });
+      it('should return a value', () => {
+          expect(buildHash('bla', '', null)).to.be.a.string;
+      });
+  });
+});

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -184,8 +184,7 @@ const GeneralSettings = function GeneralSettings() {
                     more likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
                     <h4>Residential-Proxy</h4>
                     High-quality proxy server located in one of the real people houses across the world. Datacenter
-                    proxies are faster and more likely to success, but they are more expensive. A call with a datacenter
+                    proxies are faster and more likely to success, but they are more expensive.
-                    proxy cost 250 credits.
                     <br />
                     <br />
                     <b>

ui/src/views/jobs/ProcessingTimes.jsx

@@ -45,7 +45,7 @@ export default function ProcessingTimes({ processingTimes }) {
               {format(new Date(processingTimes.scrapingAntData.end_date))}
               <br />
               Credits: {processingTimes.scrapingAntData.remained_credits}/
-              {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
+              {processingTimes.scrapingAntData.plan_total_credits}
             </p>
             If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
             <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              If you chose Immoscout, Immonet or NeubauKompass as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
               (See readme)
             </p>
             <p>