Update README.md

* Ability to send tracking information

.github/workflows/docker.yml

@@ -44,3 +44,4 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64

.github/workflows/stales.yml

@@ -0,0 +1,21 @@
+name: "Close stale issues and PRs"
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          days-before-stale: 30
+          days-before-close: 7
+          stale-issue-message: "This issue has been automatically marked as stale due to inactivity."
+          stale-pr-message: "This PR has been automatically marked as stale due to inactivity."
+          close-issue-message: "Closing this issue due to prolonged inactivity."
+          close-pr-message: "Closing this PR due to prolonged inactivity."
+          exempt-issue-labels: "keep-open"
+          exempt-pr-labels: "keep-open"
+          only: "pulls"

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 18
+          node-version: 20
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

.travis.yml

@@ -1,2 +0,0 @@
-sudo: false
-language: node_js

Dockerfile

@@ -1,19 +1,20 @@
-# syntax=docker/dockerfile:1.3
+FROM node:20
-FROM node:18-alpine AS builder
+
-COPY --chown=1000:1000 . /fredy
 WORKDIR  /fredy
-USER 1000
+
+COPY . /fredy
+
 RUN yarn install
+
+RUN yarn global add pm2
+
 RUN yarn run prod
 
-FROM node:16-alpine
-COPY --from=builder --chown=1000:1000 /fredy /fredy
 RUN mkdir /db /conf && \
   chown 1000:1000 /db /conf && \
   chmod 777 -R /db/ && \
   ln -s /db /fredy/db && ln -s /conf /fredy/conf
+
 EXPOSE 9998
-USER 1000
+
-VOLUME [ "/conf", "/db" ]
+CMD pm2-runtime index.js
-WORKDIR  /fredy
-CMD node index.js --no-daemon

README.md

@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use Node.js 18 or above
+- Make sure to use Node.js 20 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -78,17 +78,20 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-### Immoscout / Immonet
+### Immoscout / Immonet / NeubauKompass
-I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+I have added **experimental** support for Immoscout, Immonet and NeubauKompass. They all are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
 To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
 The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
 
 If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
 
-### Contribution guidelines  
+# Analytics
-
+Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 
-See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
+Before you freak out, let me explain...         
+If you agree, Fredy will send a ping to my Mixpanel project each time it runs.   
+The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.</p>
+**Thanks**🤘
 
 # Docker   
 Use the Dockerfile in this repository to build an image.  
@@ -109,6 +112,15 @@ Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+### 👐 Contributing
+Thanks to all the people who already contributed!
+
+<a href="https://github.com/orangecoding/fredy/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
+</a>
+
+See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
+
 ## Logs  
 
 You can browse the logs with  `docker logs fredy -f`.

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.3'
+version: '3.8'
 services:
   fredy:
     container_name: fredy
@@ -13,3 +13,4 @@ services:
       - ./db:/db
     ports:
         - 9998:9998
+    restart: unless-stopped

index.js

@@ -6,6 +6,7 @@ import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyRuntime from './lib/FredyRuntime.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import './lib/api/api.js';
+import {track} from './lib/services/tracking/Tracker.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
@@ -25,6 +26,7 @@ setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
     if (isDuringWorkingHoursOrNotSet) {
+      track();
       config.lastRun = Date.now();
       jobStorage
         .getJobs()

lib/FredyRuntime.js

@@ -87,7 +87,10 @@ class FredyRuntime {
     return listings.map(this._providerConfig.normalize);
   }
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);

lib/api/routes/generalSettingsRoute.js

@@ -1,5 +1,5 @@
 import restana from 'restana';
-import { config, getDirName } from '../../utils.js';
+import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
 import fs from 'fs';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
@@ -10,7 +10,9 @@ generalSettingsRouter.get('/', async (req, res) => {
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
   try {
-    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
+    await refreshConfig();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));

lib/api/routes/jobRouter.js

@@ -5,6 +5,7 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import * as immoscoutProvider from '../../provider/immoscout.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
+import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -25,8 +26,8 @@ jobRouter.get('/', async (req, res) => {
   res.send();
 });
 jobRouter.get('/processingTimes', async (req, res) => {
-  let scrapingAntData = null;
+  let scrapingAntData = {};
-  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+  if (isScrapingAntApiKeySet()) {
     try {
       const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
       scrapingAntData = await response.json();
@@ -38,6 +39,7 @@ jobRouter.get('/processingTimes', async (req, res) => {
     interval: config.interval,
     lastRun: config.lastRun || null,
     scrapingAntData,
+    error: scrapingAntData?.detail == null ? null : scrapingAntData?.detail
   };
   res.send();
 });

lib/defaultConfig.js

@@ -0,0 +1,8 @@
+export const DEFAULT_CONFIG = {
+            'interval': '60',
+            'port': 9998,
+            'scrapingAnt': {'apiKey': '', 'proxy': 'datacenter'},
+            'workingHours': {'from': '', 'to': ''},
+            'demoMode': false,
+            'analyticsEnabled': null
+    };

lib/notification/adapter/apprise.js

@@ -0,0 +1,36 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        body: message,
+        title: title,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'apprise',
+  name: 'Apprise',
+  readme: markdown2Html('lib/notification/adapter/apprise.md'),
+  description: 'Fredy will send new listings to your Apprise instance.',
+  fields: {
+    server: {
+      type: 'text',
+      label: 'Server',
+      description: 'The server URL to send the notification to.',
+    },
+  },
+};

lib/notification/adapter/apprise.md

@@ -0,0 +1,3 @@
+### Apprise Adapter
+
+Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.

lib/notification/adapter/pushover.js

@@ -0,0 +1,50 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+    return fetch('https://api.pushover.net/1/messages.json', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        token: token,
+        user: user,
+        message: message,
+        device: device,
+        title: title,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+
+export const config = {
+  id: 'pushover',
+  name: 'Pushover',
+  readme: markdown2Html('lib/notification/adapter/pushover.md'),
+  description: 'Fredy will send new listings to your mobile using Pushover.',
+  fields: {
+    token: {
+      type: 'text',
+      label: 'API token',
+      description: 'Your application\'s API token.',
+    },
+    user: {
+      type: 'text',
+      label: 'User key',
+      description: 'Your user/group key.',
+    },
+    device: {
+      type: 'text',
+      label: 'Device name',
+      description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
+    },
+  },
+};

lib/notification/adapter/pushover.md

@@ -0,0 +1,5 @@
+### Pushover Adapter
+
+Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+
+After setting up the application, please enter both your newly created User key and API token.

lib/notification/adapter/sqlite.js

@@ -0,0 +1,25 @@
+import { markdown2Html } from '../../services/markdown.js';
+import Database from 'better-sqlite3';
+export const send = ({ serviceName, newListings, jobKey }) => {
+  const db = new Database('db/listings.db');
+  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+export const config = {
+  id: 'sqlite',
+  name: 'Sqlite',
+  description: 'This adapter stores listings in a local sqlite3 database.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,7 @@
+### Sqlite Adapter
+This adapter stores search results in a sqlite database located in db/listings.db. This file can be used for further analysis later on.   
+
+Fields are:
+```
+['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description']
+```

lib/provider/einsAImmobilien.js

@@ -1,18 +1,40 @@
-import utils from '../utils.js';
+import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
+
 function normalize(o) {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
   if (o.rooms != null) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-  return Object.assign(o, { size, link });
+  const price = normalizePrice(o.price);
+  const id = buildHash(o.id, price);
+  return Object.assign(o, { id, price, size, link });
+}
+
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
 }
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
   url: null,
   crawlContainer: '.tabelle',
@@ -23,7 +45,6 @@ const config = {
     size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
     rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
     title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immobilienDe.js

@@ -1,4 +1,4 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
@@ -7,12 +7,12 @@ function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
 function normalize(o) {
-  const id = parseId(shortenLink(o.link));
   const size = o.size || 'N/A m²';
   const price = o.price || 'N/A €';
   const title = o.title || 'No title available';
   const address = o.address || 'No address available';
   const link = shortenLink(o.link);
+  const id = buildHash(parseId(shortenLink(o.link)), o.price);
   return Object.assign(o, { id, price, size, title, address, link });
 }
 function applyBlacklist(o) {

lib/provider/immonet.js

@@ -1,12 +1,12 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function normalize(o) {
-  const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
   const link = o.id;
+  const id = buildHash(o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length), price);
   return Object.assign(o, { id, address, price, size, title, link });
 }
 function applyBlacklist(o) {

lib/provider/immoscout.js

@@ -1,4 +1,4 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
@@ -6,8 +6,9 @@ function nullOrEmpty(val) {
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
-  return Object.assign(o, { title, address, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, title, address, link });
 }
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);

lib/provider/immoswp.js

@@ -1,44 +1,48 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+    const size = o.size || 'N/A m²';
-  const size = o.size || 'N/A m²';
+    const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
-  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+    const title = o.title || 'No title available';
-  const address = o.address || 'No address available';
+    const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
-  const title = o.title || 'No title available';
+    const link = `https://immo.swp.de/immobilien/${immoId}`;
-  const link = `https://immo.swp.de/immobilien/${id}`;
+    const description = o.description;
-  const description = o.description;
+    const id = buildHash(immoId, price);
-  return Object.assign(o, { id, address, price, size, title, link, description });
+    return Object.assign(o, {id, price, size, title, link, description});
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
+    return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.js-serp-item',
+    crawlContainer: '.js-serp-item',
-  sortByDateParam: 's=most_recently_updated_first',
+    sortByDateParam: 's=most_recently_updated_first',
-  crawlFields: {
+    crawlFields: {
-    id: '@id',
+        id: '.js-bookmark-btn@data-id',
-    price: 'div.item__spec.item-spec-price | trim',
+        price: 'div.align-items-start div:first-child | trim',
-    size: 'div.item__spec.item-spec-area | trim',
+        size: 'div.align-items-start div:nth-child(3) | trim',
-    title: 'a.js-item-title-link@title',
+        title: '.card-title h2 | trim',
-    address: 'div.item__locality | removeNewline | trim',
+        link: '.ci-search-result__link@href',
-    description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
+        description: '.js-show-more-item-sm | removeNewline | trim',
-  },
+    },
-  paginate: 'li.page-item.pagination__item a.page-link@href',
+    paginate: 'li.page-item.pagination__item a.page-link@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Immo Südwest Presse',
+    name: 'Immo Südwest Presse',
-  baseUrl: 'https://immo.swp.de/',
+    baseUrl: 'https://immo.swp.de/',
-  id: 'immoswp',
+    id: 'immoswp',
 };
-export { config };
+export {config};

lib/provider/immowelt.js

@@ -1,24 +1,30 @@
-import utils from '../utils.js';
+import utils, { buildHash } from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  return o;
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id });
 }
+
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
   url: null,
-  crawlContainer: "div[class^='EstateItem-']",
+  crawlContainer:
-  sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
+    'div[data-testid="serp-card-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"])',
+  sortByDateParam: 'order=DateDesc',
   crawlFields: {
     id: 'a@id',
-    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
+    price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
-    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: "div[class^='FactsMain-'] h2",
+    title: '.css-1cbj9xw',
     link: 'a@href',
-    address: "div[class^='estateFacts-'] span | removeNewline | trim",
+    address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },
   paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,

lib/provider/kleinanzeigen.js

@@ -1,44 +1,49 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
+
 function normalize(o) {
-  const size = o.size || '--- m²';
+    const size = o.size || '--- m²';
-  return Object.assign(o, { size });
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id, size});
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  const isBlacklistedDistrict =
+    const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
+        appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
+    return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '#srchrslt-adtable .ad-listitem ',
+    crawlContainer: '#srchrslt-adtable .ad-listitem ',
-  //sort by date is standard oO
+    //sort by date is standard oO
-  sortByDateParam: null,
+    sortByDateParam: null,
-  crawlFields: {
+    crawlFields: {
-    id: '.aditem@data-adid | int',
+        id: '.aditem@data-adid | int',
-    price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
+        price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
-    size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
+        size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
-    title: '.aditem-main .text-module-begin a | removeNewline | trim',
+        title: '.aditem-main .text-module-begin a | removeNewline | trim',
-    link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
+        link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
-    description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
+        description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
-    address: '.aditem-main--top--left | trim | removeNewline',
+        address: '.aditem-main--top--left | trim | removeNewline',
-  },
+    },
-  paginate: '#srchrslt-pagination .pagination-next@href',
+    paginate: '#srchrslt-pagination .pagination-next@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const metaInformation = {
-  name: 'Ebay Kleinanzeigen',
+    name: 'Ebay Kleinanzeigen',
-  baseUrl: 'https://www.kleinanzeigen.de/',
+    baseUrl: 'https://www.kleinanzeigen.de/',
-  id: 'kleinanzeigen',
+    id: 'kleinanzeigen',
 };
 export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlacklistedDistricts = blacklistedDistricts || [];
+    appliedBlacklistedDistricts = blacklistedDistricts || [];
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
-export { config };
+export {config};

lib/provider/neubauKompass.js

@@ -1,34 +1,44 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
+function nullOrEmpty(val) {
+    return val == null || val.length === 0;
+}
+
 function normalize(o) {
-  return o;
+    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id, link});
 }
+
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
+    return !utils.isOneOf(o.title, appliedBlackList);
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.nbk-container >div article',
+    crawlContainer: '.nbk-container >div article',
-  sortByDateParam: 'Sortierung=Id&Richtung=DESC',
+    sortByDateParam: 'Sortierung=Id&Richtung=DESC',
-  crawlFields: {
+    crawlFields: {
-    id: '@id',
+        id: '@id',
-    title: 'a.nbk-truncate@title | removeNewline | trim',
+        title: 'a.nbk-truncate@title | removeNewline | trim',
-    link: 'a.nbk-truncate@href',
+        link: 'a.nbk-truncate@href',
-    address: 'p.nbk-truncate | removeNewline | trim',
+        address: 'p.nbk-truncate | removeNewline | trim',
-    price: 'p.nbk-mb-0 | removeNewline | trim',
+        price: 'p.nbk-mb-0 | removeNewline | trim',
-  },
+    },
-  paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
+    paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Neubau Kompass',
+    name: 'Neubau Kompass',
-  baseUrl: 'https://www.neubaukompass.de/',
+    baseUrl: 'https://www.neubaukompass.de/',
-  id: 'neubauKompass',
+    id: 'neubauKompass',
 };
-export { config };
+export {config};

lib/provider/wgGesucht.js

@@ -1,36 +1,41 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  return o;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id});
 }
+
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  return o.id != null && titleNotBlacklisted && descNotBlacklisted;
+    return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '#main_column .wgg_card',
+    crawlContainer: '#main_column .wgg_card',
-  sortByDateParam: 'sort_column=0&sort_order=0',
+    sortByDateParam: 'sort_column=0&sort_order=0',
-  crawlFields: {
+    crawlFields: {
-    id: '@data-id',
+        id: '@data-id',
-    details: '.row .noprint .col-xs-11 |removeNewline |trim',
+        details: '.row .noprint .col-xs-11 |removeNewline |trim',
-    price: '.middle .col-xs-3 |removeNewline |trim',
+        price: '.middle .col-xs-3 |removeNewline |trim',
-    size: '.middle .text-right |removeNewline |trim',
+        size: '.middle .text-right |removeNewline |trim',
-    title: '.truncate_title a |removeNewline |trim',
+        title: '.truncate_title a |removeNewline |trim',
-    link: '.truncate_title a@href',
+        link: '.truncate_title a@href',
-  },
+    },
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+    config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+    config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+    appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-  name: 'Wg gesucht',
+    name: 'Wg gesucht',
-  baseUrl: 'https://www.wg-gesucht.de/',
+    baseUrl: 'https://www.wg-gesucht.de/',
-  id: 'wgGesucht',
+    id: 'wgGesucht',
 };
-export { config };
+export {config};

lib/services/requestDriver.js

@@ -24,13 +24,16 @@ function makeDriver(headers = {}) {
         },
       });
       const result = await response.text();
+      if (EXPECTED_STATUS_CODES.includes(response.status)) {
+        throw new Error(`${response.status}`);
+      }
       if (cookies.length === 0) {
         cookies = response.headers.raw()['set-cookie'] || [];
       }
       callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
-      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status) && !EXPECTED_STATUS_CODES.includes(Number(exception.message))) {
         console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
         callback(null, []);
         return;

lib/services/scrapingAnt.js

@@ -1,5 +1,6 @@
 import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
 import { metaInformation as immoNetInfo } from '../provider/immonet.js';
+import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
 import { config } from '../utils.js';
 
 const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
@@ -7,7 +8,7 @@ const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js
 ).toString('base64')}`;
 
 const needScrapingAnt = (id) => {
-  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
 };
 export const transformUrlForScrapingAnt = (url, id) => {
   let urlParams = '';
@@ -21,7 +22,7 @@ export const transformUrlForScrapingAnt = (url, id) => {
   return url;
 };
 export const isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 8;
 };
 export const makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');

lib/services/tracking/Tracker.js

@@ -0,0 +1,42 @@
+import Mixpanel from 'mixpanel';
+import {getJobs} from '../storage/jobStorage.js';
+
+import {config} from '../../utils.js';
+
+export const track = function () {
+    //only send tracking information if the user allowed to do so.
+    if (config.analyticsEnabled) {
+
+        const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+
+        const activeProvider = new Set();
+        const activeAdapter = new Set();
+        const platform = process.platform;
+        const arch = process.arch;
+        const language = process.env.LANG || 'en';
+        const nodeVersion = process.version || 'N/A';
+
+        const jobs = getJobs();
+
+        if (jobs != null && jobs.length > 0) {
+            jobs.forEach(job => {
+                job.provider.forEach(provider => {
+                    activeProvider.add(provider.id);
+                });
+                job.notificationAdapter.forEach(adapter => {
+                    activeAdapter.add(adapter.id);
+                });
+            });
+
+            mixpanelTracker.track('fredy_tracking', {
+                adapter: Array.from(activeAdapter),
+                provider: Array.from(activeProvider),
+                isDemo: config.demoMode,
+                platform,
+                arch,
+                nodeVersion,
+                language
+            });
+        }
+    }
+};

lib/utils.js

@@ -1,51 +1,86 @@
-import { dirname } from 'node:path';
+import {dirname} from 'node:path';
-import { fileURLToPath } from 'node:url';
+import {fileURLToPath} from 'node:url';
-import { readFile } from 'fs/promises';
+import {readFile} from 'fs/promises';
+import {createHash} from 'crypto';
+import {DEFAULT_CONFIG} from './defaultConfig.js';
 
 function isOneOf(word, arr) {
-  if (arr == null || arr.length === 0) {
+    if (arr == null || arr.length === 0) {
-    return false;
+        return false;
-  }
+    }
-  const expression = String.raw`\b(${arr.join('|')})\b`;
+    const expression = String.raw`\b(${arr.join('|')})\b`;
-  const blacklist = new RegExp(expression, 'ig');
+    const blacklist = new RegExp(expression, 'ig');
-  return blacklist.test(word);
+    return blacklist.test(word);
 }
+
 function nullOrEmpty(val) {
-  return val == null || val.length === 0;
+    return val == null || val.length === 0;
 }
+
 function timeStringToMs(timeString, now) {
-  const d = new Date(now);
+    const d = new Date(now);
-  const parts = timeString.split(':');
+    const parts = timeString.split(':');
-  d.setHours(parts[0]);
+    d.setHours(parts[0]);
-  d.setMinutes(parts[1]);
+    d.setMinutes(parts[1]);
-  d.setSeconds(0);
+    d.setSeconds(0);
-  return d.getTime();
+    return d.getTime();
 }
+
 function duringWorkingHoursOrNotSet(config, now) {
-  const { workingHours } = config;
+    const {workingHours} = config;
-  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+    if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
-    return true;
+        return true;
-  }
+    }
-  const toDate = timeStringToMs(workingHours.to, now);
+    const toDate = timeStringToMs(workingHours.to, now);
-  const fromDate = timeStringToMs(workingHours.from, now);
+    const fromDate = timeStringToMs(workingHours.from, now);
-  return fromDate <= now && toDate >= now;
+    return fromDate <= now && toDate >= now;
 }
 
 function getDirName() {
-  return dirname(fileURLToPath(import.meta.url));
+    return dirname(fileURLToPath(import.meta.url));
 }
 
-const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+function buildHash(...inputs) {
+    if (inputs == null) {
+        return null;
+    }
+    const cleaned = inputs.filter(i => i != null && i.length > 0);
+    if (cleaned.length === 0) {
+        return null;
+    }
+    return createHash('sha256')
+        .update(cleaned.join(','))
+        .digest('hex');
+}
 
-export { isOneOf };
+let config = {};
-export { nullOrEmpty };
+export async function readConfigFromStorage(){
-export { duringWorkingHoursOrNotSet };
+    return JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
-export { getDirName };
+}
-export { config };
+
+export async function refreshConfig(){
+    try {
+        config = await readConfigFromStorage();
+        //backwards compatability...
+        config.analyticsEnabled ??= null;
+        config.demoMode ??= false;
+    } catch (error) {
+        config = {...DEFAULT_CONFIG};
+        console.error('Error reading config file', error);
+    }
+}
+await refreshConfig();
+
+export {isOneOf};
+export {nullOrEmpty};
+export {duringWorkingHoursOrNotSet};
+export {getDirName};
+export {config};
+export {buildHash};
 export default {
-  isOneOf,
+    isOneOf,
-  nullOrEmpty,
+    nullOrEmpty,
-  duringWorkingHoursOrNotSet,
+    duringWorkingHoursOrNotSet,
-  getDirName,
+    getDirName,
-  config,
+    config,
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "8.0.2",
+  "version": "10.3.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -11,11 +11,6 @@
     "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
     "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
-  },
   "type": "module",
   "lint-staged": {
     "*.js": [
@@ -45,7 +40,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=20.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -55,54 +50,55 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.52.0",
+    "@douyinfe/semi-ui": "2.69.2",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "8.1.0",
+    "@sendgrid/mail": "8.1.4",
-    "@vitejs/plugin-react": "4.2.1",
+    "@vitejs/plugin-react": "4.3.3",
-    "better-sqlite3": "8.6.0",
+    "better-sqlite3": "^11.5.0",
-    "body-parser": "1.20.2",
+    "body-parser": "1.20.3",
     "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "11.3.0",
+    "highcharts": "11.4.8",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "5.0.5",
+    "mixpanel": "^0.18.0",
+    "nanoid": "5.0.8",
     "node-fetch": "3.3.2",
-    "node-mailjet": "6.0.5",
+    "node-mailjet": "6.0.6",
-    "query-string": "8.2.0",
+    "query-string": "9.1.1",
-    "react": "18.2.0",
+    "react": "18.3.1",
-    "react-dom": "18.2.0",
+    "react-dom": "18.3.1",
-    "react-redux": "9.1.0",
+    "react-redux": "9.1.2",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
     "redux": "5.0.1",
     "redux-thunk": "3.1.0",
-    "restana": "4.9.7",
+    "restana": "4.9.9",
-    "serve-static": "1.15.0",
+    "serve-static": "1.16.2",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.0.12",
+    "vite": "5.4.11",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.23.9",
+    "@babel/core": "7.26.0",
-    "@babel/eslint-parser": "7.23.10",
+    "@babel/eslint-parser": "7.25.9",
-    "@babel/preset-env": "7.23.9",
+    "@babel/preset-env": "7.26.0",
-    "@babel/preset-react": "7.23.3",
+    "@babel/preset-react": "7.25.9",
-    "chai": "5.0.3",
+    "chai": "5.1.2",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.33.2",
+    "eslint-plugin-react": "7.37.2",
-    "esmock": "2.6.3",
+    "esmock": "2.6.9",
     "history": "5.3.0",
-    "husky": "4.3.8",
+    "husky": "9.1.7",
     "less": "4.2.0",
-    "lint-staged": "13.2.2",
+    "lint-staged": "15.2.10",
-    "mocha": "10.2.0",
+    "mocha": "10.8.2",
-    "prettier": "3.2.5",
+    "prettier": "3.3.3",
     "redux-logger": "3.0.6"
   }
 }

test/provider/einsAImmobilien.test.js

@@ -20,7 +20,7 @@ describe('#einsAImmobilien testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');

test/provider/immoswp.test.js

@@ -25,12 +25,10 @@ describe('#immoswp testsuite()', () => {
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://immo.swp.de');
-          expect(notify.address).to.be.not.empty;
         });
         resolve();
       });

test/provider/kleinanzeigen.test.js

@@ -20,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');

test/provider/neubauKompass.test.js

@@ -1,36 +1,44 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
+import {get} from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
+import {mockFredy, providerConfig} from '../utils.js';
-import { expect } from 'chai';
+import {expect} from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
 
 describe('#neubauKompass testsuite()', () => {
-  after(() => {
+    after(() => {
-    similarityCache.stopCacheCleanup();
+        similarityCache.stopCacheCleanup();
-  });
+    });
-  provider.init(providerConfig.neubauKompass, [], []);
+    provider.init(providerConfig.neubauKompass, [], []);
-  it('should test neubauKompass provider', async () => {
+    it('should test neubauKompass provider', async () => {
-    const Fredy = await mockFredy();
+        const Fredy = await mockFredy();
-    return await new Promise((resolve) => {
+        return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
+            if (!scrapingAnt.isScrapingAntApiKeySet()) {
-      fredy.execute().then((listing) => {
+                /* eslint-disable no-console */
-        expect(listing).to.be.a('array');
+                console.info('Skipping Neubaukompass test as ScrapingAnt Api Key is not set.');
-        const notificationObj = get();
+                /* eslint-enable no-console */
-        expect(notificationObj.serviceName).to.equal('neubauKompass');
+                resolve();
-        notificationObj.payload.forEach((notify) => {
+                return;
-          expect(notify).to.be.a('object');
+            }
-          /** check the actual structure **/
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
-          expect(notify.id).to.be.a('string');
+            fredy.execute().then((listing) => {
-          expect(notify.title).to.be.a('string');
+                expect(listing).to.be.a('array');
-          expect(notify.link).to.be.a('string');
+                const notificationObj = get();
-          expect(notify.address).to.be.a('string');
+                expect(notificationObj.serviceName).to.equal('neubauKompass');
-          /** check the values if possible **/
+                notificationObj.payload.forEach((notify) => {
-          expect(notify.title).to.be.not.empty;
+                    expect(notify).to.be.a('object');
-          expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    /** check the actual structure **/
-          expect(notify.address).to.be.not.empty;
+                    expect(notify.id).to.be.a('string');
-        });
+                    expect(notify.title).to.be.a('string');
-        resolve();
+                    expect(notify.link).to.be.a('string');
-      });
+                    expect(notify.address).to.be.a('string');
+                    /** check the values if possible **/
+                    expect(notify.title).to.be.not.empty;
+                    expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    expect(notify.address).to.be.not.empty;
+                });
+                resolve();
+            });
+        });
     });
-  });
 });

test/provider/testProvider.json

@@ -13,7 +13,7 @@
     "enabled": true
   },
   "immowelt": {
-    "url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?d=true&rmi=3&sd=DESC&sf=TIMESTAMP&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
     "enabled": true
   },
   "immoscout": {

test/queryStringMutator/testData.json

@@ -1,7 +1,7 @@
 [
   {
-    "url": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=PRIMARY_PRICE_AMOUNT&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
-    "shouldBecome": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=TIMESTAMP&sp=1",
+    "shouldBecome": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350&order=DateDesc",
     "id": "immowelt"
   },
   {

test/utils/utils.test.js

@@ -0,0 +1,16 @@
+import { expect } from 'chai';
+import {buildHash} from '../../lib/utils.js';
+
+describe('utilsCheck', () => {
+  describe('#utilsCheck()', () => {
+      it('should be null when null input', () => {
+          expect(buildHash(null)).to.be.null;
+      });
+      it('should be null when null empty', () => {
+          expect(buildHash('')).to.be.null;
+      });
+      it('should return a value', () => {
+          expect(buildHash('bla', '', null)).to.be.a.string;
+      });
+  });
+});

ui/src/App.jsx

@@ -17,11 +17,13 @@ import Jobs from './views/jobs/Jobs';
 import { Route } from 'react-router';
 
 import './App.less';
+import TrackingModal from './components/tracking/TrackingModal.jsx';
 
 export default function FredyApp() {
   const dispatch = useDispatch();
   const [loading, setLoading] = React.useState(true);
   const currentUser = useSelector((state) => state.user.currentUser);
+  const settings = useSelector((state) => state.generalSettings.settings);
 
   useEffect(() => {
     async function init() {
@@ -31,6 +33,7 @@ export default function FredyApp() {
         await dispatch.jobs.getJobs();
         await dispatch.jobs.getProcessingTimes();
         await dispatch.notificationAdapter.getAdapter();
+        await dispatch.generalSettings.getGeneralSettings();
       }
       setLoading(false);
     }
@@ -59,6 +62,7 @@ export default function FredyApp() {
         <Logout />
         <Logo width={190} white />
         <Menu isAdmin={isAdmin()} />
+        {settings.analyticsEnabled === null && <TrackingModal/>}
         <Switch>
           <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
           <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />

ui/src/components/tracking/TrackingModal.jsx

@@ -0,0 +1,48 @@
+import React from 'react';
+import {Modal} from '@douyinfe/semi-ui';
+import Logo from '../logo/Logo.jsx';
+import {xhrPost} from '../../services/xhr.js';
+
+import './TrackingModal.less';
+
+const saveResponse = async (analyticsEnabled) => {
+    await xhrPost('/api/admin/generalSettings', {
+        analyticsEnabled
+    });
+};
+
+export default function TrackingModal() {
+
+    return <Modal
+        visible={true}
+        onOk={async () => {
+            await saveResponse(true);
+            location.reload();
+        }}
+        onCancel={async () => {
+            await saveResponse(false);
+            location.reload();
+        }}
+        maskClosable={false}
+        closable={false}
+        okText="Yes! I want to help"
+        cancelText="No, thanks"
+    >
+        <Logo white/>
+        <div className="trackingModal__description">
+            <p>Hey 👋</p>
+            <p>Fed up with popups? Yeah, me too. But this one’s important, and I promise it will only appear once ;)</p>
+            <p>Fredy is completely free (and will always remain free). If you’d like, you can support me by donating
+                through my GitHub, but there’s absolutely no obligation to do so.</p>
+            <p>However, it would be a huge
+                help if you’d allow me to collect some analytical data. Wait, before you click "no", let me explain. If
+                you
+                agree, Fredy will send a ping to my Mixpanel project each time it runs.</p>
+            <p>The data includes: names of
+                active adapters/providers, OS, architecture, Node version, and language. The information is entirely
+                anonymous and helps me understand which adapters/providers are most frequently used.</p>
+            <p>Thanks🤘</p>
+        </div>
+    </Modal>;
+
+}

ui/src/components/tracking/TrackingModal.less

@@ -0,0 +1,5 @@
+.trackingModal {
+  &__description {
+    margin-top:10rem;
+  }
+}

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -1,246 +1,332 @@
 import React from 'react';
 
-import { useDispatch, useSelector } from 'react-redux';
+import {useDispatch, useSelector} from 'react-redux';
 
-import { Divider, Input, Radio, TimePicker, Button, RadioGroup } from '@douyinfe/semi-ui';
+import {Divider, Input, Radio, TimePicker, Button, RadioGroup, Checkbox} from '@douyinfe/semi-ui';
-import { InputNumber } from '@douyinfe/semi-ui';
+import {InputNumber} from '@douyinfe/semi-ui';
 import Headline from '../../components/headline/Headline';
-import { xhrPost } from '../../services/xhr';
+import {xhrPost} from '../../services/xhr';
-import { SegmentPart } from '../../components/segment/SegmentPart';
+import {SegmentPart} from '../../components/segment/SegmentPart';
-import { Banner, Toast } from '@douyinfe/semi-ui';
+import {Banner, Toast} from '@douyinfe/semi-ui';
-import { IconSave, IconCalendar, IconKey, IconRefresh, IconSignal } from '@douyinfe/semi-icons';
+import {IconSave, IconCalendar, IconKey, IconRefresh, IconSignal, IconLineChartStroked, IconSearch} from '@douyinfe/semi-icons';
 import './GeneralSettings.less';
 
 function formatFromTimestamp(ts) {
-  const date = new Date(ts);
+    const date = new Date(ts);
-  return `${date.getHours()}:${date.getMinutes() > 9 ? date.getMinutes() : '0' + date.getMinutes()}`;
+    return `${date.getHours()}:${date.getMinutes() > 9 ? date.getMinutes() : '0' + date.getMinutes()}`;
 }
 
 function formatFromTBackend(time) {
-  if (time == null || time.length === 0) {
+    if (time == null || time.length === 0) {
-    return null;
+        return null;
-  }
+    }
-  const date = new Date();
+    const date = new Date();
-  const split = time.split(':');
+    const split = time.split(':');
-  date.setHours(split[0]);
+    date.setHours(split[0]);
-  date.setMinutes(split[1]);
+    date.setMinutes(split[1]);
-  return date.getTime();
+    return date.getTime();
 }
 
 const GeneralSettings = function GeneralSettings() {
-  const dispatch = useDispatch();
+    const dispatch = useDispatch();
-  const [loading, setLoading] = React.useState(true);
+    const [loading, setLoading] = React.useState(true);
 
-  const settings = useSelector((state) => state.generalSettings.settings);
+    const settings = useSelector((state) => state.generalSettings.settings);
 
-  const [interval, setInterval] = React.useState('');
+    const [interval, setInterval] = React.useState('');
-  const [port, setPort] = React.useState('');
+    const [port, setPort] = React.useState('');
-  const [scrapingAntApiKey, setScrapingAntApiKey] = React.useState('');
+    const [scrapingAntApiKey, setScrapingAntApiKey] = React.useState('');
-  const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
+    const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
-  const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
+    const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
-  const [workingHourTo, setWorkingHourTo] = React.useState(null);
+    const [workingHourTo, setWorkingHourTo] = React.useState(null);
-  React.useEffect(() => {
+    const [demoMode, setDemoMode] = React.useState(null);
-    async function init() {
+    const [analyticsEnabled, setAnalyticsEnabled] = React.useState(null);
-      await dispatch.generalSettings.getGeneralSettings();
-      setLoading(false);
-    }
 
-    init();
+    React.useEffect(() => {
-  }, []);
+        async function init() {
+            await dispatch.generalSettings.getGeneralSettings();
+            setLoading(false);
+        }
 
-  React.useEffect(() => {
+        init();
-    async function init() {
+    }, []);
-      setInterval(settings?.interval);
-      setPort(settings?.port);
-      setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
-      setWorkingHourFrom(settings?.workingHours?.from);
-      setWorkingHourTo(settings?.workingHours?.to);
-      setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
-    }
 
-    init();
+    React.useEffect(() => {
-  }, [settings]);
+        async function init() {
+            setInterval(settings?.interval);
+            setPort(settings?.port);
+            setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
+            setWorkingHourFrom(settings?.workingHours?.from);
+            setWorkingHourTo(settings?.workingHours?.to);
+            setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
+            setAnalyticsEnabled(settings?.analytics || false);
+            setDemoMode(settings?.demoMode || false);
+        }
 
-  const nullOrEmpty = (val) => val == null || val.length === 0;
+        init();
+    }, [settings]);
 
-  const throwMessage = (message, type) => {
+    const nullOrEmpty = (val) => val == null || val.length === 0;
-    if (type === 'error') {
-      Toast.error(message);
-    } else {
-      Toast.success(message);
-    }
-  };
 
-  const onStore = async () => {
+    const throwMessage = (message, type) => {
-    if (nullOrEmpty(interval)) {
+        if (type === 'error') {
-      throwMessage('Interval may not be empty.', 'error');
+            Toast.error(message);
-      return;
+        } else {
-    }
+            Toast.success(message);
-    if (nullOrEmpty(port)) {
+        }
-      throwMessage('Port may not be empty.', 'error');
+    };
-      return;
-    }
-    if (
-      (!nullOrEmpty(workingHourFrom) && nullOrEmpty(workingHourTo)) ||
-      (nullOrEmpty(workingHourFrom) && !nullOrEmpty(workingHourTo))
-    ) {
-      throwMessage('Working hours to and from must be set if either to or from has been set before.', 'error');
-      return;
-    }
-    try {
-      await xhrPost('/api/admin/generalSettings', {
-        interval,
-        port,
-        scrapingAnt: {
-          apiKey: scrapingAntApiKey,
-          proxy: scrapingAntProxy,
-        },
-        workingHours: {
-          from: workingHourFrom,
-          to: workingHourTo,
-        },
-      });
-    } catch (exception) {
-      console.error(exception);
-      throwMessage('Error while trying to store settings.', 'error');
-      return;
-    }
-    throwMessage('Settings stored successfully. You MUST restart Fredy.', 'success');
-  };
 
-  return (
+    const onStore = async () => {
-    <div>
+        if (nullOrEmpty(interval)) {
-      {!loading && (
+            throwMessage('Interval may not be empty.', 'error');
-        <React.Fragment>
+            return;
-          <Headline text="General Settings" />
+        }
-          <Banner
+        if (nullOrEmpty(port)) {
-            fullMode={false}
+            throwMessage('Port may not be empty.', 'error');
-            type="info"
+            return;
-            closeIcon={null}
+        }
-            title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Info</div>}
+        if (
-            style={{ marginBottom: '1rem' }}
+            (!nullOrEmpty(workingHourFrom) && nullOrEmpty(workingHourTo)) ||
-            description="If you change any settings, you must restart Fredy afterwards."
+            (nullOrEmpty(workingHourFrom) && !nullOrEmpty(workingHourTo))
-          />
+        ) {
-          <div>
+            throwMessage('Working hours to and from must be set if either to or from has been set before.', 'error');
-            <SegmentPart
+            return;
-              name="Interval"
+        }
-              helpText="Interval in minutes for running queries against the configured services."
+        try {
-              Icon={IconRefresh}
+            await xhrPost('/api/admin/generalSettings', {
-            >
+                interval,
-              <InputNumber
+                port,
-                min={0}
+                scrapingAnt: {
-                max={1440}
+                    apiKey: scrapingAntApiKey,
-                placeholder="Interval in minutes"
+                    proxy: scrapingAntProxy,
-                value={interval}
+                },
-                formatter={(value) => `${value}`.replace(/\D/g, '')}
+                workingHours: {
-                onChange={(value) => setInterval(value)}
+                    from: workingHourFrom,
-                suffix={'minutes'}
+                    to: workingHourTo,
-              />
+                },
-            </SegmentPart>
+                demoMode,
-            <Divider margin="1rem" />
+                analyticsEnabled
-            <SegmentPart name="Port" helpText="Port on which Fredy is running." Icon={IconSignal}>
+            });
-              <InputNumber
+        } catch (exception) {
-                min={0}
+            console.error(exception);
-                max={99999}
+            throwMessage('Error while trying to store settings.', 'error');
-                placeholder="Port"
+            return;
-                value={port}
+        }
-                formatter={(value) => `${value}`.replace(/\D/g, '')}
+        throwMessage('Settings stored successfully.', 'success');
-                onChange={(value) => setPort(value)}
+    };
-              />
-            </SegmentPart>
-            <Divider margin="1rem" />
-            <SegmentPart
-              name="ScrapingAnt Api Key"
-              helpText="The api key for ScrapingAnt is used to be able to scrape Immoscout."
-              Icon={IconKey}
-            >
-              <Input
-                type="text"
-                placeholder="ScrapingAnt Api Key"
-                value={scrapingAntApiKey}
-                onChange={(val) => setScrapingAntApiKey(val)}
-              />
-            </SegmentPart>
-            <Divider margin="1rem" />
-            <SegmentPart
-              name="ScrapingAnt proxy settings"
-              helpText="Scraping ant provides different proxies."
-              Icon={IconKey}
-            >
-              <Banner
-                fullMode={false}
-                type="info"
-                closeIcon={null}
-                title={
-                  <div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>
-                    ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies
-                  </div>
-                }
-                style={{ marginBottom: '1rem' }}
-                description={
-                  <div>
-                    <h4>Datacenter-Proxy</h4>
-                    Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and
-                    more likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
-                    <h4>Residential-Proxy</h4>
-                    High-quality proxy server located in one of the real people houses across the world. Datacenter
-                    proxies are faster and more likely to success, but they are more expensive. A call with a datacenter
-                    proxy cost 250 credits.
-                    <br />
-                    <br />
-                    <b>
-                      On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only
-                      successful calls will be charged.
-                    </b>
-                  </div>
-                }
-              />
 
-              <RadioGroup value={scrapingAntProxy} onChange={(e) => setScrapingAntProxy(e.target.value)}>
+    return (
-                <Radio name="datacenter" value="datacenter" checked={scrapingAntProxy === 'datacenter'}>
+        <div>
-                  Datacenter proxy
+            {!loading && (
-                </Radio>
+                <React.Fragment>
-                <Radio name="residential" value="residential" checked={scrapingAntProxy === 'residential'}>
+                    <Headline text="General Settings"/>
-                  Residential proxy
+                    <Banner
-                </Radio>
+                        fullMode={false}
-              </RadioGroup>
+                        type="info"
-            </SegmentPart>
+                        closeIcon={null}
-            <Divider margin="1rem" />
+                        title={<div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>Info</div>}
-            <SegmentPart
+                        style={{marginBottom: '1rem'}}
-              name="Working hours"
+                        description="If you change any settings, you must restart Fredy afterwards."
-              helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
+                    />
-              Icon={IconCalendar}
+                    <div>
-            >
+                        <SegmentPart
-              <div className="generalSettings__timePickerContainer">
+                            name="Interval"
-                <TimePicker
+                            helpText="Interval in minutes for running queries against the configured services."
-                  format={'HH:mm'}
+                            Icon={IconRefresh}
-                  insetLabel="From"
+                        >
-                  value={formatFromTBackend(workingHourFrom)}
+                            <InputNumber
-                  placeholder=""
+                                min={0}
-                  onChange={(val) => {
+                                max={1440}
-                    setWorkingHourFrom(val == null ? null : formatFromTimestamp(val));
+                                placeholder="Interval in minutes"
-                  }}
+                                value={interval}
-                />
+                                formatter={(value) => `${value}`.replace(/\D/g, '')}
-                <TimePicker
+                                onChange={(value) => setInterval(value)}
-                  format={'HH:mm'}
+                                suffix={'minutes'}
-                  insetLabel="Until"
+                            />
-                  value={formatFromTBackend(workingHourTo)}
+                        </SegmentPart>
-                  placeholder=""
+                        <Divider margin="1rem"/>
-                  onChange={(val) => {
+                        <SegmentPart name="Port" helpText="Port on which Fredy is running." Icon={IconSignal}>
-                    setWorkingHourTo(val == null ? null : formatFromTimestamp(val));
+                            <InputNumber
-                  }}
+                                min={0}
-                />
+                                max={99999}
-              </div>
+                                placeholder="Port"
-            </SegmentPart>
+                                value={port}
-            <Divider margin="1rem" />
+                                formatter={(value) => `${value}`.replace(/\D/g, '')}
-            <Button type="primary" theme="solid" onClick={onStore} icon={<IconSave />}>
+                                onChange={(value) => setPort(value)}
-              Save
+                            />
-            </Button>
+                        </SegmentPart>
-          </div>
+                        <Divider margin="1rem"/>
-        </React.Fragment>
+                        <SegmentPart
-      )}
+                            name="ScrapingAnt Api Key"
-    </div>
+                            helpText="The api key for ScrapingAnt is used to be able to scrape Immoscout."
-  );
+                            Icon={IconKey}
+                        >
+                            <Input
+                                type="text"
+                                placeholder="ScrapingAnt Api Key"
+                                value={scrapingAntApiKey}
+                                onChange={(val) => setScrapingAntApiKey(val)}
+                            />
+                        </SegmentPart>
+                        <Divider margin="1rem"/>
+                        <SegmentPart
+                            name="ScrapingAnt proxy settings"
+                            helpText="Scraping ant provides different proxies."
+                            Icon={IconKey}
+                        >
+                            <Banner
+                                fullMode={false}
+                                type="info"
+                                closeIcon={null}
+                                title={
+                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
+                                        ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2
+                                        different types of proxies
+                                    </div>
+                                }
+                                style={{marginBottom: '1rem'}}
+                                description={
+                                    <div>
+                                        <h4>Datacenter-Proxy</h4>
+                                        Proxy server located in one of the datacenters across the world. Datacenter
+                                        proxies are slower and
+                                        more likely to fail, but they are cheaper. A call with a datacenter proxy cost
+                                        10 credits.
+                                        <h4>Residential-Proxy</h4>
+                                        High-quality proxy server located in one of the real people houses across the
+                                        world. Datacenter
+                                        proxies are faster and more likely to success, but they are more expensive.
+                                        <br/>
+                                        <br/>
+                                        <b>
+                                            On the free tier, you have 10.000 credits, so chose your option wisely. Keep
+                                            in mind, only
+                                            successful calls will be charged.
+                                        </b>
+                                    </div>
+                                }
+                            />
+
+                            <RadioGroup value={scrapingAntProxy} onChange={(e) => setScrapingAntProxy(e.target.value)}>
+                                <Radio name="datacenter" value="datacenter" checked={scrapingAntProxy === 'datacenter'}>
+                                    Datacenter proxy
+                                </Radio>
+                                <Radio name="residential" value="residential"
+                                       checked={scrapingAntProxy === 'residential'}>
+                                    Residential proxy
+                                </Radio>
+                            </RadioGroup>
+                        </SegmentPart>
+                        <Divider margin="1rem"/>
+                        <SegmentPart
+                            name="Working hours"
+                            helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
+                            Icon={IconCalendar}
+                        >
+                            <div className="generalSettings__timePickerContainer">
+                                <TimePicker
+                                    format={'HH:mm'}
+                                    insetLabel="From"
+                                    value={formatFromTBackend(workingHourFrom)}
+                                    placeholder=""
+                                    onChange={(val) => {
+                                        setWorkingHourFrom(val == null ? null : formatFromTimestamp(val));
+                                    }}
+                                />
+                                <TimePicker
+                                    format={'HH:mm'}
+                                    insetLabel="Until"
+                                    value={formatFromTBackend(workingHourTo)}
+                                    placeholder=""
+                                    onChange={(val) => {
+                                        setWorkingHourTo(val == null ? null : formatFromTimestamp(val));
+                                    }}
+                                />
+                            </div>
+                        </SegmentPart>
+                        <Divider margin="1rem"/>
+
+                        <SegmentPart
+                            name="Analytics"
+                            helpText="Insights into the usage of Fredy."
+                            Icon={IconLineChartStroked}
+                        >
+                            <Banner
+                                fullMode={false}
+                                type="info"
+                                closeIcon={null}
+                                title={
+                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
+                                        Explanation
+                                    </div>
+                                }
+                                style={{marginBottom: '1rem'}}
+                                description={
+                                    <div>
+                                        Analytics are disabled by default. If you choose to enable them, we will begin tracking the following:<br/>
+                                        <ul>
+                                            <li>Name of active provider (e.g. Immoscout)</li>
+                                            <li>Name of active adapter (e.g. Console)</li>
+                                            <li>language</li>
+                                            <li>os</li>
+                                            <li>node version</li>
+                                            <li>arch</li>
+                                        </ul>
+                                        The data is sent anonymously and helps me understand which providers or adapters are being used the most. In the end it helps me to improve fredy.
+                                    </div>
+                                }
+                            />
+
+                            <Checkbox
+                                checked={analyticsEnabled}
+                                onChange={(e) => setAnalyticsEnabled(e.target.checked)}
+                            > Enabled
+                            </Checkbox>
+
+                        </SegmentPart>
+
+                        <Divider margin="1rem"/>
+
+                        <SegmentPart
+                            name="Demo Mode"
+                            helpText="If enabled, Fredy runs in demo mode."
+                            Icon={IconSearch}
+                        >
+                            <Banner
+                                fullMode={false}
+                                type="info"
+                                closeIcon={null}
+                                title={
+                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
+                                        Explanation
+                                    </div>
+                                }
+                                style={{marginBottom: '1rem'}}
+                                description={
+                                    <div>
+                                        In demo mode, Fredy will not (really) search for any real estates. Fredy is in a lockdown mode. Also
+                                        all database files will be set back to the default values at midnight.
+                                    </div>
+                                }
+                            />
+
+                            <Checkbox
+                                checked={demoMode}
+                                onChange={(e) => setDemoMode(e.target.checked)}
+                            > Enabled
+                            </Checkbox>
+
+                        </SegmentPart>
+
+                        <Divider margin="1rem"/>
+                        <Button type="primary" theme="solid" onClick={onStore} icon={<IconSave/>}>
+                            Save
+                        </Button>
+                    </div>
+                </React.Fragment>
+            )}
+        </div>
+    );
 };
 
 export default GeneralSettings;

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,63 +1,86 @@
 import React from 'react';
-import { format } from '../../services/time/timeService';
+import {format} from '../../services/time/timeService';
-import { Card, Descriptions, Divider } from '@douyinfe/semi-ui';
+import {Banner, Card, Descriptions, Divider} from '@douyinfe/semi-ui';
-import { IconBolt } from '@douyinfe/semi-icons';
+import {IconBolt} from '@douyinfe/semi-icons';
-export default function ProcessingTimes({ processingTimes }) {
-  const { Meta } = Card;
-  return (
-    <>
-      <Descriptions
-        row
-        size="small"
-        style={{
-          backgroundColor: '#35363c',
-          borderRadius: '4px',
-          padding: '10px',
-        }}
-      >
-        <Descriptions.Item itemKey="Processing Interval">{processingTimes.interval} min</Descriptions.Item>
-        {processingTimes.lastRun && (
-          <>
-            <Descriptions.Item itemKey="Last run">{format(processingTimes.lastRun)}</Descriptions.Item>
-            <Descriptions.Item itemKey="Next run">
-              {format(processingTimes.lastRun + processingTimes.interval * 60000)}
-            </Descriptions.Item>
-          </>
-        )}
-      </Descriptions>
 
-      {processingTimes.scrapingAntData != null && (
+export default function ProcessingTimes({processingTimes = {}}) {
-        <>
+    const {Meta} = Card;
-          <Divider margin="1rem" />
+    if (Object.keys(processingTimes).length === 0) {
-          <Card
+        return null;
-            style={{ backgroundColor: '#35363c' }}
+    }
+    if (processingTimes.error != null) {
+        return <Banner
+            fullMode={false}
+            type="danger"
+            closeIcon={null}
             title={
-              <Meta
+                <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
-                title="Remaining ScrapingAnt calls"
+                    Scraping Ant Error
-                description="Information about your Scraping Ant Plan"
+                </div>
-                avatar={<IconBolt />}
-              />
             }
-          >
+            style={{marginBottom: '1rem'}}
-            <p>Plan: {processingTimes.scrapingAntData.plan_name}</p>
+            description={
-            <p>
+                <div>
-              Duration: {format(new Date(processingTimes.scrapingAntData.start_date))} -{' '}
+                    {processingTimes.error}
-              {format(new Date(processingTimes.scrapingAntData.end_date))}
+                </div>
-              <br />
+            }
-              Credits: {processingTimes.scrapingAntData.remained_credits}/
+        />;
-              {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
+    }
-            </p>
+    return (
-            If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
+        <>
-            <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
+            <Descriptions
-              ScrapingAnt
+                row
-            </a>
+                size="small"
-            . You can use the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to
+                style={{
-            recommend ScrapingAnt.)
+                    backgroundColor: '#35363c',
-          </Card>
+                    borderRadius: '4px',
+                    padding: '10px',
+                }}
+            >
+                <Descriptions.Item itemKey="Processing Interval">{processingTimes.interval} min</Descriptions.Item>
+                {processingTimes.lastRun && (
+                    <>
+                        <Descriptions.Item itemKey="Last run">{format(processingTimes.lastRun)}</Descriptions.Item>
+                        <Descriptions.Item itemKey="Next run">
+                            {format(processingTimes.lastRun + processingTimes.interval * 60000)}
+                        </Descriptions.Item>
+                    </>
+                )}
+            </Descriptions>
+
+            {(processingTimes.scrapingAntData != null && Object.keys(processingTimes.scrapingAntData).length > 0) &&(
+                <>
+                    <Divider margin="1rem"/>
+                    <Card
+                        style={{backgroundColor: '#35363c'}}
+                        title={
+                            <Meta
+                                title="Remaining ScrapingAnt calls"
+                                description="Information about your Scraping Ant Plan"
+                                avatar={<IconBolt/>}
+                            />
+                        }
+                    >
+                        <p>Plan: {processingTimes.scrapingAntData.plan_name}</p>
+                        <p>
+                            Duration: {format(new Date(processingTimes.scrapingAntData.start_date))} -{' '}
+                            {format(new Date(processingTimes.scrapingAntData.end_date))}
+                            <br/>
+                            Credits: {processingTimes.scrapingAntData.remained_credits}/
+                            {processingTimes.scrapingAntData.plan_total_credits}
+                        </p>
+                        If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account
+                        of{' '}
+                        <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
+                            ScrapingAnt
+                        </a>
+                        . You can use the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting
+                        paid by ScrapingAnt.)
+                    </Card>
+                </>
+            )}
         </>
-      )}
+    );
-    </>
-  );
 }
 
 /*

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              If you chose Immoscout, Immonet or NeubauKompass as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
               (See readme)
             </p>
             <p>