upgrading dependencies

* Moving to puppeteer | removing scrapingAnt

.eslintrc.cjs

@@ -1,7 +1,6 @@
 module.exports = {
   env: {
-    commonjs: true,
+    es2021: true,
-    es6: true,
     node: true,
     browser: true,
     mocha: true,
@@ -17,7 +16,6 @@ module.exports = {
     fetch: true,
   },
   parserOptions: {
-    ecmaVersion: 2018,
     sourceType: 'module',
   },
   rules: {
@@ -205,10 +203,6 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
     'react/self-closing-comp': 'warn',
 
-    // Enforce spaces before the closing bracket of self-closing JSX elements
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
-    'react/jsx-space-before-closing': ['warn', 'always'],
-
     // Enforce component methods order
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
     'react/sort-comp': 'off',
@@ -239,7 +233,7 @@ module.exports = {
 
     // only .jsx files may have JSX
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
-    'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
+    'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
 
     // prevent accidental JS comments from being injected into JSX as text
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
@@ -284,15 +278,5 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
     'react/no-children-prop': 'warn',
 
-    // Validate whitespace in and around the JSX opening and closing brackets
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
-    'react/jsx-tag-spacing': [
-      'warn',
-      {
-        closingSlash: 'never',
-        beforeSelfClosing: 'always',
-        afterOpening: 'never',
-      },
-    ],
   },
 };

.github/workflows/docker.yml

@@ -44,3 +44,4 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64

.github/workflows/stales.yml

@@ -0,0 +1,21 @@
+name: "Close stale issues and PRs"
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          days-before-stale: 30
+          days-before-close: 7
+          stale-issue-message: "This issue has been automatically marked as stale due to inactivity."
+          stale-pr-message: "This PR has been automatically marked as stale due to inactivity."
+          close-issue-message: "Closing this issue due to prolonged inactivity."
+          close-pr-message: "Closing this PR due to prolonged inactivity."
+          exempt-issue-labels: "keep-open"
+          exempt-pr-labels: "keep-open"
+          only: "pulls"

.github/workflows/test.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches:
       - master
+  schedule:
+    - cron: '0 12 * * *'      
 jobs:
   test:
     name: Test
@@ -15,7 +17,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 16
+          node-version: 20
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

.travis.yml

@@ -1,2 +0,0 @@
-sudo: false
-language: node_js

CONTRIBUTING.md

@@ -106,9 +106,7 @@ exports.config = {
 ```
 
 #### Running Tests
-If you've written a new provider you are an awesome person. You know it and I do. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
+If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
-
-To write tests for provider, you need to use Node 8 as the tests are using `async / await`
 
 #### Codestyle
 I'm using Eslint to maintain quote style and quality. Do not skip it...

Dockerfile

@@ -1,18 +1,25 @@
-# syntax=docker/dockerfile:1.3
+FROM node:20
-FROM node:16-alpine AS builder
+
-COPY --chown=1000:1000 . /fredy
 WORKDIR  /fredy
-USER 1000
+
+COPY . /fredy
+
+RUN apt-get update && apt-get install -y chromium
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+
 RUN yarn install
+
+RUN yarn global add pm2
+
 RUN yarn run prod
 
-FROM node:16-alpine
-COPY --from=builder --chown=1000:1000 /fredy /fredy
 RUN mkdir /db /conf && \
   chown 1000:1000 /db /conf && \
+  chmod 777 -R /db/ && \
   ln -s /db /fredy/db && ln -s /conf /fredy/conf
+
 EXPOSE 9998
-USER 1000
+
-VOLUME [ "/conf", "/db" ]
+CMD pm2-runtime index.js
-WORKDIR  /fredy
-CMD node index.js --no-daemon

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Christian Kellner
+Copyright (c) 2025 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -11,13 +11,16 @@ If _Fredy_ finds matching results, it will send them to you via Slack, Email, Te
 # Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
 If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
 
-<img src="https://github.com/orangecoding/fredy/blob/master/doc/jetbrains.png" width="200">   
+[![JetBrains logo.](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg)](https://jb.gg/OpenSourceSupport)
 
 _Fredy_ is supported by JetBrains under Open Source Support Program
 
+## Demo
+If you want to try out _Fredy_, you can access the demo version [here](https://fredy.orange-coding.net) 🤘
+
 ## Usage  
 
-- Make sure to use Node.js 16 or above
+- Make sure to use Node.js 20 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -27,14 +30,11 @@ yarn run start
 _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
 
 <p align="center">
-  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot1.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;
-  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
+  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_2.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;  
-  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
+  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
-</p>
-<p align="center">
-
 </p>
 
 ## Understanding the fundamentals
@@ -82,16 +82,14 @@ yarn run test
 ![Architecture](/doc/architecture.jpg "Architecture")
 
 ### Immoscout
-I have added **experimental** support for Immoscout. Immoscout is somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+Immoscout has implemented advanced bot detection. I’m actively working on bypassing these measures, but until then, selecting Immoscout as a provider will not return any results. I apologize for the inconvenience. 😉
 
-To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
+# Analytics
-The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
+Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 
-
+Before you freak out, let me explain...         
-If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
+If you agree, Fredy will send a ping to my Mixpanel project each time it runs.   
-
+The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.</p>
-### Contribution guidelines  
+**Thanks**🤘
-
-See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
 # Docker   
 Use the Dockerfile in this repository to build an image.  
@@ -112,6 +110,15 @@ Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+### 👐 Contributing
+Thanks to all the people who already contributed!
+
+<a href="https://github.com/orangecoding/fredy/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
+</a>
+
+See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
+
 ## Logs  
 
 You can browse the logs with  `docker logs fredy -f`.

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.3'
+version: '3.8'
 services:
   fredy:
     container_name: fredy
@@ -13,3 +13,4 @@ services:
       - ./db:/db
     ports:
         - 9998:9998
+    restart: unless-stopped

index.html

@@ -4,12 +4,11 @@
     <meta charset="UTF-8"
           name="viewport"
           content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui@2/dist/semantic.min.css">
     <meta name="google" content="notranslate">
 
     <title>Fredy</title>
 </head>
-<body>
+<body theme-mode="dark">
 
 <div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
 </body>

index.js

@@ -1,59 +1,61 @@
-const fs = require('fs');
+import fs from 'fs';
-
+import {config} from './lib/utils.js';
+import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
+import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
+import * as jobStorage from './lib/services/storage/jobStorage.js';
+import FredyRuntime from './lib/FredyRuntime.js';
+import { duringWorkingHoursOrNotSet } from './lib/utils.js';
+import './lib/api/api.js';
+import {track} from './lib/services/tracking/Tracker.js';
+import {handleDemoUser} from './lib/services/storage/userStorage.js';
+import {cleanupDemoAtMidnight} from './lib/services/demoCleanup.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
 }
-
 const path = './lib/provider';
 const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
-const config = require('./conf/config.json');
-
-const similarityCache = require('./lib/services/similarity-check/similarityCache');
-const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
-const jobStorage = require('./lib/services/storage/jobStorage');
-const FredyRuntime = require('./lib/FredyRuntime');
-
-const { duringWorkingHoursOrNotSet } = require('./lib/utils');
-
-//starting the api service
-require('./lib/api/api');
-
 //assuming interval is always in minutes
-
 const INTERVAL = config.interval * 60 * 1000;
-
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+if(config.demoMode){
+    console.info('Running in demo mode');
+    cleanupDemoAtMidnight();
+}
 /* eslint-enable no-console */
+const fetchedProvider = await Promise.all(
+  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
+);
+
+handleDemoUser();
+
 setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-
+        if(!config.demoMode) {
-    if (isDuringWorkingHoursOrNotSet) {
+            if (isDuringWorkingHoursOrNotSet) {
-      config.lastRun = Date.now();
+                track();
-      const fetchedProvider = provider
+                config.lastRun = Date.now();
-        .filter((provider) => provider.endsWith('.js'))
+                jobStorage
-        .map((pro) => require(`${path}/${pro}`));
+                    .getJobs()
-
+                    .filter((job) => job.enabled)
-      jobStorage
+                    .forEach((job) => {
-        .getJobs()
+                        job.provider
-        .filter((job) => job.enabled)
+                            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-        .forEach((job) => {
+                            .forEach(async (prov) => {
-          job.provider
+                                const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+                                pro.init(prov, job.blacklist);
-            .forEach(async (prov) => {
+                                await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
-              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+                                setLastJobExecution(job.id);
-              pro.init(prov, job.blacklist);
+                            });
-              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+                    });
-              setLastJobExecution(job.id);
+            } else {
-            });
+                /* eslint-disable no-console */
-        });
+                console.debug('Working hours set. Skipping as outside of working hours.');
-    } else {
+                /* eslint-enable no-console */
-      /* eslint-disable no-console */
+            }
-      console.debug('Working hours set. Skipping as outside of working hours.');
+        }
-      /* eslint-enable no-console */
-    }
     return exec;
   })(),
   INTERVAL

lib/FredyRuntime.js

@@ -1,10 +1,8 @@
-const { NoNewListingsWarning } = require('./errors');
+import { NoNewListingsWarning } from './errors.js';
-const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
+import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
-
+import * as notify from './notification/notify.js';
-const notify = require('./notification/notify');
+import Extractor from './services/extractor/extractor.js';
-const xray = require('./services/scraper');
+import urlModifier from './services/queryStringMutator.js';
-const scrapingAnt = require('./services/scrapingAnt');
-const urlModifier = require('./services/queryStringMutator');
 
 class FredyRuntime {
   /**
@@ -47,44 +45,24 @@ class FredyRuntime {
   }
 
   _getListings(url) {
+    const extractor = new Extractor();
     return new Promise((resolve, reject) => {
-      const id = this._providerId;
+      extractor
-      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
+        .execute(url, this._providerConfig.waitForSelector)
-        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
+        .then(() => {
-        /* eslint-disable no-console */
+          const listings = extractor.parseResponseText(
-        console.log(error);
+            this._providerConfig.crawlContainer,
-        /* eslint-enable no-console */
+            this._providerConfig.crawlFields,
-        reject(error);
+            url,
-        return;
+          );
-      }
+          resolve(listings == null ? [] : listings);
-      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
+        })
-      try {
+        .catch((err) => {
-        if (this._providerConfig.paginate != null) {
+          reject(err);
-          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
+          /* eslint-disable no-console */
-            //the first 2 pages should be enough here
+          console.error(err);
-            .limit(2)
+          /* eslint-enable no-console */
-            .paginate(this._providerConfig.paginate)
+        });
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        } else {
-          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        }
-      } catch (error) {
-        reject(error);
-        console.error(error);
-      }
     });
   }
 
@@ -93,16 +71,17 @@ class FredyRuntime {
   }
 
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
 
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
-
     if (newListings.length === 0) {
       throw new NoNewListingsWarning();
     }
-
     return newListings;
   }
 
@@ -142,4 +121,4 @@ class FredyRuntime {
   }
 }
 
-module.exports = FredyRuntime;
+export default FredyRuntime;

lib/api/api.js

@@ -1,44 +1,40 @@
-const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
+import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
-const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
+import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
-const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
+import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
-const { analyticsRouter } = require('./routes/analyticsRouter');
+import { analyticsRouter } from './routes/analyticsRouter.js';
-const { providerRouter } = require('./routes/providerRouter');
+import { providerRouter } from './routes/providerRouter.js';
-const { loginRouter } = require('./routes/loginRoute');
+import { loginRouter } from './routes/loginRoute.js';
-const config = require('../../conf/config.json');
+import { config } from '../utils.js';
-const { userRouter } = require('./routes/userRoute');
+import { userRouter } from './routes/userRoute.js';
-const { jobRouter } = require('./routes/jobRouter');
+import { jobRouter } from './routes/jobRouter.js';
-const bodyParser = require('body-parser');
+import bodyParser from 'body-parser';
-const service = require('restana')();
+import restana from 'restana';
-const files = require('serve-static');
+import files from 'serve-static';
-const path = require('path');
+import path from 'path';
-
+import { getDirName } from '../utils.js';
-const staticService = files(path.join(__dirname, '../../ui/public'));
+import {demoRouter} from './routes/demoRouter.js';
-
+const service = restana();
+const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
 
 service.use(bodyParser.json());
-
 service.use(cookieSession());
-
 service.use(staticService);
-
 service.use('/api/admin', authInterceptor());
 service.use('/api/jobs', authInterceptor());
-
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
-
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
 service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
 service.use('/api/jobs', jobRouter);
-
 service.use('/api/login', loginRouter);
+//this route is unsecured intentionally as it is being queried from the login page
+service.use('/api/demo', demoRouter);
 
 /* eslint-disable no-console */
 service.start(PORT).then(() => {
   console.info(`Started API service on port ${PORT}`);
 });
-/* eslint-enable no-console */

lib/api/routes/analyticsRouter.js

@@ -1,12 +1,10 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as listingStorage from '../../services/storage/listingsStorage.js';
+const service = restana();
 const analyticsRouter = service.newRouter();
-const listingStorage = require('../../services/storage/listingsStorage');
-
 analyticsRouter.get('/:jobId', async (req, res) => {
   const { jobId } = req.params;
-
   res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
   res.send();
 });
-
+export { analyticsRouter };
-exports.analyticsRouter = analyticsRouter;

lib/api/routes/demoRouter.js

@@ -0,0 +1,11 @@
+import restana from 'restana';
+import {config} from '../../utils.js';
+const service = restana();
+const demoRouter = service.newRouter();
+
+demoRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, {demoMode: config.demoMode});
+  res.send();
+});
+
+export { demoRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,18 +1,24 @@
-const service = require('restana')();
+import restana from 'restana';
+import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
+import fs from 'fs';
+import {handleDemoUser} from '../../services/storage/userStorage.js';
+const service = restana();
 const generalSettingsRouter = service.newRouter();
-const config = require('../../../conf/config.json');
-const fs = require('fs');
-
 generalSettingsRouter.get('/', async (req, res) => {
   res.body = Object.assign({}, config);
   res.send();
 });
-
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
-
   try {
-    fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
+    if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+      return;
+    }
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
+    await refreshConfig();
+    handleDemoUser();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));
@@ -20,5 +26,4 @@ generalSettingsRouter.post('/', async (req, res) => {
   }
   res.send();
 });
-
+export { generalSettingsRouter };
-exports.generalSettingsRouter = generalSettingsRouter;

lib/api/routes/jobRouter.js

@@ -1,12 +1,11 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import * as userStorage from '../../services/storage/userStorage.js';
+import { config } from '../../utils.js';
+import { isAdmin } from '../security.js';
+import { trackDemoJobCreated } from '../../services/tracking/Tracker.js';
+const service = restana();
 const jobRouter = service.newRouter();
-const fetch = require('node-fetch');
-const jobStorage = require('../../services/storage/jobStorage');
-const userStorage = require('../../services/storage/userStorage');
-const immoscoutProvider = require('../../provider/immoscout');
-const config = require('../../../conf/config.json');
-const { isAdmin } = require('../security');
-
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -16,51 +15,23 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-
   return user.isAdmin || job.userId === job.userId;
 }
-
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
-
   //show only the jobs which belongs to the user (or all of the user is an admin)
   res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
-
   res.send();
 });
-
 jobRouter.get('/processingTimes', async (req, res) => {
-  let scrapingAntData = null;
-
-  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
-    try {
-      const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
-      scrapingAntData = await response.json();
-    } catch (Exception) {
-      console.error('Could not query plan data from scraping ant.', Exception);
-    }
-  }
-
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
-    scrapingAntData,
   };
-
   res.send();
 });
-
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
-  if (
-    provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
-    (config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
-  ) {
-    res.send(
-      new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
-    );
-    return;
-  }
   try {
     jobStorage.upsertJob({
       userId: req.session.currentUser,
@@ -75,9 +46,13 @@ jobRouter.post('/', async (req, res) => {
     res.send(new Error(error));
     console.error(error);
   }
+  trackDemoJobCreated({
+    name,
+    provider,
+    adapter: notificationAdapter,
+  });
   res.send();
 });
-
 jobRouter.delete('', async (req, res) => {
   const { jobId } = req.body;
   try {
@@ -93,7 +68,6 @@ jobRouter.delete('', async (req, res) => {
   }
   res.send();
 });
-
 jobRouter.put('/:jobId/status', async (req, res) => {
   const { status } = req.body;
   const { jobId } = req.params;
@@ -113,5 +87,4 @@ jobRouter.put('/:jobId/status', async (req, res) => {
   }
   res.send();
 });
-
+export { jobRouter };
-exports.jobRouter = jobRouter;

lib/api/routes/loginRoute.js

@@ -1,8 +1,10 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as hasher from '../../services/security/hash.js';
+import {config} from '../../utils.js';
+import {trackDemoAccessed} from '../../services/tracking/Tracker.js';
+const service = restana();
 const loginRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const hasher = require('../../services/security/hash');
-
 loginRouter.get('/user', async (req, res) => {
   const currentUserId = req.session.currentUser;
   const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
@@ -16,18 +18,19 @@ loginRouter.get('/user', async (req, res) => {
   }
   res.send();
 });
-
 loginRouter.post('/', async (req, res) => {
   const { username, password } = req.body;
-
   const user = userStorage.getUsers(true).find((user) => user.username === username);
-
   if (user == null) {
     res.send(401);
     return;
   }
-
   if (user.password === hasher.hash(password)) {
+
+    if(config.demoMode){
+      trackDemoAccessed();
+    }
+
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
     res.send(200);
@@ -35,13 +38,10 @@ loginRouter.post('/', async (req, res) => {
   } else {
     console.error(`User ${username} tried to login, but password was wrong.`);
   }
-
   res.send(401);
 });
-
 loginRouter.post('/logout', async (req, res) => {
   req.session = null;
   res.send(200);
 });
-
+export { loginRouter };
-exports.loginRouter = loginRouter;

lib/api/routes/notificationAdapterRouter.js

@@ -1,13 +1,13 @@
-const fs = require('fs');
+import fs from 'fs';
-const service = require('restana')();
+import restana from 'restana';
+const service = restana();
 const notificationAdapterRouter = service.newRouter();
-
 const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
-
+const notificationAdapter = await Promise.all(
-const notificationAdapter = notificationAdapterList.map((pro) => {
+  notificationAdapterList.map(async (pro) => {
-  return require(`../../notification/adapter/${pro}`);
+    return await import(`../../notification/adapter/${pro}`);
-});
+  })
-
+);
 notificationAdapterRouter.post('/try', async (req, res) => {
   const { id, fields } = req.body;
   const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
@@ -24,7 +24,6 @@ notificationAdapterRouter.post('/try', async (req, res) => {
     enabled: true,
     id,
   });
-
   try {
     await adapter.send({
       serviceName: 'TestCall',
@@ -40,16 +39,13 @@ notificationAdapterRouter.post('/try', async (req, res) => {
       notificationConfig,
       jobKey: 'TestJob',
     });
-
     res.send();
   } catch (Exception) {
     res.send(new Error(Exception));
   }
 });
-
 notificationAdapterRouter.get('/', async (req, res) => {
   res.body = notificationAdapter.map((adapter) => adapter.config);
   res.send();
 });
-
+export { notificationAdapterRouter };
-exports.notificationAdapterRouter = notificationAdapterRouter;

lib/api/routes/providerRouter.js

@@ -1,16 +1,15 @@
-const fs = require('fs');
+import fs from 'fs';
-const service = require('restana')();
+import restana from 'restana';
+const service = restana();
 const providerRouter = service.newRouter();
-
 const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
-
+const provider = await Promise.all(
-const provider = providerList.map((pro) => {
+  providerList.map(async (pro) => {
-  return require(`../../provider/${pro}`).metaInformation;
+    return await import(`../../provider/${pro}`);
-});
+  })
-
+);
 providerRouter.get('/', async (req, res) => {
-  res.body = provider;
+  res.body = provider.map((p) => p.metaInformation);
   res.send();
 });
-
+export { providerRouter };
-exports.providerRouter = providerRouter;

lib/api/routes/userRoute.js

@@ -1,33 +1,33 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import {config} from '../../utils.js';
+const service = restana();
 const userRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const jobStorage = require('../../services/storage/jobStorage');
-
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
   return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
 }
-
 function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
   return req.session.currentUser === userIdToBeRemoved;
 }
-
 const nullOrEmpty = (str) => str == null || str.length === 0;
-
 userRouter.get('/', async (req, res) => {
   res.body = userStorage.getUsers(false);
   res.send();
 });
-
 userRouter.get('/:userId', async (req, res) => {
   const { userId } = req.params;
   res.body = userStorage.getUser(userId);
   res.send();
 });
-
 userRouter.delete('/', async (req, res) => {
+  if(config.demoMode){
+    res.send(new Error('In demo mode, it is not allowed to remove user.'));
+    return;
+  }
+
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
-
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
     return;
@@ -36,15 +36,18 @@ userRouter.delete('/', async (req, res) => {
     res.send(new Error('You are trying to remove yourself. This is prohibited.'));
     return;
   }
-
   //TODO: Remove also analytics
   jobStorage.removeJobsByUserId(userId);
   userStorage.removeUser(userId);
-
   res.send();
 });
-
 userRouter.post('/', async (req, res) => {
+
+  if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change or add user.'));
+      return;
+  }
+
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
     res.send(new Error('Passwords does not match'));
@@ -55,22 +58,18 @@ userRouter.post('/', async (req, res) => {
     return;
   }
   const allUser = userStorage.getUsers(false);
-
   if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(
       new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
     );
     return;
   }
-
   userStorage.upsertUser({
     userId,
     username,
     password,
     isAdmin,
   });
-
   res.send();
 });
-
+export { userRouter };
-exports.userRouter = userRouter;

lib/api/security.js

@@ -1,15 +1,12 @@
-const userStorage = require('../services/storage/userStorage');
+import * as userStorage from '../services/storage/userStorage.js';
-const cookieSession = require('cookie-session');
+import cookieSession from 'cookie-session';
-const { nanoid } = require('nanoid');
+import { nanoid } from 'nanoid';
-
 const unauthorized = (res) => {
   return res.send(401);
 };
-
 const isUnauthorized = (req) => {
   return req.session.currentUser == null;
 };
-
 const isAdmin = (req) => {
   if (!isUnauthorized(req)) {
     const user = userStorage.getUser(req.session.currentUser);
@@ -17,7 +14,6 @@ const isAdmin = (req) => {
   }
   return false;
 };
-
 const authInterceptor = () => {
   return (req, res, next) => {
     if (isUnauthorized(req)) {
@@ -27,7 +23,6 @@ const authInterceptor = () => {
     }
   };
 };
-
 const adminInterceptor = () => {
   return (req, res, next) => {
     if (!isAdmin(req)) {
@@ -37,8 +32,7 @@ const adminInterceptor = () => {
     }
   };
 };
-
+const cookieSession$0 = (userId) => {
-exports.cookieSession = (userId) => {
   return cookieSession({
     name: 'fredy-admin-session',
     keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
@@ -46,8 +40,8 @@ exports.cookieSession = (userId) => {
     maxAge: 8 * 60 * 60 * 1000, // 8 hours
   });
 };
-
+export { cookieSession$0 as cookieSession };
-exports.adminInterceptor = adminInterceptor;
+export { adminInterceptor };
-exports.authInterceptor = authInterceptor;
+export { authInterceptor };
-exports.isUnauthorized = isUnauthorized;
+export { isUnauthorized };
-exports.isAdmin = isAdmin;
+export { isAdmin };

lib/defaultConfig.js

@@ -0,0 +1,7 @@
+export const DEFAULT_CONFIG = {
+            'interval': '60',
+            'port': 9998,
+            'workingHours': {'from': '', 'to': ''},
+            'demoMode': false,
+            'analyticsEnabled': null
+    };

lib/errors.js

@@ -9,7 +9,8 @@ class ExtendableError extends Error {
     }
   }
 }
-
 class NoNewListingsWarning extends ExtendableError {}
-
+export { NoNewListingsWarning };
-module.exports = { NoNewListingsWarning };
+export default {
+  NoNewListingsWarning,
+};

lib/notification/adapter/apprise.js

@@ -0,0 +1,36 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        body: message,
+        title: title,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'apprise',
+  name: 'Apprise',
+  readme: markdown2Html('lib/notification/adapter/apprise.md'),
+  description: 'Fredy will send new listings to your Apprise instance.',
+  fields: {
+    server: {
+      type: 'text',
+      label: 'Server',
+      description: 'The server URL to send the notification to.',
+    },
+  },
+};

lib/notification/adapter/apprise.md

@@ -0,0 +1,3 @@
+### Apprise Adapter
+
+Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.

lib/notification/adapter/console.js

@@ -1,19 +1,12 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
 
-/**
+export const send = ({ serviceName, newListings, jobKey }) => {
- * simply prints out the found data to the console
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
   /* eslint-disable no-console */
   return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
   /* eslint-enable no-console */
 };
-
+export const config = {
-exports.config = {
+  id: 'console',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Console',
   description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
   config: {},

lib/notification/adapter/mailJet.js

@@ -1,35 +1,24 @@
-const mailjet = require('node-mailjet');
+import mailjet from 'node-mailjet';
-
+import path from 'path';
-const path = require('path');
+import fs from 'fs';
-const fs = require('fs');
+import Handlebars from 'handlebars';
-const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/template.hbs'), 'utf8');
+import { markdown2Html } from '../../services/markdown.js';
-
+import { getDirName } from '../../utils.js';
-const Handlebars = require('handlebars');
+const __dirname = getDirName();
+const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
 const emailTemplate = Handlebars.compile(template);
-const { markdown2Html } = require('../../services/markdown');
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-
-/**
- * sends a new listing using MailJet
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailJet'
+    (adapter) => adapter.id === config.id,
   ).fields;
-
   const to = receiver
     .trim()
     .split(',')
     .map((r) => ({
       Email: r.trim(),
     }));
-
   return mailjet
-    .connect(apiPublicKey, apiPrivateKey)
+    .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [
@@ -49,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
       ],
     });
 };
-
+export const config = {
-exports.config = {
+  id: 'mailjet',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'MailJet',
   description: 'MailJet is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/mailJet.md'),

lib/notification/adapter/mattermost.js

@@ -1,26 +1,15 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-const { getJob } = require('../../services/storage/jobStorage');
+import { getJob } from '../../services/storage/jobStorage.js';
-const fetch = require('node-fetch');
+import fetch from 'node-fetch';
-
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-/**
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
- * sends new listings to mattermost
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-
   let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
   message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
   message += newListings.map(
-    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
   );
-
   return fetch(webhook, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
@@ -30,14 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     },
   });
 };
-
+export const config = {
-/**
+  id: 'mattermost',
- * exported config is being used in the frontend to generate the fields
- * incoming values will be the keys (and values) of the fields
- *
- */
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Mattermost',
   readme: markdown2Html('lib/notification/adapter/mattermost.md'),
   description: 'Fredy will send new listings to your mattermost team chat.',

lib/notification/adapter/ntfy.js

@@ -0,0 +1,53 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const message = `
+    Address: ${newListing.address} 
+    Size: ${newListing.size.replace(/2m/g, '$m^2$')} 
+    Price: ${newListing.price}
+    Link: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      body: JSON.stringify({
+        topic: topic,
+        message: message,
+        title: newListing.title,
+        tags: [serviceName, jobName],
+        priority: parseInt(priority),
+        click: newListing.link,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'ntfy',
+  name: 'ntfy',
+  readme: markdown2Html('lib/notification/adapter/ntfy.md'),
+  description: 'Fredy will send new listings to your ntfy.',
+  fields: {
+    priority: {
+      type: 'number',
+      label: 'Priority',
+      description: 'The priority of the send notification.',
+    },
+    server: {
+      type: 'text',
+      label: 'Server-URL',
+      description: 'The server url to the send the notification to.',
+    },
+    topic: {
+      type: 'text',
+      label: 'topic',
+      description:
+        'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
+    },
+  },
+};

lib/notification/adapter/ntfy.md

@@ -0,0 +1,5 @@
+### ntfy Adapter
+
+For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the  [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+
+As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.

lib/notification/adapter/pushover.js

@@ -0,0 +1,73 @@
+import {markdown2Html} from '../../services/markdown.js';
+import {getJob} from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({serviceName, newListings, notificationConfig, jobKey}) => {
+    const {token, user, device} = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+    const job = getJob(jobKey);
+    const jobName = job == null ? jobKey : job.name;
+    const promises = newListings.map((newListing) => {
+        const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+        const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+        return fetch('https://api.pushover.net/1/messages.json', {
+            method: 'POST',
+            headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify({
+                token: token,
+                user: user,
+                message: message,
+                device: device,
+                title: title,
+            }),
+        });
+    });
+
+    return Promise.all(promises)
+        .then((responses) => {
+            // Convert all responses to JSON
+            return Promise.all(responses.map((response) => response.json()));
+        })
+        .then((data) => {
+            // Check for errors in the data
+            const error = data
+                .map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
+                .filter((err) => err !== null);
+
+            if (error.length > 0) {
+                // Reject with the combined error messages
+                return Promise.reject(error.join('; '));
+            }
+
+            return data;
+        })
+        .then(() => {
+            return Promise.resolve();
+        })
+        .catch((error) => {
+            return Promise.reject(error);
+        });
+};
+
+export const config = {
+    id: 'pushover',
+    name: 'Pushover',
+    readme: markdown2Html('lib/notification/adapter/pushover.md'),
+    description: 'Fredy will send new listings to your mobile using Pushover.',
+    fields: {
+        token: {
+            type: 'text',
+            label: 'API token',
+            description: 'Your application\'s API token.',
+        },
+        user: {
+            type: 'text',
+            label: 'User key',
+            description: 'Your user/group key.',
+        },
+        device: {
+            type: 'text',
+            label: 'Device name',
+            description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
+        },
+    },
+};

lib/notification/adapter/pushover.md

@@ -0,0 +1,5 @@
+### Pushover Adapter
+
+Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+
+After setting up the application, please enter both your newly created User key and API token.

lib/notification/adapter/sendGrid.js

@@ -1,16 +1,7 @@
-const sgMail = require('@sendgrid/mail');
+import sgMail from '@sendgrid/mail';
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-/**
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
- * sends a new listing using SendGrid
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,
@@ -28,9 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   };
   return sgMail.send(msg);
 };
-
+export const config = {
-exports.config = {
+  id: 'sendgrid',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'SendGrid',
   description: 'SendGrid is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/sendGrid.md'),

lib/notification/adapter/slack.js

@@ -1,17 +1,8 @@
-const Slack = require('slack');
+import Slack from 'slack';
+import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
-const { markdown2Html } = require('../../services/markdown');
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
-/**
- * sends a new listing to slack
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
   return newListings.map((payload) =>
     msg({
       token,
@@ -44,12 +35,11 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
           ts: new Date().getTime() / 1000,
         },
       ],
-    })
+    }),
   );
 };
-
+export const config = {
-exports.config = {
+  id: 'slack',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Slack',
   readme: markdown2Html('lib/notification/adapter/slack.md'),
   description: 'Fredy will send new listings to the slack channel of your choice..',

lib/notification/adapter/sqlite.js

@@ -1,13 +1,6 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-const Database = require('better-sqlite3');
+import Database from 'better-sqlite3';
-
+export const send = ({ serviceName, newListings, jobKey }) => {
-/**
- * Stores data in a sqlite db in order to use the search results for later analytics
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
   const db = new Database('db/listings.db');
   const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
   db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
@@ -23,9 +16,8 @@ exports.send = ({ serviceName, newListings, jobKey }) => {
   });
   return Promise.resolve();
 };
-
+export const config = {
-exports.config = {
+  id: 'sqlite',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Sqlite',
   description: 'This adapter stores listings in a local sqlite3 database.',
   config: {},

lib/notification/adapter/sqlite.md

@@ -1,3 +1,7 @@
 ### Sqlite Adapter
+This adapter stores search results in a sqlite database located in db/listings.db. This file can be used for further analysis later on.   
 
-This adapter stores search results in an sqlite database in db/listings.db
+Fields are:
+```
+['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description']
+```

lib/notification/adapter/telegram.js

@@ -1,7 +1,6 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-const { getJob } = require('../../services/storage/jobStorage');
+import { getJob } from '../../services/storage/jobStorage.js';
-const fetch = require('node-fetch');
+import fetch from 'node-fetch';
-
 const MAX_ENTITIES_PER_CHUNK = 8;
 const RATE_LIMIT_INTERVAL = 1010;
 /**
@@ -16,32 +15,23 @@ const arrayChunks = (inputArray, perChunk) =>
     all[ch] = [].concat(all[ch] || [], one);
     return all;
   }, []);
-
+function shorten(str, len = 30) {
-/**
+  return str.length > len ? str.substring(0, len) + '...' : str;
- * sends new listings to telegram
+}
- * @param serviceName e.g immowelt
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
- * @param newListings an array with newly found listings
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
   const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
-
   const promises = chunks.map((chunk) => {
     let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
     message += chunk.map(
       (o) =>
-        `<a href="${o.link}"><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
+        `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
         [o.address, o.price, o.size].join(' | ') +
-        '\n\n'
+        '\n\n',
     );
-
     /**
      * This is to not break the rate limit. It is to only send 1 message per second
      */
@@ -66,21 +56,10 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
       }, RATE_LIMIT_INTERVAL);
     });
   });
-
   return Promise.all(promises);
 };
-
+export const config = {
-function shorten(str, len = 30) {
+  id: 'telegram',
-  return str.length > len ? str.substring(0, len) + '...' : str;
-}
-
-/**
- * exported config is being used in the frontend to generate the fields
- * incoming values will be the keys (and values) of the fields
- *
- */
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Telegram',
   readme: markdown2Html('lib/notification/adapter/telegram.md'),
   description: 'Fredy will send new listings to your mobile, using Telegram.',

lib/notification/notify.js

@@ -1,24 +1,24 @@
-const fs = require('fs');
+import fs from 'fs';
 const path = './adapter';
 
 /** Read every integration existing in ./adapter **/
-const adapter = fs
+const adapter = await Promise.all(
-  .readdirSync('./lib/notification/adapter')
+  fs
-  .filter((file) => file.endsWith('.js'))
+    .readdirSync('./lib/notification/adapter')
-  .map((integPath) => require(`${path}/${integPath}`));
+    .filter((file) => file.endsWith('.js'))
+    .map(async (integPath) => await import(`${path}/${integPath}`))
+);
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
-
+const findAdapter = (notificationAdapter) => {
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
+  return adapter.find((a) => a.config.id === notificationAdapter.id);
+};
+export const send = (serviceName, newListings, notificationConfig, jobKey) => {
   //this is not being used in tests, therefore adapter are always set
   return notificationConfig
     .filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
     .map((notificationAdapter) => findAdapter(notificationAdapter))
     .map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
 };
-
-const findAdapter = (notificationAdapter) => {
-  return adapter.find((a) => a.config.id === notificationAdapter.id);
-};

lib/provider/einsAImmobilien.js

@@ -1,21 +1,33 @@
-const utils = require('../utils');
+import utils, { buildHash } from '../utils.js';
-
 let appliedBlackList = [];
 
 function normalize(o) {
-  let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
-  if (o.rooms != null) {
-    size += ` / / ${o.rooms.trim()}`;
-  }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-
+  const price = normalizePrice(o.price);
-  return Object.assign(o, { size, link });
+  const id = buildHash(o.id, price);
+  return Object.assign(o, { id, price, link });
 }
 
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
+}
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
@@ -23,28 +35,24 @@ const config = {
   url: null,
   crawlContainer: '.tabelle',
   sortByDateParam: 'sort_type=newest',
+  waitForSelector: 'body',
   crawlFields: {
     id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
-    price: '.tabelle .inner_object_data .single_data_price | removeNewline | trim',
+    price: '.inner_object_data .single_data_price | removeNewline | trim',
-    size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
+    size: '.tabelle .tabelle_inhalt_infos .single_data_box  | removeNewline | trim',
-    rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
+    title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: '1a Immobilien',
   baseUrl: 'https://www.1a-immobilienmarkt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'einsAImmobilien',
 };
-
+export { config };
-exports.config = config;

lib/provider/immobilienDe.js

@@ -1,37 +1,33 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
-
 let appliedBlackList = [];
-
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
 }
-
 function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
-
 function normalize(o) {
-  const id = parseId(shortenLink(o.link));
   const size = o.size || 'N/A m²';
   const price = o.price || 'N/A €';
   const title = o.title || 'No title available';
   const address = o.address || 'No address available';
-  const link = shortenLink(o.link);
+  const shortLink = shortenLink(o.link);
+  const link = `https://www.immobilien.de/${shortLink}`;
+  const id = buildHash(parseId(shortLink), o.price);
   return Object.assign(o, { id, price, size, title, address, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '.estates_list .list_immo a._ref',
+  crawlContainer: '._ref',
   sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
+  waitForSelector: 'body',
   crawlFields: {
+    id: '@href', //will be transformed later
     price: '.list_entry .immo_preis .label_info',
     size: '.list_entry .flaeche .label_info | removeNewline | trim',
     title: '.list_entry .part_text h3 span',
@@ -39,21 +35,17 @@ const config = {
     link: '@href',
     address: '.list_entry .place',
   },
-  paginate: '.list_immo .blocknav .blocknav_list li.next a@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immobilien.de',
   baseUrl: 'https://www.immobilien.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immobilienDe',
 };
-
+export { config };
-exports.config = config;

lib/provider/immonet.js

@@ -1,52 +1,50 @@
-const utils = require('../utils');
+import utils, { buildHash } from '../utils.js';
-
 let appliedBlackList = [];
 
+/**
+ * Note, Immonet is rly a piece of sh*t. It is using a weird combination of React and some buttons (instead of links),
+ * so that if somebody clicks the listing, a new page will open with the actual link to the listing. Of course, a scraper
+ * cannot do this (which is why I always just return the link to the whole list of listings).
+ * This is not only bad for us, but also bad for ppl with disabilities...
+ */
+
 function normalize(o) {
-  const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
-  //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
+  const link = config.url;
-  //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
+  const id = buildHash(title, price);
-  const link = `https://www.immonet.de/angebot/${id}`;
   return Object.assign(o, { id, address, price, size, title, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '#result-list-stage .item',
+  crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
   sortByDateParam: 'sortby=19',
+  waitForSelector: 'div[data-testid="serp-resultscount-testid"]',
   crawlFields: {
-    id: '@id',
+    id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
-    price: 'div[id*="selPrice_"] | trim',
+    title: 'button@title |trim',
-    size: 'div[id*="selArea_"] | trim',
+    price: 'div[data-testid="cardmfe-price-testid"] | trim',
-    title: '.item a img@title',
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
+    address: 'div[data-testid="cardmfe-description-box-address"] | trim',
   },
-  paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immonet',
   baseUrl: 'https://www.immonet.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immonet',
 };
-
+export { config };
-exports.config = config;

lib/provider/immoscout.js

@@ -1,49 +1,42 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
-
 let appliedBlackList = [];
-
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
-
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
-  return Object.assign(o, { title, address, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, title, address, link });
 }
-
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
-
 const config = {
   url: null,
   crawlContainer: '#resultListItems li.result-list__listing',
   sortByDateParam: 'sorting=2',
+  waitForSelector: 'body',
   crawlFields: {
     id: '.result-list-entry@data-obid | int',
     price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
     size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
+    title: '.result-list-entry .result-list-entry__brand-title-container h2 | removeNewline | trim',
     link: '.result-list-entry .result-list-entry__brand-title-container@href',
     address: '.result-list-entry .result-list-entry__map-link',
   },
-  paginate: '#pager .align-right a@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immoscout',
   baseUrl: 'https://www.immobilienscout24.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immoscout',
 };
-
+export { config };
-exports.config = config;

lib/provider/immoswp.js

@@ -1,52 +1,48 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+    const size = o.size || 'N/A m²';
-  const size = o.size || 'N/A m²';
+    const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
-  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+    const title = o.title || 'No title available';
-  const address = o.address || 'No address available';
+    const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
-  const title = o.title || 'No title available';
+    const link = `https://immo.swp.de/immobilien/${immoId}`;
-  const link = `https://immo.swp.de/immobilien/${id}`;
+    const description = o.description;
-  const description = o.description;
+    const id = buildHash(immoId, price);
-  return Object.assign(o, { id, address, price, size, title, link, description });
+    return Object.assign(o, {id, price, size, title, link, description});
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
+    return titleNotBlacklisted && descNotBlacklisted;
-  return titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.js-serp-item',
+    crawlContainer: '.js-serp-item',
-  sortByDateParam: 's=most_recently_updated_first',
+    sortByDateParam: 's=most_recently_updated_first',
-  crawlFields: {
+    waitForSelector: 'body',
-    id: '@id',
+    crawlFields: {
-    price: 'div.item__spec.item-spec-price | trim',
+        id: '.js-bookmark-btn@data-id',
-    size: 'div.item__spec.item-spec-area | trim',
+        price: 'div.align-items-start div:first-child | trim',
-    title: 'a.js-item-title-link@title',
+        size: 'div.align-items-start div:nth-child(3) | trim',
-    address: 'div.item__locality | removeNewline | trim',
+        title: '.card-title h2 | trim',
-    description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
+        link: '.ci-search-result__link@href',
-  },
+        description: '.js-show-more-item-sm | removeNewline | trim',
-  paginate: 'li.page-item.pagination__item a.page-link@href',
+    },
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
+    config.enabled = sourceConfig.enabled;
-  config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
-  config.url = sourceConfig.url;
+    appliedBlackList = blacklist || [];
-  appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
+    name: 'Immo Südwest Presse',
-  name: 'Immo Südwest Presse',
+    baseUrl: 'https://immo.swp.de/',
-  baseUrl: 'https://immo.swp.de/',
+    id: 'immoswp',
-  id: __filename.slice(__dirname.length + 1, -3),
 };
-
+export {config};
-exports.config = config;

lib/provider/immowelt.js

@@ -1,44 +1,43 @@
-const utils = require('../utils');
+import utils, { buildHash } from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  return o;
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id });
 }
 
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
   url: null,
-  crawlContainer: "div[class^='EstateItem-']",
+  crawlContainer:
-  sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
+    'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
+  sortByDateParam: 'order=DateDesc',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
-    id: 'a@id',
+    id: 'a@href',
-    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
+    price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
-    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: "div[class^='FactsMain-'] h2",
+    title: '.css-1cbj9xw',
     link: 'a@href',
-    address: "div[class^='estateFacts-'] span | removeNewline | trim",
+    address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },
-  paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immowelt',
   baseUrl: 'https://www.immowelt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immowelt',
 };
-
+export { config };
-exports.config = config;

lib/provider/kleinanzeigen.js

@@ -1,53 +1,50 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
 
 function normalize(o) {
-  const size = o.size || '--- m²';
+    const size = o.size || '--- m²';
-
+    const id = buildHash(o.id, o.price);
-  return Object.assign(o, { size });
+    const link = `https://www.kleinanzeigen.de${o.link}`;
+    return Object.assign(o, {id, size, link});
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  const isBlacklistedDistrict =
+    const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
+        appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-
+    return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '#srchrslt-adtable .ad-listitem ',
+    crawlContainer: '#srchrslt-adtable .ad-listitem ',
-  //sort by date is standard oO
+    //sort by date is standard oO
-  sortByDateParam: null,
+    sortByDateParam: null,
-  crawlFields: {
+    waitForSelector: 'body',
-    id: '.aditem@data-adid | int',
+    crawlFields: {
-    price: '.aditem-main--middle--price | removeNewline | trim',
+        id: '.aditem@data-adid | int',
-    size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
+        price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
-    title: '.aditem-main .text-module-begin a | removeNewline | trim',
+        size: '.aditem-main .text-module-end | removeNewline | trim',
-    link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
+        title: '.aditem-main .text-module-begin a | removeNewline | trim',
-    description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
+        link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
-    address: '.aditem-main--top--left | trim | removeNewline',
+        description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
-  },
+        address: '.aditem-main--top--left | trim | removeNewline',
-  paginate: '#srchrslt-pagination .pagination-next@href',
+    },
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
-
+export const metaInformation = {
-exports.metaInformation = {
+    name: 'Ebay Kleinanzeigen',
-  name: 'Ebay Kleinanzeigen',
+    baseUrl: 'https://www.kleinanzeigen.de/',
-  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
+    id: 'kleinanzeigen',
-  id: __filename.slice(__dirname.length + 1, -3),
 };
-
+export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
+    config.enabled = sourceConfig.enabled;
-  config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
-  config.url = sourceConfig.url;
+    appliedBlacklistedDistricts = blacklistedDistricts || [];
-  appliedBlacklistedDistricts = blacklistedDistricts || [];
+    appliedBlackList = blacklist || [];
-  appliedBlackList = blacklist || [];
 };
-
+export {config};
-exports.config = config;

lib/provider/neubauKompass.js

@@ -1,41 +1,44 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 
+function nullOrEmpty(val) {
+    return val == null || val.length === 0;
+}
+
 function normalize(o) {
-  return o;
+    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+    const id = buildHash(o.link, o.price);
+    return Object.assign(o, {id, link});
 }
 
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
+    return !utils.isOneOf(o.title, appliedBlackList);
 }
 
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '.nbk-container >div article',
+    crawlContainer: '.col-12.mb-4',
-  sortByDateParam: 'Sortierung=Id&Richtung=DESC',
+    sortByDateParam: 'Sortierung=Id&Richtung=DESC',
-  crawlFields: {
+    waitForSelector: '.nbk-section',
-    id: '@id',
+    crawlFields: {
-    title: 'a.nbk-truncate@title | removeNewline | trim',
+        id: 'a@href',
-    link: 'a.nbk-truncate@href',
+        title: 'a@title | removeNewline | trim',
-    address: 'p.nbk-truncate | removeNewline | trim',
+        link: 'a@href',
-    price: 'p.nbk-mb-0 | removeNewline | trim',
+        address: '.nbk-project-card__description | removeNewline | trim',
-  },
+        price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
-  paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
+    },
-  normalize: normalize,
+    normalize: normalize,
-  filter: applyBlacklist,
+    filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
+    config.enabled = sourceConfig.enabled;
-  config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
-  config.url = sourceConfig.url;
+    appliedBlackList = blacklist || [];
-  appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
+    name: 'Neubau Kompass',
-  name: 'Neubau Kompass',
+    baseUrl: 'https://www.neubaukompass.de/',
-  baseUrl: 'https://www.neubaukompass.de/',
+    id: 'neubauKompass',
-  id: __filename.slice(__dirname.length + 1, -3),
 };
-
+export {config};
-exports.config = config;

lib/provider/wgGesucht.js

@@ -1,44 +1,43 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  return o;
+    const id = buildHash(o.id, o.price);
+    const link = `https://www.wg-gesucht.de${o.link}`;
+    return Object.assign(o, { id, link });
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
+    return o.id != null && titleNotBlacklisted && descNotBlacklisted;
-  return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-  url: null,
+    url: null,
-  crawlContainer: '#main_column .wgg_card',
+    crawlContainer: '#main_column .wgg_card',
-  sortByDateParam: 'sort_column=0&sort_order=0',
+    sortByDateParam: 'sort_column=0&sort_order=0',
-  crawlFields: {
+    waitForSelector: 'body',
-    id: '@data-id',
+    crawlFields: {
-    details: '.row .noprint .col-xs-11 |removeNewline |trim',
+        id: '@data-id',
-    price: '.middle .col-xs-3 |removeNewline |trim',
+        details: '.row .noprint .col-xs-11 |removeNewline |trim',
-    size: '.middle .text-right |removeNewline |trim',
+        price: '.middle .col-xs-3 |removeNewline |trim',
-    title: '.truncate_title a |removeNewline |trim',
+        size: '.middle .text-right |removeNewline |trim',
-    link: '.truncate_title a@href',
+        title: '.truncate_title a |removeNewline |trim',
-  },
+        link: '.truncate_title a@href',
-  normalize: normalize,
+    },
-  filter: applyBlacklist,
+    normalize: normalize,
+    filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
+    config.enabled = sourceConfig.enabled;
-  config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
-  config.url = sourceConfig.url;
+    appliedBlackList = blacklist || [];
-  appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
+    name: 'Wg gesucht',
-  name: 'Wg gesucht',
+    baseUrl: 'https://www.wg-gesucht.de/',
-  baseUrl: 'https://www.wg-gesucht.de/',
+    id: 'wgGesucht',
-  id: __filename.slice(__dirname.length + 1, -3),
 };
-
+export {config};
-exports.config = config;

lib/services/demoCleanup.js

@@ -0,0 +1,29 @@
+import { setInterval } from 'node:timers';
+import {removeJobsByUserName} from './storage/jobStorage.js';
+import {config} from '../utils.js';
+
+/**
+ * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
+ */
+export function cleanupDemoAtMidnight() {
+    const now = new Date();
+    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+        - now.getUTCMinutes() * 60 * 1000
+        - now.getUTCSeconds() * 1000
+        - now.getUTCMilliseconds();
+
+    setTimeout(() => {
+        cleanup();
+
+        setInterval(() => {
+            cleanup();
+        }, 24 * 60 * 60 * 1000);
+
+    }, millisUntilMidnightUTC);
+}
+
+function cleanup(){
+    if(config.demoMode){
+        removeJobsByUserName('demo');
+    }
+}

lib/services/extractor/extractor.js

@@ -0,0 +1,43 @@
+import { setDebug } from './utils.js';
+import puppeteerExtractor from './puppeteerExtractor.js';
+import { loadParser, parse } from './parser/parser.js';
+
+const DEFAULT_OPTIONS = {
+  debug: false,
+  puppeteerTimeout: 60_000,
+  puppeteerHeadless: true,
+};
+
+export default class Extractor {
+  constructor(options) {
+    this.options = {
+      ...DEFAULT_OPTIONS,
+      ...options,
+    };
+    this.responseText = null;
+    setDebug(this.options);
+  }
+
+  /**
+   * if you are extracting data from a SPA, you must provide a selector, otherwise
+   * your response will never contain what you are really looking for
+   * @param url
+   * @param waitForSelector
+   */
+  execute = async (url, waitForSelector = null) => {
+    this.responseText = null;
+    try {
+      this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+      if (this.responseText != null) {
+        loadParser(this.responseText);
+      }
+    } catch (error) {
+      console.error('Error trying to load page.', error);
+    }
+    return this;
+  };
+
+  parseResponseText = (crawlContainer, crawlFields, url) => {
+    return parse(crawlContainer, crawlFields, this.responseText, url);
+  };
+}

lib/services/extractor/parser/parser.js

@@ -0,0 +1,97 @@
+import * as cheerio from 'cheerio';
+
+let $ = null;
+
+export function loadParser(text) {
+  $ = cheerio.load(text);
+}
+
+export function parse(crawlContainer, crawlFields, text, url) {
+  if (!text) {
+    console.warn('Cannot parse, text was empty for url ', url);
+    return null;
+  }
+
+  if (!crawlContainer || !crawlFields) {
+    console.warn('Cannot parse, selector was empty for url ', url);
+    return null;
+  }
+
+  const result = [];
+
+  if ($(crawlContainer).length === 0) {
+    console.warn('No elements in crawl container found for url ', url);
+    return null;
+  }
+
+  $(crawlContainer).each((_, element) => {
+    const container = $(element);
+    const parsedObject = {};
+
+    // Parse fields based on crawlFields
+    for (const [key, fieldSelector] of Object.entries(crawlFields)) {
+      let value;
+
+      try {
+        const selector = fieldSelector.includes('|')
+          ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim()
+          : fieldSelector;
+
+        if (selector.includes('@')) {
+          const [sel, attr] = selector.split('@');
+          if (sel.length === 0) {
+            value = container.attr(attr.trim());
+          } else {
+            value = container.find(sel.trim()).attr(attr.trim());
+          }
+        } else {
+          value = container.find(selector.trim()).text();
+        }
+
+        // Apply modifiers if specified
+        if (fieldSelector.includes('|')) {
+          /* eslint-disable no-unused-vars */
+          const [_, ...modifiers] = fieldSelector.split('|').map((s) => s.trim());
+          /* eslint-disable no-unused-vars */
+          value = applyModifiers(value, modifiers);
+        }
+
+        parsedObject[key] = value || null;
+      } catch (error) {
+        console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
+        parsedObject[key] = null;
+      }
+    }
+
+    if (parsedObject.id != null) {
+      result.push(parsedObject);
+    } else {
+      console.warn('ID not found. Not relaying object.');
+    }
+  });
+
+  return result;
+}
+
+// Helper function to apply modifiers
+function applyModifiers(value, modifiers) {
+  if (!value) return value;
+
+  modifiers.forEach((modifier) => {
+    switch (modifier) {
+      case 'int':
+        value = parseInt(value, 10);
+        break;
+      case 'trim':
+        value = value.replace(/\s+/g, ' ').trim();
+        break;
+      case 'removeNewline':
+        value = value.replace(/\n/g, ' ');
+        break;
+      default:
+        console.warn(`Unknown modifier: ${modifier}`);
+    }
+  });
+
+  return value;
+}

lib/services/extractor/puppeteerExtractor.js

@@ -0,0 +1,49 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
+
+puppeteer.use(StealthPlugin());
+
+export default async function execute(url, waitForSelector, options) {
+  let browser;
+  try {
+    debug(`Sending request to ${url} using Puppeteer.`);
+
+    browser = await puppeteer.launch({
+      headless: options.puppeteerHeadless ?? true,
+      args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox'],
+      timeout: options.puppeteerTimeout || 30_000,
+    });
+    let page = await browser.newPage();
+    await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+    const response = await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+    });
+    let pageSource;
+    //if we're extracting data from a spa, we must wait for the selector
+    if (waitForSelector != null) {
+      await page.waitForSelector(waitForSelector);
+      pageSource = await page.evaluate((selector) => {
+        return document.querySelector(selector).innerHTML;
+      }, waitForSelector);
+    } else {
+      pageSource = await page.content();
+    }
+
+    const statusCode = response.status();
+
+    if (botDetected(pageSource, statusCode)) {
+      console.warn('We have been detected as a bot :-/ Tried url: => ', url);
+      return null;
+    }
+
+    return await page.content();
+  } catch (error) {
+    console.error('Error executing with puppeteer executor', error);
+    return null;
+  } finally {
+    if (browser != null) {
+      await browser.close();
+    }
+  }
+}

lib/services/extractor/utils.js

@@ -0,0 +1,32 @@
+let debuggingOn = false;
+
+export const DEFAULT_HEADER = {
+  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.5',
+  Connection: 'keep-alive',
+  'Upgrade-Insecure-Requests': '1',
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+};
+
+export const setDebug = (options) => {
+  debuggingOn = !!options?.debug;
+};
+
+export const debug = (message) => {
+  if (debuggingOn) {
+    /* eslint-disable no-console */
+    console.debug(message);
+    /* eslint-enable no-console */
+  }
+};
+
+export const botDetected = (pageSource, statusCode) => {
+  const suspiciousStatusCodes = [403, 429];
+  const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
+
+  const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
+  const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
+
+  return detectedInSource || detectedByStatus;
+};

lib/services/markdown.js

@@ -1,6 +1,6 @@
-const markdown = require('markdown').markdown;
+import markdown$0 from 'markdown';
-const fs = require('fs');
+import fs from 'fs';
-
+const markdown = markdown$0.markdown;
-exports.markdown2Html = function markdown2Html(filePath) {
+export function markdown2Html(filePath) {
   return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
-};
+}

lib/services/queryStringMutator.js

@@ -1,21 +1,9 @@
-const queryString = require('query-string');
+import queryString from 'query-string';
-
+export default (_url, sortByDateParam) => {
-/**
- * for Fredy, it is important to sort search results by date, starting with the latest listing. if it is not sorted, we
- * might never actually find the newest results, no matter how many pages we crawl.
- * It has been written in the documentation, but obviously nobody reads docu theses days which is why it's been done
- * automagically now.
- *
- * @param _url actual provider url containing the searchParams
- * @param sortByDateParam param(s) indicating the correct sort order
- * @returns {`${string}?${string}`} correctly formatted url
- */
-module.exports = (_url, sortByDateParam) => {
   //if no mutation is necessary, just return the original url
   if (sortByDateParam == null) {
     return _url;
   }
-
   const original = queryString.parseUrl(_url);
   const mutate = queryString.parse(sortByDateParam);
   return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;

lib/services/requestDriver.js

@@ -1,78 +0,0 @@
-const fetch = require('node-fetch');
-const config = require('../../conf/config.json');
-
-const { makeUrlResidential } = require('./scrapingAnt');
-//if ScrapingAnt got blocked, this http status is returned
-const BLOCKED_HTTP_STATUS = 423;
-const NOT_FOUND_HTTP_STATUS = 404;
-const MAX_RETRIES_SCRAPING_ANT = 10;
-const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
-
-function makeDriver(headers = {}) {
-  let cookies = '';
-
-  async function scrapingAntDriver(context, callback, retryCounter = 0) {
-    const proxyType = config.scrapingAnt?.proxy || 'datacenter';
-
-    try {
-      const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
-      const response = await fetch(url, {
-        headers: {
-          ...headers,
-          cookie: cookies,
-        },
-      });
-
-      const result = await response.text();
-      if (cookies.length === 0) {
-        cookies = response.headers.raw()['set-cookie'] || [];
-      }
-
-      callback(null, result);
-    } catch (exception) {
-      /* eslint-disable no-console */
-      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
-        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
-        callback(null, []);
-        return;
-      }
-
-      if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
-        retryCounter++;
-        console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
-        await scrapingAntDriver(context, callback, retryCounter);
-      } else {
-        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
-        callback(null, []);
-      }
-      /* eslint-enable no-console */
-    }
-  }
-
-  /**
-   * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
-   * everything != Immoscout as of writing this)
-   */
-  return async function driver(context, callback) {
-    if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
-      return scrapingAntDriver(context, callback);
-    }
-
-    try {
-      const response = await fetch(context.url, {
-        headers: {
-          ...headers,
-          Cookie: cookies,
-        },
-      });
-
-      const result = await response.text();
-      callback(null, result);
-    } catch (exception) {
-      console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
-      callback(null, []);
-    }
-  };
-}
-
-module.exports = makeDriver;

lib/services/scraper.js

@@ -1,46 +0,0 @@
-const config = require('../../conf/config.json');
-const makeDriver = require('./requestDriver');
-const Xray = require('x-ray');
-
-class Scraper {
-  constructor() {
-    const filters = {
-      removeNewline: this._removeNewline,
-      trim: this._trim,
-      int: this._int,
-    };
-
-    const headers = {
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-    };
-
-    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
-      headers['x-api-key'] = config.scrapingAnt.apiKey;
-    }
-    const driver = makeDriver(headers);
-
-    const xray = Xray({ filters });
-    xray.driver(driver);
-
-    this.xray = xray;
-  }
-
-  get x() {
-    return this.xray;
-  }
-
-  _removeNewline(value) {
-    return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
-  }
-
-  _trim(value) {
-    return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
-  }
-
-  _int(value) {
-    return typeof value === 'string' ? parseInt(value, 10) : value;
-  }
-}
-
-module.exports = new Scraper().x;

lib/services/scrapingAnt.js

@@ -1,25 +0,0 @@
-const { metaInformation } = require('../provider/immoscout');
-//to better configure re-capture chose a random proxy each time we do a call
-const config = require('../../conf/config.json');
-
-const isImmoscout = (id) => {
-  return id.toLowerCase() === metaInformation.id;
-};
-
-exports.transformUrlForScrapingAnt = (url, id) => {
-  if (isImmoscout(id)) {
-    //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
-  }
-  return url;
-};
-
-exports.isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
-};
-
-exports.isImmoscout = isImmoscout;
-
-exports.makeUrlResidential = (url) => {
-  return url.replace('datacenter', 'residential');
-};

lib/services/security/hash.js

@@ -1,3 +1,2 @@
-const crypto = require('crypto');
+import crypto from 'crypto';
-
+export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
-exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/similarity-check/SimilarityCacheEntry.js

@@ -1,27 +1,17 @@
-const stringSimilarity = require('string-similarity');
+import stringSimilarity from 'string-similarity';
-
 //if the score is higher than this, it will be considered a match
 const MAX_DICE_INDEX = 0.7;
-
+export default (class SimilarityCacheEntry {
-/**
- * The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
- *
- * @type {module.SimilarityCacheEntry}
- */
-module.exports = class SimilarityCacheEntry {
   constructor(time) {
     this.time = time;
     this.values = [];
   }
-
   setCacheEntry = (entry) => {
     this.values.push(entry);
   };
-
   getTime = () => {
     return this.time;
   };
-
   hasSimilarEntries = (value) => {
     if (this.values.length > 0) {
       for (let i = 0; i < this.values.length; i++) {
@@ -33,4 +23,4 @@ module.exports = class SimilarityCacheEntry {
     }
     return false;
   };
-};
+});

lib/services/similarity-check/similarityCache.js

@@ -1,63 +1,40 @@
-/**
+import SimilarityCacheEntry from './SimilarityCacheEntry.js';
- * each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
+import { config } from '../../utils.js';
- * a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
- * the user won't see any duplicates
- *
- * The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
- *
- * @type {module.SimilarityCacheEntry|{}}
- */
-const SimilarityCacheEntry = require('./SimilarityCacheEntry');
-const config = require('../../../conf/config.json');
-
 //5 minutes
 let retention = 5 * 60 * 1000;
-
 const intervalInMs = config.interval * 60 * 1000;
 //an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
 if (intervalInMs <= retention) {
   retention = Math.floor(intervalInMs / 2);
 }
-
 //jobid -> SimilarityCacheEntry
 const cache = {};
-
 let intervalId;
-
-exports.addCacheEntry = (jobId, value) => {
-  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
-  cache[jobId].setCacheEntry(value);
-};
-
-exports.hasSimilarEntries = (jobId, value) => {
-  if (cache[jobId] == null) {
-    return false;
-  }
-
-  return cache[jobId].hasSimilarEntries(value);
-};
-
 /**
  * cleanup
  */
 intervalId = setInterval(() => {
   const keysToBeRemoved = [];
   const now = Date.now();
-
   Object.keys(cache).forEach((key) => {
     if (cache[key].getTime() + retention < now) {
       keysToBeRemoved.push(key);
     }
   });
-
   if (keysToBeRemoved.length > 0) {
     keysToBeRemoved.forEach((key) => delete cache[key]);
   }
 }, 10000);
-
+export const addCacheEntry = (jobId, value) => {
-/**
+  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
- * mostly used for tests
+  cache[jobId].setCacheEntry(value);
- */
+};
-exports.stopCacheCleanup = () => {
+export const hasSimilarEntries = (jobId, value) => {
+  if (cache[jobId] == null) {
+    return false;
+  }
+  return cache[jobId].hasSimilarEntries(value);
+};
+export const stopCacheCleanup = () => {
   clearInterval(intervalId);
 };

lib/services/storage/LowDashAdapter.js

@@ -0,0 +1,8 @@
+import lodash from 'lodash';
+import { LowSync } from 'lowdb';
+export default class LowdashAdapter extends LowSync {
+  constructor(adapter, defaultData = {}) {
+    super(adapter, defaultData);
+    this.chain = lodash.chain(this).get('data');
+  }
+}

lib/services/storage/jobStorage.js

@@ -1,28 +1,29 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
+import { nanoid } from 'nanoid';
-const FileSync = require('lowdb/adapters/FileSync');
+import * as listingStorage from './listingsStorage.js';
-const adapter = new FileSync(DB_PATH);
+import { getDirName } from '../../utils.js';
-const low = require('lowdb');
+import path from 'path';
-const db = low(adapter);
+import LowdashAdapter from './LowDashAdapter.js';
-const { nanoid } = require('nanoid');
-const listingStorage = require('./listingsStorage');
 
-db.defaults({ jobs: [] }).write();
+const file = path.join(getDirName(), '../', 'db/jobs.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, { jobs: [] });
 
-exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+db.read();
+
+
+export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
       ? null
-      : db
+      : db.chain
           .get('jobs')
           .find((job) => job.id === jobId)
           .value();
-
+  const jobs = db.chain
-  const jobs = db
     .get('jobs')
-    .value()
+    .filter((job) => job.id !== jobId)
-    .filter((job) => job.id !== jobId);
+    .value();
-
   jobs.push({
     id: jobId || nanoid(),
     //make sure to not overwrite the user id in case an admin changes the job
@@ -33,57 +34,66 @@ exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, no
     provider,
     notificationAdapter,
   });
-
+  db.chain.set('jobs', jobs).value();
-  db.set('jobs', jobs).write();
+  db.write();
 };
-
+export const getJob = (jobId) => {
-exports.getJob = (jobId) => {
+  const job = db.chain
-  const job = db
     .get('jobs')
     .find((job) => job.id === jobId)
     .value();
-
   if (job == null) {
     return null;
   }
-
   return {
     ...job,
     numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
   };
 };
-
+export const setJobStatus = ({ jobId, status }) => {
-exports.setJobStatus = ({ jobId, status }) => {
+  db.chain
-  db.get('jobs')
+    .get('jobs')
     .find((job) => job.id === jobId)
     .assign({ enabled: status })
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeJob = (jobId) => {
-exports.removeJob = (jobId) => {
   listingStorage.removeListings(jobId);
-  db.get('jobs')
+  db.chain
+    .get('jobs')
     .remove((job) => job.id === jobId)
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeJobsByUserId = (userId) => {
-exports.removeJobsByUserId = (userId) => {
+  db.chain
-  db.get('jobs')
+    .get('jobs')
-    .value()
     .filter((job) => job.userId === userId)
     .forEach((job) => listingStorage.removeListings(job.id));
-
+  db.chain
-  db.get('jobs')
+    .get('jobs')
-    .remove((job) => job.userId === userId)
+    .remove((job) => job.userId === userId)
-    .write();
+    .value();
-};
+  db.write();
-
+};
-exports.getJobs = () => {
+export const removeJobsByUserName = (userName) => {
-  return db
+  db.chain
+      .get('jobs')
+      .filter((job) => job.username === userName)
+      .forEach((job) => listingStorage.removeListings(job.id));
+  db.chain
+      .get('jobs')
+      .remove((job) => job.username === userName)
+      .value();
+  db.write();
+};
+export const getJobs = () => {
+  return db.chain
     .get('jobs')
-    .value()
     .map((job) => ({
       ...job,
       numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
-    }));
+    }))
+    .value();
 };

lib/services/storage/listingsStorage.js

@@ -1,10 +1,13 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
+const file = path.join(getDirName(), '../', 'db/jobListingData.json');
-const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new JSONFileSync(file);
-const adapter = new FileSync(DB_PATH);
+const db = new LowdashAdapter(adapter, {});
-const low = require('lowdb');
+
-const db = low(adapter);
+db.read();
 
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
@@ -19,35 +22,31 @@ const buildKey = (jobKey, providerId, endpoint) => {
   }
   return key;
 };
-
+export const getNumberOfAllKnownListings = (jobId) => {
-exports.getNumberOfAllKnownListings = (jobId) => {
+  const data = db.chain.get(`${jobId}.providerData`).value() || {};
-  const data = db.get(`${jobId}.providerData`).value() || {};
   return Object.values(data)
     .map((values) => Object.keys(values).length)
     .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
 };
-
+export const getListingProviderDataForAnalytics = (jobId) => {
-exports.getListingProviderDataForAnalytics = (jobId) => {
   const key = buildKey(jobId, 'providerData');
-  return db.get(key).value() || {};
+  return db.chain.get(key).value() || {};
 };
-
+export const getKnownListings = (jobId, providerId) => {
-exports.getKnownListings = (jobId, providerId) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-  return db.get(providerListingsKey).value() || {};
+  return db.chain.get(providerListingsKey).value() || {};
 };
-
+export const setKnownListings = (jobId, providerId, listings) => {
-exports.setKnownListings = (jobId, providerId, listings) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-
+  db.chain.set(providerListingsKey, listings).value();
-  return db.set(providerListingsKey, listings).write();
+  return db.write();
 };
-
+export const setLastJobExecution = (jobId) => {
-exports.setLastJobExecution = (jobId) => {
   const key = buildKey(jobId, null, 'lastExecution');
-  return db.set(key, Date.now()).write();
+  db.chain.set(key, Date.now()).value();
+  return db.write();
 };
-
+export const removeListings = (jobId) => {
-exports.removeListings = (jobId) => {
+  db.chain.unset(jobId).value();
-  db.unset(jobId).write();
+  db.write();
 };

lib/services/storage/userStorage.js

@@ -1,30 +1,40 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
-const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
+import {config, getDirName} from '../../utils.js';
-const FileSync = require('lowdb/adapters/FileSync');
+import * as hasher from '../security/hash.js';
-const adapter = new FileSync(DB_PATH);
+import { nanoid } from 'nanoid';
-const low = require('lowdb');
+import * as jobStorage from './jobStorage.js';
-const db = low(adapter);
+import path from 'path';
-const hasher = require('../security/hash');
+import LowdashAdapter from './LowDashAdapter.js';
-const { nanoid } = require('nanoid');
-const jobStorage = require('./jobStorage');
 
-db.defaults({
+const defaultData = {
-  user: [
+    user: [
-    //you probably want to change the default password ;)
+        //you probably want to change the default password ;)
-    {
+        {
-      id: nanoid(),
+            id: nanoid(),
-      lastLogin: Date.now(),
+            lastLogin: Date.now(),
-      username: 'admin',
+            username: 'admin',
-      password: hasher.hash('admin'),
+            password: hasher.hash('admin'),
-      isAdmin: true,
+            isAdmin: true,
-      isDemo: false,
+        },
-    },
+        {
-  ],
+            id: nanoid(),
-}).write();
+            lastLogin: Date.now(),
+            username: 'demo',
+            password: hasher.hash('demo'),
+            isAdmin: true,
+        },
+    ],
+};
 
-exports.getUsers = (withPassword) => {
+const file = path.join(getDirName(), '../', 'db/users.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, defaultData);
+
+db.read();
+
+export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();
-  return db
+  return db.chain
     .get('user')
     .value()
     .map((user) => ({
@@ -34,13 +44,12 @@ exports.getUsers = (withPassword) => {
       numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
     }));
 };
-
+export const getUser = (id) => {
-exports.getUser = (id) => {
   const jobs = jobStorage.getJobs();
-  const user = db
+  const user = db.chain
     .get('user')
-    .value()
+    .find((user) => user.id === id)
-    .find((user) => user.id === id);
+    .value();
   if (user == null) {
     return null;
   }
@@ -49,13 +58,11 @@ exports.getUser = (id) => {
     numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
   };
 };
-
+export const upsertUser = ({ username, password, userId, isAdmin }) => {
-exports.upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db.chain
-  const user = db
     .get('user')
-    .value()
+    .filter((u) => u.id !== userId)
-    .filter((u) => u.id !== userId);
+    .value();
-
   user.push({
     id: userId || nanoid(),
     username,
@@ -63,21 +70,50 @@ exports.upsertUser = ({ username, password, userId, isAdmin }) => {
     password: hasher.hash(password),
     isAdmin,
   });
-
+  db.chain.set('user', user).value();
-  db.set('user', user).write();
+  db.write();
 };
-
+export const setLastLoginToNow = ({ userId }) => {
-exports.setLastLoginToNow = ({ userId }) => {
+  db.chain
-  db.get('user')
+    .get('user')
     .find((u) => u.id === userId)
     .assign({ lastLogin: Date.now() })
-    .write();
+    .value();
+  db.write();
+};
+export const removeUser = (userId) => {
+  const user = db.chain.get('user').value();
+  db.chain
+    .set(
+      'user',
+      user.filter((u) => u.id !== userId)
+    )
+    .value();
+  db.write();
 };
 
-exports.removeUser = (userId) => {
+export const handleDemoUser = () => {
-  const user = db.get('user').value();
+    if(!config.demoMode){
-  db.set(
+        const user = db.chain.get('user').value();
-    'user',
+        db.chain.get('user').value();
-    user.filter((u) => u.id !== userId)
+        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
-  ).write();
+        db.write();
+    }else {
+        const demoUser = db.chain
+            .get('user')
+            .filter((u) => u.username === 'demo')
+            .value();
+        if (demoUser == null || demoUser.length === 0) {
+            db.chain.get('user')
+                .value()
+                .push({
+                    id: nanoid(),
+                    username: 'demo',
+                    password: hasher.hash('demo'),
+                    isAdmin: true,
+                });
+            db.write();
+        }
+    }
 };
+

lib/services/tracking/Tracker.js

@@ -0,0 +1,90 @@
+import Mixpanel from 'mixpanel';
+import {getJobs} from '../storage/jobStorage.js';
+import {getUniqueId} from './uniqueId.js';
+import {config, inDevMode} from '../../utils.js';
+import os from 'os';
+import {readFileSync} from 'fs';
+import {packageUp} from 'package-up';
+
+const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+const distinct_id = getUniqueId() || 'N/A';
+const version = await getPackageVersion();
+
+export const track = function () {
+    //only send tracking information if the user allowed to do so.
+    if (config.analyticsEnabled && !inDevMode()) {
+        const activeProvider = new Set();
+        const activeAdapter = new Set();
+
+        const jobs = getJobs();
+
+        if (jobs != null && jobs.length > 0) {
+            jobs.forEach((job) => {
+                job.provider.forEach((provider) => {
+                    activeProvider.add(provider.id);
+                });
+                job.notificationAdapter.forEach((adapter) => {
+                    activeAdapter.add(adapter.id);
+                });
+            });
+
+            mixpanelTracker.track(
+                'fredy_tracking',
+                enrichTrackingObject({
+                    adapter: Array.from(activeAdapter),
+                    provider: Array.from(activeProvider),
+                }),
+            );
+        }
+    }
+};
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoJobCreated(jobData) {
+    if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+        mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
+    }
+}
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoAccessed() {
+    if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+        mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
+    }
+}
+
+function enrichTrackingObject(trackingObject) {
+    const operating_system = os.platform();
+    const os_version = os.release();
+    const arch = process.arch;
+    const language = process.env.LANG || 'en';
+    const nodeVersion = process.version || 'N/A';
+
+    return {
+        ...trackingObject,
+        isDemo: config.demoMode,
+        operating_system,
+        os_version,
+        arch,
+        nodeVersion,
+        language,
+        distinct_id,
+        fredy_version: version
+    };
+}
+
+async function getPackageVersion() {
+    try {
+        const packagePath = await packageUp();
+        const packageJson = readFileSync(packagePath, 'utf8');
+        const json = JSON.parse(packageJson);
+        return json.version;
+    } catch (error) {
+        console.error('Error reading version from package.json', error);
+    }
+    return 'N/A';
+}

lib/services/tracking/uniqueId.js

@@ -0,0 +1,19 @@
+import { hostname, arch, cpus, platform } from 'os';
+import { createHash } from 'crypto';
+
+/**
+ * Don't worry, we are not evil ;) We however need a unique id per running instance
+ * @returns {string}
+ */
+export const getUniqueId = () => {
+  const systemInfo = {
+    hostname: hostname(),
+    architecture: arch(),
+    cpuCount: cpus().length,
+    platform: platform(),
+  };
+
+  const baseData = JSON.stringify(systemInfo);
+
+  return createHash('sha256').update(baseData).digest('hex');
+};

lib/utils.js

@@ -1,36 +1,91 @@
-function isOneOf(word, arr) {
+import {dirname} from 'node:path';
-  if (arr == null || arr.length === 0) {
+import {fileURLToPath} from 'node:url';
-    return false;
+import {readFile} from 'fs/promises';
-  }
+import {createHash} from 'crypto';
-  const expression = String.raw`\b(${arr.join('|')})\b`;
+import {DEFAULT_CONFIG} from './defaultConfig.js';
-  const blacklist = new RegExp(expression, 'ig');
 
-  return blacklist.test(word);
+function inDevMode(){
+    return process.env.NODE_ENV == null || process.env.NODE_ENV !== 'production';
+}
+
+function isOneOf(word, arr) {
+    if (arr == null || arr.length === 0) {
+        return false;
+    }
+    const expression = String.raw`\b(${arr.join('|')})\b`;
+    const blacklist = new RegExp(expression, 'ig');
+    return blacklist.test(word);
 }
 
 function nullOrEmpty(val) {
-  return val == null || val.length === 0;
+    return val == null || val.length === 0;
 }
 
 function timeStringToMs(timeString, now) {
-  const d = new Date(now);
+    const d = new Date(now);
-  const parts = timeString.split(':');
+    const parts = timeString.split(':');
-  d.setHours(parts[0]);
+    d.setHours(parts[0]);
-  d.setMinutes(parts[1]);
+    d.setMinutes(parts[1]);
-  d.setSeconds(0);
+    d.setSeconds(0);
-  return d.getTime();
+    return d.getTime();
 }
 
 function duringWorkingHoursOrNotSet(config, now) {
-  const { workingHours } = config;
+    const {workingHours} = config;
-  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+    if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
-    return true;
+        return true;
-  }
+    }
-
+    const toDate = timeStringToMs(workingHours.to, now);
-  const toDate = timeStringToMs(workingHours.to, now);
+    const fromDate = timeStringToMs(workingHours.from, now);
-  const fromDate = timeStringToMs(workingHours.from, now);
+    return fromDate <= now && toDate >= now;
-
-  return fromDate <= now && toDate >= now;
 }
 
-module.exports = { isOneOf, nullOrEmpty, duringWorkingHoursOrNotSet };
+function getDirName() {
+    return dirname(fileURLToPath(import.meta.url));
+}
+
+function buildHash(...inputs) {
+    if (inputs == null) {
+        return null;
+    }
+    const cleaned = inputs.filter(i => i != null && i.length > 0);
+    if (cleaned.length === 0) {
+        return null;
+    }
+    return createHash('sha256')
+        .update(cleaned.join(','))
+        .digest('hex');
+}
+
+let config = {};
+export async function readConfigFromStorage(){
+    return JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+}
+
+export async function refreshConfig(){
+    try {
+        config = await readConfigFromStorage();
+        //backwards compatability...
+        config.analyticsEnabled ??= null;
+        config.demoMode ??= false;
+    } catch (error) {
+        config = {...DEFAULT_CONFIG};
+        console.error('Error reading config file', error);
+    }
+}
+await refreshConfig();
+
+export {isOneOf};
+export {inDevMode};
+export {nullOrEmpty};
+export {duringWorkingHoursOrNotSet};
+export {getDirName};
+export {config};
+export {buildHash};
+export default {
+    isOneOf,
+    nullOrEmpty,
+    duringWorkingHoursOrNotSet,
+    getDirName,
+    config,
+};

package.json

@@ -1,21 +1,17 @@
 {
   "name": "fredy",
-  "version": "6.0.2",
+  "version": "11.0.3",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
-    "start": "node index.js",
+    "start": "node prod.js",
     "dev": "yarn && rm -rf ./ui/public/* && vite",
     "ui": "rm -rf ./ui/public/* && vite",
     "prod": "yarn && vite build --emptyOutDir",
-    "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
+    "format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
-    "test": "mocha --timeout 3000000 test/**/*.test.js",
+    "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
-    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
+    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
   },
+  "type": "module",
   "lint-staged": {
     "*.js": [
       "eslint ./index.js ./lib/**/*.js ./test/**/*.js",
@@ -44,7 +40,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=20.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -54,54 +50,59 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@douyinfe/semi-ui": "2.73.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.7.0",
+    "@sendgrid/mail": "8.1.4",
-    "@vitejs/plugin-react": "3.1.0",
+    "@vitejs/plugin-react": "4.3.4",
-    "better-sqlite3": "8.1.0",
+    "better-sqlite3": "^11.8.0",
-    "body-parser": "1.20.1",
+    "body-parser": "1.20.3",
-    "cookie-session": "2.0.0",
+    "cheerio": "^1.0.0",
-    "handlebars": "4.7.7",
+    "cookie-session": "2.1.0",
-    "highcharts": "10.3.3",
+    "handlebars": "4.7.8",
-    "highcharts-react-official": "3.1.0",
+    "highcharts": "12.1.2",
-    "lowdb": "1.0.0",
+    "highcharts-react-official": "3.2.1",
+    "lodash": "4.17.21",
+    "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "3.3.3",
+    "mixpanel": "^0.18.0",
-    "node-fetch": "2.6.9",
+    "nanoid": "5.0.9",
-    "node-mailjet": "3.3.13",
+    "node-fetch": "3.3.2",
-    "query-string": "7.1.3",
+    "node-mailjet": "6.0.6",
-    "react": "18.2.0",
+    "package-up": "^5.0.0",
-    "react-dom": "18.2.0",
+    "puppeteer": "^24.1.0",
-    "react-redux": "8.0.5",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "query-string": "9.1.1",
+    "react": "18.3.1",
+    "react-dom": "18.3.1",
+    "react-redux": "9.2.0",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "react-switch": "7.0.0",
+    "redux": "5.0.1",
-    "redux": "4.2.1",
+    "redux-thunk": "3.1.0",
-    "redux-thunk": "2.4.2",
+    "restana": "4.9.9",
-    "restana": "4.9.7",
+    "serve-static": "1.16.2",
-    "semantic-ui-react": "2.1.4",
-    "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "4.1.1",
+    "vite": "5.4.11"
-    "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.20.12",
+    "@babel/core": "7.26.0",
-    "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-parser": "7.26.5",
-    "@babel/preset-env": "7.20.2",
+    "@babel/preset-env": "7.26.0",
-    "@babel/preset-react": "7.18.6",
+    "@babel/preset-react": "7.26.3",
-    "chai": "4.3.7",
+    "chai": "5.1.2",
-    "eslint": "8.34.0",
+    "eslint": "8.56.0",
-    "eslint-config-prettier": "8.6.0",
+    "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.32.2",
+    "eslint-plugin-react": "7.37.4",
+    "esmock": "2.6.9",
     "history": "5.3.0",
-    "husky": "4.3.8",
+    "husky": "9.1.7",
-    "less": "4.1.3",
+    "less": "4.2.1",
-    "lint-staged": "13.1.2",
+    "lint-staged": "15.4.1",
-    "mocha": "10.2.0",
+    "mocha": "10.8.2",
-    "prettier": "2.8.4",
+    "prettier": "3.4.2",
-    "proxyquire": "2.1.3",
     "redux-logger": "3.0.6"
   }
 }

prod.js

@@ -0,0 +1,2 @@
+process.env.NODE_ENV = 'production';
+import('./index.js');

test/mocks/mockNotification.js

@@ -1,12 +1,10 @@
-module.exports = {
+let tmpStore = {};
-  _tmpStore: {},
 
-  send: (serviceName, payload) => {
+export const send = (serviceName, payload) => {
-    this._tmpStore = { serviceName, payload };
+  tmpStore = { serviceName, payload };
-    return [Promise.resolve()];
+  return [Promise.resolve()];
-  },
+};
-
+
-  get: () => {
+export const get = () => {
-    return this._tmpStore;
+  return tmpStore;
-  },
 };

test/mocks/mockStore.js

@@ -1,11 +1,8 @@
 const db = {};
-
+export const setKnownListings = (jobKey, providerId, listings) => {
-exports.setKnownListings = (jobKey, providerId, listings) => {
   if (!Array.isArray(listings)) throw Error('Not a valid array');
-
   db[providerId] = listings;
 };
-
+export const getKnownListings = (jobKey, providerId) => {
-exports.getKnownListings = (jobKey, providerId) => {
   return db[providerId] || [];
 };

test/provider/einsAImmobilien.test.js

@@ -1,43 +1,30 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { providerConfig, mockFredy } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/einsAImmobilien.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/einsAImmobilien');
 
 describe('#einsAImmobilien testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.einsAImmobilien, [], []);
-
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test einsAImmobilien provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
       fredy.execute().then((listings) => {
         expect(listings).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;

test/provider/immobilienDe.test.js

@@ -1,31 +1,21 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { providerConfig, mockFredy } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immobilienDe.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immobilienDe');
 
 describe('#immobilien.de testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.immobilienDe, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immobilien.de provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immobilienDe');
         notificationObj.payload.forEach((notify) => {
@@ -36,7 +26,6 @@ describe('#immobilien.de testsuite()', () => {
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');

test/provider/immonet.test.js

@@ -1,37 +1,26 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immonet.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immonet');
 
 describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.immonet, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immonet provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immonet');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
@@ -42,7 +31,6 @@ describe('#immonet testsuite()', () => {
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immonet.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/immoscout.test.js

@@ -1,60 +1,43 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+//import {get} from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import {/*mockFredy, */providerConfig} from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+//import {expect} from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immoscout.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immoscout');
-const scrapingAnt = require('../../lib/services/scrapingAnt');
 
 describe('#immoscout testsuite()', () => {
-  after(() => {
+    after(() => {
-    similarityCache.stopCacheCleanup();
+        similarityCache.stopCacheCleanup();
-  });
+    });
-  provider.init(providerConfig.immoscout, [], []);
+    provider.init(providerConfig.immoscout, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    it('should test immoscout provider', async () => {
-    './services/storage/listingsStorage': {
+        //const Fredy = await mockFredy();
-      ...mockStore,
+        return await new Promise((resolve) => {
-    },
+            /* eslint-disable no-console */
-    './notification/notify': mockNotification,
+            console.info('Skipping Immoscout test for now until we figured out how to surpass bot detection.');
-  });
+            /* eslint-enable no-console */
-
+            resolve();
-  it('should test immoscout provider', async () => {
+            /*
-    return await new Promise((resolve) => {
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
-      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+            fredy.execute().then((listing) => {
-        /* eslint-disable no-console */
+                expect(listing).to.be.a('array');
-        console.info('Skipping Immoscout test as ScrapingAnt Api Key is not set.');
+                const notificationObj = get();
-        /* eslint-enable no-console */
+                expect(notificationObj).to.be.a('object');
-        resolve();
+                expect(notificationObj.serviceName).to.equal('immoscout');
-        return;
+                notificationObj.payload.forEach((notify) => {
-      }
+                    expect(notify.id).to.be.a('number');
-
+                    expect(notify.price).to.be.a('string');
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
+                    expect(notify.size).to.be.a('string');
-      fredy.execute().then((listing) => {
+                    expect(notify.title).to.be.a('string');
-        expect(listing).to.be.a('array');
+                    expect(notify.link).to.be.a('string');
-
+                    expect(notify.address).to.be.a('string');
-        const notificationObj = mockNotification.get();
+                    expect(notify.price).that.does.include('€');
-        expect(notificationObj).to.be.a('object');
+                    expect(notify.size).that.does.include('m²');
-        expect(notificationObj.serviceName).to.equal('immoscout');
+                    expect(notify.title).to.be.not.empty;
-
+                    expect(notify.link).that.does.include('https://www.immobilienscout24.de');
-        notificationObj.payload.forEach((notify) => {
+                    expect(notify.address).to.be.not.empty;
-          /** check the actual structure **/
+                });
-          expect(notify.id).to.be.a('number');
+                resolve();
-          expect(notify.price).to.be.a('string');
+            });*/
-          expect(notify.size).to.be.a('string');
+        });
-          expect(notify.title).to.be.a('string');
-          expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
-
-          /** check the values if possible **/
-          expect(notify.price).that.does.include('€');
-          expect(notify.size).that.does.include('m²');
-          expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immobilienscout24.de');
-          expect(notify.address).to.be.not.empty;
-        });
-        resolve();
-      });
     });
-  });
 });

test/provider/immoswp.test.js

@@ -1,34 +1,23 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immoswp.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immoswp');
 
 describe('#immoswp testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.immoswp, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immoswp provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immoswp');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
@@ -36,13 +25,10 @@ describe('#immoswp testsuite()', () => {
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://immo.swp.de');
-          expect(notify.address).to.be.not.empty;
         });
         resolve();
       });

test/provider/immowelt.test.js

@@ -1,42 +1,30 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immowelt.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immowelt');
 
 describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   it('should test immowelt provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immowelt');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.title).to.be.a('string');
-
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');

test/provider/kleinanzeigen.test.js

@@ -1,43 +1,32 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/kleinanzeigen.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/kleinanzeigen');
 
 describe('#kleinanzeigen testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   it('should test kleinanzeigen provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.kleinanzeigen, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
+          expect(notify.link).that.does.include('https://www.kleinanzeigen.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/neubauKompass.test.js

@@ -1,48 +1,36 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import {get} from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import {mockFredy, providerConfig} from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import {expect} from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/neubauKompass.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/neubauKompass');
 
 describe('#neubauKompass testsuite()', () => {
-  after(() => {
+    after(() => {
-    similarityCache.stopCacheCleanup();
+        similarityCache.stopCacheCleanup();
-  });
+    });
-  provider.init(providerConfig.neubauKompass, [], []);
+    provider.init(providerConfig.neubauKompass, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    it('should test neubauKompass provider', async () => {
-    './services/storage/listingsStorage': {
+        const Fredy = await mockFredy();
-      ...mockStore,
+        return await new Promise((resolve) => {
-    },
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
-    './notification/notify': mockNotification,
+            fredy.execute().then((listing) => {
-  });
+                expect(listing).to.be.a('array');
-
+                const notificationObj = get();
-  it('should test neubauKompass provider', async () => {
+                expect(notificationObj.serviceName).to.equal('neubauKompass');
-    return await new Promise((resolve) => {
+                notificationObj.payload.forEach((notify) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
+                    expect(notify).to.be.a('object');
-      fredy.execute().then((listing) => {
+                    /** check the actual structure **/
-        expect(listing).to.be.a('array');
+                    expect(notify.id).to.be.a('string');
-
+                    expect(notify.title).to.be.a('string');
-        const notificationObj = mockNotification.get();
+                    expect(notify.link).to.be.a('string');
-        expect(notificationObj.serviceName).to.equal('neubauKompass');
+                    expect(notify.address).to.be.a('string');
-
+                    /** check the values if possible **/
-        notificationObj.payload.forEach((notify) => {
+                    expect(notify.title).to.be.not.empty;
-          expect(notify).to.be.a('object');
+                    expect(notify.link).that.does.include('https://www.neubaukompass.de');
-
+                    expect(notify.address).to.be.not.empty;
-          /** check the actual structure **/
+                });
-          expect(notify.id).to.be.a('string');
+                resolve();
-          expect(notify.title).to.be.a('string');
+            });
-          expect(notify.link).to.be.a('string');
+        });
-          expect(notify.address).to.be.a('string');
-
-          /** check the values if possible **/
-          expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.neubaukompass.de');
-          expect(notify.address).to.be.not.empty;
-        });
-        resolve();
-      });
     });
-  });
 });

test/provider/testProvider.json

@@ -9,11 +9,11 @@
     "enabled": true
   },
   "immonet": {
-    "url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
+    "url": "https://www.immonet.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2112&order=Default&m=homepage_new_search_classified_search_result",
     "enabled": true
   },
   "immowelt": {
-    "url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?d=true&rmi=3&sd=DESC&sf=TIMESTAMP&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
     "enabled": true
   },
   "immoscout": {
@@ -29,7 +29,7 @@
     "enabled": true
   },
   "kleinanzeigen": {
-    "url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
+    "url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
     "enabled": true
   },
   "neubauKompass": {

test/provider/utils.test.js

@@ -1,6 +1,6 @@
-const utils = require('../../lib/utils');
+import utils from '../../lib/utils.js';
-const assert = require('assert');
+import assert from 'assert';
-const expect = require('chai').expect;
+import { expect } from 'chai';
 
 const fakeWorkingHoursConfig = (from, to) => ({
   workingHours: {
@@ -8,7 +8,6 @@ const fakeWorkingHoursConfig = (from, to) => ({
     from,
   },
 });
-
 describe('utils', () => {
   describe('#isOneOf()', () => {
     it('should be false', () => {
@@ -18,7 +17,6 @@ describe('utils', () => {
       assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
     });
   });
-
   describe('#duringWorkingHoursOrNotSet()', () => {
     it('should be false', () => {
       expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;

test/provider/wgGesucht.test.js

@@ -1,39 +1,28 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import { expect } from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/wgGesucht.js';
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/wgGesucht');
 
 describe('#wgGesucht testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   provider.init(providerConfig.wgGesucht, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test wgGesucht provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-        const notificationObj = mockNotification.get();
+        const notificationObj = get();
         expect(notificationObj.serviceName).to.equal('wgGesucht');
         notificationObj.payload.forEach((notify) => {
           expect(notify).to.be.a('object');
-
           /** check the actual structure **/
-
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.details).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.link).to.be.a('string');
         });

test/queryStringMutator/queryStringMutator.test.js

@@ -1,9 +1,16 @@
-const testData = require('./testData.json');
+import fs from 'fs';
-const expect = require('chai').expect;
+import { expect } from 'chai';
-const fs = require('fs');
+import { readFile } from 'fs/promises';
+import mutator from '../../lib/services/queryStringMutator.js';
+import queryString from 'query-string';
 
-const mutator = require('../../lib/services/queryStringMutator.js');
+const data = await readFile(new URL('./testData.json', import.meta.url));
-const queryString = require('query-string');
+
+const testData = JSON.parse(data);
+
+let _provider = await Promise.all(
+  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`)),
+);
 
 /**
  * Test test might look a bit weird at first, but listen stranger...
@@ -12,18 +19,14 @@ const queryString = require('query-string');
  */
 describe('queryStringMutator', () => {
   it('should fix all urls', () => {
-    let _provider = fs.readdirSync('./lib/provider/').map((integPath) => require(`../../lib/provider/${integPath}`));
-
     for (let test of testData) {
       const provider = _provider.find((p) => p.metaInformation.id === test.id);
       if (provider == null) {
         throw new Error(`Cannot find provider for given id: ${test.id}`);
       }
-
       const fixedUrl = mutator(test.url, provider.config.sortByDateParam);
       const expectedParams = queryString.parseUrl(test.shouldBecome);
       const actualParams = queryString.parseUrl(fixedUrl);
-
       //check if all new params are existing
       expect(Object.keys(expectedParams.query)).to.include.members(Object.keys(actualParams.query));
       expect(Object.values(expectedParams.query)).to.include.members(Object.values(actualParams.query));

test/queryStringMutator/testData.json

@@ -1,7 +1,7 @@
 [
   {
-    "url": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=PRIMARY_PRICE_AMOUNT&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
-    "shouldBecome": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=TIMESTAMP&sp=1",
+    "shouldBecome": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350&order=DateDesc",
     "id": "immowelt"
   },
   {

test/similarity/similarity.test.js

@@ -1,5 +1,5 @@
-const SimilarityCacheEntry = require('../../lib/services/similarity-check/SimilarityCacheEntry');
+import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
-const expect = require('chai').expect;
+import { expect } from 'chai';
 
 describe('similarityCheck', () => {
   describe('#similarityCheck()', () => {
@@ -29,10 +29,10 @@ describe('similarityCheck', () => {
     it('should be false', () => {
       const check = new SimilarityCacheEntry(0);
       check.setCacheEntry(
-        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.'
+        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.',
       );
       check.setCacheEntry(
-        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
+        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
     });
   });

test/utils.js

@@ -0,0 +1,17 @@
+import { readFile } from 'fs/promises';
+import esmock from 'esmock';
+import * as mockStore from './mocks/mockStore.js';
+import { send } from './mocks/mockNotification.js';
+
+export const providerConfig = JSON.parse(await readFile(new URL('./provider/testProvider.json', import.meta.url)));
+
+export const mockFredy = async () => {
+  return await esmock('../lib/FredyRuntime', {
+    '../lib/services/storage/listingsStorage.js': {
+      ...mockStore,
+    },
+    '../lib/notification/notify.js': {
+      send,
+    },
+  });
+};

test/utils/utils.test.js

@@ -0,0 +1,16 @@
+import { expect } from 'chai';
+import {buildHash} from '../../lib/utils.js';
+
+describe('utilsCheck', () => {
+  describe('#utilsCheck()', () => {
+      it('should be null when null input', () => {
+          expect(buildHash(null)).to.be.null;
+      });
+      it('should be null when null empty', () => {
+          expect(buildHash('')).to.be.null;
+      });
+      it('should return a value', () => {
+          expect(buildHash('bla', '', null)).to.be.a.string;
+      });
+  });
+});

ui/src/App.jsx

@@ -1,100 +1,111 @@
-import React, { useEffect } from 'react';
+import React, {useEffect} from 'react';
 
 import InsufficientPermission from './components/permission/InsufficientPermission';
 import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
 import GeneralSettings from './views/generalSettings/GeneralSettings';
-import ToastsContainer from './components/toasts/ToastContainer';
 import JobMutation from './views/jobs/mutation/JobMutation';
 import UserMutator from './views/user/mutation/UserMutator';
-import ToastContext from './components/toasts/ToastContext';
 import JobInsight from './views/jobs/insights/JobInsight.jsx';
-import { useDispatch, useSelector } from 'react-redux';
+import {useDispatch, useSelector} from 'react-redux';
-import useToast from './components/toasts/useToast';
+import {Switch, Redirect} from 'react-router-dom';
-import { Switch, Redirect } from 'react-router-dom';
 import Logout from './components/logout/Logout';
 import Logo from './components/logo/Logo';
 import Menu from './components/menu/Menu';
 import Login from './views/login/Login';
 import Users from './views/user/Users';
 import Jobs from './views/jobs/Jobs';
-import { Route } from 'react-router';
+import {Route} from 'react-router';
 
 import './App.less';
+import TrackingModal from './components/tracking/TrackingModal.jsx';
+import {Banner} from '@douyinfe/semi-ui';
 
 export default function FredyApp() {
-  const dispatch = useDispatch();
+    const dispatch = useDispatch();
-  const [showToast, onToastFinished, toasts] = useToast();
+    const [loading, setLoading] = React.useState(true);
-  const [loading, setLoading] = React.useState(true);
+    const currentUser = useSelector((state) => state.user.currentUser);
-  const currentUser = useSelector((state) => state.user.currentUser);
+    const settings = useSelector((state) => state.generalSettings.settings);
 
-  useEffect(() => {
+    useEffect(() => {
-    async function init() {
+        async function init() {
-      await dispatch.provider.getProvider();
+            await dispatch.user.getCurrentUser();
-      await dispatch.jobs.getJobs();
+            if (!needsLogin()) {
-      await dispatch.jobs.getProcessingTimes();
+                await dispatch.provider.getProvider();
-      await dispatch.notificationAdapter.getAdapter();
+                await dispatch.jobs.getJobs();
-      await dispatch.user.getCurrentUser();
+                await dispatch.jobs.getProcessingTimes();
+                await dispatch.notificationAdapter.getAdapter();
+                await dispatch.generalSettings.getGeneralSettings();
+            }
+            setLoading(false);
+        }
 
-      setLoading(false);
+        init();
-    }
+    }, [currentUser?.userId]);
-    init();
-  }, [currentUser?.userId]);
 
-  const needsLogin = () => {
+    const needsLogin = () => {
-    return currentUser == null || Object.keys(currentUser).length === 0;
+        return currentUser == null || Object.keys(currentUser).length === 0;
-  };
+    };
 
-  const isAdmin = () => currentUser != null && currentUser.isAdmin;
+    const isAdmin = () => currentUser != null && currentUser.isAdmin;
 
-  const login = () => (
+    const login = () => (
-    <Switch>
+        <Switch>
-      <Route name="Login" path={'/login'} component={Login} />
+            <Route name="Login" path={'/login'} component={Login}/>
-      <Redirect from="*" to={'/login'} />
+            <Redirect from="*" to={'/login'}/>
-    </Switch>
+        </Switch>
-  );
+    );
 
-  return loading ? null : needsLogin() ? (
+    return loading ? null : needsLogin() ? (
-    login()
+        login()
-  ) : (
+    ) : (
-    <ToastContext.Provider value={{ showToast }}>
+        <div className="app">
-      <div className="app">
+            <div className="app__container">
-        <div className="app__container">
+                <Logout/>
-          <Logout />
+                <Logo width={190} white/>
-          <Logo width={190} white />
+                <Menu isAdmin={isAdmin()}/>
-          <Menu isAdmin={isAdmin()} />
-          <ToastsContainer toasts={toasts} onToastFinished={onToastFinished} />
-          <Switch>
-            <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
-            <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
-            <Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
-            <Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
-            <Route name="Job overview" path={'/jobs'} component={Jobs} />
-            <PermissionAwareRoute
-              name="Create new User"
-              path="/users/new"
-              component={<UserMutator />}
-              currentUser={currentUser}
-            />
-            <PermissionAwareRoute
-              name="Edit a user"
-              path="/users/edit/:userId"
-              component={<UserMutator />}
-              currentUser={currentUser}
-            />
-            <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
-            <PermissionAwareRoute
-              name="General Settings"
-              path="/generalSettings"
-              component={<GeneralSettings />}
-              currentUser={currentUser}
-            />
 
-            <Redirect from="/" to={'/jobs'} />
+                {settings.demoMode && (
-          </Switch>
+                <>
+                    <Banner fullMode={true}
+                            type="info"
+                            bordered
+                            closeIcon={null}
+                            description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
+                    />
+                    <br/>
+                </>)}
+                {(settings.analyticsEnabled === null && !settings.demoMode) && <TrackingModal/>}
+                <Switch>
+                    <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission}/>
+                    <Route name="Create new Job" path={'/jobs/new'} component={JobMutation}/>
+                    <Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation}/>
+                    <Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight}/>
+                    <Route name="Job overview" path={'/jobs'} component={Jobs}/>
+                    <PermissionAwareRoute
+                        name="Create new User"
+                        path="/users/new"
+                        component={<UserMutator/>}
+                        currentUser={currentUser}
+                    />
+                    <PermissionAwareRoute
+                        name="Edit a user"
+                        path="/users/edit/:userId"
+                        component={<UserMutator/>}
+                        currentUser={currentUser}
+                    />
+                    <PermissionAwareRoute name="Users" path="/users" component={<Users/>} currentUser={currentUser}/>
+                    <PermissionAwareRoute
+                        name="General Settings"
+                        path="/generalSettings"
+                        component={<GeneralSettings/>}
+                        currentUser={currentUser}
+                    />
+
+                    <Redirect from="/" to={'/jobs'}/>
+                </Switch>
+            </div>
         </div>
-      </div>
+    );
-    </ToastContext.Provider>
-  );
 }
 
 FredyApp.displayName = 'FredyApp';

ui/src/App.less

@@ -4,11 +4,9 @@
   width:100%;
 
   &__container {
-    width: 100%;
-
     padding: 1rem 1rem;
-    background-color: #595959f5;
+    color: var(--semi-color-text-0);
-    color: #f1f1f1;
+    background-color: #232429;
   }
 }
 
@@ -19,3 +17,27 @@
 .ui.black.label, .ui.black.labels .label {
   background-color: #31303078!important;
 }
+
+a:link {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: none;
+}
+
+a:visited {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: none;
+}
+
+a:hover {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: underline;
+}
+
+a:active {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: underline;
+}

ui/src/Index.jsx

@@ -5,9 +5,11 @@ import { HashRouter } from 'react-router-dom';
 import { createHashHistory } from 'history';
 import { Provider } from 'react-redux';
 import { createRoot } from 'react-dom/client';
+import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
+import { LocaleProvider } from '@douyinfe/semi-ui';
+
 const container = document.getElementById('fredy');
 const root = createRoot(container);
-
 const history = createHashHistory();
 
 import App from './App';
@@ -17,7 +19,9 @@ import './Index.less';
 root.render(
   <Provider store={reduxStore}>
     <HashRouter history={history}>
-      <App />
+      <LocaleProvider locale={en_US}>
+        <App />
+      </LocaleProvider>
     </HashRouter>
   </Provider>
 );