next release version

Modernizing ui

.eslintrc.cjs

@@ -1,7 +1,6 @@
 module.exports = {
   env: {
-    commonjs: true,
+    es2021: true,
-    es6: true,
     node: true,
     browser: true,
     mocha: true,
@@ -17,7 +16,6 @@ module.exports = {
     fetch: true,
   },
   parserOptions: {
-    ecmaVersion: 2018,
     sourceType: 'module',
   },
   rules: {
@@ -205,10 +203,6 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
     'react/self-closing-comp': 'warn',
 
-    // Enforce spaces before the closing bracket of self-closing JSX elements
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
-    'react/jsx-space-before-closing': ['warn', 'always'],
-
     // Enforce component methods order
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
     'react/sort-comp': 'off',
@@ -239,7 +233,7 @@ module.exports = {
 
     // only .jsx files may have JSX
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
-    'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
+    'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
 
     // prevent accidental JS comments from being injected into JSX as text
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
@@ -284,15 +278,5 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
     'react/no-children-prop': 'warn',
 
-    // Validate whitespace in and around the JSX opening and closing brackets
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
-    'react/jsx-tag-spacing': [
-      'warn',
-      {
-        closingSlash: 'never',
-        beforeSelfClosing: 'always',
-        afterOpening: 'never',
-      },
-    ],
   },
 };

README.md

@@ -27,11 +27,11 @@ yarn run start
 _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
 
 <p align="center">
-  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot1.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;
-  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
+  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_2.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;  
-  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
+  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
 </p>
 <p align="center">
 
@@ -81,10 +81,10 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-### Immoscout
+### Immoscout / Immonet
-I have added **experimental** support for Immoscout. Immoscout is somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
-To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
+To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
 The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
 
 If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).

index.html

@@ -4,12 +4,11 @@
     <meta charset="UTF-8"
           name="viewport"
           content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui@2/dist/semantic.min.css">
     <meta name="google" content="notranslate">
 
     <title>Fredy</title>
 </head>
-<body>
+<body theme-mode="dark">
 
 <div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
 </body>

index.js

@@ -1,41 +1,31 @@
-const fs = require('fs');
+import fs from 'fs';
-
+import { config } from './lib/utils.js';
+import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
+import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
+import * as jobStorage from './lib/services/storage/jobStorage.js';
+import FredyRuntime from './lib/FredyRuntime.js';
+import { duringWorkingHoursOrNotSet } from './lib/utils.js';
+import './lib/api/api.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
 }
-
 const path = './lib/provider';
 const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
-const config = require('./conf/config.json');
-
-const similarityCache = require('./lib/services/similarity-check/similarityCache');
-const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
-const jobStorage = require('./lib/services/storage/jobStorage');
-const FredyRuntime = require('./lib/FredyRuntime');
-
-const { duringWorkingHoursOrNotSet } = require('./lib/utils');
-
-//starting the api service
-require('./lib/api/api');
-
 //assuming interval is always in minutes
-
 const INTERVAL = config.interval * 60 * 1000;
-
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
 /* eslint-enable no-console */
+const fetchedProvider = await Promise.all(
+  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
+);
+
 setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-
     if (isDuringWorkingHoursOrNotSet) {
       config.lastRun = Date.now();
-      const fetchedProvider = provider
-        .filter((provider) => provider.endsWith('.js'))
-        .map((pro) => require(`${path}/${pro}`));
-
       jobStorage
         .getJobs()
         .filter((job) => job.enabled)

lib/FredyRuntime.js

@@ -1,11 +1,9 @@
-const { NoNewListingsWarning } = require('./errors');
+import { NoNewListingsWarning } from './errors.js';
-const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
+import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
-
+import * as notify from './notification/notify.js';
-const notify = require('./notification/notify');
+import xray from './services/scraper.js';
-const xray = require('./services/scraper');
+import * as scrapingAnt from './services/scrapingAnt.js';
-const scrapingAnt = require('./services/scrapingAnt');
+import urlModifier from './services/queryStringMutator.js';
-const urlModifier = require('./services/queryStringMutator');
-
 class FredyRuntime {
   /**
    *
@@ -22,7 +20,6 @@ class FredyRuntime {
     this._jobKey = jobKey;
     this._similarityCache = similarityCache;
   }
-
   execute() {
     return (
       //modify the url to make sure search order is correctly set
@@ -45,19 +42,18 @@ class FredyRuntime {
         .catch(this._handleError.bind(this))
     );
   }
-
   _getListings(url) {
     return new Promise((resolve, reject) => {
       const id = this._providerId;
-      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
+      if (scrapingAnt.needScrapingAnt(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
-        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
+        const error = 'Immoscout or Immonet can only be used with if you have set an apikey for scrapingAnt.';
         /* eslint-disable no-console */
         console.log(error);
         /* eslint-enable no-console */
         reject(error);
         return;
       }
-      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
+      const u = scrapingAnt.needScrapingAnt(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
       try {
         if (this._providerConfig.paginate != null) {
           xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
@@ -87,25 +83,19 @@ class FredyRuntime {
       }
     });
   }
-
   _normalize(listings) {
     return listings.map(this._providerConfig.normalize);
   }
-
   _filter(listings) {
     return listings.filter(this._providerConfig.filter);
   }
-
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
-
     if (newListings.length === 0) {
       throw new NoNewListingsWarning();
     }
-
     return newListings;
   }
-
   _notify(newListings) {
     if (newListings.length === 0) {
       throw new NoNewListingsWarning();
@@ -113,7 +103,6 @@ class FredyRuntime {
     const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
     return Promise.all(sendNotifications).then(() => newListings);
   }
-
   _save(newListings) {
     const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
     newListings.forEach((listing) => {
@@ -122,7 +111,6 @@ class FredyRuntime {
     setKnownListings(this._jobKey, this._providerId, currentListings);
     return newListings;
   }
-
   _filterBySimilarListings(listings) {
     const filteredList = listings.filter((listing) => {
       const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
@@ -136,10 +124,8 @@ class FredyRuntime {
     filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
     return filteredList;
   }
-
   _handleError(err) {
     if (err.name !== 'NoNewListingsWarning') console.error(err);
   }
 }
-
+export default FredyRuntime;
-module.exports = FredyRuntime;

lib/api/api.js

@@ -1,44 +1,36 @@
-const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
+import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
-const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
+import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
-const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
+import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
-const { analyticsRouter } = require('./routes/analyticsRouter');
+import { analyticsRouter } from './routes/analyticsRouter.js';
-const { providerRouter } = require('./routes/providerRouter');
+import { providerRouter } from './routes/providerRouter.js';
-const { loginRouter } = require('./routes/loginRoute');
+import { loginRouter } from './routes/loginRoute.js';
-const config = require('../../conf/config.json');
+import { config } from '../utils.js';
-const { userRouter } = require('./routes/userRoute');
+import { userRouter } from './routes/userRoute.js';
-const { jobRouter } = require('./routes/jobRouter');
+import { jobRouter } from './routes/jobRouter.js';
-const bodyParser = require('body-parser');
+import bodyParser from 'body-parser';
-const service = require('restana')();
+import restana from 'restana';
-const files = require('serve-static');
+import files from 'serve-static';
-const path = require('path');
+import path from 'path';
-
+import { getDirName } from '../utils.js';
-const staticService = files(path.join(__dirname, '../../ui/public'));
+const service = restana();
-
+const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
 
 service.use(bodyParser.json());
-
 service.use(cookieSession());
-
 service.use(staticService);
-
 service.use('/api/admin', authInterceptor());
 service.use('/api/jobs', authInterceptor());
-
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
-
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
 service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
 service.use('/api/jobs', jobRouter);
-
 service.use('/api/login', loginRouter);
-
 /* eslint-disable no-console */
 service.start(PORT).then(() => {
   console.info(`Started API service on port ${PORT}`);
 });
-/* eslint-enable no-console */

lib/api/routes/analyticsRouter.js

@@ -1,12 +1,10 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as listingStorage from '../../services/storage/listingsStorage.js';
+const service = restana();
 const analyticsRouter = service.newRouter();
-const listingStorage = require('../../services/storage/listingsStorage');
-
 analyticsRouter.get('/:jobId', async (req, res) => {
   const { jobId } = req.params;
-
   res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
   res.send();
 });
-
+export { analyticsRouter };
-exports.analyticsRouter = analyticsRouter;

lib/api/routes/generalSettingsRoute.js

@@ -1,18 +1,16 @@
-const service = require('restana')();
+import restana from 'restana';
+import { config, getDirName } from '../../utils.js';
+import fs from 'fs';
+const service = restana();
 const generalSettingsRouter = service.newRouter();
-const config = require('../../../conf/config.json');
-const fs = require('fs');
-
 generalSettingsRouter.get('/', async (req, res) => {
   res.body = Object.assign({}, config);
   res.send();
 });
-
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
-
   try {
-    fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));
@@ -20,5 +18,4 @@ generalSettingsRouter.post('/', async (req, res) => {
   }
   res.send();
 });
-
+export { generalSettingsRouter };
-exports.generalSettingsRouter = generalSettingsRouter;

lib/api/routes/jobRouter.js

@@ -1,12 +1,12 @@
-const service = require('restana')();
+import restana from 'restana';
+import fetch from 'node-fetch';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as immoscoutProvider from '../../provider/immoscout.js';
+import { config } from '../../utils.js';
+import { isAdmin } from '../security.js';
+const service = restana();
 const jobRouter = service.newRouter();
-const fetch = require('node-fetch');
-const jobStorage = require('../../services/storage/jobStorage');
-const userStorage = require('../../services/storage/userStorage');
-const immoscoutProvider = require('../../provider/immoscout');
-const config = require('../../../conf/config.json');
-const { isAdmin } = require('../security');
-
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -16,40 +16,31 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-
   return user.isAdmin || job.userId === job.userId;
 }
-
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
-
   //show only the jobs which belongs to the user (or all of the user is an admin)
   res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
-
   res.send();
 });
-
 jobRouter.get('/processingTimes', async (req, res) => {
   let scrapingAntData = null;
-
   if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
     try {
-      const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
+      const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
       scrapingAntData = await response.json();
     } catch (Exception) {
       console.error('Could not query plan data from scraping ant.', Exception);
     }
   }
-
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
     scrapingAntData,
   };
-
   res.send();
 });
-
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
   if (
@@ -77,7 +68,6 @@ jobRouter.post('/', async (req, res) => {
   }
   res.send();
 });
-
 jobRouter.delete('', async (req, res) => {
   const { jobId } = req.body;
   try {
@@ -93,7 +83,6 @@ jobRouter.delete('', async (req, res) => {
   }
   res.send();
 });
-
 jobRouter.put('/:jobId/status', async (req, res) => {
   const { status } = req.body;
   const { jobId } = req.params;
@@ -113,5 +102,4 @@ jobRouter.put('/:jobId/status', async (req, res) => {
   }
   res.send();
 });
-
+export { jobRouter };
-exports.jobRouter = jobRouter;

lib/api/routes/loginRoute.js

@@ -1,8 +1,8 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as hasher from '../../services/security/hash.js';
+const service = restana();
 const loginRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const hasher = require('../../services/security/hash');
-
 loginRouter.get('/user', async (req, res) => {
   const currentUserId = req.session.currentUser;
   const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
@@ -16,17 +16,13 @@ loginRouter.get('/user', async (req, res) => {
   }
   res.send();
 });
-
 loginRouter.post('/', async (req, res) => {
   const { username, password } = req.body;
-
   const user = userStorage.getUsers(true).find((user) => user.username === username);
-
   if (user == null) {
     res.send(401);
     return;
   }
-
   if (user.password === hasher.hash(password)) {
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
@@ -35,13 +31,10 @@ loginRouter.post('/', async (req, res) => {
   } else {
     console.error(`User ${username} tried to login, but password was wrong.`);
   }
-
   res.send(401);
 });
-
 loginRouter.post('/logout', async (req, res) => {
   req.session = null;
   res.send(200);
 });
-
+export { loginRouter };
-exports.loginRouter = loginRouter;

lib/api/routes/notificationAdapterRouter.js

@@ -1,13 +1,13 @@
-const fs = require('fs');
+import fs from 'fs';
-const service = require('restana')();
+import restana from 'restana';
+const service = restana();
 const notificationAdapterRouter = service.newRouter();
-
 const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
-
+const notificationAdapter = await Promise.all(
-const notificationAdapter = notificationAdapterList.map((pro) => {
+  notificationAdapterList.map(async (pro) => {
-  return require(`../../notification/adapter/${pro}`);
+    return await import(`../../notification/adapter/${pro}`);
-});
+  })
-
+);
 notificationAdapterRouter.post('/try', async (req, res) => {
   const { id, fields } = req.body;
   const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
@@ -24,7 +24,6 @@ notificationAdapterRouter.post('/try', async (req, res) => {
     enabled: true,
     id,
   });
-
   try {
     await adapter.send({
       serviceName: 'TestCall',
@@ -40,16 +39,13 @@ notificationAdapterRouter.post('/try', async (req, res) => {
       notificationConfig,
       jobKey: 'TestJob',
     });
-
     res.send();
   } catch (Exception) {
     res.send(new Error(Exception));
   }
 });
-
 notificationAdapterRouter.get('/', async (req, res) => {
   res.body = notificationAdapter.map((adapter) => adapter.config);
   res.send();
 });
-
+export { notificationAdapterRouter };
-exports.notificationAdapterRouter = notificationAdapterRouter;

lib/api/routes/providerRouter.js

@@ -1,16 +1,15 @@
-const fs = require('fs');
+import fs from 'fs';
-const service = require('restana')();
+import restana from 'restana';
+const service = restana();
 const providerRouter = service.newRouter();
-
 const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
-
+const provider = await Promise.all(
-const provider = providerList.map((pro) => {
+  providerList.map(async (pro) => {
-  return require(`../../provider/${pro}`).metaInformation;
+    return await import(`../../provider/${pro}`);
-});
+  })
-
+);
 providerRouter.get('/', async (req, res) => {
-  res.body = provider;
+  res.body = provider.map((p) => p.metaInformation);
   res.send();
 });
-
+export { providerRouter };
-exports.providerRouter = providerRouter;

lib/api/routes/userRoute.js

@@ -1,33 +1,27 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+const service = restana();
 const userRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const jobStorage = require('../../services/storage/jobStorage');
-
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
   return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
 }
-
 function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
   return req.session.currentUser === userIdToBeRemoved;
 }
-
 const nullOrEmpty = (str) => str == null || str.length === 0;
-
 userRouter.get('/', async (req, res) => {
   res.body = userStorage.getUsers(false);
   res.send();
 });
-
 userRouter.get('/:userId', async (req, res) => {
   const { userId } = req.params;
   res.body = userStorage.getUser(userId);
   res.send();
 });
-
 userRouter.delete('/', async (req, res) => {
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
-
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
     return;
@@ -36,14 +30,11 @@ userRouter.delete('/', async (req, res) => {
     res.send(new Error('You are trying to remove yourself. This is prohibited.'));
     return;
   }
-
   //TODO: Remove also analytics
   jobStorage.removeJobsByUserId(userId);
   userStorage.removeUser(userId);
-
   res.send();
 });
-
 userRouter.post('/', async (req, res) => {
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
@@ -55,22 +46,18 @@ userRouter.post('/', async (req, res) => {
     return;
   }
   const allUser = userStorage.getUsers(false);
-
   if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(
       new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
     );
     return;
   }
-
   userStorage.upsertUser({
     userId,
     username,
     password,
     isAdmin,
   });
-
   res.send();
 });
-
+export { userRouter };
-exports.userRouter = userRouter;

lib/api/security.js

@@ -1,15 +1,12 @@
-const userStorage = require('../services/storage/userStorage');
+import * as userStorage from '../services/storage/userStorage.js';
-const cookieSession = require('cookie-session');
+import cookieSession from 'cookie-session';
-const { nanoid } = require('nanoid');
+import { nanoid } from 'nanoid';
-
 const unauthorized = (res) => {
   return res.send(401);
 };
-
 const isUnauthorized = (req) => {
   return req.session.currentUser == null;
 };
-
 const isAdmin = (req) => {
   if (!isUnauthorized(req)) {
     const user = userStorage.getUser(req.session.currentUser);
@@ -17,7 +14,6 @@ const isAdmin = (req) => {
   }
   return false;
 };
-
 const authInterceptor = () => {
   return (req, res, next) => {
     if (isUnauthorized(req)) {
@@ -27,7 +23,6 @@ const authInterceptor = () => {
     }
   };
 };
-
 const adminInterceptor = () => {
   return (req, res, next) => {
     if (!isAdmin(req)) {
@@ -37,8 +32,7 @@ const adminInterceptor = () => {
     }
   };
 };
-
+const cookieSession$0 = (userId) => {
-exports.cookieSession = (userId) => {
   return cookieSession({
     name: 'fredy-admin-session',
     keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
@@ -46,8 +40,8 @@ exports.cookieSession = (userId) => {
     maxAge: 8 * 60 * 60 * 1000, // 8 hours
   });
 };
-
+export { cookieSession$0 as cookieSession };
-exports.adminInterceptor = adminInterceptor;
+export { adminInterceptor };
-exports.authInterceptor = authInterceptor;
+export { authInterceptor };
-exports.isUnauthorized = isUnauthorized;
+export { isUnauthorized };
-exports.isAdmin = isAdmin;
+export { isAdmin };

lib/errors.js

@@ -9,7 +9,8 @@ class ExtendableError extends Error {
     }
   }
 }
-
 class NoNewListingsWarning extends ExtendableError {}
-
+export { NoNewListingsWarning };
-module.exports = { NoNewListingsWarning };
+export default {
+  NoNewListingsWarning,
+};

lib/notification/adapter/console.js

@@ -1,19 +1,12 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
 
-/**
+export const send = ({ serviceName, newListings, jobKey }) => {
- * simply prints out the found data to the console
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
   /* eslint-disable no-console */
   return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
   /* eslint-enable no-console */
 };
-
+export const config = {
-exports.config = {
+  id: 'console',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Console',
   description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
   config: {},

lib/notification/adapter/mailJet.js

@@ -1,35 +1,24 @@
-const mailjet = require('node-mailjet');
+import mailjet from 'node-mailjet';
-
+import path from 'path';
-const path = require('path');
+import fs from 'fs';
-const fs = require('fs');
+import Handlebars from 'handlebars';
-const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/template.hbs'), 'utf8');
+import { markdown2Html } from '../../services/markdown.js';
-
+import { getDirName } from '../../utils.js';
-const Handlebars = require('handlebars');
+const __dirname = getDirName();
+const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
 const emailTemplate = Handlebars.compile(template);
-const { markdown2Html } = require('../../services/markdown');
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-
-/**
- * sends a new listing using MailJet
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailJet'
+    (adapter) => adapter.id === 'mailjet'
   ).fields;
-
   const to = receiver
     .trim()
     .split(',')
     .map((r) => ({
       Email: r.trim(),
     }));
-
   return mailjet
-    .connect(apiPublicKey, apiPrivateKey)
+    .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [
@@ -49,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
       ],
     });
 };
-
+export const config = {
-exports.config = {
+  id: 'mailjet',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'MailJet',
   description: 'MailJet is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/mailJet.md'),

lib/notification/adapter/mattermost.js

@@ -1,26 +1,15 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-const { getJob } = require('../../services/storage/jobStorage');
+import { getJob } from '../../services/storage/jobStorage.js';
-const fetch = require('node-fetch');
+import fetch from 'node-fetch';
-
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-/**
- * sends new listings to mattermost
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-
   let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
   message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
   message += newListings.map(
     (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
   );
-
   return fetch(webhook, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
@@ -30,14 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     },
   });
 };
-
+export const config = {
-/**
+  id: 'mattermost',
- * exported config is being used in the frontend to generate the fields
- * incoming values will be the keys (and values) of the fields
- *
- */
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Mattermost',
   readme: markdown2Html('lib/notification/adapter/mattermost.md'),
   description: 'Fredy will send new listings to your mattermost team chat.',

lib/notification/adapter/ntfy.js

@@ -0,0 +1,51 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const message = `Address: ${newListing.address} Size: ${newListing.size.replace(/2m/g, '$m^2$')} Price: ${
+      newListing.price
+    }`;
+    return fetch(server, {
+      method: 'POST',
+      body: JSON.stringify({
+        topic: topic,
+        message: message,
+        title: newListing.title,
+        tags: [serviceName, jobName],
+        priority: parseInt(priority),
+        click: newListing.link,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'ntfy',
+  name: 'ntfy',
+  readme: markdown2Html('lib/notification/adapter/ntfy.md'),
+  description: 'Fredy will send new listings to your ntfy.',
+  fields: {
+    priority: {
+      type: 'number',
+      label: 'Priority',
+      description: 'The priority of the send notification.',
+    },
+    server: {
+      type: 'text',
+      label: 'Server-URL',
+      description: 'The server url to the send the notification to.',
+    },
+    topic: {
+      type: 'text',
+      label: 'topic',
+      description:
+        'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
+    },
+  },
+};

lib/notification/adapter/ntfy.md

@@ -0,0 +1,5 @@
+### ntfy Adapter
+
+For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the  [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+
+As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.

lib/notification/adapter/sendGrid.js

@@ -1,15 +1,6 @@
-const sgMail = require('@sendgrid/mail');
+import sgMail from '@sendgrid/mail';
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-/**
- * sends a new listing using SendGrid
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
   sgMail.setApiKey(apiKey);
   const msg = {
@@ -28,9 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   };
   return sgMail.send(msg);
 };
-
+export const config = {
-exports.config = {
+  id: 'sendgrid',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'SendGrid',
   description: 'SendGrid is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/sendGrid.md'),

lib/notification/adapter/slack.js

@@ -1,16 +1,7 @@
-const Slack = require('slack');
+import Slack from 'slack';
+import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
-const { markdown2Html } = require('../../services/markdown');
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-
-/**
- * sends a new listing to slack
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
   return newListings.map((payload) =>
     msg({
@@ -47,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     })
   );
 };
-
+export const config = {
-exports.config = {
+  id: 'slack',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Slack',
   readme: markdown2Html('lib/notification/adapter/slack.md'),
   description: 'Fredy will send new listings to the slack channel of your choice..',

lib/notification/adapter/sqlite.js

@@ -1,13 +1,6 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-const Database = require('better-sqlite3');
+import Database from 'better-sqlite3';
-
+export const send = ({ serviceName, newListings, jobKey }) => {
-/**
- * Stores data in a sqlite db in order to use the search results for later analytics
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
   const db = new Database('db/listings.db');
   const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
   db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
@@ -23,9 +16,8 @@ exports.send = ({ serviceName, newListings, jobKey }) => {
   });
   return Promise.resolve();
 };
-
+export const config = {
-exports.config = {
+  id: 'sqlite',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Sqlite',
   description: 'This adapter stores listings in a local sqlite3 database.',
   config: {},

lib/notification/adapter/telegram.js

@@ -1,7 +1,6 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-const { getJob } = require('../../services/storage/jobStorage');
+import { getJob } from '../../services/storage/jobStorage.js';
-const fetch = require('node-fetch');
+import fetch from 'node-fetch';
-
 const MAX_ENTITIES_PER_CHUNK = 8;
 const RATE_LIMIT_INTERVAL = 1010;
 /**
@@ -16,32 +15,23 @@ const arrayChunks = (inputArray, perChunk) =>
     all[ch] = [].concat(all[ch] || [], one);
     return all;
   }, []);
-
+function shorten(str, len = 30) {
-/**
+  return str.length > len ? str.substring(0, len) + '...' : str;
- * sends new listings to telegram
+}
- * @param serviceName e.g immowelt
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
   const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
-
   const promises = chunks.map((chunk) => {
     let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
     message += chunk.map(
       (o) =>
-        `<a href="${o.link}"><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
+        `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
         [o.address, o.price, o.size].join(' | ') +
         '\n\n'
     );
-
     /**
      * This is to not break the rate limit. It is to only send 1 message per second
      */
@@ -66,21 +56,10 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
       }, RATE_LIMIT_INTERVAL);
     });
   });
-
   return Promise.all(promises);
 };
-
+export const config = {
-function shorten(str, len = 30) {
+  id: 'telegram',
-  return str.length > len ? str.substring(0, len) + '...' : str;
-}
-
-/**
- * exported config is being used in the frontend to generate the fields
- * incoming values will be the keys (and values) of the fields
- *
- */
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Telegram',
   readme: markdown2Html('lib/notification/adapter/telegram.md'),
   description: 'Fredy will send new listings to your mobile, using Telegram.',

lib/notification/notify.js

@@ -1,24 +1,24 @@
-const fs = require('fs');
+import fs from 'fs';
 const path = './adapter';
 
 /** Read every integration existing in ./adapter **/
-const adapter = fs
+const adapter = await Promise.all(
-  .readdirSync('./lib/notification/adapter')
+  fs
-  .filter((file) => file.endsWith('.js'))
+    .readdirSync('./lib/notification/adapter')
-  .map((integPath) => require(`${path}/${integPath}`));
+    .filter((file) => file.endsWith('.js'))
+    .map(async (integPath) => await import(`${path}/${integPath}`))
+);
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
-
+const findAdapter = (notificationAdapter) => {
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
+  return adapter.find((a) => a.config.id === notificationAdapter.id);
+};
+export const send = (serviceName, newListings, notificationConfig, jobKey) => {
   //this is not being used in tests, therefore adapter are always set
   return notificationConfig
     .filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
     .map((notificationAdapter) => findAdapter(notificationAdapter))
     .map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
 };
-
-const findAdapter = (notificationAdapter) => {
-  return adapter.find((a) => a.config.id === notificationAdapter.id);
-};

lib/provider/einsAImmobilien.js

@@ -1,24 +1,18 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
   if (o.rooms != null) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-
   return Object.assign(o, { size, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '.tabelle',
@@ -34,17 +28,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: '1a Immobilien',
   baseUrl: 'https://www.1a-immobilienmarkt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'einsAImmobilien',
 };
-
+export { config };
-exports.config = config;

lib/provider/immobilienDe.js

@@ -1,15 +1,11 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
 }
-
 function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
-
 function normalize(o) {
   const id = parseId(shortenLink(o.link));
   const size = o.size || 'N/A m²';
@@ -19,14 +15,11 @@ function normalize(o) {
   const link = shortenLink(o.link);
   return Object.assign(o, { id, price, size, title, address, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '.estates_list .list_immo a._ref',
@@ -43,17 +36,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immobilien.de',
   baseUrl: 'https://www.immobilien.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immobilienDe',
 };
-
+export { config };
-exports.config = config;

lib/provider/immonet.js

@@ -1,52 +1,42 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
-  const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
+  const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
-  //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
+  const link = o.id;
-  //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
-  const link = `https://www.immonet.de/angebot/${id}`;
   return Object.assign(o, { id, address, price, size, title, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '#result-list-stage .item',
+  crawlContainer: '.content-wrapper-tiles .ng-star-inserted',
   sortByDateParam: 'sortby=19',
   crawlFields: {
-    id: '@id',
+    id: '.card a@href',
-    price: 'div[id*="selPrice_"] | trim',
+    title: '.card h3 |trim',
-    size: 'div[id*="selArea_"] | trim',
+    price: '.card .has-font-300 .is-bold | trim',
-    title: '.item a img@title',
+    size: '.card .has-font-300 .ml-100 | trim',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
+    address: '.card span:nth-child(2) | trim',
   },
   paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immonet',
   baseUrl: 'https://www.immonet.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immonet',
 };
-
+export { config };
-exports.config = config;

lib/provider/immoscout.js

@@ -1,22 +1,17 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
-
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
   const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
   return Object.assign(o, { title, address, link });
 }
-
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
-
 const config = {
   url: null,
   crawlContainer: '#resultListItems li.result-list__listing',
@@ -33,17 +28,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immoscout',
   baseUrl: 'https://www.immobilienscout24.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immoscout',
 };
-
+export { config };
-exports.config = config;

lib/provider/immoswp.js

@@ -1,7 +1,5 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
   const size = o.size || 'N/A m²';
@@ -12,14 +10,11 @@ function normalize(o) {
   const description = o.description;
   return Object.assign(o, { id, address, price, size, title, link, description });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '.js-serp-item',
@@ -36,17 +31,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immo Südwest Presse',
   baseUrl: 'https://immo.swp.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immoswp',
 };
-
+export { config };
-exports.config = config;

lib/provider/immowelt.js

@@ -1,18 +1,13 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   return o;
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: "div[class^='EstateItem-']",
@@ -29,16 +24,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immowelt',
   baseUrl: 'https://www.immowelt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immowelt',
 };
-
+export { config };
-exports.config = config;

lib/provider/kleinanzeigen.js

@@ -1,23 +1,17 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
-
 function normalize(o) {
   const size = o.size || '--- m²';
-
   return Object.assign(o, { size });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   const isBlacklistedDistrict =
     appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-
   return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '#srchrslt-adtable .ad-listitem ',
@@ -36,18 +30,15 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Ebay Kleinanzeigen',
   baseUrl: 'https://www.ebay-kleinanzeigen.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'kleinanzeigen',
 };
-
+export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlacklistedDistricts = blacklistedDistricts || [];
   appliedBlackList = blacklist || [];
 };
-
+export { config };
-exports.config = config;

lib/provider/neubauKompass.js

@@ -1,15 +1,11 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   return o;
 }
-
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
-
 const config = {
   url: null,
   crawlContainer: '.nbk-container >div article',
@@ -25,17 +21,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Neubau Kompass',
   baseUrl: 'https://www.neubaukompass.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'neubauKompass',
 };
-
+export { config };
-exports.config = config;

lib/provider/wgGesucht.js

@@ -1,18 +1,13 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   return o;
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '#main_column .wgg_card',
@@ -28,17 +23,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Wg gesucht',
   baseUrl: 'https://www.wg-gesucht.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'wgGesucht',
 };
-
+export { config };
-exports.config = config;

lib/services/markdown.js

@@ -1,6 +1,6 @@
-const markdown = require('markdown').markdown;
+import markdown$0 from 'markdown';
-const fs = require('fs');
+import fs from 'fs';
-
+const markdown = markdown$0.markdown;
-exports.markdown2Html = function markdown2Html(filePath) {
+export function markdown2Html(filePath) {
   return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
-};
+}

lib/services/queryStringMutator.js

@@ -1,21 +1,9 @@
-const queryString = require('query-string');
+import queryString from 'query-string';
-
+export default (_url, sortByDateParam) => {
-/**
- * for Fredy, it is important to sort search results by date, starting with the latest listing. if it is not sorted, we
- * might never actually find the newest results, no matter how many pages we crawl.
- * It has been written in the documentation, but obviously nobody reads docu theses days which is why it's been done
- * automagically now.
- *
- * @param _url actual provider url containing the searchParams
- * @param sortByDateParam param(s) indicating the correct sort order
- * @returns {`${string}?${string}`} correctly formatted url
- */
-module.exports = (_url, sortByDateParam) => {
   //if no mutation is necessary, just return the original url
   if (sortByDateParam == null) {
     return _url;
   }
-
   const original = queryString.parseUrl(_url);
   const mutate = queryString.parse(sortByDateParam);
   return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;

lib/services/requestDriver.js

@@ -1,19 +1,20 @@
-const fetch = require('node-fetch');
+import fetch from 'node-fetch';
-const config = require('../../conf/config.json');
+import { config } from '../utils.js';
-
+import { makeUrlResidential } from './scrapingAnt.js';
-const { makeUrlResidential } = require('./scrapingAnt');
+import https from 'https';
 //if ScrapingAnt got blocked, this http status is returned
 const BLOCKED_HTTP_STATUS = 423;
 const NOT_FOUND_HTTP_STATUS = 404;
 const MAX_RETRIES_SCRAPING_ANT = 10;
 const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
+const agent = new https.Agent({
+  rejectUnauthorized: false,
+});
 
 function makeDriver(headers = {}) {
   let cookies = '';
-
   async function scrapingAntDriver(context, callback, retryCounter = 0) {
     const proxyType = config.scrapingAnt?.proxy || 'datacenter';
-
     try {
       const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
       const response = await fetch(url, {
@@ -22,12 +23,10 @@ function makeDriver(headers = {}) {
           cookie: cookies,
         },
       });
-
       const result = await response.text();
       if (cookies.length === 0) {
         cookies = response.headers.raw()['set-cookie'] || [];
       }
-
       callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
@@ -36,7 +35,6 @@ function makeDriver(headers = {}) {
         callback(null, []);
         return;
       }
-
       if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
         retryCounter++;
         console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
@@ -51,21 +49,20 @@ function makeDriver(headers = {}) {
 
   /**
    * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
-   * everything != Immoscout as of writing this)
+   * everything != Immoscout & Immonet as of writing this)
    */
   return async function driver(context, callback) {
     if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
       return scrapingAntDriver(context, callback);
     }
-
     try {
       const response = await fetch(context.url, {
         headers: {
           ...headers,
           Cookie: cookies,
         },
+        agent,
       });
-
       const result = await response.text();
       callback(null, result);
     } catch (exception) {
@@ -74,5 +71,4 @@ function makeDriver(headers = {}) {
     }
   };
 }
-
+export default makeDriver;
-module.exports = makeDriver;

lib/services/scraper.js

@@ -1,7 +1,6 @@
-const config = require('../../conf/config.json');
+import { config } from '../utils.js';
-const makeDriver = require('./requestDriver');
+import makeDriver from './requestDriver.js';
-const Xray = require('x-ray');
+import Xray from 'x-ray';
-
 class Scraper {
   constructor() {
     const filters = {
@@ -9,38 +8,29 @@ class Scraper {
       trim: this._trim,
       int: this._int,
     };
-
     const headers = {
       'User-Agent':
         'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
     };
-
     if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
       headers['x-api-key'] = config.scrapingAnt.apiKey;
     }
     const driver = makeDriver(headers);
-
     const xray = Xray({ filters });
     xray.driver(driver);
-
     this.xray = xray;
   }
-
   get x() {
     return this.xray;
   }
-
   _removeNewline(value) {
     return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
   }
-
   _trim(value) {
     return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
   }
-
   _int(value) {
     return typeof value === 'string' ? parseInt(value, 10) : value;
   }
 }
-
+export default new Scraper().x;
-module.exports = new Scraper().x;

lib/services/scrapingAnt.js

@@ -1,25 +1,29 @@
-const { metaInformation } = require('../provider/immoscout');
+import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
-//to better configure re-capture chose a random proxy each time we do a call
+import { metaInformation as immoNetInfo } from '../provider/immonet.js';
-const config = require('../../conf/config.json');
+import { config } from '../utils.js';
 
-const isImmoscout = (id) => {
+const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${new Buffer(
-  return id.toLowerCase() === metaInformation.id;
+  'window.scrollTo(0,document.body.scrollHeight);'
+).toString('base64')}`;
+
+const needScrapingAnt = (id) => {
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
 };
-
+export const transformUrlForScrapingAnt = (url, id) => {
-exports.transformUrlForScrapingAnt = (url, id) => {
+  let urlParams = '';
-  if (isImmoscout(id)) {
+  if (needScrapingAnt(id)) {
-    //only do calls to scrapingAnt when dealing with Immoscout
+    if (id.toLowerCase() === immoNetInfo.id) {
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
+      urlParams = additionalImmonetUrlParams;
+    }
+    //only do calls to scrapingAnt when dealing with Immoscout/Immonet
+    url = `https://api.scrapingant.com/v2/general?url=${encodeURIComponent(url)}&proxy_type=datacenter${urlParams}`;
   }
   return url;
 };
-
+export const isScrapingAntApiKeySet = () => {
-exports.isScrapingAntApiKeySet = () => {
   return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
 };
-
+export const makeUrlResidential = (url) => {
-exports.isImmoscout = isImmoscout;
-
-exports.makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');
 };
+export { needScrapingAnt };

lib/services/security/hash.js

@@ -1,3 +1,2 @@
-const crypto = require('crypto');
+import crypto from 'crypto';
-
+export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
-exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/similarity-check/SimilarityCacheEntry.js

@@ -1,27 +1,17 @@
-const stringSimilarity = require('string-similarity');
+import stringSimilarity from 'string-similarity';
-
 //if the score is higher than this, it will be considered a match
 const MAX_DICE_INDEX = 0.7;
-
+export default (class SimilarityCacheEntry {
-/**
- * The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
- *
- * @type {module.SimilarityCacheEntry}
- */
-module.exports = class SimilarityCacheEntry {
   constructor(time) {
     this.time = time;
     this.values = [];
   }
-
   setCacheEntry = (entry) => {
     this.values.push(entry);
   };
-
   getTime = () => {
     return this.time;
   };
-
   hasSimilarEntries = (value) => {
     if (this.values.length > 0) {
       for (let i = 0; i < this.values.length; i++) {
@@ -33,4 +23,4 @@ module.exports = class SimilarityCacheEntry {
     }
     return false;
   };
-};
+});

lib/services/similarity-check/similarityCache.js

@@ -1,63 +1,40 @@
-/**
+import SimilarityCacheEntry from './SimilarityCacheEntry.js';
- * each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
+import { config } from '../../utils.js';
- * a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
- * the user won't see any duplicates
- *
- * The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
- *
- * @type {module.SimilarityCacheEntry|{}}
- */
-const SimilarityCacheEntry = require('./SimilarityCacheEntry');
-const config = require('../../../conf/config.json');
-
 //5 minutes
 let retention = 5 * 60 * 1000;
-
 const intervalInMs = config.interval * 60 * 1000;
 //an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
 if (intervalInMs <= retention) {
   retention = Math.floor(intervalInMs / 2);
 }
-
 //jobid -> SimilarityCacheEntry
 const cache = {};
-
 let intervalId;
-
-exports.addCacheEntry = (jobId, value) => {
-  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
-  cache[jobId].setCacheEntry(value);
-};
-
-exports.hasSimilarEntries = (jobId, value) => {
-  if (cache[jobId] == null) {
-    return false;
-  }
-
-  return cache[jobId].hasSimilarEntries(value);
-};
-
 /**
  * cleanup
  */
 intervalId = setInterval(() => {
   const keysToBeRemoved = [];
   const now = Date.now();
-
   Object.keys(cache).forEach((key) => {
     if (cache[key].getTime() + retention < now) {
       keysToBeRemoved.push(key);
     }
   });
-
   if (keysToBeRemoved.length > 0) {
     keysToBeRemoved.forEach((key) => delete cache[key]);
   }
 }, 10000);
-
+export const addCacheEntry = (jobId, value) => {
-/**
+  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
- * mostly used for tests
+  cache[jobId].setCacheEntry(value);
- */
+};
-exports.stopCacheCleanup = () => {
+export const hasSimilarEntries = (jobId, value) => {
+  if (cache[jobId] == null) {
+    return false;
+  }
+  return cache[jobId].hasSimilarEntries(value);
+};
+export const stopCacheCleanup = () => {
   clearInterval(intervalId);
 };

lib/services/storage/LowDashAdapter.js

@@ -0,0 +1,8 @@
+import lodash from 'lodash';
+import { LowSync } from 'lowdb';
+export default class LowdashAdapter extends LowSync {
+  constructor(adapter) {
+    super(adapter);
+    this.chain = lodash.chain(this).get('data');
+  }
+}

lib/services/storage/jobStorage.js

@@ -1,28 +1,30 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
+import { nanoid } from 'nanoid';
-const FileSync = require('lowdb/adapters/FileSync');
+import * as listingStorage from './listingsStorage.js';
-const adapter = new FileSync(DB_PATH);
+import { getDirName } from '../../utils.js';
-const low = require('lowdb');
+import path from 'path';
-const db = low(adapter);
+import LowdashAdapter from './LowDashAdapter.js';
-const { nanoid } = require('nanoid');
-const listingStorage = require('./listingsStorage');
 
-db.defaults({ jobs: [] }).write();
+const file = path.join(getDirName(), '../', 'db/jobs.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter);
 
-exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+db.read();
+
+db.data ||= { jobs: [] };
+
+export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
       ? null
-      : db
+      : db.chain
           .get('jobs')
           .find((job) => job.id === jobId)
           .value();
-
+  const jobs = db.chain
-  const jobs = db
     .get('jobs')
-    .value()
+    .filter((job) => job.id !== jobId)
-    .filter((job) => job.id !== jobId);
+    .value();
-
   jobs.push({
     id: jobId || nanoid(),
     //make sure to not overwrite the user id in case an admin changes the job
@@ -33,57 +35,55 @@ exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, no
     provider,
     notificationAdapter,
   });
-
+  db.chain.set('jobs', jobs).value();
-  db.set('jobs', jobs).write();
+  db.write();
 };
-
+export const getJob = (jobId) => {
-exports.getJob = (jobId) => {
+  const job = db.chain
-  const job = db
     .get('jobs')
     .find((job) => job.id === jobId)
     .value();
-
   if (job == null) {
     return null;
   }
-
   return {
     ...job,
     numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
   };
 };
-
+export const setJobStatus = ({ jobId, status }) => {
-exports.setJobStatus = ({ jobId, status }) => {
+  db.chain
-  db.get('jobs')
+    .get('jobs')
     .find((job) => job.id === jobId)
     .assign({ enabled: status })
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeJob = (jobId) => {
-exports.removeJob = (jobId) => {
   listingStorage.removeListings(jobId);
-  db.get('jobs')
+  db.chain
+    .get('jobs')
     .remove((job) => job.id === jobId)
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeJobsByUserId = (userId) => {
-exports.removeJobsByUserId = (userId) => {
+  db.chain
-  db.get('jobs')
+    .get('jobs')
-    .value()
     .filter((job) => job.userId === userId)
     .forEach((job) => listingStorage.removeListings(job.id));
-
+  db.chain
-  db.get('jobs')
+    .get('jobs')
-    .remove((job) => job.userId === userId)
+    .remove((job) => job.userId === userId)
-    .write();
+    .value();
-};
+  db.write();
-
+};
-exports.getJobs = () => {
+export const getJobs = () => {
-  return db
+  return db.chain
     .get('jobs')
-    .value()
     .map((job) => ({
       ...job,
       numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
-    }));
+    }))
+    .value();
 };

lib/services/storage/listingsStorage.js

@@ -1,10 +1,15 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
+const file = path.join(getDirName(), '../', 'db/jobListingData.json');
-const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new JSONFileSync(file);
-const adapter = new FileSync(DB_PATH);
+const db = new LowdashAdapter(adapter);
-const low = require('lowdb');
+
-const db = low(adapter);
+db.read();
+
+db.data ||= {};
 
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
@@ -19,35 +24,31 @@ const buildKey = (jobKey, providerId, endpoint) => {
   }
   return key;
 };
-
+export const getNumberOfAllKnownListings = (jobId) => {
-exports.getNumberOfAllKnownListings = (jobId) => {
+  const data = db.chain.get(`${jobId}.providerData`).value() || {};
-  const data = db.get(`${jobId}.providerData`).value() || {};
   return Object.values(data)
     .map((values) => Object.keys(values).length)
     .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
 };
-
+export const getListingProviderDataForAnalytics = (jobId) => {
-exports.getListingProviderDataForAnalytics = (jobId) => {
   const key = buildKey(jobId, 'providerData');
-  return db.get(key).value() || {};
+  return db.chain.get(key).value() || {};
 };
-
+export const getKnownListings = (jobId, providerId) => {
-exports.getKnownListings = (jobId, providerId) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-  return db.get(providerListingsKey).value() || {};
+  return db.chain.get(providerListingsKey).value() || {};
 };
-
+export const setKnownListings = (jobId, providerId, listings) => {
-exports.setKnownListings = (jobId, providerId, listings) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-
+  db.chain.set(providerListingsKey, listings).value();
-  return db.set(providerListingsKey, listings).write();
+  return db.write();
 };
-
+export const setLastJobExecution = (jobId) => {
-exports.setLastJobExecution = (jobId) => {
   const key = buildKey(jobId, null, 'lastExecution');
-  return db.set(key, Date.now()).write();
+  db.chain.set(key, Date.now()).value();
+  return db.write();
 };
-
+export const removeListings = (jobId) => {
-exports.removeListings = (jobId) => {
+  db.chain.unset(jobId).value();
-  db.unset(jobId).write();
+  db.write();
 };

lib/services/storage/userStorage.js

@@ -1,14 +1,17 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
-const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
+import { getDirName } from '../../utils.js';
-const FileSync = require('lowdb/adapters/FileSync');
+import * as hasher from '../security/hash.js';
-const adapter = new FileSync(DB_PATH);
+import { nanoid } from 'nanoid';
-const low = require('lowdb');
+import * as jobStorage from './jobStorage.js';
-const db = low(adapter);
+import path from 'path';
-const hasher = require('../security/hash');
+import LowdashAdapter from './LowDashAdapter.js';
-const { nanoid } = require('nanoid');
-const jobStorage = require('./jobStorage');
 
-db.defaults({
+const file = path.join(getDirName(), '../', 'db/users.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter);
+
+db.read();
+db.data ||= {
   user: [
     //you probably want to change the default password ;)
     {
@@ -20,11 +23,11 @@ db.defaults({
       isDemo: false,
     },
   ],
-}).write();
+};
 
-exports.getUsers = (withPassword) => {
+export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();
-  return db
+  return db.chain
     .get('user')
     .value()
     .map((user) => ({
@@ -34,13 +37,12 @@ exports.getUsers = (withPassword) => {
       numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
     }));
 };
-
+export const getUser = (id) => {
-exports.getUser = (id) => {
   const jobs = jobStorage.getJobs();
-  const user = db
+  const user = db.chain
     .get('user')
-    .value()
+    .find((user) => user.id === id)
-    .find((user) => user.id === id);
+    .value();
   if (user == null) {
     return null;
   }
@@ -49,13 +51,11 @@ exports.getUser = (id) => {
     numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
   };
 };
-
+export const upsertUser = ({ username, password, userId, isAdmin }) => {
-exports.upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db.chain
-  const user = db
     .get('user')
-    .value()
+    .filter((u) => u.id !== userId)
-    .filter((u) => u.id !== userId);
+    .value();
-
   user.push({
     id: userId || nanoid(),
     username,
@@ -63,21 +63,24 @@ exports.upsertUser = ({ username, password, userId, isAdmin }) => {
     password: hasher.hash(password),
     isAdmin,
   });
-
+  db.chain.set('user', user).value();
-  db.set('user', user).write();
+  db.write();
 };
-
+export const setLastLoginToNow = ({ userId }) => {
-exports.setLastLoginToNow = ({ userId }) => {
+  db.chain
-  db.get('user')
+    .get('user')
     .find((u) => u.id === userId)
     .assign({ lastLogin: Date.now() })
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeUser = (userId) => {
-exports.removeUser = (userId) => {
+  const user = db.chain.get('user').value();
-  const user = db.get('user').value();
+  db.chain
-  db.set(
+    .set(
-    'user',
+      'user',
-    user.filter((u) => u.id !== userId)
+      user.filter((u) => u.id !== userId)
-  ).write();
+    )
+    .value();
+  db.write();
 };

lib/utils.js

@@ -1,17 +1,18 @@
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { readFile } from 'fs/promises';
+
 function isOneOf(word, arr) {
   if (arr == null || arr.length === 0) {
     return false;
   }
   const expression = String.raw`\b(${arr.join('|')})\b`;
   const blacklist = new RegExp(expression, 'ig');
-
   return blacklist.test(word);
 }
-
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
-
 function timeStringToMs(timeString, now) {
   const d = new Date(now);
   const parts = timeString.split(':');
@@ -20,17 +21,31 @@ function timeStringToMs(timeString, now) {
   d.setSeconds(0);
   return d.getTime();
 }
-
 function duringWorkingHoursOrNotSet(config, now) {
   const { workingHours } = config;
   if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
     return true;
   }
-
   const toDate = timeStringToMs(workingHours.to, now);
   const fromDate = timeStringToMs(workingHours.from, now);
-
   return fromDate <= now && toDate >= now;
 }
 
-module.exports = { isOneOf, nullOrEmpty, duringWorkingHoursOrNotSet };
+function getDirName() {
+  return dirname(fileURLToPath(import.meta.url));
+}
+
+const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+
+export { isOneOf };
+export { nullOrEmpty };
+export { duringWorkingHoursOrNotSet };
+export { getDirName };
+export { config };
+export default {
+  isOneOf,
+  nullOrEmpty,
+  duringWorkingHoursOrNotSet,
+  getDirName,
+  config,
+};

package.json

@@ -1,21 +1,22 @@
 {
   "name": "fredy",
-  "version": "6.0.2",
+  "version": "7.2.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
     "dev": "yarn && rm -rf ./ui/public/* && vite",
     "ui": "rm -rf ./ui/public/* && vite",
     "prod": "yarn && vite build --emptyOutDir",
-    "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
+    "format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
-    "test": "mocha --timeout 3000000 test/**/*.test.js",
+    "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
-    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
+    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
   },
   "husky": {
     "hooks": {
       "pre-commit": "lint-staged"
     }
   },
+  "type": "module",
   "lint-staged": {
     "*.js": [
       "eslint ./index.js ./lib/**/*.js ./test/**/*.js",
@@ -54,54 +55,54 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@douyinfe/semi-ui": "2.31.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "7.7.0",
     "@vitejs/plugin-react": "3.1.0",
-    "better-sqlite3": "8.1.0",
+    "better-sqlite3": "8.2.0",
-    "body-parser": "1.20.1",
+    "body-parser": "1.20.2",
     "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
     "highcharts": "10.3.3",
-    "highcharts-react-official": "3.1.0",
+    "highcharts-react-official": "3.2.0",
-    "lowdb": "1.0.0",
+    "lodash": "4.17.21",
+    "lowdb": "5.1.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.3.3",
+    "nanoid": "4.0.1",
-    "node-fetch": "2.6.9",
+    "node-fetch": "3.3.1",
-    "node-mailjet": "3.3.13",
+    "node-mailjet": "6.0.2",
-    "query-string": "7.1.3",
+    "query-string": "8.1.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-redux": "8.0.5",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "react-switch": "7.0.0",
     "redux": "4.2.1",
     "redux-thunk": "2.4.2",
     "restana": "4.9.7",
-    "semantic-ui-react": "2.1.4",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "4.1.1",
+    "vite": "4.2.0",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.20.12",
+    "@babel/core": "7.21.3",
-    "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-parser": "7.21.3",
     "@babel/preset-env": "7.20.2",
     "@babel/preset-react": "7.18.6",
     "chai": "4.3.7",
-    "eslint": "8.34.0",
+    "eslint": "8.36.0",
-    "eslint-config-prettier": "8.6.0",
+    "eslint-config-prettier": "8.7.0",
     "eslint-plugin-react": "7.32.2",
+    "esmock": "2.1.0",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.3",
-    "lint-staged": "13.1.2",
+    "lint-staged": "13.2.0",
     "mocha": "10.2.0",
-    "prettier": "2.8.4",
+    "prettier": "2.8.5",
-    "proxyquire": "2.1.3",
     "redux-logger": "3.0.6"
   }
 }

test/mocks/mockNotification.js

@@ -1,12 +1,10 @@
-module.exports = {
+let tmpStore = {};
-  _tmpStore: {},
 
-  send: (serviceName, payload) => {
+export const send = (serviceName, payload) => {
-    this._tmpStore = { serviceName, payload };
+  tmpStore = { serviceName, payload };
-    return [Promise.resolve()];
+  return [Promise.resolve()];
-  },
+};
-
+
-  get: () => {
+export const get = () => {
-    return this._tmpStore;
+  return tmpStore;
-  },
 };

test/mocks/mockStore.js

@@ -1,11 +1,8 @@
 const db = {};
-
+export const setKnownListings = (jobKey, providerId, listings) => {
-exports.setKnownListings = (jobKey, providerId, listings) => {
   if (!Array.isArray(listings)) throw Error('Not a valid array');
-
   db[providerId] = listings;
 };
-
+export const getKnownListings = (jobKey, providerId) => {
-exports.getKnownListings = (jobKey, providerId) => {
   return db[providerId] || [];
 };

test/provider/einsAImmobilien.test.js

@@ -1,35 +1,25 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { providerConfig, mockFredy } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/einsAImmobilien.js';
-const expect = require('chai').expect;
+
-const provider = require('../../lib/provider/einsAImmobilien');
+const expect = chai.expect;
 
 describe('#einsAImmobilien testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.einsAImmobilien, [], []);
-
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test einsAImmobilien provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
       fredy.execute().then((listings) => {
         expect(listings).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
@@ -37,7 +27,6 @@ describe('#einsAImmobilien testsuite()', () => {
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;

test/provider/immobilienDe.test.js

@@ -1,31 +1,21 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { providerConfig, mockFredy } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immobilienDe.js';
-const expect = require('chai').expect;
+const expect = chai.expect;
-const provider = require('../../lib/provider/immobilienDe');
-
 describe('#immobilien.de testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.immobilienDe, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immobilien.de provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immobilienDe');
         notificationObj.payload.forEach((notify) => {
@@ -36,7 +26,6 @@ describe('#immobilien.de testsuite()', () => {
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');

test/provider/immonet.test.js

@@ -1,37 +1,34 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immonet.js';
-const expect = require('chai').expect;
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const provider = require('../../lib/provider/immonet');
+const expect = chai.expect;
-
 describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.immonet, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immonet provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
+      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+        /* eslint-disable no-console */
+        console.info('Skipping Immonet test as ScrapingAnt Api Key is not set.');
+        /* eslint-enable no-console */
+        resolve();
+        return;
+      }
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immonet');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
@@ -42,7 +39,6 @@ describe('#immonet testsuite()', () => {
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immonet.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/immoscout.test.js

@@ -1,25 +1,17 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immoscout.js';
-const expect = require('chai').expect;
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const provider = require('../../lib/provider/immoscout');
+const expect = chai.expect;
-const scrapingAnt = require('../../lib/services/scrapingAnt');
-
 describe('#immoscout testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   provider.init(providerConfig.immoscout, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immoscout provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       if (!scrapingAnt.isScrapingAntApiKeySet()) {
         /* eslint-disable no-console */
@@ -28,15 +20,12 @@ describe('#immoscout testsuite()', () => {
         resolve();
         return;
       }
-
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immoscout');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
@@ -45,7 +34,6 @@ describe('#immoscout testsuite()', () => {
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');

test/provider/immoswp.test.js

@@ -1,34 +1,23 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immoswp.js';
-const expect = require('chai').expect;
+const expect = chai.expect;
-const provider = require('../../lib/provider/immoswp');
-
 describe('#immoswp testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.immoswp, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immoswp provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immoswp');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
@@ -37,7 +26,6 @@ describe('#immoswp testsuite()', () => {
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.price).that.does.include('€');
           expect(notify.title).to.be.not.empty;

test/provider/immowelt.test.js

@@ -1,42 +1,30 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/immowelt.js';
-const expect = require('chai').expect;
+const expect = chai.expect;
-const provider = require('../../lib/provider/immowelt');
-
 describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   it('should test immowelt provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immowelt');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.title).to.be.a('string');
-
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');

test/provider/kleinanzeigen.test.js

@@ -1,40 +1,29 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/kleinanzeigen.js';
-const expect = require('chai').expect;
+const expect = chai.expect;
-const provider = require('../../lib/provider/kleinanzeigen');
-
 describe('#kleinanzeigen testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   it('should test kleinanzeigen provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.kleinanzeigen, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');

test/provider/neubauKompass.test.js

@@ -1,41 +1,29 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/neubauKompass.js';
-const expect = require('chai').expect;
+const expect = chai.expect;
-const provider = require('../../lib/provider/neubauKompass');
-
 describe('#neubauKompass testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   provider.init(providerConfig.neubauKompass, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test neubauKompass provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj.serviceName).to.equal('neubauKompass');
-
         notificationObj.payload.forEach((notify) => {
           expect(notify).to.be.a('object');
-
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.neubaukompass.de');

test/provider/testProvider.json

@@ -9,7 +9,7 @@
     "enabled": true
   },
   "immonet": {
-    "url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
+    "url": "https://www.immonet.de/immobiliensuche/beta?pageoffset=1&listsize=100&objecttype=1&locationname=D%C3%BCsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
     "enabled": true
   },
   "immowelt": {

test/provider/utils.test.js

@@ -1,14 +1,13 @@
-const utils = require('../../lib/utils');
+import utils from '../../lib/utils.js';
-const assert = require('assert');
+import assert from 'assert';
-const expect = require('chai').expect;
+import chai from 'chai';
-
+const expect = chai.expect;
 const fakeWorkingHoursConfig = (from, to) => ({
   workingHours: {
     to,
     from,
   },
 });
-
 describe('utils', () => {
   describe('#isOneOf()', () => {
     it('should be false', () => {
@@ -18,7 +17,6 @@ describe('utils', () => {
       assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
     });
   });
-
   describe('#duringWorkingHoursOrNotSet()', () => {
     it('should be false', () => {
       expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;

test/provider/wgGesucht.test.js

@@ -1,35 +1,25 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const mockNotification = require('../mocks/mockNotification');
+import { get } from '../mocks/mockNotification.js';
-const providerConfig = require('./testProvider.json');
+import { mockFredy, providerConfig } from '../utils.js';
-const mockStore = require('../mocks/mockStore');
+import chai from 'chai';
-const proxyquire = require('proxyquire').noCallThru();
+import * as provider from '../../lib/provider/wgGesucht.js';
-const expect = require('chai').expect;
+const expect = chai.expect;
-const provider = require('../../lib/provider/wgGesucht');
-
 describe('#wgGesucht testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   provider.init(providerConfig.wgGesucht, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test wgGesucht provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
       const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-        const notificationObj = mockNotification.get();
+        const notificationObj = get();
         expect(notificationObj.serviceName).to.equal('wgGesucht');
         notificationObj.payload.forEach((notify) => {
           expect(notify).to.be.a('object');
-
           /** check the actual structure **/
-
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.details).to.be.a('string');

test/queryStringMutator/queryStringMutator.test.js

@@ -1,9 +1,17 @@
-const testData = require('./testData.json');
+import fs from 'fs';
-const expect = require('chai').expect;
+import chai from 'chai';
-const fs = require('fs');
+import { readFile } from 'fs/promises';
+import mutator from '../../lib/services/queryStringMutator.js';
+import queryString from 'query-string';
+const expect = chai.expect;
 
-const mutator = require('../../lib/services/queryStringMutator.js');
+const data = await readFile(new URL('./testData.json', import.meta.url));
-const queryString = require('query-string');
+
+const testData = JSON.parse(data);
+
+let _provider = await Promise.all(
+  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
+);
 
 /**
  * Test test might look a bit weird at first, but listen stranger...
@@ -12,18 +20,14 @@ const queryString = require('query-string');
  */
 describe('queryStringMutator', () => {
   it('should fix all urls', () => {
-    let _provider = fs.readdirSync('./lib/provider/').map((integPath) => require(`../../lib/provider/${integPath}`));
-
     for (let test of testData) {
       const provider = _provider.find((p) => p.metaInformation.id === test.id);
       if (provider == null) {
         throw new Error(`Cannot find provider for given id: ${test.id}`);
       }
-
       const fixedUrl = mutator(test.url, provider.config.sortByDateParam);
       const expectedParams = queryString.parseUrl(test.shouldBecome);
       const actualParams = queryString.parseUrl(fixedUrl);
-
       //check if all new params are existing
       expect(Object.keys(expectedParams.query)).to.include.members(Object.keys(actualParams.query));
       expect(Object.values(expectedParams.query)).to.include.members(Object.values(actualParams.query));

test/similarity/similarity.test.js

@@ -1,6 +1,6 @@
-const SimilarityCacheEntry = require('../../lib/services/similarity-check/SimilarityCacheEntry');
+import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
-const expect = require('chai').expect;
+import chai from 'chai';
-
+const expect = chai.expect;
 describe('similarityCheck', () => {
   describe('#similarityCheck()', () => {
     it('should be false', () => {

test/utils.js

@@ -0,0 +1,17 @@
+import { readFile } from 'fs/promises';
+import esmock from 'esmock';
+import * as mockStore from './mocks/mockStore.js';
+import { send } from './mocks/mockNotification.js';
+
+export const providerConfig = JSON.parse(await readFile(new URL('./provider/testProvider.json', import.meta.url)));
+
+export const mockFredy = async () => {
+  return await esmock('../lib/FredyRuntime', {
+    '../lib/services/storage/listingsStorage.js': {
+      ...mockStore,
+    },
+    '../lib/notification/notify.js': {
+      send,
+    },
+  });
+};

ui/src/App.jsx

@@ -3,13 +3,10 @@ import React, { useEffect } from 'react';
 import InsufficientPermission from './components/permission/InsufficientPermission';
 import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
 import GeneralSettings from './views/generalSettings/GeneralSettings';
-import ToastsContainer from './components/toasts/ToastContainer';
 import JobMutation from './views/jobs/mutation/JobMutation';
 import UserMutator from './views/user/mutation/UserMutator';
-import ToastContext from './components/toasts/ToastContext';
 import JobInsight from './views/jobs/insights/JobInsight.jsx';
 import { useDispatch, useSelector } from 'react-redux';
-import useToast from './components/toasts/useToast';
 import { Switch, Redirect } from 'react-router-dom';
 import Logout from './components/logout/Logout';
 import Logo from './components/logo/Logo';
@@ -23,20 +20,21 @@ import './App.less';
 
 export default function FredyApp() {
   const dispatch = useDispatch();
-  const [showToast, onToastFinished, toasts] = useToast();
   const [loading, setLoading] = React.useState(true);
   const currentUser = useSelector((state) => state.user.currentUser);
 
   useEffect(() => {
     async function init() {
-      await dispatch.provider.getProvider();
-      await dispatch.jobs.getJobs();
-      await dispatch.jobs.getProcessingTimes();
-      await dispatch.notificationAdapter.getAdapter();
       await dispatch.user.getCurrentUser();
-
+      if (!needsLogin()) {
+        await dispatch.provider.getProvider();
+        await dispatch.jobs.getJobs();
+        await dispatch.jobs.getProcessingTimes();
+        await dispatch.notificationAdapter.getAdapter();
+      }
       setLoading(false);
     }
+
     init();
   }, [currentUser?.userId]);
 
@@ -56,44 +54,41 @@ export default function FredyApp() {
   return loading ? null : needsLogin() ? (
     login()
   ) : (
-    <ToastContext.Provider value={{ showToast }}>
+    <div className="app">
-      <div className="app">
+      <div className="app__container">
-        <div className="app__container">
+        <Logout />
-          <Logout />
+        <Logo width={190} white />
-          <Logo width={190} white />
+        <Menu isAdmin={isAdmin()} />
-          <Menu isAdmin={isAdmin()} />
+        <Switch>
-          <ToastsContainer toasts={toasts} onToastFinished={onToastFinished} />
+          <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
-          <Switch>
+          <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
-            <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
+          <Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
-            <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
+          <Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
-            <Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
+          <Route name="Job overview" path={'/jobs'} component={Jobs} />
-            <Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
+          <PermissionAwareRoute
-            <Route name="Job overview" path={'/jobs'} component={Jobs} />
+            name="Create new User"
-            <PermissionAwareRoute
+            path="/users/new"
-              name="Create new User"
+            component={<UserMutator />}
-              path="/users/new"
+            currentUser={currentUser}
-              component={<UserMutator />}
+          />
-              currentUser={currentUser}
+          <PermissionAwareRoute
-            />
+            name="Edit a user"
-            <PermissionAwareRoute
+            path="/users/edit/:userId"
-              name="Edit a user"
+            component={<UserMutator />}
-              path="/users/edit/:userId"
+            currentUser={currentUser}
-              component={<UserMutator />}
+          />
-              currentUser={currentUser}
+          <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
-            />
+          <PermissionAwareRoute
-            <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
+            name="General Settings"
-            <PermissionAwareRoute
+            path="/generalSettings"
-              name="General Settings"
+            component={<GeneralSettings />}
-              path="/generalSettings"
+            currentUser={currentUser}
-              component={<GeneralSettings />}
+          />
-              currentUser={currentUser}
-            />
 
-            <Redirect from="/" to={'/jobs'} />
+          <Redirect from="/" to={'/jobs'} />
-          </Switch>
+        </Switch>
-        </div>
       </div>
-    </ToastContext.Provider>
+    </div>
   );
 }
 

ui/src/App.less

@@ -4,11 +4,9 @@
   width:100%;
 
   &__container {
-    width: 100%;
-
     padding: 1rem 1rem;
-    background-color: #595959f5;
+    color: var(--semi-color-text-0);
-    color: #f1f1f1;
+    background-color: #232429;
   }
 }
 
@@ -19,3 +17,27 @@
 .ui.black.label, .ui.black.labels .label {
   background-color: #31303078!important;
 }
+
+a:link {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: none;
+}
+
+a:visited {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: none;
+}
+
+a:hover {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: underline;
+}
+
+a:active {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: underline;
+}

ui/src/Index.jsx

@@ -5,9 +5,11 @@ import { HashRouter } from 'react-router-dom';
 import { createHashHistory } from 'history';
 import { Provider } from 'react-redux';
 import { createRoot } from 'react-dom/client';
+import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
+import { LocaleProvider } from '@douyinfe/semi-ui';
+
 const container = document.getElementById('fredy');
 const root = createRoot(container);
-
 const history = createHashHistory();
 
 import App from './App';
@@ -17,7 +19,9 @@ import './Index.less';
 root.render(
   <Provider store={reduxStore}>
     <HashRouter history={history}>
-      <App />
+      <LocaleProvider locale={en_US}>
+        <App />
+      </LocaleProvider>
     </HashRouter>
   </Provider>
 );

ui/src/Index.less

@@ -2,5 +2,14 @@ body, html {
   margin: 0;
   height: 100%;
   width: 100%;
-  background-color: #595959f5;
+  background-color: #232429;
+}
+
+.semi-table-row-head{
+  background-color: #2b2b2b !important;
+  color: #fff !important;
+}
+
+.semi-table-row-cell {
+  background-color: #333333 !important;
 }

ui/src/components/headline/Headline.jsx

@@ -1,12 +1,11 @@
 import React from 'react';
-import { Header } from 'semantic-ui-react';
+import { Typography } from '@douyinfe/semi-ui';
 
-import './Headline.less';
+export default function Headline({ text, size = 3 } = {}) {
-
+  const { Title } = Typography;
-export default function Headline({ text, size = 'medium', className = '' } = {}) {
   return (
-    <Header className={`headline ${className}`} size={size}>
+    <Title heading={size} style={{ marginBottom: '1rem' }}>
       {text}
-    </Header>
+    </Title>
   );
 }

ui/src/components/headline/Headline.less

@@ -1,3 +0,0 @@
-.headline{
-  color: #f1f1f1 !important;
-}

ui/src/components/logout/Logout.jsx

@@ -1,20 +1,20 @@
 import React from 'react';
-import { Button } from 'semantic-ui-react';
+import { Button } from '@douyinfe/semi-ui';
 import { xhrPost } from '../../services/xhr';
-
+import { IconUser } from '@douyinfe/semi-icons';
 const Logout = function Logout() {
   return (
     <Button
-      content="Logout"
+      icon={<IconUser />}
-      labelPosition="left"
+      type="danger"
-      icon="user"
+      theme="solid"
-      size="mini"
       onClick={async () => {
         await xhrPost('/api/login/logout');
         location.reload();
       }}
-      negative
+    >
-    />
+      Logout
+    </Button>
   );
 };
 

ui/src/components/menu/Menu.jsx

@@ -1,49 +1,54 @@
 import React from 'react';
 import { useHistory } from 'react-router-dom';
-import { Icon, Menu } from 'semantic-ui-react';
+import { Tabs, TabPane } from '@douyinfe/semi-ui';
 
-import './Menu.less';
 import { useLocation } from 'react-router';
+import { IconUser, IconTerminal, IconSetting } from '@douyinfe/semi-icons';
+
+function parsePathName(name) {
+  const split = name.split('/').filter((s) => s.length !== 0);
+  return '/' + split[0];
+}
 
 const TopMenu = function TopMenu({ isAdmin }) {
   const history = useHistory();
   const location = useLocation();
-
-  const isActiveRoute = (name) => location.pathname.indexOf(name) !== -1;
-
   return (
-    <Menu pointing secondary className="topMenu">
+    <Tabs type="line" activeKey={parsePathName(location.pathname)} onTabClick={(key) => history.push(key)}>
-      <Menu.Item
+      <TabPane
-        name="jobs"
+        itemKey="/jobs"
-        active={isActiveRoute('jobs')}
+        tab={
-        className={isActiveRoute('jobs') ? 'topMenu__active' : 'topMenu__item'}
+          <span>
-        onClick={() => history.push('/jobs')}
+            <IconTerminal />
-      >
+            Jobs
-        <Icon name="search" /> Job Configuration
+          </span>
-      </Menu.Item>
+        }
+      />
 
       {isAdmin && (
-        <Menu.Item
+        <TabPane
-          name="user"
+          itemKey="/users"
-          active={isActiveRoute('users')}
+          tab={
-          className={isActiveRoute('users') ? 'topMenu__active' : 'topMenu__item'}
+            <span>
-          onClick={() => history.push('/users')}
+              <IconUser />
-        >
+              User
-          <Icon name="user" /> User configuration
+            </span>
-        </Menu.Item>
+          }
+        />
       )}
 
       {isAdmin && (
-        <Menu.Item
+        <TabPane
-          name="general"
+          itemKey="/generalSettings"
-          active={isActiveRoute('general')}
+          tab={
-          className={isActiveRoute('general') ? 'topMenu__active' : 'topMenu__item'}
+            <span>
-          onClick={() => history.push('/generalSettings')}
+              <IconSetting />
-        >
+              General
-          <Icon name="cog" /> General Settings
+            </span>
-        </Menu.Item>
+          }
+        />
       )}
-    </Menu>
+    </Tabs>
   );
 };
 

ui/src/components/menu/Menu.less

@@ -1,15 +0,0 @@
-.topMenu {
-  border-bottom: 1px solid #b7b7b7f2 !important;
-
-  &__active {
-    border-bottom: 1px solid #06dcfff2 !important;
-    font-weight: 550 !important;
-    color: #3ed7ff !important;
-    margin: 0 0 -1px !important;
-  }
-
-  &__item {
-    color: #fffffff2 !important;
-    border-color: transparent !important;
-  }
-}

ui/src/components/permission/InsufficientPermission.jsx

@@ -1,5 +1,4 @@
 import React from 'react';
-import { Header } from 'semantic-ui-react';
 import insufficientPermission from '../../assets/insufficient_permission.png';
 
 export default function InsufficientPermission() {
@@ -7,9 +6,7 @@ export default function InsufficientPermission() {
     <div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center', flexDirection: 'column' }}>
       <img src={insufficientPermission} height={250} />
       <br />
-      <Header as="h4" inverted>
+      <h4>Insufficient permission :(</h4>
-        Insufficient permission :(
-      </Header>
     </div>
   );
 }

ui/src/components/segment/SegmentPart.jsx

@@ -1,27 +1,18 @@
 import React from 'react';
-import { Header, Icon, Popup, Segment } from 'semantic-ui-react';
+import { Card } from '@douyinfe/semi-ui';
 
 import './SegmentParts.less';
 
-export const SegmentPart = ({ name, icon = null, children, helpText }) => (
+export const SegmentPart = ({ name, Icon = null, children, helpText }) => {
-  <Segment inverted>
+  const { Meta } = Card;
-    <Header as="h5" inverted sub>
-      {icon && <Icon name={icon} inverted size="mini" />}
-      <Header.Content>{name}</Header.Content>
-    </Header>
 
-    <Popup
+  return (
-      content={helpText}
+    <Card
-      trigger={
+      title={
-        <span className="generalSettings__help">
+        <Meta title={name} description={helpText} avatar={Icon == null ? null : <Icon size="extra-extra-small" />} />
-          {' '}
-          <Icon name="help circle" inverted />
-          What is this?
-        </span>
       }
-    />
+    >
-    <Segment inverted className="segmentParts">
       {children}
-    </Segment>
+    </Card>
-  </Segment>
+  );
-);
+};

ui/src/components/table/JobTable.jsx

@@ -1,66 +1,79 @@
-import React, { Fragment } from 'react';
+import React from 'react';
 
-import { Table, Button } from 'semantic-ui-react';
+import { Button, Empty, Table, Switch } from '@douyinfe/semi-ui';
-import Switch from 'react-switch';
+import { IconDelete, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
-
+import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
-const emptyTable = () => {
+const empty = (
-  return (
+  <Empty
-    <Table.Row>
+    image={<IllustrationNoResult />}
-      <Table.Cell collapsing colSpan={6} style={{ textAlign: 'center' }}>
+    darkModeImage={<IllustrationNoResultDark />}
-        No Data
+    description={'No jobs available'}
-      </Table.Cell>
+  />
-    </Table.Row>
+);
-  );
-};
-
-const content = (jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight) => {
-  return (
-    <Fragment>
-      {Object.keys(jobs).map((jobKey) => {
-        const job = jobs[jobKey];
-
-        return (
-          <Table.Row key={jobKey}>
-            <Table.Cell collapsing>
-              <Switch onChange={(checked) => onJobStatusChanged(job.id, checked)} checked={job.enabled} />
-            </Table.Cell>
-            <Table.Cell>{job.name}</Table.Cell>
-            <Table.Cell>{job.numberOfFoundListings || 0}</Table.Cell>
-            <Table.Cell>{job.provider.length || 0}</Table.Cell>
-            <Table.Cell>{job.notificationAdapter.length || 0}</Table.Cell>
-            <Table.Cell>
-              <div style={{ float: 'right' }}>
-                <Button circular color="teal" icon="chart line" onClick={() => onJobInsight(job.id)} />
-                <Button circular color="blue" icon="edit" onClick={() => onJobEdit(job.id)} />
-                <Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
-              </div>
-            </Table.Cell>
-          </Table.Row>
-        );
-      })}
-    </Fragment>
-  );
-};
 
 export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight } = {}) {
   return (
-    <Table singleLine inverted>
+    <Table
-      <Table.Header>
+      pagination={false}
-        <Table.Row>
+      empty={empty}
-          <Table.HeaderCell />
+      columns={[
-          <Table.HeaderCell>Job Name</Table.HeaderCell>
+        {
-          <Table.HeaderCell>Number of findings</Table.HeaderCell>
+          title: '',
-          <Table.HeaderCell>Active provider</Table.HeaderCell>
+          dataIndex: '',
-          <Table.HeaderCell>Active notification adapter</Table.HeaderCell>
+          render: (job) => {
-          <Table.HeaderCell></Table.HeaderCell>
+            return <Switch onChange={(checked) => onJobStatusChanged(job.id, checked)} checked={job.enabled} />;
-        </Table.Row>
+          },
-      </Table.Header>
+        },
-
+        {
-      <Table.Body>
+          title: 'Job Name',
-        {Object.keys(jobs).length === 0
+          dataIndex: 'name',
-          ? emptyTable()
+        },
-          : content(jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight)}
+        {
-      </Table.Body>
+          title: 'Number of findings',
-    </Table>
+          dataIndex: 'numberOfFoundListings',
+          render: (value) => {
+            return value || 0;
+          },
+        },
+        {
+          title: 'Active provider',
+          dataIndex: 'provider',
+          render: (value) => {
+            return value.length || 0;
+          },
+        },
+        {
+          title: 'Active notification adapter',
+          dataIndex: 'notificationAdapter',
+          render: (value) => {
+            return value.length || 0;
+          },
+        },
+        {
+          title: '',
+          dataIndex: 'tools',
+          render: (_, job) => {
+            return (
+              <div style={{ float: 'right' }}>
+                <Button
+                  type="primary"
+                  icon={<IconHistogram />}
+                  onClick={() => onJobInsight(job.id)}
+                  style={{ marginRight: '1rem' }}
+                />
+                <Button
+                  type="secondary"
+                  icon={<IconEdit />}
+                  onClick={() => onJobEdit(job.id)}
+                  style={{ marginRight: '1rem' }}
+                />
+                <Button type="danger" icon={<IconDelete />} onClick={() => onJobRemoval(job.id)} />
+              </div>
+            );
+          },
+        },
+      ]}
+      dataSource={jobs}
+    />
   );
 }

ui/src/components/table/NotificationAdapterTable.jsx

@@ -1,50 +1,38 @@
-import React, { Fragment } from 'react';
+import React from 'react';
 
-import { Table, Button } from 'semantic-ui-react';
+import { Empty, Table, Button } from '@douyinfe/semi-ui';
-
+import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
-const emptyTable = () => {
-  return (
-    <Table.Row>
-      <Table.Cell collapsing colSpan="3" style={{ textAlign: 'center' }}>
-        No Data
-      </Table.Cell>
-    </Table.Row>
-  );
-};
-
-const content = (adapterData, onRemove, onEdit) => {
-  return (
-    <Fragment>
-      {adapterData.map((data) => {
-        return (
-          <Table.Row key={data.id}>
-            <Table.Cell>{data.name}</Table.Cell>
-            <Table.Cell>
-              <div style={{ float: 'right' }}>
-                <Button circular color="blue" icon="edit" onClick={() => onEdit(data.id)} />
-                <Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
-              </div>
-            </Table.Cell>
-          </Table.Row>
-        );
-      })}
-    </Fragment>
-  );
-};
 
 export default function NotificationAdapterTable({ notificationAdapter = [], onRemove, onEdit } = {}) {
   return (
-    <Table singleLine inverted>
+    <Table
-      <Table.Header>
+      pagination={false}
-        <Table.Row>
+      empty={<Empty description="No Data" />}
-          <Table.HeaderCell>Notification Adapter Name</Table.HeaderCell>
+      columns={[
-          <Table.HeaderCell></Table.HeaderCell>
+        {
-        </Table.Row>
+          title: 'Notification Adapter Name',
-      </Table.Header>
+          dataIndex: 'name',
+        },
 
-      <Table.Body>
+        {
-        {notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove, onEdit)}
+          title: '',
-      </Table.Body>
+          dataIndex: 'tools',
-    </Table>
+          render: (_, record) => {
+            return (
+              <div style={{ float: 'right' }}>
+                <Button
+                  type="secondary"
+                  icon={<IconEdit />}
+                  onClick={() => onEdit(record.id)}
+                  style={{ marginRight: '1rem' }}
+                />
+                <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
+              </div>
+            );
+          },
+        },
+      ]}
+      dataSource={notificationAdapter}
+    />
   );
 }

ui/src/components/table/ProviderTable.jsx

@@ -1,53 +1,42 @@
-import React, { Fragment } from 'react';
+import React from 'react';
 
-import { Table, Button } from 'semantic-ui-react';
+import { Empty, Table, Button } from '@douyinfe/semi-ui';
-
+import { IconDelete } from '@douyinfe/semi-icons';
-const emptyTable = () => {
-  return (
-    <Table.Row>
-      <Table.Cell collapsing colSpan="3" style={{ textAlign: 'center' }}>
-        No Data
-      </Table.Cell>
-    </Table.Row>
-  );
-};
-
-const content = (providerData, onRemove) => {
-  return (
-    <Fragment>
-      {providerData.map((data) => {
-        return (
-          <Table.Row key={data.id}>
-            <Table.Cell>{data.name}</Table.Cell>
-            <Table.Cell>
-              <a href={data.url} target="_blank" rel="noopener noreferrer">
-                Visit site
-              </a>
-            </Table.Cell>
-            <Table.Cell>
-              <div style={{ float: 'right' }}>
-                <Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
-              </div>
-            </Table.Cell>
-          </Table.Row>
-        );
-      })}
-    </Fragment>
-  );
-};
 
 export default function ProviderTable({ providerData = [], onRemove } = {}) {
   return (
-    <Table singleLine inverted>
+    <Table
-      <Table.Header>
+      pagination={false}
-        <Table.Row>
+      empty={<Empty description="No Provider available" />}
-          <Table.HeaderCell>Provider Name</Table.HeaderCell>
+      columns={[
-          <Table.HeaderCell>Url</Table.HeaderCell>
+        {
-          <Table.HeaderCell></Table.HeaderCell>
+          title: 'Provider Name',
-        </Table.Row>
+          dataIndex: 'name',
-      </Table.Header>
+        },
-
+        {
-      <Table.Body>{providerData.length === 0 ? emptyTable() : content(providerData, onRemove)}</Table.Body>
+          title: 'Provider Url',
-    </Table>
+          dataIndex: 'url',
+          render: (_, data) => {
+            return (
+              <a href={data.url} target="_blank" rel="noopener noreferrer">
+                Visit site
+              </a>
+            );
+          },
+        },
+        {
+          title: '',
+          dataIndex: 'tools',
+          render: (_, record) => {
+            return (
+              <div style={{ float: 'right' }}>
+                <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
+              </div>
+            );
+          },
+        },
+      ]}
+      dataSource={providerData}
+    />
   );
 }

ui/src/components/table/UserTable.jsx

@@ -1,49 +1,58 @@
 import React from 'react';
 
-import { Table, Button } from 'semantic-ui-react';
+import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
 import { format } from '../../services/time/timeService';
+import { Table, Button, Empty } from '@douyinfe/semi-ui';
+import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
 
-const emptyTable = () => {
+const empty = (
-  return (
+  <Empty
-    <Table.Row>
+    image={<IllustrationNoResult />}
-      <Table.Cell collapsing colSpan={4} style={{ textAlign: 'center' }}>
+    darkModeImage={<IllustrationNoResultDark />}
-        No Data
+    description={'No user available'}
-      </Table.Cell>
+  />
-    </Table.Row>
+);
-  );
-};
-
-const content = (user, onUserRemoval, onUserEdit) => {
-  return user.map((user) => {
-    return (
-      <Table.Row key={user.id}>
-        <Table.Cell>{user.username}</Table.Cell>
-        <Table.Cell>{user.lastLogin == null ? '---' : format(user.lastLogin)}</Table.Cell>
-        <Table.Cell>{user.numberOfJobs}</Table.Cell>
-        <Table.Cell>
-          <div style={{ float: 'right' }}>
-            <Button circular color="red" icon="trash" onClick={() => onUserRemoval(user.id)} />
-            <Button circular color="blue" icon="edit" onClick={() => onUserEdit(user.id)} />
-          </div>
-        </Table.Cell>
-      </Table.Row>
-    );
-  });
-};
 
 export default function UserTable({ user = [], onUserRemoval, onUserEdit } = {}) {
   return (
-    <Table inverted>
+    <Table
-      <Table.Header>
+      pagination={false}
-        <Table.Row>
+      empty={empty}
-          <Table.HeaderCell>Username</Table.HeaderCell>
+      columns={[
-          <Table.HeaderCell>Last login</Table.HeaderCell>
+        {
-          <Table.HeaderCell>Number of jobs</Table.HeaderCell>
+          title: 'Username',
-          <Table.HeaderCell></Table.HeaderCell>
+          dataIndex: 'username',
-        </Table.Row>
+        },
-      </Table.Header>
+        {
-
+          title: 'Last login',
-      <Table.Body>{user.length === 0 ? emptyTable() : content(user, onUserRemoval, onUserEdit)}</Table.Body>
+          dataIndex: 'lastLogin',
-    </Table>
+          render: (value) => {
+            return format(value);
+          },
+        },
+        {
+          title: 'Number of jobs',
+          dataIndex: 'numberOfJobs',
+        },
+        {
+          title: '',
+          dataIndex: 'tools',
+          render: (value, user) => {
+            return (
+              <div style={{ float: 'right' }}>
+                <Button
+                  type="danger"
+                  icon={<IconDelete />}
+                  onClick={() => onUserRemoval(user.id)}
+                  style={{ marginRight: '1rem' }}
+                />
+                <Button type="primary" icon={<IconEdit />} onClick={() => onUserEdit(user.id)} />
+              </div>
+            );
+          },
+        },
+      ]}
+      dataSource={user}
+    />
   );
 }

ui/src/components/toasts/Toast.jsx

@@ -1,27 +0,0 @@
-import React from 'react';
-
-import './Toasts.css';
-
-export default function Toast({ id, delay = 5500, message, onHide, backgroundColor, color, title }) {
-  const [className, setClassname] = React.useState('toast-container show-toast');
-
-  React.useEffect(() => {
-    let hideTimeout = null;
-    const timeout = setTimeout(() => {
-      setClassname('toast-container hide-toast');
-      hideTimeout = setTimeout(() => {
-        onHide && onHide(id);
-      }, 500);
-    }, delay);
-    return () => {
-      clearTimeout(timeout);
-      clearTimeout(hideTimeout);
-    };
-  }, [id, delay, onHide]);
-  return (
-    <div className={className} style={{ backgroundColor, color }}>
-      <h5>{title}</h5>
-      {message}
-    </div>
-  );
-}

ui/src/components/toasts/ToastContainer.jsx

@@ -1,12 +0,0 @@
-import Toast from './Toast';
-import React from 'react';
-
-export default function ToastsContainer({ toasts, onToastFinished }) {
-  return (
-    <div className="toasts-container">
-      {toasts.map((toast, index) => (
-        <Toast key={index} {...toast} onHide={onToastFinished} />
-      ))}
-    </div>
-  );
-}

ui/src/components/toasts/ToastContext.jsx

@@ -1,7 +0,0 @@
-import { createContext } from 'react';
-
-const CheckoutDrawerContext = createContext({
-  showToast: () => {},
-});
-
-export default CheckoutDrawerContext;

ui/src/components/toasts/Toasts.css

@@ -1,63 +0,0 @@
-.toasts-container {
-    position: fixed;
-    z-index: 65535;
-    right: 0;
-    max-width: 250px;
-    display: flex;
-    flex-direction: column-reverse;
-}
-
-.toasts-container > .toast-container {
-    margin-bottom: 10px;
-}
-
-.toasts-container:last-child {
-    margin-bottom: 0;
-}
-
-.toast-container {
-    visibility: hidden;
-    position: relative;
-    z-index: 65535;
-    right: -1000px;
-
-    background-color: skyblue;
-    border-radius: 10px;
-    padding: 10px;
-
-    min-width: 10rem;
-    min-height: 3rem;
-}
-
-.toast-container.show-toast {
-    visibility: visible;
-    right: 24px;
-    animation: slidein 0.5s;
-}
-
-.toast-container.hide-toast {
-    visibility: visible;
-    animation: slideout 0.5s;
-}
-
-@keyframes slidein {
-    from {
-        right: -1000px;
-        opacity: 0;
-    }
-    to {
-        right: 24px;
-        opacity: 1;
-    }
-}
-
-@keyframes slideout {
-    from {
-        right: 24px;
-        opacity: 1;
-    }
-    to {
-        right: -1000px;
-        opacity: 0;
-    }
-}

ui/src/components/toasts/useToast.jsx

@@ -1,23 +0,0 @@
-import React from 'react';
-
-export default function useToast() {
-  const [toasts, setToasts] = React.useState([]);
-
-  const showToast = ({ message, delay, color, backgroundColor, title }) => {
-    const toast = {
-      id: toasts.length,
-      message,
-      delay,
-      backgroundColor,
-      color,
-      title,
-    };
-    setToasts([...toasts, toast].reverse());
-  };
-
-  const onToastFinished = (id) => {
-    setToasts(toasts.filter((toast) => toast.id !== id));
-  };
-
-  return [showToast, onToastFinished, toasts];
-}

ui/src/services/rematch/models/generalSettings.js

@@ -1,5 +1,4 @@
 import { xhrGet } from '../../xhr';
-
 export const generalSettings = {
   state: {
     settings: {},

ui/src/services/rematch/models/jobs.js

@@ -1,5 +1,4 @@
 import { xhrGet } from '../../xhr';
-
 export const jobs = {
   state: {
     jobs: [],

ui/src/services/rematch/models/user.js

@@ -1,5 +1,4 @@
 import { xhrGet } from '../../xhr';
-
 export const user = {
   state: {
     users: [],

ui/src/services/rematch/store.js

@@ -6,14 +6,11 @@ import { createLogger } from 'redux-logger';
 import { jobs } from './models/jobs';
 import { user } from './models/user';
 import { init } from '@rematch/core';
-
 const middleware = [];
-
 if (process.env.NODE_ENV === 'development') {
   // eslint-disable-line no-redeclare
   middleware.push(createLogger({ duration: false, collapsed: (getState, action, logEntry) => !logEntry.error }));
 }
-
 const store = init({
   name: 'fredy',
   models: {
@@ -28,5 +25,4 @@ const store = init({
     middlewares: middleware,
   },
 });
-
 export const reduxStore = store;

ui/src/services/time/timeService.js

@@ -8,5 +8,4 @@ export function format(ts) {
     second: 'numeric',
   }).format(ts);
 }
-
 export const roundToNext5Minute = (ts) => Math.ceil(ts / (1000 * 60 * 5)) * (1000 * 60 * 5);

ui/src/services/transformer/notificationAdapterTransformer.js

@@ -1,10 +1,8 @@
 export function transform({ id, name, fields }) {
   const fieldValues = {};
-
   Object.keys(fields).map((key) => {
     fieldValues[key] = fields[key].value;
   });
-
   return {
     id,
     name,

ui/src/services/xhr.js

@@ -9,7 +9,6 @@
 export function xhrPost(url, data, contentType = 'application/json; charset=utf-8', isJson = true) {
   return executePostOrPutCall(url, contentType, data, isJson, true);
 }
-
 /**
  * put request to backend.
  *
@@ -21,7 +20,6 @@ export function xhrPost(url, data, contentType = 'application/json; charset=utf-
 export function xhrPut(url, data, contentType = 'application/json; charset=utf-8', isJson = true) {
   return executePostOrPutCall(url, contentType, data, isJson, false);
 }
-
 function executePostOrPutCall(url, contentType, data, isJson, isPost) {
   return new Promise((resolve, reject) => {
     fetch(url, {
@@ -41,7 +39,6 @@ function executePostOrPutCall(url, contentType, data, isJson, isPost) {
       });
   });
 }
-
 /**
  * get request to backend
  * returns a Promise with
@@ -75,7 +72,6 @@ export function xhrGet(url, contentType = 'application/json; charset=utf-8', isJ
       });
   });
 }
-
 /**
  * delete request to backend
  * returns a Promise with
@@ -114,7 +110,6 @@ export function xhrDelete(url, data, contentType = 'application/json; charset=ut
       });
   });
 }
-
 function parseJSON(response) {
   return new Promise((resolve, reject) =>
     response
@@ -122,7 +117,6 @@ function parseJSON(response) {
       .then((text) => {
         //some responses doesn't contain a body. .json() would throw errors here...
         const json = text != null && text.length > 0 ? JSON.parse(text) : {};
-
         if (response.ok) {
           resolve({
             status: response.status,

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -2,14 +2,32 @@ import React from 'react';
 
 import { useDispatch, useSelector } from 'react-redux';
 
-import { Button, Form, Icon, Message, Segment, Radio } from 'semantic-ui-react';
+import { Divider, Input, Radio, TimePicker, Button, RadioGroup } from '@douyinfe/semi-ui';
-import ToastContext from '../../components/toasts/ToastContext';
+import { InputNumber } from '@douyinfe/semi-ui';
 import Headline from '../../components/headline/Headline';
 import { xhrPost } from '../../services/xhr';
 import { SegmentPart } from '../../components/segment/SegmentPart';
+import { Banner, Toast } from '@douyinfe/semi-ui';
+import { IconSave, IconCalendar, IconKey, IconRefresh, IconSignal } from '@douyinfe/semi-icons';
 import './GeneralSettings.less';
 
-const GeneralSettings = function Users() {
+function formatFromTimestamp(ts) {
+  const date = new Date(ts);
+  return `${date.getHours()}:${date.getMinutes() > 9 ? date.getMinutes() : '0' + date.getMinutes()}`;
+}
+
+function formatFromTBackend(time) {
+  if (time == null || time.length === 0) {
+    return null;
+  }
+  const date = new Date();
+  const split = time.split(':');
+  date.setHours(split[0]);
+  date.setMinutes(split[1]);
+  return date.getTime();
+}
+
+const GeneralSettings = function GeneralSettings() {
   const dispatch = useDispatch();
   const [loading, setLoading] = React.useState(true);
 
@@ -21,13 +39,12 @@ const GeneralSettings = function Users() {
   const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
   const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
   const [workingHourTo, setWorkingHourTo] = React.useState(null);
-  const ctx = React.useContext(ToastContext);
-
   React.useEffect(() => {
     async function init() {
       await dispatch.generalSettings.getGeneralSettings();
       setLoading(false);
     }
+
     init();
   }, []);
 
@@ -40,19 +57,18 @@ const GeneralSettings = function Users() {
       setWorkingHourTo(settings?.workingHours?.to);
       setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
     }
+
     init();
   }, [settings]);
 
   const nullOrEmpty = (val) => val == null || val.length === 0;
 
   const throwMessage = (message, type) => {
-    ctx.showToast({
+    if (type === 'error') {
-      title: type === 'error' ? 'Error' : 'Success',
+      Toast.error(message);
-      message: message,
+    } else {
-      delay: 5000,
+      Toast.success(message);
-      backgroundColor: type === 'error' ? '#db2828' : '#87eb8f',
+    }
-      color: type === 'error' ? '#fff' : '#000',
-    });
   };
 
   const onStore = async () => {
@@ -97,139 +113,130 @@ const GeneralSettings = function Users() {
       {!loading && (
         <React.Fragment>
           <Headline text="General Settings" />
-          <Message className="generalSettings__message">
+          <Banner
-            <h5>
+            fullMode={false}
-              <Icon name="info circle" />
+            type="info"
-              Info
+            closeIcon={null}
-            </h5>
+            title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Info</div>}
-            <p>If you change any settings, you must restart Fredy afterwards.</p>
+            style={{ marginBottom: '1rem' }}
-          </Message>
+            description="If you change any settings, you must restart Fredy afterwards."
-          <Form>
+          />
+          <div>
             <SegmentPart
               name="Interval"
               helpText="Interval in minutes for running queries against the configured services."
-              icon="refresh"
+              Icon={IconRefresh}
             >
-              <Form.Input
+              <InputNumber
-                type="number"
+                min={0}
-                min="0"
+                max={1440}
-                max="1440"
                 placeholder="Interval in minutes"
-                inverted
+                value={interval}
-                size="mini"
+                formatter={(value) => `${value}`.replace(/\D/g, '')}
-                width={6}
+                onChange={(value) => setInterval(value)}
-                defaultValue={interval}
+                suffix={'minutes'}
-                onChange={(e) => setInterval(e.target.value)}
               />
             </SegmentPart>
-
+            <Divider margin="1rem" />
-            <SegmentPart name="Port" helpText="Port on which Fredy is running." icon="connectdevelop">
+            <SegmentPart name="Port" helpText="Port on which Fredy is running." Icon={IconSignal}>
-              <Form.Input
+              <InputNumber
-                type="number"
+                min={0}
-                min="0"
+                max={99999}
-                max="99999"
                 placeholder="Port"
-                inverted
+                value={port}
-                size="mini"
+                formatter={(value) => `${value}`.replace(/\D/g, '')}
-                width={6}
+                onChange={(value) => setPort(value)}
-                defaultValue={port}
-                onChange={(e) => setPort(e.target.value)}
               />
             </SegmentPart>
-
+            <Divider margin="1rem" />
             <SegmentPart
               name="ScrapingAnt Api Key"
               helpText="The api key for ScrapingAnt is used to be able to scrape Immoscout."
-              icon="key"
+              Icon={IconKey}
             >
-              <Form.Input
+              <Input
                 type="text"
                 placeholder="ScrapingAnt Api Key"
-                inverted
+                value={scrapingAntApiKey}
-                size="mini"
+                onChange={(val) => setScrapingAntApiKey(val)}
-                width={6}
-                defaultValue={scrapingAntApiKey}
-                onChange={(e) => setScrapingAntApiKey(e.target.value)}
               />
             </SegmentPart>
-
+            <Divider margin="1rem" />
             <SegmentPart
               name="ScrapingAnt proxy settings"
               helpText="Scraping ant provides different proxies."
-              icon="key"
+              Icon={IconKey}
             >
-              <Message info>
+              <Banner
-                ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies.{' '}
+                fullMode={false}
-                <br />
+                type="info"
-                <h4>Datacenter-Proxy</h4>
+                closeIcon={null}
-                Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and more
+                title={
-                likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
+                  <div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>
-                <h4>Residential-Proxy</h4>
+                    ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies
-                High-quality proxy server located in one of the real people houses across the world. Datacenter proxies
+                  </div>
-                are faster and more likely to success, but they are more expensive. A call with a datacenter proxy cost
+                }
-                250 credits.
+                style={{ marginBottom: '1rem' }}
-                <br />
+                description={
-                <br />
+                  <div>
-                <b>
+                    <h4>Datacenter-Proxy</h4>
-                  On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only successful
+                    Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and
-                  calls will be charged.
+                    more likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
-                </b>
+                    <h4>Residential-Proxy</h4>
-              </Message>
+                    High-quality proxy server located in one of the real people houses across the world. Datacenter
-              <Form.Field>
+                    proxies are faster and more likely to success, but they are more expensive. A call with a datacenter
-                <Radio
+                    proxy cost 250 credits.
-                  label="Datacenter proxy"
+                    <br />
-                  name="scrapingAntProxy"
+                    <br />
-                  value="datacenter"
+                    <b>
-                  checked={scrapingAntProxy === 'datacenter'}
+                      On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only
-                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                      successful calls will be charged.
-                />
+                    </b>
-              </Form.Field>
+                  </div>
-              <Form.Field>
+                }
-                <Radio
+              />
-                  label="Residential proxy"
-                  name="scrapingAntProxy"
-                  value="residential"
-                  checked={scrapingAntProxy === 'residential'}
-                  onChange={(e, { value }) => setScrapingAntProxy(value)}
-                />
-              </Form.Field>
-            </SegmentPart>
 
+              <RadioGroup value={scrapingAntProxy} onChange={(e) => setScrapingAntProxy(e.target.value)}>
+                <Radio name="datacenter" value="datacenter" checked={scrapingAntProxy === 'datacenter'}>
+                  Datacenter proxy
+                </Radio>
+                <Radio name="residential" value="residential" checked={scrapingAntProxy === 'residential'}>
+                  Residential proxy
+                </Radio>
+              </RadioGroup>
+            </SegmentPart>
+            <Divider margin="1rem" />
             <SegmentPart
               name="Working hours"
               helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
-              icon="calendar outline"
+              Icon={IconCalendar}
             >
               <div className="generalSettings__timePickerContainer">
-                <Form.Input
+                <TimePicker
-                  className="generalSettings__time"
+                  format={'HH:mm'}
-                  type="time"
+                  insetLabel="From"
-                  placeholder="Working hours from"
+                  value={formatFromTBackend(workingHourFrom)}
-                  inverted
+                  placeholder=""
-                  size="mini"
+                  onChange={(val) => {
-                  width={2}
+                    setWorkingHourFrom(val == null ? null : formatFromTimestamp(val));
-                  defaultValue={workingHourFrom}
+                  }}
-                  onChange={(e) => setWorkingHourFrom(e.target.value)}
                 />
-                <div className="generalSettings__until">until</div>
+                <TimePicker
-                <Form.Input
+                  format={'HH:mm'}
-                  type="time"
+                  insetLabel="Until"
-                  placeholder="Working hours to"
+                  value={formatFromTBackend(workingHourTo)}
-                  inverted
+                  placeholder=""
-                  size="mini"
+                  onChange={(val) => {
-                  width={2}
+                    setWorkingHourTo(val == null ? null : formatFromTimestamp(val));
-                  defaultValue={workingHourTo}
+                  }}
-                  onChange={(e) => setWorkingHourTo(e.target.value)}
                 />
               </div>
             </SegmentPart>
-
+            <Divider margin="1rem" />
-            <Segment inverted floated="right">
+            <Button type="primary" theme="solid" onClick={onStore} icon={<IconSave />}>
-              <Button color="teal" onClick={onStore}>
+              Save
-                Save
+            </Button>
-              </Button>
+          </div>
-            </Segment>
-          </Form>
         </React.Fragment>
       )}
     </div>

ui/src/views/generalSettings/GeneralSettings.less

@@ -2,11 +2,7 @@
   &__timePickerContainer {
     display: flex;
     align-items: baseline;
-  }
+    gap: 1rem;
-
-  &__until {
-    margin-left: 1rem;
-    margin-right: 1rem;
   }
 
   &__help{
@@ -14,8 +10,4 @@
     margin-left: 1rem;
   }
 
-  &__message{
-    background: #8fe8ff!important;
-  }
-
 }

ui/src/views/jobs/Jobs.jsx

@@ -1,13 +1,12 @@
 import React from 'react';
 
-import ToastContext from '../../components/toasts/ToastContext';
 import JobTable from '../../components/table/JobTable';
 import { useSelector, useDispatch } from 'react-redux';
 import { xhrDelete, xhrPut } from '../../services/xhr';
-import { Button, Icon } from 'semantic-ui-react';
 import { useHistory } from 'react-router-dom';
 import ProcessingTimes from './ProcessingTimes';
-
+import { Button, Toast } from '@douyinfe/semi-ui';
+import { IconPlusCircle } from '@douyinfe/semi-icons';
 import './Jobs.less';
 
 export default function Jobs() {
@@ -15,49 +14,24 @@ export default function Jobs() {
   const processingTimes = useSelector((state) => state.jobs.processingTimes);
   const history = useHistory();
   const dispatch = useDispatch();
-  const ctx = React.useContext(ToastContext);
 
   const onJobRemoval = async (jobId) => {
     try {
       await xhrDelete('/api/jobs', { jobId });
-      ctx.showToast({
+      Toast.success('Job successfully remove');
-        title: 'Success',
-        message: 'Job successfully remove',
-        delay: 5000,
-        backgroundColor: '#87eb8f',
-        color: '#000',
-      });
       await dispatch.jobs.getJobs();
     } catch (error) {
-      ctx.showToast({
+      Toast.error(error);
-        title: 'Error',
-        message: error,
-        delay: 35000,
-        backgroundColor: '#db2828',
-        color: '#fff',
-      });
     }
   };
 
   const onJobStatusChanged = async (jobId, status) => {
     try {
       await xhrPut(`/api/jobs/${jobId}/status`, { status });
-      ctx.showToast({
+      Toast.success('Job status successfully changed');
-        title: 'Success',
-        message: 'Job status successfully changed',
-        delay: 5000,
-        backgroundColor: '#87eb8f',
-        color: '#000',
-      });
       await dispatch.jobs.getJobs();
     } catch (error) {
-      ctx.showToast({
+      Toast.error(error);
-        title: 'Error',
-        message: error,
-        delay: 35000,
-        backgroundColor: '#db2828',
-        color: '#fff',
-      });
     }
   };
 
@@ -65,8 +39,12 @@ export default function Jobs() {
     <div>
       <div>
         {processingTimes != null && <ProcessingTimes processingTimes={processingTimes} />}
-        <Button primary className="jobs__newButton" onClick={() => history.push('/jobs/new')}>
+        <Button
-          <Icon name="plus" />
+          type="primary"
+          icon={<IconPlusCircle />}
+          className="jobs__newButton"
+          onClick={() => history.push('/jobs/new')}
+        >
           New Job
         </Button>
       </div>