General settings (#28)

adding ui for general settings | adding 'working hours' as new feature

.eslintrc.js

@@ -6,7 +6,7 @@ module.exports = {
     browser: true,
   },
   parser: 'babel-eslint',
-  extends: ['eslint:recommended', 'prettier', 'prettier/react'],
+  extends: ['eslint:recommended', 'prettier'],
   plugins: ['react'],
   globals: {
     Promise: false,

CHANGELOG.md

@@ -1,3 +1,16 @@
+###### [V5.1.0]
+- Upgrading dependencies
+- NodeJS 12.13 is now the minimum supported version
+- Adding general settings as new configuration page to ui
+- Adding new feature working hours 
+
+###### [V5.0.0]
+- Upgrading dependencies
+- NodeJS 12 is now the minimum supported version
+
+###### [V4.0.0]
+Bringing back Immoscout :tada:
+
 ###### [V3.0.0]
 This is basically a re-write, your old config file will not be compatible anymore. Please re-created your search jobs
 on the new ui and use the values from your previous config file if needed.

README.md

@@ -17,8 +17,28 @@ yarn run start
 ```
 _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` for username and password. (You should change the password asap when you plan to run Fredy on your server.)
 
+<p align="center">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
+&nbsp; &nbsp; &nbsp; &nbsp;
+  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
+&nbsp; &nbsp; &nbsp; &nbsp;  
+  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
+</p>
+<p align="center">
+
+</p>
+
+## Immoscout
+I have added **EXPERIMENTAL** support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successful validated) re-send the cookies each time.
+
+To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the ApiKey in the "General Settings" tab (visible when logged in as administrator).
+The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time it works pretty good :)
+
+If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefor they've decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (No I don't get any money for recommending good services...)
+
+
 ## Understanding the fundamentals
-There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
+There are 3 important parts in Fredy, that you need to understand leveraging the full power of _Fredy_.
 
 #### Adapter
 _Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few. Those services are called adapter within _Fredy_. When creating a new job, you can choose 1 or many adapter.    
@@ -57,8 +77,15 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-## Why is Immoscout missing
-Immoscout decided to add "robot protection" to their service. Meaning if Fredy tries to check for listings, it will be recognized as a bot. I haven't found a way around it (yet) ;)
+## Immoscout 
+I have added EXPERIMENTAL support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way
+around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/).
+
+To be able to use Immoscout, you need to create an account and copy the apiKey into the config file under /conf/config.json.
+The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time
+it works pretty good :)
+
+If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... (No I don't get any money for recommending good service)
 
 #### Contribution guidelines  
 

conf/config.json

@@ -1,4 +1 @@
-{
-  "interval": 30,
-  "port": 9998
-}
+{"interval":"30","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}

index.js

@@ -13,6 +13,8 @@ const jobStorage = require('./lib/services/storage/jobStorage');
 const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
 const FredyRuntime = require('./lib/FredyRuntime');
 
+const { duringWorkingHoursOrNotSet } = require('./lib/utils');
+
 //starting the api service
 require('./lib/api/api');
 
@@ -24,30 +26,39 @@ console.log(`Started Fredy successfully. Ui can be accessed via http://localhost
 /* eslint-enable no-console */
 setInterval(
   (function exec() {
-    jobStorage
-      .getJobs()
-      .filter((job) => job.enabled)
-      .forEach((job) => {
-        const providerIds = job.provider.map((provider) => provider.id);
+    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
 
-        provider
-          .filter((provider) => provider.endsWith('.js'))
-          .map((pro) => require(`${path}/${pro}`))
-          .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
-          .forEach(async (pro) => {
-            const providerId = pro.metaInformation.id;
-            if (providerId == null || providerId.length === 0) {
-              throw new Error('Provider id must not be empty. => ' + pro);
-            }
-            const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
-            if (providerConfig == null) {
-              throw new Error(`Provider Config for provider with id ${providerId} not found.`);
-            }
-            pro.init(providerConfig, job.blacklist);
-            await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute();
-            setLastJobExecution(job.id);
-          });
-      });
+    if (isDuringWorkingHoursOrNotSet) {
+      config.lastRun = Date.now();
+      jobStorage
+        .getJobs()
+        .filter((job) => job.enabled)
+        .forEach((job) => {
+          const providerIds = job.provider.map((provider) => provider.id);
+
+          provider
+            .filter((provider) => provider.endsWith('.js'))
+            .map((pro) => require(`${path}/${pro}`))
+            .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
+            .forEach(async (pro) => {
+              const providerId = pro.metaInformation.id;
+              if (providerId == null || providerId.length === 0) {
+                throw new Error('Provider id must not be empty. => ' + pro);
+              }
+              const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
+              if (providerConfig == null) {
+                throw new Error(`Provider Config for provider with id ${providerId} not found.`);
+              }
+              pro.init(providerConfig, job.blacklist);
+              await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute();
+              setLastJobExecution(job.id);
+            });
+        });
+    } else {
+      /* eslint-disable no-console */
+      console.debug('Working hours set. Skipping as outside of working hours.');
+      /* eslint-enable no-console */
+    }
     return exec;
   })(),
   INTERVAL

lib/FredyRuntime.js

@@ -3,6 +3,7 @@ const { setKnownListings, getKnownListings } = require('./services/storage/listi
 
 const notify = require('./notification/notify');
 const xray = require('./services/scraper');
+const scrapingAnt = require('./services/scrapingAnt');
 
 class FredyRuntime {
   /**
@@ -41,15 +42,29 @@ class FredyRuntime {
 
   _getListings(url) {
     return new Promise((resolve, reject) => {
-      let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
-
-      x((err, listings) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(listings);
-        }
-      });
+      const id = this._providerId;
+      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
+        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
+        /* eslint-disable no-console */
+        console.log(error);
+        /* eslint-enable no-console */
+        reject(error);
+        return;
+      }
+      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
+      try {
+        xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
+          .then((listings) => {
+            resolve(listings == null ? [] : listings);
+          })
+          .catch((err) => {
+            reject(err);
+            console.error(err);
+          });
+      } catch (error) {
+        reject(error);
+        console.error(error);
+      }
     });
   }
 

lib/api/api.js

@@ -1,5 +1,6 @@
 const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
 const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
+const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
 const { analyticsRouter } = require('./routes/analyticsRouter');
 const { providerRouter } = require('./routes/providerRouter');
 const { loginRouter } = require('./routes/loginRoute');
@@ -28,6 +29,7 @@ service.use('/api/jobs', authInterceptor());
 service.use('/api/admin', adminInterceptor());
 
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
+service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);

lib/api/routes/generalSettingsRoute.js

@@ -0,0 +1,24 @@
+const service = require('restana')();
+const generalSettingsRouter = service.newRouter();
+const config = require('../../../conf/config.json');
+const fs = require('fs');
+
+generalSettingsRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, config);
+  res.send();
+});
+
+generalSettingsRouter.post('/', async (req, res) => {
+  const settings = req.body;
+
+  try {
+    fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
+  } catch (err) {
+    console.error(err);
+    res.send(new Error('Error while trying to write settings.'));
+    return;
+  }
+  res.send();
+});
+
+exports.generalSettingsRouter = generalSettingsRouter;

lib/api/routes/jobRouter.js

@@ -2,6 +2,9 @@ const service = require('restana')();
 const jobRouter = service.newRouter();
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
+const immoscoutProvider = require('../../provider/immoscout');
+const config = require('../../../conf/config.json');
+
 const { isAdmin } = require('../security');
 
 function doesJobBelongsToUser(job, req) {
@@ -26,8 +29,26 @@ jobRouter.get('/', async (req, res) => {
   res.send();
 });
 
+jobRouter.get('/processingTimes', async (req, res) => {
+  res.body = {
+    interval: config.interval,
+    lastRun: config.lastRun || null,
+  };
+
+  res.send();
+});
+
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
+  if (
+    provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
+    (config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
+  ) {
+    res.send(
+      new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
+    );
+    return;
+  }
   try {
     jobStorage.upsertJob({
       userId: req.session.currentUser,

lib/notification/adapter/telegram.js

@@ -1,6 +1,5 @@
-const TelegramBot = require('tg-yarl');
 const { markdown2Html } = require('../../services/markdown');
-const opts = { parse_mode: 'Markdown' };
+const axios = require('axios');
 
 /**
  * sends new listings to telegram
@@ -13,19 +12,22 @@ const opts = { parse_mode: 'Markdown' };
 exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
 
-  const bot = new TelegramBot(token);
-
-  let message = `Job: ${jobKey} | Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
+  let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
 
   message += newListings.map(
     (o) =>
-      `*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
+      `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
       [o.address, o.price, o.size].join(' | ') +
       '\n' +
-      `[LINK](${o.link})\n\n`
+      `<a href="${o.link}">${o.link}</a>\n\n`
   );
 
-  return bot.sendMessage(chatId, message, opts);
+  return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
+    chat_id: chatId,
+    text: message,
+    parse_mode: 'HTML',
+    disable_web_page_preview: true,
+  });
 };
 
 function shorten(str, len = 30) {

lib/provider/immoscout.js

@@ -0,0 +1,44 @@
+const utils = require('../utils');
+
+let appliedBlackList = [];
+
+function normalize(o) {
+  const title = o.title.replace('NEU', '');
+  const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
+  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  return Object.assign(o, { title, address, link });
+}
+
+function applyBlacklist(o) {
+  return !utils.isOneOf(o.title, appliedBlackList);
+}
+
+const config = {
+  url: null,
+  crawlContainer: '#resultListItems li.result-list__listing',
+  crawlFields: {
+    id: '.result-list-entry@data-obid | int',
+    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
+    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
+    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
+    link: '.result-list-entry .result-list-entry__brand-title-container@href',
+    address: '.result-list-entry .result-list-entry__map-link',
+  },
+  paginate: '#pager .align-right a@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+
+exports.metaInformation = {
+  name: 'Immoscout',
+  baseUrl: 'https://www.immobilienscout24.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
+
+exports.config = config;

lib/services/requestDriver.js

@@ -0,0 +1,33 @@
+const axios = require('axios');
+
+function makeDriver(headers = {}) {
+  let cookies = '';
+
+  return async function driver(context, callback) {
+    const url = context.url;
+    let result;
+    try {
+      result = await axios({
+        url,
+        headers: {
+          ...headers,
+          Cookie: cookies,
+        },
+      });
+    } catch (exception) {
+      callback(exception, null);
+    }
+
+    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
+      //assume we have gotten a response from scrapingAnt
+      if (cookies.length === 0) {
+        cookies = result.data.cookies;
+      }
+      callback(null, result.data.content);
+    } else {
+      callback(null, result.data);
+    }
+  };
+}
+
+module.exports = makeDriver;

lib/services/scraper.js

@@ -1,4 +1,5 @@
-const makeDriver = require('request-x-ray');
+const config = require('../../conf/config.json');
+const makeDriver = require('./requestDriver');
 const Xray = require('x-ray');
 
 class Scraper {
@@ -9,14 +10,15 @@ class Scraper {
       int: this._int,
     };
 
-    const driver = makeDriver({
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-        cookie:
-          'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
-      },
-    });
+    const headers = {
+      'User-Agent':
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
+    };
+
+    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
+      headers['x-api-key'] = config.scrapingAnt.apiKey;
+    }
+    const driver = makeDriver(headers);
 
     const xray = Xray({ filters });
     xray.driver(driver);

lib/services/scrapingAnt.js

@@ -0,0 +1,24 @@
+const { metaInformation } = require('../provider/immoscout');
+//to better confure re-capture chose a random proxy each time we do a call
+const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
+const config = require('../../conf/config.json');
+
+const isImmoscout = (id) => {
+  return id.toLowerCase() === metaInformation.id;
+};
+
+exports.transformUrlForScrapingAnt = (url, id) => {
+  const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
+
+  if (isImmoscout(id)) {
+    //only do calls to scrapingAnt when dealing with Immoscout
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
+  }
+  return url;
+};
+
+exports.isScrapingAntApiKeySet = () => {
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+};
+
+exports.isImmoscout = isImmoscout;

lib/utils.js

@@ -8,4 +8,29 @@ function isOneOf(word, arr) {
   return blacklist.test(word);
 }
 
-module.exports = { isOneOf };
+function nullOrEmpty(val) {
+  return val == null || val.length === 0;
+}
+
+function timeStringToMs(timeString, now) {
+  const d = new Date(now);
+  const parts = timeString.split(':');
+  d.setHours(parts[0]);
+  d.setMinutes(parts[1]);
+  d.setSeconds(0);
+  return d.getTime();
+}
+
+function duringWorkingHoursOrNotSet(config, now) {
+  const { workingHours } = config;
+  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+    return true;
+  }
+
+  const toDate = timeStringToMs(workingHours.to, now);
+  const fromDate = timeStringToMs(workingHours.from, now);
+
+  return fromDate <= now && toDate >= now;
+}
+
+module.exports = { isOneOf, nullOrEmpty, duringWorkingHoursOrNotSet };

package.json

@@ -1,11 +1,11 @@
 {
   "name": "fredy",
-  "version": "3.0.0",
+  "version": "5.1.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
     "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
-    "prod": "export BUILD_DEV='false' && export NODE_ENV='production' && webpack --config ./webpack.prod.js",
+    "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
     "test": "mocha --timeout 20000 test/**/*.test.js"
   },
@@ -41,7 +41,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=11.0.0",
+    "node": ">=12.13.0",
     "npm": ">=6.0.0"
   },
   "browserslist": [
@@ -53,59 +53,57 @@
   "dependencies": {
     "@rematch/core": "2.0.1",
     "@rematch/loading": "2.0.1",
-    "@sendgrid/mail": "7.4.2",
-    "axios": "^0.21.1",
+    "@sendgrid/mail": "7.4.4",
+    "axios": "0.21.1",
     "body-parser": "1.19.0",
-    "cookie-session": "^1.4.0",
+    "cookie-session": "1.4.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.0.1",
-    "highcharts-react-official": "^3.0.0",
+    "highcharts": "9.1.0",
+    "highcharts-react-official": "3.0.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "^3.1.22",
-    "node-mailjet": "3.3.1",
+    "nanoid": "3.1.23",
+    "node-mailjet": "3.3.4",
     "react": "17.0.2",
     "react-dom": "17.0.2",
-    "react-redux": "7.2.3",
+    "react-redux": "7.2.4",
     "react-router": "5.2.0",
     "react-router-dom": "5.2.0",
     "react-switch": "^6.0.0",
-    "redux": "4.0.5",
+    "redux": "4.1.0",
     "redux-thunk": "2.3.0",
-    "request-x-ray": "0.1.4",
-    "restana": "4.8.1",
+    "restana": "4.9.1",
     "semantic-ui-react": "2.0.3",
     "serve-static": "^1.14.1",
     "slack": "11.0.2",
-    "tg-yarl": "1.3.0",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.13.13",
-    "@babel/preset-env": "7.13.12",
+    "@babel/core": "7.14.3",
+    "@babel/preset-env": "7.14.2",
     "@babel/preset-react": "7.13.13",
     "babel-eslint": "10.1.0",
     "babel-loader": "8.2.2",
     "chai": "4.3.4",
     "clean-webpack-plugin": "3.0.0",
-    "copy-webpack-plugin": "6.3.0",
-    "css-loader": "5.0.1",
-    "eslint": "7.23.0",
-    "eslint-config-prettier": "7.1.0",
-    "eslint-plugin-react": "7.23.1",
+    "copy-webpack-plugin": "9.0.0",
+    "css-loader": "5.2.6",
+    "eslint": "7.27.0",
+    "eslint-config-prettier": "8.3.0",
+    "eslint-plugin-react": "7.23.2",
     "file-loader": "6.2.0",
     "history": "5.0.0",
     "husky": "4.3.8",
     "less": "4.1.1",
-    "less-loader": "7.2.1",
-    "lint-staged": "10.5.4",
-    "mocha": "8.3.2",
-    "prettier": "2.2.1",
+    "less-loader": "9.0.0",
+    "lint-staged": "11.0.0",
+    "mocha": "8.4.0",
+    "prettier": "2.3.0",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
     "style-loader": "2.0.0",
     "url-loader": "4.1.1",
-    "webpack": "4.44.2",
+    "webpack": "5.37.1",
     "webpack-cli": "3.3.12",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.7.3"

test/provider/immoscout.test.js

@@ -0,0 +1,56 @@
+const mockNotification = require('../mocks/mockNotification');
+const providerConfig = require('./testProvider.json');
+const mockStore = require('../mocks/mockStore');
+const proxyquire = require('proxyquire').noCallThru();
+const expect = require('chai').expect;
+const provider = require('../../lib/provider/immoscout');
+const scrapingAnt = require('../../lib/services/scrapingAnt');
+
+describe('#immoscout testsuite()', () => {
+  provider.init(providerConfig.immoscout, [], []);
+  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    './services/storage/listingsStorage': {
+      ...mockStore,
+    },
+    './notification/notify': mockNotification,
+  });
+
+  it('should test immoscout provider', async () => {
+    return await new Promise((resolve) => {
+      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+        /* eslint-disable no-console */
+        console.info('Skipping Immoscout test as ScrapingAnt Api Key is not set.');
+        /* eslint-enable no-console */
+        resolve();
+        return;
+      }
+
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+
+        const notificationObj = mockNotification.get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immoscout');
+
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('number');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.size).that.does.include('m²');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilienscout24.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/testProvider.json

@@ -12,6 +12,10 @@
     "url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
     "enabled": true
   },
+  "immoscout": {
+    "url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search",
+    "enabled": true
+  },
   "kalaydo": {
     "url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
     "enabled": true

test/provider/utils.test.js

@@ -1,5 +1,13 @@
 const utils = require('../../lib/utils');
 const assert = require('assert');
+const expect = require('chai').expect;
+
+const fakeWorkingHoursConfig = (from, to) => ({
+  workingHours: {
+    to,
+    from,
+  },
+});
 
 describe('utils', () => {
   describe('#isOneOf()', () => {
@@ -10,4 +18,22 @@ describe('utils', () => {
       assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
     });
   });
+
+  describe('#duringWorkingHoursOrNotSet()', () => {
+    it('should be false', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
+    });
+    it('should be true', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
+    });
+    it('should be true if nothing set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
+    });
+    it('should be true if only to is set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
+    });
+    it('should be true if only from is set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
+    });
+  });
 });

ui/src/App.js

@@ -2,6 +2,7 @@ import React, { useEffect } from 'react';
 
 import InsufficientPermission from './components/permission/InsufficientPermission';
 import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
+import GeneralSettings from './views/generalSettings/GeneralSettings';
 import ToastsContainer from './components/toasts/ToastContainer';
 import JobMutation from './views/jobs/mutation/JobMutation';
 import UserMutator from './views/user/mutation/UserMutator';
@@ -29,6 +30,7 @@ export default function FredyApp() {
   useEffect(async () => {
     await dispatch.provider.getProvider();
     await dispatch.jobs.getJobs();
+    await dispatch.jobs.getProcessingTimes();
     await dispatch.notificationAdapter.getAdapter();
     await dispatch.user.getCurrentUser();
 
@@ -77,6 +79,12 @@ export default function FredyApp() {
               currentUser={currentUser}
             />
             <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
+            <PermissionAwareRoute
+              name="General Settings"
+              path="/generalSettings"
+              component={<GeneralSettings />}
+              currentUser={currentUser}
+            />
 
             <Redirect from="/" to={'/jobs'} />
           </Switch>

ui/src/components/menu/Menu.js

@@ -1,6 +1,6 @@
 import React from 'react';
 import { useHistory } from 'react-router-dom';
-import { Menu } from 'semantic-ui-react';
+import { Icon, Menu } from 'semantic-ui-react';
 
 import './Menu.less';
 import { useLocation } from 'react-router';
@@ -19,7 +19,7 @@ const TopMenu = function TopMenu({ isAdmin }) {
         className={isActiveRoute('jobs') ? 'topMenu__active' : 'topMenu__item'}
         onClick={() => history.push('/jobs')}
       >
-        Job Configuration
+        <Icon name="search" /> Job Configuration
       </Menu.Item>
 
       {isAdmin && (
@@ -29,7 +29,18 @@ const TopMenu = function TopMenu({ isAdmin }) {
           className={isActiveRoute('users') ? 'topMenu__active' : 'topMenu__item'}
           onClick={() => history.push('/users')}
         >
-          User configuration
+          <Icon name="user" /> User configuration
+        </Menu.Item>
+      )}
+
+      {isAdmin && (
+        <Menu.Item
+          name="general"
+          active={isActiveRoute('general')}
+          className={isActiveRoute('general') ? 'topMenu__active' : 'topMenu__item'}
+          onClick={() => history.push('/generalSettings')}
+        >
+          <Icon name="cog" /> General Settings
         </Menu.Item>
       )}
     </Menu>

ui/src/components/table/JobTable.js

@@ -31,8 +31,8 @@ const content = (jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight
             <Table.Cell>
               <div style={{ float: 'right' }}>
                 <Button circular color="teal" icon="chart line" onClick={() => onJobInsight(job.id)} />
-                <Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
                 <Button circular color="blue" icon="edit" onClick={() => onJobEdit(job.id)} />
+                <Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
               </div>
             </Table.Cell>
           </Table.Row>

ui/src/components/table/NotificationAdapterTable.js

@@ -12,7 +12,7 @@ const emptyTable = () => {
   );
 };
 
-const content = (adapterData, onRemove) => {
+const content = (adapterData, onRemove, onEdit) => {
   return (
     <Fragment>
       {adapterData.map((data) => {
@@ -21,6 +21,7 @@ const content = (adapterData, onRemove) => {
             <Table.Cell>{data.name}</Table.Cell>
             <Table.Cell>
               <div style={{ float: 'right' }}>
+                <Button circular color="blue" icon="edit" onClick={() => onEdit(data.id)} />
                 <Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
               </div>
             </Table.Cell>
@@ -31,7 +32,7 @@ const content = (adapterData, onRemove) => {
   );
 };
 
-export default function NotificationAdapterTable({ notificationAdapter = [], onRemove } = {}) {
+export default function NotificationAdapterTable({ notificationAdapter = [], onRemove, onEdit } = {}) {
   return (
     <Table singleLine inverted>
       <Table.Header>
@@ -42,7 +43,7 @@ export default function NotificationAdapterTable({ notificationAdapter = [], onR
       </Table.Header>
 
       <Table.Body>
-        {notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove)}
+        {notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove, onEdit)}
       </Table.Body>
     </Table>
   );

ui/src/services/rematch/models/generalSettings.js

@@ -0,0 +1,26 @@
+import { xhrGet } from '../../xhr';
+
+export const generalSettings = {
+  state: {
+    settings: {},
+  },
+  reducers: {
+    //only admins
+    setGeneralSettings: (state, payload) => {
+      return {
+        ...state,
+        settings: payload,
+      };
+    },
+  },
+  effects: {
+    async getGeneralSettings() {
+      try {
+        const response = await xhrGet('/api/admin/generalSettings');
+        this.setGeneralSettings(response.json);
+      } catch (Exception) {
+        console.error('Error while trying to get resource for api/admin/generalSettings. Error:', Exception);
+      }
+    },
+  },
+};

ui/src/services/rematch/models/jobs.js

@@ -4,6 +4,7 @@ export const jobs = {
   state: {
     jobs: [],
     insights: {},
+    processingTimes: {},
   },
   reducers: {
     setJobs: (state, payload) => {
@@ -12,6 +13,12 @@ export const jobs = {
         jobs: Object.freeze(payload),
       };
     },
+    setProcessingTimes: (state, payload) => {
+      return {
+        ...state,
+        processingTimes: Object.freeze(payload),
+      };
+    },
     setJobInsights: (state, payload, jobId) => {
       return {
         ...state,
@@ -31,6 +38,14 @@ export const jobs = {
         console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
       }
     },
+    async getProcessingTimes() {
+      try {
+        const response = await xhrGet('/api/jobs/processingTimes');
+        this.setProcessingTimes(response.json);
+      } catch (Exception) {
+        console.error(`Error while trying to get resource for api/processingTimes. Error:`, Exception);
+      }
+    },
     async getInsightDataForJob(jobId) {
       try {
         const response = await xhrGet(`/api/jobs/insights/${jobId}`);

ui/src/services/rematch/store.js

@@ -1,4 +1,5 @@
 import { notificationAdapter } from './models/notificationAdapter';
+import { generalSettings } from './models/generalSettings';
 import createLoadingPlugin from '@rematch/loading';
 import { provider } from './models/provider';
 import { createLogger } from 'redux-logger';
@@ -17,6 +18,7 @@ const store = init({
   name: 'fredy',
   models: {
     notificationAdapter,
+    generalSettings,
     provider,
     jobs,
     user,

ui/src/views/generalSettings/GeneralSettings.js

@@ -0,0 +1,211 @@
+import React from 'react';
+
+import { useDispatch, useSelector } from 'react-redux';
+
+import { Button, Form, Header, Icon, Message, Popup, Segment } from 'semantic-ui-react';
+import ToastContext from '../../components/toasts/ToastContext';
+import Headline from '../../components/headline/Headline';
+import { xhrPost } from '../../services/xhr';
+
+import './GeneralSettings.less';
+
+const SegmentPart = ({ name, icon, children, helpText }) => (
+  <React.Fragment>
+    <Header as="h5" inverted attached="top" sub>
+      <Icon name={icon} inverted size="mini" />
+      <Header.Content>{name}</Header.Content>
+    </Header>
+
+    <Popup
+      content={helpText}
+      trigger={
+        <span className="generalSettings__help">
+          {' '}
+          <Icon name="help circle" inverted />
+          What is this?
+        </span>
+      }
+    />
+    <Segment inverted attached>
+      {children}
+    </Segment>
+  </React.Fragment>
+);
+
+const GeneralSettings = function Users() {
+  const dispatch = useDispatch();
+  const [loading, setLoading] = React.useState(true);
+
+  const settings = useSelector((state) => state.generalSettings.settings);
+
+  const [interval, setInterval] = React.useState('');
+  const [port, setPort] = React.useState('');
+  const [scrapingAntApiKey, setScrapingAntApiKey] = React.useState('');
+  const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
+  const [workingHourTo, setWorkingHourTo] = React.useState(null);
+  const ctx = React.useContext(ToastContext);
+
+  React.useEffect(async () => {
+    await dispatch.generalSettings.getGeneralSettings();
+    setLoading(false);
+  }, []);
+
+  React.useEffect(async () => {
+    setInterval(settings?.interval);
+    setPort(settings?.port);
+    setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
+    setWorkingHourFrom(settings?.workingHours?.from);
+    setWorkingHourTo(settings?.workingHours?.to);
+  }, [settings]);
+
+  const nullOrEmpty = (val) => val == null || val.length === 0;
+
+  const throwMessage = (message, type) => {
+    ctx.showToast({
+      title: type === 'error' ? 'Error' : 'Success',
+      message: message,
+      delay: 5000,
+      backgroundColor: type === 'error' ? '#db2828' : '#87eb8f',
+      color: type === 'error' ? '#fff' : '#000',
+    });
+  };
+
+  const onStore = async () => {
+    if (nullOrEmpty(interval)) {
+      throwMessage('Interval may not be empty.', 'error');
+      return;
+    }
+    if (nullOrEmpty(port)) {
+      throwMessage('Port may not be empty.', 'error');
+      return;
+    }
+    if (
+      (!nullOrEmpty(workingHourFrom) && nullOrEmpty(workingHourTo)) ||
+      (nullOrEmpty(workingHourFrom) && !nullOrEmpty(workingHourTo))
+    ) {
+      throwMessage('Working hours to and from must be set if either to or from has been set before.', 'error');
+      return;
+    }
+    try {
+      await xhrPost('/api/admin/generalSettings', {
+        interval,
+        port,
+        scrapingAnt: {
+          apiKey: scrapingAntApiKey,
+        },
+        workingHours: {
+          from: workingHourFrom,
+          to: workingHourTo,
+        },
+      });
+    } catch (exception) {
+      console.error(exception);
+      throwMessage('Error while trying to store settings.', 'error');
+      return;
+    }
+    throwMessage('Settings stored successfully. You MUST restart Fredy.', 'success');
+  };
+
+  return (
+    <div>
+      {!loading && (
+        <React.Fragment>
+          <Headline text="General Settings" />
+          <Message info>
+            <h5>
+              <Icon name="info circle" />
+              Info
+            </h5>
+            <p>If you change any settings, you must restart Fredy afterwards.</p>
+          </Message>
+          <Form>
+            <SegmentPart
+              name="Interval"
+              helpText="Interval in minutes for running queries against the configured services."
+              icon="refresh"
+            >
+              <Form.Input
+                type="number"
+                min="0"
+                max="1440"
+                placeholder="Interval in minutes"
+                inverted
+                size="mini"
+                width={6}
+                defaultValue={interval}
+                onChange={(e) => setInterval(e.target.value)}
+              />
+            </SegmentPart>
+
+            <SegmentPart name="Port" helpText="Port on which Fredy is running." icon="connectdevelop">
+              <Form.Input
+                type="number"
+                min="0"
+                max="99999"
+                placeholder="Port"
+                inverted
+                size="mini"
+                width={6}
+                defaultValue={port}
+                onChange={(e) => setPort(e.target.value)}
+              />
+            </SegmentPart>
+
+            <SegmentPart
+              name="ScrapingAnt Api Key"
+              helpText="The api key for ScrapingAnt is used to be able to scrape Immoscout."
+              icon="key"
+            >
+              <Form.Input
+                type="text"
+                placeholder="ScrapingAnt Api Key"
+                inverted
+                size="mini"
+                width={6}
+                defaultValue={scrapingAntApiKey}
+                onChange={(e) => setScrapingAntApiKey(e.target.value)}
+              />
+            </SegmentPart>
+
+            <SegmentPart
+              name="Working hours"
+              helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
+              icon="calendar outline"
+            >
+              <div className="generalSettings__timePickerContainer">
+                <Form.Input
+                  className="generalSettings__time"
+                  type="time"
+                  placeholder="ScrapingAnt Api Key"
+                  inverted
+                  size="mini"
+                  width={2}
+                  defaultValue={workingHourFrom}
+                  onChange={(e) => setWorkingHourFrom(e.target.value)}
+                />
+                <div className="generalSettings__until">until</div>
+                <Form.Input
+                  type="time"
+                  placeholder="ScrapingAnt Api Key"
+                  inverted
+                  size="mini"
+                  width={2}
+                  defaultValue={workingHourTo}
+                  onChange={(e) => setWorkingHourTo(e.target.value)}
+                />
+              </div>
+            </SegmentPart>
+
+            <Segment inverted floated="right">
+              <Button color="teal" onClick={onStore}>
+                Save
+              </Button>
+            </Segment>
+          </Form>
+        </React.Fragment>
+      )}
+    </div>
+  );
+};
+
+export default GeneralSettings;

ui/src/views/generalSettings/GeneralSettings.less

@@ -0,0 +1,17 @@
+.generalSettings {
+  &__timePickerContainer {
+    display: flex;
+    align-items: baseline;
+  }
+
+  &__until {
+    margin-left: 1rem;
+    margin-right: 1rem;
+  }
+
+  &__help{
+    font-size: 11px;
+    margin-left: 1rem;
+  }
+
+}

ui/src/views/jobs/Jobs.js

@@ -6,11 +6,13 @@ import { useSelector, useDispatch } from 'react-redux';
 import { xhrDelete, xhrPut } from '../../services/xhr';
 import { Button, Icon } from 'semantic-ui-react';
 import { useHistory } from 'react-router-dom';
+import ProcessingTimes from './ProcessingTimes';
 
 import './Jobs.less';
 
 export default function Jobs() {
   const jobs = useSelector((state) => state.jobs.jobs);
+  const processingTimes = useSelector((state) => state.jobs.processingTimes);
   const history = useHistory();
   const dispatch = useDispatch();
   const ctx = React.useContext(ToastContext);
@@ -61,10 +63,13 @@ export default function Jobs() {
 
   return (
     <div>
-      <Button primary className="jobs__newButton" onClick={() => history.push('/jobs/new')}>
-        <Icon name="plus" />
-        New Job
-      </Button>
+      <div>
+        {processingTimes != null && <ProcessingTimes processingTimes={processingTimes} />}
+        <Button primary className="jobs__newButton" onClick={() => history.push('/jobs/new')}>
+          <Icon name="plus" />
+          New Job
+        </Button>
+      </div>
 
       <JobTable
         jobs={jobs || []}

ui/src/views/jobs/ProcessingTimes.js

@@ -0,0 +1,26 @@
+import React from 'react';
+import { format } from '../../services/time/timeService';
+import { Label } from 'semantic-ui-react';
+
+export default function ProcessingTimes({ processingTimes }) {
+  return (
+    <React.Fragment>
+      <Label as="span" color="black">
+        Processing Interval:
+        <Label.Detail>{processingTimes.interval} min</Label.Detail>
+      </Label>
+      {processingTimes.lastRun && (
+        <React.Fragment>
+          <Label as="span" color="black">
+            Last run:
+            <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
+          </Label>
+          <Label as="span" color="black">
+            Next run:
+            <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
+          </Label>
+        </React.Fragment>
+      )}
+    </React.Fragment>
+  );
+}

ui/src/views/jobs/mutation/JobMutation.js

@@ -29,6 +29,7 @@ export default function JobMutator() {
 
   const [providerCreationVisible, setProviderCreationVisibility] = useState(false);
   const [notificationCreationVisible, setNotificationCreationVisibility] = useState(false);
+  const [editNotificationAdapter, setEditNotificationAdapter] = useState(null);
   const [providerData, setProviderData] = useState(defaultProviderData);
   const [name, setName] = useState(defaultName);
   const [blacklist, setBlacklist] = useState(defaultBlacklist);
@@ -83,11 +84,12 @@ export default function JobMutator() {
       });
       history.push('/jobs');
     } catch (Exception) {
-      console.error(Exception);
+      console.error(Exception.json.message);
+
       ctx.showToast({
         title: 'Error',
-        message: Exception,
-        delay: 35000,
+        message: Exception.json != null ? Exception.json.message : Exception,
+        delay: 8000,
         backgroundColor: '#db2828',
         color: '#fff',
       });
@@ -105,14 +107,25 @@ export default function JobMutator() {
         }}
       />
 
-      <NotificationAdapterMutator
-        visible={notificationCreationVisible}
-        onVisibilityChanged={(visible) => setNotificationCreationVisibility(visible)}
-        selected={providerData}
-        onData={(data) => {
-          setNotificationAdapterData([...notificationAdapterData, data]);
-        }}
-      />
+      {notificationCreationVisible && (
+        <NotificationAdapterMutator
+          visible={notificationCreationVisible}
+          onVisibilityChanged={(visible) => {
+            setEditNotificationAdapter(null);
+            setNotificationCreationVisibility(visible);
+          }}
+          selected={notificationAdapterData}
+          editNotificationAdapter={
+            editNotificationAdapter == null
+              ? null
+              : notificationAdapterData.find((adapter) => adapter.id === editNotificationAdapter)
+          }
+          onData={(data) => {
+            const oldData = [...notificationAdapterData].filter((o) => o.id !== data.id);
+            setNotificationAdapterData([...oldData, data]);
+          }}
+        />
+      )}
 
       <Headline text={jobToBeEdit ? 'Edit a Job' : 'Create a new Job'} />
       <Form className="jobMutation__form">
@@ -173,8 +186,13 @@ export default function JobMutator() {
           <NotificationAdapterTable
             notificationAdapter={notificationAdapterData}
             onRemove={(adapterId) => {
+              setEditNotificationAdapter(null);
               setNotificationAdapterData(notificationAdapterData.filter((adapter) => adapter.id !== adapterId));
             }}
+            onEdit={(adapterId) => {
+              setEditNotificationAdapter(adapterId);
+              setNotificationCreationVisibility(true);
+            }}
           />
         </div>
 

ui/src/views/jobs/mutation/JobMutation.less

@@ -22,7 +22,7 @@
   }
 
   &__separator{
-    background-color: #3a3a3a;
+    background-color: #2b2b2b;
     border-radius: 10px;
     padding: .8rem;
   }

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.js

@@ -42,15 +42,29 @@ const validate = (selectedAdapter) => {
   return [...new Set(results)];
 };
 
+function spreadPrefilledAdapterWithValues(prefilled, fields) {
+  if (prefilled != null && fields != null) {
+    Object.keys(fields).forEach((fieldKey) => {
+      prefilled.fields[fieldKey].value = fields[fieldKey];
+    });
+  }
+}
+
 export default function NotificationAdapterMutator({
   onVisibilityChanged,
   visible = false,
   selected = [],
+  editNotificationAdapter,
   onData,
 } = {}) {
   const adapter = useSelector((state) => state.notificationAdapter);
 
-  const [selectedAdapter, setSelectedAdapter] = useState(null);
+  const preFilledSelectedAdapter =
+    editNotificationAdapter == null ? null : adapter.find((a) => a.id === editNotificationAdapter.id);
+
+  spreadPrefilledAdapterWithValues(preFilledSelectedAdapter, editNotificationAdapter?.fields);
+
+  const [selectedAdapter, setSelectedAdapter] = useState(preFilledSelectedAdapter);
   const [validationMessage, setValidationMessage] = useState(null);
   const [successMessage, setSuccessMessage] = useState(null);
 
@@ -107,9 +121,12 @@ export default function NotificationAdapterMutator({
 
     setSelectedAdapter({
       ...selectedAdapter,
-      config: {
+      fields: {
         ...selectedAdapter.fields,
-        [key]: uiElement,
+        [key]: {
+          ...selectedAdapter.fields[key],
+          value,
+        },
       },
     });
   };
@@ -174,7 +191,7 @@ export default function NotificationAdapterMutator({
             There are multiple ways how Fredy can send new listings to you. Chose your weapon...
           </p>
           <Dropdown
-            placeholder="Select a notification adapteer"
+            placeholder="Select a notification adapter"
             className="providerMutator__fields"
             selection
             value={selectedAdapter == null ? '' : selectedAdapter.id}
@@ -187,7 +204,11 @@ export default function NotificationAdapterMutator({
                 };
               })
               //filter out those, that have already been selected
-              .filter((option) => selected.find((selectedOption) => selectedOption.id === option.key) == null)
+              .filter((option) =>
+                editNotificationAdapter != null
+                  ? true
+                  : selected.find((selectedOption) => selectedOption.id === option.key) == null
+              )
               .sort(sortAdapter)}
             onChange={(e, { value }) => {
               setSuccessMessage(null);
@@ -215,7 +236,7 @@ export default function NotificationAdapterMutator({
       </Modal.Content>
       <Modal.Actions>
         <Button
-          content="Try"
+          content="Try Notification Adapter"
           labelPosition="left"
           floated="left"
           icon="hand spock"

ui/src/views/jobs/mutation/components/provider/ProviderMutator.js

@@ -84,6 +84,11 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
             <br />
             When the search results are shown on the website, copy the url and paste it into the textfield below.
             <br />
+            <span style={{ color: '#ff0000' }}>
+              If you chose Immoscout as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              (See readme)
+            </span>
+            <br />
             <span style={{ color: '#ff0000' }}>
               Do not forget to sort the results by date before copying the url to Fredy, so that Fredy always captures
               the latest search results.

webpack.common.js

@@ -18,6 +18,7 @@ module.exports = {
     publicPath: '/',
     filename: 'fredy.bundle.js',
   },
+  performance: { hints: false },
   module: {
     rules: [
       {