UI dependencies upgrade (#26)

* upgrading dependencies

* making notification adapter editable

.eslintrc.js

@@ -6,7 +6,7 @@ module.exports = {
     browser: true,
   },
   parser: 'babel-eslint',
-  extends: ['eslint:recommended', 'prettier', 'prettier/react'],
+  extends: ['eslint:recommended', 'prettier'],
   plugins: ['react'],
   globals: {
     Promise: false,

CHANGELOG.md

@@ -1,3 +1,10 @@
+###### [V5.0.0]
+- Upgrading dependencies
+- NodeJS 12 is now the minimum supported version
+
+###### [V4.0.0]
+Bringing back Immoscout :tada:
+
 ###### [V3.0.0]
 This is basically a re-write, your old config file will not be compatible anymore. Please re-created your search jobs
 on the new ui and use the values from your previous config file if needed.

README.md

@@ -17,8 +17,28 @@ yarn run start
 ```
 _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` for username and password. (You should change the password asap when you plan to run Fredy on your server.)
 
+<p align="center">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_1.png" width="30%">
+&nbsp; &nbsp; &nbsp; &nbsp;
+  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
+&nbsp; &nbsp; &nbsp; &nbsp;  
+  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
+</p>
+<p align="center">
+
+</p>
+
+## Immoscout
+I have added **EXPERIMENTAL** support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successful validated) re-send the cookies each time.
+
+To be able to use Immoscout, you need to create an account and copy the apiKey into the config file under /conf/config.json.
+The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time it works pretty good :)
+
+If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefor they've decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (No I don't get any money for recommending good services...)
+
+
 ## Understanding the fundamentals
-There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
+There are 3 important parts in Fredy, that you need to understand leveraging the full power of _Fredy_.
 
 #### Adapter
 _Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few. Those services are called adapter within _Fredy_. When creating a new job, you can choose 1 or many adapter.    
@@ -57,8 +77,15 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-## Why is Immoscout missing
-Immoscout decided to add "robot protection" to their service. Meaning if Fredy tries to check for listings, it will be recognized as a bot. I haven't found a way around it (yet) ;)
+## Immoscout 
+I have added EXPERIMENTAL support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way
+around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/).
+
+To be able to use Immoscout, you need to create an account and copy the apiKey into the config file under /conf/config.json.
+The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time
+it works pretty good :)
+
+If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... (No I don't get any money for recommending good service)
 
 #### Contribution guidelines  
 

conf/config.json

@@ -1,4 +1,7 @@
 {
   "interval": 30,
-  "port": 9998
+  "port": 9998,
+  "scrapingAnt": {
+    "apiKey": ""
+  }
 }

index.js

@@ -24,6 +24,7 @@ console.log(`Started Fredy successfully. Ui can be accessed via http://localhost
 /* eslint-enable no-console */
 setInterval(
   (function exec() {
+    config.lastRun = Date.now();
     jobStorage
       .getJobs()
       .filter((job) => job.enabled)

lib/FredyRuntime.js

@@ -3,6 +3,7 @@ const { setKnownListings, getKnownListings } = require('./services/storage/listi
 
 const notify = require('./notification/notify');
 const xray = require('./services/scraper');
+const scrapingAnt = require('./services/scrapingAnt');
 
 class FredyRuntime {
   /**
@@ -41,15 +42,29 @@ class FredyRuntime {
 
   _getListings(url) {
     return new Promise((resolve, reject) => {
-      let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
-
-      x((err, listings) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(listings);
-        }
-      });
+      const id = this._providerId;
+      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
+        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
+        /* eslint-disable no-console */
+        console.log(error);
+        /* eslint-enable no-console */
+        reject(error);
+        return;
+      }
+      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
+      try {
+        xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
+          .then((listings) => {
+            resolve(listings == null ? [] : listings);
+          })
+          .catch((err) => {
+            reject(err);
+            console.error(err);
+          });
+      } catch (error) {
+        reject(error);
+        console.error(error);
+      }
     });
   }
 

lib/api/routes/jobRouter.js

@@ -2,6 +2,9 @@ const service = require('restana')();
 const jobRouter = service.newRouter();
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
+const immoscoutProvider = require('../../provider/immoscout');
+const config = require('../../../conf/config.json');
+
 const { isAdmin } = require('../security');
 
 function doesJobBelongsToUser(job, req) {
@@ -26,8 +29,26 @@ jobRouter.get('/', async (req, res) => {
   res.send();
 });
 
+jobRouter.get('/processingTimes', async (req, res) => {
+  res.body = {
+    interval: config.interval,
+    lastRun: config.lastRun || null,
+  };
+
+  res.send();
+});
+
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
+  if (
+    provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
+    (config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
+  ) {
+    res.send(
+      new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
+    );
+    return;
+  }
   try {
     jobStorage.upsertJob({
       userId: req.session.currentUser,

lib/notification/adapter/telegram.js

@@ -1,6 +1,5 @@
-const TelegramBot = require('tg-yarl');
 const { markdown2Html } = require('../../services/markdown');
-const opts = { parse_mode: 'Markdown' };
+const axios = require('axios');
 
 /**
  * sends new listings to telegram
@@ -13,19 +12,22 @@ const opts = { parse_mode: 'Markdown' };
 exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
 
-  const bot = new TelegramBot(token);
-
-  let message = `Job: ${jobKey} | Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
+  let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
 
   message += newListings.map(
     (o) =>
-      `*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
+      `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
       [o.address, o.price, o.size].join(' | ') +
       '\n' +
-      `[LINK](${o.link})\n\n`
+      `<a href="${o.link}">${o.link}</a>\n\n`
   );
 
-  return bot.sendMessage(chatId, message, opts);
+  return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
+    chat_id: chatId,
+    text: message,
+    parse_mode: 'HTML',
+    disable_web_page_preview: true,
+  });
 };
 
 function shorten(str, len = 30) {

lib/provider/immoscout.js

@@ -0,0 +1,44 @@
+const utils = require('../utils');
+
+let appliedBlackList = [];
+
+function normalize(o) {
+  const title = o.title.replace('NEU', '');
+  const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
+  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  return Object.assign(o, { title, address, link });
+}
+
+function applyBlacklist(o) {
+  return !utils.isOneOf(o.title, appliedBlackList);
+}
+
+const config = {
+  url: null,
+  crawlContainer: '#resultListItems li.result-list__listing',
+  crawlFields: {
+    id: '.result-list-entry@data-obid | int',
+    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
+    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
+    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
+    link: '.result-list-entry .result-list-entry__brand-title-container@href',
+    address: '.result-list-entry .result-list-entry__map-link',
+  },
+  paginate: '#pager .align-right a@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+
+exports.metaInformation = {
+  name: 'Immoscout',
+  baseUrl: 'https://www.immobilienscout24.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
+
+exports.config = config;

lib/services/requestDriver.js

@@ -0,0 +1,33 @@
+const axios = require('axios');
+
+function makeDriver(headers = {}) {
+  let cookies = '';
+
+  return async function driver(context, callback) {
+    const url = context.url;
+    let result;
+    try {
+      result = await axios({
+        url,
+        headers: {
+          ...headers,
+          Cookie: cookies,
+        },
+      });
+    } catch (exception) {
+      callback(exception, null);
+    }
+
+    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
+      //assume we have gotten a response from scrapingAnt
+      if (cookies.length === 0) {
+        cookies = result.data.cookies;
+      }
+      callback(null, result.data.content);
+    } else {
+      callback(null, result.data);
+    }
+  };
+}
+
+module.exports = makeDriver;

lib/services/scraper.js

@@ -1,4 +1,5 @@
-const makeDriver = require('request-x-ray');
+const config = require('../../conf/config.json');
+const makeDriver = require('./requestDriver');
 const Xray = require('x-ray');
 
 class Scraper {
@@ -9,14 +10,15 @@ class Scraper {
       int: this._int,
     };
 
-    const driver = makeDriver({
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-        cookie:
-          'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
-      },
-    });
+    const headers = {
+      'User-Agent':
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
+    };
+
+    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
+      headers['x-api-key'] = config.scrapingAnt.apiKey;
+    }
+    const driver = makeDriver(headers);
 
     const xray = Xray({ filters });
     xray.driver(driver);

lib/services/scrapingAnt.js

@@ -0,0 +1,24 @@
+const { metaInformation } = require('../provider/immoscout');
+//to better confure re-capture chose a random proxy each time we do a call
+const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
+const config = require('../../conf/config.json');
+
+const isImmoscout = (id) => {
+  return id.toLowerCase() === metaInformation.id;
+};
+
+exports.transformUrlForScrapingAnt = (url, id) => {
+  const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
+
+  if (isImmoscout(id)) {
+    //only do calls to scrapingAnt when dealing with Immoscout
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
+  }
+  return url;
+};
+
+exports.isScrapingAntApiKeySet = () => {
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+};
+
+exports.isImmoscout = isImmoscout;

package.json

@@ -1,11 +1,11 @@
 {
   "name": "fredy",
-  "version": "3.0.0",
+  "version": "5.0.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
     "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
-    "prod": "export BUILD_DEV='false' && export NODE_ENV='production' && webpack --config ./webpack.prod.js",
+    "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
     "test": "mocha --timeout 20000 test/**/*.test.js"
   },
@@ -41,7 +41,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=11.0.0",
+    "node": ">=12.0.0",
     "npm": ">=6.0.0"
   },
   "browserslist": [
@@ -53,59 +53,57 @@
   "dependencies": {
     "@rematch/core": "2.0.1",
     "@rematch/loading": "2.0.1",
-    "@sendgrid/mail": "7.4.2",
-    "axios": "^0.21.1",
+    "@sendgrid/mail": "7.4.4",
+    "axios": "0.21.1",
     "body-parser": "1.19.0",
-    "cookie-session": "^1.4.0",
+    "cookie-session": "1.4.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.0.1",
-    "highcharts-react-official": "^3.0.0",
+    "highcharts": "9.1.0",
+    "highcharts-react-official": "3.0.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "^3.1.22",
+    "nanoid": "3.1.23",
     "node-mailjet": "3.3.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
-    "react-redux": "7.2.3",
+    "react-redux": "7.2.4",
     "react-router": "5.2.0",
     "react-router-dom": "5.2.0",
     "react-switch": "^6.0.0",
-    "redux": "4.0.5",
+    "redux": "4.1.0",
     "redux-thunk": "2.3.0",
-    "request-x-ray": "0.1.4",
-    "restana": "4.8.1",
+    "restana": "4.9.1",
     "semantic-ui-react": "2.0.3",
     "serve-static": "^1.14.1",
     "slack": "11.0.2",
-    "tg-yarl": "1.3.0",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.13.13",
-    "@babel/preset-env": "7.13.12",
+    "@babel/core": "7.14.3",
+    "@babel/preset-env": "7.14.2",
     "@babel/preset-react": "7.13.13",
     "babel-eslint": "10.1.0",
     "babel-loader": "8.2.2",
     "chai": "4.3.4",
     "clean-webpack-plugin": "3.0.0",
-    "copy-webpack-plugin": "6.3.0",
-    "css-loader": "5.0.1",
-    "eslint": "7.23.0",
-    "eslint-config-prettier": "7.1.0",
-    "eslint-plugin-react": "7.23.1",
+    "copy-webpack-plugin": "8.1.1",
+    "css-loader": "5.2.4",
+    "eslint": "7.26.0",
+    "eslint-config-prettier": "8.3.0",
+    "eslint-plugin-react": "7.23.2",
     "file-loader": "6.2.0",
     "history": "5.0.0",
     "husky": "4.3.8",
     "less": "4.1.1",
-    "less-loader": "7.2.1",
-    "lint-staged": "10.5.4",
-    "mocha": "8.3.2",
-    "prettier": "2.2.1",
+    "less-loader": "9.0.0",
+    "lint-staged": "11.0.0",
+    "mocha": "8.4.0",
+    "prettier": "2.3.0",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
     "style-loader": "2.0.0",
     "url-loader": "4.1.1",
-    "webpack": "4.44.2",
+    "webpack": "5.37.1",
     "webpack-cli": "3.3.12",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.7.3"

test/provider/immoscout.test.js

@@ -0,0 +1,56 @@
+const mockNotification = require('../mocks/mockNotification');
+const providerConfig = require('./testProvider.json');
+const mockStore = require('../mocks/mockStore');
+const proxyquire = require('proxyquire').noCallThru();
+const expect = require('chai').expect;
+const provider = require('../../lib/provider/immoscout');
+const scrapingAnt = require('../../lib/services/scrapingAnt');
+
+describe('#immoscout testsuite()', () => {
+  provider.init(providerConfig.immoscout, [], []);
+  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    './services/storage/listingsStorage': {
+      ...mockStore,
+    },
+    './notification/notify': mockNotification,
+  });
+
+  it('should test immoscout provider', async () => {
+    return await new Promise((resolve) => {
+      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+        /* eslint-disable no-console */
+        console.info('Skipping Immoscout test as ScrapingAnt Api Key is not set.');
+        /* eslint-enable no-console */
+        resolve();
+        return;
+      }
+
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+
+        const notificationObj = mockNotification.get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immoscout');
+
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('number');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.size).that.does.include('m²');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilienscout24.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/testProvider.json

@@ -12,6 +12,10 @@
     "url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
     "enabled": true
   },
+  "immoscout": {
+    "url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search",
+    "enabled": true
+  },
   "kalaydo": {
     "url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
     "enabled": true

ui/src/App.js

@@ -29,6 +29,7 @@ export default function FredyApp() {
   useEffect(async () => {
     await dispatch.provider.getProvider();
     await dispatch.jobs.getJobs();
+    await dispatch.jobs.getProcessingTimes();
     await dispatch.notificationAdapter.getAdapter();
     await dispatch.user.getCurrentUser();
 

ui/src/components/table/JobTable.js

@@ -31,8 +31,8 @@ const content = (jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight
             <Table.Cell>
               <div style={{ float: 'right' }}>
                 <Button circular color="teal" icon="chart line" onClick={() => onJobInsight(job.id)} />
-                <Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
                 <Button circular color="blue" icon="edit" onClick={() => onJobEdit(job.id)} />
+                <Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
               </div>
             </Table.Cell>
           </Table.Row>

ui/src/components/table/NotificationAdapterTable.js

@@ -12,7 +12,7 @@ const emptyTable = () => {
   );
 };
 
-const content = (adapterData, onRemove) => {
+const content = (adapterData, onRemove, onEdit) => {
   return (
     <Fragment>
       {adapterData.map((data) => {
@@ -21,6 +21,7 @@ const content = (adapterData, onRemove) => {
             <Table.Cell>{data.name}</Table.Cell>
             <Table.Cell>
               <div style={{ float: 'right' }}>
+                <Button circular color="blue" icon="edit" onClick={() => onEdit(data.id)} />
                 <Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
               </div>
             </Table.Cell>
@@ -31,7 +32,7 @@ const content = (adapterData, onRemove) => {
   );
 };
 
-export default function NotificationAdapterTable({ notificationAdapter = [], onRemove } = {}) {
+export default function NotificationAdapterTable({ notificationAdapter = [], onRemove, onEdit } = {}) {
   return (
     <Table singleLine inverted>
       <Table.Header>
@@ -42,7 +43,7 @@ export default function NotificationAdapterTable({ notificationAdapter = [], onR
       </Table.Header>
 
       <Table.Body>
-        {notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove)}
+        {notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove, onEdit)}
       </Table.Body>
     </Table>
   );

ui/src/services/rematch/models/jobs.js

@@ -4,6 +4,7 @@ export const jobs = {
   state: {
     jobs: [],
     insights: {},
+    processingTimes: {},
   },
   reducers: {
     setJobs: (state, payload) => {
@@ -12,6 +13,12 @@ export const jobs = {
         jobs: Object.freeze(payload),
       };
     },
+    setProcessingTimes: (state, payload) => {
+      return {
+        ...state,
+        processingTimes: Object.freeze(payload),
+      };
+    },
     setJobInsights: (state, payload, jobId) => {
       return {
         ...state,
@@ -31,6 +38,14 @@ export const jobs = {
         console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
       }
     },
+    async getProcessingTimes() {
+      try {
+        const response = await xhrGet('/api/jobs/processingTimes');
+        this.setProcessingTimes(response.json);
+      } catch (Exception) {
+        console.error(`Error while trying to get resource for api/processingTimes. Error:`, Exception);
+      }
+    },
     async getInsightDataForJob(jobId) {
       try {
         const response = await xhrGet(`/api/jobs/insights/${jobId}`);

ui/src/views/jobs/Jobs.js

@@ -6,11 +6,13 @@ import { useSelector, useDispatch } from 'react-redux';
 import { xhrDelete, xhrPut } from '../../services/xhr';
 import { Button, Icon } from 'semantic-ui-react';
 import { useHistory } from 'react-router-dom';
+import ProcessingTimes from './ProcessingTimes';
 
 import './Jobs.less';
 
 export default function Jobs() {
   const jobs = useSelector((state) => state.jobs.jobs);
+  const processingTimes = useSelector((state) => state.jobs.processingTimes);
   const history = useHistory();
   const dispatch = useDispatch();
   const ctx = React.useContext(ToastContext);
@@ -61,10 +63,13 @@ export default function Jobs() {
 
   return (
     <div>
-      <Button primary className="jobs__newButton" onClick={() => history.push('/jobs/new')}>
-        <Icon name="plus" />
-        New Job
-      </Button>
+      <div>
+        {processingTimes != null && <ProcessingTimes processingTimes={processingTimes} />}
+        <Button primary className="jobs__newButton" onClick={() => history.push('/jobs/new')}>
+          <Icon name="plus" />
+          New Job
+        </Button>
+      </div>
 
       <JobTable
         jobs={jobs || []}

ui/src/views/jobs/ProcessingTimes.js

@@ -0,0 +1,26 @@
+import React from 'react';
+import { format } from '../../services/time/timeService';
+import { Label } from 'semantic-ui-react';
+
+export default function ProcessingTimes({ processingTimes }) {
+  return (
+    <React.Fragment>
+      <Label as="span" color="black">
+        Processing Interval:
+        <Label.Detail>{processingTimes.interval} min</Label.Detail>
+      </Label>
+      {processingTimes.lastRun && (
+        <React.Fragment>
+          <Label as="span" color="black">
+            Last run:
+            <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
+          </Label>
+          <Label as="span" color="black">
+            Next run:
+            <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
+          </Label>
+        </React.Fragment>
+      )}
+    </React.Fragment>
+  );
+}

ui/src/views/jobs/mutation/JobMutation.js

@@ -29,6 +29,7 @@ export default function JobMutator() {
 
   const [providerCreationVisible, setProviderCreationVisibility] = useState(false);
   const [notificationCreationVisible, setNotificationCreationVisibility] = useState(false);
+  const [editNotificationAdapter, setEditNotificationAdapter] = useState(null);
   const [providerData, setProviderData] = useState(defaultProviderData);
   const [name, setName] = useState(defaultName);
   const [blacklist, setBlacklist] = useState(defaultBlacklist);
@@ -83,11 +84,12 @@ export default function JobMutator() {
       });
       history.push('/jobs');
     } catch (Exception) {
-      console.error(Exception);
+      console.error(Exception.json.message);
+
       ctx.showToast({
         title: 'Error',
-        message: Exception,
-        delay: 35000,
+        message: Exception.json != null ? Exception.json.message : Exception,
+        delay: 8000,
         backgroundColor: '#db2828',
         color: '#fff',
       });
@@ -105,14 +107,25 @@ export default function JobMutator() {
         }}
       />
 
-      <NotificationAdapterMutator
-        visible={notificationCreationVisible}
-        onVisibilityChanged={(visible) => setNotificationCreationVisibility(visible)}
-        selected={providerData}
-        onData={(data) => {
-          setNotificationAdapterData([...notificationAdapterData, data]);
-        }}
-      />
+      {notificationCreationVisible && (
+        <NotificationAdapterMutator
+          visible={notificationCreationVisible}
+          onVisibilityChanged={(visible) => {
+            setEditNotificationAdapter(null);
+            setNotificationCreationVisibility(visible);
+          }}
+          selected={notificationAdapterData}
+          editNotificationAdapter={
+            editNotificationAdapter == null
+              ? null
+              : notificationAdapterData.find((adapter) => adapter.id === editNotificationAdapter)
+          }
+          onData={(data) => {
+            const oldData = [...notificationAdapterData].filter((o) => o.id !== data.id);
+            setNotificationAdapterData([...oldData, data]);
+          }}
+        />
+      )}
 
       <Headline text={jobToBeEdit ? 'Edit a Job' : 'Create a new Job'} />
       <Form className="jobMutation__form">
@@ -173,8 +186,13 @@ export default function JobMutator() {
           <NotificationAdapterTable
             notificationAdapter={notificationAdapterData}
             onRemove={(adapterId) => {
+              setEditNotificationAdapter(null);
               setNotificationAdapterData(notificationAdapterData.filter((adapter) => adapter.id !== adapterId));
             }}
+            onEdit={(adapterId) => {
+              setEditNotificationAdapter(adapterId);
+              setNotificationCreationVisibility(true);
+            }}
           />
         </div>
 

ui/src/views/jobs/mutation/JobMutation.less

@@ -22,7 +22,7 @@
   }
 
   &__separator{
-    background-color: #3a3a3a;
+    background-color: #2b2b2b;
     border-radius: 10px;
     padding: .8rem;
   }

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.js

@@ -42,15 +42,29 @@ const validate = (selectedAdapter) => {
   return [...new Set(results)];
 };
 
+function spreadPrefilledAdapterWithValues(prefilled, fields) {
+  if (prefilled != null && fields != null) {
+    Object.keys(fields).forEach((fieldKey) => {
+      prefilled.fields[fieldKey].value = fields[fieldKey];
+    });
+  }
+}
+
 export default function NotificationAdapterMutator({
   onVisibilityChanged,
   visible = false,
   selected = [],
+  editNotificationAdapter,
   onData,
 } = {}) {
   const adapter = useSelector((state) => state.notificationAdapter);
 
-  const [selectedAdapter, setSelectedAdapter] = useState(null);
+  const preFilledSelectedAdapter =
+    editNotificationAdapter == null ? null : adapter.find((a) => a.id === editNotificationAdapter.id);
+
+  spreadPrefilledAdapterWithValues(preFilledSelectedAdapter, editNotificationAdapter?.fields);
+
+  const [selectedAdapter, setSelectedAdapter] = useState(preFilledSelectedAdapter);
   const [validationMessage, setValidationMessage] = useState(null);
   const [successMessage, setSuccessMessage] = useState(null);
 
@@ -107,9 +121,12 @@ export default function NotificationAdapterMutator({
 
     setSelectedAdapter({
       ...selectedAdapter,
-      config: {
+      fields: {
         ...selectedAdapter.fields,
-        [key]: uiElement,
+        [key]: {
+          ...selectedAdapter.fields[key],
+          value,
+        },
       },
     });
   };
@@ -174,7 +191,7 @@ export default function NotificationAdapterMutator({
             There are multiple ways how Fredy can send new listings to you. Chose your weapon...
           </p>
           <Dropdown
-            placeholder="Select a notification adapteer"
+            placeholder="Select a notification adapter"
             className="providerMutator__fields"
             selection
             value={selectedAdapter == null ? '' : selectedAdapter.id}
@@ -187,7 +204,11 @@ export default function NotificationAdapterMutator({
                 };
               })
               //filter out those, that have already been selected
-              .filter((option) => selected.find((selectedOption) => selectedOption.id === option.key) == null)
+              .filter((option) =>
+                editNotificationAdapter != null
+                  ? true
+                  : selected.find((selectedOption) => selectedOption.id === option.key) == null
+              )
               .sort(sortAdapter)}
             onChange={(e, { value }) => {
               setSuccessMessage(null);
@@ -215,7 +236,7 @@ export default function NotificationAdapterMutator({
       </Modal.Content>
       <Modal.Actions>
         <Button
-          content="Try"
+          content="Try Notification Adapter"
           labelPosition="left"
           floated="left"
           icon="hand spock"

ui/src/views/jobs/mutation/components/provider/ProviderMutator.js

@@ -84,6 +84,11 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
             <br />
             When the search results are shown on the website, copy the url and paste it into the textfield below.
             <br />
+            <span style={{ color: '#ff0000' }}>
+              If you chose Immoscout as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              (See readme)
+            </span>
+            <br />
             <span style={{ color: '#ff0000' }}>
               Do not forget to sort the results by date before copying the url to Fredy, so that Fredy always captures
               the latest search results.

webpack.common.js

@@ -18,6 +18,7 @@ module.exports = {
     publicPath: '/',
     filename: 'fredy.bundle.js',
   },
+  performance: { hints: false },
   module: {
     rules: [
       {