correct version

* preparing scraping ant proxies

* adding general settings for scraping ant proxy

* retrying with new ui settings

CHANGELOG.md

@@ -1,3 +1,16 @@
+Newer release changelog see https://github.com/orangecoding/fredy/releases
+
+------------
+
+###### [V5.5.0]
+- Upgrading dependencies
+- fixing provider
+- allow multiple instances of 1 provider  
+- __BREAKING__: Minimum node version is now 16
+
+###### [V5.4.6]
+- Adding Instana node.js monitoring
+- 
 ###### [V5.4.5]
 - Adding Instana node.js monitoring 
 

README.md

@@ -81,12 +81,6 @@ If you need more than the 1000 API calls allowed per month, I'd suggest opting f
 
 See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
-### Monitoring
-
-_Fredy_ can be monitored by [Instana](https://www.instana.com). If you are interested, sign up for a free trial. This is totally optional of course :)  
-If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.   
-If you want to know more, head over to the [Instana docs](https://www.ibm.com/docs/en/obi/current?topic=technologies-monitoring-nodejs).
-
 # Docker   
 Use the Dockerfile in this repository to build an image.  
 

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}

index.js

@@ -31,33 +31,20 @@ setInterval(
 
     if (isDuringWorkingHoursOrNotSet) {
       config.lastRun = Date.now();
+      const fetchedProvider = provider
+        .filter((provider) => provider.endsWith('.js'))
+        .map((pro) => require(`${path}/${pro}`));
+
       jobStorage
         .getJobs()
         .filter((job) => job.enabled)
         .forEach((job) => {
-          const providerIds = job.provider.map((provider) => provider.id);
-
-          provider
-            .filter((provider) => provider.endsWith('.js'))
-            .map((pro) => require(`${path}/${pro}`))
-            .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
-            .forEach(async (pro) => {
-              const providerId = pro.metaInformation.id;
-              if (providerId == null || providerId.length === 0) {
-                throw new Error('Provider id must not be empty. => ' + pro);
-              }
-              const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
-              if (providerConfig == null) {
-                throw new Error(`Provider Config for provider with id ${providerId} not found.`);
-              }
-              pro.init(providerConfig, job.blacklist);
-              await new FredyRuntime(
-                pro.config,
-                job.notificationAdapter,
-                providerId,
-                job.id,
-                similarityCache
-              ).execute();
+          job.provider
+            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+            .forEach(async (prov) => {
+              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+              pro.init(prov, job.blacklist);
+              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
               setLastJobExecution(job.id);
             });
         });

lib/notification/adapter/telegram.js

@@ -2,6 +2,8 @@ const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
 const axios = require('axios');
 
+const MAX_ENTITIES_PER_CHUNK = 8;
+const RATE_LIMIT_INTERVAL = 1010;
 /**
  * splitting an array into chunks because Telegram only allows for messages up to
  * 4096 chars, thus we have to split messages into chunks
@@ -29,7 +31,7 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const jobName = job == null ? jobKey : job.name;
 
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
-  const chunks = arrayChunks(newListings, 3);
+  const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
 
   const promises = chunks.map((chunk) => {
     let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
@@ -40,11 +42,21 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
         '\n\n'
     );
 
-    return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
-      chat_id: chatId,
-      text: message,
-      parse_mode: 'HTML',
-      disable_web_page_preview: true,
+    /**
+     * This is to not break the rate limit. It is to only send 1 message per second
+     */
+    return new Promise((resolve, reject) => {
+      setTimeout(() => {
+        axios
+          .post(`https://api.telegram.org/bot${token}/sendMessage`, {
+            chat_id: chatId,
+            text: message,
+            parse_mode: 'HTML',
+            disable_web_page_preview: true,
+          })
+          .then(() => resolve())
+          .catch(() => reject());
+      }, RATE_LIMIT_INTERVAL);
     });
   });
 

lib/services/requestDriver.js

@@ -1,14 +1,21 @@
 const axios = require('axios');
-const axiosRetry = require('axios-retry');
+const config = require('../../conf/config.json');
 
-axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
+const { makeUrlResidential } = require('./scrapingAnt');
+//if ScrapingAnt got blocked, this http status is returned
+const BLOCKED_HTTP_STATUS = 423;
+const NOT_FOUND_HTTP_STATUS = 404;
+const MAX_RETRIES_SCRAPING_ANT = 10;
+const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
 
 function makeDriver(headers = {}) {
   let cookies = '';
 
-  return async function driver(context, callback) {
+  async function scrapingAntDriver(context, callback, retryCounter = 0) {
+    const proxyType = config.scrapingAnt?.proxy || 'datacenter';
+
     try {
-      const url = context.url;
+      const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
       const result = await axios({
         url,
         headers: {
@@ -17,15 +24,50 @@ function makeDriver(headers = {}) {
         },
       });
 
-      if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
-        //assume we have gotten a response from scrapingAnt
-        if (cookies.length === 0) {
-          cookies = result.data.cookies;
-        }
-        callback(null, result.data.content);
-      } else {
-        callback(null, result.data);
+      if (cookies.length === 0) {
+        cookies = result.data.cookies;
       }
+
+      callback(null, result.data.content);
+    } catch (exception) {
+      /* eslint-disable no-console */
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
+        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
+        callback(null, []);
+        return;
+      }
+
+      if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
+        retryCounter++;
+        console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
+        await scrapingAntDriver(context, callback, retryCounter);
+      } else {
+        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
+        callback(null, []);
+      }
+      /* eslint-enable no-console */
+    }
+  }
+
+  /**
+   * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
+   * everything != Immoscout as of writing this)
+   */
+  return async function driver(context, callback) {
+    if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
+      return scrapingAntDriver(context, callback);
+    }
+
+    try {
+      const result = await axios({
+        url: context.url,
+        headers: {
+          ...headers,
+          Cookie: cookies,
+        },
+      });
+
+      callback(null, result.data);
     } catch (exception) {
       console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
       callback(null, []);

lib/services/scrapingAnt.js

@@ -1,6 +1,5 @@
 const { metaInformation } = require('../provider/immoscout');
 //to better configure re-capture chose a random proxy each time we do a call
-const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
 const config = require('../../conf/config.json');
 
 const isImmoscout = (id) => {
@@ -8,13 +7,9 @@ const isImmoscout = (id) => {
 };
 
 exports.transformUrlForScrapingAnt = (url, id) => {
-  const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
-
   if (isImmoscout(id)) {
     //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(
-      url
-    )}&proxy_country=${randomProxy}&proxy_type=residential`;
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
   }
   return url;
 };
@@ -24,3 +19,7 @@ exports.isScrapingAntApiKeySet = () => {
 };
 
 exports.isImmoscout = isImmoscout;
+
+exports.makeUrlResidential = (url) => {
+  return url.replace('datacenter', 'residential');
+};

package.json

@@ -1,13 +1,17 @@
 {
   "name": "fredy",
-  "version": "5.4.8",
+  "version": "5.8.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
-    "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
+    "dev": "run-script-os",
+    "dev:win32": "yarn && set BUILD_DEV='true' && set NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
+    "dev:default": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
+    "prod": "run-script-os",
+    "prod:win32": "set BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
+    "prod:default": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
-    "test": "mocha --timeout 20000 test/**/*.test.js",
+    "test": "mocha --timeout 3000000 test/**/*.test.js",
     "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
   },
   "husky": {
@@ -43,7 +47,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=14.0.0",
+    "node": ">=16.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -55,61 +59,61 @@
   "dependencies": {
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.2",
-    "axios": "0.26.1",
-    "axios-retry": "^3.2.4",
-    "better-sqlite3": "^7.5.0",
-    "body-parser": "1.19.2",
+    "@sendgrid/mail": "7.7.0",
+    "axios": "0.27.2",
+    "better-sqlite3": "8.0.1",
+    "body-parser": "1.20.1",
     "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "10.0.0",
+    "highcharts": "10.3.2",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.3.1",
-    "node-mailjet": "3.3.7",
-    "query-string": "7.1.1",
+    "nanoid": "3.3.3",
+    "node-mailjet": "3.3.13",
+    "query-string": "7.1.3",
     "react": "17.0.2",
     "react-dom": "17.0.2",
-    "react-redux": "7.2.6",
+    "react-redux": "8.0.5",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "react-switch": "^6.0.0",
-    "redux": "4.1.2",
-    "redux-thunk": "2.4.1",
-    "restana": "4.9.3",
-    "semantic-ui-react": "2.1.2",
+    "react-switch": "6.0.0",
+    "redux": "4.2.0",
+    "redux-thunk": "2.4.2",
+    "restana": "4.9.7",
+    "semantic-ui-react": "2.1.4",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.17.8",
-    "@babel/preset-env": "7.16.11",
-    "@babel/preset-react": "7.16.7",
+    "@babel/core": "7.20.5",
+    "@babel/preset-env": "7.20.2",
+    "@babel/preset-react": "7.18.6",
     "babel-eslint": "10.1.0",
-    "babel-loader": "8.2.4",
-    "chai": "4.3.6",
+    "babel-loader": "8.2.5",
+    "chai": "4.3.7",
     "clean-webpack-plugin": "4.0.0",
     "copy-webpack-plugin": "10.2.4",
-    "css-loader": "6.7.1",
+    "css-loader": "6.7.2",
     "eslint": "7.32.0",
     "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-react": "7.29.3",
+    "eslint-plugin-react": "7.31.11",
     "file-loader": "6.2.0",
     "history": "5.3.0",
     "husky": "4.3.8",
-    "less": "4.1.2",
+    "less": "4.1.3",
     "less-loader": "10.2.0",
-    "lint-staged": "12.3.7",
+    "lint-staged": "12.4.1",
     "mocha": "9.2.2",
-    "prettier": "2.6.1",
+    "prettier": "2.8.1",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
+    "run-script-os": "^1.1.6",
     "style-loader": "3.3.1",
     "url-loader": "4.1.1",
-    "webpack": "5.70.0",
+    "webpack": "5.75.0",
     "webpack-cli": "4.9.2",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.8.0"

test/provider/einsAImmobilien.test.js

@@ -22,7 +22,7 @@ describe('#einsAImmobilien testsuite()', () => {
 
   it('should test einsAImmobilien provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
       fredy.execute().then((listings) => {
         expect(listings).to.be.a('array');
 
@@ -39,7 +39,6 @@ describe('#einsAImmobilien testsuite()', () => {
           expect(notify.link).to.be.a('string');
 
           /** check the values if possible **/
-          expect(notify.price).that.does.include('EUR');
           expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');

test/provider/immonet.test.js

@@ -21,7 +21,7 @@ describe('#immonet testsuite()', () => {
 
   it('should test immonet provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/immoscout.test.js

@@ -29,7 +29,7 @@ describe('#immoscout testsuite()', () => {
         return;
       }
 
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/immoswp.test.js

@@ -21,7 +21,7 @@ describe('#immoswp testsuite()', () => {
 
   it('should test immoswp provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/immowelt.test.js

@@ -20,7 +20,7 @@ describe('#immowelt testsuite()', () => {
     });
 
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/kleinanzeigen.test.js

@@ -20,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => {
     });
 
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 
@@ -31,10 +31,8 @@ describe('#kleinanzeigen testsuite()', () => {
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.price).to.be.a('string');
           expect(notify.address).to.be.a('string');
 
           /** check the values if possible **/

test/provider/neubauKompass.test.js

@@ -20,7 +20,7 @@ describe('#neubauKompass testsuite()', () => {
 
   it('should test neubauKompass provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/wgGesucht.test.js

@@ -20,7 +20,7 @@ describe('#wgGesucht testsuite()', () => {
 
   it('should test wgGesucht provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
         const notificationObj = mockNotification.get();

ui/src/App.less

@@ -10,4 +10,12 @@
     background-color: #3f3e3ef5;
     color: #f1f1f1;
   }
+}
+
+.ui.inverted.segment{
+  background: #31303078!important;
+}
+
+.ui.black.label, .ui.black.labels .label {
+  background-color: #31303078!important;
 }

ui/src/components/menu/Menu.less

@@ -4,7 +4,7 @@
   &__active {
     border-bottom: 1px solid #06dcfff2 !important;
     font-weight: 550 !important;
-    color: #78e5ff !important;
+    color: #3ed7ff !important;
     margin: 0 0 -1px !important;
   }
 

ui/src/components/segment/SegmentPart.js

@@ -0,0 +1,27 @@
+import React from 'react';
+import { Header, Icon, Popup, Segment } from 'semantic-ui-react';
+
+import './SegmentParts.less';
+
+export const SegmentPart = ({ name, icon = null, children, helpText }) => (
+  <Segment inverted>
+    <Header as="h5" inverted sub>
+      {icon && <Icon name={icon} inverted size="mini" />}
+      <Header.Content>{name}</Header.Content>
+    </Header>
+
+    <Popup
+      content={helpText}
+      trigger={
+        <span className="generalSettings__help">
+          {' '}
+          <Icon name="help circle" inverted />
+          What is this?
+        </span>
+      }
+    />
+    <Segment inverted className="segmentParts">
+      {children}
+    </Segment>
+  </Segment>
+);

ui/src/components/segment/SegmentParts.less

@@ -0,0 +1,4 @@
+.segmentParts {
+  border: 1px solid #323232 !important;
+  border-radius: 5px !important;
+}

ui/src/views/generalSettings/GeneralSettings.js

@@ -2,36 +2,13 @@ import React from 'react';
 
 import { useDispatch, useSelector } from 'react-redux';
 
-import { Button, Form, Header, Icon, Message, Popup, Segment } from 'semantic-ui-react';
+import { Button, Form, Icon, Message, Segment, Radio } from 'semantic-ui-react';
 import ToastContext from '../../components/toasts/ToastContext';
 import Headline from '../../components/headline/Headline';
 import { xhrPost } from '../../services/xhr';
-
+import { SegmentPart } from '../../components/segment/SegmentPart';
 import './GeneralSettings.less';
 
-const SegmentPart = ({ name, icon, children, helpText }) => (
-  <React.Fragment>
-    <Header as="h5" inverted attached="top" sub>
-      <Icon name={icon} inverted size="mini" />
-      <Header.Content>{name}</Header.Content>
-    </Header>
-
-    <Popup
-      content={helpText}
-      trigger={
-        <span className="generalSettings__help">
-          {' '}
-          <Icon name="help circle" inverted />
-          What is this?
-        </span>
-      }
-    />
-    <Segment inverted attached>
-      {children}
-    </Segment>
-  </React.Fragment>
-);
-
 const GeneralSettings = function Users() {
   const dispatch = useDispatch();
   const [loading, setLoading] = React.useState(true);
@@ -41,6 +18,7 @@ const GeneralSettings = function Users() {
   const [interval, setInterval] = React.useState('');
   const [port, setPort] = React.useState('');
   const [scrapingAntApiKey, setScrapingAntApiKey] = React.useState('');
+  const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
   const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
   const [workingHourTo, setWorkingHourTo] = React.useState(null);
   const ctx = React.useContext(ToastContext);
@@ -56,6 +34,7 @@ const GeneralSettings = function Users() {
     setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
     setWorkingHourFrom(settings?.workingHours?.from);
     setWorkingHourTo(settings?.workingHours?.to);
+    setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
   }, [settings]);
 
   const nullOrEmpty = (val) => val == null || val.length === 0;
@@ -92,6 +71,7 @@ const GeneralSettings = function Users() {
         port,
         scrapingAnt: {
           apiKey: scrapingAntApiKey,
+          proxy: scrapingAntProxy,
         },
         workingHours: {
           from: workingHourFrom,
@@ -111,7 +91,7 @@ const GeneralSettings = function Users() {
       {!loading && (
         <React.Fragment>
           <Headline text="General Settings" />
-          <Message info>
+          <Message className="generalSettings__message">
             <h5>
               <Icon name="info circle" />
               Info
@@ -167,6 +147,48 @@ const GeneralSettings = function Users() {
               />
             </SegmentPart>
 
+            <SegmentPart
+              name="ScrapingAnt proxy settings"
+              helpText="Scraping ant provides different proxies."
+              icon="key"
+            >
+              <Message info>
+                ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies.{' '}
+                <br />
+                <h4>Datacenter-Proxy</h4>
+                Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and more
+                likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
+                <h4>Residential-Proxy</h4>
+                High-quality proxy server located in one of the real people houses across the world. Datacenter proxies
+                are faster and more likely to success, but they are more expensive. A call with a datacenter proxy cost
+                250 credits.
+                <br />
+                <br />
+                <b>
+                  On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only successful
+                  calls will be charged.
+                </b>
+              </Message>
+              <Form.Field>
+                <Radio
+                  label="Datacenter proxy"
+                  name="scrapingAntProxy"
+                  value="datacenter"
+                  checked={scrapingAntProxy === 'datacenter'}
+                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                />
+              </Form.Field>
+              <Form.Field>
+                <Radio
+                  label="Residential proxy"
+                  name="scrapingAntProxy"
+                  value="residential"
+                  checked={scrapingAntProxy === 'residential'}
+                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                />
+              </Form.Field>
+            </SegmentPart>
+
             <SegmentPart
               name="Working hours"
               helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
@@ -176,7 +198,7 @@ const GeneralSettings = function Users() {
                 <Form.Input
                   className="generalSettings__time"
                   type="time"
-                  placeholder="ScrapingAnt Api Key"
+                  placeholder="Working hours from"
                   inverted
                   size="mini"
                   width={2}
@@ -186,7 +208,7 @@ const GeneralSettings = function Users() {
                 <div className="generalSettings__until">until</div>
                 <Form.Input
                   type="time"
-                  placeholder="ScrapingAnt Api Key"
+                  placeholder="Working hours to"
                   inverted
                   size="mini"
                   width={2}

ui/src/views/generalSettings/GeneralSettings.less

@@ -14,4 +14,8 @@
     margin-left: 1rem;
   }
 
+  &__message{
+    background: #60c5df!important;
+  }
+
 }

ui/src/views/jobs/ProcessingTimes.js

@@ -37,9 +37,13 @@ export default function ProcessingTimes({ processingTimes }) {
               {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
             </Message.Item>
           </Message.List>
-          If you want to scrape Immoscout more often, you have to purchase a premium account of ScrapingAnt. You can use
-          the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to recommend
-          ScrapingAnt.
+          If you want to scrape Immoscout more often, you have to purchase a premium account of{' '}
+          <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
+            {' '}
+            ScrapingAnt
+          </a>
+          . You can use the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to
+          recommend ScrapingAnt.)
         </Segment>
       )}
     </React.Fragment>

ui/src/views/jobs/mutation/JobMutation.js

@@ -2,7 +2,7 @@ import React, { Fragment, useState } from 'react';
 
 import NotificationAdapterMutator from './components/notificationAdapter/NotificationAdapterMutator';
 import NotificationAdapterTable from '../../../components/table/NotificationAdapterTable';
-import { Header, Icon, Form, Popup, Button, Label } from 'semantic-ui-react';
+import { Icon, Form, Button, Label } from 'semantic-ui-react';
 import ProviderTable from '../../../components/table/ProviderTable';
 import ProviderMutator from './components/provider/ProviderMutator';
 import ToastContext from '../../../components/toasts/ToastContext';
@@ -14,6 +14,7 @@ import { useParams } from 'react-router';
 
 import './JobMutation.less';
 import Switch from 'react-switch';
+import { SegmentPart } from '../../../components/segment/SegmentPart';
 
 export default function JobMutator() {
   const jobs = useSelector((state) => state.jobs.jobs);
@@ -39,27 +40,6 @@ export default function JobMutator() {
   const dispatch = useDispatch();
   const ctx = React.useContext(ToastContext);
 
-  const header = (name, icon) => (
-    <Header as="h5" inverted>
-      <Icon name={icon} inverted />
-      {name}
-    </Header>
-  );
-
-  const help = (helpText) => (
-    <div>
-      <Popup
-        content={helpText}
-        trigger={
-          <Header as="h6" inverted>
-            <Icon name="help circle" inverted />
-            What is this?
-          </Header>
-        }
-      />
-    </div>
-  );
-
   const isSavingEnabled = () => {
     return notificationAdapterData.length > 0 && providerData.length > 0 && name != null && name.length > 0;
   };
@@ -101,7 +81,6 @@ export default function JobMutator() {
       <ProviderMutator
         visible={providerCreationVisible}
         onVisibilityChanged={(visible) => setProviderCreationVisibility(visible)}
-        selected={providerData}
         onData={(data) => {
           setProviderData([...providerData, data]);
         }}
@@ -128,8 +107,8 @@ export default function JobMutator() {
       )}
 
       <Headline text={jobToBeEdit ? 'Edit a Job' : 'Create a new Job'} />
-      <Form className="jobMutation__form">
-        <div className="jobMutation__block">
+      <Form>
+        <SegmentPart name="Name">
           <Form.Input
             type="text"
             maxLength={40}
@@ -140,48 +119,43 @@ export default function JobMutator() {
             defaultValue={name}
             onChange={(e) => setName(e.target.value)}
           />
-        </div>
+        </SegmentPart>
 
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Provider', 'briefcase')}
+        <SegmentPart
+          name="Provider"
+          icon="briefcase"
+          helpText={
+            'A provider is essentially the service (Immowelt etc.) that Fredy is using to search for new listings. When adding a new provider, Fredy will open a new tab pointing ' +
+            'to the website of this provider. You have to adjust your search parameter and click on "Search". If the results are being shown, copy the browser url. This is the url, Fredy will use ' +
+            'to search for new listings.'
+          }
+        >
+          <Form.Button primary className="jobMutation__newButton" onClick={() => setProviderCreationVisibility(true)}>
+            <Icon name="plus" />
+            Add new Provider
+          </Form.Button>
 
-          <div className="jobMutation__helpContainer">
-            {help(
-              'A provider is essentially the service (Immowelt etc.) that Fredy is using to search for new listings. When adding a new provider, Fredy will open a new tab pointing ' +
-                'to the website of this provider. You have to adjust your search parameter and click on "Search". If the results are being shown, copy the browser url. This is the url, Fredy will use ' +
-                'to search for new listings.'
-            )}
-
-            <Form.Button primary className="jobMutation__newButton" onClick={() => setProviderCreationVisibility(true)}>
-              <Icon name="plus" />
-              Add new Provider
-            </Form.Button>
-          </div>
           <ProviderTable
             providerData={providerData}
             onRemove={(providerId) => {
               setProviderData(providerData.filter((provider) => provider.id !== providerId));
             }}
           />
-        </div>
+        </SegmentPart>
 
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Notification Adapter', 'bell')}
-
-          <div className="jobMutation__helpContainer">
-            {help(
-              'Fredy supports multiple ways to notify you about new findings. These are called notification adapter. You can chose between email, Telegram etc.'
-            )}
-
-            <Form.Button
-              primary
-              className="jobMutation__newButton"
-              onClick={() => setNotificationCreationVisibility(true)}
-            >
-              <Icon name="plus" />
-              Add new Notification Adapter
-            </Form.Button>
-          </div>
+        <SegmentPart
+          icon="bell"
+          name="Notification Adapter"
+          helpText="Fredy supports multiple ways to notify you about new findings. These are called notification adapter. You can chose between email, Telegram etc."
+        >
+          <Form.Button
+            primary
+            className="jobMutation__newButton"
+            onClick={() => setNotificationCreationVisibility(true)}
+          >
+            <Icon name="plus" />
+            Add new Notification Adapter
+          </Form.Button>
 
           <NotificationAdapterTable
             notificationAdapter={notificationAdapterData}
@@ -194,20 +168,15 @@ export default function JobMutator() {
               setNotificationCreationVisibility(true);
             }}
           />
-        </div>
-
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Blacklist', 'bell')}
-
-          <div className="jobMutation__helpContainer">
-            {help(
-              'If a listing contains one of these words, it will be filtered out. Words must be comma separated. To remove a word from the black list, just click the red label(s).'
-            )}
-          </div>
+        </SegmentPart>
 
+        <SegmentPart
+          icon="bell"
+          name="Blacklist"
+          helpText="If a listing contains one of these words, it will be filtered out. Words must be comma separated. To remove a word from the black list, just click the red label(s)."
+        >
           <Form.Input
             type="text"
-            className="jobMutation__spaceTop"
             maxLength={40}
             placeholder="Comma separated list of blacklisted words"
             autoFocus
@@ -232,19 +201,15 @@ export default function JobMutator() {
               color="red"
             />
           ))}
-        </div>
-
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Job activation', 'play circle outline')}
-
-          <div className="jobMutation__helpContainer">
-            {help(
-              'Whether or not the job is activated. If it is not activated, it will be ignored when Fredy checks for new listings.'
-            )}
-          </div>
+        </SegmentPart>
 
+        <SegmentPart
+          icon="play circle outline"
+          name="Job activation"
+          helpText="Whether or not the job is activated. If it is not activated, it will be ignored when Fredy checks for new listings."
+        >
           <Switch className="jobMutation__spaceTop" onChange={(checked) => setEnabled(checked)} checked={enabled} />
-        </div>
+        </SegmentPart>
 
         <Button color="red" onClick={() => history.push('/jobs')}>
           Cancel

ui/src/views/jobs/mutation/JobMutation.less

@@ -1,29 +1,5 @@
 .jobMutation {
-
-  &__form {
-    margin-top:2rem;
-  }
-
-  &__block {
-    margin-bottom: 2rem;
-  }
   &__newButton{
     float: right;
   }
-
-  &__helpContainer {
-    display: flex;
-    justify-content: space-between;
-    align-items: flex-end;
-  }
-
-  &__spaceTop{
-    margin-top:1rem !important;
-  }
-
-  &__separator{
-    background-color: #2b2b2b;
-    border-radius: 10px;
-    padding: .8rem;
-  }
 }

ui/src/views/jobs/mutation/components/provider/ProviderMutator.js

@@ -16,7 +16,7 @@ const sortProvider = (a, b) => {
   return 0;
 };
 
-export default function ProviderMutator({ onVisibilityChanged, visible = false, selected = [], onData } = {}) {
+export default function ProviderMutator({ onVisibilityChanged, visible = false, onData } = {}) {
   const provider = useSelector((state) => state.provider);
   const [selectedProvider, setSelectedProvider] = useState(null);
   const [providerUrl, setProviderUrl] = useState(null);
@@ -107,8 +107,6 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
                   text: pro.name,
                 };
               })
-              //filter out those, that have already been selected
-              .filter((option) => selected.find((selectedOption) => selectedOption.id === option.key) == null)
               .sort(sortProvider)}
             onChange={(e, { value }) => {
               const selectedProvider = provider.find((pro) => pro.id === value);

ui/src/views/user/mutation/UserMutator.js

@@ -8,6 +8,7 @@ import { useDispatch } from 'react-redux';
 import Switch from 'react-switch';
 
 import './UserMutator.less';
+import { SegmentPart } from '../../../components/segment/SegmentPart';
 
 const UserMutator = function UserMutator() {
   const params = useParams();
@@ -69,40 +70,47 @@ const UserMutator = function UserMutator() {
 
   return (
     <Form inverted className="userMutator">
-      <Form.Input
-        type="text"
-        label="Username"
-        maxLength={30}
-        placeholder="Username"
-        autoFocus
-        inverted
-        width={6}
-        defaultValue={username}
-        onChange={(e) => setUsername(e.target.value)}
-      />
-      <Form.Input
-        type="password"
-        label="Password"
-        placeholder="Password"
-        inverted
-        width={6}
-        defaultValue={password}
-        onChange={(e) => setPassword(e.target.value)}
-      />
-      <Form.Input
-        type="password"
-        label="Retype password"
-        placeholder="Retype password"
-        inverted
-        width={6}
-        defaultValue={password2}
-        onChange={(e) => setPassword2(e.target.value)}
-      />
-      <Form.Field>
-        <label>Is user an admin?</label>
-        <Switch checked={isAdmin} onChange={(checked) => setIsAdmin(checked)} />
-      </Form.Field>
-
+      <SegmentPart name="Username" helpText="The username used to login to Fredy">
+        <Form.Input
+          type="text"
+          label="Username"
+          maxLength={30}
+          placeholder="Username"
+          autoFocus
+          inverted
+          width={6}
+          defaultValue={username}
+          onChange={(e) => setUsername(e.target.value)}
+        />
+      </SegmentPart>
+      <SegmentPart name="Password" helpText="The password used to login to Fredy">
+        <Form.Input
+          type="password"
+          label="Password"
+          placeholder="Password"
+          inverted
+          width={6}
+          defaultValue={password}
+          onChange={(e) => setPassword(e.target.value)}
+        />
+      </SegmentPart>
+      <SegmentPart name="Retype password" helpText="Retype the password to make sure they match">
+        <Form.Input
+          type="password"
+          label="Retype password"
+          placeholder="Retype password"
+          inverted
+          width={6}
+          defaultValue={password2}
+          onChange={(e) => setPassword2(e.target.value)}
+        />
+      </SegmentPart>
+      <SegmentPart name="Admin use" helpText="Check this if the user is an administrator">
+        <Form.Field>
+          <label>Is user an admin?</label>
+          <Switch checked={isAdmin} onChange={(checked) => setIsAdmin(checked)} />
+        </Form.Field>
+      </SegmentPart>
       <Button color="red" onClick={() => history.push('/users')}>
         Cancel
       </Button>