Scrapingant proxies (#59)

* preparing scraping ant proxies

* adding general settings for scraping ant proxy

* retrying with new ui settings

.env

@@ -1 +0,0 @@
-INSTANA_MONITORING=false

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

CHANGELOG.md

@@ -1,3 +1,10 @@
+Newer release changelog see https://github.com/orangecoding/fredy/releases
+
+------------
+
+###### [V5.4.6]
+- Adding Instana node.js monitoring
+- 
 ###### [V5.4.5]
 - Adding Instana node.js monitoring 
 

README.md

@@ -81,12 +81,6 @@ If you need more than the 1000 API calls allowed per month, I'd suggest opting f
 
 See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
-### Monitoring
-
-_Fredy_ can be monitored by [Instana](https://www.instana.com). If you are interested, sign up for a free trial. This is totally optional of course :)  
-If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.   
-If you want to know more, head over to the [Instana docs](https://www.ibm.com/docs/en/obi/current?topic=technologies-monitoring-nodejs).
-
 # Docker   
 Use the Dockerfile in this repository to build an image.  
 

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}

index.js

@@ -1,16 +1,3 @@
-require('dotenv').config();
-/********OPTIONAL INSTANA INITIALIZATION BEGIN********/
-//if you want to use Instana to monitor fredy, go to https://www.instana.com and
-// try it yourself by signing up for a free trial
-const { INSTANA_MONITORING } = process.env;
-if (INSTANA_MONITORING != null && INSTANA_MONITORING === 'true') {
-  /* eslint-disable no-console */
-  console.debug('Starting Instana monitoring');
-  /* eslint-enable no-console */
-  require('@instana/collector')();
-}
-/********OPTIONAL INSTANA INITIALIZATION END********/
-
 const fs = require('fs');
 
 //if db folder does not exist, ensure to create it before loading anything else

lib/api/routes/jobRouter.js

@@ -1,10 +1,10 @@
 const service = require('restana')();
 const jobRouter = service.newRouter();
+const axios = require('axios');
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
 const immoscoutProvider = require('../../provider/immoscout');
 const config = require('../../../conf/config.json');
-
 const { isAdmin } = require('../security');
 
 function doesJobBelongsToUser(job, req) {
@@ -30,9 +30,23 @@ jobRouter.get('/', async (req, res) => {
 });
 
 jobRouter.get('/processingTimes', async (req, res) => {
+  let scrapingAntData = null;
+
+  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+    try {
+      const result = await axios({
+        url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
+      });
+      scrapingAntData = result.data;
+    } catch (Exception) {
+      console.error('Could not query plan data from scraping ant.', Exception);
+    }
+  }
+
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
+    scrapingAntData,
   };
 
   res.send();

lib/notification/adapter/telegram.js

@@ -2,6 +2,8 @@ const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
 const axios = require('axios');
 
+const MAX_ENTITIES_PER_CHUNK = 8;
+const RATE_LIMIT_INTERVAL = 1010;
 /**
  * splitting an array into chunks because Telegram only allows for messages up to
  * 4096 chars, thus we have to split messages into chunks
@@ -29,7 +31,7 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const jobName = job == null ? jobKey : job.name;
 
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
-  const chunks = arrayChunks(newListings, 3);
+  const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
 
   const promises = chunks.map((chunk) => {
     let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
@@ -40,11 +42,21 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
         '\n\n'
     );
 
-    return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
-      chat_id: chatId,
-      text: message,
-      parse_mode: 'HTML',
-      disable_web_page_preview: true,
+    /**
+     * This is to not break the rate limit. It is to only send 1 message per second
+     */
+    return new Promise((resolve, reject) => {
+      setTimeout(() => {
+        axios
+          .post(`https://api.telegram.org/bot${token}/sendMessage`, {
+            chat_id: chatId,
+            text: message,
+            parse_mode: 'HTML',
+            disable_web_page_preview: true,
+          })
+          .then(() => resolve())
+          .catch(() => reject());
+      }, RATE_LIMIT_INTERVAL);
     });
   });
 

lib/services/requestDriver.js

@@ -1,14 +1,21 @@
 const axios = require('axios');
-const axiosRetry = require('axios-retry');
+const config = require('../../conf/config.json');
 
-axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
+const { makeUrlResidential } = require('./scrapingAnt');
+//if ScrapingAnt got blocked, this http status is returned
+const BLOCKED_HTTP_STATUS = 423;
+const NOT_FOUND_HTTP_STATUS = 404;
+const MAX_RETRIES_SCRAPING_ANT = 10;
+const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
 
 function makeDriver(headers = {}) {
   let cookies = '';
 
-  return async function driver(context, callback) {
+  async function scrapingAntDriver(context, callback, retryCounter = 0) {
+    const proxyType = config.scrapingAnt?.proxy || 'datacenter';
+
     try {
-      const url = context.url;
+      const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
       const result = await axios({
         url,
         headers: {
@@ -17,15 +24,50 @@ function makeDriver(headers = {}) {
         },
       });
 
-      if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
-        //assume we have gotten a response from scrapingAnt
-        if (cookies.length === 0) {
-          cookies = result.data.cookies;
-        }
-        callback(null, result.data.content);
-      } else {
-        callback(null, result.data);
+      if (cookies.length === 0) {
+        cookies = result.data.cookies;
       }
+
+      callback(null, result.data.content);
+    } catch (exception) {
+      /* eslint-disable no-console */
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
+        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
+        callback(null, []);
+        return;
+      }
+
+      if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
+        retryCounter++;
+        console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
+        await scrapingAntDriver(context, callback, retryCounter);
+      } else {
+        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
+        callback(null, []);
+      }
+      /* eslint-enable no-console */
+    }
+  }
+
+  /**
+   * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
+   * everything != Immoscout as of writing this)
+   */
+  return async function driver(context, callback) {
+    if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
+      return scrapingAntDriver(context, callback);
+    }
+
+    try {
+      const result = await axios({
+        url: context.url,
+        headers: {
+          ...headers,
+          Cookie: cookies,
+        },
+      });
+
+      callback(null, result.data);
     } catch (exception) {
       console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
       callback(null, []);

lib/services/scrapingAnt.js

@@ -1,6 +1,5 @@
 const { metaInformation } = require('../provider/immoscout');
-//to better confure re-capture chose a random proxy each time we do a call
-const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
+//to better configure re-capture chose a random proxy each time we do a call
 const config = require('../../conf/config.json');
 
 const isImmoscout = (id) => {
@@ -8,11 +7,9 @@ const isImmoscout = (id) => {
 };
 
 exports.transformUrlForScrapingAnt = (url, id) => {
-  const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
-
   if (isImmoscout(id)) {
     //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
   }
   return url;
 };
@@ -22,3 +19,7 @@ exports.isScrapingAntApiKeySet = () => {
 };
 
 exports.isImmoscout = isImmoscout;
+
+exports.makeUrlResidential = (url) => {
+  return url.replace('datacenter', 'residential');
+};

package.json

@@ -1,13 +1,17 @@
 {
   "name": "fredy",
-  "version": "5.4.6",
+  "version": "5.7.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
-    "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
+    "dev": "run-script-os",
+    "dev:win32": "yarn && set BUILD_DEV='true' && set NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
+    "dev:default": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
+    "prod": "run-script-os",
+    "prod:win32": "set BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
+    "prod:default": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
-    "test": "mocha --timeout 20000 test/**/*.test.js",
+    "test": "mocha --timeout 3000000 test/**/*.test.js",
     "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
   },
   "husky": {
@@ -43,8 +47,8 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=12.13.0",
-    "npm": ">=6.0.0"
+    "node": ">=14.0.0",
+    "npm": ">=7.0.0"
   },
   "browserslist": [
     "> 0.5%",
@@ -53,66 +57,64 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@instana/collector": "^1.137.5",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.0",
-    "axios": "0.24.0",
-    "axios-retry": "^3.2.4",
-    "better-sqlite3": "^7.5.0",
-    "body-parser": "1.19.0",
-    "cookie-session": "1.4.0",
-    "dotenv": "^15.0.0",
+    "@sendgrid/mail": "7.6.2",
+    "axios": "0.27.2",
+    "better-sqlite3": "7.5.1",
+    "body-parser": "1.20.0",
+    "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.3.1",
+    "highcharts": "10.0.0",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.1.30",
-    "node-mailjet": "3.3.4",
-    "query-string": "^7.0.1",
+    "nanoid": "3.3.3",
+    "node-mailjet": "3.3.13",
+    "query-string": "7.1.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
-    "react-redux": "7.2.6",
+    "react-redux": "8.0.1",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
     "react-switch": "^6.0.0",
-    "redux": "4.1.2",
-    "redux-thunk": "2.4.0",
-    "restana": "4.9.2",
-    "semantic-ui-react": "2.0.4",
-    "serve-static": "^1.14.1",
+    "redux": "4.2.0",
+    "redux-thunk": "2.4.1",
+    "restana": "4.9.4",
+    "semantic-ui-react": "2.1.2",
+    "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.16.0",
-    "@babel/preset-env": "7.16.4",
-    "@babel/preset-react": "7.16.0",
+    "@babel/core": "7.17.9",
+    "@babel/preset-env": "7.16.11",
+    "@babel/preset-react": "7.16.7",
     "babel-eslint": "10.1.0",
-    "babel-loader": "8.2.3",
-    "chai": "4.3.4",
+    "babel-loader": "8.2.5",
+    "chai": "4.3.6",
     "clean-webpack-plugin": "4.0.0",
-    "copy-webpack-plugin": "10.0.0",
-    "css-loader": "6.5.1",
+    "copy-webpack-plugin": "10.2.4",
+    "css-loader": "6.7.1",
     "eslint": "7.32.0",
-    "eslint-config-prettier": "8.3.0",
-    "eslint-plugin-react": "7.27.1",
+    "eslint-config-prettier": "8.5.0",
+    "eslint-plugin-react": "7.29.4",
     "file-loader": "6.2.0",
-    "history": "5.1.0",
+    "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.2",
     "less-loader": "10.2.0",
-    "lint-staged": "12.1.2",
-    "mocha": "9.1.3",
-    "prettier": "2.5.0",
+    "lint-staged": "12.4.1",
+    "mocha": "9.2.2",
+    "prettier": "2.6.2",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
+    "run-script-os": "^1.1.6",
     "style-loader": "3.3.1",
     "url-loader": "4.1.1",
-    "webpack": "5.64.4",
-    "webpack-cli": "4.9.1",
+    "webpack": "5.72.0",
+    "webpack-cli": "4.9.2",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.8.0"
   }

test/provider/einsAImmobilien.test.js

@@ -39,7 +39,6 @@ describe('#einsAImmobilien testsuite()', () => {
           expect(notify.link).to.be.a('string');
 
           /** check the values if possible **/
-          expect(notify.price).that.does.include('EUR');
           expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');

ui/src/App.less

@@ -10,4 +10,12 @@
     background-color: #3f3e3ef5;
     color: #f1f1f1;
   }
+}
+
+.ui.inverted.segment{
+  background: #31303078!important;
+}
+
+.ui.black.label, .ui.black.labels .label {
+  background-color: #31303078!important;
 }

ui/src/components/menu/Menu.less

@@ -4,7 +4,7 @@
   &__active {
     border-bottom: 1px solid #06dcfff2 !important;
     font-weight: 550 !important;
-    color: #78e5ff !important;
+    color: #3ed7ff !important;
     margin: 0 0 -1px !important;
   }
 

ui/src/components/segment/SegmentPart.js

@@ -0,0 +1,27 @@
+import React from 'react';
+import { Header, Icon, Popup, Segment } from 'semantic-ui-react';
+
+import './SegmentParts.less';
+
+export const SegmentPart = ({ name, icon = null, children, helpText }) => (
+  <Segment inverted>
+    <Header as="h5" inverted sub>
+      {icon && <Icon name={icon} inverted size="mini" />}
+      <Header.Content>{name}</Header.Content>
+    </Header>
+
+    <Popup
+      content={helpText}
+      trigger={
+        <span className="generalSettings__help">
+          {' '}
+          <Icon name="help circle" inverted />
+          What is this?
+        </span>
+      }
+    />
+    <Segment inverted className="segmentParts">
+      {children}
+    </Segment>
+  </Segment>
+);

ui/src/components/segment/SegmentParts.less

@@ -0,0 +1,4 @@
+.segmentParts {
+  border: 1px solid #323232 !important;
+  border-radius: 5px !important;
+}

ui/src/views/generalSettings/GeneralSettings.js

@@ -2,36 +2,13 @@ import React from 'react';
 
 import { useDispatch, useSelector } from 'react-redux';
 
-import { Button, Form, Header, Icon, Message, Popup, Segment } from 'semantic-ui-react';
+import { Button, Form, Icon, Message, Segment, Radio } from 'semantic-ui-react';
 import ToastContext from '../../components/toasts/ToastContext';
 import Headline from '../../components/headline/Headline';
 import { xhrPost } from '../../services/xhr';
-
+import { SegmentPart } from '../../components/segment/SegmentPart';
 import './GeneralSettings.less';
 
-const SegmentPart = ({ name, icon, children, helpText }) => (
-  <React.Fragment>
-    <Header as="h5" inverted attached="top" sub>
-      <Icon name={icon} inverted size="mini" />
-      <Header.Content>{name}</Header.Content>
-    </Header>
-
-    <Popup
-      content={helpText}
-      trigger={
-        <span className="generalSettings__help">
-          {' '}
-          <Icon name="help circle" inverted />
-          What is this?
-        </span>
-      }
-    />
-    <Segment inverted attached>
-      {children}
-    </Segment>
-  </React.Fragment>
-);
-
 const GeneralSettings = function Users() {
   const dispatch = useDispatch();
   const [loading, setLoading] = React.useState(true);
@@ -41,6 +18,7 @@ const GeneralSettings = function Users() {
   const [interval, setInterval] = React.useState('');
   const [port, setPort] = React.useState('');
   const [scrapingAntApiKey, setScrapingAntApiKey] = React.useState('');
+  const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
   const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
   const [workingHourTo, setWorkingHourTo] = React.useState(null);
   const ctx = React.useContext(ToastContext);
@@ -56,6 +34,7 @@ const GeneralSettings = function Users() {
     setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
     setWorkingHourFrom(settings?.workingHours?.from);
     setWorkingHourTo(settings?.workingHours?.to);
+    setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
   }, [settings]);
 
   const nullOrEmpty = (val) => val == null || val.length === 0;
@@ -92,6 +71,7 @@ const GeneralSettings = function Users() {
         port,
         scrapingAnt: {
           apiKey: scrapingAntApiKey,
+          proxy: scrapingAntProxy,
         },
         workingHours: {
           from: workingHourFrom,
@@ -111,7 +91,7 @@ const GeneralSettings = function Users() {
       {!loading && (
         <React.Fragment>
           <Headline text="General Settings" />
-          <Message info>
+          <Message className="generalSettings__message">
             <h5>
               <Icon name="info circle" />
               Info
@@ -167,6 +147,48 @@ const GeneralSettings = function Users() {
               />
             </SegmentPart>
 
+            <SegmentPart
+              name="ScrapingAnt proxy settings"
+              helpText="Scraping ant provides different proxies."
+              icon="key"
+            >
+              <Message info>
+                ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies.{' '}
+                <br />
+                <h4>Datacenter-Proxy</h4>
+                Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and more
+                likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
+                <h4>Residential-Proxy</h4>
+                High-quality proxy server located in one of the real people houses across the world. Datacenter proxies
+                are faster and more likely to success, but they are more expensive. A call with a datacenter proxy cost
+                250 credits.
+                <br />
+                <br />
+                <b>
+                  On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only successful
+                  calls will be charged.
+                </b>
+              </Message>
+              <Form.Field>
+                <Radio
+                  label="Datacenter proxy"
+                  name="scrapingAntProxy"
+                  value="datacenter"
+                  checked={scrapingAntProxy === 'datacenter'}
+                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                />
+              </Form.Field>
+              <Form.Field>
+                <Radio
+                  label="Residential proxy"
+                  name="scrapingAntProxy"
+                  value="residential"
+                  checked={scrapingAntProxy === 'residential'}
+                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                />
+              </Form.Field>
+            </SegmentPart>
+
             <SegmentPart
               name="Working hours"
               helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
@@ -176,7 +198,7 @@ const GeneralSettings = function Users() {
                 <Form.Input
                   className="generalSettings__time"
                   type="time"
-                  placeholder="ScrapingAnt Api Key"
+                  placeholder="Working hours from"
                   inverted
                   size="mini"
                   width={2}
@@ -186,7 +208,7 @@ const GeneralSettings = function Users() {
                 <div className="generalSettings__until">until</div>
                 <Form.Input
                   type="time"
-                  placeholder="ScrapingAnt Api Key"
+                  placeholder="Working hours to"
                   inverted
                   size="mini"
                   width={2}

ui/src/views/generalSettings/GeneralSettings.less

@@ -14,4 +14,8 @@
     margin-left: 1rem;
   }
 
+  &__message{
+    background: #60c5df!important;
+  }
+
 }

ui/src/views/jobs/ProcessingTimes.js

@@ -1,25 +1,50 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import { Label } from 'semantic-ui-react';
+import { Header, Label, Message, Segment } from 'semantic-ui-react';
 
 export default function ProcessingTimes({ processingTimes }) {
   return (
     <React.Fragment>
-      <Label as="span" color="black">
-        Processing Interval:
-        <Label.Detail>{processingTimes.interval} min</Label.Detail>
-      </Label>
-      {processingTimes.lastRun && (
-        <React.Fragment>
-          <Label as="span" color="black">
-            Last run:
-            <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
-          </Label>
-          <Label as="span" color="black">
-            Next run:
-            <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
-          </Label>
-        </React.Fragment>
+      <div>
+        <Label as="span" color="black">
+          Processing Interval:
+          <Label.Detail>{processingTimes.interval} min</Label.Detail>
+        </Label>
+        {processingTimes.lastRun && (
+          <React.Fragment>
+            <Label as="span" color="black">
+              Last run:
+              <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
+            </Label>
+            <Label as="span" color="black">
+              Next run:
+              <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
+            </Label>
+          </React.Fragment>
+        )}
+      </div>
+      {processingTimes.scrapingAntData != null && (
+        <Segment inverted>
+          <Header as="h5">Remaining ScrapingAnt calls</Header>
+          <Message.List>
+            <Message.Item>Plan: {processingTimes.scrapingAntData.plan_name}</Message.Item>
+            <Message.Item>
+              Duration: {format(new Date(processingTimes.scrapingAntData.start_date))} -{' '}
+              {format(new Date(processingTimes.scrapingAntData.end_date))}
+            </Message.Item>
+            <Message.Item>
+              Credits: {processingTimes.scrapingAntData.remained_credits}/
+              {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
+            </Message.Item>
+          </Message.List>
+          If you want to scrape Immoscout more often, you have to purchase a premium account of{' '}
+          <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
+            {' '}
+            ScrapingAnt
+          </a>
+          . You can use the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to
+          recommend ScrapingAnt.)
+        </Segment>
       )}
     </React.Fragment>
   );

ui/src/views/jobs/mutation/JobMutation.js

@@ -2,7 +2,7 @@ import React, { Fragment, useState } from 'react';
 
 import NotificationAdapterMutator from './components/notificationAdapter/NotificationAdapterMutator';
 import NotificationAdapterTable from '../../../components/table/NotificationAdapterTable';
-import { Header, Icon, Form, Popup, Button, Label } from 'semantic-ui-react';
+import { Icon, Form, Button, Label } from 'semantic-ui-react';
 import ProviderTable from '../../../components/table/ProviderTable';
 import ProviderMutator from './components/provider/ProviderMutator';
 import ToastContext from '../../../components/toasts/ToastContext';
@@ -14,6 +14,7 @@ import { useParams } from 'react-router';
 
 import './JobMutation.less';
 import Switch from 'react-switch';
+import { SegmentPart } from '../../../components/segment/SegmentPart';
 
 export default function JobMutator() {
   const jobs = useSelector((state) => state.jobs.jobs);
@@ -39,27 +40,6 @@ export default function JobMutator() {
   const dispatch = useDispatch();
   const ctx = React.useContext(ToastContext);
 
-  const header = (name, icon) => (
-    <Header as="h5" inverted>
-      <Icon name={icon} inverted />
-      {name}
-    </Header>
-  );
-
-  const help = (helpText) => (
-    <div>
-      <Popup
-        content={helpText}
-        trigger={
-          <Header as="h6" inverted>
-            <Icon name="help circle" inverted />
-            What is this?
-          </Header>
-        }
-      />
-    </div>
-  );
-
   const isSavingEnabled = () => {
     return notificationAdapterData.length > 0 && providerData.length > 0 && name != null && name.length > 0;
   };
@@ -128,8 +108,8 @@ export default function JobMutator() {
       )}
 
       <Headline text={jobToBeEdit ? 'Edit a Job' : 'Create a new Job'} />
-      <Form className="jobMutation__form">
-        <div className="jobMutation__block">
+      <Form>
+        <SegmentPart name="Name">
           <Form.Input
             type="text"
             maxLength={40}
@@ -140,48 +120,43 @@ export default function JobMutator() {
             defaultValue={name}
             onChange={(e) => setName(e.target.value)}
           />
-        </div>
+        </SegmentPart>
 
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Provider', 'briefcase')}
+        <SegmentPart
+          name="Provider"
+          icon="briefcase"
+          helpText={
+            'A provider is essentially the service (Immowelt etc.) that Fredy is using to search for new listings. When adding a new provider, Fredy will open a new tab pointing ' +
+            'to the website of this provider. You have to adjust your search parameter and click on "Search". If the results are being shown, copy the browser url. This is the url, Fredy will use ' +
+            'to search for new listings.'
+          }
+        >
+          <Form.Button primary className="jobMutation__newButton" onClick={() => setProviderCreationVisibility(true)}>
+            <Icon name="plus" />
+            Add new Provider
+          </Form.Button>
 
-          <div className="jobMutation__helpContainer">
-            {help(
-              'A provider is essentially the service (Immowelt etc.) that Fredy is using to search for new listings. When adding a new provider, Fredy will open a new tab pointing ' +
-                'to the website of this provider. You have to adjust your search parameter and click on "Search". If the results are being shown, copy the browser url. This is the url, Fredy will use ' +
-                'to search for new listings.'
-            )}
-
-            <Form.Button primary className="jobMutation__newButton" onClick={() => setProviderCreationVisibility(true)}>
-              <Icon name="plus" />
-              Add new Provider
-            </Form.Button>
-          </div>
           <ProviderTable
             providerData={providerData}
             onRemove={(providerId) => {
               setProviderData(providerData.filter((provider) => provider.id !== providerId));
             }}
           />
-        </div>
+        </SegmentPart>
 
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Notification Adapter', 'bell')}
-
-          <div className="jobMutation__helpContainer">
-            {help(
-              'Fredy supports multiple ways to notify you about new findings. These are called notification adapter. You can chose between email, Telegram etc.'
-            )}
-
-            <Form.Button
-              primary
-              className="jobMutation__newButton"
-              onClick={() => setNotificationCreationVisibility(true)}
-            >
-              <Icon name="plus" />
-              Add new Notification Adapter
-            </Form.Button>
-          </div>
+        <SegmentPart
+          icon="bell"
+          name="Notification Adapter"
+          helpText="Fredy supports multiple ways to notify you about new findings. These are called notification adapter. You can chose between email, Telegram etc."
+        >
+          <Form.Button
+            primary
+            className="jobMutation__newButton"
+            onClick={() => setNotificationCreationVisibility(true)}
+          >
+            <Icon name="plus" />
+            Add new Notification Adapter
+          </Form.Button>
 
           <NotificationAdapterTable
             notificationAdapter={notificationAdapterData}
@@ -194,20 +169,15 @@ export default function JobMutator() {
               setNotificationCreationVisibility(true);
             }}
           />
-        </div>
-
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Blacklist', 'bell')}
-
-          <div className="jobMutation__helpContainer">
-            {help(
-              'If a listing contains one of these words, it will be filtered out. Words must be comma separated. To remove a word from the black list, just click the red label(s).'
-            )}
-          </div>
+        </SegmentPart>
 
+        <SegmentPart
+          icon="bell"
+          name="Blacklist"
+          helpText="If a listing contains one of these words, it will be filtered out. Words must be comma separated. To remove a word from the black list, just click the red label(s)."
+        >
           <Form.Input
             type="text"
-            className="jobMutation__spaceTop"
             maxLength={40}
             placeholder="Comma separated list of blacklisted words"
             autoFocus
@@ -232,19 +202,15 @@ export default function JobMutator() {
               color="red"
             />
           ))}
-        </div>
-
-        <div className="jobMutation__block jobMutation__separator">
-          {header('Job activation', 'play circle outline')}
-
-          <div className="jobMutation__helpContainer">
-            {help(
-              'Whether or not the job is activated. If it is not activated, it will be ignored when Fredy checks for new listings.'
-            )}
-          </div>
+        </SegmentPart>
 
+        <SegmentPart
+          icon="play circle outline"
+          name="Job activation"
+          helpText="Whether or not the job is activated. If it is not activated, it will be ignored when Fredy checks for new listings."
+        >
           <Switch className="jobMutation__spaceTop" onChange={(checked) => setEnabled(checked)} checked={enabled} />
-        </div>
+        </SegmentPart>
 
         <Button color="red" onClick={() => history.push('/jobs')}>
           Cancel

ui/src/views/jobs/mutation/JobMutation.less

@@ -1,29 +1,5 @@
 .jobMutation {
-
-  &__form {
-    margin-top:2rem;
-  }
-
-  &__block {
-    margin-bottom: 2rem;
-  }
   &__newButton{
     float: right;
   }
-
-  &__helpContainer {
-    display: flex;
-    justify-content: space-between;
-    align-items: flex-end;
-  }
-
-  &__spaceTop{
-    margin-top:1rem !important;
-  }
-
-  &__separator{
-    background-color: #2b2b2b;
-    border-radius: 10px;
-    padding: .8rem;
-  }
 }

ui/src/views/user/mutation/UserMutator.js

@@ -8,6 +8,7 @@ import { useDispatch } from 'react-redux';
 import Switch from 'react-switch';
 
 import './UserMutator.less';
+import { SegmentPart } from '../../../components/segment/SegmentPart';
 
 const UserMutator = function UserMutator() {
   const params = useParams();
@@ -69,40 +70,47 @@ const UserMutator = function UserMutator() {
 
   return (
     <Form inverted className="userMutator">
-      <Form.Input
-        type="text"
-        label="Username"
-        maxLength={30}
-        placeholder="Username"
-        autoFocus
-        inverted
-        width={6}
-        defaultValue={username}
-        onChange={(e) => setUsername(e.target.value)}
-      />
-      <Form.Input
-        type="password"
-        label="Password"
-        placeholder="Password"
-        inverted
-        width={6}
-        defaultValue={password}
-        onChange={(e) => setPassword(e.target.value)}
-      />
-      <Form.Input
-        type="password"
-        label="Retype password"
-        placeholder="Retype password"
-        inverted
-        width={6}
-        defaultValue={password2}
-        onChange={(e) => setPassword2(e.target.value)}
-      />
-      <Form.Field>
-        <label>Is user an admin?</label>
-        <Switch checked={isAdmin} onChange={(checked) => setIsAdmin(checked)} />
-      </Form.Field>
-
+      <SegmentPart name="Username" helpText="The username used to login to Fredy">
+        <Form.Input
+          type="text"
+          label="Username"
+          maxLength={30}
+          placeholder="Username"
+          autoFocus
+          inverted
+          width={6}
+          defaultValue={username}
+          onChange={(e) => setUsername(e.target.value)}
+        />
+      </SegmentPart>
+      <SegmentPart name="Password" helpText="The password used to login to Fredy">
+        <Form.Input
+          type="password"
+          label="Password"
+          placeholder="Password"
+          inverted
+          width={6}
+          defaultValue={password}
+          onChange={(e) => setPassword(e.target.value)}
+        />
+      </SegmentPart>
+      <SegmentPart name="Retype password" helpText="Retype the password to make sure they match">
+        <Form.Input
+          type="password"
+          label="Retype password"
+          placeholder="Retype password"
+          inverted
+          width={6}
+          defaultValue={password2}
+          onChange={(e) => setPassword2(e.target.value)}
+        />
+      </SegmentPart>
+      <SegmentPart name="Admin use" helpText="Check this if the user is an administrator">
+        <Form.Field>
+          <label>Is user an admin?</label>
+          <Switch checked={isAdmin} onChange={(checked) => setIsAdmin(checked)} />
+        </Form.Field>
+      </SegmentPart>
       <Button color="red" onClick={() => history.push('/users')}>
         Cancel
       </Button>