Merge branch 'master' of https://github.com/orangecoding/fredy

* Add provider for immobilien.de

.eslintrc.js

@@ -6,7 +6,7 @@ module.exports = {
     browser: true,
     mocha: true,
   },
-  parser: 'babel-eslint',
+  parser: '@babel/eslint-parser',
   extends: ['eslint:recommended', 'prettier'],
   plugins: ['react'],
   globals: {

CHANGELOG.md

@@ -2,6 +2,12 @@ Newer release changelog see https://github.com/orangecoding/fredy/releases
 
 ------------
 
+###### [V5.5.0]
+- Upgrading dependencies
+- fixing provider
+- allow multiple instances of 1 provider  
+- __BREAKING__: Minimum node version is now 16
+
 ###### [V5.4.6]
 - Adding Instana node.js monitoring
 - 

README.md

@@ -8,9 +8,12 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
 
 If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
 
+[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
+If you like my work, consider sponsoring this or other projects. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of these projects in my spare time :) Thanks.
+
 ## Usage  
 
-- Make sure to use Node.js 12 or above
+- Make sure to use Node.js 16 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -33,20 +36,20 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
 ## Understanding the fundamentals
 There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
 
-#### Adapter
-_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few examples. Those services are called adapters within _Fredy_. When creating a new job, you can choose one or more adapters.    
-An adapter contains the URL that points to the search results for the respective service. If you go to immonet.de and search for something, the displayed URL in the browser is what the adapter needs to do its magic.   
+#### Provider
+_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few examples. Those services are called providers within _Fredy_. When creating a new job, you can choose one or more providers.
+A provider contains the URL that points to the search results for the respective service. If you go to immonet.de and search for something, the displayed URL in the browser is what the provider needs to do its magic.
 **It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
 
-#### Provider
-_Fredy_ supports multiple providers, such as Slack, SendGrid, Telegram etc. A search job can have as many providers as supported by _Fredy_. Each provider needs different configuration values, which you have to provide when using them. A provider dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+#### Adapter
+_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
 
 #### Jobs
 A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
 
 ## Creating your first job
 To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self-explanatory, however there is one important thing.
-When configuring adapters, before copying the URL from your browser, make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
+When configuring providers, before copying the URL from your browser, make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
 
 ## User management
 As an administrator, you can create, edit and remove users from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, their jobs will also be removed.
@@ -54,7 +57,12 @@ As an administrator, you can create, edit and remove users from _Fredy_. Be care
 # Development
 
 ### Running Fredy in development mode
-To run _Fredy_ in development mode, you need to run the backend & frontend separately. Run the backend in your favorite IDE, the frontend can be started from the terminal.
+To run _Fredy_ in development mode, you need to run the backend & frontend separately.
+Start the backend with:
+```shell
+yarn run start
+```
+For the frontend, run:
 ```shell
 yarn run dev
 ```

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}

index.js

@@ -31,33 +31,20 @@ setInterval(
 
     if (isDuringWorkingHoursOrNotSet) {
       config.lastRun = Date.now();
+      const fetchedProvider = provider
+        .filter((provider) => provider.endsWith('.js'))
+        .map((pro) => require(`${path}/${pro}`));
+
       jobStorage
         .getJobs()
         .filter((job) => job.enabled)
         .forEach((job) => {
-          const providerIds = job.provider.map((provider) => provider.id);
-
-          provider
-            .filter((provider) => provider.endsWith('.js'))
-            .map((pro) => require(`${path}/${pro}`))
-            .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
-            .forEach(async (pro) => {
-              const providerId = pro.metaInformation.id;
-              if (providerId == null || providerId.length === 0) {
-                throw new Error('Provider id must not be empty. => ' + pro);
-              }
-              const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
-              if (providerConfig == null) {
-                throw new Error(`Provider Config for provider with id ${providerId} not found.`);
-              }
-              pro.init(providerConfig, job.blacklist);
-              await new FredyRuntime(
-                pro.config,
-                job.notificationAdapter,
-                providerId,
-                job.id,
-                similarityCache
-              ).execute();
+          job.provider
+            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+            .forEach(async (prov) => {
+              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+              pro.init(prov, job.blacklist);
+              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
               setLastJobExecution(job.id);
             });
         });

lib/provider/immobilienDe.js

@@ -0,0 +1,59 @@
+const utils = require('../utils');
+
+let appliedBlackList = [];
+
+function shortenLink(link) {
+  return link.substring(0, link.indexOf('?'));
+}
+
+function parseId(shortenedLink) {
+  return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
+}
+
+function normalize(o) {
+  const id = parseId(shortenLink(o.link));
+  const size = o.size || 'N/A m²';
+  const price = o.price || 'N/A €';
+  const title = o.title || 'No title available';
+  const address = o.address || 'No address available';
+  const link = shortenLink(o.link);
+  return Object.assign(o, { id, price, size, title, address, link });
+}
+
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+
+const config = {
+  url: null,
+  crawlContainer: '.estates_list .list_immo a._ref',
+  sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
+  crawlFields: {
+    price: '.list_entry .immo_preis .label_info',
+    size: '.list_entry .flaeche .label_info | removeNewline | trim',
+    title: '.list_entry .part_text h3 span',
+    description: '.list_entry .description | trim',
+    link: '@href',
+    address: '.list_entry .place',
+  },
+  paginate: '.list_immo .blocknav .blocknav_list li.next a@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+
+exports.metaInformation = {
+  name: 'Immobilien.de',
+  baseUrl: 'https://www.immobilien.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
+
+exports.config = config;

lib/services/requestDriver.js

@@ -1,16 +1,21 @@
 const axios = require('axios');
+const config = require('../../conf/config.json');
 
 const { makeUrlResidential } = require('./scrapingAnt');
 //if ScrapingAnt got blocked, this http status is returned
 const BLOCKED_HTTP_STATUS = 423;
-const MAX_RETRIES_SCRAPING_ANT = 3;
+const NOT_FOUND_HTTP_STATUS = 404;
+const MAX_RETRIES_SCRAPING_ANT = 10;
+const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
 
 function makeDriver(headers = {}) {
   let cookies = '';
 
-  async function scrapingAntDriver(context, callback, tryResidentialProxy, retryCounter = 0) {
+  async function scrapingAntDriver(context, callback, retryCounter = 0) {
+    const proxyType = config.scrapingAnt?.proxy || 'datacenter';
+
     try {
-      const url = context.url;
+      const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
       const result = await axios({
         url,
         headers: {
@@ -26,27 +31,16 @@ function makeDriver(headers = {}) {
       callback(null, result.data.content);
     } catch (exception) {
       /* eslint-disable no-console */
-      if (exception.response?.status !== BLOCKED_HTTP_STATUS) {
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
         console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
         callback(null, []);
         return;
       }
 
-      if (!tryResidentialProxy) {
-        console.debug('ScrapingAnt got blocked out. Retrying with residential Proxy...');
-        await scrapingAntDriver({ ...context, url: makeUrlResidential(context.url) }, callback, true, 0);
-      } else if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
+      if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
         retryCounter++;
-        console.debug(`ScrapingAnt still got blocked retry ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
-        await scrapingAntDriver(
-          {
-            ...context,
-            url: makeUrlResidential(context.url),
-          },
-          callback,
-          true,
-          retryCounter
-        );
+        console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
+        await scrapingAntDriver(context, callback, retryCounter);
       } else {
         console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
         callback(null, []);

lib/services/scrapingAnt.js

@@ -9,7 +9,7 @@ const isImmoscout = (id) => {
 exports.transformUrlForScrapingAnt = (url, id) => {
   if (isImmoscout(id)) {
     //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=residential`;
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
   }
   return url;
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "5.6.1",
+  "version": "5.9.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -11,7 +11,7 @@
     "prod:win32": "set BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "prod:default": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
-    "test": "mocha --timeout 20000 test/**/*.test.js",
+    "test": "mocha --timeout 3000000 test/**/*.test.js",
     "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
   },
   "husky": {
@@ -47,7 +47,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=14.0.0",
+    "node": ">=16.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -59,63 +59,63 @@
   "dependencies": {
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.2",
-    "axios": "0.27.2",
-    "better-sqlite3": "7.5.1",
-    "body-parser": "1.20.0",
+    "@sendgrid/mail": "7.7.0",
+    "axios": "1.2.1",
+    "better-sqlite3": "8.0.1",
+    "body-parser": "1.20.1",
     "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "10.0.0",
+    "highcharts": "10.3.2",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
     "nanoid": "3.3.3",
     "node-mailjet": "3.3.13",
-    "query-string": "7.1.1",
-    "react": "17.0.2",
-    "react-dom": "17.0.2",
-    "react-redux": "8.0.1",
+    "query-string": "7.1.3",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-redux": "8.0.5",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "react-switch": "^6.0.0",
+    "react-switch": "7.0.0",
     "redux": "4.2.0",
-    "redux-thunk": "2.4.1",
-    "restana": "4.9.4",
-    "semantic-ui-react": "2.1.2",
+    "redux-thunk": "2.4.2",
+    "restana": "4.9.7",
+    "semantic-ui-react": "2.1.4",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.17.9",
-    "@babel/preset-env": "7.16.11",
-    "@babel/preset-react": "7.16.7",
-    "babel-eslint": "10.1.0",
-    "babel-loader": "8.2.5",
-    "chai": "4.3.6",
+    "@babel/core": "^7.20.5",
+    "@babel/eslint-parser": "^7.19.1",
+    "@babel/preset-env": "7.20.2",
+    "@babel/preset-react": "7.18.6",
+    "babel-loader": "9.1.0",
+    "chai": "4.3.7",
     "clean-webpack-plugin": "4.0.0",
-    "copy-webpack-plugin": "10.2.4",
-    "css-loader": "6.7.1",
-    "eslint": "7.32.0",
+    "copy-webpack-plugin": "11.0.0",
+    "css-loader": "6.7.3",
+    "eslint": "^8.30.0",
     "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-react": "7.29.4",
+    "eslint-plugin-react": "7.31.11",
     "file-loader": "6.2.0",
     "history": "5.3.0",
     "husky": "4.3.8",
-    "less": "4.1.2",
-    "less-loader": "10.2.0",
-    "lint-staged": "12.4.1",
-    "mocha": "9.2.2",
-    "prettier": "2.6.2",
+    "less": "4.1.3",
+    "less-loader": "11.1.0",
+    "lint-staged": "13.1.0",
+    "mocha": "10.2.0",
+    "prettier": "2.8.1",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
     "run-script-os": "^1.1.6",
     "style-loader": "3.3.1",
     "url-loader": "4.1.1",
-    "webpack": "5.72.0",
-    "webpack-cli": "4.9.2",
-    "webpack-dev-server": "3.11.2",
+    "webpack": "5.75.0",
+    "webpack-cli": "5.0.1",
+    "webpack-dev-server": "4.11.1",
     "webpack-merge": "5.8.0"
   }
 }

test/provider/einsAImmobilien.test.js

@@ -22,7 +22,7 @@ describe('#einsAImmobilien testsuite()', () => {
 
   it('should test einsAImmobilien provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
       fredy.execute().then((listings) => {
         expect(listings).to.be.a('array');
 

test/provider/immobilienDe.test.js

@@ -0,0 +1,51 @@
+const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+const mockNotification = require('../mocks/mockNotification');
+const providerConfig = require('./testProvider.json');
+const mockStore = require('../mocks/mockStore');
+const proxyquire = require('proxyquire').noCallThru();
+const expect = require('chai').expect;
+const provider = require('../../lib/provider/immobilienDe');
+
+describe('#immobilien.de testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+
+  provider.init(providerConfig.immobilienDe, [], []);
+  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    './services/storage/listingsStorage': {
+      ...mockStore,
+    },
+    './notification/notify': mockNotification,
+  });
+
+  it('should test immobilien.de provider', async () => {
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+
+        const notificationObj = mockNotification.get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immobilienDe');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.size).that.does.include('m²');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilien.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/immonet.test.js

@@ -21,7 +21,7 @@ describe('#immonet testsuite()', () => {
 
   it('should test immonet provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/immoscout.test.js

@@ -29,7 +29,7 @@ describe('#immoscout testsuite()', () => {
         return;
       }
 
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/immoswp.test.js

@@ -21,7 +21,7 @@ describe('#immoswp testsuite()', () => {
 
   it('should test immoswp provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/immowelt.test.js

@@ -20,7 +20,7 @@ describe('#immowelt testsuite()', () => {
     });
 
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/kleinanzeigen.test.js

@@ -20,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => {
     });
 
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 
@@ -31,10 +31,8 @@ describe('#kleinanzeigen testsuite()', () => {
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.price).to.be.a('string');
           expect(notify.address).to.be.a('string');
 
           /** check the values if possible **/

test/provider/neubauKompass.test.js

@@ -20,7 +20,7 @@ describe('#neubauKompass testsuite()', () => {
 
   it('should test neubauKompass provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
 

test/provider/testProvider.json

@@ -4,6 +4,10 @@
     "enabled": true,
     "id": "einsAImmobilien"
   },
+  "immobilienDe": {
+    "url": "https://www.immobilien.de/Wohnen/Suchergebnisse-51797.html?search._digest=true&search._filter=wohnen&search.flaeche_von=50&search.objektart=wohnung&search.preis_bis=1200&search.typ=mieten&search.umkreis=15&search.wo=district%3A2434%2C2695%2C2621%2C2700%2C2967%2C2734%2C2909%2C2955%2C2392%2C2746%2C2767%2C2982%2C2904%2C2612%2C2892%2C2587%2C2871%2C2975%2C2591%2C2887%2C2569%2C2640%2C2735&sort_col=*created_ts&sort_dir=desc",
+    "enabled": true
+  },
   "immonet": {
     "url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
     "enabled": true
@@ -36,4 +40,4 @@
     "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
     "enabled": true
   }
-}
+}

test/provider/wgGesucht.test.js

@@ -20,7 +20,7 @@ describe('#wgGesucht testsuite()', () => {
 
   it('should test wgGesucht provider', async () => {
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
         const notificationObj = mockNotification.get();

ui/src/App.js

@@ -27,14 +27,17 @@ export default function FredyApp() {
   const [loading, setLoading] = React.useState(true);
   const currentUser = useSelector((state) => state.user.currentUser);
 
-  useEffect(async () => {
-    await dispatch.provider.getProvider();
-    await dispatch.jobs.getJobs();
-    await dispatch.jobs.getProcessingTimes();
-    await dispatch.notificationAdapter.getAdapter();
-    await dispatch.user.getCurrentUser();
+  useEffect(() => {
+    async function init() {
+      await dispatch.provider.getProvider();
+      await dispatch.jobs.getJobs();
+      await dispatch.jobs.getProcessingTimes();
+      await dispatch.notificationAdapter.getAdapter();
+      await dispatch.user.getCurrentUser();
 
-    setLoading(false);
+      setLoading(false);
+    }
+    init();
   }, [currentUser?.userId]);
 
   const needsLogin = () => {

ui/src/Index.js

@@ -4,7 +4,9 @@ import { reduxStore } from './services/rematch/store';
 import { HashRouter } from 'react-router-dom';
 import { createHashHistory } from 'history';
 import { Provider } from 'react-redux';
-import ReactDOM from 'react-dom';
+import { createRoot } from 'react-dom/client';
+const container = document.getElementById('fredy');
+const root = createRoot(container);
 
 const history = createHashHistory();
 
@@ -12,11 +14,10 @@ import App from './App';
 
 import './Index.less';
 
-ReactDOM.render(
+root.render(
   <Provider store={reduxStore}>
     <HashRouter history={history}>
       <App />
     </HashRouter>
-  </Provider>,
-  document.getElementById('fredy')
+  </Provider>
 );

ui/src/views/generalSettings/GeneralSettings.js

@@ -2,7 +2,7 @@ import React from 'react';
 
 import { useDispatch, useSelector } from 'react-redux';
 
-import { Button, Form, Icon, Message, Segment } from 'semantic-ui-react';
+import { Button, Form, Icon, Message, Segment, Radio } from 'semantic-ui-react';
 import ToastContext from '../../components/toasts/ToastContext';
 import Headline from '../../components/headline/Headline';
 import { xhrPost } from '../../services/xhr';
@@ -18,21 +18,29 @@ const GeneralSettings = function Users() {
   const [interval, setInterval] = React.useState('');
   const [port, setPort] = React.useState('');
   const [scrapingAntApiKey, setScrapingAntApiKey] = React.useState('');
+  const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
   const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
   const [workingHourTo, setWorkingHourTo] = React.useState(null);
   const ctx = React.useContext(ToastContext);
 
-  React.useEffect(async () => {
-    await dispatch.generalSettings.getGeneralSettings();
-    setLoading(false);
+  React.useEffect(() => {
+    async function init() {
+      await dispatch.generalSettings.getGeneralSettings();
+      setLoading(false);
+    }
+    init();
   }, []);
 
-  React.useEffect(async () => {
-    setInterval(settings?.interval);
-    setPort(settings?.port);
-    setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
-    setWorkingHourFrom(settings?.workingHours?.from);
-    setWorkingHourTo(settings?.workingHours?.to);
+  React.useEffect(() => {
+    async function init() {
+      setInterval(settings?.interval);
+      setPort(settings?.port);
+      setScrapingAntApiKey(settings?.scrapingAnt?.apiKey);
+      setWorkingHourFrom(settings?.workingHours?.from);
+      setWorkingHourTo(settings?.workingHours?.to);
+      setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
+    }
+    init();
   }, [settings]);
 
   const nullOrEmpty = (val) => val == null || val.length === 0;
@@ -69,6 +77,7 @@ const GeneralSettings = function Users() {
         port,
         scrapingAnt: {
           apiKey: scrapingAntApiKey,
+          proxy: scrapingAntProxy,
         },
         workingHours: {
           from: workingHourFrom,
@@ -144,6 +153,48 @@ const GeneralSettings = function Users() {
               />
             </SegmentPart>
 
+            <SegmentPart
+              name="ScrapingAnt proxy settings"
+              helpText="Scraping ant provides different proxies."
+              icon="key"
+            >
+              <Message info>
+                ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies.{' '}
+                <br />
+                <h4>Datacenter-Proxy</h4>
+                Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and more
+                likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
+                <h4>Residential-Proxy</h4>
+                High-quality proxy server located in one of the real people houses across the world. Datacenter proxies
+                are faster and more likely to success, but they are more expensive. A call with a datacenter proxy cost
+                250 credits.
+                <br />
+                <br />
+                <b>
+                  On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only successful
+                  calls will be charged.
+                </b>
+              </Message>
+              <Form.Field>
+                <Radio
+                  label="Datacenter proxy"
+                  name="scrapingAntProxy"
+                  value="datacenter"
+                  checked={scrapingAntProxy === 'datacenter'}
+                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                />
+              </Form.Field>
+              <Form.Field>
+                <Radio
+                  label="Residential proxy"
+                  name="scrapingAntProxy"
+                  value="residential"
+                  checked={scrapingAntProxy === 'residential'}
+                  onChange={(e, { value }) => setScrapingAntProxy(value)}
+                />
+              </Form.Field>
+            </SegmentPart>
+
             <SegmentPart
               name="Working hours"
               helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
@@ -153,7 +204,7 @@ const GeneralSettings = function Users() {
                 <Form.Input
                   className="generalSettings__time"
                   type="time"
-                  placeholder="ScrapingAnt Api Key"
+                  placeholder="Working hours from"
                   inverted
                   size="mini"
                   width={2}
@@ -163,7 +214,7 @@ const GeneralSettings = function Users() {
                 <div className="generalSettings__until">until</div>
                 <Form.Input
                   type="time"
-                  placeholder="ScrapingAnt Api Key"
+                  placeholder="Working hours to"
                   inverted
                   size="mini"
                   width={2}

ui/src/views/jobs/mutation/JobMutation.js

@@ -81,7 +81,6 @@ export default function JobMutator() {
       <ProviderMutator
         visible={providerCreationVisible}
         onVisibilityChanged={(visible) => setProviderCreationVisibility(visible)}
-        selected={providerData}
         onData={(data) => {
           setProviderData([...providerData, data]);
         }}

ui/src/views/jobs/mutation/components/provider/ProviderMutator.js

@@ -16,7 +16,7 @@ const sortProvider = (a, b) => {
   return 0;
 };
 
-export default function ProviderMutator({ onVisibilityChanged, visible = false, selected = [], onData } = {}) {
+export default function ProviderMutator({ onVisibilityChanged, visible = false, onData } = {}) {
   const provider = useSelector((state) => state.provider);
   const [selectedProvider, setSelectedProvider] = useState(null);
   const [providerUrl, setProviderUrl] = useState(null);
@@ -107,8 +107,6 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
                   text: pro.name,
                 };
               })
-              //filter out those, that have already been selected
-              .filter((option) => selected.find((selectedOption) => selectedOption.id === option.key) == null)
               .sort(sortProvider)}
             onChange={(e, { value }) => {
               const selectedProvider = provider.find((pro) => pro.id === value);

ui/src/views/user/Users.js

@@ -18,9 +18,12 @@ const Users = function Users() {
   const [userIdToBeRemoved, setUserIdToBeRemoved] = React.useState(null);
   const history = useHistory();
 
-  React.useEffect(async () => {
-    await dispatch.user.getUsers();
-    setLoading(false);
+  React.useEffect(() => {
+    async function init() {
+      await dispatch.user.getUsers();
+      setLoading(false);
+    }
+    init();
   }, []);
 
   const onUserRemoval = async () => {

ui/src/views/user/mutation/UserMutator.js

@@ -21,21 +21,24 @@ const UserMutator = function UserMutator() {
   const ctx = React.useContext(ToastContext);
   const dispatch = useDispatch();
 
-  React.useEffect(async () => {
-    if (params.userId != null) {
-      try {
-        const userJson = await xhrGet(`/api/admin/users/${params.userId}`);
-        const user = userJson.json;
+  React.useEffect(() => {
+    async function init() {
+      if (params.userId != null) {
+        try {
+          const userJson = await xhrGet(`/api/admin/users/${params.userId}`);
+          const user = userJson.json;
 
-        const defaultName = user?.username || '';
-        const defaultIsAdmin = user?.isAdmin || false;
+          const defaultName = user?.username || '';
+          const defaultIsAdmin = user?.isAdmin || false;
 
-        setUsername(defaultName);
-        setIsAdmin(defaultIsAdmin);
-      } catch (Exception) {
-        console.error(Exception);
+          setUsername(defaultName);
+          setIsAdmin(defaultIsAdmin);
+        } catch (Exception) {
+          console.error(Exception);
+        }
       }
     }
+    init();
   }, [params.userId]);
 
   const saveUser = async () => {

webpack.dev.js

@@ -6,7 +6,7 @@ const path = require('path');
 module.exports = merge(common, {
   devtool: 'inline-source-map',
   devServer: {
-    contentBase: path.join(__dirname, 'ui', 'public'),
+    static: path.join(__dirname, 'ui', 'public'),
     port: 9000,
     proxy: {
       '/api': {