adding scraping ant infos as we now need to use residental proxies

.env

@@ -1 +0,0 @@
-INSTANA_MONITORING=false

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

index.js

@@ -1,16 +1,3 @@
-require('dotenv').config();
-/********OPTIONAL INSTANA INITIALIZATION BEGIN********/
-//if you want to use Instana to monitor fredy, go to https://www.instana.com and
-// try it yourself by signing up for a free trial
-const { INSTANA_MONITORING } = process.env;
-if (INSTANA_MONITORING != null && INSTANA_MONITORING === 'true') {
-  /* eslint-disable no-console */
-  console.debug('Starting Instana monitoring');
-  /* eslint-enable no-console */
-  require('@instana/collector')();
-}
-/********OPTIONAL INSTANA INITIALIZATION END********/
-
 const fs = require('fs');
 
 //if db folder does not exist, ensure to create it before loading anything else

lib/api/routes/jobRouter.js

@@ -1,10 +1,10 @@
 const service = require('restana')();
 const jobRouter = service.newRouter();
+const axios = require('axios');
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
 const immoscoutProvider = require('../../provider/immoscout');
 const config = require('../../../conf/config.json');
-
 const { isAdmin } = require('../security');
 
 function doesJobBelongsToUser(job, req) {
@@ -30,9 +30,23 @@ jobRouter.get('/', async (req, res) => {
 });
 
 jobRouter.get('/processingTimes', async (req, res) => {
+  let scrapingAntData = null;
+
+  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+    try {
+      const result = await axios({
+        url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
+      });
+      scrapingAntData = result.data;
+    } catch (Exception) {
+      console.error('Could not query plan data from scraping ant.', Exception);
+    }
+  }
+
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
+    scrapingAntData,
   };
 
   res.send();

lib/services/scrapingAnt.js

@@ -1,5 +1,5 @@
 const { metaInformation } = require('../provider/immoscout');
-//to better confure re-capture chose a random proxy each time we do a call
+//to better configure re-capture chose a random proxy each time we do a call
 const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
 const config = require('../../conf/config.json');
 
@@ -12,7 +12,9 @@ exports.transformUrlForScrapingAnt = (url, id) => {
 
   if (isImmoscout(id)) {
     //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(
+      url
+    )}&proxy_country=${randomProxy}&proxy_type=residential`;
   }
   return url;
 };

package.json

@@ -1,10 +1,10 @@
 {
   "name": "fredy",
-  "version": "5.4.6",
+  "version": "5.4.8",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
+    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
     "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
     "test": "mocha --timeout 20000 test/**/*.test.js",
@@ -43,8 +43,8 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=12.13.0",
+    "node": ">=14.0.0",
-    "npm": ">=6.0.0"
+    "npm": ">=7.0.0"
   },
   "browserslist": [
     "> 0.5%",
@@ -53,24 +53,22 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@instana/collector": "^1.137.5",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.0",
+    "@sendgrid/mail": "7.6.2",
-    "axios": "0.24.0",
+    "axios": "0.26.1",
     "axios-retry": "^3.2.4",
     "better-sqlite3": "^7.5.0",
-    "body-parser": "1.19.0",
+    "body-parser": "1.19.2",
-    "cookie-session": "1.4.0",
+    "cookie-session": "2.0.0",
-    "dotenv": "^15.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.3.1",
+    "highcharts": "10.0.0",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.1.30",
+    "nanoid": "3.3.1",
-    "node-mailjet": "3.3.4",
+    "node-mailjet": "3.3.7",
-    "query-string": "^7.0.1",
+    "query-string": "7.1.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-redux": "7.2.6",
@@ -78,41 +76,41 @@
     "react-router-dom": "5.3.0",
     "react-switch": "^6.0.0",
     "redux": "4.1.2",
-    "redux-thunk": "2.4.0",
+    "redux-thunk": "2.4.1",
-    "restana": "4.9.2",
+    "restana": "4.9.3",
-    "semantic-ui-react": "2.0.4",
+    "semantic-ui-react": "2.1.2",
-    "serve-static": "^1.14.1",
+    "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.16.0",
+    "@babel/core": "7.17.8",
-    "@babel/preset-env": "7.16.4",
+    "@babel/preset-env": "7.16.11",
-    "@babel/preset-react": "7.16.0",
+    "@babel/preset-react": "7.16.7",
     "babel-eslint": "10.1.0",
-    "babel-loader": "8.2.3",
+    "babel-loader": "8.2.4",
-    "chai": "4.3.4",
+    "chai": "4.3.6",
     "clean-webpack-plugin": "4.0.0",
-    "copy-webpack-plugin": "10.0.0",
+    "copy-webpack-plugin": "10.2.4",
-    "css-loader": "6.5.1",
+    "css-loader": "6.7.1",
     "eslint": "7.32.0",
-    "eslint-config-prettier": "8.3.0",
+    "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-react": "7.27.1",
+    "eslint-plugin-react": "7.29.3",
     "file-loader": "6.2.0",
-    "history": "5.1.0",
+    "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.2",
     "less-loader": "10.2.0",
-    "lint-staged": "12.1.2",
+    "lint-staged": "12.3.7",
-    "mocha": "9.1.3",
+    "mocha": "9.2.2",
-    "prettier": "2.5.0",
+    "prettier": "2.6.1",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
     "style-loader": "3.3.1",
     "url-loader": "4.1.1",
-    "webpack": "5.64.4",
+    "webpack": "5.70.0",
-    "webpack-cli": "4.9.1",
+    "webpack-cli": "4.9.2",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.8.0"
   }

ui/src/views/jobs/ProcessingTimes.js

@@ -1,25 +1,46 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import { Label } from 'semantic-ui-react';
+import { Header, Label, Message, Segment } from 'semantic-ui-react';
 
 export default function ProcessingTimes({ processingTimes }) {
   return (
     <React.Fragment>
-      <Label as="span" color="black">
+      <div>
-        Processing Interval:
+        <Label as="span" color="black">
-        <Label.Detail>{processingTimes.interval} min</Label.Detail>
+          Processing Interval:
-      </Label>
+          <Label.Detail>{processingTimes.interval} min</Label.Detail>
-      {processingTimes.lastRun && (
+        </Label>
-        <React.Fragment>
+        {processingTimes.lastRun && (
-          <Label as="span" color="black">
+          <React.Fragment>
-            Last run:
+            <Label as="span" color="black">
-            <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
+              Last run:
-          </Label>
+              <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
-          <Label as="span" color="black">
+            </Label>
-            Next run:
+            <Label as="span" color="black">
-            <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
+              Next run:
-          </Label>
+              <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
-        </React.Fragment>
+            </Label>
+          </React.Fragment>
+        )}
+      </div>
+      {processingTimes.scrapingAntData != null && (
+        <Segment inverted>
+          <Header as="h5">Remaining ScrapingAnt calls</Header>
+          <Message.List>
+            <Message.Item>Plan: {processingTimes.scrapingAntData.plan_name}</Message.Item>
+            <Message.Item>
+              Duration: {format(new Date(processingTimes.scrapingAntData.start_date))} -{' '}
+              {format(new Date(processingTimes.scrapingAntData.end_date))}
+            </Message.Item>
+            <Message.Item>
+              Credits: {processingTimes.scrapingAntData.remained_credits}/
+              {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
+            </Message.Item>
+          </Message.List>
+          If you want to scrape Immoscout more often, you have to purchase a premium account of ScrapingAnt. You can use
+          the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to recommend
+          ScrapingAnt.
+        </Segment>
       )}
     </React.Fragment>
   );