adding scraping ant infos as we now need to use residental proxies

next version

.env

@@ -1 +0,0 @@
-INSTANA_MONITORING=false

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

CONTRIBUTING.md

@@ -17,7 +17,7 @@ function normalize(o) {
   return Object.assign(o, { id });
 }
 
-//apply blaclist if needed
+//apply blacklist if needed
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);

README.md

@@ -84,7 +84,8 @@ See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTIN
 ### Monitoring
 
 _Fredy_ can be monitored by [Instana](https://www.instana.com). If you are interested, sign up for a free trial. This is totally optional of course :)  
-If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.
+If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.   
+If you want to know more, head over to the [Instana docs](https://www.ibm.com/docs/en/obi/current?topic=technologies-monitoring-nodejs).
 
 # Docker   
 Use the Dockerfile in this repository to build an image.  

index.js

@@ -1,16 +1,3 @@
-require('dotenv').config();
-/********OPTIONAL INSTANA INITIALIZATION BEGIN********/
-//if you want to use Instana to monitor fredy, go to https://www.instana.com and
-// try it yourself by signing up for a free trial
-const { INSTANA_MONITORING } = process.env;
-if (INSTANA_MONITORING != null && INSTANA_MONITORING === 'true') {
-  /* eslint-disable no-console */
-  console.debug('Starting Instana monitoring');
-  /* eslint-enable no-console */
-  require('@instana/collector')();
-}
-/********OPTIONAL INSTANA INITIALIZATION END********/
-
 const fs = require('fs');
 
 //if db folder does not exist, ensure to create it before loading anything else

lib/api/routes/jobRouter.js

@@ -1,10 +1,10 @@
 const service = require('restana')();
 const jobRouter = service.newRouter();
+const axios = require('axios');
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
 const immoscoutProvider = require('../../provider/immoscout');
 const config = require('../../../conf/config.json');
-
 const { isAdmin } = require('../security');
 
 function doesJobBelongsToUser(job, req) {
@@ -30,9 +30,23 @@ jobRouter.get('/', async (req, res) => {
 });
 
 jobRouter.get('/processingTimes', async (req, res) => {
+  let scrapingAntData = null;
+
+  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+    try {
+      const result = await axios({
+        url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
+      });
+      scrapingAntData = result.data;
+    } catch (Exception) {
+      console.error('Could not query plan data from scraping ant.', Exception);
+    }
+  }
+
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
+    scrapingAntData,
   };
 
   res.send();

lib/notification/adapter/mattermost.js

@@ -0,0 +1,52 @@
+const { markdown2Html } = require('../../services/markdown');
+const { getJob } = require('../../services/storage/jobStorage');
+const axios = require('axios');
+
+/**
+ * sends new listings to mattermost
+ * @param serviceName e.g immowelt
+ * @param newListings an array with newly found listings
+ * @param notificationConfig config of this notification adapter
+ * @param jobKey name of the current job that is being executed
+ * @returns {Promise<Void> | void}
+ */
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+
+  let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
+  message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
+  message += newListings.map(
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+  );
+
+  return axios.post(`${webhook}`, {
+    channel: channel,
+    text: message,
+  });
+};
+
+/**
+ * exported config is being used in the frontend to generate the fields
+ * incoming values will be the keys (and values) of the fields
+ *
+ */
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Mattermost',
+  readme: markdown2Html('lib/notification/adapter/mattermost.md'),
+  description: 'Fredy will send new listings to your mattermost team chat.',
+  fields: {
+    webhook: {
+      type: 'text',
+      label: 'Webhook-URL',
+      description: 'The incoming webhook url',
+    },
+    channel: {
+      type: 'text',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/mattermost.md

@@ -0,0 +1,5 @@
+### Mattermost Adapter
+
+For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the  [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+
+As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.

lib/notification/adapter/sqlite.js

@@ -0,0 +1,33 @@
+const { markdown2Html } = require('../../services/markdown');
+const Database = require('better-sqlite3');
+
+/**
+ * Stores data in a sqlite db in order to use the search results for later analytics
+ * @param serviceName e.g immowelt
+ * @param newListings an array with newly found listings
+ * @param jobKey name of the current job that is being executed
+ */
+exports.send = ({ serviceName, newListings, jobKey }) => {
+  const db = new Database('db/listings.db');
+  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Sqlite',
+  description: 'This adapter stores listings in a local sqlite3 database.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,3 @@
+### Sqlite Adapter
+
+This adapter stores search results in an sqlite database in db/listings.db

lib/provider/immonet.js

@@ -6,7 +6,7 @@ function normalize(o) {
   const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
-  const address = o.address.split(' • ')[1];
+  const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
   //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
   //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)

lib/services/scrapingAnt.js

@@ -1,5 +1,5 @@
 const { metaInformation } = require('../provider/immoscout');
-//to better confure re-capture chose a random proxy each time we do a call
+//to better configure re-capture chose a random proxy each time we do a call
 const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
 const config = require('../../conf/config.json');
 
@@ -12,7 +12,9 @@ exports.transformUrlForScrapingAnt = (url, id) => {
 
   if (isImmoscout(id)) {
     //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(
+      url
+    )}&proxy_country=${randomProxy}&proxy_type=residential`;
   }
   return url;
 };

package.json

@@ -1,10 +1,10 @@
 {
   "name": "fredy",
-  "version": "5.4.5",
+  "version": "5.4.8",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
+    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
     "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
     "test": "mocha --timeout 20000 test/**/*.test.js",
@@ -43,8 +43,8 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=12.13.0",
-    "npm": ">=6.0.0"
+    "node": ">=14.0.0",
+    "npm": ">=7.0.0"
   },
   "browserslist": [
     "> 0.5%",
@@ -53,23 +53,22 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@instana/collector": "^1.137.5",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.0",
-    "axios": "0.24.0",
+    "@sendgrid/mail": "7.6.2",
+    "axios": "0.26.1",
     "axios-retry": "^3.2.4",
-    "body-parser": "1.19.0",
-    "cookie-session": "1.4.0",
-    "dotenv": "^15.0.0",
+    "better-sqlite3": "^7.5.0",
+    "body-parser": "1.19.2",
+    "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.3.1",
+    "highcharts": "10.0.0",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.1.30",
-    "node-mailjet": "3.3.4",
-    "query-string": "^7.0.1",
+    "nanoid": "3.3.1",
+    "node-mailjet": "3.3.7",
+    "query-string": "7.1.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-redux": "7.2.6",
@@ -77,41 +76,41 @@
     "react-router-dom": "5.3.0",
     "react-switch": "^6.0.0",
     "redux": "4.1.2",
-    "redux-thunk": "2.4.0",
-    "restana": "4.9.2",
-    "semantic-ui-react": "2.0.4",
-    "serve-static": "^1.14.1",
+    "redux-thunk": "2.4.1",
+    "restana": "4.9.3",
+    "semantic-ui-react": "2.1.2",
+    "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.16.0",
-    "@babel/preset-env": "7.16.4",
-    "@babel/preset-react": "7.16.0",
+    "@babel/core": "7.17.8",
+    "@babel/preset-env": "7.16.11",
+    "@babel/preset-react": "7.16.7",
     "babel-eslint": "10.1.0",
-    "babel-loader": "8.2.3",
-    "chai": "4.3.4",
+    "babel-loader": "8.2.4",
+    "chai": "4.3.6",
     "clean-webpack-plugin": "4.0.0",
-    "copy-webpack-plugin": "10.0.0",
-    "css-loader": "6.5.1",
+    "copy-webpack-plugin": "10.2.4",
+    "css-loader": "6.7.1",
     "eslint": "7.32.0",
-    "eslint-config-prettier": "8.3.0",
-    "eslint-plugin-react": "7.27.1",
+    "eslint-config-prettier": "8.5.0",
+    "eslint-plugin-react": "7.29.3",
     "file-loader": "6.2.0",
-    "history": "5.1.0",
+    "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.2",
     "less-loader": "10.2.0",
-    "lint-staged": "12.1.2",
-    "mocha": "9.1.3",
-    "prettier": "2.5.0",
+    "lint-staged": "12.3.7",
+    "mocha": "9.2.2",
+    "prettier": "2.6.1",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
     "style-loader": "3.3.1",
     "url-loader": "4.1.1",
-    "webpack": "5.64.4",
-    "webpack-cli": "4.9.1",
+    "webpack": "5.70.0",
+    "webpack-cli": "4.9.2",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.8.0"
   }

test/provider/immowelt.test.js

@@ -32,13 +32,13 @@ describe('#immowelt testsuite()', () => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
+
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
 
           /** check the values if possible **/
-          if (notify.size.trim().toLowerCase() !== 'k.a.') {
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');
           }
           expect(notify.title).to.be.not.empty;

ui/src/views/jobs/ProcessingTimes.js

@@ -1,25 +1,46 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import { Label } from 'semantic-ui-react';
+import { Header, Label, Message, Segment } from 'semantic-ui-react';
 
 export default function ProcessingTimes({ processingTimes }) {
   return (
     <React.Fragment>
-      <Label as="span" color="black">
-        Processing Interval:
-        <Label.Detail>{processingTimes.interval} min</Label.Detail>
-      </Label>
-      {processingTimes.lastRun && (
-        <React.Fragment>
-          <Label as="span" color="black">
-            Last run:
-            <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
-          </Label>
-          <Label as="span" color="black">
-            Next run:
-            <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
-          </Label>
-        </React.Fragment>
+      <div>
+        <Label as="span" color="black">
+          Processing Interval:
+          <Label.Detail>{processingTimes.interval} min</Label.Detail>
+        </Label>
+        {processingTimes.lastRun && (
+          <React.Fragment>
+            <Label as="span" color="black">
+              Last run:
+              <Label.Detail>{format(processingTimes.lastRun)}</Label.Detail>
+            </Label>
+            <Label as="span" color="black">
+              Next run:
+              <Label.Detail>{format(processingTimes.lastRun + processingTimes.interval * 60000)}</Label.Detail>
+            </Label>
+          </React.Fragment>
+        )}
+      </div>
+      {processingTimes.scrapingAntData != null && (
+        <Segment inverted>
+          <Header as="h5">Remaining ScrapingAnt calls</Header>
+          <Message.List>
+            <Message.Item>Plan: {processingTimes.scrapingAntData.plan_name}</Message.Item>
+            <Message.Item>
+              Duration: {format(new Date(processingTimes.scrapingAntData.start_date))} -{' '}
+              {format(new Date(processingTimes.scrapingAntData.end_date))}
+            </Message.Item>
+            <Message.Item>
+              Credits: {processingTimes.scrapingAntData.remained_credits}/
+              {processingTimes.scrapingAntData.plan_total_credits} (250 credits per call)
+            </Message.Item>
+          </Message.List>
+          If you want to scrape Immoscout more often, you have to purchase a premium account of ScrapingAnt. You can use
+          the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to recommend
+          ScrapingAnt.
+        </Segment>
       )}
     </React.Fragment>
   );