next release

CHANGELOG.md

@@ -1,3 +1,8 @@
+###### [V5.3.0]
+- Upgrading dependencies
+- It's now possible to send mails to multiple receiver using comma separation for MailJet & Sendgrid
+- Fixing Immowelt scraping
+
 ###### [V5.2.0]
 - Upgrading dependencies
 - Adding new similarity check layer (Duplicates are being removed now)

lib/notification/adapter/mailJet.js

@@ -21,6 +21,13 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     (adapter) => adapter.id === 'mailJet'
   ).fields;
 
+  const to = receiver
+    .trim()
+    .split(',')
+    .map((r) => ({
+      Email: r.trim(),
+    }));
+
   return mailjet
     .connect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
@@ -31,11 +38,7 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
             Email: from,
             Name: 'Fredy',
           },
-          To: [
+          To: to,
-            {
-              Email: receiver,
-            },
-          ],
           Subject: `Fredy found ${newListings.length} new listings for ${serviceName}`,
           HTMLPart: emailTemplate({
             serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,

lib/notification/adapter/mailJet.md

@@ -4,3 +4,5 @@ To use [MailJet](https://mailjet.com), you need to create an account. You'll nee
  
 E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.     
 The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
+
+If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com). 

lib/notification/adapter/sendGrid.js

@@ -14,7 +14,10 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,
-    to: receiver,
+    to: receiver
+      .trim()
+      .split(',')
+      .map((r) => r.trim()),
     from,
     subject: `Job ${jobKey} | Service ${serviceName} found ${newListings.length} new listing(s)`,
     dynamic_template_data: {

lib/notification/adapter/sendGrid.md

@@ -6,3 +6,5 @@ SendGrid is a free email service (free as in "you cannot send more than 100(Send
 To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.     
 
 Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new dynamic template. For this new template, I recommend copying and pasting the code from the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
+
+If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).

lib/notification/adapter/telegram.js

@@ -1,6 +1,19 @@
 const { markdown2Html } = require('../../services/markdown');
 const axios = require('axios');
 
+/**
+ * splitting an array into chunks because Telegram only allows for messages up to
+ * 4096 chars, thus we have to split messages into chunks
+ * @param inputArray
+ * @param perChunk
+ */
+const arrayChunks = (inputArray, perChunk) =>
+  inputArray.reduce((all, one, i) => {
+    const ch = Math.floor(i / perChunk);
+    all[ch] = [].concat(all[ch] || [], one);
+    return all;
+  }, []);
+
 /**
  * sends new listings to telegram
  * @param serviceName e.g immowelt
@@ -12,22 +25,28 @@ const axios = require('axios');
 exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
 
-  let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
+  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
+  const chunks = arrayChunks(newListings, 3);
 
-  message += newListings.map(
+  const promises = chunks.map((chunk) => {
-    (o) =>
+    let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
-      `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
+    message += chunk.map(
-      [o.address, o.price, o.size].join(' | ') +
+      (o) =>
-      '\n' +
+        `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
-      `<a href="${o.link}">${o.link}</a>\n\n`
+        [o.address, o.price, o.size].join(' | ') +
-  );
+        '\n' +
+        `<a href="${o.link}">${o.link}</a>\n\n`
+    );
 
-  return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
+    return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
-    chat_id: chatId,
+      chat_id: chatId,
-    text: message,
+      text: message,
-    parse_mode: 'HTML',
+      parse_mode: 'HTML',
-    disable_web_page_preview: true,
+      disable_web_page_preview: true,
+    });
   });
+
+  return Promise.all(promises);
 };
 
 function shorten(str, len = 30) {

lib/provider/immoscout.js

@@ -2,9 +2,13 @@ const utils = require('../utils');
 
 let appliedBlackList = [];
 
+function nullOrEmpty(val) {
+  return val == null || val.length === 0;
+}
+
 function normalize(o) {
-  const title = o.title.replace('NEU', '');
+  const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
-  const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
+  const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
   const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
   return Object.assign(o, { title, address, link });
 }

lib/provider/immowelt.js

@@ -3,10 +3,7 @@ const utils = require('../utils');
 let appliedBlackList = [];
 
 function normalize(o) {
-  const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
+  return o;
-  const address = o.address;
-
-  return Object.assign(o, { size, address });
 }
 
 function applyBlacklist(o) {
@@ -18,14 +15,14 @@ function applyBlacklist(o) {
 
 const config = {
   url: null,
-  crawlContainer: '.immoliste .js-object.listitem_wrap ',
+  crawlContainer: "div[class^='EstateItem-']",
   crawlFields: {
-    id: '@data-estateid | int',
+    id: 'a@id',
-    price: '.hardfacts_3 strong | removeNewline | trim',
+    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
-    size: '.js-object.listitem_wrap .hardfacts_3 div:nth-child(2)| removeNewline | trim',
+    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
-    title: '.listcontent.clear h2',
+    title: "div[class^='FactsMain-'] h2",
     link: 'a@href',
-    address: '.listcontent .details .listlocation| removeNewline | trim',
+    address: "div[class^='estateFacts-'] span | removeNewline | trim",
   },
   paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,

lib/services/requestDriver.js

@@ -1,4 +1,7 @@
 const axios = require('axios');
+const axiosRetry = require('axios-retry');
+
+axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
 
 function makeDriver(headers = {}) {
   let cookies = '';
@@ -15,7 +18,8 @@ function makeDriver(headers = {}) {
         },
       });
     } catch (exception) {
-      callback(exception, null);
+      console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
+      callback(null, []);
     }
 
     if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "5.2.0",
+  "version": "5.3.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -32,6 +32,7 @@
     "house",
     "rent",
     "immoscout",
+    "scraper",
     "immonet",
     "immowelt",
     "immobilienscout24"
@@ -51,60 +52,61 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@rematch/core": "2.0.1",
+    "@rematch/core": "2.1.0",
-    "@rematch/loading": "2.0.1",
+    "@rematch/loading": "2.1.0",
-    "@sendgrid/mail": "7.4.5",
+    "@sendgrid/mail": "7.4.7",
-    "axios": "0.21.1",
+    "axios": "0.24.0",
+    "axios-retry": "^3.2.4",
     "body-parser": "1.19.0",
     "cookie-session": "1.4.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.1.2",
+    "highcharts": "9.2.2",
     "highcharts-react-official": "3.0.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.1.23",
+    "nanoid": "3.1.28",
     "node-mailjet": "3.3.4",
     "react": "17.0.2",
     "react-dom": "17.0.2",
-    "react-redux": "7.2.4",
+    "react-redux": "7.2.5",
-    "react-router": "5.2.0",
+    "react-router": "5.2.1",
-    "react-router-dom": "5.2.0",
+    "react-router-dom": "5.3.0",
     "react-switch": "^6.0.0",
-    "redux": "4.1.0",
+    "redux": "4.1.1",
     "redux-thunk": "2.3.0",
     "restana": "4.9.1",
-    "semantic-ui-react": "2.0.3",
+    "semantic-ui-react": "2.0.4",
     "serve-static": "^1.14.1",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.14.6",
+    "@babel/core": "7.15.5",
-    "@babel/preset-env": "7.14.7",
+    "@babel/preset-env": "7.15.6",
     "@babel/preset-react": "7.14.5",
     "babel-eslint": "10.1.0",
     "babel-loader": "8.2.2",
     "chai": "4.3.4",
-    "clean-webpack-plugin": "3.0.0",
+    "clean-webpack-plugin": "4.0.0",
     "copy-webpack-plugin": "9.0.1",
-    "css-loader": "5.2.6",
+    "css-loader": "6.3.0",
-    "eslint": "7.29.0",
+    "eslint": "7.32.0",
     "eslint-config-prettier": "8.3.0",
-    "eslint-plugin-react": "7.24.0",
+    "eslint-plugin-react": "7.26.1",
     "file-loader": "6.2.0",
-    "history": "5.0.0",
+    "history": "5.0.1",
     "husky": "4.3.8",
     "less": "4.1.1",
-    "less-loader": "10.0.0",
+    "less-loader": "10.0.1",
-    "lint-staged": "11.0.0",
+    "lint-staged": "11.1.2",
-    "mocha": "9.0.1",
+    "mocha": "9.1.2",
-    "prettier": "2.3.2",
+    "prettier": "2.4.1",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
-    "style-loader": "3.0.0",
+    "style-loader": "3.3.0",
     "url-loader": "4.1.1",
-    "webpack": "5.40.0",
+    "webpack": "5.56.0",
     "webpack-cli": "3.3.12",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.8.0"

test/provider/immowelt.test.js

@@ -30,7 +30,7 @@ describe('#immowelt testsuite()', () => {
 
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');

test/provider/testProvider.json

@@ -9,7 +9,7 @@
     "enabled": true
   },
   "immowelt": {
-    "url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
+    "url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?d=true&rmi=3&sd=DESC&sf=TIMESTAMP&sp=1",
     "enabled": true
   },
   "immoscout": {