dependency management | fixing dev mode

next version

.env

@@ -1 +0,0 @@
-INSTANA_MONITORING=false

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

CONTRIBUTING.md

@@ -17,7 +17,7 @@ function normalize(o) {
   return Object.assign(o, { id });
 }
 
-//apply blaclist if needed
+//apply blacklist if needed
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);

README.md

@@ -84,7 +84,8 @@ See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTIN
 ### Monitoring
 
 _Fredy_ can be monitored by [Instana](https://www.instana.com). If you are interested, sign up for a free trial. This is totally optional of course :)  
-If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.
+If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.   
+If you want to know more, head over to the [Instana docs](https://www.ibm.com/docs/en/obi/current?topic=technologies-monitoring-nodejs).
 
 # Docker   
 Use the Dockerfile in this repository to build an image.  

index.js

@@ -1,16 +1,3 @@
-require('dotenv').config();
-/********OPTIONAL INSTANA INITIALIZATION BEGIN********/
-//if you want to use Instana to monitor fredy, go to https://www.instana.com and
-// try it yourself by signing up for a free trial
-const { INSTANA_MONITORING } = process.env;
-if (INSTANA_MONITORING != null && INSTANA_MONITORING === 'true') {
-  /* eslint-disable no-console */
-  console.debug('Starting Instana monitoring');
-  /* eslint-enable no-console */
-  require('@instana/collector')();
-}
-/********OPTIONAL INSTANA INITIALIZATION END********/
-
 const fs = require('fs');
 
 //if db folder does not exist, ensure to create it before loading anything else

lib/notification/adapter/mattermost.js

@@ -0,0 +1,52 @@
+const { markdown2Html } = require('../../services/markdown');
+const { getJob } = require('../../services/storage/jobStorage');
+const axios = require('axios');
+
+/**
+ * sends new listings to mattermost
+ * @param serviceName e.g immowelt
+ * @param newListings an array with newly found listings
+ * @param notificationConfig config of this notification adapter
+ * @param jobKey name of the current job that is being executed
+ * @returns {Promise<Void> | void}
+ */
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+
+  let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
+  message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
+  message += newListings.map(
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+  );
+
+  return axios.post(`${webhook}`, {
+    channel: channel,
+    text: message,
+  });
+};
+
+/**
+ * exported config is being used in the frontend to generate the fields
+ * incoming values will be the keys (and values) of the fields
+ *
+ */
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Mattermost',
+  readme: markdown2Html('lib/notification/adapter/mattermost.md'),
+  description: 'Fredy will send new listings to your mattermost team chat.',
+  fields: {
+    webhook: {
+      type: 'text',
+      label: 'Webhook-URL',
+      description: 'The incoming webhook url',
+    },
+    channel: {
+      type: 'text',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/mattermost.md

@@ -0,0 +1,5 @@
+### Mattermost Adapter
+
+For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the  [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+
+As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.

lib/notification/adapter/sqlite.js

@@ -0,0 +1,33 @@
+const { markdown2Html } = require('../../services/markdown');
+const Database = require('better-sqlite3');
+
+/**
+ * Stores data in a sqlite db in order to use the search results for later analytics
+ * @param serviceName e.g immowelt
+ * @param newListings an array with newly found listings
+ * @param jobKey name of the current job that is being executed
+ */
+exports.send = ({ serviceName, newListings, jobKey }) => {
+  const db = new Database('db/listings.db');
+  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Sqlite',
+  description: 'This adapter stores listings in a local sqlite3 database.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,3 @@
+### Sqlite Adapter
+
+This adapter stores search results in an sqlite database in db/listings.db

lib/provider/immonet.js

@@ -6,7 +6,7 @@ function normalize(o) {
   const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
-  const address = o.address.split(' • ')[1];
+  const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
   //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
   //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)

package.json

@@ -1,10 +1,10 @@
 {
   "name": "fredy",
-  "version": "5.4.5",
+  "version": "5.4.7",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
+    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
     "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
     "test": "mocha --timeout 20000 test/**/*.test.js",
@@ -43,8 +43,8 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=12.13.0",
+    "node": ">=14.0.0",
-    "npm": ">=6.0.0"
+    "npm": ">=7.0.0"
   },
   "browserslist": [
     "> 0.5%",
@@ -53,23 +53,22 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@instana/collector": "^1.137.5",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.0",
+    "@sendgrid/mail": "7.6.1",
-    "axios": "0.24.0",
+    "axios": "0.26.0",
     "axios-retry": "^3.2.4",
-    "body-parser": "1.19.0",
+    "better-sqlite3": "^7.5.0",
-    "cookie-session": "1.4.0",
+    "body-parser": "1.19.2",
-    "dotenv": "^15.0.0",
+    "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "9.3.1",
+    "highcharts": "10.0.0",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
-    "nanoid": "3.1.30",
+    "nanoid": "3.3.1",
-    "node-mailjet": "3.3.4",
+    "node-mailjet": "3.3.7",
-    "query-string": "^7.0.1",
+    "query-string": "7.1.1",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-redux": "7.2.6",
@@ -77,41 +76,41 @@
     "react-router-dom": "5.3.0",
     "react-switch": "^6.0.0",
     "redux": "4.1.2",
-    "redux-thunk": "2.4.0",
+    "redux-thunk": "2.4.1",
-    "restana": "4.9.2",
+    "restana": "4.9.3",
-    "semantic-ui-react": "2.0.4",
+    "semantic-ui-react": "2.1.2",
-    "serve-static": "^1.14.1",
+    "serve-static": "1.14.2",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.16.0",
+    "@babel/core": "7.17.5",
-    "@babel/preset-env": "7.16.4",
+    "@babel/preset-env": "7.16.11",
-    "@babel/preset-react": "7.16.0",
+    "@babel/preset-react": "7.16.7",
     "babel-eslint": "10.1.0",
     "babel-loader": "8.2.3",
-    "chai": "4.3.4",
+    "chai": "4.3.6",
     "clean-webpack-plugin": "4.0.0",
-    "copy-webpack-plugin": "10.0.0",
+    "copy-webpack-plugin": "10.2.4",
-    "css-loader": "6.5.1",
+    "css-loader": "6.7.1",
     "eslint": "7.32.0",
-    "eslint-config-prettier": "8.3.0",
+    "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-react": "7.27.1",
+    "eslint-plugin-react": "7.29.3",
     "file-loader": "6.2.0",
-    "history": "5.1.0",
+    "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.2",
     "less-loader": "10.2.0",
-    "lint-staged": "12.1.2",
+    "lint-staged": "12.3.5",
-    "mocha": "9.1.3",
+    "mocha": "9.2.1",
-    "prettier": "2.5.0",
+    "prettier": "2.5.1",
     "proxyquire": "2.1.3",
     "redux-logger": "3.0.6",
     "style-loader": "3.3.1",
     "url-loader": "4.1.1",
-    "webpack": "5.64.4",
+    "webpack": "5.70.0",
-    "webpack-cli": "4.9.1",
+    "webpack-cli": "4.9.2",
     "webpack-dev-server": "3.11.2",
     "webpack-merge": "5.8.0"
   }

test/provider/immowelt.test.js

@@ -32,13 +32,13 @@ describe('#immowelt testsuite()', () => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
+
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
 
           /** check the values if possible **/
-          if (notify.size.trim().toLowerCase() !== 'k.a.') {
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');
           }
           expect(notify.title).to.be.not.empty;