Update package.json

next version

.env

@@ -0,0 +1 @@
+INSTANA_MONITORING=false

CHANGELOG.md

@@ -1,4 +1,21 @@
-###### [V5.4.0]
+###### [V5.4.5]
+- Adding Instana node.js monitoring 
+
+###### [V5.4.4]
+- Add support for Immo Südwest Presse (immo.swp.de)
+- Telegram: Use job name instead of ID and link in title
+- Fix race condition if user ID is in session but not in user store
+- Allow visiting the original provider URL
+
+###### [V5.4.3]
+- re-writing readme
+- improving docker build
+- using github's actions to build docker and test automatically
+
+###### [V5.4.2]
+- Fixing prod build
+
+###### [V5.4.1]
 - Upgrading dependencies
 - Provider urls are now automagically been changed to include the correct sort order for search results
 
@@ -45,4 +62,4 @@ on the new ui and use the values from your previous config file if needed.
 [BREAKING CHANGES]
 - The config has been changed, the config of V1.x will not work any longer
 - Sources have been renamed to provider
-```
+```

CONTRIBUTING.md

@@ -17,7 +17,7 @@ function normalize(o) {
   return Object.assign(o, { id });
 }
 
-//apply blaclist if needed
+//apply blacklist if needed
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);

README.md

@@ -69,7 +69,7 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-## Immoscout
+### Immoscout
 I have added **experimental** support for Immoscout. Immoscout is somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
 To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
@@ -77,9 +77,15 @@ The rest will be handled by _Fredy_. Keep in mind, the support is experimental.
 
 If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
 
-#### Contribution guidelines  
+### Contribution guidelines  
 
-See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)  
+See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
+
+### Monitoring
+
+_Fredy_ can be monitored by [Instana](https://www.instana.com). If you are interested, sign up for a free trial. This is totally optional of course :)  
+If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.   
+If you want to know more, head over to the [Instana docs](https://www.ibm.com/docs/en/obi/current?topic=technologies-monitoring-nodejs).
 
 # Docker   
 Use the Dockerfile in this repository to build an image.  

doc/Untitled Diagram.drawio

@@ -0,0 +1,84 @@
+<mxfile host="app.diagrams.net" modified="2022-01-29T18:34:51.211Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" etag="W0jmvptvMSkuHq89hwUy" version="16.5.2" type="github">
+  <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
+    <mxGraphModel dx="850" dy="907" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-3" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="Job1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="100" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-2">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-3">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-4">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-7" value="FredyRuntime" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="110" y="120" width="360" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-0" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-0" value="Job2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-7" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-1" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-1" value="Job3" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-2" target="4kAlOAlRylSy7JMoHAEd-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-2" value="Provider1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="100" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-3">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="290" y="290" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-3" value="Provider2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-4" target="4kAlOAlRylSy7JMoHAEd-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-4" value="Provider3" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-12" target="4kAlOAlRylSy7JMoHAEd-16">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-12" value="Similarity check" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="110" y="290" width="360" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-16" target="4kAlOAlRylSy7JMoHAEd-18">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-16" value="Found similarity" style="rhombus;whiteSpace=wrap;html=1;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="250" y="360" width="80" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-18" target="4kAlOAlRylSy7JMoHAEd-19">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-18" value="Notification Adapter1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="460" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-19" value="Notification Adapter2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="520" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-22" value="No" style="text;html=1;resizable=0;autosize=1;align=center;verticalAlign=middle;points=[];fillColor=none;strokeColor=none;rounded=0;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="300" y="440" width="30" height="20" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

index.js

@@ -1,3 +1,16 @@
+require('dotenv').config();
+/********OPTIONAL INSTANA INITIALIZATION BEGIN********/
+//if you want to use Instana to monitor fredy, go to https://www.instana.com and
+// try it yourself by signing up for a free trial
+const { INSTANA_MONITORING } = process.env;
+if (INSTANA_MONITORING != null && INSTANA_MONITORING === 'true') {
+  /* eslint-disable no-console */
+  console.debug('Starting Instana monitoring');
+  /* eslint-enable no-console */
+  require('@instana/collector')();
+}
+/********OPTIONAL INSTANA INITIALIZATION END********/
+
 const fs = require('fs');
 
 //if db folder does not exist, ensure to create it before loading anything else

lib/FredyRuntime.js

@@ -62,7 +62,6 @@ class FredyRuntime {
         if (this._providerConfig.paginate != null) {
           xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
             //the first 2 pages should be enough here
-            //TODO: Think about automagically sort by date
             .limit(2)
             .paginate(this._providerConfig.paginate)
             .then((listings) => {

lib/notification/adapter/mattermost.js

@@ -0,0 +1,52 @@
+const { markdown2Html } = require('../../services/markdown');
+const { getJob } = require('../../services/storage/jobStorage');
+const axios = require('axios');
+
+/**
+ * sends new listings to mattermost
+ * @param serviceName e.g immowelt
+ * @param newListings an array with newly found listings
+ * @param notificationConfig config of this notification adapter
+ * @param jobKey name of the current job that is being executed
+ * @returns {Promise<Void> | void}
+ */
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+
+  let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
+  message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
+  message += newListings.map(
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+  );
+
+  return axios.post(`${webhook}`, {
+    channel: channel,
+    text: message,
+  });
+};
+
+/**
+ * exported config is being used in the frontend to generate the fields
+ * incoming values will be the keys (and values) of the fields
+ *
+ */
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Mattermost',
+  readme: markdown2Html('lib/notification/adapter/mattermost.md'),
+  description: 'Fredy will send new listings to your mattermost team chat.',
+  fields: {
+    webhook: {
+      type: 'text',
+      label: 'Webhook-URL',
+      description: 'The incoming webhook url',
+    },
+    channel: {
+      type: 'text',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/mattermost.md

@@ -0,0 +1,5 @@
+### Mattermost Adapter
+
+For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the  [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+
+As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.

lib/notification/adapter/sqlite.js

@@ -0,0 +1,33 @@
+const { markdown2Html } = require('../../services/markdown');
+const Database = require('better-sqlite3');
+
+/**
+ * Stores data in a sqlite db in order to use the search results for later analytics
+ * @param serviceName e.g immowelt
+ * @param newListings an array with newly found listings
+ * @param jobKey name of the current job that is being executed
+ */
+exports.send = ({ serviceName, newListings, jobKey }) => {
+  const db = new Database('db/listings.db');
+  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Sqlite',
+  description: 'This adapter stores listings in a local sqlite3 database.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,3 @@
+### Sqlite Adapter
+
+This adapter stores search results in an sqlite database in db/listings.db

lib/provider/immonet.js

@@ -6,7 +6,7 @@ function normalize(o) {
   const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
-  const address = o.address.split(' • ')[1];
+  const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
   //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
   //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)

lib/services/requestDriver.js

@@ -7,30 +7,29 @@ function makeDriver(headers = {}) {
   let cookies = '';
 
   return async function driver(context, callback) {
-    const url = context.url;
-    let result;
     try {
-      result = await axios({
+      const url = context.url;
+      const result = await axios({
         url,
         headers: {
           ...headers,
           Cookie: cookies,
         },
       });
+
+      if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
+        //assume we have gotten a response from scrapingAnt
+        if (cookies.length === 0) {
+          cookies = result.data.cookies;
+        }
+        callback(null, result.data.content);
+      } else {
+        callback(null, result.data);
+      }
     } catch (exception) {
       console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
       callback(null, []);
     }
-
-    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
-      //assume we have gotten a response from scrapingAnt
-      if (cookies.length === 0) {
-        cookies = result.data.cookies;
-      }
-      callback(null, result.data.content);
-    } else {
-      callback(null, result.data);
-    }
   };
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "5.4.4",
+  "version": "5.4.6",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -53,13 +53,16 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@instana/collector": "^1.137.5",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "7.6.0",
     "axios": "0.24.0",
     "axios-retry": "^3.2.4",
+    "better-sqlite3": "^7.5.0",
     "body-parser": "1.19.0",
     "cookie-session": "1.4.0",
+    "dotenv": "^15.0.0",
     "handlebars": "4.7.7",
     "highcharts": "9.3.1",
     "highcharts-react-official": "3.1.0",

test/provider/immowelt.test.js

@@ -32,13 +32,13 @@ describe('#immowelt testsuite()', () => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
+
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
 
           /** check the values if possible **/
-          if (notify.size.trim().toLowerCase() !== 'k.a.') {
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');
           }
           expect(notify.title).to.be.not.empty;