next version

Instead of truncating the original URL of each provider in the job configuration to 60 characters and losing a lot of context information, put a link to the original URL in the provider table which can be opened directly to verify what is being scraped by Fredy.

lib/api/routes/loginRoute.js

@@ -5,13 +5,13 @@ const hasher = require('../../services/security/hash');
 
 loginRouter.get('/user', async (req, res) => {
   const currentUserId = req.session.currentUser;
-  const isAdmin = currentUserId == null ? false : userStorage.getUser(currentUserId).isAdmin;
+  const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
-  if (currentUserId == null) {
+  if (currentUser == null) {
     res.body = {};
   } else {
     res.body = {
-      userId: currentUserId,
+      userId: currentUser.id,
-      isAdmin,
+      isAdmin: currentUser.isAdmin,
     };
   }
   res.send();

lib/notification/adapter/telegram.js

@@ -1,4 +1,5 @@
 const { markdown2Html } = require('../../services/markdown');
+const { getJob } = require('../../services/storage/jobStorage');
 const axios = require('axios');
 
 /**
@@ -19,23 +20,24 @@ const arrayChunks = (inputArray, perChunk) =>
  * @param serviceName e.g immowelt
  * @param newListings an array with newly found listings
  * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
+ * @param jobKey name of the current job that is being executed
  * @returns {Promise<Void> | void}
  */
 exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
 
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
   const chunks = arrayChunks(newListings, 3);
 
   const promises = chunks.map((chunk) => {
-    let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
+    let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
     message += chunk.map(
       (o) =>
-        `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
+        `<a href="${o.link}"><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
         [o.address, o.price, o.size].join(' | ') +
-        '\n' +
+        '\n\n'
-        `<a href="${o.link}">${o.link}</a>\n\n`
     );
 
     return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {

lib/provider/immoswp.js

@@ -0,0 +1,52 @@
+const utils = require('../utils');
+
+let appliedBlackList = [];
+
+function normalize(o) {
+  const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+  const size = o.size || 'N/A m²';
+  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+  const address = o.address || 'No address available';
+  const title = o.title || 'No title available';
+  const link = `https://immo.swp.de/immobilien/${id}`;
+  const description = o.description;
+  return Object.assign(o, { id, address, price, size, title, link, description });
+}
+
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+
+const config = {
+  url: null,
+  crawlContainer: '.js-serp-item',
+  sortByDateParam: 's=most_recently_updated_first',
+  crawlFields: {
+    id: '@id',
+    price: 'div.item__spec.item-spec-price | trim',
+    size: 'div.item__spec.item-spec-area | trim',
+    title: 'a.js-item-title-link@title',
+    address: 'div.item__locality | removeNewline | trim',
+    description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
+  },
+  paginate: 'li.page-item.pagination__item a.page-link@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+
+exports.metaInformation = {
+  name: 'Immo Südwest Presse',
+  baseUrl: 'https://immo.swp.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
+
+exports.config = config;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "5.4.3",
+  "version": "5.4.4",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",

test/provider/immoswp.test.js

@@ -0,0 +1,51 @@
+const similarityCache = require('../../lib/services/similarity-check/similarityCache');
+const mockNotification = require('../mocks/mockNotification');
+const providerConfig = require('./testProvider.json');
+const mockStore = require('../mocks/mockStore');
+const proxyquire = require('proxyquire').noCallThru();
+const expect = require('chai').expect;
+const provider = require('../../lib/provider/immoswp');
+
+describe('#immoswp testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+
+  provider.init(providerConfig.immoswp, [], []);
+  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    './services/storage/listingsStorage': {
+      ...mockStore,
+    },
+    './notification/notify': mockNotification,
+  });
+
+  it('should test immoswp provider', async () => {
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+
+        const notificationObj = mockNotification.get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immoswp');
+
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://immo.swp.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/testProvider.json

@@ -16,6 +16,10 @@
     "url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search",
     "enabled": true
   },
+  "immoswp": {
+    "url": "https://immo.swp.de/suchergebnisse?l=M%C3%BCnchen&r=0km&_multiselect_r=0km&ut=private&t=apartment%3Arental&a=de.muenchen&pf=&pt=&rf=0&rt=0&sf=50&st=&yf=&yt=&ff=&ft=&s=most_recently_updated_first&pa=&o=&ad=&u=",
+    "enabled": true
+  },
   "kalaydo": {
     "url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
     "enabled": true

ui/src/components/table/ProviderTable.js

@@ -12,10 +12,6 @@ const emptyTable = () => {
   );
 };
 
-const truncate = (str, n) => {
-  return str.length > n ? str.substr(0, n - 1) + '…' : str;
-};
-
 const content = (providerData, onRemove) => {
   return (
     <Fragment>
@@ -23,7 +19,11 @@ const content = (providerData, onRemove) => {
         return (
           <Table.Row key={data.id}>
             <Table.Cell>{data.name}</Table.Cell>
-            <Table.Cell>{truncate(data.url, 60)}</Table.Cell>
+            <Table.Cell>
+              <a href={data.url} target="_blank" rel="noopener noreferrer">
+                Visit site
+              </a>
+            </Table.Cell>
             <Table.Cell>
               <div style={{ float: 'right' }}>
                 <Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />