From 8a1d86c90efa9c0fccbc1eba624851cab9e9c1d5 Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Tue, 13 Feb 2018 12:43:37 +0100 Subject: [PATCH] ADD: new Neubaukompass provider | FIX: tests, console notifier --- README.md | 4 +++ conf/config.example | 3 ++ conf/config.test.json | 4 +++ lib/notification/adapter/console.js | 3 +- lib/provider/neubauKompass.js | 35 ++++++++++++++++++++ package.json | 8 ++--- test/einsAImmobilien.test.js | 27 +++++++++------- test/immonet.test.js | 30 +++++++++-------- test/immoscout.test.js | 34 +++++++++++--------- test/immowelt.test.js | 34 +++++++++++--------- test/kalaydo.test.js | 26 ++++++++------- test/kleinanzeigen.test.js | 34 +++++++++++--------- test/mocks/mockNotification.js | 1 + test/neubauKompass.test.js | 50 +++++++++++++++++++++++++++++ 14 files changed, 201 insertions(+), 92 deletions(-) create mode 100755 lib/provider/neubauKompass.js create mode 100644 test/neubauKompass.test.js diff --git a/README.md b/README.md index 62118cf..f3ab81f 100755 --- a/README.md +++ b/README.md @@ -91,6 +91,10 @@ These are the current provider that are already implemented within _Fredy_ "einsAImmobilien": { "url": "https://www.1a-immobilienmarkt.de/...", "enabled": true +}, +"neubauKompass": { + "url": "https://www.neubaukompass.de/...", + "enabled": true } ``` diff --git a/conf/config.example b/conf/config.example index 997ba64..a81bcd4 100755 --- a/conf/config.example +++ b/conf/config.example @@ -41,6 +41,9 @@ "einsAImmobilien":{ "url": "https://www.1a-immobilienmarkt.de/...", "enabled": true + },"neubauKompass": { + "url": "https://www.neubaukompass.de/...", + "enabled": true } }, "blacklist": [ diff --git a/conf/config.test.json b/conf/config.test.json index 8f13f7a..cf88e2c 100755 --- a/conf/config.test.json +++ b/conf/config.test.json @@ -41,6 +41,10 @@ "einsAImmobilien":{ "url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=fcfa4ee1e6cfaf791051a6f342eec1f8&sort_type=newest", "enabled": true + }, + "neubauKompass": { + "url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/", + "enabled": true } }, "blacklist": [ diff --git a/lib/notification/adapter/console.js b/lib/notification/adapter/console.js index 8548b56..3ce3660 100755 --- a/lib/notification/adapter/console.js +++ b/lib/notification/adapter/console.js @@ -4,10 +4,9 @@ const config = require('../../../conf/config.json'); * simply prints out the found data to the console * @param serviceName e.g immoscout * @param newListings an array with newly found listings - * @returns {Promise | void} */ exports.send = (serviceName, newListings) => { - return Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings)) + return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))]; }; /** diff --git a/lib/provider/neubauKompass.js b/lib/provider/neubauKompass.js new file mode 100755 index 0000000..cf4b7ec --- /dev/null +++ b/lib/provider/neubauKompass.js @@ -0,0 +1,35 @@ +const config = require('../../conf/config.json'); +const Fredy = require('../fredy'); +const utils = require('../utils'); + +function normalize(o) { + const title = o.title + '| '+o.subTitle; + //this is a bit nasty, but we do not have a size, therefor take the availability and set it as size to not modify notifications any furter + const size = o.available; + return Object.assign(o, { title, size }); +} + +function applyBlacklist(o) { + return !utils.isOneOf(o.title, config.blacklist); +} + +const neubauKompass = { + name: 'neubauKompass', + enabled: config.sources.neubauKompass.enabled, + url: config.sources.neubauKompass.url, + crawlContainer: '.md__property-list .post-list__item', + crawlFields: { + id: '@id', + price: '.entry__main .entry__data li:nth-child(1) span:nth-child(2) | removeNewline | trim', + available: '.entry__main .entry__data li:nth-child(3) span:nth-child(2) | removeNewline | trim', + title: '.entry__main .entry__title | removeNewline | trim', + link: '.entry__main .entry__title a@href', + subTitle: '.entry__main .entry__subtitle | removeNewline | trim', + address: '.entry__main .entry__info | removeNewline | trim' + }, + paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href', + normalize: normalize, + filter: applyBlacklist +}; + +module.exports = new Fredy(neubauKompass); diff --git a/package.json b/package.json index 80cf854..8ae5438 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Fredy", - "version": "1.1.0", + "version": "1.2.0", "description": "[F]ind [R]eal [E]states [d]amn eas[y].", "scripts": { "start": "node index.js", @@ -30,14 +30,14 @@ "dependencies": { "chai": "^4.1.2", "lowdb": "^1.0.0", - "mocha": "^4.1.0", + "mocha": "^5.0.0", "request-x-ray": "^0.1.4", - "slack": "^10.0.0", + "slack": "^10.1.1", "tg-yarl": "^1.3.0", "x-ray": "^2.3.2" }, "devDependencies": { - "prettier": "^1.0.0", + "prettier": "^1.10.2", "proxyquire": "^1.8.0" } } diff --git a/test/einsAImmobilien.test.js b/test/einsAImmobilien.test.js index e233d17..663d125 100644 --- a/test/einsAImmobilien.test.js +++ b/test/einsAImmobilien.test.js @@ -26,19 +26,22 @@ describe('#einsAImmobilien testsuite()', () => { expect(notificationObj).to.be.a('object'); expect(notificationObj.serviceName).to.equal('einsAImmobilien'); - /** check the actual structure **/ - expect(notificationObj.payload.id).to.be.a('number'); - expect(notificationObj.payload.price).to.be.a('string'); - expect(notificationObj.payload.size).to.be.a('string'); - expect(notificationObj.payload.title).to.be.a('string'); - expect(notificationObj.payload.link).to.be.a('string'); + notificationObj.payload.forEach((notify, idx) => { - /** check the values if possible **/ - expect(notificationObj.payload.id).to.equal(immonetDbContent.einsAImmobilien[immonetDbContent.einsAImmobilien.length - 1]); - expect(notificationObj.payload.price).that.does.include('EUR'); - expect(notificationObj.payload.size).that.does.include('m²'); - expect(notificationObj.payload.title).to.be.not.empty; - expect(notificationObj.payload.link).that.does.include('https://www.1a-immobilienmarkt.de'); + /** check the actual structure **/ + expect(notify.id).to.be.a('number'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + + /** check the values if possible **/ + expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]); + expect(notify.price).that.does.include('EUR'); + expect(notify.size).that.does.include('m²'); + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de'); + }); resolve(); }); }); diff --git a/test/immonet.test.js b/test/immonet.test.js index dbc02a0..1db3025 100644 --- a/test/immonet.test.js +++ b/test/immonet.test.js @@ -26,22 +26,24 @@ describe('#immonet testsuite()', () => { expect(notificationObj).to.be.a('object'); expect(notificationObj.serviceName).to.equal('immonet'); - /** check the actual structure **/ - expect(notificationObj.payload.id).to.be.a('number'); - expect(notificationObj.payload.price).to.be.a('string'); - expect(notificationObj.payload.size).to.be.a('string'); - expect(notificationObj.payload.title).to.be.a('string'); - expect(notificationObj.payload.link).to.be.a('string'); - expect(notificationObj.payload.address).to.be.a('string'); - /** check the values if possible **/ - expect(notificationObj.payload.id).to.equal(immonetDbContent.immonet[immonetDbContent.immonet.length - 1]); - expect(notificationObj.payload.price).that.does.include('€'); - expect(notificationObj.payload.size).that.does.include('m²'); - expect(notificationObj.payload.title).to.be.not.empty; - expect(notificationObj.payload.link).that.does.include('https://www.immonet.de'); - expect(notificationObj.payload.address).to.be.not.empty; + notificationObj.payload.forEach((notify, idx) => { + /** check the actual structure **/ + expect(notify.id).to.be.a('number'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + expect(notify.address).to.be.a('string'); + /** check the values if possible **/ + expect(notify.id).to.equal(immonetDbContent.immonet[idx]); + expect(notify.price).that.does.include('€'); + expect(notify.size).that.does.include('m²'); + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.immonet.de'); + expect(notify.address).to.be.not.empty; + }); resolve(); }); }); diff --git a/test/immoscout.test.js b/test/immoscout.test.js index 26b3ea2..18cbf25 100644 --- a/test/immoscout.test.js +++ b/test/immoscout.test.js @@ -25,24 +25,26 @@ describe('#immoscout testsuite()', () => { expect(notificationObj).to.be.a('object'); expect(notificationObj.serviceName).to.equal('immoscout'); - /** check the actual structure **/ - expect(notificationObj.payload.id).to.be.a('number'); - expect(notificationObj.payload.price).to.be.a('string'); - expect(notificationObj.payload.size).to.be.a('string'); - expect(notificationObj.payload.title).to.be.a('string'); - expect(notificationObj.payload.link).to.be.a('string'); - expect(notificationObj.payload.address).to.be.a('string'); - /** check the values if possible **/ - expect(notificationObj.payload.id).to.equal( - immoscoutDbContent.immoscout[immoscoutDbContent.immoscout.length - 1] - ); - expect(notificationObj.payload.price).that.does.include('€'); - expect(notificationObj.payload.size).that.does.include('m²'); - expect(notificationObj.payload.title).to.be.not.empty; - expect(notificationObj.payload.link).that.does.include('https://www.immobilienscout24.de'); - expect(notificationObj.payload.address).to.be.not.empty; + notificationObj.payload.forEach((notify, idx) => { + /** check the actual structure **/ + expect(notify.id).to.be.a('number'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + expect(notify.address).to.be.a('string'); + /** check the values if possible **/ + expect(notify.id).to.equal( + immoscoutDbContent.immoscout[idx] + ); + expect(notify.price).that.does.include('€'); + expect(notify.size).that.does.include('m²'); + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.immobilienscout24.de'); + expect(notify.address).to.be.not.empty; + }); resolve(); }).catch(resolve); }); diff --git a/test/immowelt.test.js b/test/immowelt.test.js index 1ff7258..cb70691 100644 --- a/test/immowelt.test.js +++ b/test/immowelt.test.js @@ -26,24 +26,26 @@ describe('#immowelt testsuite()', () => { expect(notificationObj).to.be.a('object'); expect(notificationObj.serviceName).to.equal('immowelt'); - /** check the actual structure **/ - expect(notificationObj.payload.id).to.be.a('number'); - expect(notificationObj.payload.price).to.be.a('string'); - expect(notificationObj.payload.size).to.be.a('string'); - expect(notificationObj.payload.title).to.be.a('string'); - expect(notificationObj.payload.link).to.be.a('string'); - expect(notificationObj.payload.address).to.be.a('string'); + notificationObj.payload.forEach((notify, idx) => { - /** check the values if possible **/ - expect(notificationObj.payload.id).to.equal( - immoweltDbContent.immowelt[immoweltDbContent.immowelt.length - 1] - ); - expect(notificationObj.payload.price).that.does.include('€'); - expect(notificationObj.payload.size).that.does.include('m²'); - expect(notificationObj.payload.title).to.be.not.empty; - expect(notificationObj.payload.link).that.does.include('https://www.immowelt.de'); - expect(notificationObj.payload.address).to.be.not.empty; + /** check the actual structure **/ + expect(notify.id).to.be.a('number'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + expect(notify.address).to.be.a('string'); + /** check the values if possible **/ + expect(notify.id).to.equal( + immoweltDbContent.immowelt[idx] + ); + expect(notify.price).that.does.include('€'); + expect(notify.size).that.does.include('m²'); + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.immowelt.de'); + expect(notify.address).to.be.not.empty; + }); resolve(); }); }); diff --git a/test/kalaydo.test.js b/test/kalaydo.test.js index 4abd10c..b852ead 100644 --- a/test/kalaydo.test.js +++ b/test/kalaydo.test.js @@ -27,20 +27,22 @@ describe('#kalaydo testsuite()', () => { expect(notificationObj).to.be.a('object'); expect(notificationObj.serviceName).to.equal('kalaydo'); - /** check the actual structure **/ - expect(notificationObj.payload.id).to.be.a('string'); - expect(notificationObj.payload.price).to.be.a('string'); - expect(notificationObj.payload.size).to.be.a('string'); - expect(notificationObj.payload.title).to.be.a('string'); - expect(notificationObj.payload.link).to.be.a('string'); + notificationObj.payload.forEach((notify, idx) => { - /** check the values if possible **/ - expect(notificationObj.payload.id).to.equal(kalaydoDbContent.kalaydo[kalaydoDbContent.kalaydo.length - 1]); - expect(notificationObj.payload.price).that.does.include('€'); - expect(notificationObj.payload.size).that.does.include('m²'); - expect(notificationObj.payload.title).to.be.not.empty; - expect(notificationObj.payload.link).that.does.include('https://www.kalaydo.de'); + /** check the actual structure **/ + expect(notify.id).to.be.a('string'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + /** check the values if possible **/ + expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]); + expect(notify.price).that.does.include('€'); + expect(notify.size).that.does.include('m²'); + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.kalaydo.de'); + }); resolve(); }); }); diff --git a/test/kleinanzeigen.test.js b/test/kleinanzeigen.test.js index ef35f3a..53bd7b2 100644 --- a/test/kleinanzeigen.test.js +++ b/test/kleinanzeigen.test.js @@ -26,24 +26,26 @@ describe('#kleinanzeigen testsuite()', () => { expect(notificationObj).to.be.a('object'); expect(notificationObj.serviceName).to.equal('kleinanzeigen'); - /** check the actual structure **/ - expect(notificationObj.payload.id).to.be.a('number'); - expect(notificationObj.payload.price).to.be.a('string'); - expect(notificationObj.payload.size).to.be.a('string'); - expect(notificationObj.payload.title).to.be.a('string'); - expect(notificationObj.payload.link).to.be.a('string'); - expect(notificationObj.payload.address).to.be.a('string'); + notificationObj.payload.forEach((notify, idx) => { - /** check the values if possible **/ - expect(notificationObj.payload.id).to.equal( - kleinanzeigenDbContent.kleinanzeigen[kleinanzeigenDbContent.kleinanzeigen.length - 1] - ); - expect(notificationObj.payload.price).that.does.include('€'); - expect(notificationObj.payload.size).that.does.include('m²'); - expect(notificationObj.payload.title).to.be.not.empty; - expect(notificationObj.payload.link).that.does.include('https://www.ebay-kleinanzeigen.de'); - expect(notificationObj.payload.address).to.be.not.empty; + /** check the actual structure **/ + expect(notify.id).to.be.a('number'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + expect(notify.address).to.be.a('string'); + /** check the values if possible **/ + expect(notify.id).to.equal( + kleinanzeigenDbContent.kleinanzeigen[idx] + ); + expect(notify.price).that.does.include('€'); + expect(notify.size).that.does.include('m²'); + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de'); + expect(notify.address).to.be.not.empty; + }); resolve(); }); }); diff --git a/test/mocks/mockNotification.js b/test/mocks/mockNotification.js index 2cd6f42..9bee5f8 100644 --- a/test/mocks/mockNotification.js +++ b/test/mocks/mockNotification.js @@ -3,6 +3,7 @@ module.exports = { send: (serviceName, payload) => { this._tmpStore = { serviceName, payload }; + return [Promise.resolve()]; }, get: () => { diff --git a/test/neubauKompass.test.js b/test/neubauKompass.test.js new file mode 100644 index 0000000..dd27c86 --- /dev/null +++ b/test/neubauKompass.test.js @@ -0,0 +1,50 @@ +const mockNotification = require('./mocks/mockNotification'); +const mockConfig = require('../conf/config.test'); +const mockStore = require('./mocks/mockStore'); +const mockStats = require('./mocks/mockStats'); +const proxyquire = require('proxyquire').noCallThru(); +const expect = require('chai').expect; + +describe('#neubauKompass testsuite()', () => { + + const neubauKompass = proxyquire('../lib/provider/neubauKompass', { + '../../conf/config.json': mockConfig, + '../lib/fredy': proxyquire('../lib/fredy', { + './services/store': mockStore, + './notification/notify': mockNotification + }) + }); + + it('should test neubauKompass provider', async () => { + return await new Promise(resolve => { + neubauKompass.run(mockStats).then(() => { + const neubauKompassDbContent = neubauKompass._getStore()._db; + expect(neubauKompassDbContent.neubauKompass).to.be.a('array'); + + const notificationObj = mockNotification.get(); + expect(notificationObj.serviceName).to.equal('neubauKompass'); + + notificationObj.payload.forEach((notify, idx) => { + expect(notify).to.be.a('object'); + + /** check the actual structure **/ + expect(notify.id).to.be.a('string'); + expect(notify.price).to.be.a('string'); + expect(notify.size).to.be.a('string'); + expect(notify.title).to.be.a('string'); + expect(notify.link).to.be.a('string'); + expect(notify.address).to.be.a('string'); + + /** check the values if possible **/ + expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]); + expect(notify.price).to.be.not.empty; + expect(notify.size).to.be.not.empty; + expect(notify.title).to.be.not.empty; + expect(notify.link).that.does.include('https://www.neubaukompass.de'); + expect(notify.address).to.be.not.empty; + }); + resolve(); + }); + }); + }); +});