External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

ADD: new Neubaukompass provider | FIX: tests, console notifier

Browse Source
This commit is contained in:
Christian Kellner
2018-02-13 12:43:37 +01:00
parent 35bdc9e78c
commit 8a1d86c90e
14 changed files with 201 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -91,6 +91,10 @@ These are the current provider that are already implemented within _Fredy_
"einsAImmobilien": { "einsAImmobilien": {
"url": "https://www.1a-immobilienmarkt.de/...", "url": "https://www.1a-immobilienmarkt.de/...",
"enabled": true "enabled": true
},
"neubauKompass": {
"url": "https://www.neubaukompass.de/...",
"enabled": true
} }
``` ```

3
conf/config.example
View File

@@ -41,6 +41,9 @@
"einsAImmobilien":{ "einsAImmobilien":{
"url": "https://www.1a-immobilienmarkt.de/...", "url": "https://www.1a-immobilienmarkt.de/...",
"enabled": true "enabled": true
},"neubauKompass": {
"url": "https://www.neubaukompass.de/...",
"enabled": true
} }
}, },
"blacklist": [ "blacklist": [

4
conf/config.test.json
View File

@@ -41,6 +41,10 @@
"einsAImmobilien":{ "einsAImmobilien":{
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=fcfa4ee1e6cfaf791051a6f342eec1f8&sort_type=newest", "url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=fcfa4ee1e6cfaf791051a6f342eec1f8&sort_type=newest",
"enabled": true "enabled": true
},
"neubauKompass": {
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
"enabled": true
} }
}, },
"blacklist": [ "blacklist": [

3
lib/notification/adapter/console.js
View File

@@ -4,10 +4,9 @@ const config = require('../../../conf/config.json');
* simply prints out the found data to the console * simply prints out the found data to the console
* @param serviceName e.g immoscout * @param serviceName e.g immoscout
* @param newListings an array with newly found listings * @param newListings an array with newly found listings
* @returns {Promise<Void> | void}
*/ */
exports.send = (serviceName, newListings) => { exports.send = (serviceName, newListings) => {
return Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings)) return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
}; };
/** /**

35
lib/provider/neubauKompass.js Executable file
View File

@@ -0,0 +1,35 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
function normalize(o) {
const title = o.title + '| '+o.subTitle;
//this is a bit nasty, but we do not have a size, therefor take the availability and set it as size to not modify notifications any furter
const size = o.available;
return Object.assign(o, { title, size });
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, config.blacklist);
}
const neubauKompass = {
name: 'neubauKompass',
enabled: config.sources.neubauKompass.enabled,
url: config.sources.neubauKompass.url,
crawlContainer: '.md__property-list .post-list__item',
crawlFields: {
id: '@id',
price: '.entry__main .entry__data li:nth-child(1) span:nth-child(2) | removeNewline | trim',
available: '.entry__main .entry__data li:nth-child(3) span:nth-child(2) | removeNewline | trim',
title: '.entry__main .entry__title | removeNewline | trim',
link: '.entry__main .entry__title a@href',
subTitle: '.entry__main .entry__subtitle | removeNewline | trim',
address: '.entry__main .entry__info | removeNewline | trim'
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,
filter: applyBlacklist
};
module.exports = new Fredy(neubauKompass);

8
package.json
View File

@@ -1,6 +1,6 @@
{ {
"name": "Fredy", "name": "Fredy",
"version": "1.1.0", "version": "1.2.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"start": "node index.js", "start": "node index.js",
@@ -30,14 +30,14 @@
"dependencies": { "dependencies": {
"chai": "^4.1.2", "chai": "^4.1.2",
"lowdb": "^1.0.0", "lowdb": "^1.0.0",
"mocha": "^4.1.0", "mocha": "^5.0.0",
"request-x-ray": "^0.1.4", "request-x-ray": "^0.1.4",
"slack": "^10.0.0", "slack": "^10.1.1",
"tg-yarl": "^1.3.0", "tg-yarl": "^1.3.0",
"x-ray": "^2.3.2" "x-ray": "^2.3.2"
}, },
"devDependencies": { "devDependencies": {
"prettier": "^1.0.0", "prettier": "^1.10.2",
"proxyquire": "^1.8.0" "proxyquire": "^1.8.0"
} }
} }

27
test/einsAImmobilien.test.js
View File

@@ -26,19 +26,22 @@ describe('#einsAImmobilien testsuite()', () => {
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('einsAImmobilien'); expect(notificationObj.serviceName).to.equal('einsAImmobilien');
/** check the actual structure **/ notificationObj.payload.forEach((notify, idx) => {
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
/** check the values if possible **/ /** check the actual structure **/
expect(notificationObj.payload.id).to.equal(immonetDbContent.einsAImmobilien[immonetDbContent.einsAImmobilien.length - 1]); expect(notify.id).to.be.a('number');
expect(notificationObj.payload.price).that.does.include('EUR'); expect(notify.price).to.be.a('string');
expect(notificationObj.payload.size).that.does.include('m²'); expect(notify.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.not.empty; expect(notify.title).to.be.a('string');
expect(notificationObj.payload.link).that.does.include('https://www.1a-immobilienmarkt.de'); expect(notify.link).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
expect(notify.price).that.does.include('EUR');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');
});
resolve(); resolve();
}); });
}); });

30
test/immonet.test.js
View File

@@ -26,22 +26,24 @@ describe('#immonet testsuite()', () => {
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet'); expect(notificationObj.serviceName).to.equal('immonet');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
/** check the values if possible **/ notificationObj.payload.forEach((notify, idx) => {
expect(notificationObj.payload.id).to.equal(immonetDbContent.immonet[immonetDbContent.immonet.length - 1]); /** check the actual structure **/
expect(notificationObj.payload.price).that.does.include('€'); expect(notify.id).to.be.a('number');
expect(notificationObj.payload.size).that.does.include('m²'); expect(notify.price).to.be.a('string');
expect(notificationObj.payload.title).to.be.not.empty; expect(notify.size).to.be.a('string');
expect(notificationObj.payload.link).that.does.include('https://www.immonet.de'); expect(notify.title).to.be.a('string');
expect(notificationObj.payload.address).to.be.not.empty; expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immonet.de');
expect(notify.address).to.be.not.empty;
});
resolve(); resolve();
}); });
}); });

34
test/immoscout.test.js
View File

@@ -25,24 +25,26 @@ describe('#immoscout testsuite()', () => {
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immoscout'); expect(notificationObj.serviceName).to.equal('immoscout');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
/** check the values if possible **/ notificationObj.payload.forEach((notify, idx) => {
expect(notificationObj.payload.id).to.equal( /** check the actual structure **/
immoscoutDbContent.immoscout[immoscoutDbContent.immoscout.length - 1] expect(notify.id).to.be.a('number');
); expect(notify.price).to.be.a('string');
expect(notificationObj.payload.price).that.does.include('€'); expect(notify.size).to.be.a('string');
expect(notificationObj.payload.size).that.does.include('m²'); expect(notify.title).to.be.a('string');
expect(notificationObj.payload.title).to.be.not.empty; expect(notify.link).to.be.a('string');
expect(notificationObj.payload.link).that.does.include('https://www.immobilienscout24.de'); expect(notify.address).to.be.a('string');
expect(notificationObj.payload.address).to.be.not.empty;
/** check the values if possible **/
expect(notify.id).to.equal(
immoscoutDbContent.immoscout[idx]
);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
expect(notify.address).to.be.not.empty;
});
resolve(); resolve();
}).catch(resolve); }).catch(resolve);
}); });

34
test/immowelt.test.js
View File

@@ -26,24 +26,26 @@ describe('#immowelt testsuite()', () => {
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immowelt'); expect(notificationObj.serviceName).to.equal('immowelt');
/** check the actual structure **/ notificationObj.payload.forEach((notify, idx) => {
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
/** check the values if possible **/ /** check the actual structure **/
expect(notificationObj.payload.id).to.equal( expect(notify.id).to.be.a('number');
immoweltDbContent.immowelt[immoweltDbContent.immowelt.length - 1] expect(notify.price).to.be.a('string');
); expect(notify.size).to.be.a('string');
expect(notificationObj.payload.price).that.does.include('€'); expect(notify.title).to.be.a('string');
expect(notificationObj.payload.size).that.does.include('m²'); expect(notify.link).to.be.a('string');
expect(notificationObj.payload.title).to.be.not.empty; expect(notify.address).to.be.a('string');
expect(notificationObj.payload.link).that.does.include('https://www.immowelt.de');
expect(notificationObj.payload.address).to.be.not.empty;
/** check the values if possible **/
expect(notify.id).to.equal(
immoweltDbContent.immowelt[idx]
);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immowelt.de');
expect(notify.address).to.be.not.empty;
});
resolve(); resolve();
}); });
}); });

26
test/kalaydo.test.js
View File

@@ -27,20 +27,22 @@ describe('#kalaydo testsuite()', () => {
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('kalaydo'); expect(notificationObj.serviceName).to.equal('kalaydo');
/** check the actual structure **/ notificationObj.payload.forEach((notify, idx) => {
expect(notificationObj.payload.id).to.be.a('string');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
/** check the values if possible **/ /** check the actual structure **/
expect(notificationObj.payload.id).to.equal(kalaydoDbContent.kalaydo[kalaydoDbContent.kalaydo.length - 1]); expect(notify.id).to.be.a('string');
expect(notificationObj.payload.price).that.does.include('€'); expect(notify.price).to.be.a('string');
expect(notificationObj.payload.size).that.does.include('m²'); expect(notify.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.not.empty; expect(notify.title).to.be.a('string');
expect(notificationObj.payload.link).that.does.include('https://www.kalaydo.de'); expect(notify.link).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.kalaydo.de');
});
resolve(); resolve();
}); });
}); });

34
test/kleinanzeigen.test.js
View File

@@ -26,24 +26,26 @@ describe('#kleinanzeigen testsuite()', () => {
expect(notificationObj).to.be.a('object'); expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('kleinanzeigen'); expect(notificationObj.serviceName).to.equal('kleinanzeigen');
/** check the actual structure **/ notificationObj.payload.forEach((notify, idx) => {
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
/** check the values if possible **/ /** check the actual structure **/
expect(notificationObj.payload.id).to.equal( expect(notify.id).to.be.a('number');
kleinanzeigenDbContent.kleinanzeigen[kleinanzeigenDbContent.kleinanzeigen.length - 1] expect(notify.price).to.be.a('string');
); expect(notify.size).to.be.a('string');
expect(notificationObj.payload.price).that.does.include('€'); expect(notify.title).to.be.a('string');
expect(notificationObj.payload.size).that.does.include('m²'); expect(notify.link).to.be.a('string');
expect(notificationObj.payload.title).to.be.not.empty; expect(notify.address).to.be.a('string');
expect(notificationObj.payload.link).that.does.include('https://www.ebay-kleinanzeigen.de');
expect(notificationObj.payload.address).to.be.not.empty;
/** check the values if possible **/
expect(notify.id).to.equal(
kleinanzeigenDbContent.kleinanzeigen[idx]
);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
expect(notify.address).to.be.not.empty;
});
resolve(); resolve();
}); });
}); });

1
test/mocks/mockNotification.js
View File

@@ -3,6 +3,7 @@ module.exports = {
send: (serviceName, payload) => { send: (serviceName, payload) => {
this._tmpStore = { serviceName, payload }; this._tmpStore = { serviceName, payload };
return [Promise.resolve()];
}, },
get: () => { get: () => {

50
test/neubauKompass.test.js Normal file
View File

@@ -0,0 +1,50 @@
const mockNotification = require('./mocks/mockNotification');
const mockConfig = require('../conf/config.test');
const mockStore = require('./mocks/mockStore');
const mockStats = require('./mocks/mockStats');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
describe('#neubauKompass testsuite()', () => {
const neubauKompass = proxyquire('../lib/provider/neubauKompass', {
'../../conf/config.json': mockConfig,
'../lib/fredy': proxyquire('../lib/fredy', {
'./services/store': mockStore,
'./notification/notify': mockNotification
})
});
it('should test neubauKompass provider', async () => {
return await new Promise(resolve => {
neubauKompass.run(mockStats).then(() => {
const neubauKompassDbContent = neubauKompass._getStore()._db;
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
const notificationObj = mockNotification.get();
expect(notificationObj.serviceName).to.equal('neubauKompass');
notificationObj.payload.forEach((notify, idx) => {
expect(notify).to.be.a('object');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
expect(notify.price).to.be.not.empty;
expect(notify.size).to.be.not.empty;
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.neubaukompass.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});
});
});