External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

ADD: new Neubaukompass provider | FIX: tests, console notifier

Browse Source
This commit is contained in:
Christian Kellner
2018-02-13 12:43:37 +01:00
parent 35bdc9e78c
commit 8a1d86c90e
14 changed files with 201 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -91,6 +91,10 @@ These are the current provider that are already implemented within _Fredy_
"einsAImmobilien": {
"url": "https://www.1a-immobilienmarkt.de/...",
"enabled": true
},
"neubauKompass": {
"url": "https://www.neubaukompass.de/...",
"enabled": true
}
```

3
conf/config.example
View File

@@ -41,6 +41,9 @@
"einsAImmobilien":{
"url": "https://www.1a-immobilienmarkt.de/...",
"enabled": true
},"neubauKompass": {
"url": "https://www.neubaukompass.de/...",
"enabled": true
}
},
"blacklist": [

4
conf/config.test.json
View File

@@ -41,6 +41,10 @@
"einsAImmobilien":{
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=fcfa4ee1e6cfaf791051a6f342eec1f8&sort_type=newest",
"enabled": true
},
"neubauKompass": {
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
"enabled": true
}
},
"blacklist": [

3
lib/notification/adapter/console.js
View File

@@ -4,10 +4,9 @@ const config = require('../../../conf/config.json');
* simply prints out the found data to the console
* @param serviceName e.g immoscout
* @param newListings an array with newly found listings
* @returns {Promise<Void> | void}
*/
exports.send = (serviceName, newListings) => {
return Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))
return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
};
/**

35
lib/provider/neubauKompass.js Executable file
View File

@@ -0,0 +1,35 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
function normalize(o) {
const title = o.title + '| '+o.subTitle;
//this is a bit nasty, but we do not have a size, therefor take the availability and set it as size to not modify notifications any furter
const size = o.available;
return Object.assign(o, { title, size });
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, config.blacklist);
}
const neubauKompass = {
name: 'neubauKompass',
enabled: config.sources.neubauKompass.enabled,
url: config.sources.neubauKompass.url,
crawlContainer: '.md__property-list .post-list__item',
crawlFields: {
id: '@id',
price: '.entry__main .entry__data li:nth-child(1) span:nth-child(2) | removeNewline | trim',
available: '.entry__main .entry__data li:nth-child(3) span:nth-child(2) | removeNewline | trim',
title: '.entry__main .entry__title | removeNewline | trim',
link: '.entry__main .entry__title a@href',
subTitle: '.entry__main .entry__subtitle | removeNewline | trim',
address: '.entry__main .entry__info | removeNewline | trim'
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,
filter: applyBlacklist
};
module.exports = new Fredy(neubauKompass);

8
package.json
View File

@@ -1,6 +1,6 @@
{
"name": "Fredy",
"version": "1.1.0",
"version": "1.2.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"start": "node index.js",
@@ -30,14 +30,14 @@
"dependencies": {
"chai": "^4.1.2",
"lowdb": "^1.0.0",
"mocha": "^4.1.0",
"mocha": "^5.0.0",
"request-x-ray": "^0.1.4",
"slack": "^10.0.0",
"slack": "^10.1.1",
"tg-yarl": "^1.3.0",
"x-ray": "^2.3.2"
},
"devDependencies": {
"prettier": "^1.0.0",
"prettier": "^1.10.2",
"proxyquire": "^1.8.0"
}
}

27
test/einsAImmobilien.test.js
View File

@@ -26,19 +26,22 @@ describe('#einsAImmobilien testsuite()', () => {
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
notificationObj.payload.forEach((notify, idx) => {
/** check the values if possible **/
expect(notificationObj.payload.id).to.equal(immonetDbContent.einsAImmobilien[immonetDbContent.einsAImmobilien.length - 1]);
expect(notificationObj.payload.price).that.does.include('EUR');
expect(notificationObj.payload.size).that.does.include('m²');
expect(notificationObj.payload.title).to.be.not.empty;
expect(notificationObj.payload.link).that.does.include('https://www.1a-immobilienmarkt.de');
/** check the actual structure **/
expect(notify.id).to.be.a('number');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
expect(notify.price).that.does.include('EUR');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');
});
resolve();
});
});

30
test/immonet.test.js
View File

@@ -26,22 +26,24 @@ describe('#immonet testsuite()', () => {
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
/** check the values if possible **/
expect(notificationObj.payload.id).to.equal(immonetDbContent.immonet[immonetDbContent.immonet.length - 1]);
expect(notificationObj.payload.price).that.does.include('€');
expect(notificationObj.payload.size).that.does.include('m²');
expect(notificationObj.payload.title).to.be.not.empty;
expect(notificationObj.payload.link).that.does.include('https://www.immonet.de');
expect(notificationObj.payload.address).to.be.not.empty;
notificationObj.payload.forEach((notify, idx) => {
/** check the actual structure **/
expect(notify.id).to.be.a('number');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immonet.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});

34
test/immoscout.test.js
View File

@@ -25,24 +25,26 @@ describe('#immoscout testsuite()', () => {
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immoscout');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
/** check the values if possible **/
expect(notificationObj.payload.id).to.equal(
immoscoutDbContent.immoscout[immoscoutDbContent.immoscout.length - 1]
);
expect(notificationObj.payload.price).that.does.include('€');
expect(notificationObj.payload.size).that.does.include('m²');
expect(notificationObj.payload.title).to.be.not.empty;
expect(notificationObj.payload.link).that.does.include('https://www.immobilienscout24.de');
expect(notificationObj.payload.address).to.be.not.empty;
notificationObj.payload.forEach((notify, idx) => {
/** check the actual structure **/
expect(notify.id).to.be.a('number');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(
immoscoutDbContent.immoscout[idx]
);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
expect(notify.address).to.be.not.empty;
});
resolve();
}).catch(resolve);
});

34
test/immowelt.test.js
View File

@@ -26,24 +26,26 @@ describe('#immowelt testsuite()', () => {
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immowelt');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
notificationObj.payload.forEach((notify, idx) => {
/** check the values if possible **/
expect(notificationObj.payload.id).to.equal(
immoweltDbContent.immowelt[immoweltDbContent.immowelt.length - 1]
);
expect(notificationObj.payload.price).that.does.include('€');
expect(notificationObj.payload.size).that.does.include('m²');
expect(notificationObj.payload.title).to.be.not.empty;
expect(notificationObj.payload.link).that.does.include('https://www.immowelt.de');
expect(notificationObj.payload.address).to.be.not.empty;
/** check the actual structure **/
expect(notify.id).to.be.a('number');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(
immoweltDbContent.immowelt[idx]
);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immowelt.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});

26
test/kalaydo.test.js
View File

@@ -27,20 +27,22 @@ describe('#kalaydo testsuite()', () => {
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('kalaydo');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('string');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
notificationObj.payload.forEach((notify, idx) => {
/** check the values if possible **/
expect(notificationObj.payload.id).to.equal(kalaydoDbContent.kalaydo[kalaydoDbContent.kalaydo.length - 1]);
expect(notificationObj.payload.price).that.does.include('€');
expect(notificationObj.payload.size).that.does.include('m²');
expect(notificationObj.payload.title).to.be.not.empty;
expect(notificationObj.payload.link).that.does.include('https://www.kalaydo.de');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.kalaydo.de');
});
resolve();
});
});

34
test/kleinanzeigen.test.js
View File

@@ -26,24 +26,26 @@ describe('#kleinanzeigen testsuite()', () => {
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
/** check the actual structure **/
expect(notificationObj.payload.id).to.be.a('number');
expect(notificationObj.payload.price).to.be.a('string');
expect(notificationObj.payload.size).to.be.a('string');
expect(notificationObj.payload.title).to.be.a('string');
expect(notificationObj.payload.link).to.be.a('string');
expect(notificationObj.payload.address).to.be.a('string');
notificationObj.payload.forEach((notify, idx) => {
/** check the values if possible **/
expect(notificationObj.payload.id).to.equal(
kleinanzeigenDbContent.kleinanzeigen[kleinanzeigenDbContent.kleinanzeigen.length - 1]
);
expect(notificationObj.payload.price).that.does.include('€');
expect(notificationObj.payload.size).that.does.include('m²');
expect(notificationObj.payload.title).to.be.not.empty;
expect(notificationObj.payload.link).that.does.include('https://www.ebay-kleinanzeigen.de');
expect(notificationObj.payload.address).to.be.not.empty;
/** check the actual structure **/
expect(notify.id).to.be.a('number');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(
kleinanzeigenDbContent.kleinanzeigen[idx]
);
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});

1
test/mocks/mockNotification.js
View File

@@ -3,6 +3,7 @@ module.exports = {
send: (serviceName, payload) => {
this._tmpStore = { serviceName, payload };
return [Promise.resolve()];
},
get: () => {

50
test/neubauKompass.test.js Normal file
View File

@@ -0,0 +1,50 @@
const mockNotification = require('./mocks/mockNotification');
const mockConfig = require('../conf/config.test');
const mockStore = require('./mocks/mockStore');
const mockStats = require('./mocks/mockStats');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
describe('#neubauKompass testsuite()', () => {
const neubauKompass = proxyquire('../lib/provider/neubauKompass', {
'../../conf/config.json': mockConfig,
'../lib/fredy': proxyquire('../lib/fredy', {
'./services/store': mockStore,
'./notification/notify': mockNotification
})
});
it('should test neubauKompass provider', async () => {
return await new Promise(resolve => {
neubauKompass.run(mockStats).then(() => {
const neubauKompassDbContent = neubauKompass._getStore()._db;
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
const notificationObj = mockNotification.get();
expect(notificationObj.serviceName).to.equal('neubauKompass');
notificationObj.payload.forEach((notify, idx) => {
expect(notify).to.be.a('object');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
expect(notify.price).to.be.not.empty;
expect(notify.size).to.be.not.empty;
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.neubaukompass.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});
});
});