External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixing ebay kleinanzeigen || fiing neubauKompass

Browse Source
This commit is contained in:
Christian Kellner
2018-06-10 11:17:01 +02:00
parent acdbfd54cf
commit 5ac33aa009
4 changed files with 29 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
lib/provider/kleinanzeigen.js
View File

@@ -3,37 +3,37 @@ const config = require('../../conf/config.json');
const utils = require('../utils'); const utils = require('../utils');
function normalize(o) { function normalize(o) {
const address = o.address.split('\n')[4].trim(); const size = o.size || '--- m²';
return Object.assign(o, { address }); return Object.assign(o, {size});
} }
function applyBlacklist(o) { function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist); const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist); const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const isBlacklistedDistrict = const isBlacklistedDistrict =
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics); config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted; return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
} }
const kleinanzeigen = { const kleinanzeigen = {
name: 'kleinanzeigen', name: 'kleinanzeigen',
enabled: config.sources.kleinanzeigen.enabled, enabled: config.sources.kleinanzeigen.enabled,
url: config.sources.kleinanzeigen.url, url: config.sources.kleinanzeigen.url,
crawlContainer: '#srchrslt-adtable .ad-listitem', crawlContainer: '#srchrslt-adtable .ad-listitem',
crawlFields: { crawlFields: {
id: '.aditem@data-adid | int', id: '.aditem@data-adid | int',
price: '.aditem-details strong | removeNewline | trim', price: '.aditem-details strong | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim', size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim', title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim', link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim', description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
address: '.aditem-details' address: '.aditem-details | trim | removeNewline'
}, },
paginate: '#srchrslt-pagination .pagination-next@href', paginate: '#srchrslt-pagination .pagination-next@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist filter: applyBlacklist
}; };
module.exports = new Fredy(kleinanzeigen); module.exports = new Fredy(kleinanzeigen);

16
lib/provider/neubauKompass.js
View File

@@ -4,9 +4,7 @@ const utils = require('../utils');
function normalize(o) { function normalize(o) {
const title = o.title + '| '+o.subTitle; const title = o.title + '| '+o.subTitle;
//this is a bit nasty, but we do not have a size, therefor take the availability and set it as size to not modify notifications any furter return Object.assign(o, { title });
const size = o.available;
return Object.assign(o, { title, size });
} }
function applyBlacklist(o) { function applyBlacklist(o) {
@@ -17,15 +15,13 @@ const neubauKompass = {
name: 'neubauKompass', name: 'neubauKompass',
enabled: config.sources.neubauKompass.enabled, enabled: config.sources.neubauKompass.enabled,
url: config.sources.neubauKompass.url, url: config.sources.neubauKompass.url,
crawlContainer: '.md__property-list .post-list__item', crawlContainer: '.row article',
crawlFields: { crawlFields: {
id: '@id', id: '@id',
price: '.entry__main .entry__data li:nth-child(1) span:nth-child(2) | removeNewline | trim', title: 'a@title | removeNewline | trim',
available: '.entry__main .entry__data li:nth-child(3) span:nth-child(2) | removeNewline | trim', link: 'a@href',
title: '.entry__main .entry__title | removeNewline | trim', subTitle: '.p-3 .mb-2 | removeNewline | trim',
link: '.entry__main .entry__title a@href', address: 'div.p-3 > p:nth-child(3) | removeNewline | trim'
subTitle: '.entry__main .entry__subtitle | removeNewline | trim',
address: '.entry__main .entry__info | removeNewline | trim'
}, },
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href', paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize, normalize: normalize,

1
test/kleinanzeigen.test.js
View File

@@ -40,7 +40,6 @@ describe('#kleinanzeigen testsuite()', () => {
expect(notify.id).to.equal( expect(notify.id).to.equal(
kleinanzeigenDbContent.kleinanzeigen[idx] kleinanzeigenDbContent.kleinanzeigen[idx]
); );
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²'); expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty; expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de'); expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');

4
test/neubauKompass.test.js
View File

@@ -29,16 +29,12 @@ describe('#neubauKompass testsuite()', () => {
/** check the actual structure **/ /** check the actual structure **/
expect(notify.id).to.be.a('string'); expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string'); expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string'); expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string'); expect(notify.address).to.be.a('string');
/** check the values if possible **/ /** check the values if possible **/
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]); expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
expect(notify.price).to.be.not.empty;
expect(notify.size).to.be.not.empty;
expect(notify.title).to.be.not.empty; expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.neubaukompass.de'); expect(notify.link).that.does.include('https://www.neubaukompass.de');
expect(notify.address).to.be.not.empty; expect(notify.address).to.be.not.empty;