External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixing ebay kleinanzeigen || fiing neubauKompass

Browse Source
This commit is contained in:
Christian Kellner
2018-06-10 11:17:01 +02:00
parent acdbfd54cf
commit 5ac33aa009
4 changed files with 29 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
lib/provider/kleinanzeigen.js
View File

@@ -3,37 +3,37 @@ const config = require('../../conf/config.json');
const utils = require('../utils');
function normalize(o) {
const address = o.address.split('\n')[4].trim();
const size = o.size || '--- m²';
return Object.assign(o, { address });
return Object.assign(o, {size});
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const isBlacklistedDistrict =
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const isBlacklistedDistrict =
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
const kleinanzeigen = {
name: 'kleinanzeigen',
enabled: config.sources.kleinanzeigen.enabled,
url: config.sources.kleinanzeigen.url,
crawlContainer: '#srchrslt-adtable .ad-listitem',
crawlFields: {
id: '.aditem@data-adid | int',
price: '.aditem-details strong | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
address: '.aditem-details'
},
paginate: '#srchrslt-pagination .pagination-next@href',
normalize: normalize,
filter: applyBlacklist
name: 'kleinanzeigen',
enabled: config.sources.kleinanzeigen.enabled,
url: config.sources.kleinanzeigen.url,
crawlContainer: '#srchrslt-adtable .ad-listitem',
crawlFields: {
id: '.aditem@data-adid | int',
price: '.aditem-details strong | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
address: '.aditem-details | trim | removeNewline'
},
paginate: '#srchrslt-pagination .pagination-next@href',
normalize: normalize,
filter: applyBlacklist
};
module.exports = new Fredy(kleinanzeigen);

16
lib/provider/neubauKompass.js
View File

@@ -4,9 +4,7 @@ const utils = require('../utils');
function normalize(o) {
const title = o.title + '| '+o.subTitle;
//this is a bit nasty, but we do not have a size, therefor take the availability and set it as size to not modify notifications any furter
const size = o.available;
return Object.assign(o, { title, size });
return Object.assign(o, { title });
}
function applyBlacklist(o) {
@@ -17,15 +15,13 @@ const neubauKompass = {
name: 'neubauKompass',
enabled: config.sources.neubauKompass.enabled,
url: config.sources.neubauKompass.url,
crawlContainer: '.md__property-list .post-list__item',
crawlContainer: '.row article',
crawlFields: {
id: '@id',
price: '.entry__main .entry__data li:nth-child(1) span:nth-child(2) | removeNewline | trim',
available: '.entry__main .entry__data li:nth-child(3) span:nth-child(2) | removeNewline | trim',
title: '.entry__main .entry__title | removeNewline | trim',
link: '.entry__main .entry__title a@href',
subTitle: '.entry__main .entry__subtitle | removeNewline | trim',
address: '.entry__main .entry__info | removeNewline | trim'
title: 'a@title | removeNewline | trim',
link: 'a@href',
subTitle: '.p-3 .mb-2 | removeNewline | trim',
address: 'div.p-3 > p:nth-child(3) | removeNewline | trim'
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,