External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixing crawling of immonet

Browse Source
This commit is contained in:
Christian Kellner
2018-06-10 10:36:29 +02:00
parent 8a1d86c90e
commit bb08d036af
1 changed files with 23 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
lib/provider/immonet.js
View File

@@ -3,36 +3,37 @@ const Fredy = require('../fredy');
const utils = require('../utils'); const utils = require('../utils');
function normalize(o) { function normalize(o) {
const id = parseInt(o.id.split('_')[1], 10); const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
const title = o.title.replace('NEU ', ''); const size = o.size.replace('Wohnfläche ', '');
const address = o.address.split(' - ')[1]; const price = o.price.replace('Kaufpreis ', '');
const address = o.address.split(' • ')[1];
return Object.assign(o, { id, title, address }); return Object.assign(o, {id, address, price, size});
} }
function applyBlacklist(o) { function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist); const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist); const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
return titleNotBlacklisted && descNotBlacklisted; return titleNotBlacklisted && descNotBlacklisted;
} }
const immonet = { const immonet = {
name: 'immonet', name: 'immonet',
enabled: config.sources.immonet.enabled, enabled: config.sources.immonet.enabled,
url: config.sources.immonet.url, url: config.sources.immonet.url,
crawlContainer: '#idResultList .search-object', crawlContainer: '#result-list-stage .item',
crawlFields: { crawlFields: {
id: '.search-info a@id', id: '@id',
price: '#keyfacts-bar div:first-child span', price: 'div[id*="selPrice_"] | trim',
size: '#keyfacts-bar div:nth-child(2) .text-primary-highlight', size: 'div[id*="selArea_"] | trim',
title: '.search-info a | removeNewline | trim', title: '.item a img@title',
link: '.search-info a@href', link: '.item a@href',
address: '.search-info p | removeNewline | trim' address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim'
}, },
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href', paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist filter: applyBlacklist
}; };
module.exports = new Fredy(immonet); module.exports = new Fredy(immonet);