mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
fixing immowelt provider
This commit is contained in:
@@ -9,7 +9,9 @@ import checkIfListingIsActive from '../services/listings/listingActiveTester.js'
|
||||
let appliedBlackList = [];
|
||||
|
||||
function shortenLink(link) {
|
||||
return link.substring(0, link.indexOf('?'));
|
||||
if (!link) return '';
|
||||
const index = link.indexOf('?');
|
||||
return index === -1 ? link : link.substring(0, index);
|
||||
}
|
||||
|
||||
function parseId(shortenedLink) {
|
||||
@@ -23,7 +25,7 @@ function normalize(o) {
|
||||
const title = o.title || 'No title available';
|
||||
const address = o.address || null;
|
||||
const shortLink = shortenLink(o.link);
|
||||
const link = `${baseUrl}/${shortLink}`;
|
||||
const link = baseUrl + shortLink;
|
||||
const image = baseUrl + o.image;
|
||||
const id = buildHash(parseId(shortLink), o.price);
|
||||
return Object.assign(o, { id, price, size, title, address, link, image });
|
||||
@@ -37,18 +39,18 @@ function applyBlacklist(o) {
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '._ref',
|
||||
crawlContainer: 'a:has(div.list_entry)',
|
||||
sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
|
||||
waitForSelector: 'body',
|
||||
crawlFields: {
|
||||
id: '@href', //will be transformed later
|
||||
price: '.list_entry .immo_preis .label_info',
|
||||
size: '.list_entry .flaeche .label_info | removeNewline | trim',
|
||||
title: '.list_entry .part_text h3 span',
|
||||
description: '.list_entry .description | trim',
|
||||
price: '.immo_preis .label_info',
|
||||
size: '.flaeche .label_info | removeNewline | trim',
|
||||
title: 'h3 span',
|
||||
description: '.description | trim',
|
||||
link: '@href',
|
||||
address: '.list_entry .place',
|
||||
image: '.list_entry img@src',
|
||||
address: '.place',
|
||||
image: 'img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
|
||||
Reference in New Issue
Block a user