mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Adding images to scraping data (#157)
* Fredy now supports pulling the main Image from the listing and send it together with the usual information
This commit is contained in:
committed by
GitHub
parent
da743c8279
commit
f0b146fd7f
@@ -1,26 +1,34 @@
|
||||
import utils, { buildHash } from '../utils.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function shortenLink(link) {
|
||||
return link.substring(0, link.indexOf('?'));
|
||||
}
|
||||
|
||||
function parseId(shortenedLink) {
|
||||
return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
|
||||
}
|
||||
|
||||
function normalize(o) {
|
||||
const baseUrl = 'https://www.immobilien.de';
|
||||
const size = o.size || 'N/A m²';
|
||||
const price = o.price || 'N/A €';
|
||||
const title = o.title || 'No title available';
|
||||
const address = o.address || 'No address available';
|
||||
const shortLink = shortenLink(o.link);
|
||||
const link = `https://www.immobilien.de/${shortLink}`;
|
||||
const link = `${baseUrl}/${shortLink}`;
|
||||
const image = baseUrl + o.image;
|
||||
const id = buildHash(parseId(shortLink), o.price);
|
||||
return Object.assign(o, { id, price, size, title, address, link });
|
||||
return Object.assign(o, { id, price, size, title, address, link, image });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '._ref',
|
||||
@@ -34,6 +42,7 @@ const config = {
|
||||
description: '.list_entry .description | trim',
|
||||
link: '@href',
|
||||
address: '.list_entry .place',
|
||||
image: '.list_entry img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
|
||||
Reference in New Issue
Block a user