mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
committed by
GitHub
parent
7d0ec72a0c
commit
2c5eceb0c1
@@ -1,15 +1,11 @@
|
||||
const utils = require('../utils');
|
||||
|
||||
import utils from '../utils.js';
|
||||
let appliedBlackList = [];
|
||||
|
||||
function shortenLink(link) {
|
||||
return link.substring(0, link.indexOf('?'));
|
||||
}
|
||||
|
||||
function parseId(shortenedLink) {
|
||||
return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
|
||||
}
|
||||
|
||||
function normalize(o) {
|
||||
const id = parseId(shortenLink(o.link));
|
||||
const size = o.size || 'N/A m²';
|
||||
@@ -19,14 +15,11 @@ function normalize(o) {
|
||||
const link = shortenLink(o.link);
|
||||
return Object.assign(o, { id, price, size, title, address, link });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '.estates_list .list_immo a._ref',
|
||||
@@ -43,17 +36,14 @@ const config = {
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
exports.metaInformation = {
|
||||
export const metaInformation = {
|
||||
name: 'Immobilien.de',
|
||||
baseUrl: 'https://www.immobilien.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
id: 'immobilienDe',
|
||||
};
|
||||
|
||||
exports.config = config;
|
||||
export { config };
|
||||
|
||||
Reference in New Issue
Block a user