External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

upgrading dependencies, fixing immowelt, using hash of price and id as unique identifier for listings

Browse Source
This commit is contained in:
Christian Kellner
2024-09-05 13:34:14 +02:00
parent 1bf012f13e
commit 35feb772d7
15 changed files with 680 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
lib/provider/neubauKompass.js
View File

@@ -1,38 +1,44 @@
import utils from '../utils.js';
import utils, {buildHash} from '../utils.js';
let appliedBlackList = [];
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function normalize(o) {
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
return {...o, link};
const id = buildHash(o.id, o.price);
return Object.assign(o, {id, link});
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
return !utils.isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
crawlContainer: '.nbk-container >div article',
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
crawlFields: {
id: '@id',
title: 'a.nbk-truncate@title | removeNewline | trim',
link: 'a.nbk-truncate@href',
address: 'p.nbk-truncate | removeNewline | trim',
price: 'p.nbk-mb-0 | removeNewline | trim',
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,
filter: applyBlacklist,
url: null,
crawlContainer: '.nbk-container >div article',
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
crawlFields: {
id: '@id',
title: 'a.nbk-truncate@title | removeNewline | trim',
link: 'a.nbk-truncate@href',
address: 'p.nbk-truncate | removeNewline | trim',
price: 'p.nbk-mb-0 | removeNewline | trim',
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,
filter: applyBlacklist,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
export const metaInformation = {
name: 'Neubau Kompass',
baseUrl: 'https://www.neubaukompass.de/',
id: 'neubauKompass',
name: 'Neubau Kompass',
baseUrl: 'https://www.neubaukompass.de/',
id: 'neubauKompass',
};
export { config };
export {config};