External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Puppeteer rewrite (#119)

Browse Source 
* Moving to puppeteer | removing scrapingAnt
This commit is contained in:
Christian Kellner
2024-12-17 12:38:28 +01:00
committed by GitHub
parent 58965a6f1b
commit 214e714c03
30 changed files with 1367 additions and 1190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
lib/provider/neubauKompass.js
View File

@@ -8,7 +8,7 @@ function nullOrEmpty(val) {
function normalize(o) {
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
const id = buildHash(o.id, o.price);
const id = buildHash(o.link, o.price);
return Object.assign(o, {id, link});
}
@@ -18,16 +18,16 @@ function applyBlacklist(o) {
const config = {
url: null,
crawlContainer: '.nbk-container >div article',
crawlContainer: '.col-12.mb-4',
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
waitForSelector: '.nbk-section',
crawlFields: {
id: '@id',
title: 'a.nbk-truncate@title | removeNewline | trim',
link: 'a.nbk-truncate@href',
address: 'p.nbk-truncate | removeNewline | trim',
price: 'p.nbk-mb-0 | removeNewline | trim',
id: 'a@href',
title: 'a@title | removeNewline | trim',
link: 'a@href',
address: '.nbk-project-card__description | removeNewline | trim',
price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,
filter: applyBlacklist,
};