External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Puppeteer rewrite (#119)

Browse Source 
* Moving to puppeteer | removing scrapingAnt
This commit is contained in:
Christian Kellner
2024-12-17 12:38:28 +01:00
committed by GitHub
parent 58965a6f1b
commit 214e714c03
30 changed files with 1367 additions and 1190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lib/provider/kleinanzeigen.js
View File

@@ -6,7 +6,8 @@ let appliedBlacklistedDistricts = [];
function normalize(o) {
const size = o.size || '--- m²';
const id = buildHash(o.id, o.price);
return Object.assign(o, {id, size});
const link = `https://www.kleinanzeigen.de${o.link}`;
return Object.assign(o, {id, size, link});
}
function applyBlacklist(o) {
@@ -22,16 +23,16 @@ const config = {
crawlContainer: '#srchrslt-adtable .ad-listitem ',
//sort by date is standard oO
sortByDateParam: null,
waitForSelector: 'body',
crawlFields: {
id: '.aditem@data-adid | int',
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
size: '.aditem-main .text-module-end | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
address: '.aditem-main--top--left | trim | removeNewline',
},
paginate: '#srchrslt-pagination .pagination-next@href',
normalize: normalize,
filter: applyBlacklist,
};