External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Puppeteer rewrite (#119)

Browse Source 
* Moving to puppeteer | removing scrapingAnt
This commit is contained in:
Christian Kellner
2024-12-17 12:38:28 +01:00
committed by GitHub
parent 58965a6f1b
commit 214e714c03
30 changed files with 1367 additions and 1190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
lib/provider/immonet.js
View File

@@ -1,12 +1,20 @@
import utils, {buildHash} from '../utils.js';
let appliedBlackList = [];
/**
* Note, Immonet is rly a piece of sh*t. It is using a weird combination of React and some buttons (instead of links),
* so that if somebody clicks the listing, a new page will open with the actual link to the listing. Of course, a scraper
* cannot do this (which is why I always just return the link to the whole list of listings).
* This is not only bad for us, but also bad for ppl with disabilities...
*/
function normalize(o) {
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
const price = o.price.replace('Kaufpreis ', '');
const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
const title = o.title || 'No title available';
const link = o.id;
const id = buildHash(o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length), price);
const link = config.url;
const id = buildHash(title, price);
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
@@ -16,16 +24,16 @@ function applyBlacklist(o) {
}
const config = {
url: null,
crawlContainer: '.content-wrapper-tiles .ng-star-inserted',
crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
sortByDateParam: 'sortby=19',
waitForSelector: 'div[data-testid="serp-core-classified-card-testid"]',
crawlFields: {
id: '.card a@href',
title: '.card h3 |trim',
price: '.card .has-font-300 .is-bold | trim',
size: '.card .has-font-300 .ml-100 | trim',
address: '.card span:nth-child(2) | trim',
id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
title: 'button@title |trim',
price: 'div[data-testid="cardmfe-price-testid"] | trim',
size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
address: 'div[data-testid="cardmfe-description-box-address"] | trim',
},
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
normalize: normalize,
filter: applyBlacklist,
};