External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Puppeteer rewrite (#119)

Browse Source 
* Moving to puppeteer | removing scrapingAnt
This commit is contained in:
Christian Kellner
2024-12-17 12:38:28 +01:00
committed by GitHub
parent 58965a6f1b
commit 214e714c03
30 changed files with 1367 additions and 1190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
lib/provider/einsAImmobilien.js
View File

@@ -2,14 +2,10 @@ import utils, { buildHash } from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
if (o.rooms != null) {
size += ` / / ${o.rooms.trim()}`;
}
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
const price = normalizePrice(o.price);
const id = buildHash(o.id, price);
return Object.assign(o, { id, price, size, link });
return Object.assign(o, { id, price, link });
}
/**
@@ -39,12 +35,12 @@ const config = {
url: null,
crawlContainer: '.tabelle',
sortByDateParam: 'sort_type=newest',
waitForSelector: 'body',
crawlFields: {
id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
price: '.tabelle .inner_object_data .single_data_price | removeNewline | trim',
size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
price: '.inner_object_data .single_data_price | removeNewline | trim',
size: '.tabelle .tabelle_inhalt_infos .single_data_box | removeNewline | trim',
title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
},
normalize: normalize,
filter: applyBlacklist,

10
lib/provider/immobilienDe.js
View File

@@ -11,8 +11,9 @@ function normalize(o) {
const price = o.price || 'N/A €';
const title = o.title || 'No title available';
const address = o.address || 'No address available';
const link = shortenLink(o.link);
const id = buildHash(parseId(shortenLink(o.link)), o.price);
const shortLink = shortenLink(o.link);
const link = `https://www.immobilien.de/${shortLink}`;
const id = buildHash(parseId(shortLink), o.price);
return Object.assign(o, { id, price, size, title, address, link });
}
function applyBlacklist(o) {
@@ -22,9 +23,11 @@ function applyBlacklist(o) {
}
const config = {
url: null,
crawlContainer: '.estates_list .list_immo a._ref',
crawlContainer: '._ref',
sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
waitForSelector: 'body',
crawlFields: {
id: '@href', //will be transformed later
price: '.list_entry .immo_preis .label_info',
size: '.list_entry .flaeche .label_info | removeNewline | trim',
title: '.list_entry .part_text h3 span',
@@ -32,7 +35,6 @@ const config = {
link: '@href',
address: '.list_entry .place',
},
paginate: '.list_immo .blocknav .blocknav_list li.next a@href',
normalize: normalize,
filter: applyBlacklist,
};

26
lib/provider/immonet.js
View File

@@ -1,12 +1,20 @@
import utils, {buildHash} from '../utils.js';
let appliedBlackList = [];
/**
* Note, Immonet is rly a piece of sh*t. It is using a weird combination of React and some buttons (instead of links),
* so that if somebody clicks the listing, a new page will open with the actual link to the listing. Of course, a scraper
* cannot do this (which is why I always just return the link to the whole list of listings).
* This is not only bad for us, but also bad for ppl with disabilities...
*/
function normalize(o) {
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
const price = o.price.replace('Kaufpreis ', '');
const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
const title = o.title || 'No title available';
const link = o.id;
const id = buildHash(o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length), price);
const link = config.url;
const id = buildHash(title, price);
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
@@ -16,16 +24,16 @@ function applyBlacklist(o) {
}
const config = {
url: null,
crawlContainer: '.content-wrapper-tiles .ng-star-inserted',
crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
sortByDateParam: 'sortby=19',
waitForSelector: 'div[data-testid="serp-core-classified-card-testid"]',
crawlFields: {
id: '.card a@href',
title: '.card h3 |trim',
price: '.card .has-font-300 .is-bold | trim',
size: '.card .has-font-300 .ml-100 | trim',
address: '.card span:nth-child(2) | trim',
id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
title: 'button@title |trim',
price: 'div[data-testid="cardmfe-price-testid"] | trim',
size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
address: 'div[data-testid="cardmfe-description-box-address"] | trim',
},
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
normalize: normalize,
filter: applyBlacklist,
};

2
lib/provider/immoscout.js
View File

@@ -17,6 +17,7 @@ const config = {
url: null,
crawlContainer: '#resultListItems li.result-list__listing',
sortByDateParam: 'sorting=2',
waitForSelector: 'body',
crawlFields: {
id: '.result-list-entry@data-obid | int',
price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
@@ -25,7 +26,6 @@ const config = {
link: '.result-list-entry .result-list-entry__brand-title-container@href',
address: '.result-list-entry .result-list-entry__map-link',
},
paginate: '#pager .align-right a@href',
normalize: normalize,
filter: applyBlacklist,
};

2
lib/provider/immoswp.js
View File

@@ -23,6 +23,7 @@ const config = {
url: null,
crawlContainer: '.js-serp-item',
sortByDateParam: 's=most_recently_updated_first',
waitForSelector: 'body',
crawlFields: {
id: '.js-bookmark-btn@data-id',
price: 'div.align-items-start div:first-child | trim',
@@ -31,7 +32,6 @@ const config = {
link: '.ci-search-result__link@href',
description: '.js-show-more-item-sm | removeNewline | trim',
},
paginate: 'li.page-item.pagination__item a.page-link@href',
normalize: normalize,
filter: applyBlacklist,
};

6
lib/provider/immowelt.js
View File

@@ -16,17 +16,17 @@ function applyBlacklist(o) {
const config = {
url: null,
crawlContainer:
'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"])',
'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
sortByDateParam: 'order=DateDesc',
waitForSelector: 'div[data-testid="cardmfe-price-testid"]',
crawlFields: {
id: 'a@id',
id: 'a@href',
price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
title: '.css-1cbj9xw',
link: 'a@href',
address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
},
paginate: '#pnlPaging #nlbPlus@href',
normalize: normalize,
filter: applyBlacklist,
};

9
lib/provider/kleinanzeigen.js
View File

@@ -6,7 +6,8 @@ let appliedBlacklistedDistricts = [];
function normalize(o) {
const size = o.size || '--- m²';
const id = buildHash(o.id, o.price);
return Object.assign(o, {id, size});
const link = `https://www.kleinanzeigen.de${o.link}`;
return Object.assign(o, {id, size, link});
}
function applyBlacklist(o) {
@@ -22,16 +23,16 @@ const config = {
crawlContainer: '#srchrslt-adtable .ad-listitem ',
//sort by date is standard oO
sortByDateParam: null,
waitForSelector: 'body',
crawlFields: {
id: '.aditem@data-adid | int',
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
size: '.aditem-main .text-module-end | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
address: '.aditem-main--top--left | trim | removeNewline',
},
paginate: '#srchrslt-pagination .pagination-next@href',
normalize: normalize,
filter: applyBlacklist,
};

16
lib/provider/neubauKompass.js
View File

@@ -8,7 +8,7 @@ function nullOrEmpty(val) {
function normalize(o) {
const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
const id = buildHash(o.id, o.price);
const id = buildHash(o.link, o.price);
return Object.assign(o, {id, link});
}
@@ -18,16 +18,16 @@ function applyBlacklist(o) {
const config = {
url: null,
crawlContainer: '.nbk-container >div article',
crawlContainer: '.col-12.mb-4',
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
waitForSelector: '.nbk-section',
crawlFields: {
id: '@id',
title: 'a.nbk-truncate@title | removeNewline | trim',
link: 'a.nbk-truncate@href',
address: 'p.nbk-truncate | removeNewline | trim',
price: 'p.nbk-mb-0 | removeNewline | trim',
id: 'a@href',
title: 'a@title | removeNewline | trim',
link: 'a@href',
address: '.nbk-project-card__description | removeNewline | trim',
price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
},
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize,
filter: applyBlacklist,
};

1
lib/provider/wgGesucht.js
View File

@@ -17,6 +17,7 @@ const config = {
url: null,
crawlContainer: '#main_column .wgg_card',
sortByDateParam: 'sort_column=0&sort_order=0',
waitForSelector: 'body',
crawlFields: {
id: '@data-id',
details: '.row .noprint .col-xs-11 |removeNewline |trim',