mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Feature/kleinanzeigen new (#292)
* Feature/Kleinanzeigen addresses (#289) * upgrade dependencies * immoscout_details -> provider_details * fetching details more generic * removing claude action * fixing sparkassen selector * improvements * fixing immobilienDE test * upgrading dependencies * settings for many provider --------- Co-authored-by: Adrian Bach <65734063+realDayaa@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
7888c5b340
commit
cdc0cbda2f
@@ -5,9 +5,49 @@
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import logger from '../services/logger.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
async function fetchDetails(listing, browser) {
|
||||
try {
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser });
|
||||
if (!html) return listing;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
const nextDataRaw = $('#__NEXT_DATA__').text();
|
||||
if (!nextDataRaw) return listing;
|
||||
|
||||
const classified = JSON.parse(nextDataRaw)?.props?.pageProps?.classified;
|
||||
if (!classified) return listing;
|
||||
|
||||
const description = (classified.Texts || [])
|
||||
.map((t) => [t.Title, t.Content].filter(Boolean).join('\n'))
|
||||
.filter(Boolean)
|
||||
.join('\n\n');
|
||||
|
||||
const addr = classified.EstateAddress;
|
||||
let address = listing.address;
|
||||
if (addr) {
|
||||
const street = [addr.Street, addr.HouseNumber].filter(Boolean).join(' ');
|
||||
const cityLine = [addr.ZipCode, addr.District || addr.City].filter(Boolean).join(' ');
|
||||
const full = [street, cityLine].filter(Boolean).join(', ');
|
||||
if (full) address = full;
|
||||
}
|
||||
|
||||
return {
|
||||
...listing,
|
||||
address,
|
||||
description: description || listing.description,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.warn(`Could not fetch immowelt detail page for listing '${listing.id}'.`, error?.message || error);
|
||||
return listing;
|
||||
}
|
||||
}
|
||||
|
||||
function normalize(o) {
|
||||
const id = buildHash(o.id, o.price);
|
||||
return Object.assign(o, { id });
|
||||
@@ -37,6 +77,7 @@ const config = {
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
fetchDetails: fetchDetails,
|
||||
activeTester: checkIfListingIsActive,
|
||||
};
|
||||
export const init = (sourceConfig, blacklist) => {
|
||||
|
||||
Reference in New Issue
Block a user