Feature/kleinanzeigen new (#292)

* Feature/Kleinanzeigen addresses (#289)

* upgrade dependencies

* immoscout_details -> provider_details

* fetching details more generic

* removing claude action

* fixing sparkassen selector

* improvements

* fixing immobilienDE test

* upgrading dependencies

* settings for many provider

---------

Co-authored-by: Adrian Bach <65734063+realDayaa@users.noreply.github.com>
This commit is contained in:
Christian Kellner
2026-04-07 19:53:40 +02:00
committed by GitHub
parent 7888c5b340
commit cdc0cbda2f
35 changed files with 1098 additions and 501 deletions

View File

@@ -5,9 +5,49 @@
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
import * as cheerio from 'cheerio';
import logger from '../services/logger.js';
let appliedBlackList = [];
async function fetchDetails(listing, browser) {
try {
const html = await puppeteerExtractor(listing.link, null, { browser });
if (!html) return listing;
const $ = cheerio.load(html);
const nextDataRaw = $('#__NEXT_DATA__').text();
if (!nextDataRaw) return listing;
const classified = JSON.parse(nextDataRaw)?.props?.pageProps?.classified;
if (!classified) return listing;
const description = (classified.Texts || [])
.map((t) => [t.Title, t.Content].filter(Boolean).join('\n'))
.filter(Boolean)
.join('\n\n');
const addr = classified.EstateAddress;
let address = listing.address;
if (addr) {
const street = [addr.Street, addr.HouseNumber].filter(Boolean).join(' ');
const cityLine = [addr.ZipCode, addr.District || addr.City].filter(Boolean).join(' ');
const full = [street, cityLine].filter(Boolean).join(', ');
if (full) address = full;
}
return {
...listing,
address,
description: description || listing.description,
};
} catch (error) {
logger.warn(`Could not fetch immowelt detail page for listing '${listing.id}'.`, error?.message || error);
return listing;
}
}
function normalize(o) {
const id = buildHash(o.id, o.price);
return Object.assign(o, { id });
@@ -37,6 +77,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
fetchDetails: fetchDetails,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {