Feature/spec filter (#276)

* feat(): create map component, add area filtering to the job config

* feat(): filter listings by area filter

* chore(): cleanup

* feat(): solve feedback

* feat(): solve most providers

* feat(): solve maybe other providers

* feat(): add specFilter config, also add rooms to listing

* feat(): change tests

* feat(): fix kleinanzeigen parser

* feat(): add spec filter switch for listing overviiews

* feat(): add rooms and size to the overview and detail of a listing

* feat(): rem label

* feat(): add types, update providers, they now return specs as numbers

* feat(): add jsonconfig to enable type checks

* feat: add type for prividerConfig, add fieldNames per provider

* feat: fix tests, provider, add formatListing

* chore: remov duplicates

* feat(): fix tests

* feat: fix immoscout

* chore: geojson typing

* feat: solve requested changes
This commit is contained in:
Stephan
2026-04-12 09:17:23 +02:00
committed by GitHub
parent 05f74f99ef
commit 10c94eea0a
49 changed files with 1004 additions and 250 deletions

View File

@@ -8,6 +8,9 @@ import checkIfListingIsActive from '../services/listings/listingActiveTester.js'
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
import * as cheerio from 'cheerio';
import logger from '../services/logger.js';
import { extractNumber } from '../utils/extract-number.js';
/** @import { ParsedListing } from '../types/listing.js' */
/** @import { ProviderConfig } from '../types/providerConfig.js' */
let appliedBlackList = [];
@@ -55,20 +58,39 @@ async function fetchDetails(listing, browser) {
}
}
/**
* @param {any} o
* @returns {ParsedListing}
*/
function normalize(o) {
const originalId = o.id.split('/').pop().replace('.html', '');
const id = buildHash(originalId, o.price);
const size = o.size?.replace(' Wohnfläche', '').replace(' m²', 'm²') ?? null;
const title = o.title || 'No title available';
const link = o.link != null ? `https://immobilien.sparkasse.de${o.link}` : config.url;
return Object.assign(o, { id, size, title, link });
return {
id,
link,
title: o.title || '',
price: extractNumber(o.price),
size: extractNumber(o.size),
rooms: extractNumber(o.rooms),
address: o.address,
image: o.image,
description: o.description,
};
}
/**
* @param {ParsedListing} o
* @returns {boolean}
*/
function applyBlacklist(o) {
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
/** @type {ProviderConfig} */
const config = {
fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
url: null,
crawlContainer: 'div[data-testid="estate-link"]',
sortByDateParam: 'sortBy=date_desc',
@@ -77,7 +99,8 @@ const config = {
id: 'a@href',
title: 'h3 | trim',
price: '.estate-list-price | trim',
size: '.estate-mainfact span | trim',
size: '.estate-mainfact:nth-child(1) span | trim',
rooms: '.estate-mainfact:nth-child(2) span | trim',
address: 'h6 | trim',
image: 'img@src',
link: 'a@href',