External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/spec filter (#276)

Browse Source 
* feat(): create map component, add area filtering to the job config

* feat(): filter listings by area filter

* chore(): cleanup

* feat(): solve feedback

* feat(): solve most providers

* feat(): solve maybe other providers

* feat(): add specFilter config, also add rooms to listing

* feat(): change tests

* feat(): fix kleinanzeigen parser

* feat(): add spec filter switch for listing overviiews

* feat(): add rooms and size to the overview and detail of a listing

* feat(): rem label

* feat(): add types, update providers, they now return specs as numbers

* feat(): add jsonconfig to enable type checks

* feat: add type for prividerConfig, add fieldNames per provider

* feat: fix tests, provider, add formatListing

* chore: remov duplicates

* feat(): fix tests

* feat: fix immoscout

* chore: geojson typing

* feat: solve requested changes
This commit is contained in:
Stephan
2026-04-12 09:17:23 +02:00
committed by GitHub
parent 05f74f99ef
commit 10c94eea0a
49 changed files with 1004 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
lib/provider/immobilienDe.js
View File

@@ -5,9 +5,12 @@
import { buildHash, isOneOf } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
import { extractNumber } from '../utils/extract-number.js';
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
import * as cheerio from 'cheerio';
import logger from '../services/logger.js';
/** @import { ParsedListing } from '../types/listing.js' */
/** @import { ProviderConfig } from '../types/providerConfig.js' */
let appliedBlackList = [];
@@ -65,27 +68,44 @@ async function fetchDetails(listing, browser) {
return listing;
}
}
/**
* @param {any} o
* @returns {ParsedListing}
*/
function normalize(o) {
const baseUrl = 'https://www.immobilien.de';
const size = o.size || null;
const price = o.price || null;
const title = o.title || 'No title available';
const title = o.title || '';
const address = o.address || null;
const shortLink = shortenLink(o.link);
const link = shortLink ? (shortLink.startsWith('http') ? shortLink : baseUrl + shortLink) : baseUrl;
const image = o.image ? (o.image.startsWith('http') ? o.image : baseUrl + o.image) : null;
const id = buildHash(parseId(shortLink), o.price);
return Object.assign(o, { id, price, size, title, address, link, image });
return {
id,
link,
title,
price: extractNumber(o.price),
size: extractNumber(o.size),
rooms: extractNumber(o.rooms),
address,
image,
description: o.description,
};
}
/**
* @param {ParsedListing} o
* @returns {boolean}
*/
function applyBlacklist(o) {
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
/** @type {ProviderConfig} */
const config = {
fieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
url: null,
crawlContainer: 'a.lr-card',
sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
@@ -94,6 +114,7 @@ const config = {
id: '@href', //will be transformed later
price: '.lr-card__price-amount | trim',
size: '.lr-card__fact:has(.lr-card__fact-label:contains("Fläche")) .lr-card__fact-value | trim',
rooms: '.zimmer .label_info',
title: '.lr-card__title | trim',
description: '.description | trim',
link: '@href',