External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature/kleinanzeigen new (#292)

Browse Source 
* Feature/Kleinanzeigen addresses (#289)

* upgrade dependencies

* immoscout_details -> provider_details

* fetching details more generic

* removing claude action

* fixing sparkassen selector

* improvements

* fixing immobilienDE test

* upgrading dependencies

* settings for many provider

---------

Co-authored-by: Adrian Bach <65734063+realDayaa@users.noreply.github.com>
This commit is contained in:
Christian Kellner
2026-04-07 19:53:40 +02:00
committed by GitHub
parent 7888c5b340
commit cdc0cbda2f
35 changed files with 1098 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
lib/FredyPipelineExecutioner.js
View File

@@ -63,6 +63,7 @@ class FredyPipelineExecutioner {
* @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape.
* @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings.
* @param {(url:string, waitForSelector?:string)=>Promise<void>|Promise<Listing[]>} [providerConfig.getListings] Optional override to fetch listings.
* @param {(listing:Listing, browser:any)=>Promise<Listing>} [providerConfig.fetchDetails] Optional per-listing detail enrichment. Called in parallel for each new listing after deduplication. Receives the shared browser instance. Must always resolve (never reject).
* @param {Object} notificationConfig Notification configuration passed to notification adapters.
* @param {Object} spatialFilter Optional spatial filter configuration.
* @param {string} providerId The ID of the provider currently in use.
@@ -92,6 +93,7 @@ class FredyPipelineExecutioner {
.then(this._normalize.bind(this))
.then(this._filter.bind(this))
.then(this._findNew.bind(this))
.then(this._fetchDetails.bind(this))
.then(this._geocode.bind(this))
.then(this._save.bind(this))
.then(this._calculateDistance.bind(this))
@@ -101,6 +103,32 @@ class FredyPipelineExecutioner {
.catch(this._handleError.bind(this));
}
/**
* Optionally enrich new listings with data from their detail pages.
* Only called when the provider config defines a `fetchDetails` function.
* Runs all fetches in parallel. Each individual fetch must handle its own errors
* and always resolve (never reject) to avoid aborting other listings.
*
* @param {Listing[]} newListings New listings to enrich.
* @returns {Promise<Listing[]>} Resolves with enriched listings.
*/
async _fetchDetails(newListings) {
if (typeof this._providerConfig.fetchDetails !== 'function') {
return newListings;
}
const userId = getJob(this._jobKey)?.userId;
const enabledProviders = getUserSettings(userId)?.provider_details ?? [];
if (!userId || !Array.isArray(enabledProviders) || !enabledProviders.includes(this._providerId)) {
return newListings;
}
const listingsToEnrich = process.env.NODE_ENV === 'test' ? newListings.slice(0, 1) : newListings;
const enriched = [];
for (const listing of listingsToEnrich) {
enriched.push(await this._providerConfig.fetchDetails(listing, this._browser));
}
return enriched;
}
/**
* Geocode new listings.
*