mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Feature/spec filter (#276)
* feat(): create map component, add area filtering to the job config * feat(): filter listings by area filter * chore(): cleanup * feat(): solve feedback * feat(): solve most providers * feat(): solve maybe other providers * feat(): add specFilter config, also add rooms to listing * feat(): change tests * feat(): fix kleinanzeigen parser * feat(): add spec filter switch for listing overviiews * feat(): add rooms and size to the overview and detail of a listing * feat(): rem label * feat(): add types, update providers, they now return specs as numbers * feat(): add jsonconfig to enable type checks * feat: add type for prividerConfig, add fieldNames per provider * feat: fix tests, provider, add formatListing * chore: remov duplicates * feat(): fix tests * feat: fix immoscout * chore: geojson typing * feat: solve requested changes
This commit is contained in:
10
lib/types/browser.js
Normal file
10
lib/types/browser.js
Normal file
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import('puppeteer').Browser} Browser
|
||||
*/
|
||||
|
||||
export {};
|
||||
19
lib/types/filter.js
Normal file
19
lib/types/filter.js
Normal file
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SpecFilter
|
||||
* @property {number} [minRooms] Minimum number of rooms.
|
||||
* @property {number} [minSize] Minimum size in m².
|
||||
* @property {number} [maxPrice] Maximum price.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SpatialFilter GeoJSON FeatureCollection.
|
||||
* @property {Array<Object>} [features] GeoJSON features for spatial filtering (typically Polygons).
|
||||
* @property {string} [type] Type 'FeatureCollection'.
|
||||
*/
|
||||
|
||||
export {};
|
||||
23
lib/types/job.js
Normal file
23
lib/types/job.js
Normal file
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/** @import { SpecFilter, SpatialFilter } from './filter.js' */
|
||||
|
||||
/**
|
||||
* @typedef {Object} Job
|
||||
* @property {string} id Job ID.
|
||||
* @property {string} [userId] Owner user id.
|
||||
* @property {string} [name] Job display name.
|
||||
* @property {boolean} [enabled] Whether the job is enabled.
|
||||
* @property {Array<any>} [blacklist] Blacklist entries.
|
||||
* @property {Array<any>} [provider] Provider configuration list.
|
||||
* @property {Object} [notificationAdapter] Notification configuration.
|
||||
* @property {Array<string>} [shared_with_user] Users this job is shared with.
|
||||
* @property {SpatialFilter | null} [spatialFilter] Optional spatial filter configuration as GeoJSON FeatureCollection.
|
||||
* @property {SpecFilter | null} [specFilter] Optional listing specifications.
|
||||
* @property {number} [numberOfFoundListings] Count of active listings for this job.
|
||||
*/
|
||||
|
||||
export {};
|
||||
22
lib/types/listing.js
Normal file
22
lib/types/listing.js
Normal file
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} ParsedListing
|
||||
* @property {string} id Stable unique identifier (hash) of the listing.
|
||||
* @property {string} link Link to the listing detail page.
|
||||
* @property {string} image Link to the listing image.
|
||||
* @property {string} title Title or headline of the listing.
|
||||
* @property {string} [description] Description of the listing.
|
||||
* @property {string} [address] Optional address/location text.
|
||||
* @property {number} [price] Optional price of the listing.
|
||||
* @property {number} [size] Optional size of the listing.
|
||||
* @property {number} [rooms] Optional number of rooms.
|
||||
* @property {number} [latitude] Optional latitude.
|
||||
* @property {number} [longitude] Optional longitude.
|
||||
* @property {number} [distance_to_destination] Optional distance to destination.
|
||||
*/
|
||||
|
||||
export {};
|
||||
25
lib/types/providerConfig.js
Normal file
25
lib/types/providerConfig.js
Normal file
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/** @import { ParsedListing } from './listing.js' */
|
||||
|
||||
/**
|
||||
* @typedef {Object} ProviderConfig
|
||||
* @property {string} [url] Base URL to crawl.
|
||||
* @property {string} [sortByDateParam] Query parameter used to enforce sorting by date.
|
||||
* @property {string} [waitForSelector] CSS selector to wait for before parsing content.
|
||||
* @property {Object.<string, string>} crawlFields Mapping of field names to selectors/paths.
|
||||
* @property {string[]} fieldNames List of field names that this provider supports.
|
||||
* @property {string} [crawlContainer] CSS selector for the container holding listing items.
|
||||
* @property {(raw: any) => ParsedListing} normalize Function to convert raw scraped data into a ParsedListing shape.
|
||||
* @property {(listing: ParsedListing) => boolean} filter Function to filter out unwanted listings.
|
||||
* @property {(url: string, waitForSelector?: string) => Promise<any[]>} [getListings] Optional override to fetch listings.
|
||||
* @property {(listing:ParsedListing, browser:any)=>Promise<ParsedListing>} [providerConfig.fetchDetails] Optional per-listing detail enrichment. Called in parallel for each new listing after deduplication. Receives the shared browser instance. Must always resolve (never reject).
|
||||
* @property {Object} [puppeteerOptions] Puppeteer specific options.
|
||||
* @property {boolean} [enabled] Whether the provider is enabled.
|
||||
* @property {(url: string) => Promise<number> | number} [activeTester] Function to check if a listing is still active.
|
||||
*/
|
||||
|
||||
export {};
|
||||
11
lib/types/similarityCache.js
Normal file
11
lib/types/similarityCache.js
Normal file
@@ -0,0 +1,11 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SimilarityCache
|
||||
* @property {(params: { title?: string, address?: string, price?: number|string }) => boolean} checkAndAddEntry Checks if a listing is similar and adds it if not.
|
||||
*/
|
||||
|
||||
export {};
|
||||
Reference in New Issue
Block a user