External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

improving similarity cache

Browse Source
This commit is contained in:
orangecoding
2025-10-29 09:36:05 +01:00
parent d433b13db6
commit 79a8420dfb
20 changed files with 169 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/FredyPipeline.js
View File

@@ -183,8 +183,12 @@ class FredyPipeline {
* @returns {Listing[]} Listings considered unique enough to keep.
*/
_filterBySimilarListings(listings) {
const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
return listings.filter((listing) => {
const similar = this._similarityCache.checkAndAddEntry({
title: listing.title,
address: listing.address,
price: listing.price,
});
if (similar) {
logger.debug(
`Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
@@ -192,8 +196,6 @@ class FredyPipeline {
}
return !similar;
});
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
return filteredList;
}
/**