External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding similarity check (#29)

Browse Source 
* adding similarity check

* adding paging

* fixing tests

* docu

* better error handling

* fixing tests

* adjusting page limit

* fixing login screen

* cleanup

* upgrade browser list

* prevent spamming the log

* fixing tests

* removing job listings when removing a job or the user
This commit is contained in:
Christian Kellner
2021-06-28 08:52:09 +02:00
committed by GitHub
parent 88c046dbd4
commit 59e6d287fc
26 changed files with 1114 additions and 878 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
lib/FredyRuntime.js
View File

@@ -1,4 +1,4 @@
const { NoNewListingsError } = require('./errors');
const { NoNewListingsWarning } = require('./errors');
const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
const notify = require('./notification/notify');
@@ -12,12 +12,14 @@ class FredyRuntime {
* @param notificationConfig the config for all notifications
* @param providerId the id of the provider currently in use
* @param jobKey key of the job that is currently running (from within the config)
* @param similarityCache cache instance holding values to check for similarity of entries
*/
constructor(providerConfig, notificationConfig, providerId, jobKey) {
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
this._providerConfig = providerConfig;
this._notificationConfig = notificationConfig;
this._providerId = providerId;
this._jobKey = jobKey;
this._similarityCache = similarityCache;
}
execute() {
@@ -33,6 +35,8 @@ class FredyRuntime {
.then(this._findNew.bind(this))
//store everything in db
.then(this._save.bind(this))
//check for similar listings. if found, remove them before notifying
.then(this._filterBySimilarListings.bind(this))
//notify the user using the configured notification adapter
.then(this._notify.bind(this))
//if an error occurred on the way, handle it here.
@@ -53,14 +57,29 @@ class FredyRuntime {
}
const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
try {
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
.then((listings) => {
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
if (this._providerConfig.paginate != null) {
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
//the first 2 pages should be enough here
//TODO: Think about automagically sort by date
.limit(2)
.paginate(this._providerConfig.paginate)
.then((listings) => {
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
} else {
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
.then((listings) => {
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
}
} catch (error) {
reject(error);
console.error(error);
@@ -80,7 +99,7 @@ class FredyRuntime {
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
if (newListings.length === 0) {
throw new NoNewListingsError();
throw new NoNewListingsWarning();
}
return newListings;
@@ -100,8 +119,22 @@ class FredyRuntime {
return newListings;
}
_filterBySimilarListings(listings) {
const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
if (similar) {
/* eslint-disable no-console */
console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
/* eslint-enable no-console */
}
return !similar;
});
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
return filteredList;
}
_handleError(err) {
if (err.name !== 'NoNewListingsError') console.error(err);
if (err.name !== 'NoNewListingsWarning') console.error(err);
}
}