External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

UI (#15)

Browse Source 
Adding new Admin UI. Updating Fredy to V3.0.0 as it has been a large rewrite. Thanks for all contributions and help on the way!
This commit is contained in:
Christian Kellner
2021-01-21 16:09:23 +01:00
committed by GitHub
parent 8185bfe818
commit b2847f6834
124 changed files with 9768 additions and 1495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
lib/FredyRuntime.js
View File

@@ -1,10 +1,5 @@
const { NoNewListingsError } = require('./errors');
const {
setKnownListings,
getKnownListings,
setNumberOfTotalFoundProviderListings,
getForTesting,
} = require('./services/store');
const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
const notify = require('./notification/notify');
const xray = require('./services/scraper');
@@ -13,7 +8,7 @@ class FredyRuntime {
/**
*
* @param providerConfig the config for the specific provider, we're going to query at the moment
* @param notificationConfig the config for all notifications (because all could be applied to a provider)
* @param notificationConfig the config for all notifications
* @param providerId the id of the provider currently in use
* @param jobKey key of the job that is currently running (from within the config)
*/
@@ -25,8 +20,6 @@ class FredyRuntime {
}
execute() {
if (!this._providerConfig.enabled) return Promise.resolve();
return (
Promise.resolve(this._providerConfig.url)
//scraping the site and try finding new listings
@@ -37,8 +30,6 @@ class FredyRuntime {
.then(this._filter.bind(this))
//check if new listings available. if so proceed
.then(this._findNew.bind(this))
//store update of number of found listings
.then(this._storeStats.bind(this))
//store everything in db
.then(this._save.bind(this))
//notify the user using the configured notification adapter
@@ -52,24 +43,16 @@ class FredyRuntime {
return new Promise((resolve, reject) => {
let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
if (this._providerConfig.paginage) {
x = x.paginate(this._providerConfig.paginage);
}
x((err, listings) => {
if (err) reject(err);
else {
if (err) {
reject(err);
} else {
resolve(listings);
}
});
});
}
_storeStats(listings) {
setNumberOfTotalFoundProviderListings(this._jobKey, this._providerId, listings.length);
return Promise.resolve(listings);
}
_normalize(listings) {
return listings.map(this._providerConfig.normalize);
}
@@ -79,7 +62,7 @@ class FredyRuntime {
}
_findNew(listings) {
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId).indexOf(o.id) === -1);
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
if (newListings.length === 0) {
throw new NoNewListingsError();
@@ -94,25 +77,17 @@ class FredyRuntime {
}
_save(newListings) {
setKnownListings(this._jobKey, this._providerId, [
...getKnownListings(this._jobKey, this._providerId),
...newListings.map((l) => l.id),
]);
const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
newListings.forEach((listing) => {
currentListings[listing.id] = Date.now();
});
setKnownListings(this._jobKey, this._providerId, currentListings);
return newListings;
}
_handleError(err) {
if (err.name !== 'NoNewListingsError') console.error(err);
}
/**
* for testing purposes only
* @returns {Store}
* @private
*/
_getStore() {
return getForTesting();
}
}
module.exports = FredyRuntime;