Concurrent-Jobs (#7)

Fredy 2.0.0 introduces the concept of search jobs. You can now configure multiple of these jobs running in the same instance of Fredy. Also a new API has been created 🎉

lib/services/stats.js

@@ -1,35 +0,0 @@
-const config = require('../../conf/config.json');
-let stats = {
-  lastScrape: {},
-  foundScrapes: {}
-};
-
-if (config.enableStats) {
-  const http = require('http');
-  http
-    .createServer((req, res) => {
-      res.writeHead(200, { 'Content-Type': 'application/json' });
-      res.end(
-        JSON.stringify({
-          config,
-          stats
-        })
-      );
-    })
-    .listen(config.statsPort, '127.0.0.1');
-}
-
-const datetime = date => {
-  return `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()} ${date.getHours()}:${date.getMinutes()}`;
-};
-
-exports.setLastScrape = (serviceName, numberOfNewListsings) => {
-  const d = new Date();
-  const dt = datetime(d);
-  stats.lastScrape[serviceName] = d.toString();
-
-  if (numberOfNewListsings > 0) {
-    stats.foundScrapes[dt] = stats.foundScrapes[dt] || {};
-    stats.foundScrapes[dt][serviceName] = numberOfNewListsings;
-  }
-};

lib/services/store.js

@@ -7,30 +7,79 @@ const low = require('lowdb');
 
 const lowdb = low(adapter);
 
-class Store {
-  constructor(name) {
-    this._name = name;
-    this._db = null;
-  }
+let db = null;
 
-  get warmup() {
-    return new Promise(resolve => {
-      lowdb.then(db => {
-        this._db = db;
-        resolve();
-      });
+const buildKey = (jobKey, providerId, endpoint) => {
+  let key = `${jobKey}`;
+  if (jobKey == null && endpoint == null) {
+    return key;
+  }
+  if (providerId != null) {
+    key += `.${providerId}`;
+  }
+  if (endpoint != null) {
+    key += `.${endpoint}`;
+  }
+  return key;
+};
+
+exports.init = () => {
+  return new Promise(resolve => {
+    //warmup
+    lowdb.then(database => {
+      db = database;
+      /* eslint-disable no-console */
+      console.info('Warming up database successful');
+      /* eslint-enable no-console */
+      resolve();
     });
+  });
+};
+
+exports.setKnownListings = (jobKey, providerId, listings) => {
+  if (!Array.isArray(listings)) throw Error('Not a valid array');
+  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
+  const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
+
+  return db
+    .set(providerListingsKey, listings)
+    .set(providerLastScrapeKey, Date.now())
+    .write();
+};
+
+exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
+  if (numberOfNewListings > 0) {
+    const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
+    const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
+    db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
   }
+};
 
-  set knownListings(value) {
-    if (!Array.isArray(value)) throw Error('Not a valid array');
+exports.setLastJobExecution = jobKey => {
+  const key = buildKey(jobKey, null, 'lastJobExecution');
+  return db.set(key, Date.now()).write();
+};
 
-    return this._db.set(this._name, value).write();
-  }
+exports.getKnownListings = (jobKey, providerId) => {
+  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
+  return db.get(providerListingsKey).value() || [];
+};
 
-  get knownListings() {
-    return this._db.get(this._name).value() || [];
-  }
-}
+exports.getLastProviderExecution = (jobKey, providerId) => {
+  const key = buildKey(jobKey, providerId, 'lastProviderExecution');
+  return db.get(key).value() || 0;
+};
 
-module.exports = Store;
+exports.getLastJobExecution = jobKey => {
+  const key = buildKey(jobKey, null, 'lastJobExecution');
+  return db.get(key).value() || 0;
+};
+
+exports.getTotalNumberOfListings = (jobKey, providerId) => {
+  const key = buildKey(jobKey, providerId, 'foundListings');
+  return db.get(key).value() || 0;
+};
+
+exports.getForTesting = () => {
+  return db;
+};