Concurrent-Jobs (#7)

Fredy 2.0.0 introduces the concept of search jobs. You can now configure multiple of these jobs running in the same instance of Fredy. Also a new API has been created 🎉

lib/FredyRuntime.js

@@ -0,0 +1,114 @@
+const { NoNewListingsError } = require('./errors');
+const {
+  setKnownListings,
+  getKnownListings,
+  setNumberOfTotalFoundProviderListings,
+  getForTesting
+} = require('./services/store');
+
+const notify = require('./notification/notify');
+const xray = require('./services/scraper');
+
+class FredyRuntime {
+  /**
+   *
+   * @param providerConfig the config for the specific provider, we're going to query at the moment
+   * @param notificationConfig the config for all notifications (because all could be applied to a provider)
+   * @param providerId the id of the provider currently in use
+   * @param jobKey key of the job that is currently running (from within the config)
+   */
+  constructor(providerConfig, notificationConfig, providerId, jobKey) {
+    this._providerConfig = providerConfig;
+    this._notificationConfig = notificationConfig;
+    this._providerId = providerId;
+    this._jobKey = jobKey;
+  }
+
+  execute() {
+    if (!this._providerConfig.enabled) return Promise.resolve();
+
+    return Promise.resolve(this._providerConfig.url)
+      .then(this._getListings.bind(this))
+      .then(this._normalize.bind(this))
+      .then(this._filter.bind(this))
+      .then(this._findNew.bind(this))
+      .then(this._storeStats.bind(this))
+      .then(this._save.bind(this))
+      .then(this._notify.bind(this))
+      .then(this._updateStates.bind(this))
+      .catch(this._handleError.bind(this));
+  }
+
+  _getListings(url) {
+    return new Promise((resolve, reject) => {
+      let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
+
+      if (this._providerConfig.paginage) {
+        x = x.paginate(this._providerConfig.paginage);
+      }
+
+      x((err, listings) => {
+        if (err) reject(err);
+        else {
+          resolve(listings);
+        }
+      });
+    });
+  }
+
+  _storeStats(listings) {
+    setNumberOfTotalFoundProviderListings(this._jobKey, this._providerId, listings.length);
+    return Promise.resolve(listings);
+  }
+
+  _normalize(listings) {
+    return listings.map(this._providerConfig.normalize);
+  }
+
+  _filter(listings) {
+    return listings.filter(this._providerConfig.filter);
+  }
+
+  _findNew(listings) {
+    const newListings = listings.filter(o => getKnownListings(this._jobKey, this._providerId).indexOf(o.id) === -1);
+
+    if (newListings.length === 0) {
+      this._updateStates([]);
+      throw new NoNewListingsError();
+    }
+
+    return newListings;
+  }
+
+  _notify(newListings) {
+    const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig);
+    return Promise.all(sendNotifications).then(() => newListings);
+  }
+
+  _updateStates(newListings) {
+    return newListings;
+  }
+
+  _save(newListings) {
+    setKnownListings(this._jobKey, this._providerId, [
+      ...getKnownListings(this._jobKey, this._providerId),
+      ...newListings.map(l => l.id)
+    ]);
+    return newListings;
+  }
+
+  _handleError(err) {
+    if (err.name !== 'NoNewListingsError') console.error(err);
+  }
+
+  /**
+   * for testing purposes only
+   * @returns {Store}
+   * @private
+   */
+  _getStore() {
+    return getForTesting();
+  }
+}
+
+module.exports = FredyRuntime;

lib/api/api.js

@@ -0,0 +1,62 @@
+const bodyParser = require('body-parser');
+const config = require('../../conf/config');
+const { getLastJobExecution, getLastProviderExecution, getTotalNumberOfListings } = require('../services/store');
+const PORT = 9988;
+const service = require('restana')();
+service.use(bodyParser.json());
+
+service.get('/', async (req, res) => {
+  const result = {};
+  Object.keys(config.jobs).forEach(job => {
+    result[job] = {
+      lastExecution: getLastJobExecution(job),
+      enabledProvider: Object.keys(config.jobs[job].provider)
+        .filter(providerKey => config.jobs[job].provider[providerKey].enabled)
+        .map(providerKey => {
+          return {
+            name: providerKey,
+            lastExecution: getLastProviderExecution(job, providerKey),
+            totalFindings: getTotalNumberOfListings(job, providerKey)
+          };
+        })
+    };
+  });
+  res.body = result;
+  res.send();
+});
+
+service.get('/jobs/:name', async (req, res) => {
+  const { name: jobKey } = req.params;
+  if (Object.keys(config.jobs).indexOf(jobKey) === -1) {
+    console.error(`Cannot find job with name ${jobKey}. Available Jobs are [${Object.keys(config.jobs)}]`);
+    res.send(404);
+    return;
+  }
+  res.body = {
+    lastExecution: getLastJobExecution(jobKey),
+    enabledProvider: Object.keys(config.jobs[jobKey].provider)
+      .filter(providerKey => config.jobs[jobKey].provider[providerKey].enabled)
+      .map(providerKey => {
+        return {
+          name: providerKey,
+          url: config.jobs[jobKey].provider[providerKey].url,
+          lastExecution: getLastProviderExecution(jobKey, providerKey),
+          totalFindings: getTotalNumberOfListings(jobKey, providerKey)
+        };
+      })
+  };
+  res.send();
+});
+
+service.get('/ping', function(req, res) {
+  res.body = {
+    pong: 'pong'
+  };
+  res.send();
+});
+
+service.start(PORT).then(() => {
+  /* eslint-disable no-console */
+  console.info(`Started API service on port ${PORT}`);
+  /* eslint-enable no-console */
+});

lib/errors.js

@@ -1,16 +1,15 @@
 class ExtendableError extends Error {
-    constructor(message) {
-        super(message);
-        this.name = this.constructor.name;
-        if (typeof Error.captureStackTrace === 'function') {
-            Error.captureStackTrace(this, this.constructor)
-        } else {
-            this.stack = (new Error(message)).stack
-        }
+  constructor(message) {
+    super(message);
+    this.name = this.constructor.name;
+    if (typeof Error.captureStackTrace === 'function') {
+      Error.captureStackTrace(this, this.constructor);
+    } else {
+      this.stack = new Error(message).stack;
     }
+  }
 }
 
-class NoNewListingsError extends ExtendableError {
-}
+class NoNewListingsError extends ExtendableError {}
 
-module.exports = {NoNewListingsError};
+module.exports = { NoNewListingsError };

lib/fredy.js

@@ -1,106 +0,0 @@
-const {NoNewListingsError} = require('./errors');
-const Store = require('./services/store');
-
-const notify = require('./notification/notify');
-const xray = require('./services/scraper');
-
-class Fredy {
-    constructor(source) {
-        this._store = new Store(source.name);
-        this._fullCrawl = true;
-        this._source = source;
-        this._stats = null;
-    }
-
-    run(stats) {
-
-        if(!this._stats){
-            this._stats = stats;
-        }
-
-        if (!this._source.enabled) return Promise.resolve();
-
-        return Promise.resolve(this._source.url)
-            .then(this._store.warmup)
-            .then(this._getListings.bind(this))
-            .then(this._normalize.bind(this))
-            .then(this._filter.bind(this))
-            .then(this._findNew.bind(this))
-            .then(this._save.bind(this))
-            .then(this._notify.bind(this))
-            .then(this._updateStates.bind(this))
-            .catch(this._handleError.bind(this))
-    }
-
-    _getListings(url) {
-        return new Promise((resolve, reject) => {
-            let x = xray(url, this._source.crawlContainer, [this._source.crawlFields]);
-
-            if (this._source.paginage && this._fullCrawl) {
-                this._fullCrawl = false;
-                x = x.paginate(this._source.paginage)
-            }
-
-            x((err, listings) => {
-                if (err) reject(err);
-                else {
-                    resolve(listings);
-                }
-            })
-        })
-    }
-
-    _normalize(listings) {
-        return listings.map(this._source.normalize)
-    }
-
-    _filter(listings) {
-        return listings.filter(this._source.filter)
-    }
-
-    _findNew(listings) {
-        const newListings = listings.filter(
-            o => this._store.knownListings.indexOf(o.id) === -1
-        );
-
-        if (newListings.length === 0) {
-            this._updateStates([]);
-            throw new NoNewListingsError();
-        }
-
-        return newListings
-    }
-
-    _notify(newListings) {
-        const sendNotifications = notify.send(this._source.name, newListings);
-        return Promise.all(sendNotifications).then(() => newListings)
-    }
-
-    _updateStates(newListings){
-        this._stats.setLastScrape(this._source.name, newListings.length);
-        return newListings;
-    }
-
-    _save(newListings) {
-        this._store.knownListings = [
-            ...this._store.knownListings,
-            ...newListings.map(l => l.id)
-        ];
-        return newListings;
-    }
-
-    _handleError(err) {
-        if (err.name !== 'NoNewListingsError') console.error(err)
-    }
-
-    /**
-     * for testing purposes only
-     * @returns {Store}
-     * @private
-     */
-    _getStore(){
-        return this._store;
-    }
-}
-
-module.exports = Fredy;

lib/notification/adapter/console.js

@@ -1,17 +1,15 @@
-const config = require('../../../conf/config.json');
-
 /**
  * simply prints out the found data to the console
  * @param serviceName e.g immoscout
  * @param newListings an array with newly found listings
+ * @param notificationConfig config of this notification adapter
  */
-exports.send = (serviceName, newListings) => {
-    return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
-};
-
-/**
- * each integration needs to implement this method
- */
-exports.enabled = () => {
-  return config.notification.console.enabled;
+exports.send = (serviceName, newListings, notificationConfig) => {
+  const { enabled } = notificationConfig.console;
+  if (!enabled) {
+    return [Promise.resolve()];
+  }
+  /* eslint-disable no-console */
+  return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
+  /* eslint-enable no-console */
 };

lib/notification/adapter/slack.js

@@ -1,54 +1,50 @@
 const Slack = require('slack');
-const config = require('../../../conf/config.json');
 const msg = Slack.chat.postMessage;
 
-const {token, channel} = config.notification.slack;
-
 /**
  * sends a new listing to slack
  * @param serviceName e.g immoscout
  * @param newListings an array with newly found listings
+ * @param notificationConfig config of this notification adapter
  * @returns {Promise<Chat.PostMessage.Response> | void}
  */
-exports.send = (serviceName, newListings) => {
-    return newListings.map(payload => msg({
-            token,
-            channel,
-            text: `*(${serviceName})* - ${payload.title}`,
-            "attachments": [
-                {
-                    "fallback": payload.title,
-                    "color": "#36a64f",
-                    "title": "Link to Exposé",
-                    "title_link": payload.link,
-                    "fields": [
-                        {
-                            "title": "Price",
-                            "value": payload.price,
-                            "short": false
-                        },
-                        {
-                            "title": "Size",
-                            "value": payload.size,
-                            "short": false
-                        },
-                        {
-                            "title": "Address",
-                            "value": payload.address,
-                            "short": false
-                        }
-                    ],
-                    "footer": "Powered by Fredy",
-                    ts: new Date().getTime() / 1000
-                }
-            ]
-        })
-    );
-};
-
-/**
- * each integration needs to implement this method
- */
-exports.enabled = () => {
-    return config.notification.slack.enabled;
+exports.send = (serviceName, newListings, notificationConfig) => {
+  const { token, channel, enabled } = notificationConfig.slack;
+  if (!enabled) {
+    return [Promise.resolve()];
+  }
+  return newListings.map(payload =>
+    msg({
+      token,
+      channel,
+      text: `*(${serviceName})* - ${payload.title}`,
+      attachments: [
+        {
+          fallback: payload.title,
+          color: '#36a64f',
+          title: 'Link to Exposé',
+          title_link: payload.link,
+          fields: [
+            {
+              title: 'Price',
+              value: payload.price,
+              short: false
+            },
+            {
+              title: 'Size',
+              value: payload.size,
+              short: false
+            },
+            {
+              title: 'Address',
+              value: payload.address,
+              short: false
+            }
+          ],
+          footer: 'Powered by Fredy',
+          ts: new Date().getTime() / 1000
+        }
+      ]
+    })
+  );
 };

lib/notification/adapter/telegram.js

@@ -1,35 +1,33 @@
-const config = require('../../../conf/config.json');
 const TelegramBot = require('tg-yarl');
 
-const opts = { parse_mode: 'Markdown' };
-const bot = new TelegramBot(config.notification.telegram.token);
+const opts = {parse_mode: 'Markdown'};
 
 
 /**
  * sends new listings to telegram
  * @param serviceName e.g immoscout
  * @param newListings an array with newly found listings
+ * @param notificationConfig config of this notification adapter
  * @returns {Promise<Void> | void}
  */
-exports.send = (serviceName, newListings) => {
+exports.send = (serviceName, newListings, notificationConfig) => {
+    const {enabled, token, chatId} = notificationConfig.telegram;
+    if (!enabled) {
+        return [Promise.resolve()];
+    }
+
+    const bot = new TelegramBot(token);
+
     let message = `Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
 
     message += newListings.map(o =>
-            `*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
-            [o.address, o.price, o.size].join(' | ') + '\n' +
-            `[LINK](${o.link})\n\n`);
+        `*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
+        [o.address, o.price, o.size].join(' | ') + '\n' +
+        `[LINK](${o.link})\n\n`);
 
-    return bot.sendMessage(config.notification.telegram.chatId, message, opts);
+    return bot.sendMessage(chatId, message, opts);
 };
 
-/**
- * each integration needs to implement this method
- */
-exports.enabled = () => {
-    return config.notification.telegram.enabled;
-};
-
-
-function shorten (str, len = 30) {
+function shorten(str, len = 30) {
     return str.length > len ? str.substring(0, len) + '...' : str;
 }

lib/notification/notify.js

@@ -4,14 +4,13 @@ const path = './adapter';
 /** Read every integration existing in ./adapter **/
 const adapter = fs
   .readdirSync('./lib/notification/adapter')
-  .map(integPath => require(`${path}/${integPath}`))
-  .filter(integration => integration.enabled());
+  .map(integPath => require(`${path}/${integPath}`));
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
 
-exports.send = (serviceName, payload) => {
+exports.send = (serviceName, payload, notificationConfig) => {
   //this is not being used in tests, therefor adapter are always set
-  return adapter.map(a => a.send(serviceName, payload));
+  return adapter.map(a => a.send(serviceName, payload, notificationConfig));
 };

lib/provider/einsAImmobilien.js

@@ -1,11 +1,11 @@
-const config = require('../../conf/config.json');
-const Fredy = require('../fredy');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+
 function normalize(o) {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
-  if(o.rooms != null){
-      size+=` / / ${o.rooms.trim()}`;
+  if (o.rooms != null) {
+    size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
 
@@ -13,16 +13,15 @@ function normalize(o) {
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
-  const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
 
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
-const einsAImmobilien = {
-  name: 'einsAImmobilien',
-  enabled: config.sources.einsAImmobilien.enabled,
-  url: config.sources.einsAImmobilien.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '.tabelle',
   crawlFields: {
     id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
@@ -37,4 +36,13 @@ const einsAImmobilien = {
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(einsAImmobilien);
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'einsAImmobilien';
+
+exports.config = config;

lib/provider/immonet.js

@@ -1,7 +1,7 @@
-const config = require('../../conf/config.json');
-const Fredy = require('../fredy');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+
 function normalize(o) {
   const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
@@ -13,16 +13,15 @@ function normalize(o) {
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
-  const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
 
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
-const immonet = {
-  name: 'immonet',
-  enabled: config.sources.immonet.enabled,
-  url: config.sources.immonet.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '#result-list-stage .item',
   crawlFields: {
     id: '@id',
@@ -37,4 +36,13 @@ const immonet = {
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(immonet);
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'immonet';
+
+exports.config = config;

lib/provider/immoscout.js

@@ -1,7 +1,7 @@
-const config = require('../../conf/config.json');
-const Fredy = require('../fredy');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+
 function normalize(o) {
   const title = o.title.replace('NEU', '');
   const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
@@ -10,25 +10,33 @@ function normalize(o) {
 }
 
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, config.blacklist);
+  return !utils.isOneOf(o.title, appliedBlackList);
 }
 
-const immoscout = {
-    name: 'immoscout',
-    enabled: config.sources.immoscout.enabled,
-    url: config.sources.immoscout.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '#resultListItems li.result-list__listing',
-    crawlFields: {
+  crawlFields: {
     id: '.result-list-entry@data-obid | int',
     price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
     size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
     title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
     link: '.result-list-entry .result-list-entry__brand-title-container@href',
     address: '.result-list-entry .result-list-entry__map-link'
-    },
+  },
   paginate: '#pager .align-right a@href',
-    normalize: normalize,
-    filter: applyBlacklist
+  normalize: normalize,
+  filter: applyBlacklist
 };
 
-module.exports = new Fredy(immoscout);
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'immoscout';
+
+exports.config = config;

lib/provider/immowelt.js

@@ -1,7 +1,7 @@
-const Fredy = require('../fredy');
-const config = require('../../conf/config.json');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+
 function normalize(o) {
   const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
   const address = o.address;
@@ -10,16 +10,15 @@ function normalize(o) {
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
-  const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
 
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
-const immowelt = {
-  name: 'immowelt',
-  enabled: config.sources.immowelt.enabled,
-  url: config.sources.immowelt.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '.immoliste .js-object.listitem_wrap ',
   crawlFields: {
     id: '@data-estateid | int',
@@ -34,4 +33,13 @@ const immowelt = {
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(immowelt);
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'immowelt';
+
+exports.config = config;

lib/provider/kalaydo.js

@@ -1,7 +1,8 @@
-const config = require('../../conf/config.json');
-const Fredy = require('../fredy');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+let appliedBlacklistedDistricts = [];
+
 function normalize(o) {
   const id = o.id
     .split('/')
@@ -16,19 +17,18 @@ function normalize(o) {
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
-  const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
 
   const isBlacklistedDistrict =
-    config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.title, config.blacklistedDistrics);
+      appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
 
   return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
-const kalaydo = {
-  name: 'kalaydo',
-  enabled: config.sources.kalaydo.enabled,
-  url: config.sources.kalaydo.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '#resultList .resultitem-content-container',
   crawlFields: {
     id: '.resultitem-content-container a@href',
@@ -43,4 +43,15 @@ const kalaydo = {
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(kalaydo);
+exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
+    config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
+    appliedBlackList = blacklist;
+    appliedBlacklistedDistricts = blacklistedDistricts;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'kalaydo';
+
+exports.config = config;
+

lib/provider/kleinanzeigen.js

@@ -1,7 +1,8 @@
-const Fredy = require('../fredy');
-const config = require('../../conf/config.json');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+let appliedBlacklistedDistricts = [];
+
 function normalize(o) {
   const size = o.size || '--- m²';
 
@@ -9,18 +10,17 @@ function normalize(o) {
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
-  const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   const isBlacklistedDistrict =
-    config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
+      appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
 
   return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
-const kleinanzeigen = {
-  name: 'kleinanzeigen',
-  enabled: config.sources.kleinanzeigen.enabled,
-  url: config.sources.kleinanzeigen.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '#srchrslt-adtable .ad-listitem',
   crawlFields: {
     id: '.aditem@data-adid | int',
@@ -36,4 +36,15 @@ const kleinanzeigen = {
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(kleinanzeigen);
+exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
+    config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
+    appliedBlacklistedDistricts = blacklistedDistricts;
+    appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'kleinanzeigen';
+
+exports.config = config;
+

lib/provider/neubauKompass.js

@@ -1,19 +1,18 @@
-const config = require('../../conf/config.json');
-const Fredy = require('../fredy');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+
 function normalize(o) {
   return o;
 }
 
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, config.blacklist);
+  return !utils.isOneOf(o.title, appliedBlackList);
 }
 
-const neubauKompass = {
-  name: 'neubauKompass',
-  enabled: config.sources.neubauKompass.enabled,
-  url: config.sources.neubauKompass.url,
+const config = {
+  enabled: null,
+  url: null,
   crawlContainer: '.row article',
   crawlFields: {
     id: '@id',
@@ -26,4 +25,13 @@ const neubauKompass = {
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(neubauKompass);
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'neubauKompass';
+
+exports.config = config;

lib/provider/wgGesucht.js

@@ -1,34 +1,42 @@
-const config = require('../../conf/config.json');
-const Fredy = require('../fredy');
 const utils = require('../utils');
 
+let appliedBlackList = [];
+
 function normalize(o) {
   return o;
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
-  const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
-
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
 
   return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
 
-const wgGesucht = {
-  name: 'wgGesucht',
-  enabled: config.sources.wgGesucht.enabled,
-  url: config.sources.wgGesucht.url,
-  crawlContainer: '#main_column .panel:not(.display-none):not(.noprint)',
+const config = {
+  enabled: null,
+  url: null,
+  crawlContainer: '#main_column .wgg_card',
   crawlFields: {
     id: '@data-id',
-    details: ' .list-details-costs-col |removeNewline |trim',
-    title: '.headline .detailansicht |removeNewline |trim',
-    description: '.list-details-category-location |removeNewline |trim',
-    link: '.headline .detailansicht@href'
+    details: '.row .noprint .col-xs-11 |removeNewline |trim',
+    price: '.middle .col-xs-3 |removeNewline |trim',
+    size: '.middle .text-right |removeNewline |trim',
+    title: '.truncate_title a |removeNewline |trim',
+    link: '.truncate_title a@href'
   },
   paginate: '.pagination-sm:first a:last@href',
   normalize: normalize,
   filter: applyBlacklist
 };
 
-module.exports = new Fredy(wgGesucht);
+exports.init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist;
+};
+
+//must match the id of the source given in the config!
+exports.id = () => 'wgGesucht';
+
+exports.config = config;

lib/services/stats.js

@@ -1,35 +0,0 @@
-const config = require('../../conf/config.json');
-let stats = {
-  lastScrape: {},
-  foundScrapes: {}
-};
-
-if (config.enableStats) {
-  const http = require('http');
-  http
-    .createServer((req, res) => {
-      res.writeHead(200, { 'Content-Type': 'application/json' });
-      res.end(
-        JSON.stringify({
-          config,
-          stats
-        })
-      );
-    })
-    .listen(config.statsPort, '127.0.0.1');
-}
-
-const datetime = date => {
-  return `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()} ${date.getHours()}:${date.getMinutes()}`;
-};
-
-exports.setLastScrape = (serviceName, numberOfNewListsings) => {
-  const d = new Date();
-  const dt = datetime(d);
-  stats.lastScrape[serviceName] = d.toString();
-
-  if (numberOfNewListsings > 0) {
-    stats.foundScrapes[dt] = stats.foundScrapes[dt] || {};
-    stats.foundScrapes[dt][serviceName] = numberOfNewListsings;
-  }
-};

lib/services/store.js

@@ -7,30 +7,79 @@ const low = require('lowdb');
 
 const lowdb = low(adapter);
 
-class Store {
-  constructor(name) {
-    this._name = name;
-    this._db = null;
-  }
+let db = null;
 
-  get warmup() {
-    return new Promise(resolve => {
-      lowdb.then(db => {
-        this._db = db;
-        resolve();
-      });
+const buildKey = (jobKey, providerId, endpoint) => {
+  let key = `${jobKey}`;
+  if (jobKey == null && endpoint == null) {
+    return key;
+  }
+  if (providerId != null) {
+    key += `.${providerId}`;
+  }
+  if (endpoint != null) {
+    key += `.${endpoint}`;
+  }
+  return key;
+};
+
+exports.init = () => {
+  return new Promise(resolve => {
+    //warmup
+    lowdb.then(database => {
+      db = database;
+      /* eslint-disable no-console */
+      console.info('Warming up database successful');
+      /* eslint-enable no-console */
+      resolve();
     });
+  });
+};
+
+exports.setKnownListings = (jobKey, providerId, listings) => {
+  if (!Array.isArray(listings)) throw Error('Not a valid array');
+  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
+  const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
+
+  return db
+    .set(providerListingsKey, listings)
+    .set(providerLastScrapeKey, Date.now())
+    .write();
+};
+
+exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
+  if (numberOfNewListings > 0) {
+    const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
+    const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
+    db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
   }
+};
 
-  set knownListings(value) {
-    if (!Array.isArray(value)) throw Error('Not a valid array');
+exports.setLastJobExecution = jobKey => {
+  const key = buildKey(jobKey, null, 'lastJobExecution');
+  return db.set(key, Date.now()).write();
+};
 
-    return this._db.set(this._name, value).write();
-  }
+exports.getKnownListings = (jobKey, providerId) => {
+  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
+  return db.get(providerListingsKey).value() || [];
+};
 
-  get knownListings() {
-    return this._db.get(this._name).value() || [];
-  }
-}
+exports.getLastProviderExecution = (jobKey, providerId) => {
+  const key = buildKey(jobKey, providerId, 'lastProviderExecution');
+  return db.get(key).value() || 0;
+};
 
-module.exports = Store;
+exports.getLastJobExecution = jobKey => {
+  const key = buildKey(jobKey, null, 'lastJobExecution');
+  return db.get(key).value() || 0;
+};
+
+exports.getTotalNumberOfListings = (jobKey, providerId) => {
+  const key = buildKey(jobKey, providerId, 'foundListings');
+  return db.get(key).value() || 0;
+};
+
+exports.getForTesting = () => {
+  return db;
+};

lib/utils.js

@@ -1,8 +1,11 @@
-function isOneOf (word, arr) {
+function isOneOf(word, arr) {
+  if (arr == null || arr.length === 0) {
+    return false;
+  }
   const expression = String.raw`\b(${arr.join('|')})\b`;
   const blacklist = new RegExp(expression, 'ig');
 
-  return blacklist.test(word)
+  return blacklist.test(word);
 }
 
 module.exports = { isOneOf };