Release v1.0.0 🎉

lib/fredy.js

@@ -0,0 +1,108 @@
+const {NoNewListingsError} = require('./errors');
+const Store = require('./services/store');
+
+const notify = require('./notification/notify');
+const xray = require('./services/scraper');
+
+class Fredy {
+    constructor(source) {
+        this._store = new Store(source.name);
+        this._fullCrawl = true;
+        this._source = source;
+        this._stats = null;
+    }
+
+    run(stats) {
+
+        if(!this._stats){
+            this._stats = stats;
+        }
+
+        if (!this._source.enabled) return Promise.resolve();
+
+        return Promise.resolve(this._source.url)
+            .then(this._store.warmup)
+            .then(this._getListings.bind(this))
+            .then(this._normalize.bind(this))
+            .then(this._filter.bind(this))
+            .then(this._findNew.bind(this))
+            .then(this._save.bind(this))
+            .then(this._notify.bind(this))
+            .then(this._updateStates.bind(this))
+            .catch(this._handleError.bind(this))
+    }
+
+    _getListings(url) {
+        return new Promise((resolve, reject) => {
+            let x = xray(url, this._source.crawlContainer, [this._source.crawlFields]);
+
+            if (this._source.paginage && this._fullCrawl) {
+                this._fullCrawl = false;
+                x = x.paginate(this._source.paginage)
+            }
+
+            x((err, listings) => {
+                if (err) reject(err);
+                else resolve(listings)
+            })
+        })
+    }
+
+    _normalize(listings) {
+        return listings.map(this._source.normalize)
+    }
+
+    _filter(listings) {
+        return listings.filter(this._source.filter)
+    }
+
+    _findNew(listings) {
+        const newListings = listings.filter(
+            o => this._store.knownListings.indexOf(o.id) === -1
+        );
+
+        if (newListings.length === 0) {
+            this._updateStates([]);
+            throw new NoNewListingsError();
+        }
+
+        return newListings
+    }
+
+    _notify(newListings) {
+        const sendNotifications = newListings.map(payload => {
+            return notify.send(this._source.name, payload);
+        }
+        );
+
+        return Promise.all(sendNotifications).then(() => newListings)
+    }
+
+    _updateStates(newListings){
+        this._stats.setLastScrape(this._source.name, newListings.length);
+        return newListings;
+    }
+
+    _save(newListings) {
+        this._store.knownListings = [
+            ...this._store.knownListings,
+            ...newListings.map(l => l.id)
+        ];
+        return newListings;
+    }
+
+    _handleError(err) {
+        if (err.name !== 'NoNewListingsError') console.error(err)
+    }
+
+    /**
+     * for testing purposes only
+     * @returns {Store}
+     * @private
+     */
+    _getStore(){
+        return this._store;
+    }
+}
+
+module.exports = Fredy;