UI (#15)

Adding new Admin UI. Updating Fredy to V3.0.0 as it has been a large rewrite. Thanks for all contributions and help on the way!

lib/services/markdown.js

@@ -0,0 +1,6 @@
+const markdown = require('markdown').markdown;
+const fs = require('fs');
+
+exports.markdown2Html = function markdown2Html(filePath) {
+  return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
+};

lib/services/scraper.js

@@ -6,14 +6,16 @@ class Scraper {
     const filters = {
       removeNewline: this._removeNewline,
       trim: this._trim,
-      int: this._int
+      int: this._int,
     };
 
     const driver = makeDriver({
       headers: {
         'User-Agent':
-          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36'
-      }
+          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
+        cookie:
+          'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
+      },
     });
 
     const xray = Xray({ filters });

lib/services/security/hash.js

@@ -0,0 +1,3 @@
+const crypto = require('crypto');
+
+exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/storage/jobStorage.js

@@ -0,0 +1,83 @@
+const path = require('path');
+const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
+const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new FileSync(DB_PATH);
+const low = require('lowdb');
+const db = low(adapter);
+const { nanoid } = require('nanoid');
+const listingStorage = require('./listingsStorage');
+
+db.defaults({ jobs: [] }).write();
+
+exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+  const currentJob =
+    jobId == null
+      ? null
+      : db
+          .get('jobs')
+          .find((job) => job.id === jobId)
+          .value();
+
+  const jobs = db
+    .get('jobs')
+    .value()
+    .filter((job) => job.id !== jobId);
+
+  jobs.push({
+    id: jobId || nanoid(),
+    //make sure to not overwrite the user id in case an admin changes the job
+    userId: currentJob == null ? userId : currentJob.userId,
+    enabled,
+    name,
+    blacklist,
+    provider,
+    notificationAdapter,
+  });
+
+  db.set('jobs', jobs).write();
+};
+
+exports.getJob = (jobId) => {
+  const job = db
+    .get('jobs')
+    .find((job) => job.id === jobId)
+    .value();
+
+  if (job == null) {
+    return null;
+  }
+
+  return {
+    ...job,
+    numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
+  };
+};
+
+exports.setJobStatus = ({ jobId, status }) => {
+  db.get('jobs')
+    .find((job) => job.id === jobId)
+    .assign({ enabled: status })
+    .write();
+};
+
+exports.removeJob = (jobId) => {
+  db.get('jobs')
+    .remove((job) => job.id === jobId)
+    .write();
+};
+
+exports.removeJobsByUserId = (userId) => {
+  db.get('jobs')
+    .remove((job) => job.userId === userId)
+    .write();
+};
+
+exports.getJobs = () => {
+  return db
+    .get('jobs')
+    .value()
+    .map((job) => ({
+      ...job,
+      numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
+    }));
+};

lib/services/storage/listingsStorage.js

@@ -0,0 +1,49 @@
+const path = require('path');
+
+const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
+const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new FileSync(DB_PATH);
+const low = require('lowdb');
+const db = low(adapter);
+
+const buildKey = (jobKey, providerId, endpoint) => {
+  let key = `${jobKey}`;
+  if (jobKey == null && endpoint == null) {
+    return key;
+  }
+  if (providerId != null) {
+    key += `.${providerId}`;
+  }
+  if (endpoint != null) {
+    key += `.${endpoint}`;
+  }
+  return key;
+};
+
+exports.getNumberOfAllKnownListings = (jobId) => {
+  const data = db.get(`${jobId}.providerData`).value() || {};
+  return Object.values(data)
+    .map((values) => Object.keys(values).length)
+    .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
+};
+
+exports.getListingProviderDataForAnalytics = (jobId) => {
+  const key = buildKey(jobId, 'providerData');
+  return db.get(key).value() || {};
+};
+
+exports.getKnownListings = (jobId, providerId) => {
+  const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
+  return db.get(providerListingsKey).value() || {};
+};
+
+exports.setKnownListings = (jobId, providerId, listings) => {
+  const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
+
+  return db.set(providerListingsKey, listings).write();
+};
+
+exports.setLastJobExecution = (jobId) => {
+  const key = buildKey(jobId, null, 'lastExecution');
+  return db.set(key, Date.now()).write();
+};

lib/services/storage/userStorage.js

@@ -0,0 +1,83 @@
+const path = require('path');
+const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
+const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new FileSync(DB_PATH);
+const low = require('lowdb');
+const db = low(adapter);
+const hasher = require('../security/hash');
+const { nanoid } = require('nanoid');
+const jobStorage = require('./jobStorage');
+
+db.defaults({
+  user: [
+    //you probably want to change the default password ;)
+    {
+      id: nanoid(),
+      lastLogin: Date.now(),
+      username: 'admin',
+      password: hasher.hash('admin'),
+      isAdmin: true,
+      isDemo: false,
+    },
+  ],
+}).write();
+
+exports.getUsers = (withPassword) => {
+  const jobs = jobStorage.getJobs();
+  return db
+    .get('user')
+    .value()
+    .map((user) => ({
+      //we dont want the password in the frontend, even tho it's hashed
+      ...user,
+      password: withPassword ? user.password : null,
+      numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
+    }));
+};
+
+exports.getUser = (id) => {
+  const jobs = jobStorage.getJobs();
+  const user = db
+    .get('user')
+    .value()
+    .find((user) => user.id === id);
+  if (user == null) {
+    return null;
+  }
+  return {
+    ...user,
+    numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
+  };
+};
+
+exports.upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db
+    .get('user')
+    .value()
+    .filter((u) => u.id !== userId);
+
+  user.push({
+    id: userId || nanoid(),
+    username,
+    lastLogin: user.lastLogin,
+    password: hasher.hash(password),
+    isAdmin,
+  });
+
+  db.set('user', user).write();
+};
+
+exports.setLastLoginToNow = ({ userId }) => {
+  db.get('user')
+    .find((u) => u.id === userId)
+    .assign({ lastLogin: Date.now() })
+    .write();
+};
+
+exports.removeUser = (userId) => {
+  const user = db.get('user').value();
+  db.set(
+    'user',
+    user.filter((u) => u.id !== userId)
+  ).write();
+};

lib/services/store.js

@@ -1,85 +0,0 @@
-const path = require('path');
-const DB_PATH = path.dirname(require.main.filename) + '/conf/store.json';
-
-const FileAsync = require('lowdb/adapters/FileAsync');
-const adapter = new FileAsync(DB_PATH);
-const low = require('lowdb');
-
-const lowdb = low(adapter);
-
-let db = null;
-
-const buildKey = (jobKey, providerId, endpoint) => {
-  let key = `${jobKey}`;
-  if (jobKey == null && endpoint == null) {
-    return key;
-  }
-  if (providerId != null) {
-    key += `.${providerId}`;
-  }
-  if (endpoint != null) {
-    key += `.${endpoint}`;
-  }
-  return key;
-};
-
-exports.init = () => {
-  return new Promise(resolve => {
-    //warmup
-    lowdb.then(database => {
-      db = database;
-      /* eslint-disable no-console */
-      console.info('Warming up database successful');
-      /* eslint-enable no-console */
-      resolve();
-    });
-  });
-};
-
-exports.setKnownListings = (jobKey, providerId, listings) => {
-  if (!Array.isArray(listings)) throw Error('Not a valid array');
-  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
-  const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
-
-  return db
-    .set(providerListingsKey, listings)
-    .set(providerLastScrapeKey, Date.now())
-    .write();
-};
-
-exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
-  if (numberOfNewListings > 0) {
-    const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
-    const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
-    db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
-  }
-};
-
-exports.setLastJobExecution = jobKey => {
-  const key = buildKey(jobKey, null, 'lastJobExecution');
-  return db.set(key, Date.now()).write();
-};
-
-exports.getKnownListings = (jobKey, providerId) => {
-  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
-  return db.get(providerListingsKey).value() || [];
-};
-
-exports.getLastProviderExecution = (jobKey, providerId) => {
-  const key = buildKey(jobKey, providerId, 'lastProviderExecution');
-  return db.get(key).value() || 0;
-};
-
-exports.getLastJobExecution = jobKey => {
-  const key = buildKey(jobKey, null, 'lastJobExecution');
-  return db.get(key).value() || 0;
-};
-
-exports.getTotalNumberOfListings = (jobKey, providerId) => {
-  const key = buildKey(jobKey, providerId, 'foundListings');
-  return db.get(key).value() || 0;
-};
-
-exports.getForTesting = () => {
-  return db;
-};