UI (#15)

Adding new Admin UI. Updating Fredy to V3.0.0 as it has been a large rewrite. Thanks for all contributions and help on the way!

lib/FredyRuntime.js

@@ -1,10 +1,5 @@
 const { NoNewListingsError } = require('./errors');
-const {
-  setKnownListings,
-  getKnownListings,
-  setNumberOfTotalFoundProviderListings,
-  getForTesting,
-} = require('./services/store');
+const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
 
 const notify = require('./notification/notify');
 const xray = require('./services/scraper');
@@ -13,7 +8,7 @@ class FredyRuntime {
   /**
    *
    * @param providerConfig the config for the specific provider, we're going to query at the moment
-   * @param notificationConfig the config for all notifications (because all could be applied to a provider)
+   * @param notificationConfig the config for all notifications
    * @param providerId the id of the provider currently in use
    * @param jobKey key of the job that is currently running (from within the config)
    */
@@ -25,8 +20,6 @@ class FredyRuntime {
   }
 
   execute() {
-    if (!this._providerConfig.enabled) return Promise.resolve();
-
     return (
       Promise.resolve(this._providerConfig.url)
         //scraping the site and try finding new listings
@@ -37,8 +30,6 @@ class FredyRuntime {
         .then(this._filter.bind(this))
         //check if new listings available. if so proceed
         .then(this._findNew.bind(this))
-        //store update of number of found listings
-        .then(this._storeStats.bind(this))
         //store everything in db
         .then(this._save.bind(this))
         //notify the user using the configured notification adapter
@@ -52,24 +43,16 @@ class FredyRuntime {
     return new Promise((resolve, reject) => {
       let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
 
-      if (this._providerConfig.paginage) {
-        x = x.paginate(this._providerConfig.paginage);
-      }
-
       x((err, listings) => {
-        if (err) reject(err);
-        else {
+        if (err) {
+          reject(err);
+        } else {
           resolve(listings);
         }
       });
     });
   }
 
-  _storeStats(listings) {
-    setNumberOfTotalFoundProviderListings(this._jobKey, this._providerId, listings.length);
-    return Promise.resolve(listings);
-  }
-
   _normalize(listings) {
     return listings.map(this._providerConfig.normalize);
   }
@@ -79,7 +62,7 @@ class FredyRuntime {
   }
 
   _findNew(listings) {
-    const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId).indexOf(o.id) === -1);
+    const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
 
     if (newListings.length === 0) {
       throw new NoNewListingsError();
@@ -94,25 +77,17 @@ class FredyRuntime {
   }
 
   _save(newListings) {
-    setKnownListings(this._jobKey, this._providerId, [
-      ...getKnownListings(this._jobKey, this._providerId),
-      ...newListings.map((l) => l.id),
-    ]);
+    const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
+    newListings.forEach((listing) => {
+      currentListings[listing.id] = Date.now();
+    });
+    setKnownListings(this._jobKey, this._providerId, currentListings);
     return newListings;
   }
 
   _handleError(err) {
     if (err.name !== 'NoNewListingsError') console.error(err);
   }
-
-  /**
-   * for testing purposes only
-   * @returns {Store}
-   * @private
-   */
-  _getStore() {
-    return getForTesting();
-  }
 }
 
 module.exports = FredyRuntime;

lib/api/api.js

@@ -1,68 +1,42 @@
+const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
+const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
+const { analyticsRouter } = require('./routes/analyticsRouter');
+const { providerRouter } = require('./routes/providerRouter');
+const { loginRouter } = require('./routes/loginRoute');
+const config = require('../../conf/config.json');
+const { userRouter } = require('./routes/userRoute');
+const { jobRouter } = require('./routes/jobRouter');
 const bodyParser = require('body-parser');
-const config = require('../../conf/config');
-const { getLastJobExecution, getLastProviderExecution, getTotalNumberOfListings } = require('../services/store');
-const PORT = config.infoApiPort || 9998;
 const service = require('restana')();
-const enabled = config.infoApi == null ? false : config.infoApi;
+const files = require('serve-static');
+const path = require('path');
+
+const staticService = files(path.join(__dirname, '../../ui/public'));
+
+const PORT = config.port || 9998;
 
 service.use(bodyParser.json());
 
-service.get('/', async (req, res) => {
-  const result = {};
-  Object.keys(config.jobs).forEach((job) => {
-    result[job] = {
-      lastExecution: getLastJobExecution(job),
-      enabledProvider: Object.keys(config.jobs[job].provider)
-        .filter((providerKey) => config.jobs[job].provider[providerKey].enabled)
-        .map((providerKey) => {
-          return {
-            name: providerKey,
-            lastExecution: getLastProviderExecution(job, providerKey),
-            totalFindings: getTotalNumberOfListings(job, providerKey),
-          };
-        }),
-    };
-  });
-  res.body = result;
-  res.send();
-});
+service.use(cookieSession());
 
-service.get('/jobs/:name', async (req, res) => {
-  const { name: jobKey } = req.params;
-  if (Object.keys(config.jobs).indexOf(jobKey) === -1) {
-    console.error(`Cannot find job with name ${jobKey}. Available Jobs are [${Object.keys(config.jobs)}]`);
-    res.send(404);
-    return;
-  }
-  res.body = {
-    lastExecution: getLastJobExecution(jobKey),
-    enabledProvider: Object.keys(config.jobs[jobKey].provider)
-      .filter((providerKey) => config.jobs[jobKey].provider[providerKey].enabled)
-      .map((providerKey) => {
-        return {
-          name: providerKey,
-          url: config.jobs[jobKey].provider[providerKey].url,
-          lastExecution: getLastProviderExecution(jobKey, providerKey),
-          totalFindings: getTotalNumberOfListings(jobKey, providerKey),
-        };
-      }),
-  };
-  res.send();
-});
+service.use(staticService);
 
-service.get('/ping', function (req, res) {
-  res.body = {
-    pong: 'pong',
-  };
-  res.send();
-});
+service.use('/api/admin', authInterceptor());
+service.use('/api/jobs', authInterceptor());
+
+// /admin can only be accessed when user is having admin permissions
+service.use('/api/admin', adminInterceptor());
+
+service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
+service.use('/api/jobs/provider', providerRouter);
+service.use('/api/jobs/insights', analyticsRouter);
+service.use('/api/admin/users', userRouter);
+service.use('/api/jobs', jobRouter);
+
+service.use('/api/login', loginRouter);
 
 /* eslint-disable no-console */
-if (enabled) {
-  service.start(PORT).then(() => {
-    console.info(`Started API service on port ${PORT}`);
-  });
-} else {
-  console.info('Info Api is disabled.');
-}
+service.start(PORT).then(() => {
+  console.info(`Started API service on port ${PORT}`);
+});
 /* eslint-enable no-console */

lib/api/routes/analyticsRouter.js

@@ -0,0 +1,12 @@
+const service = require('restana')();
+const analyticsRouter = service.newRouter();
+const listingStorage = require('../../services/storage/listingsStorage');
+
+analyticsRouter.get('/:jobId', async (req, res) => {
+  const { jobId } = req.params;
+
+  res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
+  res.send();
+});
+
+exports.analyticsRouter = analyticsRouter;

lib/api/routes/jobRouter.js

@@ -0,0 +1,84 @@
+const service = require('restana')();
+const jobRouter = service.newRouter();
+const jobStorage = require('../../services/storage/jobStorage');
+const userStorage = require('../../services/storage/userStorage');
+const { isAdmin } = require('../security');
+
+function doesJobBelongsToUser(job, req) {
+  const userId = req.session.currentUser;
+  if (userId == null) {
+    return false;
+  }
+  const user = userStorage.getUser(userId);
+  if (user == null) {
+    return false;
+  }
+
+  return user.isAdmin || job.userId === job.userId;
+}
+
+jobRouter.get('/', async (req, res) => {
+  const isUserAdmin = isAdmin(req);
+
+  //show only the jobs which belongs to the user (or all of the user is an admin)
+  res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
+
+  res.send();
+});
+
+jobRouter.post('/', async (req, res) => {
+  const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
+  try {
+    jobStorage.upsertJob({
+      userId: req.session.currentUser,
+      jobId,
+      enabled,
+      name,
+      blacklist,
+      provider,
+      notificationAdapter,
+    });
+  } catch (error) {
+    res.send(new Error(error));
+    console.error(error);
+  }
+  res.send();
+});
+
+jobRouter.delete('', async (req, res) => {
+  const { jobId } = req.body;
+  try {
+    const job = jobStorage.getJob(jobId);
+    if (!doesJobBelongsToUser(job, req)) {
+      res.send(new Error('You are trying to remove a job that is not associated to your user'));
+    } else {
+      jobStorage.removeJob(jobId);
+    }
+  } catch (error) {
+    res.send(new Error(error));
+    console.error(error);
+  }
+  res.send();
+});
+
+jobRouter.put('/:jobId/status', async (req, res) => {
+  const { status } = req.body;
+  const { jobId } = req.params;
+  try {
+    const job = jobStorage.getJob(jobId);
+    if (!doesJobBelongsToUser(job, req)) {
+      res.send(new Error('You are trying change a job that is not associated to your user'));
+    } else {
+      jobStorage.setJobStatus({
+        jobId,
+        status,
+      });
+    }
+  } catch (error) {
+    res.send(new Error(error));
+    console.error(error);
+  }
+  res.send();
+});
+
+exports.jobRouter = jobRouter;

lib/api/routes/loginRoute.js

@@ -0,0 +1,47 @@
+const service = require('restana')();
+const loginRouter = service.newRouter();
+const userStorage = require('../../services/storage/userStorage');
+const hasher = require('../../services/security/hash');
+
+loginRouter.get('/user', async (req, res) => {
+  const currentUserId = req.session.currentUser;
+  const isAdmin = currentUserId == null ? false : userStorage.getUser(currentUserId).isAdmin;
+  if (currentUserId == null) {
+    res.body = {};
+  } else {
+    res.body = {
+      userId: currentUserId,
+      isAdmin,
+    };
+  }
+  res.send();
+});
+
+loginRouter.post('/', async (req, res) => {
+  const { username, password } = req.body;
+
+  const user = userStorage.getUsers(true).find((user) => user.username === username);
+
+  if (user == null) {
+    res.send(401);
+    return;
+  }
+
+  if (user.password === hasher.hash(password)) {
+    req.session.currentUser = user.id;
+    userStorage.setLastLoginToNow({ userId: user.id });
+    res.send(200);
+    return;
+  } else {
+    console.error(`User ${username} tried to login, but password was wrong.`);
+  }
+
+  res.send(401);
+});
+
+loginRouter.post('/logout', async (req, res) => {
+  req.session = null;
+  res.send(200);
+});
+
+exports.loginRouter = loginRouter;

lib/api/routes/notificationAdapterRouter.js

@@ -0,0 +1,54 @@
+const fs = require('fs');
+const service = require('restana')();
+const notificationAdapterRouter = service.newRouter();
+
+const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
+
+const notificationAdapter = notificationAdapterList.map((pro) => {
+  return require(`../../notification/adapter/${pro}`);
+});
+
+notificationAdapterRouter.post('/try', async (req, res) => {
+  const { id, fields } = req.body;
+  const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
+  if (adapter == null) {
+    res.send(404);
+  }
+  const notificationConfig = [];
+  const notificationObject = {};
+  Object.keys(fields).forEach((key) => {
+    notificationObject[key] = fields[key].value;
+  });
+  notificationConfig.push({
+    ...notificationObject,
+    enabled: true,
+    id,
+  });
+  Promise.all(
+    adapter.send({
+      serviceName: 'TestCall',
+      newListings: [
+        {
+          price: '42 €',
+          title: 'This is a test listing',
+          address: 'some address',
+          size: '666 2m',
+          link: 'https://www.orange-coding.net',
+        },
+      ],
+      notificationConfig,
+      jobKey: 'TestJob',
+    })
+  )
+    .then(() => res.send())
+    .catch((error) => {
+      res.send(new Error(error));
+    });
+});
+
+notificationAdapterRouter.get('/', async (req, res) => {
+  res.body = notificationAdapter.map((adapter) => adapter.config);
+  res.send();
+});
+
+exports.notificationAdapterRouter = notificationAdapterRouter;

lib/api/routes/providerRouter.js

@@ -0,0 +1,16 @@
+const fs = require('fs');
+const service = require('restana')();
+const providerRouter = service.newRouter();
+
+const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
+
+const provider = providerList.map((pro) => {
+  return require(`../../provider/${pro}`).metaInformation;
+});
+
+providerRouter.get('/', async (req, res) => {
+  res.body = provider;
+  res.send();
+});
+
+exports.providerRouter = providerRouter;

lib/api/routes/userRoute.js

@@ -0,0 +1,76 @@
+const service = require('restana')();
+const userRouter = service.newRouter();
+const userStorage = require('../../services/storage/userStorage');
+const jobStorage = require('../../services/storage/jobStorage');
+
+function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
+  return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
+}
+
+function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
+  return req.session.currentUser === userIdToBeRemoved;
+}
+
+const nullOrEmpty = (str) => str == null || str.length === 0;
+
+userRouter.get('/', async (req, res) => {
+  res.body = userStorage.getUsers(false);
+  res.send();
+});
+
+userRouter.get('/:userId', async (req, res) => {
+  const { userId } = req.params;
+  res.body = userStorage.getUser(userId);
+  res.send();
+});
+
+userRouter.delete('/', async (req, res) => {
+  const { userId } = req.body;
+  const allUser = userStorage.getUsers(false);
+
+  if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
+    res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
+    return;
+  }
+  if (checkIfUserToBeRemovedIsLoggedIn(userId, req)) {
+    res.send(new Error('You are trying to remove yourself. This is prohibited.'));
+    return;
+  }
+
+  //TODO: Remove also analytics
+  jobStorage.removeJobsByUserId(userId);
+  userStorage.removeUser(userId);
+
+  res.send();
+});
+
+userRouter.post('/', async (req, res) => {
+  const { username, password, password2, isAdmin, userId } = req.body;
+  if (password !== password2) {
+    res.send(new Error('Passwords does not match'));
+    return;
+  }
+  if (nullOrEmpty(username) || nullOrEmpty(password) || nullOrEmpty(password2)) {
+    res.send(new Error('Username and password are mandatory.'));
+    return;
+  }
+  const allUser = userStorage.getUsers(false);
+
+  if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
+    res.send(
+      new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
+    );
+    return;
+  }
+
+  userStorage.upsertUser({
+    userId,
+    username,
+    password,
+    isAdmin,
+  });
+
+  res.send();
+});
+
+exports.userRouter = userRouter;

lib/api/security.js

@@ -0,0 +1,53 @@
+const userStorage = require('../services/storage/userStorage');
+const cookieSession = require('cookie-session');
+const { nanoid } = require('nanoid');
+
+const unauthorized = (res) => {
+  return res.send(401);
+};
+
+const isUnauthorized = (req) => {
+  return req.session.currentUser == null;
+};
+
+const isAdmin = (req) => {
+  if (!isUnauthorized(req)) {
+    const user = userStorage.getUser(req.session.currentUser);
+    return user != null && user.isAdmin;
+  }
+  return false;
+};
+
+const authInterceptor = () => {
+  return (req, res, next) => {
+    if (isUnauthorized(req)) {
+      return unauthorized(res);
+    } else {
+      next();
+    }
+  };
+};
+
+const adminInterceptor = () => {
+  return (req, res, next) => {
+    if (!isAdmin(req)) {
+      return unauthorized(res);
+    } else {
+      next();
+    }
+  };
+};
+
+exports.cookieSession = (userId) => {
+  return cookieSession({
+    name: 'fredy-admin-session',
+    keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
+    userId,
+    maxAge: 24 * 60 * 60 * 1000, // 24 hours
+  });
+};
+
+exports.adminInterceptor = adminInterceptor;
+exports.authInterceptor = authInterceptor;
+exports.isUnauthorized = isUnauthorized;
+exports.isAdmin = isAdmin;

lib/notification/adapter/console.js

@@ -1,16 +1,21 @@
+const { markdown2Html } = require('../../services/markdown');
+
 /**
  * simply prints out the found data to the console
- * @param serviceName e.g immoscout
+ * @param serviceName e.g immowelt
  * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
  * @param jobKey name of the current job that is being executed
  */
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
-  const { enabled } = notificationConfig.console;
-  if (!enabled) {
-    return [Promise.resolve()];
-  }
+exports.send = ({ serviceName, newListings, jobKey }) => {
   /* eslint-disable no-console */
   return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
   /* eslint-enable no-console */
 };
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Console',
+  description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/console.md'),
+};

lib/notification/adapter/console.md

@@ -0,0 +1,4 @@
+### Console Adapter
+
+The console adapter prints everything found by Fredy into the console (not sending any notifications to you). This can be useful when you want to check if your search
+criteria meet the expectations.

lib/notification/adapter/mailJet.js

@@ -6,21 +6,20 @@ const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/t
 
 const Handlebars = require('handlebars');
 const emailTemplate = Handlebars.compile(template);
+const { markdown2Html } = require('../../services/markdown');
 
 /**
  * sends a new listing using MailJet
- * @param serviceName e.g immoscout
+ * @param serviceName e.g immowelt
  * @param newListings an array with newly found listings
  * @param notificationConfig config of this notification adapter
  * * @param jobKey name of the current job that is being executed
  * @returns {Promise<Chat.PostMessage.Response> | void}
  */
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
-  const { apiPublicKey, apiPrivateKey, enabled, receiver, from } = notificationConfig.mailJet;
-
-  if (!enabled) {
-    return [Promise.resolve()];
-  }
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
+    (adapter) => adapter.id === 'mailJet'
+  ).fields;
 
   return mailjet
     .connect(apiPublicKey, apiPrivateKey)
@@ -47,3 +46,33 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
       ],
     });
 };
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'MailJet',
+  description: 'MailJet is being used to send new listings via mail.',
+  readme: markdown2Html('lib/notification/adapter/mailJet.md'),
+  fields: {
+    apiPublicKey: {
+      type: 'text',
+      label: 'Public Api Key',
+      description: 'The public api key needed to access this service.',
+    },
+    apiPrivateKey: {
+      type: 'text',
+      label: 'Private Api Key',
+      description: 'The private api key needed to access this service.',
+    },
+    receiver: {
+      type: 'email',
+      label: 'Receiver Email',
+      description: 'The email address (single one) which Fredy is using to send notifications to.',
+    },
+    from: {
+      type: 'email',
+      label: 'Sender email',
+      description:
+        'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
+    },
+  },
+};

lib/notification/adapter/mailJet.md

@@ -0,0 +1,9 @@
+### MailJet Adapter
+
+ 
+   
+SendGrid is a free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
+   
+To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.     
+
+Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new template. For this new template, I recommend copying and pasting the one I have provided under `/lib/notification/emailTemplate/template.hbs`.

lib/notification/adapter/sendGrid.js

@@ -1,18 +1,16 @@
 const sgMail = require('@sendgrid/mail');
+const { markdown2Html } = require('../../services/markdown');
 
 /**
  * sends a new listing using SendGrid
- * @param serviceName e.g immoscout
+ * @param serviceName e.g immowelt
  * @param newListings an array with newly found listings
  * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
+ * @param jobKey name of the current job that is being executed
  * @returns {Promise<Chat.PostMessage.Response> | void}
  */
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
-  const { apiKey, enabled, receiver, from, templateId } = notificationConfig.sendGrid;
-  if (!enabled) {
-    return [Promise.resolve()];
-  }
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,
@@ -27,3 +25,33 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
   };
   return sgMail.send(msg);
 };
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'SendGrid',
+  description: 'SendGrid is being used to send new listings via mail.',
+  readme: markdown2Html('lib/notification/adapter/sendGrid.md'),
+  fields: {
+    apiKey: {
+      type: 'text',
+      label: 'Api Key',
+      description: 'The api key needed to access this service.',
+    },
+    receiver: {
+      type: 'email',
+      label: 'Receiver Email',
+      description: 'The email address (single one) which Fredy is using to send notifications to.',
+    },
+    from: {
+      type: 'email',
+      label: 'Sender Email',
+      description:
+        'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
+    },
+    templateId: {
+      type: 'text',
+      label: 'Template Id',
+      description: 'Sendgrid supports templates which Fredy is using to send out emails that looks awesome ;)',
+    },
+  },
+};

lib/notification/adapter/sendGrid.md

@@ -0,0 +1,7 @@
+### SendGrid Adapter
+
+ 
+To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from.  
+ 
+E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.     
+The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.

lib/notification/adapter/slack.js

@@ -1,19 +1,17 @@
 const Slack = require('slack');
 const msg = Slack.chat.postMessage;
+const { markdown2Html } = require('../../services/markdown');
 
 /**
  * sends a new listing to slack
- * @param serviceName e.g immoscout
+ * @param serviceName e.g immowelt
  * @param newListings an array with newly found listings
  * @param notificationConfig config of this notification adapter
  * * @param jobKey name of the current job that is being executed
  * @returns {Promise<Chat.PostMessage.Response> | void}
  */
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
-  const { token, channel, enabled } = notificationConfig.slack;
-  if (!enabled) {
-    return [Promise.resolve()];
-  }
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
   return newListings.map((payload) =>
     msg({
       token,
@@ -49,3 +47,22 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
     })
   );
 };
+
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Slack',
+  readme: markdown2Html('lib/notification/adapter/slack.md'),
+  description: 'Fredy will send new listings to the slack channel of your choice..',
+  fields: {
+    token: {
+      type: 'text',
+      label: 'Token',
+      description: 'The token needed to send notifications to slack.',
+    },
+    channel: {
+      type: 'channel',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/slack.md

@@ -0,0 +1,6 @@
+### Slack Adapter
+
+ 
+In order to use [Slack](https://slack.com), you need to create an account. When done, you need to create a new App in your workspace. Give it the permission `chat:write:bot` and `chat:write:user`.  
+ 
+Now you need to create a user token and a channel. Make sure the bot is installed to this channel. 

lib/notification/adapter/telegram.js

@@ -1,20 +1,17 @@
 const TelegramBot = require('tg-yarl');
-
+const { markdown2Html } = require('../../services/markdown');
 const opts = { parse_mode: 'Markdown' };
 
 /**
  * sends new listings to telegram
- * @param serviceName e.g immoscout
+ * @param serviceName e.g immowelt
  * @param newListings an array with newly found listings
  * @param notificationConfig config of this notification adapter
  * * @param jobKey name of the current job that is being executed
  * @returns {Promise<Void> | void}
  */
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
-  const { enabled, token, chatId } = notificationConfig.telegram;
-  if (!enabled) {
-    return [Promise.resolve()];
-  }
+exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
 
   const bot = new TelegramBot(token);
 
@@ -34,3 +31,27 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
 function shorten(str, len = 30) {
   return str.length > len ? str.substring(0, len) + '...' : str;
 }
+
+/**
+ * exported config is being used in the frontend to generate the fields
+ * incoming values will be the keys (and values) of the fields
+ *
+ */
+exports.config = {
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'Telegram',
+  readme: markdown2Html('lib/notification/adapter/telegram.md'),
+  description: 'Fredy will send new listings to your mobile, using Telegram.',
+  fields: {
+    token: {
+      type: 'text',
+      label: 'Token',
+      description: 'The token needed to access this service.',
+    },
+    chatId: {
+      type: 'chatId',
+      label: 'Chat Id',
+      description: 'The chat id to send messages to you.',
+    },
+  },
+};

lib/notification/adapter/telegram.md

@@ -0,0 +1,13 @@
+### Telegram Adapter
+
+ 
+For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
+  
+A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.        
+After the user has send a message to your bot the first time, you can gather the chatId like this:   
+
+```  
+curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates  
+```  
+  
+A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)

lib/notification/notify.js

@@ -2,13 +2,20 @@ const fs = require('fs');
 const path = './adapter';
 
 /** Read every integration existing in ./adapter **/
-const adapter = fs.readdirSync('./lib/notification/adapter').map((integPath) => require(`${path}/${integPath}`));
+const adapter = fs
+  .readdirSync('./lib/notification/adapter')
+  .filter((file) => file.endsWith('.js'))
+  .map((integPath) => require(`${path}/${integPath}`));
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
 
-exports.send = (serviceName, payload, notificationConfig, jobKey) => {
-  //this is not being used in tests, therefor adapter are always set
-  return adapter.map((a) => a.send(serviceName, payload, notificationConfig, jobKey));
+exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
+  //this is not being used in tests, therefore adapter are always set
+  return adapter
+    .filter((notificationAdapter) => {
+      return notificationConfig.find((config) => config.id === notificationAdapter.config.id);
+    })
+    .map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
 };

lib/provider/einsAImmobilien.js

@@ -20,7 +20,6 @@ function applyBlacklist(o) {
 }
 
 const config = {
-  enabled: null,
   url: null,
   crawlContainer: '.tabelle',
   crawlFields: {
@@ -29,20 +28,23 @@ const config = {
     size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
     rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
     title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim'
+    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
   paginate: '.pagination_blocks div:last a@href',
   normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };
 
 exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };
 
-//must match the id of the source given in the config!
-exports.id = () => 'einsAImmobilien';
+exports.metaInformation = {
+  name: '1a Immobilien',
+  baseUrl: 'https://www.1a-immobilienmarkt.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
 
 exports.config = config;

lib/provider/immonet.js

@@ -20,7 +20,6 @@ function applyBlacklist(o) {
 }
 
 const config = {
-  enabled: null,
   url: null,
   crawlContainer: '#result-list-stage .item',
   crawlFields: {
@@ -29,20 +28,23 @@ const config = {
     size: 'div[id*="selArea_"] | trim',
     title: '.item a img@title',
     link: 'a[id*="lnkImgToDetails_"]@href',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim'
+    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
   },
   paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };
 
 exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };
 
-//must match the id of the source given in the config!
-exports.id = () => 'immonet';
+exports.metaInformation = {
+  name: 'Immonet',
+  baseUrl: 'https://www.immonet.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
 
 exports.config = config;

lib/provider/immoscout.js

@@ -1,42 +0,0 @@
-const utils = require('../utils');
-
-let appliedBlackList = [];
-
-function normalize(o) {
-  const title = o.title.replace('NEU', '');
-  const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
-
-  return Object.assign(o, { title, address });
-}
-
-function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
-}
-
-const config = {
-  enabled: null,
-  url: null,
-  crawlContainer: '#resultListItems li.result-list__listing',
-  crawlFields: {
-    id: '.result-list-entry@data-obid | int',
-    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
-    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
-    link: '.result-list-entry .result-list-entry__brand-title-container@href',
-    address: '.result-list-entry .result-list-entry__map-link'
-  },
-  paginate: '#pager .align-right a@href',
-  normalize: normalize,
-  filter: applyBlacklist
-};
-
-exports.init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
-};
-
-//must match the id of the source given in the config!
-exports.id = () => 'immoscout';
-
-exports.config = config;

lib/provider/immowelt.js

@@ -17,7 +17,6 @@ function applyBlacklist(o) {
 }
 
 const config = {
-  enabled: null,
   url: null,
   crawlContainer: '.immoliste .js-object.listitem_wrap ',
   crawlFields: {
@@ -26,20 +25,22 @@ const config = {
     size: '.js-object.listitem_wrap .hardfacts_3 div:nth-child(2)| removeNewline | trim',
     title: '.listcontent.clear h2',
     link: 'a@href',
-    address: '.listcontent .details .listlocation| removeNewline | trim'
+    address: '.listcontent .details .listlocation| removeNewline | trim',
   },
   paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };
 
 exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
+};
+exports.metaInformation = {
+  name: 'Immowelt',
+  baseUrl: 'https://www.immowelt.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
 };
 
-//must match the id of the source given in the config!
-exports.id = () => 'immowelt';
-
 exports.config = config;

lib/provider/kalaydo.js

@@ -1,56 +0,0 @@
-const utils = require('../utils');
-
-let appliedBlackList = [];
-let appliedBlacklistedDistricts = [];
-
-function normalize(o) {
-  const id = o.id
-    .split('/')
-    .filter(Boolean)
-    .reverse()[0];
-  const price = o.price == null ? 'unknown' : o.price.trim().replace('Preis', '');
-  let size = o.size == null ? 'unknown' : o.size.replace('Wohnfläche: ', '').replace('ca. ', '');
-  size += ' / ' + o.rooms;
-  const address = '---';
-
-  return Object.assign(o, { id, price, size, address });
-}
-
-function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
-  const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
-
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
-}
-
-const config = {
-  enabled: null,
-  url: null,
-  crawlContainer: '#resultList .resultitem-content-container',
-  crawlFields: {
-    id: '.resultitem-content-container a@href',
-    price: '.description .rent | removeNewline | trim',
-    title: '.resultitem-content-container a@title',
-    link: '.resultitem-content-container a@href',
-    rooms: '.resultitem-content-container .no-of-rooms | removeNewline  | trim',
-    size: '.resultitem-content-container .living-area | removeNewline | trim'
-  },
-  paginate: '.markt_pagination_pageLinkNext .markt_pagination_link@href',
-  normalize: normalize,
-  filter: applyBlacklist
-};
-
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
-  appliedBlacklistedDistricts = blacklistedDistricts;
-};
-
-//must match the id of the source given in the config!
-exports.id = () => 'kalaydo';
-
-exports.config = config;

lib/provider/kleinanzeigen.js

@@ -13,13 +13,12 @@ function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   const isBlacklistedDistrict =
-      appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
+    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
 
   return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-  enabled: null,
   url: null,
   crawlContainer: '#srchrslt-adtable .ad-listitem',
   crawlFields: {
@@ -29,22 +28,24 @@ const config = {
     title: '.aditem-main .text-module-begin a | removeNewline | trim',
     link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
     description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
-    address: '.aditem-details | trim | removeNewline'
+    address: '.aditem-details | trim | removeNewline',
   },
   paginate: '#srchrslt-pagination .pagination-next@href',
   normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
+};
+
+exports.metaInformation = {
+  name: 'Ebay Kleinanzeigen',
+  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
 };
 
 exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
-    config.enabled = sourceConfig.enabled;
-    config.url = sourceConfig.url;
-    appliedBlacklistedDistricts = blacklistedDistricts;
-    appliedBlackList = blacklist;
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlacklistedDistricts = blacklistedDistricts || [];
+  appliedBlackList = blacklist || [];
 };
 
-//must match the id of the source given in the config!
-exports.id = () => 'kleinanzeigen';
-
 exports.config = config;
-

lib/provider/neubauKompass.js

@@ -11,7 +11,6 @@ function applyBlacklist(o) {
 }
 
 const config = {
-  enabled: null,
   url: null,
   crawlContainer: '.nbk-container >div article',
   crawlFields: {
@@ -28,10 +27,13 @@ const config = {
 exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };
 
-//must match the id of the source given in the config!
-exports.id = () => 'neubauKompass';
+exports.metaInformation = {
+  name: 'Neubau Kompass',
+  baseUrl: 'https://www.neubaukompass.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
 
 exports.config = config;

lib/provider/wgGesucht.js

@@ -14,7 +14,6 @@ function applyBlacklist(o) {
 }
 
 const config = {
-  enabled: null,
   url: null,
   crawlContainer: '#main_column .wgg_card',
   crawlFields: {
@@ -23,20 +22,23 @@ const config = {
     price: '.middle .col-xs-3 |removeNewline |trim',
     size: '.middle .text-right |removeNewline |trim',
     title: '.truncate_title a |removeNewline |trim',
-    link: '.truncate_title a@href'
+    link: '.truncate_title a@href',
   },
   paginate: '.pagination-sm:first a:last@href',
   normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };
 
 exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };
 
-//must match the id of the source given in the config!
-exports.id = () => 'wgGesucht';
+exports.metaInformation = {
+  name: 'Wg gesucht',
+  baseUrl: 'https://www.wg-gesucht.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};
 
 exports.config = config;

lib/services/markdown.js

@@ -0,0 +1,6 @@
+const markdown = require('markdown').markdown;
+const fs = require('fs');
+
+exports.markdown2Html = function markdown2Html(filePath) {
+  return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
+};

lib/services/scraper.js

@@ -6,14 +6,16 @@ class Scraper {
     const filters = {
       removeNewline: this._removeNewline,
       trim: this._trim,
-      int: this._int
+      int: this._int,
     };
 
     const driver = makeDriver({
       headers: {
         'User-Agent':
-          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36'
-      }
+          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
+        cookie:
+          'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
+      },
     });
 
     const xray = Xray({ filters });

lib/services/security/hash.js

@@ -0,0 +1,3 @@
+const crypto = require('crypto');
+
+exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/storage/jobStorage.js

@@ -0,0 +1,83 @@
+const path = require('path');
+const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
+const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new FileSync(DB_PATH);
+const low = require('lowdb');
+const db = low(adapter);
+const { nanoid } = require('nanoid');
+const listingStorage = require('./listingsStorage');
+
+db.defaults({ jobs: [] }).write();
+
+exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+  const currentJob =
+    jobId == null
+      ? null
+      : db
+          .get('jobs')
+          .find((job) => job.id === jobId)
+          .value();
+
+  const jobs = db
+    .get('jobs')
+    .value()
+    .filter((job) => job.id !== jobId);
+
+  jobs.push({
+    id: jobId || nanoid(),
+    //make sure to not overwrite the user id in case an admin changes the job
+    userId: currentJob == null ? userId : currentJob.userId,
+    enabled,
+    name,
+    blacklist,
+    provider,
+    notificationAdapter,
+  });
+
+  db.set('jobs', jobs).write();
+};
+
+exports.getJob = (jobId) => {
+  const job = db
+    .get('jobs')
+    .find((job) => job.id === jobId)
+    .value();
+
+  if (job == null) {
+    return null;
+  }
+
+  return {
+    ...job,
+    numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
+  };
+};
+
+exports.setJobStatus = ({ jobId, status }) => {
+  db.get('jobs')
+    .find((job) => job.id === jobId)
+    .assign({ enabled: status })
+    .write();
+};
+
+exports.removeJob = (jobId) => {
+  db.get('jobs')
+    .remove((job) => job.id === jobId)
+    .write();
+};
+
+exports.removeJobsByUserId = (userId) => {
+  db.get('jobs')
+    .remove((job) => job.userId === userId)
+    .write();
+};
+
+exports.getJobs = () => {
+  return db
+    .get('jobs')
+    .value()
+    .map((job) => ({
+      ...job,
+      numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
+    }));
+};

lib/services/storage/listingsStorage.js

@@ -0,0 +1,49 @@
+const path = require('path');
+
+const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
+const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new FileSync(DB_PATH);
+const low = require('lowdb');
+const db = low(adapter);
+
+const buildKey = (jobKey, providerId, endpoint) => {
+  let key = `${jobKey}`;
+  if (jobKey == null && endpoint == null) {
+    return key;
+  }
+  if (providerId != null) {
+    key += `.${providerId}`;
+  }
+  if (endpoint != null) {
+    key += `.${endpoint}`;
+  }
+  return key;
+};
+
+exports.getNumberOfAllKnownListings = (jobId) => {
+  const data = db.get(`${jobId}.providerData`).value() || {};
+  return Object.values(data)
+    .map((values) => Object.keys(values).length)
+    .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
+};
+
+exports.getListingProviderDataForAnalytics = (jobId) => {
+  const key = buildKey(jobId, 'providerData');
+  return db.get(key).value() || {};
+};
+
+exports.getKnownListings = (jobId, providerId) => {
+  const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
+  return db.get(providerListingsKey).value() || {};
+};
+
+exports.setKnownListings = (jobId, providerId, listings) => {
+  const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
+
+  return db.set(providerListingsKey, listings).write();
+};
+
+exports.setLastJobExecution = (jobId) => {
+  const key = buildKey(jobId, null, 'lastExecution');
+  return db.set(key, Date.now()).write();
+};

lib/services/storage/userStorage.js

@@ -0,0 +1,83 @@
+const path = require('path');
+const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
+const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new FileSync(DB_PATH);
+const low = require('lowdb');
+const db = low(adapter);
+const hasher = require('../security/hash');
+const { nanoid } = require('nanoid');
+const jobStorage = require('./jobStorage');
+
+db.defaults({
+  user: [
+    //you probably want to change the default password ;)
+    {
+      id: nanoid(),
+      lastLogin: Date.now(),
+      username: 'admin',
+      password: hasher.hash('admin'),
+      isAdmin: true,
+      isDemo: false,
+    },
+  ],
+}).write();
+
+exports.getUsers = (withPassword) => {
+  const jobs = jobStorage.getJobs();
+  return db
+    .get('user')
+    .value()
+    .map((user) => ({
+      //we dont want the password in the frontend, even tho it's hashed
+      ...user,
+      password: withPassword ? user.password : null,
+      numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
+    }));
+};
+
+exports.getUser = (id) => {
+  const jobs = jobStorage.getJobs();
+  const user = db
+    .get('user')
+    .value()
+    .find((user) => user.id === id);
+  if (user == null) {
+    return null;
+  }
+  return {
+    ...user,
+    numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
+  };
+};
+
+exports.upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db
+    .get('user')
+    .value()
+    .filter((u) => u.id !== userId);
+
+  user.push({
+    id: userId || nanoid(),
+    username,
+    lastLogin: user.lastLogin,
+    password: hasher.hash(password),
+    isAdmin,
+  });
+
+  db.set('user', user).write();
+};
+
+exports.setLastLoginToNow = ({ userId }) => {
+  db.get('user')
+    .find((u) => u.id === userId)
+    .assign({ lastLogin: Date.now() })
+    .write();
+};
+
+exports.removeUser = (userId) => {
+  const user = db.get('user').value();
+  db.set(
+    'user',
+    user.filter((u) => u.id !== userId)
+  ).write();
+};

lib/services/store.js

@@ -1,85 +0,0 @@
-const path = require('path');
-const DB_PATH = path.dirname(require.main.filename) + '/conf/store.json';
-
-const FileAsync = require('lowdb/adapters/FileAsync');
-const adapter = new FileAsync(DB_PATH);
-const low = require('lowdb');
-
-const lowdb = low(adapter);
-
-let db = null;
-
-const buildKey = (jobKey, providerId, endpoint) => {
-  let key = `${jobKey}`;
-  if (jobKey == null && endpoint == null) {
-    return key;
-  }
-  if (providerId != null) {
-    key += `.${providerId}`;
-  }
-  if (endpoint != null) {
-    key += `.${endpoint}`;
-  }
-  return key;
-};
-
-exports.init = () => {
-  return new Promise(resolve => {
-    //warmup
-    lowdb.then(database => {
-      db = database;
-      /* eslint-disable no-console */
-      console.info('Warming up database successful');
-      /* eslint-enable no-console */
-      resolve();
-    });
-  });
-};
-
-exports.setKnownListings = (jobKey, providerId, listings) => {
-  if (!Array.isArray(listings)) throw Error('Not a valid array');
-  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
-  const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
-
-  return db
-    .set(providerListingsKey, listings)
-    .set(providerLastScrapeKey, Date.now())
-    .write();
-};
-
-exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
-  if (numberOfNewListings > 0) {
-    const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
-    const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
-    db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
-  }
-};
-
-exports.setLastJobExecution = jobKey => {
-  const key = buildKey(jobKey, null, 'lastJobExecution');
-  return db.set(key, Date.now()).write();
-};
-
-exports.getKnownListings = (jobKey, providerId) => {
-  const providerListingsKey = buildKey(jobKey, providerId, 'listings');
-  return db.get(providerListingsKey).value() || [];
-};
-
-exports.getLastProviderExecution = (jobKey, providerId) => {
-  const key = buildKey(jobKey, providerId, 'lastProviderExecution');
-  return db.get(key).value() || 0;
-};
-
-exports.getLastJobExecution = jobKey => {
-  const key = buildKey(jobKey, null, 'lastJobExecution');
-  return db.get(key).value() || 0;
-};
-
-exports.getTotalNumberOfListings = (jobKey, providerId) => {
-  const key = buildKey(jobKey, providerId, 'foundListings');
-  return db.get(key).value() || 0;
-};
-
-exports.getForTesting = () => {
-  return db;
-};