Making Fredy an ESM project (#70)

Making Fredy an ESM project

lib/services/markdown.js

@@ -1,6 +1,6 @@
-const markdown = require('markdown').markdown;
-const fs = require('fs');
-
-exports.markdown2Html = function markdown2Html(filePath) {
+import markdown$0 from 'markdown';
+import fs from 'fs';
+const markdown = markdown$0.markdown;
+export function markdown2Html(filePath) {
   return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
-};
+}

lib/services/queryStringMutator.js

@@ -1,21 +1,9 @@
-const queryString = require('query-string');
-
-/**
- * for Fredy, it is important to sort search results by date, starting with the latest listing. if it is not sorted, we
- * might never actually find the newest results, no matter how many pages we crawl.
- * It has been written in the documentation, but obviously nobody reads docu theses days which is why it's been done
- * automagically now.
- *
- * @param _url actual provider url containing the searchParams
- * @param sortByDateParam param(s) indicating the correct sort order
- * @returns {`${string}?${string}`} correctly formatted url
- */
-module.exports = (_url, sortByDateParam) => {
+import queryString from 'query-string';
+export default (_url, sortByDateParam) => {
   //if no mutation is necessary, just return the original url
   if (sortByDateParam == null) {
     return _url;
   }
-
   const original = queryString.parseUrl(_url);
   const mutate = queryString.parse(sortByDateParam);
   return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;

lib/services/requestDriver.js

@@ -1,19 +1,15 @@
-const fetch = require('node-fetch');
-const config = require('../../conf/config.json');
-
-const { makeUrlResidential } = require('./scrapingAnt');
+import fetch from 'node-fetch';
+import { config } from '../utils.js';
+import { makeUrlResidential } from './scrapingAnt.js';
 //if ScrapingAnt got blocked, this http status is returned
 const BLOCKED_HTTP_STATUS = 423;
 const NOT_FOUND_HTTP_STATUS = 404;
 const MAX_RETRIES_SCRAPING_ANT = 10;
 const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
-
 function makeDriver(headers = {}) {
   let cookies = '';
-
   async function scrapingAntDriver(context, callback, retryCounter = 0) {
     const proxyType = config.scrapingAnt?.proxy || 'datacenter';
-
     try {
       const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
       const response = await fetch(url, {
@@ -22,12 +18,10 @@ function makeDriver(headers = {}) {
           cookie: cookies,
         },
       });
-
       const result = await response.text();
       if (cookies.length === 0) {
         cookies = response.headers.raw()['set-cookie'] || [];
       }
-
       callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
@@ -36,7 +30,6 @@ function makeDriver(headers = {}) {
         callback(null, []);
         return;
       }
-
       if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
         retryCounter++;
         console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
@@ -48,7 +41,6 @@ function makeDriver(headers = {}) {
       /* eslint-enable no-console */
     }
   }
-
   /**
    * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
    * everything != Immoscout as of writing this)
@@ -57,7 +49,6 @@ function makeDriver(headers = {}) {
     if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
       return scrapingAntDriver(context, callback);
     }
-
     try {
       const response = await fetch(context.url, {
         headers: {
@@ -65,7 +56,6 @@ function makeDriver(headers = {}) {
           Cookie: cookies,
         },
       });
-
       const result = await response.text();
       callback(null, result);
     } catch (exception) {
@@ -74,5 +64,4 @@ function makeDriver(headers = {}) {
     }
   };
 }
-
-module.exports = makeDriver;
+export default makeDriver;

lib/services/scraper.js

@@ -1,7 +1,6 @@
-const config = require('../../conf/config.json');
-const makeDriver = require('./requestDriver');
-const Xray = require('x-ray');
-
+import { config } from '../utils.js';
+import makeDriver from './requestDriver.js';
+import Xray from 'x-ray';
 class Scraper {
   constructor() {
     const filters = {
@@ -9,38 +8,29 @@ class Scraper {
       trim: this._trim,
       int: this._int,
     };
-
     const headers = {
       'User-Agent':
         'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
     };
-
     if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
       headers['x-api-key'] = config.scrapingAnt.apiKey;
     }
     const driver = makeDriver(headers);
-
     const xray = Xray({ filters });
     xray.driver(driver);
-
     this.xray = xray;
   }
-
   get x() {
     return this.xray;
   }
-
   _removeNewline(value) {
     return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
   }
-
   _trim(value) {
     return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
   }
-
   _int(value) {
     return typeof value === 'string' ? parseInt(value, 10) : value;
   }
 }
-
-module.exports = new Scraper().x;
+export default new Scraper().x;

lib/services/scrapingAnt.js

@@ -1,25 +1,19 @@
-const { metaInformation } = require('../provider/immoscout');
-//to better configure re-capture chose a random proxy each time we do a call
-const config = require('../../conf/config.json');
-
+import { metaInformation } from '../provider/immoscout.js';
+import { config } from '../utils.js';
 const isImmoscout = (id) => {
   return id.toLowerCase() === metaInformation.id;
 };
-
-exports.transformUrlForScrapingAnt = (url, id) => {
+export const transformUrlForScrapingAnt = (url, id) => {
   if (isImmoscout(id)) {
     //only do calls to scrapingAnt when dealing with Immoscout
     url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
   }
   return url;
 };
-
-exports.isScrapingAntApiKeySet = () => {
+export const isScrapingAntApiKeySet = () => {
   return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
 };
-
-exports.isImmoscout = isImmoscout;
-
-exports.makeUrlResidential = (url) => {
+export const makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');
 };
+export { isImmoscout };

lib/services/security/hash.js

@@ -1,3 +1,2 @@
-const crypto = require('crypto');
-
-exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
+import crypto from 'crypto';
+export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/similarity-check/SimilarityCacheEntry.js

@@ -1,27 +1,17 @@
-const stringSimilarity = require('string-similarity');
-
+import stringSimilarity from 'string-similarity';
 //if the score is higher than this, it will be considered a match
 const MAX_DICE_INDEX = 0.7;
-
-/**
- * The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
- *
- * @type {module.SimilarityCacheEntry}
- */
-module.exports = class SimilarityCacheEntry {
+export default (class SimilarityCacheEntry {
   constructor(time) {
     this.time = time;
     this.values = [];
   }
-
   setCacheEntry = (entry) => {
     this.values.push(entry);
   };
-
   getTime = () => {
     return this.time;
   };
-
   hasSimilarEntries = (value) => {
     if (this.values.length > 0) {
       for (let i = 0; i < this.values.length; i++) {
@@ -33,4 +23,4 @@ module.exports = class SimilarityCacheEntry {
     }
     return false;
   };
-};
+});

lib/services/similarity-check/similarityCache.js

@@ -1,63 +1,40 @@
-/**
- * each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
- * a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
- * the user won't see any duplicates
- *
- * The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
- *
- * @type {module.SimilarityCacheEntry|{}}
- */
-const SimilarityCacheEntry = require('./SimilarityCacheEntry');
-const config = require('../../../conf/config.json');
-
+import SimilarityCacheEntry from './SimilarityCacheEntry.js';
+import { config } from '../../utils.js';
 //5 minutes
 let retention = 5 * 60 * 1000;
-
 const intervalInMs = config.interval * 60 * 1000;
 //an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
 if (intervalInMs <= retention) {
   retention = Math.floor(intervalInMs / 2);
 }
-
 //jobid -> SimilarityCacheEntry
 const cache = {};
-
 let intervalId;
-
-exports.addCacheEntry = (jobId, value) => {
-  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
-  cache[jobId].setCacheEntry(value);
-};
-
-exports.hasSimilarEntries = (jobId, value) => {
-  if (cache[jobId] == null) {
-    return false;
-  }
-
-  return cache[jobId].hasSimilarEntries(value);
-};
-
 /**
  * cleanup
  */
 intervalId = setInterval(() => {
   const keysToBeRemoved = [];
   const now = Date.now();
-
   Object.keys(cache).forEach((key) => {
     if (cache[key].getTime() + retention < now) {
       keysToBeRemoved.push(key);
     }
   });
-
   if (keysToBeRemoved.length > 0) {
     keysToBeRemoved.forEach((key) => delete cache[key]);
   }
 }, 10000);
-
-/**
- * mostly used for tests
- */
-exports.stopCacheCleanup = () => {
+export const addCacheEntry = (jobId, value) => {
+  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
+  cache[jobId].setCacheEntry(value);
+};
+export const hasSimilarEntries = (jobId, value) => {
+  if (cache[jobId] == null) {
+    return false;
+  }
+  return cache[jobId].hasSimilarEntries(value);
+};
+export const stopCacheCleanup = () => {
   clearInterval(intervalId);
 };

lib/services/storage/LowDashAdapter.js

@@ -0,0 +1,8 @@
+import lodash from 'lodash';
+import { LowSync } from 'lowdb';
+export default class LowdashAdapter extends LowSync {
+  constructor(adapter) {
+    super(adapter);
+    this.chain = lodash.chain(this).get('data');
+  }
+}

lib/services/storage/jobStorage.js

@@ -1,28 +1,30 @@
-const path = require('path');
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
-const FileSync = require('lowdb/adapters/FileSync');
-const adapter = new FileSync(DB_PATH);
-const low = require('lowdb');
-const db = low(adapter);
-const { nanoid } = require('nanoid');
-const listingStorage = require('./listingsStorage');
+import { JSONFileSync } from 'lowdb/node';
+import { nanoid } from 'nanoid';
+import * as listingStorage from './listingsStorage.js';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-db.defaults({ jobs: [] }).write();
+const file = path.join(getDirName(), '../', 'db/jobs.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter);
 
-exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+db.read();
+
+db.data ||= { jobs: [] };
+
+export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
       ? null
-      : db
+      : db.chain
           .get('jobs')
           .find((job) => job.id === jobId)
           .value();
-
-  const jobs = db
+  const jobs = db.chain
     .get('jobs')
-    .value()
-    .filter((job) => job.id !== jobId);
-
+    .filter((job) => job.id !== jobId)
+    .value();
   jobs.push({
     id: jobId || nanoid(),
     //make sure to not overwrite the user id in case an admin changes the job
@@ -33,57 +35,55 @@ exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, no
     provider,
     notificationAdapter,
   });
-
-  db.set('jobs', jobs).write();
+  db.chain.set('jobs', jobs).value();
+  db.write();
 };
-
-exports.getJob = (jobId) => {
-  const job = db
+export const getJob = (jobId) => {
+  const job = db.chain
     .get('jobs')
     .find((job) => job.id === jobId)
     .value();
-
   if (job == null) {
     return null;
   }
-
   return {
     ...job,
     numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
   };
 };
-
-exports.setJobStatus = ({ jobId, status }) => {
-  db.get('jobs')
+export const setJobStatus = ({ jobId, status }) => {
+  db.chain
+    .get('jobs')
     .find((job) => job.id === jobId)
     .assign({ enabled: status })
-    .write();
+    .value();
+  db.write();
 };
-
-exports.removeJob = (jobId) => {
+export const removeJob = (jobId) => {
   listingStorage.removeListings(jobId);
-  db.get('jobs')
+  db.chain
+    .get('jobs')
     .remove((job) => job.id === jobId)
-    .write();
+    .value();
+  db.write();
 };
-
-exports.removeJobsByUserId = (userId) => {
-  db.get('jobs')
-    .value()
+export const removeJobsByUserId = (userId) => {
+  db.chain
+    .get('jobs')
     .filter((job) => job.userId === userId)
     .forEach((job) => listingStorage.removeListings(job.id));
-
-  db.get('jobs')
-    .remove((job) => job.userId === userId)
-    .write();
-};
-
-exports.getJobs = () => {
-  return db
+  db.chain
+    .get('jobs')
+    .remove((job) => job.userId === userId)
+    .value();
+  db.write();
+};
+export const getJobs = () => {
+  return db.chain
     .get('jobs')
-    .value()
     .map((job) => ({
       ...job,
       numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
-    }));
+    }))
+    .value();
 };

lib/services/storage/listingsStorage.js

@@ -1,10 +1,15 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
-const FileSync = require('lowdb/adapters/FileSync');
-const adapter = new FileSync(DB_PATH);
-const low = require('lowdb');
-const db = low(adapter);
+const file = path.join(getDirName(), '../', 'db/jobListingData.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter);
+
+db.read();
+
+db.data ||= {};
 
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
@@ -19,35 +24,31 @@ const buildKey = (jobKey, providerId, endpoint) => {
   }
   return key;
 };
-
-exports.getNumberOfAllKnownListings = (jobId) => {
-  const data = db.get(`${jobId}.providerData`).value() || {};
+export const getNumberOfAllKnownListings = (jobId) => {
+  const data = db.chain.get(`${jobId}.providerData`).value() || {};
   return Object.values(data)
     .map((values) => Object.keys(values).length)
     .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
 };
-
-exports.getListingProviderDataForAnalytics = (jobId) => {
+export const getListingProviderDataForAnalytics = (jobId) => {
   const key = buildKey(jobId, 'providerData');
-  return db.get(key).value() || {};
+  return db.chain.get(key).value() || {};
 };
-
-exports.getKnownListings = (jobId, providerId) => {
+export const getKnownListings = (jobId, providerId) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-  return db.get(providerListingsKey).value() || {};
+  return db.chain.get(providerListingsKey).value() || {};
 };
-
-exports.setKnownListings = (jobId, providerId, listings) => {
+export const setKnownListings = (jobId, providerId, listings) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-
-  return db.set(providerListingsKey, listings).write();
+  db.chain.set(providerListingsKey, listings).value();
+  return db.write();
 };
-
-exports.setLastJobExecution = (jobId) => {
+export const setLastJobExecution = (jobId) => {
   const key = buildKey(jobId, null, 'lastExecution');
-  return db.set(key, Date.now()).write();
+  db.chain.set(key, Date.now()).value();
+  return db.write();
 };
-
-exports.removeListings = (jobId) => {
-  db.unset(jobId).write();
+export const removeListings = (jobId) => {
+  db.chain.unset(jobId).value();
+  db.write();
 };

lib/services/storage/userStorage.js

@@ -1,14 +1,17 @@
-const path = require('path');
-const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
-const FileSync = require('lowdb/adapters/FileSync');
-const adapter = new FileSync(DB_PATH);
-const low = require('lowdb');
-const db = low(adapter);
-const hasher = require('../security/hash');
-const { nanoid } = require('nanoid');
-const jobStorage = require('./jobStorage');
+import { JSONFileSync } from 'lowdb/node';
+import { getDirName } from '../../utils.js';
+import * as hasher from '../security/hash.js';
+import { nanoid } from 'nanoid';
+import * as jobStorage from './jobStorage.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-db.defaults({
+const file = path.join(getDirName(), '../', 'db/users.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter);
+
+db.read();
+db.data ||= {
   user: [
     //you probably want to change the default password ;)
     {
@@ -20,11 +23,11 @@ db.defaults({
       isDemo: false,
     },
   ],
-}).write();
+};
 
-exports.getUsers = (withPassword) => {
+export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();
-  return db
+  return db.chain
     .get('user')
     .value()
     .map((user) => ({
@@ -34,13 +37,12 @@ exports.getUsers = (withPassword) => {
       numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
     }));
 };
-
-exports.getUser = (id) => {
+export const getUser = (id) => {
   const jobs = jobStorage.getJobs();
-  const user = db
+  const user = db.chain
     .get('user')
-    .value()
-    .find((user) => user.id === id);
+    .find((user) => user.id === id)
+    .value();
   if (user == null) {
     return null;
   }
@@ -49,13 +51,11 @@ exports.getUser = (id) => {
     numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
   };
 };
-
-exports.upsertUser = ({ username, password, userId, isAdmin }) => {
-  const user = db
+export const upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db.chain
     .get('user')
-    .value()
-    .filter((u) => u.id !== userId);
-
+    .filter((u) => u.id !== userId)
+    .value();
   user.push({
     id: userId || nanoid(),
     username,
@@ -63,21 +63,24 @@ exports.upsertUser = ({ username, password, userId, isAdmin }) => {
     password: hasher.hash(password),
     isAdmin,
   });
-
-  db.set('user', user).write();
+  db.chain.set('user', user).value();
+  db.write();
 };
-
-exports.setLastLoginToNow = ({ userId }) => {
-  db.get('user')
+export const setLastLoginToNow = ({ userId }) => {
+  db.chain
+    .get('user')
     .find((u) => u.id === userId)
     .assign({ lastLogin: Date.now() })
-    .write();
+    .value();
+  db.write();
 };
-
-exports.removeUser = (userId) => {
-  const user = db.get('user').value();
-  db.set(
-    'user',
-    user.filter((u) => u.id !== userId)
-  ).write();
+export const removeUser = (userId) => {
+  const user = db.chain.get('user').value();
+  db.chain
+    .set(
+      'user',
+      user.filter((u) => u.id !== userId)
+    )
+    .value();
+  db.write();
 };