sending tracking information (#116)

* Ability to send tracking information

lib/api/routes/generalSettingsRoute.js

@@ -1,5 +1,5 @@
 import restana from 'restana';
-import { config, getDirName } from '../../utils.js';
+import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
 import fs from 'fs';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
@@ -10,7 +10,9 @@ generalSettingsRouter.get('/', async (req, res) => {
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
   try {
-    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
+    await refreshConfig();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));

lib/api/routes/jobRouter.js

@@ -5,6 +5,7 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import * as immoscoutProvider from '../../provider/immoscout.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
+import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -25,8 +26,8 @@ jobRouter.get('/', async (req, res) => {
   res.send();
 });
 jobRouter.get('/processingTimes', async (req, res) => {
-  let scrapingAntData = null;
-  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+  let scrapingAntData = {};
+  if (isScrapingAntApiKeySet()) {
     try {
       const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
       scrapingAntData = await response.json();
@@ -38,6 +39,7 @@ jobRouter.get('/processingTimes', async (req, res) => {
     interval: config.interval,
     lastRun: config.lastRun || null,
     scrapingAntData,
+    error: scrapingAntData?.detail == null ? null : scrapingAntData?.detail
   };
   res.send();
 });

lib/defaultConfig.js

@@ -0,0 +1,8 @@
+export const DEFAULT_CONFIG = {
+            'interval': '60',
+            'port': 9998,
+            'scrapingAnt': {'apiKey': '', 'proxy': 'datacenter'},
+            'workingHours': {'from': '', 'to': ''},
+            'demoMode': false,
+            'analyticsEnabled': null
+    };

lib/services/scrapingAnt.js

@@ -22,7 +22,7 @@ export const transformUrlForScrapingAnt = (url, id) => {
   return url;
 };
 export const isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 8;
 };
 export const makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');

lib/services/tracking/Tracker.js

@@ -0,0 +1,42 @@
+import Mixpanel from 'mixpanel';
+import {getJobs} from '../storage/jobStorage.js';
+
+import {config} from '../../utils.js';
+
+export const track = function () {
+    //only send tracking information if the user allowed to do so.
+    if (config.analyticsEnabled) {
+
+        const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+
+        const activeProvider = new Set();
+        const activeAdapter = new Set();
+        const platform = process.platform;
+        const arch = process.arch;
+        const language = process.env.LANG || 'en';
+        const nodeVersion = process.version || 'N/A';
+
+        const jobs = getJobs();
+
+        if (jobs != null && jobs.length > 0) {
+            jobs.forEach(job => {
+                job.provider.forEach(provider => {
+                    activeProvider.add(provider.id);
+                });
+                job.notificationAdapter.forEach(adapter => {
+                    activeAdapter.add(adapter.id);
+                });
+            });
+
+            mixpanelTracker.track('fredy_tracking', {
+                adapter: Array.from(activeAdapter),
+                provider: Array.from(activeProvider),
+                isDemo: config.demoMode,
+                platform,
+                arch,
+                nodeVersion,
+                language
+            });
+        }
+    }
+};

lib/utils.js

@@ -2,6 +2,7 @@ import {dirname} from 'node:path';
 import {fileURLToPath} from 'node:url';
 import {readFile} from 'fs/promises';
 import {createHash} from 'crypto';
+import {DEFAULT_CONFIG} from './defaultConfig.js';
 
 function isOneOf(word, arr) {
     if (arr == null || arr.length === 0) {
@@ -52,7 +53,23 @@ function buildHash(...inputs) {
         .digest('hex');
 }
 
-const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+let config = {};
+export async function readConfigFromStorage(){
+    return JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+}
+
+export async function refreshConfig(){
+    try {
+        config = await readConfigFromStorage();
+        //backwards compatability...
+        config.analyticsEnabled ??= null;
+        config.demoMode ??= false;
+    } catch (error) {
+        config = {...DEFAULT_CONFIG};
+        console.error('Error reading config file', error);
+    }
+}
+await refreshConfig();
 
 export {isOneOf};
 export {nullOrEmpty};