version update

* Adding Demo Mode to Fredy

.github/workflows/stales.yml

@@ -0,0 +1,21 @@
+name: "Close stale issues and PRs"
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          days-before-stale: 30
+          days-before-close: 7
+          stale-issue-message: "This issue has been automatically marked as stale due to inactivity."
+          stale-pr-message: "This PR has been automatically marked as stale due to inactivity."
+          close-issue-message: "Closing this issue due to prolonged inactivity."
+          close-pr-message: "Closing this PR due to prolonged inactivity."
+          exempt-issue-labels: "keep-open"
+          exempt-pr-labels: "keep-open"
+          only: "pulls"

.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

.travis.yml

@@ -1,2 +0,0 @@
-sudo: false
-language: node_js

CONTRIBUTING.md

@@ -106,9 +106,7 @@ exports.config = {
 ```
 
 #### Running Tests
-If you've written a new provider you are an awesome person. You know it and I do. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
+If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
-
-To write tests for provider, you need to use Node 8 as the tests are using `async / await`
 
 #### Codestyle
 I'm using Eslint to maintain quote style and quality. Do not skip it...

README.md

@@ -15,6 +15,9 @@ If you like my work, consider becoming a sponsor. I'm not expecting anybody to p
 
 _Fredy_ is supported by JetBrains under Open Source Support Program
 
+## Demo
+If you want to try out _Fredy_, you can access the demo version [here](https://fredy.orange-coding.net) 🤘
+
 ## Usage  
 
 - Make sure to use Node.js 20 or above
@@ -86,14 +89,12 @@ The rest will be handled by _Fredy_. Keep in mind, the support is experimental.
 
 If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
 
-### 👐 Contributing
+# Analytics
-Thanks to all the people who already contributed!
+Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 
-
+Before you freak out, let me explain...         
-<a href="https://github.com/orangecoding/fredy/graphs/contributors">
+If you agree, Fredy will send a ping to my Mixpanel project each time it runs.   
-  <img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
+The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.</p>
-</a>
+**Thanks**🤘
-
-See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
 # Docker   
 Use the Dockerfile in this repository to build an image.  
@@ -114,6 +115,15 @@ Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+### 👐 Contributing
+Thanks to all the people who already contributed!
+
+<a href="https://github.com/orangecoding/fredy/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
+</a>
+
+See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
+
 ## Logs  
 
 You can browse the logs with  `docker logs fredy -f`.

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}

index.js

@@ -6,6 +6,9 @@ import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyRuntime from './lib/FredyRuntime.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import './lib/api/api.js';
+import {track} from './lib/services/tracking/Tracker.js';
+import {handleDemoUser} from './lib/services/storage/userStorage.js';
+import {cleanupDemoAtMidnight} from './lib/services/demoCleanup.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
@@ -16,15 +19,23 @@ const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
 const INTERVAL = config.interval * 60 * 1000;
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+if(config.demoMode){
+    console.info('Running in demo mode');
+    cleanupDemoAtMidnight();
+}
 /* eslint-enable no-console */
 const fetchedProvider = await Promise.all(
   provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
 );
 
+handleDemoUser();
+
 setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+        if(!config.demoMode) {
             if (isDuringWorkingHoursOrNotSet) {
+                track();
                 config.lastRun = Date.now();
                 jobStorage
                     .getJobs()
@@ -44,6 +55,7 @@ setInterval(
                 console.debug('Working hours set. Skipping as outside of working hours.');
                 /* eslint-enable no-console */
             }
+        }
     return exec;
   })(),
   INTERVAL

lib/api/api.js

@@ -12,6 +12,7 @@ import restana from 'restana';
 import files from 'serve-static';
 import path from 'path';
 import { getDirName } from '../utils.js';
+import {demoRouter} from './routes/demoRouter.js';
 const service = restana();
 const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
@@ -30,6 +31,9 @@ service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
 service.use('/api/jobs', jobRouter);
 service.use('/api/login', loginRouter);
+//this route is unsecured intentionally as it is being queried from the login page
+service.use('/api/demo', demoRouter);
+
 /* eslint-disable no-console */
 service.start(PORT).then(() => {
   console.info(`Started API service on port ${PORT}`);

lib/api/routes/demoRouter.js

@@ -0,0 +1,11 @@
+import restana from 'restana';
+import {config} from '../../utils.js';
+const service = restana();
+const demoRouter = service.newRouter();
+
+demoRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, {demoMode: config.demoMode});
+  res.send();
+});
+
+export { demoRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,6 +1,7 @@
 import restana from 'restana';
-import { config, getDirName } from '../../utils.js';
+import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
 import fs from 'fs';
+import {handleDemoUser} from '../../services/storage/userStorage.js';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
 generalSettingsRouter.get('/', async (req, res) => {
@@ -10,7 +11,14 @@ generalSettingsRouter.get('/', async (req, res) => {
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
   try {
-    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
+    if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+      return;
+    }
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
+    await refreshConfig();
+    handleDemoUser();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));

lib/api/routes/jobRouter.js

@@ -5,6 +5,8 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import * as immoscoutProvider from '../../provider/immoscout.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
+import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
+import {trackDemoJobCreated} from '../../services/tracking/Tracker.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -25,8 +27,8 @@ jobRouter.get('/', async (req, res) => {
   res.send();
 });
 jobRouter.get('/processingTimes', async (req, res) => {
-  let scrapingAntData = null;
+  let scrapingAntData = {};
-  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+  if (isScrapingAntApiKeySet()) {
     try {
       const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
       scrapingAntData = await response.json();
@@ -38,6 +40,7 @@ jobRouter.get('/processingTimes', async (req, res) => {
     interval: config.interval,
     lastRun: config.lastRun || null,
     scrapingAntData,
+    error: scrapingAntData?.detail == null ? null : scrapingAntData?.detail
   };
   res.send();
 });
@@ -66,6 +69,11 @@ jobRouter.post('/', async (req, res) => {
     res.send(new Error(error));
     console.error(error);
   }
+  trackDemoJobCreated({
+    name,
+    provider,
+    adapter: notificationAdapter
+  });
   res.send();
 });
 jobRouter.delete('', async (req, res) => {

lib/api/routes/loginRoute.js

@@ -1,6 +1,8 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as hasher from '../../services/security/hash.js';
+import {config} from '../../utils.js';
+import {trackDemoAccessed} from '../../services/tracking/Tracker.js';
 const service = restana();
 const loginRouter = service.newRouter();
 loginRouter.get('/user', async (req, res) => {
@@ -24,6 +26,11 @@ loginRouter.post('/', async (req, res) => {
     return;
   }
   if (user.password === hasher.hash(password)) {
+
+    if(config.demoMode){
+      trackDemoAccessed();
+    }
+
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
     res.send(200);

lib/api/routes/userRoute.js

@@ -1,6 +1,7 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as jobStorage from '../../services/storage/jobStorage.js';
+import {config} from '../../utils.js';
 const service = restana();
 const userRouter = service.newRouter();
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
@@ -20,6 +21,11 @@ userRouter.get('/:userId', async (req, res) => {
   res.send();
 });
 userRouter.delete('/', async (req, res) => {
+  if(config.demoMode){
+    res.send(new Error('In demo mode, it is not allowed to remove user.'));
+    return;
+  }
+
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
@@ -36,6 +42,12 @@ userRouter.delete('/', async (req, res) => {
   res.send();
 });
 userRouter.post('/', async (req, res) => {
+
+  if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change or add user.'));
+      return;
+  }
+
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
     res.send(new Error('Passwords does not match'));

lib/defaultConfig.js

@@ -0,0 +1,8 @@
+export const DEFAULT_CONFIG = {
+            'interval': '60',
+            'port': 9998,
+            'scrapingAnt': {'apiKey': '', 'proxy': 'datacenter'},
+            'workingHours': {'from': '', 'to': ''},
+            'demoMode': false,
+            'analyticsEnabled': null
+    };

lib/services/demoCleanup.js

@@ -0,0 +1,29 @@
+import { setInterval } from 'node:timers';
+import {removeJobsByUserName} from './storage/jobStorage.js';
+import {config} from '../utils.js';
+
+/**
+ * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
+ */
+export function cleanupDemoAtMidnight() {
+    const now = new Date();
+    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+        - now.getUTCMinutes() * 60 * 1000
+        - now.getUTCSeconds() * 1000
+        - now.getUTCMilliseconds();
+
+    setTimeout(() => {
+        cleanup();
+
+        setInterval(() => {
+            cleanup();
+        }, 24 * 60 * 60 * 1000);
+
+    }, millisUntilMidnightUTC);
+}
+
+function cleanup(){
+    if(config.demoMode){
+        removeJobsByUserName('demo');
+    }
+}

lib/services/scrapingAnt.js

@@ -22,7 +22,7 @@ export const transformUrlForScrapingAnt = (url, id) => {
   return url;
 };
 export const isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 8;
 };
 export const makeUrlResidential = (url) => {
   return url.replace('datacenter', 'residential');

lib/services/storage/jobStorage.js

@@ -77,6 +77,17 @@ export const removeJobsByUserId = (userId) => {
     .value();
   db.write();
 };
+export const removeJobsByUserName = (userName) => {
+  db.chain
+      .get('jobs')
+      .filter((job) => job.username === userName)
+      .forEach((job) => listingStorage.removeListings(job.id));
+  db.chain
+      .get('jobs')
+      .remove((job) => job.username === userName)
+      .value();
+  db.write();
+};
 export const getJobs = () => {
   return db.chain
     .get('jobs')

lib/services/storage/userStorage.js

@@ -1,5 +1,5 @@
 import { JSONFileSync } from 'lowdb/node';
-import { getDirName } from '../../utils.js';
+import {config, getDirName} from '../../utils.js';
 import * as hasher from '../security/hash.js';
 import { nanoid } from 'nanoid';
 import * as jobStorage from './jobStorage.js';
@@ -16,6 +16,13 @@ const defaultData = {
             password: hasher.hash('admin'),
             isAdmin: true,
         },
+        {
+            id: nanoid(),
+            lastLogin: Date.now(),
+            username: 'demo',
+            password: hasher.hash('demo'),
+            isAdmin: true,
+        },
     ],
 };
 
@@ -84,3 +91,29 @@ export const removeUser = (userId) => {
     .value();
   db.write();
 };
+
+export const handleDemoUser = () => {
+    if(!config.demoMode){
+        const user = db.chain.get('user').value();
+        db.chain.get('user').value();
+        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+        db.write();
+    }else {
+        const demoUser = db.chain
+            .get('user')
+            .filter((u) => u.username === 'demo')
+            .value();
+        if (demoUser == null || demoUser.length === 0) {
+            db.chain.get('user')
+                .value()
+                .push({
+                    id: nanoid(),
+                    username: 'demo',
+                    password: hasher.hash('demo'),
+                    isAdmin: true,
+                });
+            db.write();
+        }
+    }
+};
+

lib/services/tracking/Tracker.js

@@ -0,0 +1,75 @@
+import Mixpanel from 'mixpanel';
+import { getJobs } from '../storage/jobStorage.js';
+import { getUniqueId } from './uniqueId.js';
+import { config, inDevMode } from '../../utils.js';
+import os from 'os';
+
+const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+
+const distinct_id = getUniqueId() || 'N/A';
+
+export const track = function () {
+  //only send tracking information if the user allowed to do so.
+  if (config.analyticsEnabled && !inDevMode()) {
+    const activeProvider = new Set();
+    const activeAdapter = new Set();
+
+    const jobs = getJobs();
+
+    if (jobs != null && jobs.length > 0) {
+      jobs.forEach((job) => {
+        job.provider.forEach((provider) => {
+          activeProvider.add(provider.id);
+        });
+        job.notificationAdapter.forEach((adapter) => {
+          activeAdapter.add(adapter.id);
+        });
+      });
+
+      mixpanelTracker.track(
+        'fredy_tracking',
+        enrichTrackingObject({
+          adapter: Array.from(activeAdapter),
+          provider: Array.from(activeProvider),
+        }),
+      );
+    }
+  }
+};
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoJobCreated(jobData) {
+  if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+    mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
+  }
+}
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoAccessed() {
+  if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+    mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
+  }
+}
+
+function enrichTrackingObject(trackingObject) {
+  const operating_system = os.platform();
+  const os_version = os.release();
+  const arch = process.arch;
+  const language = process.env.LANG || 'en';
+  const nodeVersion = process.version || 'N/A';
+
+  return {
+    ...trackingObject,
+    isDemo: config.demoMode,
+    operating_system,
+    os_version,
+    arch,
+    nodeVersion,
+    language,
+    distinct_id,
+  };
+}

lib/services/tracking/uniqueId.js

@@ -0,0 +1,19 @@
+import { hostname, arch, cpus, platform } from 'os';
+import { createHash } from 'crypto';
+
+/**
+ * Don't worry, we are not evil ;) We however need a unique id per running instance
+ * @returns {string}
+ */
+export const getUniqueId = () => {
+  const systemInfo = {
+    hostname: hostname(),
+    architecture: arch(),
+    cpuCount: cpus().length,
+    platform: platform(),
+  };
+
+  const baseData = JSON.stringify(systemInfo);
+
+  return createHash('sha256').update(baseData).digest('hex');
+};

lib/utils.js

@@ -2,6 +2,11 @@ import {dirname} from 'node:path';
 import {fileURLToPath} from 'node:url';
 import {readFile} from 'fs/promises';
 import {createHash} from 'crypto';
+import {DEFAULT_CONFIG} from './defaultConfig.js';
+
+function inDevMode(){
+    return process.env.NODE_ENV == null || process.env.NODE_ENV !== 'production';
+}
 
 function isOneOf(word, arr) {
     if (arr == null || arr.length === 0) {
@@ -52,9 +57,26 @@ function buildHash(...inputs) {
         .digest('hex');
 }
 
-const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+let config = {};
+export async function readConfigFromStorage(){
+    return JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+}
+
+export async function refreshConfig(){
+    try {
+        config = await readConfigFromStorage();
+        //backwards compatability...
+        config.analyticsEnabled ??= null;
+        config.demoMode ??= false;
+    } catch (error) {
+        config = {...DEFAULT_CONFIG};
+        console.error('Error reading config file', error);
+    }
+}
+await refreshConfig();
 
 export {isOneOf};
+export {inDevMode};
 export {nullOrEmpty};
 export {duringWorkingHoursOrNotSet};
 export {getDirName};

package.json

@@ -1,9 +1,9 @@
 {
   "name": "fredy",
-  "version": "10.2.0",
+  "version": "10.4.3",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
-    "start": "node index.js",
+    "start": "node prod.js",
     "dev": "yarn && rm -rf ./ui/public/* && vite",
     "ui": "rm -rf ./ui/public/* && vite",
     "prod": "yarn && vite build --emptyOutDir",
@@ -11,11 +11,6 @@
     "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
     "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
   },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
-  },
   "type": "module",
   "lint-staged": {
     "*.js": [
@@ -55,21 +50,22 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.68.3",
+    "@douyinfe/semi-ui": "2.70.1",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.4",
-    "@vitejs/plugin-react": "4.3.3",
+    "@vitejs/plugin-react": "4.3.4",
-    "better-sqlite3": "^11.5.0",
+    "better-sqlite3": "^11.6.0",
     "body-parser": "1.20.3",
     "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "11.4.8",
+    "highcharts": "12.0.1",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "5.0.8",
+    "mixpanel": "^0.18.0",
+    "nanoid": "5.0.9",
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.6",
     "query-string": "9.1.1",
@@ -84,7 +80,7 @@
     "serve-static": "1.16.2",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.4.10",
+    "vite": "5.4.11",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
@@ -98,9 +94,9 @@
     "eslint-plugin-react": "7.37.2",
     "esmock": "2.6.9",
     "history": "5.3.0",
-    "husky": "4.3.8",
+    "husky": "9.1.7",
-    "less": "4.2.0",
+    "less": "4.2.1",
-    "lint-staged": "13.2.2",
+    "lint-staged": "15.2.10",
     "mocha": "10.8.2",
     "prettier": "3.3.3",
     "redux-logger": "3.0.6"

prod.js

@@ -0,0 +1,2 @@
+process.env.NODE_ENV = 'production';
+import('./index.js');

ui/src/App.jsx

@@ -17,11 +17,14 @@ import Jobs from './views/jobs/Jobs';
 import {Route} from 'react-router';
 
 import './App.less';
+import TrackingModal from './components/tracking/TrackingModal.jsx';
+import {Banner} from '@douyinfe/semi-ui';
 
 export default function FredyApp() {
     const dispatch = useDispatch();
     const [loading, setLoading] = React.useState(true);
     const currentUser = useSelector((state) => state.user.currentUser);
+    const settings = useSelector((state) => state.generalSettings.settings);
 
     useEffect(() => {
         async function init() {
@@ -31,6 +34,7 @@ export default function FredyApp() {
                 await dispatch.jobs.getJobs();
                 await dispatch.jobs.getProcessingTimes();
                 await dispatch.notificationAdapter.getAdapter();
+                await dispatch.generalSettings.getGeneralSettings();
             }
             setLoading(false);
         }
@@ -59,6 +63,18 @@ export default function FredyApp() {
                 <Logout/>
                 <Logo width={190} white/>
                 <Menu isAdmin={isAdmin()}/>
+
+                {settings.demoMode && (
+                <>
+                    <Banner fullMode={true}
+                            type="info"
+                            bordered
+                            closeIcon={null}
+                            description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
+                    />
+                    <br/>
+                </>)}
+                {(settings.analyticsEnabled === null && !settings.demoMode) && <TrackingModal/>}
                 <Switch>
                     <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission}/>
                     <Route name="Create new Job" path={'/jobs/new'} component={JobMutation}/>

ui/src/components/tracking/TrackingModal.jsx

@@ -0,0 +1,48 @@
+import React from 'react';
+import {Modal} from '@douyinfe/semi-ui';
+import Logo from '../logo/Logo.jsx';
+import {xhrPost} from '../../services/xhr.js';
+
+import './TrackingModal.less';
+
+const saveResponse = async (analyticsEnabled) => {
+    await xhrPost('/api/admin/generalSettings', {
+        analyticsEnabled
+    });
+};
+
+export default function TrackingModal() {
+
+    return <Modal
+        visible={true}
+        onOk={async () => {
+            await saveResponse(true);
+            location.reload();
+        }}
+        onCancel={async () => {
+            await saveResponse(false);
+            location.reload();
+        }}
+        maskClosable={false}
+        closable={false}
+        okText="Yes! I want to help"
+        cancelText="No, thanks"
+    >
+        <Logo white/>
+        <div className="trackingModal__description">
+            <p>Hey 👋</p>
+            <p>Fed up with popups? Yeah, me too. But this one’s important, and I promise it will only appear once ;)</p>
+            <p>Fredy is completely free (and will always remain free). If you’d like, you can support me by donating
+                through my GitHub, but there’s absolutely no obligation to do so.</p>
+            <p>However, it would be a huge
+                help if you’d allow me to collect some analytical data. Wait, before you click "no", let me explain. If
+                you
+                agree, Fredy will send a ping to my Mixpanel project each time it runs.</p>
+            <p>The data includes: names of
+                active adapters/providers, OS, architecture, Node version, and language. The information is entirely
+                anonymous and helps me understand which adapters/providers are most frequently used.</p>
+            <p>Thanks🤘</p>
+        </div>
+    </Modal>;
+
+}

ui/src/components/tracking/TrackingModal.less

@@ -0,0 +1,5 @@
+.trackingModal {
+  &__description {
+    margin-top:10rem;
+  }
+}

ui/src/services/rematch/models/demoMode.js

@@ -0,0 +1,24 @@
+import { xhrGet } from '../../xhr';
+export const demoMode = {
+  state: {
+    demoMode: false,
+  },
+  reducers: {
+    setDemoMode: (state, payload) => {
+      return {
+        ...state,
+        demoMode: payload.demoMode,
+      };
+    },
+  },
+  effects: {
+    async getDemoMode() {
+      try {
+        const response = await xhrGet('/api/demo');
+        this.setDemoMode(response.json);
+      } catch (Exception) {
+        console.error('Error while trying to get resource for api/demo. Error:', Exception);
+      }
+    },
+  },
+};

ui/src/services/rematch/store.js

@@ -5,6 +5,7 @@ import { provider } from './models/provider';
 import { createLogger } from 'redux-logger';
 import { jobs } from './models/jobs';
 import { user } from './models/user';
+import { demoMode } from './models/demoMode.js';
 import { init } from '@rematch/core';
 const middleware = [];
 if (process.env.NODE_ENV === 'development') {
@@ -16,6 +17,7 @@ const store = init({
   models: {
     notificationAdapter,
     generalSettings,
+    demoMode,
     provider,
     jobs,
     user,

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -2,13 +2,13 @@ import React from 'react';
 
 import {useDispatch, useSelector} from 'react-redux';
 
-import { Divider, Input, Radio, TimePicker, Button, RadioGroup } from '@douyinfe/semi-ui';
+import {Divider, Input, Radio, TimePicker, Button, RadioGroup, Checkbox} from '@douyinfe/semi-ui';
 import {InputNumber} from '@douyinfe/semi-ui';
 import Headline from '../../components/headline/Headline';
 import {xhrPost} from '../../services/xhr';
 import {SegmentPart} from '../../components/segment/SegmentPart';
 import {Banner, Toast} from '@douyinfe/semi-ui';
-import { IconSave, IconCalendar, IconKey, IconRefresh, IconSignal } from '@douyinfe/semi-icons';
+import {IconSave, IconCalendar, IconKey, IconRefresh, IconSignal, IconLineChartStroked, IconSearch} from '@douyinfe/semi-icons';
 import './GeneralSettings.less';
 
 function formatFromTimestamp(ts) {
@@ -39,6 +39,9 @@ const GeneralSettings = function GeneralSettings() {
     const [scrapingAntProxy, setScrapingAntProxy] = React.useState('');
     const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
     const [workingHourTo, setWorkingHourTo] = React.useState(null);
+    const [demoMode, setDemoMode] = React.useState(null);
+    const [analyticsEnabled, setAnalyticsEnabled] = React.useState(null);
+
     React.useEffect(() => {
         async function init() {
             await dispatch.generalSettings.getGeneralSettings();
@@ -56,6 +59,8 @@ const GeneralSettings = function GeneralSettings() {
             setWorkingHourFrom(settings?.workingHours?.from);
             setWorkingHourTo(settings?.workingHours?.to);
             setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
+            setAnalyticsEnabled(settings?.analyticsEnabled || false);
+            setDemoMode(settings?.demoMode || false);
         }
 
         init();
@@ -99,13 +104,22 @@ const GeneralSettings = function GeneralSettings() {
                     from: workingHourFrom,
                     to: workingHourTo,
                 },
+                demoMode,
+                analyticsEnabled
             });
         } catch (exception) {
             console.error(exception);
+            if(exception?.json?.message != null){
+                throwMessage(exception.json.message, 'error');
+            }else {
                 throwMessage('Error while trying to store settings.', 'error');
+            }
             return;
         }
-    throwMessage('Settings stored successfully. You MUST restart Fredy.', 'success');
+        throwMessage('Settings stored successfully. We will reload your browser in 3 seconds.', 'success');
+        setTimeout(()=>{
+            location.reload();
+        }, 3000);
     };
 
     return (
@@ -113,14 +127,6 @@ const GeneralSettings = function GeneralSettings() {
             {!loading && (
                 <React.Fragment>
                     <Headline text="General Settings"/>
-          <Banner
-            fullMode={false}
-            type="info"
-            closeIcon={null}
-            title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Info</div>}
-            style={{ marginBottom: '1rem' }}
-            description="If you change any settings, you must restart Fredy afterwards."
-          />
                     <div>
                         <SegmentPart
                             name="Interval"
@@ -173,22 +179,27 @@ const GeneralSettings = function GeneralSettings() {
                                 closeIcon={null}
                                 title={
                                     <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
-                    ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2 different types of proxies
+                                        ScrapingAnt is needed to scrape Immoscout. ScrapingAnt itself is using 2
+                                        different types of proxies
                                     </div>
                                 }
                                 style={{marginBottom: '1rem'}}
                                 description={
                                     <div>
                                         <h4>Datacenter-Proxy</h4>
-                    Proxy server located in one of the datacenters across the world. Datacenter proxies are slower and
+                                        Proxy server located in one of the datacenters across the world. Datacenter
-                    more likely to fail, but they are cheaper. A call with a datacenter proxy cost 10 credits.
+                                        proxies are slower and
+                                        more likely to fail, but they are cheaper. A call with a datacenter proxy cost
+                                        10 credits.
                                         <h4>Residential-Proxy</h4>
-                    High-quality proxy server located in one of the real people houses across the world. Datacenter
+                                        High-quality proxy server located in one of the real people houses across the
+                                        world. Datacenter
                                         proxies are faster and more likely to success, but they are more expensive.
                                         <br/>
                                         <br/>
                                         <b>
-                      On the free tier, you have 10.000 credits, so chose your option wisely. Keep in mind, only
+                                            On the free tier, you have 10.000 credits, so chose your option wisely. Keep
+                                            in mind, only
                                             successful calls will be charged.
                                         </b>
                                     </div>
@@ -199,7 +210,8 @@ const GeneralSettings = function GeneralSettings() {
                                 <Radio name="datacenter" value="datacenter" checked={scrapingAntProxy === 'datacenter'}>
                                     Datacenter proxy
                                 </Radio>
-                <Radio name="residential" value="residential" checked={scrapingAntProxy === 'residential'}>
+                                <Radio name="residential" value="residential"
+                                       checked={scrapingAntProxy === 'residential'}>
                                     Residential proxy
                                 </Radio>
                             </RadioGroup>
@@ -231,6 +243,80 @@ const GeneralSettings = function GeneralSettings() {
                                 />
                             </div>
                         </SegmentPart>
+                        <Divider margin="1rem"/>
+
+                        <SegmentPart
+                            name="Analytics"
+                            helpText="Insights into the usage of Fredy."
+                            Icon={IconLineChartStroked}
+                        >
+                            <Banner
+                                fullMode={false}
+                                type="info"
+                                closeIcon={null}
+                                title={
+                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
+                                        Explanation
+                                    </div>
+                                }
+                                style={{marginBottom: '1rem'}}
+                                description={
+                                    <div>
+                                        Analytics are disabled by default. If you choose to enable them, we will begin tracking the following:<br/>
+                                        <ul>
+                                            <li>Name of active provider (e.g. Immoscout)</li>
+                                            <li>Name of active adapter (e.g. Console)</li>
+                                            <li>language</li>
+                                            <li>os</li>
+                                            <li>node version</li>
+                                            <li>arch</li>
+                                        </ul>
+                                        The data is sent anonymously and helps me understand which providers or adapters are being used the most. In the end it helps me to improve fredy.
+                                    </div>
+                                }
+                            />
+
+                            <Checkbox
+                                checked={analyticsEnabled}
+                                onChange={(e) => setAnalyticsEnabled(e.target.checked)}
+                            > Enabled
+                            </Checkbox>
+
+                        </SegmentPart>
+
+                        <Divider margin="1rem"/>
+
+                        <SegmentPart
+                            name="Demo Mode"
+                            helpText="If enabled, Fredy runs in demo mode."
+                            Icon={IconSearch}
+                        >
+                            <Banner
+                                fullMode={false}
+                                type="info"
+                                closeIcon={null}
+                                title={
+                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
+                                        Explanation
+                                    </div>
+                                }
+                                style={{marginBottom: '1rem'}}
+                                description={
+                                    <div>
+                                        In demo mode, Fredy will not (really) search for any real estates. Fredy is in a lockdown mode. Also
+                                        all database files will be set back to the default values at midnight.
+                                    </div>
+                                }
+                            />
+
+                            <Checkbox
+                                checked={demoMode}
+                                onChange={(e) => setDemoMode(e.target.checked)}
+                            > Enabled
+                            </Checkbox>
+
+                        </SegmentPart>
+
                         <Divider margin="1rem"/>
                         <Button type="primary" theme="solid" onClick={onStore} icon={<IconSave/>}>
                             Save

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,9 +1,31 @@
 import React from 'react';
 import {format} from '../../services/time/timeService';
-import { Card, Descriptions, Divider } from '@douyinfe/semi-ui';
+import {Banner, Card, Descriptions, Divider} from '@douyinfe/semi-ui';
 import {IconBolt} from '@douyinfe/semi-icons';
-export default function ProcessingTimes({ processingTimes }) {
+
+export default function ProcessingTimes({processingTimes = {}}) {
     const {Meta} = Card;
+    if (Object.keys(processingTimes).length === 0) {
+        return null;
+    }
+    if (processingTimes.error != null) {
+        return <Banner
+            fullMode={false}
+            type="danger"
+            closeIcon={null}
+            title={
+                <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
+                    Scraping Ant Error
+                </div>
+            }
+            style={{marginBottom: '1rem'}}
+            description={
+                <div>
+                    {processingTimes.error}
+                </div>
+            }
+        />;
+    }
     return (
         <>
             <Descriptions
@@ -26,7 +48,7 @@ export default function ProcessingTimes({ processingTimes }) {
                 )}
             </Descriptions>
 
-      {processingTimes.scrapingAntData != null && (
+            {(processingTimes.scrapingAntData != null && Object.keys(processingTimes.scrapingAntData).length > 0) &&(
                 <>
                     <Divider margin="1rem"/>
                     <Card
@@ -47,12 +69,13 @@ export default function ProcessingTimes({ processingTimes }) {
                             Credits: {processingTimes.scrapingAntData.remained_credits}/
                             {processingTimes.scrapingAntData.plan_total_credits}
                         </p>
-            If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account of{' '}
+                        If you want to scrape Immoscout or Immonet more often, you have to purchase a premium account
+                        of{' '}
                         <a href="https://scrapingant.com/" target="_blank" rel="noreferrer">
                             ScrapingAnt
                         </a>
-            . You can use the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting paid to
+                        . You can use the code <b>FREDY10</b> to get 10% off. (No affiliation, we are <b>not</b> getting
-            recommend ScrapingAnt.)
+                        paid by ScrapingAnt.)
                     </Card>
                 </>
             )}

ui/src/views/login/Login.jsx

@@ -1,10 +1,10 @@
-import React from 'react';
+import React, {useEffect} from 'react';
 
 import cityBackground from '../../assets/city_background.jpg';
 import Logo from '../../components/logo/Logo';
 import {xhrPost} from '../../services/xhr';
 import {useHistory} from 'react-router';
-import { useDispatch } from 'react-redux';
+import {useDispatch, useSelector} from 'react-redux';
 import {Input, Button, Banner} from '@douyinfe/semi-ui';
 
 import './login.less';
@@ -12,13 +12,20 @@ import { IconUser, IconLock } from '@douyinfe/semi-icons';
 
 export default function Login() {
     const dispatch = useDispatch();
-
     const [username, setUserName] = React.useState('');
     const [password, setPassword] = React.useState('');
     const [error, setError] = React.useState(null);
-
+    const demoMode = useSelector((state) => state.demoMode.demoMode || false);
     const history = useHistory();
 
+    useEffect(() => {
+        async function init() {
+            await dispatch.demoMode.getDemoMode();
+        }
+
+        init();
+    }, []);
+
     const tryLogin = async () => {
         if (username.length === 0 || password.length === 0) {
             setError('Username and password are mandatory.');
@@ -52,7 +59,7 @@ export default function Login() {
                         value={username}
                         showClear
                         style={{marginTop: error ? '1rem' : '4rem'}}
-            autofocus
+                        autoFocus
                         onChange={(value) => setUserName(value)}
                         onKeyPress={async (e) => {
                             if (e.key === 'Enter') {
@@ -79,6 +86,13 @@ export default function Login() {
                     <Button type="primary" onClick={tryLogin} theme="solid" style={{marginTop: '3rem'}}>
                         Login
                     </Button>
+                    <br/>
+                    {demoMode && <Banner fullMode={true}
+                                         type="info"
+                                         bordered
+                                         closeIcon={null}
+                                         description="This is the demo version of Fredy. Use 'demo' as both the username and password to log in."
+                    />}
                 </div>
             </form>
         </div>