diff --git a/conf/config.json b/conf/config.json
index 037f907..0e5b45f 100755
--- a/conf/config.json
+++ b/conf/config.json
@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}
\ No newline at end of file
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"d","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}
\ No newline at end of file
diff --git a/index.js b/index.js
index 511086b..717675f 100755
--- a/index.js
+++ b/index.js
@@ -1,5 +1,5 @@
import fs from 'fs';
-import { config } from './lib/utils.js';
+import {config} from './lib/utils.js';
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
import * as jobStorage from './lib/services/storage/jobStorage.js';
@@ -7,6 +7,8 @@ import FredyRuntime from './lib/FredyRuntime.js';
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
import './lib/api/api.js';
import {track} from './lib/services/tracking/Tracker.js';
+import {handleDemoUser} from './lib/services/storage/userStorage.js';
+import {cleanupDemoAtMidnight} from './lib/services/demoCleanup.js';
//if db folder does not exist, ensure to create it before loading anything else
if (!fs.existsSync('./db')) {
fs.mkdirSync('./db');
@@ -17,35 +19,43 @@ const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
const INTERVAL = config.interval * 60 * 1000;
/* eslint-disable no-console */
console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+if(config.demoMode){
+ console.info('Running in demo mode');
+ cleanupDemoAtMidnight();
+}
/* eslint-enable no-console */
const fetchedProvider = await Promise.all(
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
);
+handleDemoUser();
+
setInterval(
(function exec() {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
- if (isDuringWorkingHoursOrNotSet) {
- track();
- config.lastRun = Date.now();
- jobStorage
- .getJobs()
- .filter((job) => job.enabled)
- .forEach((job) => {
- job.provider
- .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
- .forEach(async (prov) => {
- const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
- pro.init(prov, job.blacklist);
- await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
- setLastJobExecution(job.id);
- });
- });
- } else {
- /* eslint-disable no-console */
- console.debug('Working hours set. Skipping as outside of working hours.');
- /* eslint-enable no-console */
- }
+ if(!config.demoMode) {
+ if (isDuringWorkingHoursOrNotSet) {
+ track();
+ config.lastRun = Date.now();
+ jobStorage
+ .getJobs()
+ .filter((job) => job.enabled)
+ .forEach((job) => {
+ job.provider
+ .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+ .forEach(async (prov) => {
+ const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+ pro.init(prov, job.blacklist);
+ await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+ setLastJobExecution(job.id);
+ });
+ });
+ } else {
+ /* eslint-disable no-console */
+ console.debug('Working hours set. Skipping as outside of working hours.');
+ /* eslint-enable no-console */
+ }
+ }
return exec;
})(),
INTERVAL
diff --git a/lib/api/api.js b/lib/api/api.js
index 362ef0c..7d44843 100644
--- a/lib/api/api.js
+++ b/lib/api/api.js
@@ -12,6 +12,7 @@ import restana from 'restana';
import files from 'serve-static';
import path from 'path';
import { getDirName } from '../utils.js';
+import {demoRouter} from './routes/demoRouter.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../ui/public'));
const PORT = config.port || 9998;
@@ -30,6 +31,9 @@ service.use('/api/jobs/insights', analyticsRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
+//this route is unsecured intentionally as it is being queried from the login page
+service.use('/api/demo', demoRouter);
+
/* eslint-disable no-console */
service.start(PORT).then(() => {
console.info(`Started API service on port ${PORT}`);
diff --git a/lib/api/routes/demoRouter.js b/lib/api/routes/demoRouter.js
new file mode 100644
index 0000000..6eb6e0d
--- /dev/null
+++ b/lib/api/routes/demoRouter.js
@@ -0,0 +1,11 @@
+import restana from 'restana';
+import {config} from '../../utils.js';
+const service = restana();
+const demoRouter = service.newRouter();
+
+demoRouter.get('/', async (req, res) => {
+ res.body = Object.assign({}, {demoMode: config.demoMode});
+ res.send();
+});
+
+export { demoRouter };
diff --git a/lib/api/routes/generalSettingsRoute.js b/lib/api/routes/generalSettingsRoute.js
index 10e2e49..f125924 100644
--- a/lib/api/routes/generalSettingsRoute.js
+++ b/lib/api/routes/generalSettingsRoute.js
@@ -1,6 +1,7 @@
import restana from 'restana';
import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
import fs from 'fs';
+import {handleDemoUser} from '../../services/storage/userStorage.js';
const service = restana();
const generalSettingsRouter = service.newRouter();
generalSettingsRouter.get('/', async (req, res) => {
@@ -10,9 +11,14 @@ generalSettingsRouter.get('/', async (req, res) => {
generalSettingsRouter.post('/', async (req, res) => {
const settings = req.body;
try {
+ if(config.demoMode){
+ res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+ return;
+ }
const currentConfig = await readConfigFromStorage();
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
await refreshConfig();
+ handleDemoUser();
} catch (err) {
console.error(err);
res.send(new Error('Error while trying to write settings.'));
diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js
index 37ac1f5..35f3aac 100644
--- a/lib/api/routes/jobRouter.js
+++ b/lib/api/routes/jobRouter.js
@@ -6,6 +6,7 @@ import * as immoscoutProvider from '../../provider/immoscout.js';
import { config } from '../../utils.js';
import { isAdmin } from '../security.js';
import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
+import {trackDemoJobCreated} from '../../services/tracking/Tracker.js';
const service = restana();
const jobRouter = service.newRouter();
function doesJobBelongsToUser(job, req) {
@@ -68,6 +69,11 @@ jobRouter.post('/', async (req, res) => {
res.send(new Error(error));
console.error(error);
}
+ trackDemoJobCreated({
+ name,
+ provider,
+ adapter: notificationAdapter
+ });
res.send();
});
jobRouter.delete('', async (req, res) => {
diff --git a/lib/api/routes/loginRoute.js b/lib/api/routes/loginRoute.js
index c41240b..7234021 100644
--- a/lib/api/routes/loginRoute.js
+++ b/lib/api/routes/loginRoute.js
@@ -1,6 +1,8 @@
import restana from 'restana';
import * as userStorage from '../../services/storage/userStorage.js';
import * as hasher from '../../services/security/hash.js';
+import {config} from '../../utils.js';
+import {trackDemoAccessed} from '../../services/tracking/Tracker.js';
const service = restana();
const loginRouter = service.newRouter();
loginRouter.get('/user', async (req, res) => {
@@ -24,6 +26,11 @@ loginRouter.post('/', async (req, res) => {
return;
}
if (user.password === hasher.hash(password)) {
+
+ if(config.demoMode){
+ trackDemoAccessed();
+ }
+
req.session.currentUser = user.id;
userStorage.setLastLoginToNow({ userId: user.id });
res.send(200);
diff --git a/lib/api/routes/userRoute.js b/lib/api/routes/userRoute.js
index f57035b..ade3cca 100644
--- a/lib/api/routes/userRoute.js
+++ b/lib/api/routes/userRoute.js
@@ -1,6 +1,7 @@
import restana from 'restana';
import * as userStorage from '../../services/storage/userStorage.js';
import * as jobStorage from '../../services/storage/jobStorage.js';
+import {config} from '../../utils.js';
const service = restana();
const userRouter = service.newRouter();
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
@@ -20,6 +21,11 @@ userRouter.get('/:userId', async (req, res) => {
res.send();
});
userRouter.delete('/', async (req, res) => {
+ if(config.demoMode){
+ res.send(new Error('In demo mode, it is not allowed to remove user.'));
+ return;
+ }
+
const { userId } = req.body;
const allUser = userStorage.getUsers(false);
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
@@ -36,6 +42,12 @@ userRouter.delete('/', async (req, res) => {
res.send();
});
userRouter.post('/', async (req, res) => {
+
+ if(config.demoMode){
+ res.send(new Error('In demo mode, it is not allowed to change or add user.'));
+ return;
+ }
+
const { username, password, password2, isAdmin, userId } = req.body;
if (password !== password2) {
res.send(new Error('Passwords does not match'));
diff --git a/lib/services/demoCleanup.js b/lib/services/demoCleanup.js
new file mode 100644
index 0000000..4386dd4
--- /dev/null
+++ b/lib/services/demoCleanup.js
@@ -0,0 +1,29 @@
+import { setInterval } from 'node:timers';
+import {removeJobsByUserName} from './storage/jobStorage.js';
+import {config} from '../utils.js';
+
+/**
+ * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
+ */
+export function cleanupDemoAtMidnight() {
+ const now = new Date();
+ const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+ - now.getUTCMinutes() * 60 * 1000
+ - now.getUTCSeconds() * 1000
+ - now.getUTCMilliseconds();
+
+ setTimeout(() => {
+ cleanup();
+
+ setInterval(() => {
+ cleanup();
+ }, 24 * 60 * 60 * 1000);
+
+ }, millisUntilMidnightUTC);
+}
+
+function cleanup(){
+ if(config.demoMode){
+ removeJobsByUserName('demo');
+ }
+}
\ No newline at end of file
diff --git a/lib/services/storage/jobStorage.js b/lib/services/storage/jobStorage.js
index 0300046..655ada2 100644
--- a/lib/services/storage/jobStorage.js
+++ b/lib/services/storage/jobStorage.js
@@ -77,6 +77,17 @@ export const removeJobsByUserId = (userId) => {
.value();
db.write();
};
+export const removeJobsByUserName = (userName) => {
+ db.chain
+ .get('jobs')
+ .filter((job) => job.username === userName)
+ .forEach((job) => listingStorage.removeListings(job.id));
+ db.chain
+ .get('jobs')
+ .remove((job) => job.username === userName)
+ .value();
+ db.write();
+};
export const getJobs = () => {
return db.chain
.get('jobs')
diff --git a/lib/services/storage/userStorage.js b/lib/services/storage/userStorage.js
index 0d7712d..a3ffd9d 100644
--- a/lib/services/storage/userStorage.js
+++ b/lib/services/storage/userStorage.js
@@ -1,5 +1,5 @@
import { JSONFileSync } from 'lowdb/node';
-import { getDirName } from '../../utils.js';
+import {config, getDirName} from '../../utils.js';
import * as hasher from '../security/hash.js';
import { nanoid } from 'nanoid';
import * as jobStorage from './jobStorage.js';
@@ -16,6 +16,13 @@ const defaultData = {
password: hasher.hash('admin'),
isAdmin: true,
},
+ {
+ id: nanoid(),
+ lastLogin: Date.now(),
+ username: 'demo',
+ password: hasher.hash('demo'),
+ isAdmin: true,
+ },
],
};
@@ -84,3 +91,29 @@ export const removeUser = (userId) => {
.value();
db.write();
};
+
+export const handleDemoUser = () => {
+ if(!config.demoMode){
+ const user = db.chain.get('user').value();
+ db.chain.get('user').value();
+ db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+ db.write();
+ }else {
+ const demoUser = db.chain
+ .get('user')
+ .filter((u) => u.username === 'demo')
+ .value();
+ if (demoUser == null || demoUser.length === 0) {
+ db.chain.get('user')
+ .value()
+ .push({
+ id: nanoid(),
+ username: 'demo',
+ password: hasher.hash('demo'),
+ isAdmin: true,
+ });
+ db.write();
+ }
+ }
+};
+
diff --git a/lib/services/tracking/Tracker.js b/lib/services/tracking/Tracker.js
index 1439770..8b96df3 100644
--- a/lib/services/tracking/Tracker.js
+++ b/lib/services/tracking/Tracker.js
@@ -1,20 +1,16 @@
import Mixpanel from 'mixpanel';
import {getJobs} from '../storage/jobStorage.js';
-import {config} from '../../utils.js';
+import {config, inDevMode} from '../../utils.js';
+
+const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
export const track = function () {
//only send tracking information if the user allowed to do so.
- if (config.analyticsEnabled) {
-
- const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+ if (config.analyticsEnabled && !inDevMode()) {
const activeProvider = new Set();
const activeAdapter = new Set();
- const platform = process.platform;
- const arch = process.arch;
- const language = process.env.LANG || 'en';
- const nodeVersion = process.version || 'N/A';
const jobs = getJobs();
@@ -28,15 +24,45 @@ export const track = function () {
});
});
- mixpanelTracker.track('fredy_tracking', {
+ mixpanelTracker.track('fredy_tracking', enrichTrackingObject({
adapter: Array.from(activeAdapter),
provider: Array.from(activeProvider),
- isDemo: config.demoMode,
- platform,
- arch,
- nodeVersion,
- language
- });
+ }));
}
}
-};
\ No newline at end of file
+};
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoJobCreated(jobData) {
+ if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+ mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
+ }
+}
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoAccessed() {
+ if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+ mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
+ }
+}
+
+
+function enrichTrackingObject(trackingObject) {
+ const platform = process.platform;
+ const arch = process.arch;
+ const language = process.env.LANG || 'en';
+ const nodeVersion = process.version || 'N/A';
+
+ return {
+ ...trackingObject,
+ isDemo: config.demoMode,
+ platform,
+ arch,
+ nodeVersion,
+ language
+ };
+}
diff --git a/lib/utils.js b/lib/utils.js
index ea534b6..4f8527b 100755
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -4,6 +4,10 @@ import {readFile} from 'fs/promises';
import {createHash} from 'crypto';
import {DEFAULT_CONFIG} from './defaultConfig.js';
+function inDevMode(){
+ return process.env.NODE_ENV == null || process.env.NODE_ENV !== 'production';
+}
+
function isOneOf(word, arr) {
if (arr == null || arr.length === 0) {
return false;
@@ -72,6 +76,7 @@ export async function refreshConfig(){
await refreshConfig();
export {isOneOf};
+export {inDevMode};
export {nullOrEmpty};
export {duringWorkingHoursOrNotSet};
export {getDirName};
diff --git a/package.json b/package.json
index 4cda337..9752dfb 100755
--- a/package.json
+++ b/package.json
@@ -3,7 +3,7 @@
"version": "10.3.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
- "start": "node index.js",
+ "start": "node prod.js",
"dev": "yarn && rm -rf ./ui/public/* && vite",
"ui": "rm -rf ./ui/public/* && vite",
"prod": "yarn && vite build --emptyOutDir",
diff --git a/prod.js b/prod.js
new file mode 100644
index 0000000..b496da6
--- /dev/null
+++ b/prod.js
@@ -0,0 +1,2 @@
+process.env.NODE_ENV = 'production';
+import('./index.js');
\ No newline at end of file
diff --git a/ui/src/App.jsx b/ui/src/App.jsx
index cef92eb..dadcdfa 100644
--- a/ui/src/App.jsx
+++ b/ui/src/App.jsx
@@ -1,4 +1,4 @@
-import React, { useEffect } from 'react';
+import React, {useEffect} from 'react';
import InsufficientPermission from './components/permission/InsufficientPermission';
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
@@ -6,94 +6,106 @@ import GeneralSettings from './views/generalSettings/GeneralSettings';
import JobMutation from './views/jobs/mutation/JobMutation';
import UserMutator from './views/user/mutation/UserMutator';
import JobInsight from './views/jobs/insights/JobInsight.jsx';
-import { useDispatch, useSelector } from 'react-redux';
-import { Switch, Redirect } from 'react-router-dom';
+import {useDispatch, useSelector} from 'react-redux';
+import {Switch, Redirect} from 'react-router-dom';
import Logout from './components/logout/Logout';
import Logo from './components/logo/Logo';
import Menu from './components/menu/Menu';
import Login from './views/login/Login';
import Users from './views/user/Users';
import Jobs from './views/jobs/Jobs';
-import { Route } from 'react-router';
+import {Route} from 'react-router';
import './App.less';
import TrackingModal from './components/tracking/TrackingModal.jsx';
+import {Banner} from '@douyinfe/semi-ui';
export default function FredyApp() {
- const dispatch = useDispatch();
- const [loading, setLoading] = React.useState(true);
- const currentUser = useSelector((state) => state.user.currentUser);
- const settings = useSelector((state) => state.generalSettings.settings);
+ const dispatch = useDispatch();
+ const [loading, setLoading] = React.useState(true);
+ const currentUser = useSelector((state) => state.user.currentUser);
+ const settings = useSelector((state) => state.generalSettings.settings);
- useEffect(() => {
- async function init() {
- await dispatch.user.getCurrentUser();
- if (!needsLogin()) {
- await dispatch.provider.getProvider();
- await dispatch.jobs.getJobs();
- await dispatch.jobs.getProcessingTimes();
- await dispatch.notificationAdapter.getAdapter();
- await dispatch.generalSettings.getGeneralSettings();
- }
- setLoading(false);
- }
+ useEffect(() => {
+ async function init() {
+ await dispatch.user.getCurrentUser();
+ if (!needsLogin()) {
+ await dispatch.provider.getProvider();
+ await dispatch.jobs.getJobs();
+ await dispatch.jobs.getProcessingTimes();
+ await dispatch.notificationAdapter.getAdapter();
+ await dispatch.generalSettings.getGeneralSettings();
+ }
+ setLoading(false);
+ }
- init();
- }, [currentUser?.userId]);
+ init();
+ }, [currentUser?.userId]);
- const needsLogin = () => {
- return currentUser == null || Object.keys(currentUser).length === 0;
- };
+ const needsLogin = () => {
+ return currentUser == null || Object.keys(currentUser).length === 0;
+ };
- const isAdmin = () => currentUser != null && currentUser.isAdmin;
+ const isAdmin = () => currentUser != null && currentUser.isAdmin;
- const login = () => (
-