mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2c769f786 | ||
|
|
1825a25eaa | ||
|
|
0f20b85f38 | ||
|
|
d17ef9ef1e | ||
|
|
337ee922a6 |
@@ -15,6 +15,9 @@ If you like my work, consider becoming a sponsor. I'm not expecting anybody to p
|
|||||||
|
|
||||||
_Fredy_ is supported by JetBrains under Open Source Support Program
|
_Fredy_ is supported by JetBrains under Open Source Support Program
|
||||||
|
|
||||||
|
## Demo
|
||||||
|
If you want to try out _Fredy_, you can access the demo version [here](https://fredy.orange-coding.net) 🤘
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
- Make sure to use Node.js 20 or above
|
- Make sure to use Node.js 20 or above
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}
|
{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"d","proxy":"datacenter"},"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}
|
||||||
54
index.js
54
index.js
@@ -1,5 +1,5 @@
|
|||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { config } from './lib/utils.js';
|
import {config} from './lib/utils.js';
|
||||||
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
|
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
|
||||||
import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
|
import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
|
||||||
import * as jobStorage from './lib/services/storage/jobStorage.js';
|
import * as jobStorage from './lib/services/storage/jobStorage.js';
|
||||||
@@ -7,6 +7,8 @@ import FredyRuntime from './lib/FredyRuntime.js';
|
|||||||
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
|
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
|
||||||
import './lib/api/api.js';
|
import './lib/api/api.js';
|
||||||
import {track} from './lib/services/tracking/Tracker.js';
|
import {track} from './lib/services/tracking/Tracker.js';
|
||||||
|
import {handleDemoUser} from './lib/services/storage/userStorage.js';
|
||||||
|
import {cleanupDemoAtMidnight} from './lib/services/demoCleanup.js';
|
||||||
//if db folder does not exist, ensure to create it before loading anything else
|
//if db folder does not exist, ensure to create it before loading anything else
|
||||||
if (!fs.existsSync('./db')) {
|
if (!fs.existsSync('./db')) {
|
||||||
fs.mkdirSync('./db');
|
fs.mkdirSync('./db');
|
||||||
@@ -17,35 +19,43 @@ const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
|
|||||||
const INTERVAL = config.interval * 60 * 1000;
|
const INTERVAL = config.interval * 60 * 1000;
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
|
console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
|
||||||
|
if(config.demoMode){
|
||||||
|
console.info('Running in demo mode');
|
||||||
|
cleanupDemoAtMidnight();
|
||||||
|
}
|
||||||
/* eslint-enable no-console */
|
/* eslint-enable no-console */
|
||||||
const fetchedProvider = await Promise.all(
|
const fetchedProvider = await Promise.all(
|
||||||
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
|
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
handleDemoUser();
|
||||||
|
|
||||||
setInterval(
|
setInterval(
|
||||||
(function exec() {
|
(function exec() {
|
||||||
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
|
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
|
||||||
if (isDuringWorkingHoursOrNotSet) {
|
if(!config.demoMode) {
|
||||||
track();
|
if (isDuringWorkingHoursOrNotSet) {
|
||||||
config.lastRun = Date.now();
|
track();
|
||||||
jobStorage
|
config.lastRun = Date.now();
|
||||||
.getJobs()
|
jobStorage
|
||||||
.filter((job) => job.enabled)
|
.getJobs()
|
||||||
.forEach((job) => {
|
.filter((job) => job.enabled)
|
||||||
job.provider
|
.forEach((job) => {
|
||||||
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
|
job.provider
|
||||||
.forEach(async (prov) => {
|
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
|
||||||
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
|
.forEach(async (prov) => {
|
||||||
pro.init(prov, job.blacklist);
|
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
|
||||||
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
|
pro.init(prov, job.blacklist);
|
||||||
setLastJobExecution(job.id);
|
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
|
||||||
});
|
setLastJobExecution(job.id);
|
||||||
});
|
});
|
||||||
} else {
|
});
|
||||||
/* eslint-disable no-console */
|
} else {
|
||||||
console.debug('Working hours set. Skipping as outside of working hours.');
|
/* eslint-disable no-console */
|
||||||
/* eslint-enable no-console */
|
console.debug('Working hours set. Skipping as outside of working hours.');
|
||||||
}
|
/* eslint-enable no-console */
|
||||||
|
}
|
||||||
|
}
|
||||||
return exec;
|
return exec;
|
||||||
})(),
|
})(),
|
||||||
INTERVAL
|
INTERVAL
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import restana from 'restana';
|
|||||||
import files from 'serve-static';
|
import files from 'serve-static';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { getDirName } from '../utils.js';
|
import { getDirName } from '../utils.js';
|
||||||
|
import {demoRouter} from './routes/demoRouter.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const staticService = files(path.join(getDirName(), '../ui/public'));
|
const staticService = files(path.join(getDirName(), '../ui/public'));
|
||||||
const PORT = config.port || 9998;
|
const PORT = config.port || 9998;
|
||||||
@@ -30,6 +31,9 @@ service.use('/api/jobs/insights', analyticsRouter);
|
|||||||
service.use('/api/admin/users', userRouter);
|
service.use('/api/admin/users', userRouter);
|
||||||
service.use('/api/jobs', jobRouter);
|
service.use('/api/jobs', jobRouter);
|
||||||
service.use('/api/login', loginRouter);
|
service.use('/api/login', loginRouter);
|
||||||
|
//this route is unsecured intentionally as it is being queried from the login page
|
||||||
|
service.use('/api/demo', demoRouter);
|
||||||
|
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
service.start(PORT).then(() => {
|
service.start(PORT).then(() => {
|
||||||
console.info(`Started API service on port ${PORT}`);
|
console.info(`Started API service on port ${PORT}`);
|
||||||
|
|||||||
11
lib/api/routes/demoRouter.js
Normal file
11
lib/api/routes/demoRouter.js
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
import restana from 'restana';
|
||||||
|
import {config} from '../../utils.js';
|
||||||
|
const service = restana();
|
||||||
|
const demoRouter = service.newRouter();
|
||||||
|
|
||||||
|
demoRouter.get('/', async (req, res) => {
|
||||||
|
res.body = Object.assign({}, {demoMode: config.demoMode});
|
||||||
|
res.send();
|
||||||
|
});
|
||||||
|
|
||||||
|
export { demoRouter };
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import restana from 'restana';
|
import restana from 'restana';
|
||||||
import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
|
import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
import {handleDemoUser} from '../../services/storage/userStorage.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const generalSettingsRouter = service.newRouter();
|
const generalSettingsRouter = service.newRouter();
|
||||||
generalSettingsRouter.get('/', async (req, res) => {
|
generalSettingsRouter.get('/', async (req, res) => {
|
||||||
@@ -10,9 +11,14 @@ generalSettingsRouter.get('/', async (req, res) => {
|
|||||||
generalSettingsRouter.post('/', async (req, res) => {
|
generalSettingsRouter.post('/', async (req, res) => {
|
||||||
const settings = req.body;
|
const settings = req.body;
|
||||||
try {
|
try {
|
||||||
|
if(config.demoMode){
|
||||||
|
res.send(new Error('In demo mode, it is not allowed to change these settings.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
const currentConfig = await readConfigFromStorage();
|
const currentConfig = await readConfigFromStorage();
|
||||||
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
|
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
|
||||||
await refreshConfig();
|
await refreshConfig();
|
||||||
|
handleDemoUser();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
res.send(new Error('Error while trying to write settings.'));
|
res.send(new Error('Error while trying to write settings.'));
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import * as immoscoutProvider from '../../provider/immoscout.js';
|
|||||||
import { config } from '../../utils.js';
|
import { config } from '../../utils.js';
|
||||||
import { isAdmin } from '../security.js';
|
import { isAdmin } from '../security.js';
|
||||||
import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
|
import {isScrapingAntApiKeySet} from '../../services/scrapingAnt.js';
|
||||||
|
import {trackDemoJobCreated} from '../../services/tracking/Tracker.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const jobRouter = service.newRouter();
|
const jobRouter = service.newRouter();
|
||||||
function doesJobBelongsToUser(job, req) {
|
function doesJobBelongsToUser(job, req) {
|
||||||
@@ -68,6 +69,11 @@ jobRouter.post('/', async (req, res) => {
|
|||||||
res.send(new Error(error));
|
res.send(new Error(error));
|
||||||
console.error(error);
|
console.error(error);
|
||||||
}
|
}
|
||||||
|
trackDemoJobCreated({
|
||||||
|
name,
|
||||||
|
provider,
|
||||||
|
adapter: notificationAdapter
|
||||||
|
});
|
||||||
res.send();
|
res.send();
|
||||||
});
|
});
|
||||||
jobRouter.delete('', async (req, res) => {
|
jobRouter.delete('', async (req, res) => {
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import restana from 'restana';
|
import restana from 'restana';
|
||||||
import * as userStorage from '../../services/storage/userStorage.js';
|
import * as userStorage from '../../services/storage/userStorage.js';
|
||||||
import * as hasher from '../../services/security/hash.js';
|
import * as hasher from '../../services/security/hash.js';
|
||||||
|
import {config} from '../../utils.js';
|
||||||
|
import {trackDemoAccessed} from '../../services/tracking/Tracker.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const loginRouter = service.newRouter();
|
const loginRouter = service.newRouter();
|
||||||
loginRouter.get('/user', async (req, res) => {
|
loginRouter.get('/user', async (req, res) => {
|
||||||
@@ -24,6 +26,11 @@ loginRouter.post('/', async (req, res) => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (user.password === hasher.hash(password)) {
|
if (user.password === hasher.hash(password)) {
|
||||||
|
|
||||||
|
if(config.demoMode){
|
||||||
|
trackDemoAccessed();
|
||||||
|
}
|
||||||
|
|
||||||
req.session.currentUser = user.id;
|
req.session.currentUser = user.id;
|
||||||
userStorage.setLastLoginToNow({ userId: user.id });
|
userStorage.setLastLoginToNow({ userId: user.id });
|
||||||
res.send(200);
|
res.send(200);
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import restana from 'restana';
|
import restana from 'restana';
|
||||||
import * as userStorage from '../../services/storage/userStorage.js';
|
import * as userStorage from '../../services/storage/userStorage.js';
|
||||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||||
|
import {config} from '../../utils.js';
|
||||||
const service = restana();
|
const service = restana();
|
||||||
const userRouter = service.newRouter();
|
const userRouter = service.newRouter();
|
||||||
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
|
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
|
||||||
@@ -20,6 +21,11 @@ userRouter.get('/:userId', async (req, res) => {
|
|||||||
res.send();
|
res.send();
|
||||||
});
|
});
|
||||||
userRouter.delete('/', async (req, res) => {
|
userRouter.delete('/', async (req, res) => {
|
||||||
|
if(config.demoMode){
|
||||||
|
res.send(new Error('In demo mode, it is not allowed to remove user.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const { userId } = req.body;
|
const { userId } = req.body;
|
||||||
const allUser = userStorage.getUsers(false);
|
const allUser = userStorage.getUsers(false);
|
||||||
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||||
@@ -36,6 +42,12 @@ userRouter.delete('/', async (req, res) => {
|
|||||||
res.send();
|
res.send();
|
||||||
});
|
});
|
||||||
userRouter.post('/', async (req, res) => {
|
userRouter.post('/', async (req, res) => {
|
||||||
|
|
||||||
|
if(config.demoMode){
|
||||||
|
res.send(new Error('In demo mode, it is not allowed to change or add user.'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const { username, password, password2, isAdmin, userId } = req.body;
|
const { username, password, password2, isAdmin, userId } = req.body;
|
||||||
if (password !== password2) {
|
if (password !== password2) {
|
||||||
res.send(new Error('Passwords does not match'));
|
res.send(new Error('Passwords does not match'));
|
||||||
|
|||||||
29
lib/services/demoCleanup.js
Normal file
29
lib/services/demoCleanup.js
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { setInterval } from 'node:timers';
|
||||||
|
import {removeJobsByUserName} from './storage/jobStorage.js';
|
||||||
|
import {config} from '../utils.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
|
||||||
|
*/
|
||||||
|
export function cleanupDemoAtMidnight() {
|
||||||
|
const now = new Date();
|
||||||
|
const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
|
||||||
|
- now.getUTCMinutes() * 60 * 1000
|
||||||
|
- now.getUTCSeconds() * 1000
|
||||||
|
- now.getUTCMilliseconds();
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
cleanup();
|
||||||
|
}, 24 * 60 * 60 * 1000);
|
||||||
|
|
||||||
|
}, millisUntilMidnightUTC);
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanup(){
|
||||||
|
if(config.demoMode){
|
||||||
|
removeJobsByUserName('demo');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -77,6 +77,17 @@ export const removeJobsByUserId = (userId) => {
|
|||||||
.value();
|
.value();
|
||||||
db.write();
|
db.write();
|
||||||
};
|
};
|
||||||
|
export const removeJobsByUserName = (userName) => {
|
||||||
|
db.chain
|
||||||
|
.get('jobs')
|
||||||
|
.filter((job) => job.username === userName)
|
||||||
|
.forEach((job) => listingStorage.removeListings(job.id));
|
||||||
|
db.chain
|
||||||
|
.get('jobs')
|
||||||
|
.remove((job) => job.username === userName)
|
||||||
|
.value();
|
||||||
|
db.write();
|
||||||
|
};
|
||||||
export const getJobs = () => {
|
export const getJobs = () => {
|
||||||
return db.chain
|
return db.chain
|
||||||
.get('jobs')
|
.get('jobs')
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { JSONFileSync } from 'lowdb/node';
|
import { JSONFileSync } from 'lowdb/node';
|
||||||
import { getDirName } from '../../utils.js';
|
import {config, getDirName} from '../../utils.js';
|
||||||
import * as hasher from '../security/hash.js';
|
import * as hasher from '../security/hash.js';
|
||||||
import { nanoid } from 'nanoid';
|
import { nanoid } from 'nanoid';
|
||||||
import * as jobStorage from './jobStorage.js';
|
import * as jobStorage from './jobStorage.js';
|
||||||
@@ -16,6 +16,13 @@ const defaultData = {
|
|||||||
password: hasher.hash('admin'),
|
password: hasher.hash('admin'),
|
||||||
isAdmin: true,
|
isAdmin: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: nanoid(),
|
||||||
|
lastLogin: Date.now(),
|
||||||
|
username: 'demo',
|
||||||
|
password: hasher.hash('demo'),
|
||||||
|
isAdmin: true,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -84,3 +91,29 @@ export const removeUser = (userId) => {
|
|||||||
.value();
|
.value();
|
||||||
db.write();
|
db.write();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const handleDemoUser = () => {
|
||||||
|
if(!config.demoMode){
|
||||||
|
const user = db.chain.get('user').value();
|
||||||
|
db.chain.get('user').value();
|
||||||
|
db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
|
||||||
|
db.write();
|
||||||
|
}else {
|
||||||
|
const demoUser = db.chain
|
||||||
|
.get('user')
|
||||||
|
.filter((u) => u.username === 'demo')
|
||||||
|
.value();
|
||||||
|
if (demoUser == null || demoUser.length === 0) {
|
||||||
|
db.chain.get('user')
|
||||||
|
.value()
|
||||||
|
.push({
|
||||||
|
id: nanoid(),
|
||||||
|
username: 'demo',
|
||||||
|
password: hasher.hash('demo'),
|
||||||
|
isAdmin: true,
|
||||||
|
});
|
||||||
|
db.write();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,20 +1,16 @@
|
|||||||
import Mixpanel from 'mixpanel';
|
import Mixpanel from 'mixpanel';
|
||||||
import {getJobs} from '../storage/jobStorage.js';
|
import {getJobs} from '../storage/jobStorage.js';
|
||||||
|
|
||||||
import {config} from '../../utils.js';
|
import {config, inDevMode} from '../../utils.js';
|
||||||
|
|
||||||
|
const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
|
||||||
|
|
||||||
export const track = function () {
|
export const track = function () {
|
||||||
//only send tracking information if the user allowed to do so.
|
//only send tracking information if the user allowed to do so.
|
||||||
if (config.analyticsEnabled) {
|
if (config.analyticsEnabled && !inDevMode()) {
|
||||||
|
|
||||||
const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
|
|
||||||
|
|
||||||
const activeProvider = new Set();
|
const activeProvider = new Set();
|
||||||
const activeAdapter = new Set();
|
const activeAdapter = new Set();
|
||||||
const platform = process.platform;
|
|
||||||
const arch = process.arch;
|
|
||||||
const language = process.env.LANG || 'en';
|
|
||||||
const nodeVersion = process.version || 'N/A';
|
|
||||||
|
|
||||||
const jobs = getJobs();
|
const jobs = getJobs();
|
||||||
|
|
||||||
@@ -28,15 +24,45 @@ export const track = function () {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
mixpanelTracker.track('fredy_tracking', {
|
mixpanelTracker.track('fredy_tracking', enrichTrackingObject({
|
||||||
adapter: Array.from(activeAdapter),
|
adapter: Array.from(activeAdapter),
|
||||||
provider: Array.from(activeProvider),
|
provider: Array.from(activeProvider),
|
||||||
isDemo: config.demoMode,
|
}));
|
||||||
platform,
|
|
||||||
arch,
|
|
||||||
nodeVersion,
|
|
||||||
language
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Note, this will only be used when Fredy runs in demo mode
|
||||||
|
*/
|
||||||
|
export function trackDemoJobCreated(jobData) {
|
||||||
|
if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
|
||||||
|
mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Note, this will only be used when Fredy runs in demo mode
|
||||||
|
*/
|
||||||
|
export function trackDemoAccessed() {
|
||||||
|
if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
|
||||||
|
mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function enrichTrackingObject(trackingObject) {
|
||||||
|
const platform = process.platform;
|
||||||
|
const arch = process.arch;
|
||||||
|
const language = process.env.LANG || 'en';
|
||||||
|
const nodeVersion = process.version || 'N/A';
|
||||||
|
|
||||||
|
return {
|
||||||
|
...trackingObject,
|
||||||
|
isDemo: config.demoMode,
|
||||||
|
platform,
|
||||||
|
arch,
|
||||||
|
nodeVersion,
|
||||||
|
language
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ import {readFile} from 'fs/promises';
|
|||||||
import {createHash} from 'crypto';
|
import {createHash} from 'crypto';
|
||||||
import {DEFAULT_CONFIG} from './defaultConfig.js';
|
import {DEFAULT_CONFIG} from './defaultConfig.js';
|
||||||
|
|
||||||
|
function inDevMode(){
|
||||||
|
return process.env.NODE_ENV == null || process.env.NODE_ENV !== 'production';
|
||||||
|
}
|
||||||
|
|
||||||
function isOneOf(word, arr) {
|
function isOneOf(word, arr) {
|
||||||
if (arr == null || arr.length === 0) {
|
if (arr == null || arr.length === 0) {
|
||||||
return false;
|
return false;
|
||||||
@@ -72,6 +76,7 @@ export async function refreshConfig(){
|
|||||||
await refreshConfig();
|
await refreshConfig();
|
||||||
|
|
||||||
export {isOneOf};
|
export {isOneOf};
|
||||||
|
export {inDevMode};
|
||||||
export {nullOrEmpty};
|
export {nullOrEmpty};
|
||||||
export {duringWorkingHoursOrNotSet};
|
export {duringWorkingHoursOrNotSet};
|
||||||
export {getDirName};
|
export {getDirName};
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
{
|
{
|
||||||
"name": "fredy",
|
"name": "fredy",
|
||||||
"version": "10.3.0",
|
"version": "10.4.1",
|
||||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node index.js",
|
"start": "node prod.js",
|
||||||
"dev": "yarn && rm -rf ./ui/public/* && vite",
|
"dev": "yarn && rm -rf ./ui/public/* && vite",
|
||||||
"ui": "rm -rf ./ui/public/* && vite",
|
"ui": "rm -rf ./ui/public/* && vite",
|
||||||
"prod": "yarn && vite build --emptyOutDir",
|
"prod": "yarn && vite build --emptyOutDir",
|
||||||
|
|||||||
2
prod.js
Normal file
2
prod.js
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
process.env.NODE_ENV = 'production';
|
||||||
|
import('./index.js');
|
||||||
154
ui/src/App.jsx
154
ui/src/App.jsx
@@ -1,4 +1,4 @@
|
|||||||
import React, { useEffect } from 'react';
|
import React, {useEffect} from 'react';
|
||||||
|
|
||||||
import InsufficientPermission from './components/permission/InsufficientPermission';
|
import InsufficientPermission from './components/permission/InsufficientPermission';
|
||||||
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
|
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
|
||||||
@@ -6,94 +6,106 @@ import GeneralSettings from './views/generalSettings/GeneralSettings';
|
|||||||
import JobMutation from './views/jobs/mutation/JobMutation';
|
import JobMutation from './views/jobs/mutation/JobMutation';
|
||||||
import UserMutator from './views/user/mutation/UserMutator';
|
import UserMutator from './views/user/mutation/UserMutator';
|
||||||
import JobInsight from './views/jobs/insights/JobInsight.jsx';
|
import JobInsight from './views/jobs/insights/JobInsight.jsx';
|
||||||
import { useDispatch, useSelector } from 'react-redux';
|
import {useDispatch, useSelector} from 'react-redux';
|
||||||
import { Switch, Redirect } from 'react-router-dom';
|
import {Switch, Redirect} from 'react-router-dom';
|
||||||
import Logout from './components/logout/Logout';
|
import Logout from './components/logout/Logout';
|
||||||
import Logo from './components/logo/Logo';
|
import Logo from './components/logo/Logo';
|
||||||
import Menu from './components/menu/Menu';
|
import Menu from './components/menu/Menu';
|
||||||
import Login from './views/login/Login';
|
import Login from './views/login/Login';
|
||||||
import Users from './views/user/Users';
|
import Users from './views/user/Users';
|
||||||
import Jobs from './views/jobs/Jobs';
|
import Jobs from './views/jobs/Jobs';
|
||||||
import { Route } from 'react-router';
|
import {Route} from 'react-router';
|
||||||
|
|
||||||
import './App.less';
|
import './App.less';
|
||||||
import TrackingModal from './components/tracking/TrackingModal.jsx';
|
import TrackingModal from './components/tracking/TrackingModal.jsx';
|
||||||
|
import {Banner} from '@douyinfe/semi-ui';
|
||||||
|
|
||||||
export default function FredyApp() {
|
export default function FredyApp() {
|
||||||
const dispatch = useDispatch();
|
const dispatch = useDispatch();
|
||||||
const [loading, setLoading] = React.useState(true);
|
const [loading, setLoading] = React.useState(true);
|
||||||
const currentUser = useSelector((state) => state.user.currentUser);
|
const currentUser = useSelector((state) => state.user.currentUser);
|
||||||
const settings = useSelector((state) => state.generalSettings.settings);
|
const settings = useSelector((state) => state.generalSettings.settings);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
async function init() {
|
async function init() {
|
||||||
await dispatch.user.getCurrentUser();
|
await dispatch.user.getCurrentUser();
|
||||||
if (!needsLogin()) {
|
if (!needsLogin()) {
|
||||||
await dispatch.provider.getProvider();
|
await dispatch.provider.getProvider();
|
||||||
await dispatch.jobs.getJobs();
|
await dispatch.jobs.getJobs();
|
||||||
await dispatch.jobs.getProcessingTimes();
|
await dispatch.jobs.getProcessingTimes();
|
||||||
await dispatch.notificationAdapter.getAdapter();
|
await dispatch.notificationAdapter.getAdapter();
|
||||||
await dispatch.generalSettings.getGeneralSettings();
|
await dispatch.generalSettings.getGeneralSettings();
|
||||||
}
|
}
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
init();
|
init();
|
||||||
}, [currentUser?.userId]);
|
}, [currentUser?.userId]);
|
||||||
|
|
||||||
const needsLogin = () => {
|
const needsLogin = () => {
|
||||||
return currentUser == null || Object.keys(currentUser).length === 0;
|
return currentUser == null || Object.keys(currentUser).length === 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
const isAdmin = () => currentUser != null && currentUser.isAdmin;
|
const isAdmin = () => currentUser != null && currentUser.isAdmin;
|
||||||
|
|
||||||
const login = () => (
|
const login = () => (
|
||||||
<Switch>
|
|
||||||
<Route name="Login" path={'/login'} component={Login} />
|
|
||||||
<Redirect from="*" to={'/login'} />
|
|
||||||
</Switch>
|
|
||||||
);
|
|
||||||
|
|
||||||
return loading ? null : needsLogin() ? (
|
|
||||||
login()
|
|
||||||
) : (
|
|
||||||
<div className="app">
|
|
||||||
<div className="app__container">
|
|
||||||
<Logout />
|
|
||||||
<Logo width={190} white />
|
|
||||||
<Menu isAdmin={isAdmin()} />
|
|
||||||
{settings.analyticsEnabled === null && <TrackingModal/>}
|
|
||||||
<Switch>
|
<Switch>
|
||||||
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
|
<Route name="Login" path={'/login'} component={Login}/>
|
||||||
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
|
<Redirect from="*" to={'/login'}/>
|
||||||
<Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
|
|
||||||
<Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
|
|
||||||
<Route name="Job overview" path={'/jobs'} component={Jobs} />
|
|
||||||
<PermissionAwareRoute
|
|
||||||
name="Create new User"
|
|
||||||
path="/users/new"
|
|
||||||
component={<UserMutator />}
|
|
||||||
currentUser={currentUser}
|
|
||||||
/>
|
|
||||||
<PermissionAwareRoute
|
|
||||||
name="Edit a user"
|
|
||||||
path="/users/edit/:userId"
|
|
||||||
component={<UserMutator />}
|
|
||||||
currentUser={currentUser}
|
|
||||||
/>
|
|
||||||
<PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
|
|
||||||
<PermissionAwareRoute
|
|
||||||
name="General Settings"
|
|
||||||
path="/generalSettings"
|
|
||||||
component={<GeneralSettings />}
|
|
||||||
currentUser={currentUser}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<Redirect from="/" to={'/jobs'} />
|
|
||||||
</Switch>
|
</Switch>
|
||||||
</div>
|
);
|
||||||
</div>
|
|
||||||
);
|
return loading ? null : needsLogin() ? (
|
||||||
|
login()
|
||||||
|
) : (
|
||||||
|
<div className="app">
|
||||||
|
<div className="app__container">
|
||||||
|
<Logout/>
|
||||||
|
<Logo width={190} white/>
|
||||||
|
<Menu isAdmin={isAdmin()}/>
|
||||||
|
|
||||||
|
{settings.demoMode && (
|
||||||
|
<>
|
||||||
|
<Banner fullMode={true}
|
||||||
|
type="info"
|
||||||
|
bordered
|
||||||
|
closeIcon={null}
|
||||||
|
description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
|
||||||
|
/>
|
||||||
|
<br/>
|
||||||
|
</>)}
|
||||||
|
{(settings.analyticsEnabled === null && !settings.demoMode) && <TrackingModal/>}
|
||||||
|
<Switch>
|
||||||
|
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission}/>
|
||||||
|
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation}/>
|
||||||
|
<Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation}/>
|
||||||
|
<Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight}/>
|
||||||
|
<Route name="Job overview" path={'/jobs'} component={Jobs}/>
|
||||||
|
<PermissionAwareRoute
|
||||||
|
name="Create new User"
|
||||||
|
path="/users/new"
|
||||||
|
component={<UserMutator/>}
|
||||||
|
currentUser={currentUser}
|
||||||
|
/>
|
||||||
|
<PermissionAwareRoute
|
||||||
|
name="Edit a user"
|
||||||
|
path="/users/edit/:userId"
|
||||||
|
component={<UserMutator/>}
|
||||||
|
currentUser={currentUser}
|
||||||
|
/>
|
||||||
|
<PermissionAwareRoute name="Users" path="/users" component={<Users/>} currentUser={currentUser}/>
|
||||||
|
<PermissionAwareRoute
|
||||||
|
name="General Settings"
|
||||||
|
path="/generalSettings"
|
||||||
|
component={<GeneralSettings/>}
|
||||||
|
currentUser={currentUser}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Redirect from="/" to={'/jobs'}/>
|
||||||
|
</Switch>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
FredyApp.displayName = 'FredyApp';
|
FredyApp.displayName = 'FredyApp';
|
||||||
|
|||||||
24
ui/src/services/rematch/models/demoMode.js
Normal file
24
ui/src/services/rematch/models/demoMode.js
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { xhrGet } from '../../xhr';
|
||||||
|
export const demoMode = {
|
||||||
|
state: {
|
||||||
|
demoMode: false,
|
||||||
|
},
|
||||||
|
reducers: {
|
||||||
|
setDemoMode: (state, payload) => {
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
demoMode: payload.demoMode,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
effects: {
|
||||||
|
async getDemoMode() {
|
||||||
|
try {
|
||||||
|
const response = await xhrGet('/api/demo');
|
||||||
|
this.setDemoMode(response.json);
|
||||||
|
} catch (Exception) {
|
||||||
|
console.error('Error while trying to get resource for api/demo. Error:', Exception);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -5,6 +5,7 @@ import { provider } from './models/provider';
|
|||||||
import { createLogger } from 'redux-logger';
|
import { createLogger } from 'redux-logger';
|
||||||
import { jobs } from './models/jobs';
|
import { jobs } from './models/jobs';
|
||||||
import { user } from './models/user';
|
import { user } from './models/user';
|
||||||
|
import { demoMode } from './models/demoMode.js';
|
||||||
import { init } from '@rematch/core';
|
import { init } from '@rematch/core';
|
||||||
const middleware = [];
|
const middleware = [];
|
||||||
if (process.env.NODE_ENV === 'development') {
|
if (process.env.NODE_ENV === 'development') {
|
||||||
@@ -16,6 +17,7 @@ const store = init({
|
|||||||
models: {
|
models: {
|
||||||
notificationAdapter,
|
notificationAdapter,
|
||||||
generalSettings,
|
generalSettings,
|
||||||
|
demoMode,
|
||||||
provider,
|
provider,
|
||||||
jobs,
|
jobs,
|
||||||
user,
|
user,
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ const GeneralSettings = function GeneralSettings() {
|
|||||||
setWorkingHourFrom(settings?.workingHours?.from);
|
setWorkingHourFrom(settings?.workingHours?.from);
|
||||||
setWorkingHourTo(settings?.workingHours?.to);
|
setWorkingHourTo(settings?.workingHours?.to);
|
||||||
setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
|
setScrapingAntProxy(settings?.scrapingAnt?.proxy || 'datacenter');
|
||||||
setAnalyticsEnabled(settings?.analytics || false);
|
setAnalyticsEnabled(settings?.analyticsEnabled || false);
|
||||||
setDemoMode(settings?.demoMode || false);
|
setDemoMode(settings?.demoMode || false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,10 +109,17 @@ const GeneralSettings = function GeneralSettings() {
|
|||||||
});
|
});
|
||||||
} catch (exception) {
|
} catch (exception) {
|
||||||
console.error(exception);
|
console.error(exception);
|
||||||
throwMessage('Error while trying to store settings.', 'error');
|
if(exception?.json?.message != null){
|
||||||
|
throwMessage(exception.json.message, 'error');
|
||||||
|
}else {
|
||||||
|
throwMessage('Error while trying to store settings.', 'error');
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
throwMessage('Settings stored successfully.', 'success');
|
throwMessage('Settings stored successfully. We will reload your browser in 3 seconds.', 'success');
|
||||||
|
setTimeout(()=>{
|
||||||
|
location.reload();
|
||||||
|
}, 3000);
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -120,14 +127,6 @@ const GeneralSettings = function GeneralSettings() {
|
|||||||
{!loading && (
|
{!loading && (
|
||||||
<React.Fragment>
|
<React.Fragment>
|
||||||
<Headline text="General Settings"/>
|
<Headline text="General Settings"/>
|
||||||
<Banner
|
|
||||||
fullMode={false}
|
|
||||||
type="info"
|
|
||||||
closeIcon={null}
|
|
||||||
title={<div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>Info</div>}
|
|
||||||
style={{marginBottom: '1rem'}}
|
|
||||||
description="If you change any settings, you must restart Fredy afterwards."
|
|
||||||
/>
|
|
||||||
<div>
|
<div>
|
||||||
<SegmentPart
|
<SegmentPart
|
||||||
name="Interval"
|
name="Interval"
|
||||||
|
|||||||
@@ -1,88 +1,102 @@
|
|||||||
import React from 'react';
|
import React, {useEffect} from 'react';
|
||||||
|
|
||||||
import cityBackground from '../../assets/city_background.jpg';
|
import cityBackground from '../../assets/city_background.jpg';
|
||||||
import Logo from '../../components/logo/Logo';
|
import Logo from '../../components/logo/Logo';
|
||||||
import { xhrPost } from '../../services/xhr';
|
import {xhrPost} from '../../services/xhr';
|
||||||
import { useHistory } from 'react-router';
|
import {useHistory} from 'react-router';
|
||||||
import { useDispatch } from 'react-redux';
|
import {useDispatch, useSelector} from 'react-redux';
|
||||||
import { Input, Button, Banner } from '@douyinfe/semi-ui';
|
import {Input, Button, Banner} from '@douyinfe/semi-ui';
|
||||||
|
|
||||||
import './login.less';
|
import './login.less';
|
||||||
import { IconUser, IconLock } from '@douyinfe/semi-icons';
|
import {IconUser, IconLock} from '@douyinfe/semi-icons';
|
||||||
|
|
||||||
export default function Login() {
|
export default function Login() {
|
||||||
const dispatch = useDispatch();
|
const dispatch = useDispatch();
|
||||||
|
const [username, setUserName] = React.useState('');
|
||||||
|
const [password, setPassword] = React.useState('');
|
||||||
|
const [error, setError] = React.useState(null);
|
||||||
|
const demoMode = useSelector((state) => state.demoMode.demoMode || false);
|
||||||
|
const history = useHistory();
|
||||||
|
|
||||||
const [username, setUserName] = React.useState('');
|
useEffect(() => {
|
||||||
const [password, setPassword] = React.useState('');
|
async function init() {
|
||||||
const [error, setError] = React.useState(null);
|
await dispatch.demoMode.getDemoMode();
|
||||||
|
}
|
||||||
|
|
||||||
const history = useHistory();
|
init();
|
||||||
|
}, []);
|
||||||
|
|
||||||
const tryLogin = async () => {
|
const tryLogin = async () => {
|
||||||
if (username.length === 0 || password.length === 0) {
|
if (username.length === 0 || password.length === 0) {
|
||||||
setError('Username and password are mandatory.');
|
setError('Username and password are mandatory.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await xhrPost('/api/login', {
|
await xhrPost('/api/login', {
|
||||||
username,
|
username,
|
||||||
password,
|
password,
|
||||||
});
|
});
|
||||||
setError(null);
|
setError(null);
|
||||||
} catch (Exception) {
|
} catch (Exception) {
|
||||||
setError('Login not successful...');
|
setError('Login not successful...');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await dispatch.user.getCurrentUser();
|
await dispatch.user.getCurrentUser();
|
||||||
history.push('/jobs');
|
history.push('/jobs');
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="login">
|
<div className="login">
|
||||||
<div className="login__bgImage" style={{ background: `url("${cityBackground}")` }} />
|
<div className="login__bgImage" style={{background: `url("${cityBackground}")`}}/>
|
||||||
<Logo />
|
<Logo/>
|
||||||
<form>
|
<form>
|
||||||
<div className="login__loginWrapper">
|
<div className="login__loginWrapper">
|
||||||
{error && <Banner type="danger" closeIcon={null} description={error} />}
|
{error && <Banner type="danger" closeIcon={null} description={error}/>}
|
||||||
<Input
|
<Input
|
||||||
size="large"
|
size="large"
|
||||||
prefix={<IconUser />}
|
prefix={<IconUser/>}
|
||||||
placeholder="Username"
|
placeholder="Username"
|
||||||
value={username}
|
value={username}
|
||||||
showClear
|
showClear
|
||||||
style={{ marginTop: error ? '1rem' : '4rem' }}
|
style={{marginTop: error ? '1rem' : '4rem'}}
|
||||||
autofocus
|
autoFocus
|
||||||
onChange={(value) => setUserName(value)}
|
onChange={(value) => setUserName(value)}
|
||||||
onKeyPress={async (e) => {
|
onKeyPress={async (e) => {
|
||||||
if (e.key === 'Enter') {
|
if (e.key === 'Enter') {
|
||||||
await tryLogin();
|
await tryLogin();
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<Input
|
<Input
|
||||||
size="large"
|
size="large"
|
||||||
mode="password"
|
mode="password"
|
||||||
prefix={<IconLock />}
|
prefix={<IconLock/>}
|
||||||
value={password}
|
value={password}
|
||||||
placeholder="Password"
|
placeholder="Password"
|
||||||
style={{ marginTop: '2rem' }}
|
style={{marginTop: '2rem'}}
|
||||||
onChange={(value) => setPassword(value)}
|
onChange={(value) => setPassword(value)}
|
||||||
onKeyPress={async (e) => {
|
onKeyPress={async (e) => {
|
||||||
if (e.key === 'Enter') {
|
if (e.key === 'Enter') {
|
||||||
await tryLogin();
|
await tryLogin();
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<Button type="primary" onClick={tryLogin} theme="solid" style={{ marginTop: '3rem' }}>
|
<Button type="primary" onClick={tryLogin} theme="solid" style={{marginTop: '3rem'}}>
|
||||||
Login
|
Login
|
||||||
</Button>
|
</Button>
|
||||||
|
<br/>
|
||||||
|
{demoMode && <Banner fullMode={true}
|
||||||
|
type="info"
|
||||||
|
bordered
|
||||||
|
closeIcon={null}
|
||||||
|
description="This is the demo version of Fredy. Use 'demo' as both the username and password to log in."
|
||||||
|
/>}
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</form>
|
);
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Login.displayName = 'Login';
|
Login.displayName = 'Login';
|
||||||
|
|||||||
Reference in New Issue
Block a user