Compare commits

..

33 Commits
5.4.8 ... 7.1.0

Author SHA1 Message Date
Christian Kellner
f3cded7e5d fixing npe 2023-03-20 10:56:33 +01:00
Christian Kellner
d7c9c4bf76 Modernizing ui (#73)
Modernizing ui
2023-03-20 08:52:13 +01:00
Christian Kellner
2c5eceb0c1 Making Fredy an ESM project (#70)
Making Fredy an ESM project
2023-03-13 13:42:43 +01:00
weakmap@gmail.com
7d0ec72a0c fixing end of file issue by upgrading node-fetch 2023-02-14 19:46:49 +01:00
weakmap@gmail.com
faf020bd53 typo 2023-02-11 21:33:54 +01:00
weakmap@gmail.com
7df0754217 typo 2023-02-11 21:33:30 +01:00
weakmap@gmail.com
11a3e8771b adding jetbrains as sponsor 2023-02-11 21:32:28 +01:00
weakmap@gmail.com
af996d81c9 smaller design improvements 2022-12-20 13:39:25 +01:00
weakmap@gmail.com
8a5fbcdf71 moving to node-fetch coz axios is causing issues with scrapingAnt 2022-12-20 10:21:15 +01:00
weakmap@gmail.com
60bb75da57 fixing dependencies 2022-12-19 21:49:47 +01:00
weakmap@gmail.com
45411080ab adding forgotten dependencies 2022-12-19 21:48:14 +01:00
weakmap@gmail.com
4785cf797d moving to vite as build system 🎉 2022-12-19 21:44:10 +01:00
weakmap@gmail.com
e155e992d4 Merge branch 'master' of https://github.com/orangecoding/fredy 2022-12-19 21:15:12 +01:00
weakmap@gmail.com
3ce08a3f2e next build version 2022-12-19 21:15:01 +01:00
Christian Kellner
169655800b Update config.json 2022-12-19 21:11:15 +01:00
weakmap@gmail.com
baf57b3641 upgrading to react 18 2022-12-19 21:10:00 +01:00
Quoc Duong Bui
47e4230b39 Provider for immobilien.de (#65)
* Add provider for immobilien.de
2022-12-19 19:29:13 +01:00
Quoc Duong Bui
c5f4333878 Update README information (#66) 2022-12-19 19:27:13 +01:00
Christian Kellner
c99b78fb54 Update README.md 2022-12-14 15:13:00 +01:00
Christian Kellner
88e1e1d3a9 Update README.md 2022-12-14 15:12:18 +01:00
Christian Kellner
31174b3c85 Update README.md 2022-12-14 15:11:57 +01:00
weakmap@gmail.com
265ea58bab correct version 2022-12-11 20:08:40 +01:00
weakmap@gmail.com
ab5ee59d72 ugrading dependencies | fixing tests | supporting multiple provider of the same type 2022-12-11 20:07:18 +01:00
Christian Kellner
2062aa11a3 Scrapingant proxies (#59)
* preparing scraping ant proxies

* adding general settings for scraping ant proxy

* retrying with new ui settings
2022-06-13 08:10:30 +02:00
Christian Kellner
a4501007ff next release version 2022-06-10 14:19:41 +02:00
Christian Kellner
bc01806421 fixing telegram provider not respecting rate limits 2022-06-10 14:19:20 +02:00
Christian Kellner
bfba6d4bd9 next release version 2022-04-29 13:26:29 +02:00
Christian Kellner
676d48807a scraping ant retries 2022-04-29 13:22:39 +02:00
Christian Kellner
1a37773a40 Update package.json 2022-04-05 14:36:46 +02:00
Sven
67497d9828 added run-script-os to scripts to separate win use of set from unix use of export (#52)
Co-authored-by: Sven Simonsen <contact@svensimonsen.com>
2022-04-05 09:25:51 +02:00
weakmap@gmail.com
62ea296f3b Merge branch 'master' of https://github.com/orangecoding/fredy 2022-03-27 19:43:09 +02:00
weakmap@gmail.com
52dafcef97 improving ui / ux 2022-03-27 19:42:58 +02:00
Christian Kellner
a06d20ee53 Update README.md 2022-03-26 15:09:33 +01:00
137 changed files with 4429 additions and 7775 deletions

View File

@@ -1,12 +1,11 @@
module.exports = {
env: {
commonjs: true,
es6: true,
es2021: true,
node: true,
browser: true,
mocha: true,
},
parser: 'babel-eslint',
parser: '@babel/eslint-parser',
extends: ['eslint:recommended', 'prettier'],
plugins: ['react'],
globals: {
@@ -17,7 +16,6 @@ module.exports = {
fetch: true,
},
parserOptions: {
ecmaVersion: 2018,
sourceType: 'module',
},
rules: {
@@ -205,10 +203,6 @@ module.exports = {
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
'react/self-closing-comp': 'warn',
// Enforce spaces before the closing bracket of self-closing JSX elements
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
'react/jsx-space-before-closing': ['warn', 'always'],
// Enforce component methods order
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
'react/sort-comp': 'off',
@@ -239,7 +233,7 @@ module.exports = {
// only .jsx files may have JSX
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
// prevent accidental JS comments from being injected into JSX as text
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
@@ -284,15 +278,5 @@ module.exports = {
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
'react/no-children-prop': 'warn',
// Validate whitespace in and around the JSX opening and closing brackets
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
'react/jsx-tag-spacing': [
'warn',
{
closingSlash: 'never',
beforeSelfClosing: 'always',
afterOpening: 'never',
},
],
},
};

View File

@@ -1,3 +1,16 @@
Newer release changelog see https://github.com/orangecoding/fredy/releases
------------
###### [V5.5.0]
- Upgrading dependencies
- fixing provider
- allow multiple instances of 1 provider
- __BREAKING__: Minimum node version is now 16
###### [V5.4.6]
- Adding Instana node.js monitoring
-
###### [V5.4.5]
- Adding Instana node.js monitoring

View File

@@ -8,9 +8,16 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)
If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
<img src="https://github.com/orangecoding/fredy/blob/master/doc/jetbrains.png" width="200">
_Fredy_ is supported by JetBrains under Open Source Support Program
## Usage
- Make sure to use Node.js 12 or above
- Make sure to use Node.js 16 or above
- Run the following commands:
```ssh
yarn (or npm install)
@@ -20,11 +27,11 @@ yarn run start
_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
<p align="center">
<img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
<img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot1.png" width="30%">
&nbsp; &nbsp; &nbsp; &nbsp;
<img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
<img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_2.png" width="30%">
&nbsp; &nbsp; &nbsp; &nbsp;
<img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
<img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
</p>
<p align="center">
@@ -33,20 +40,20 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
## Understanding the fundamentals
There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
#### Adapter
_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few examples. Those services are called adapters within _Fredy_. When creating a new job, you can choose one or more adapters.
An adapter contains the URL that points to the search results for the respective service. If you go to immonet.de and search for something, the displayed URL in the browser is what the adapter needs to do its magic.
#### Provider
_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few examples. Those services are called providers within _Fredy_. When creating a new job, you can choose one or more providers.
A provider contains the URL that points to the search results for the respective service. If you go to immonet.de and search for something, the displayed URL in the browser is what the provider needs to do its magic.
**It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
#### Provider
_Fredy_ supports multiple providers, such as Slack, SendGrid, Telegram etc. A search job can have as many providers as supported by _Fredy_. Each provider needs different configuration values, which you have to provide when using them. A provider dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
#### Adapter
_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
#### Jobs
A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
## Creating your first job
To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self-explanatory, however there is one important thing.
When configuring adapters, before copying the URL from your browser, make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
When configuring providers, before copying the URL from your browser, make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
## User management
As an administrator, you can create, edit and remove users from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, their jobs will also be removed.
@@ -54,11 +61,16 @@ As an administrator, you can create, edit and remove users from _Fredy_. Be care
# Development
### Running Fredy in development mode
To run _Fredy_ in development mode, you need to run the backend & frontend separately. Run the backend in your favorite IDE, the frontend can be started from the terminal.
To run _Fredy_ in development mode, you need to run the backend & frontend separately.
Start the backend with:
```shell
yarn run start
```
For the frontend, run:
```shell
yarn run dev
```
You should now be able to access _Fredy_ from your browser. Go to `http://localhost:9000`.
You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
### Running Tests
To run the tests, run
@@ -81,12 +93,6 @@ If you need more than the 1000 API calls allowed per month, I'd suggest opting f
See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
### Monitoring
_Fredy_ can be monitored by [Instana](https://www.instana.com). If you are interested, sign up for a free trial. This is totally optional of course :)
If you want to use Instana to monitor _Fredy_, please change the variable `INSTANA_MONITORING` in the `.env` file to `true`.
If you want to know more, head over to the [Instana docs](https://www.ibm.com/docs/en/obi/current?topic=technologies-monitoring-nodejs).
# Docker
Use the Dockerfile in this repository to build an image.

View File

@@ -1 +1 @@
{"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}
{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}

BIN
doc/jetbrains.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 150 KiB

BIN
doc/screenshot1.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 121 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 279 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 380 KiB

BIN
doc/screenshot_2.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 323 KiB

BIN
doc/screenshot_3.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 93 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 202 KiB

View File

@@ -4,14 +4,13 @@
<meta charset="UTF-8"
name="viewport"
content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css">
<meta name="google" content="notranslate">
<title>Fredy</title>
</head>
<body>
<body theme-mode="dark">
<div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
</body>
<script src="fredy.bundle.js"></script>
<script type="module" src="/ui/src/Index.jsx"></script>
</html>

View File

@@ -1,63 +1,41 @@
const fs = require('fs');
import fs from 'fs';
import { config } from './lib/utils.js';
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
import * as jobStorage from './lib/services/storage/jobStorage.js';
import FredyRuntime from './lib/FredyRuntime.js';
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
import './lib/api/api.js';
//if db folder does not exist, ensure to create it before loading anything else
if (!fs.existsSync('./db')) {
fs.mkdirSync('./db');
}
const path = './lib/provider';
const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
const config = require('./conf/config.json');
const similarityCache = require('./lib/services/similarity-check/similarityCache');
const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
const jobStorage = require('./lib/services/storage/jobStorage');
const FredyRuntime = require('./lib/FredyRuntime');
const { duringWorkingHoursOrNotSet } = require('./lib/utils');
//starting the api service
require('./lib/api/api');
//assuming interval is always in minutes
const INTERVAL = config.interval * 60 * 1000;
/* eslint-disable no-console */
console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
/* eslint-enable no-console */
const fetchedProvider = await Promise.all(
provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
);
setInterval(
(function exec() {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
if (isDuringWorkingHoursOrNotSet) {
config.lastRun = Date.now();
jobStorage
.getJobs()
.filter((job) => job.enabled)
.forEach((job) => {
const providerIds = job.provider.map((provider) => provider.id);
provider
.filter((provider) => provider.endsWith('.js'))
.map((pro) => require(`${path}/${pro}`))
.filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
.forEach(async (pro) => {
const providerId = pro.metaInformation.id;
if (providerId == null || providerId.length === 0) {
throw new Error('Provider id must not be empty. => ' + pro);
}
const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
if (providerConfig == null) {
throw new Error(`Provider Config for provider with id ${providerId} not found.`);
}
pro.init(providerConfig, job.blacklist);
await new FredyRuntime(
pro.config,
job.notificationAdapter,
providerId,
job.id,
similarityCache
).execute();
job.provider
.filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
.forEach(async (prov) => {
const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
pro.init(prov, job.blacklist);
await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
setLastJobExecution(job.id);
});
});

View File

@@ -1,11 +1,9 @@
const { NoNewListingsWarning } = require('./errors');
const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
const notify = require('./notification/notify');
const xray = require('./services/scraper');
const scrapingAnt = require('./services/scrapingAnt');
const urlModifier = require('./services/queryStringMutator');
import { NoNewListingsWarning } from './errors.js';
import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
import * as notify from './notification/notify.js';
import xray from './services/scraper.js';
import * as scrapingAnt from './services/scrapingAnt.js';
import urlModifier from './services/queryStringMutator.js';
class FredyRuntime {
/**
*
@@ -22,7 +20,6 @@ class FredyRuntime {
this._jobKey = jobKey;
this._similarityCache = similarityCache;
}
execute() {
return (
//modify the url to make sure search order is correctly set
@@ -45,7 +42,6 @@ class FredyRuntime {
.catch(this._handleError.bind(this))
);
}
_getListings(url) {
return new Promise((resolve, reject) => {
const id = this._providerId;
@@ -87,25 +83,19 @@ class FredyRuntime {
}
});
}
_normalize(listings) {
return listings.map(this._providerConfig.normalize);
}
_filter(listings) {
return listings.filter(this._providerConfig.filter);
}
_findNew(listings) {
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
if (newListings.length === 0) {
throw new NoNewListingsWarning();
}
return newListings;
}
_notify(newListings) {
if (newListings.length === 0) {
throw new NoNewListingsWarning();
@@ -113,7 +103,6 @@ class FredyRuntime {
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
return Promise.all(sendNotifications).then(() => newListings);
}
_save(newListings) {
const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
newListings.forEach((listing) => {
@@ -122,7 +111,6 @@ class FredyRuntime {
setKnownListings(this._jobKey, this._providerId, currentListings);
return newListings;
}
_filterBySimilarListings(listings) {
const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
@@ -136,10 +124,8 @@ class FredyRuntime {
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
return filteredList;
}
_handleError(err) {
if (err.name !== 'NoNewListingsWarning') console.error(err);
}
}
module.exports = FredyRuntime;
export default FredyRuntime;

View File

@@ -1,44 +1,36 @@
const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
const { analyticsRouter } = require('./routes/analyticsRouter');
const { providerRouter } = require('./routes/providerRouter');
const { loginRouter } = require('./routes/loginRoute');
const config = require('../../conf/config.json');
const { userRouter } = require('./routes/userRoute');
const { jobRouter } = require('./routes/jobRouter');
const bodyParser = require('body-parser');
const service = require('restana')();
const files = require('serve-static');
const path = require('path');
const staticService = files(path.join(__dirname, '../../ui/public'));
import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
import { analyticsRouter } from './routes/analyticsRouter.js';
import { providerRouter } from './routes/providerRouter.js';
import { loginRouter } from './routes/loginRoute.js';
import { config } from '../utils.js';
import { userRouter } from './routes/userRoute.js';
import { jobRouter } from './routes/jobRouter.js';
import bodyParser from 'body-parser';
import restana from 'restana';
import files from 'serve-static';
import path from 'path';
import { getDirName } from '../utils.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../../ui/public'));
const PORT = config.port || 9998;
service.use(bodyParser.json());
service.use(cookieSession());
service.use(staticService);
service.use('/api/admin', authInterceptor());
service.use('/api/jobs', authInterceptor());
// /admin can only be accessed when user is having admin permissions
service.use('/api/admin', adminInterceptor());
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
service.use('/api/admin/generalSettings', generalSettingsRouter);
service.use('/api/jobs/provider', providerRouter);
service.use('/api/jobs/insights', analyticsRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
/* eslint-disable no-console */
service.start(PORT).then(() => {
console.info(`Started API service on port ${PORT}`);
});
/* eslint-enable no-console */

View File

@@ -1,12 +1,10 @@
const service = require('restana')();
import restana from 'restana';
import * as listingStorage from '../../services/storage/listingsStorage.js';
const service = restana();
const analyticsRouter = service.newRouter();
const listingStorage = require('../../services/storage/listingsStorage');
analyticsRouter.get('/:jobId', async (req, res) => {
const { jobId } = req.params;
res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
res.send();
});
exports.analyticsRouter = analyticsRouter;
export { analyticsRouter };

View File

@@ -1,18 +1,16 @@
const service = require('restana')();
import restana from 'restana';
import { config, getDirName } from '../../utils.js';
import fs from 'fs';
const service = restana();
const generalSettingsRouter = service.newRouter();
const config = require('../../../conf/config.json');
const fs = require('fs');
generalSettingsRouter.get('/', async (req, res) => {
res.body = Object.assign({}, config);
res.send();
});
generalSettingsRouter.post('/', async (req, res) => {
const settings = req.body;
try {
fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
} catch (err) {
console.error(err);
res.send(new Error('Error while trying to write settings.'));
@@ -20,5 +18,4 @@ generalSettingsRouter.post('/', async (req, res) => {
}
res.send();
});
exports.generalSettingsRouter = generalSettingsRouter;
export { generalSettingsRouter };

View File

@@ -1,12 +1,12 @@
const service = require('restana')();
import restana from 'restana';
import fetch from 'node-fetch';
import * as jobStorage from '../../services/storage/jobStorage.js';
import * as userStorage from '../../services/storage/userStorage.js';
import * as immoscoutProvider from '../../provider/immoscout.js';
import { config } from '../../utils.js';
import { isAdmin } from '../security.js';
const service = restana();
const jobRouter = service.newRouter();
const axios = require('axios');
const jobStorage = require('../../services/storage/jobStorage');
const userStorage = require('../../services/storage/userStorage');
const immoscoutProvider = require('../../provider/immoscout');
const config = require('../../../conf/config.json');
const { isAdmin } = require('../security');
function doesJobBelongsToUser(job, req) {
const userId = req.session.currentUser;
if (userId == null) {
@@ -16,42 +16,31 @@ function doesJobBelongsToUser(job, req) {
if (user == null) {
return false;
}
return user.isAdmin || job.userId === job.userId;
}
jobRouter.get('/', async (req, res) => {
const isUserAdmin = isAdmin(req);
//show only the jobs which belongs to the user (or all of the user is an admin)
res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
res.send();
});
jobRouter.get('/processingTimes', async (req, res) => {
let scrapingAntData = null;
if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
try {
const result = await axios({
url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
});
scrapingAntData = result.data;
const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
scrapingAntData = await response.json();
} catch (Exception) {
console.error('Could not query plan data from scraping ant.', Exception);
}
}
res.body = {
interval: config.interval,
lastRun: config.lastRun || null,
scrapingAntData,
};
res.send();
});
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
if (
@@ -79,7 +68,6 @@ jobRouter.post('/', async (req, res) => {
}
res.send();
});
jobRouter.delete('', async (req, res) => {
const { jobId } = req.body;
try {
@@ -95,7 +83,6 @@ jobRouter.delete('', async (req, res) => {
}
res.send();
});
jobRouter.put('/:jobId/status', async (req, res) => {
const { status } = req.body;
const { jobId } = req.params;
@@ -115,5 +102,4 @@ jobRouter.put('/:jobId/status', async (req, res) => {
}
res.send();
});
exports.jobRouter = jobRouter;
export { jobRouter };

View File

@@ -1,8 +1,8 @@
const service = require('restana')();
import restana from 'restana';
import * as userStorage from '../../services/storage/userStorage.js';
import * as hasher from '../../services/security/hash.js';
const service = restana();
const loginRouter = service.newRouter();
const userStorage = require('../../services/storage/userStorage');
const hasher = require('../../services/security/hash');
loginRouter.get('/user', async (req, res) => {
const currentUserId = req.session.currentUser;
const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
@@ -16,17 +16,13 @@ loginRouter.get('/user', async (req, res) => {
}
res.send();
});
loginRouter.post('/', async (req, res) => {
const { username, password } = req.body;
const user = userStorage.getUsers(true).find((user) => user.username === username);
if (user == null) {
res.send(401);
return;
}
if (user.password === hasher.hash(password)) {
req.session.currentUser = user.id;
userStorage.setLastLoginToNow({ userId: user.id });
@@ -35,13 +31,10 @@ loginRouter.post('/', async (req, res) => {
} else {
console.error(`User ${username} tried to login, but password was wrong.`);
}
res.send(401);
});
loginRouter.post('/logout', async (req, res) => {
req.session = null;
res.send(200);
});
exports.loginRouter = loginRouter;
export { loginRouter };

View File

@@ -1,13 +1,13 @@
const fs = require('fs');
const service = require('restana')();
import fs from 'fs';
import restana from 'restana';
const service = restana();
const notificationAdapterRouter = service.newRouter();
const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
const notificationAdapter = notificationAdapterList.map((pro) => {
return require(`../../notification/adapter/${pro}`);
});
const notificationAdapter = await Promise.all(
notificationAdapterList.map(async (pro) => {
return await import(`../../notification/adapter/${pro}`);
})
);
notificationAdapterRouter.post('/try', async (req, res) => {
const { id, fields } = req.body;
const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
@@ -24,7 +24,6 @@ notificationAdapterRouter.post('/try', async (req, res) => {
enabled: true,
id,
});
try {
await adapter.send({
serviceName: 'TestCall',
@@ -40,16 +39,13 @@ notificationAdapterRouter.post('/try', async (req, res) => {
notificationConfig,
jobKey: 'TestJob',
});
res.send();
} catch (Exception) {
res.send(new Error(Exception));
}
});
notificationAdapterRouter.get('/', async (req, res) => {
res.body = notificationAdapter.map((adapter) => adapter.config);
res.send();
});
exports.notificationAdapterRouter = notificationAdapterRouter;
export { notificationAdapterRouter };

View File

@@ -1,16 +1,15 @@
const fs = require('fs');
const service = require('restana')();
import fs from 'fs';
import restana from 'restana';
const service = restana();
const providerRouter = service.newRouter();
const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
const provider = providerList.map((pro) => {
return require(`../../provider/${pro}`).metaInformation;
});
const provider = await Promise.all(
providerList.map(async (pro) => {
return await import(`../../provider/${pro}`);
})
);
providerRouter.get('/', async (req, res) => {
res.body = provider;
res.body = provider.map((p) => p.metaInformation);
res.send();
});
exports.providerRouter = providerRouter;
export { providerRouter };

View File

@@ -1,33 +1,27 @@
const service = require('restana')();
import restana from 'restana';
import * as userStorage from '../../services/storage/userStorage.js';
import * as jobStorage from '../../services/storage/jobStorage.js';
const service = restana();
const userRouter = service.newRouter();
const userStorage = require('../../services/storage/userStorage');
const jobStorage = require('../../services/storage/jobStorage');
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
}
function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
return req.session.currentUser === userIdToBeRemoved;
}
const nullOrEmpty = (str) => str == null || str.length === 0;
userRouter.get('/', async (req, res) => {
res.body = userStorage.getUsers(false);
res.send();
});
userRouter.get('/:userId', async (req, res) => {
const { userId } = req.params;
res.body = userStorage.getUser(userId);
res.send();
});
userRouter.delete('/', async (req, res) => {
const { userId } = req.body;
const allUser = userStorage.getUsers(false);
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
return;
@@ -36,14 +30,11 @@ userRouter.delete('/', async (req, res) => {
res.send(new Error('You are trying to remove yourself. This is prohibited.'));
return;
}
//TODO: Remove also analytics
jobStorage.removeJobsByUserId(userId);
userStorage.removeUser(userId);
res.send();
});
userRouter.post('/', async (req, res) => {
const { username, password, password2, isAdmin, userId } = req.body;
if (password !== password2) {
@@ -55,22 +46,18 @@ userRouter.post('/', async (req, res) => {
return;
}
const allUser = userStorage.getUsers(false);
if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
res.send(
new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
);
return;
}
userStorage.upsertUser({
userId,
username,
password,
isAdmin,
});
res.send();
});
exports.userRouter = userRouter;
export { userRouter };

View File

@@ -1,15 +1,12 @@
const userStorage = require('../services/storage/userStorage');
const cookieSession = require('cookie-session');
const { nanoid } = require('nanoid');
import * as userStorage from '../services/storage/userStorage.js';
import cookieSession from 'cookie-session';
import { nanoid } from 'nanoid';
const unauthorized = (res) => {
return res.send(401);
};
const isUnauthorized = (req) => {
return req.session.currentUser == null;
};
const isAdmin = (req) => {
if (!isUnauthorized(req)) {
const user = userStorage.getUser(req.session.currentUser);
@@ -17,7 +14,6 @@ const isAdmin = (req) => {
}
return false;
};
const authInterceptor = () => {
return (req, res, next) => {
if (isUnauthorized(req)) {
@@ -27,7 +23,6 @@ const authInterceptor = () => {
}
};
};
const adminInterceptor = () => {
return (req, res, next) => {
if (!isAdmin(req)) {
@@ -37,8 +32,7 @@ const adminInterceptor = () => {
}
};
};
exports.cookieSession = (userId) => {
const cookieSession$0 = (userId) => {
return cookieSession({
name: 'fredy-admin-session',
keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
@@ -46,8 +40,8 @@ exports.cookieSession = (userId) => {
maxAge: 8 * 60 * 60 * 1000, // 8 hours
});
};
exports.adminInterceptor = adminInterceptor;
exports.authInterceptor = authInterceptor;
exports.isUnauthorized = isUnauthorized;
exports.isAdmin = isAdmin;
export { cookieSession$0 as cookieSession };
export { adminInterceptor };
export { authInterceptor };
export { isUnauthorized };
export { isAdmin };

View File

@@ -9,7 +9,8 @@ class ExtendableError extends Error {
}
}
}
class NoNewListingsWarning extends ExtendableError {}
module.exports = { NoNewListingsWarning };
export { NoNewListingsWarning };
export default {
NoNewListingsWarning,
};

View File

@@ -1,19 +1,12 @@
const { markdown2Html } = require('../../services/markdown');
import { markdown2Html } from '../../services/markdown.js';
/**
* simply prints out the found data to the console
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param jobKey name of the current job that is being executed
*/
exports.send = ({ serviceName, newListings, jobKey }) => {
export const send = ({ serviceName, newListings, jobKey }) => {
/* eslint-disable no-console */
return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
/* eslint-enable no-console */
};
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'console',
name: 'Console',
description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
config: {},

View File

@@ -1,33 +1,22 @@
const mailjet = require('node-mailjet');
const path = require('path');
const fs = require('fs');
const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/template.hbs'), 'utf8');
const Handlebars = require('handlebars');
import mailjet from 'node-mailjet';
import path from 'path';
import fs from 'fs';
import Handlebars from 'handlebars';
import { markdown2Html } from '../../services/markdown.js';
import { getDirName } from '../../utils.js';
const __dirname = getDirName();
const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
const emailTemplate = Handlebars.compile(template);
const { markdown2Html } = require('../../services/markdown');
/**
* sends a new listing using MailJet
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param notificationConfig config of this notification adapter
* * @param jobKey name of the current job that is being executed
* @returns {Promise<Chat.PostMessage.Response> | void}
*/
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
(adapter) => adapter.id === 'mailJet'
).fields;
const to = receiver
.trim()
.split(',')
.map((r) => ({
Email: r.trim(),
}));
return mailjet
.connect(apiPublicKey, apiPrivateKey)
.post('send', { version: 'v3.1' })
@@ -49,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
],
});
};
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'mailjet',
name: 'MailJet',
description: 'MailJet is being used to send new listings via mail.',
readme: markdown2Html('lib/notification/adapter/mailJet.md'),

View File

@@ -1,39 +1,26 @@
const { markdown2Html } = require('../../services/markdown');
const { getJob } = require('../../services/storage/jobStorage');
const axios = require('axios');
/**
* sends new listings to mattermost
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param notificationConfig config of this notification adapter
* @param jobKey name of the current job that is being executed
* @returns {Promise<Void> | void}
*/
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
import { markdown2Html } from '../../services/markdown.js';
import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
message += newListings.map(
(o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
);
return axios.post(`${webhook}`, {
channel: channel,
text: message,
return fetch(webhook, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: {
channel: channel,
text: message,
},
});
};
/**
* exported config is being used in the frontend to generate the fields
* incoming values will be the keys (and values) of the fields
*
*/
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'mattermost',
name: 'Mattermost',
readme: markdown2Html('lib/notification/adapter/mattermost.md'),
description: 'Fredy will send new listings to your mattermost team chat.',

View File

@@ -1,15 +1,6 @@
const sgMail = require('@sendgrid/mail');
const { markdown2Html } = require('../../services/markdown');
/**
* sends a new listing using SendGrid
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param notificationConfig config of this notification adapter
* @param jobKey name of the current job that is being executed
* @returns {Promise<Chat.PostMessage.Response> | void}
*/
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
import sgMail from '@sendgrid/mail';
import { markdown2Html } from '../../services/markdown.js';
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
sgMail.setApiKey(apiKey);
const msg = {
@@ -28,9 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
};
return sgMail.send(msg);
};
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'sendgrid',
name: 'SendGrid',
description: 'SendGrid is being used to send new listings via mail.',
readme: markdown2Html('lib/notification/adapter/sendGrid.md'),

View File

@@ -1,16 +1,7 @@
const Slack = require('slack');
import Slack from 'slack';
import { markdown2Html } from '../../services/markdown.js';
const msg = Slack.chat.postMessage;
const { markdown2Html } = require('../../services/markdown');
/**
* sends a new listing to slack
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param notificationConfig config of this notification adapter
* * @param jobKey name of the current job that is being executed
* @returns {Promise<Chat.PostMessage.Response> | void}
*/
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
return newListings.map((payload) =>
msg({
@@ -47,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
})
);
};
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'slack',
name: 'Slack',
readme: markdown2Html('lib/notification/adapter/slack.md'),
description: 'Fredy will send new listings to the slack channel of your choice..',

View File

@@ -1,13 +1,6 @@
const { markdown2Html } = require('../../services/markdown');
const Database = require('better-sqlite3');
/**
* Stores data in a sqlite db in order to use the search results for later analytics
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param jobKey name of the current job that is being executed
*/
exports.send = ({ serviceName, newListings, jobKey }) => {
import { markdown2Html } from '../../services/markdown.js';
import Database from 'better-sqlite3';
export const send = ({ serviceName, newListings, jobKey }) => {
const db = new Database('db/listings.db');
const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
@@ -23,9 +16,8 @@ exports.send = ({ serviceName, newListings, jobKey }) => {
});
return Promise.resolve();
};
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'sqlite',
name: 'Sqlite',
description: 'This adapter stores listings in a local sqlite3 database.',
config: {},

View File

@@ -1,7 +1,8 @@
const { markdown2Html } = require('../../services/markdown');
const { getJob } = require('../../services/storage/jobStorage');
const axios = require('axios');
import { markdown2Html } from '../../services/markdown.js';
import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch';
const MAX_ENTITIES_PER_CHUNK = 8;
const RATE_LIMIT_INTERVAL = 1010;
/**
* splitting an array into chunks because Telegram only allows for messages up to
* 4096 chars, thus we have to split messages into chunks
@@ -14,54 +15,51 @@ const arrayChunks = (inputArray, perChunk) =>
all[ch] = [].concat(all[ch] || [], one);
return all;
}, []);
/**
* sends new listings to telegram
* @param serviceName e.g immowelt
* @param newListings an array with newly found listings
* @param notificationConfig config of this notification adapter
* @param jobKey name of the current job that is being executed
* @returns {Promise<Void> | void}
*/
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
function shorten(str, len = 30) {
return str.length > len ? str.substring(0, len) + '...' : str;
}
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name;
//we have to split messages into chunk, because otherwise messages are going to become too big and will fail
const chunks = arrayChunks(newListings, 3);
const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
const promises = chunks.map((chunk) => {
let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
message += chunk.map(
(o) =>
`<a href="${o.link}"><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
`<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
[o.address, o.price, o.size].join(' | ') +
'\n\n'
);
return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
chat_id: chatId,
text: message,
parse_mode: 'HTML',
disable_web_page_preview: true,
/**
* This is to not break the rate limit. It is to only send 1 message per second
*/
return new Promise((resolve, reject) => {
setTimeout(() => {
fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
method: 'post',
body: JSON.stringify({
chat_id: chatId,
text: message,
parse_mode: 'HTML',
disable_web_page_preview: true,
}),
headers: { 'Content-Type': 'application/json' },
})
.then(() => {
resolve();
})
.catch(() => {
reject();
});
}, RATE_LIMIT_INTERVAL);
});
});
return Promise.all(promises);
};
function shorten(str, len = 30) {
return str.length > len ? str.substring(0, len) + '...' : str;
}
/**
* exported config is being used in the frontend to generate the fields
* incoming values will be the keys (and values) of the fields
*
*/
exports.config = {
id: __filename.slice(__dirname.length + 1, -3),
export const config = {
id: 'telegram',
name: 'Telegram',
readme: markdown2Html('lib/notification/adapter/telegram.md'),
description: 'Fredy will send new listings to your mobile, using Telegram.',

View File

@@ -1,24 +1,24 @@
const fs = require('fs');
import fs from 'fs';
const path = './adapter';
/** Read every integration existing in ./adapter **/
const adapter = fs
.readdirSync('./lib/notification/adapter')
.filter((file) => file.endsWith('.js'))
.map((integPath) => require(`${path}/${integPath}`));
const adapter = await Promise.all(
fs
.readdirSync('./lib/notification/adapter')
.filter((file) => file.endsWith('.js'))
.map(async (integPath) => await import(`${path}/${integPath}`))
);
if (adapter.length === 0) {
throw new Error('Please specify at least one notification provider');
}
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
const findAdapter = (notificationAdapter) => {
return adapter.find((a) => a.config.id === notificationAdapter.id);
};
export const send = (serviceName, newListings, notificationConfig, jobKey) => {
//this is not being used in tests, therefore adapter are always set
return notificationConfig
.filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
.map((notificationAdapter) => findAdapter(notificationAdapter))
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
};
const findAdapter = (notificationAdapter) => {
return adapter.find((a) => a.config.id === notificationAdapter.id);
};

View File

@@ -1,24 +1,18 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
if (o.rooms != null) {
size += ` / / ${o.rooms.trim()}`;
}
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
return Object.assign(o, { size, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.tabelle',
@@ -34,17 +28,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: '1a Immobilien',
baseUrl: 'https://www.1a-immobilienmarkt.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'einsAImmobilien',
};
exports.config = config;
export { config };

View File

@@ -0,0 +1,49 @@
import utils from '../utils.js';
let appliedBlackList = [];
function shortenLink(link) {
return link.substring(0, link.indexOf('?'));
}
function parseId(shortenedLink) {
return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
}
function normalize(o) {
const id = parseId(shortenLink(o.link));
const size = o.size || 'N/A m²';
const price = o.price || 'N/A €';
const title = o.title || 'No title available';
const address = o.address || 'No address available';
const link = shortenLink(o.link);
return Object.assign(o, { id, price, size, title, address, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.estates_list .list_immo a._ref',
sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
crawlFields: {
price: '.list_entry .immo_preis .label_info',
size: '.list_entry .flaeche .label_info | removeNewline | trim',
title: '.list_entry .part_text h3 span',
description: '.list_entry .description | trim',
link: '@href',
address: '.list_entry .place',
},
paginate: '.list_immo .blocknav .blocknav_list li.next a@href',
normalize: normalize,
filter: applyBlacklist,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
export const metaInformation = {
name: 'Immobilien.de',
baseUrl: 'https://www.immobilien.de/',
id: 'immobilienDe',
};
export { config };

View File

@@ -1,7 +1,5 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
@@ -13,14 +11,11 @@ function normalize(o) {
const link = `https://www.immonet.de/angebot/${id}`;
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '#result-list-stage .item',
@@ -36,17 +31,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immonet',
baseUrl: 'https://www.immonet.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immonet',
};
exports.config = config;
export { config };

View File

@@ -1,22 +1,17 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function normalize(o) {
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
return Object.assign(o, { title, address, link });
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
crawlContainer: '#resultListItems li.result-list__listing',
@@ -33,17 +28,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immoscout',
baseUrl: 'https://www.immobilienscout24.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immoscout',
};
exports.config = config;
export { config };

View File

@@ -1,7 +1,5 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
const size = o.size || 'N/A m²';
@@ -12,14 +10,11 @@ function normalize(o) {
const description = o.description;
return Object.assign(o, { id, address, price, size, title, link, description });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.js-serp-item',
@@ -36,17 +31,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immo Südwest Presse',
baseUrl: 'https://immo.swp.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immoswp',
};
exports.config = config;
export { config };

View File

@@ -1,18 +1,13 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: "div[class^='EstateItem-']",
@@ -29,16 +24,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immowelt',
baseUrl: 'https://www.immowelt.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immowelt',
};
exports.config = config;
export { config };

View File

@@ -1,23 +1,17 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
let appliedBlacklistedDistricts = [];
function normalize(o) {
const size = o.size || '--- m²';
return Object.assign(o, { size });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const isBlacklistedDistrict =
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '#srchrslt-adtable .ad-listitem ',
@@ -36,18 +30,15 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.metaInformation = {
export const metaInformation = {
name: 'Ebay Kleinanzeigen',
baseUrl: 'https://www.ebay-kleinanzeigen.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'kleinanzeigen',
};
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlacklistedDistricts = blacklistedDistricts || [];
appliedBlackList = blacklist || [];
};
exports.config = config;
export { config };

View File

@@ -1,15 +1,11 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
crawlContainer: '.nbk-container >div article',
@@ -25,17 +21,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Neubau Kompass',
baseUrl: 'https://www.neubaukompass.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'neubauKompass',
};
exports.config = config;
export { config };

View File

@@ -1,18 +1,13 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '#main_column .wgg_card',
@@ -28,17 +23,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Wg gesucht',
baseUrl: 'https://www.wg-gesucht.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'wgGesucht',
};
exports.config = config;
export { config };

View File

@@ -1,6 +1,6 @@
const markdown = require('markdown').markdown;
const fs = require('fs');
exports.markdown2Html = function markdown2Html(filePath) {
import markdown$0 from 'markdown';
import fs from 'fs';
const markdown = markdown$0.markdown;
export function markdown2Html(filePath) {
return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
};
}

View File

@@ -1,21 +1,9 @@
const queryString = require('query-string');
/**
* for Fredy, it is important to sort search results by date, starting with the latest listing. if it is not sorted, we
* might never actually find the newest results, no matter how many pages we crawl.
* It has been written in the documentation, but obviously nobody reads docu theses days which is why it's been done
* automagically now.
*
* @param _url actual provider url containing the searchParams
* @param sortByDateParam param(s) indicating the correct sort order
* @returns {`${string}?${string}`} correctly formatted url
*/
module.exports = (_url, sortByDateParam) => {
import queryString from 'query-string';
export default (_url, sortByDateParam) => {
//if no mutation is necessary, just return the original url
if (sortByDateParam == null) {
return _url;
}
const original = queryString.parseUrl(_url);
const mutate = queryString.parse(sortByDateParam);
return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;

View File

@@ -1,36 +1,67 @@
const axios = require('axios');
const axiosRetry = require('axios-retry');
axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
import fetch from 'node-fetch';
import { config } from '../utils.js';
import { makeUrlResidential } from './scrapingAnt.js';
//if ScrapingAnt got blocked, this http status is returned
const BLOCKED_HTTP_STATUS = 423;
const NOT_FOUND_HTTP_STATUS = 404;
const MAX_RETRIES_SCRAPING_ANT = 10;
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
function makeDriver(headers = {}) {
let cookies = '';
return async function driver(context, callback) {
async function scrapingAntDriver(context, callback, retryCounter = 0) {
const proxyType = config.scrapingAnt?.proxy || 'datacenter';
try {
const url = context.url;
const result = await axios({
url,
const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
const response = await fetch(url, {
headers: {
...headers,
cookie: cookies,
},
});
const result = await response.text();
if (cookies.length === 0) {
cookies = response.headers.raw()['set-cookie'] || [];
}
callback(null, result);
} catch (exception) {
/* eslint-disable no-console */
if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
callback(null, []);
return;
}
if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
retryCounter++;
console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
await scrapingAntDriver(context, callback, retryCounter);
} else {
console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
callback(null, []);
}
/* eslint-enable no-console */
}
}
/**
* The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
* everything != Immoscout as of writing this)
*/
return async function driver(context, callback) {
if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
return scrapingAntDriver(context, callback);
}
try {
const response = await fetch(context.url, {
headers: {
...headers,
Cookie: cookies,
},
});
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
//assume we have gotten a response from scrapingAnt
if (cookies.length === 0) {
cookies = result.data.cookies;
}
callback(null, result.data.content);
} else {
callback(null, result.data);
}
const result = await response.text();
callback(null, result);
} catch (exception) {
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
callback(null, []);
}
};
}
module.exports = makeDriver;
export default makeDriver;

View File

@@ -1,7 +1,6 @@
const config = require('../../conf/config.json');
const makeDriver = require('./requestDriver');
const Xray = require('x-ray');
import { config } from '../utils.js';
import makeDriver from './requestDriver.js';
import Xray from 'x-ray';
class Scraper {
constructor() {
const filters = {
@@ -9,38 +8,29 @@ class Scraper {
trim: this._trim,
int: this._int,
};
const headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
};
if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
headers['x-api-key'] = config.scrapingAnt.apiKey;
}
const driver = makeDriver(headers);
const xray = Xray({ filters });
xray.driver(driver);
this.xray = xray;
}
get x() {
return this.xray;
}
_removeNewline(value) {
return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
}
_trim(value) {
return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
}
_int(value) {
return typeof value === 'string' ? parseInt(value, 10) : value;
}
}
module.exports = new Scraper().x;
export default new Scraper().x;

View File

@@ -1,26 +1,19 @@
const { metaInformation } = require('../provider/immoscout');
//to better configure re-capture chose a random proxy each time we do a call
const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
const config = require('../../conf/config.json');
import { metaInformation } from '../provider/immoscout.js';
import { config } from '../utils.js';
const isImmoscout = (id) => {
return id.toLowerCase() === metaInformation.id;
};
exports.transformUrlForScrapingAnt = (url, id) => {
const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
export const transformUrlForScrapingAnt = (url, id) => {
if (isImmoscout(id)) {
//only do calls to scrapingAnt when dealing with Immoscout
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(
url
)}&proxy_country=${randomProxy}&proxy_type=residential`;
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
}
return url;
};
exports.isScrapingAntApiKeySet = () => {
export const isScrapingAntApiKeySet = () => {
return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
};
exports.isImmoscout = isImmoscout;
export const makeUrlResidential = (url) => {
return url.replace('datacenter', 'residential');
};
export { isImmoscout };

View File

@@ -1,3 +1,2 @@
const crypto = require('crypto');
exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
import crypto from 'crypto';
export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

View File

@@ -1,27 +1,17 @@
const stringSimilarity = require('string-similarity');
import stringSimilarity from 'string-similarity';
//if the score is higher than this, it will be considered a match
const MAX_DICE_INDEX = 0.7;
/**
* The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
*
* @type {module.SimilarityCacheEntry}
*/
module.exports = class SimilarityCacheEntry {
export default (class SimilarityCacheEntry {
constructor(time) {
this.time = time;
this.values = [];
}
setCacheEntry = (entry) => {
this.values.push(entry);
};
getTime = () => {
return this.time;
};
hasSimilarEntries = (value) => {
if (this.values.length > 0) {
for (let i = 0; i < this.values.length; i++) {
@@ -33,4 +23,4 @@ module.exports = class SimilarityCacheEntry {
}
return false;
};
};
});

View File

@@ -1,63 +1,40 @@
/**
* each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
* a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
* the user won't see any duplicates
*
* The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
*
* @type {module.SimilarityCacheEntry|{}}
*/
const SimilarityCacheEntry = require('./SimilarityCacheEntry');
const config = require('../../../conf/config.json');
import SimilarityCacheEntry from './SimilarityCacheEntry.js';
import { config } from '../../utils.js';
//5 minutes
let retention = 5 * 60 * 1000;
const intervalInMs = config.interval * 60 * 1000;
//an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
if (intervalInMs <= retention) {
retention = Math.floor(intervalInMs / 2);
}
//jobid -> SimilarityCacheEntry
const cache = {};
let intervalId;
exports.addCacheEntry = (jobId, value) => {
cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
cache[jobId].setCacheEntry(value);
};
exports.hasSimilarEntries = (jobId, value) => {
if (cache[jobId] == null) {
return false;
}
return cache[jobId].hasSimilarEntries(value);
};
/**
* cleanup
*/
intervalId = setInterval(() => {
const keysToBeRemoved = [];
const now = Date.now();
Object.keys(cache).forEach((key) => {
if (cache[key].getTime() + retention < now) {
keysToBeRemoved.push(key);
}
});
if (keysToBeRemoved.length > 0) {
keysToBeRemoved.forEach((key) => delete cache[key]);
}
}, 10000);
/**
* mostly used for tests
*/
exports.stopCacheCleanup = () => {
export const addCacheEntry = (jobId, value) => {
cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
cache[jobId].setCacheEntry(value);
};
export const hasSimilarEntries = (jobId, value) => {
if (cache[jobId] == null) {
return false;
}
return cache[jobId].hasSimilarEntries(value);
};
export const stopCacheCleanup = () => {
clearInterval(intervalId);
};

View File

@@ -0,0 +1,8 @@
import lodash from 'lodash';
import { LowSync } from 'lowdb';
export default class LowdashAdapter extends LowSync {
constructor(adapter) {
super(adapter);
this.chain = lodash.chain(this).get('data');
}
}

View File

@@ -1,28 +1,30 @@
const path = require('path');
const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync(DB_PATH);
const low = require('lowdb');
const db = low(adapter);
const { nanoid } = require('nanoid');
const listingStorage = require('./listingsStorage');
import { JSONFileSync } from 'lowdb/node';
import { nanoid } from 'nanoid';
import * as listingStorage from './listingsStorage.js';
import { getDirName } from '../../utils.js';
import path from 'path';
import LowdashAdapter from './LowDashAdapter.js';
db.defaults({ jobs: [] }).write();
const file = path.join(getDirName(), '../', 'db/jobs.json');
const adapter = new JSONFileSync(file);
const db = new LowdashAdapter(adapter);
exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
db.read();
db.data ||= { jobs: [] };
export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
const currentJob =
jobId == null
? null
: db
: db.chain
.get('jobs')
.find((job) => job.id === jobId)
.value();
const jobs = db
const jobs = db.chain
.get('jobs')
.value()
.filter((job) => job.id !== jobId);
.filter((job) => job.id !== jobId)
.value();
jobs.push({
id: jobId || nanoid(),
//make sure to not overwrite the user id in case an admin changes the job
@@ -33,57 +35,55 @@ exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, no
provider,
notificationAdapter,
});
db.set('jobs', jobs).write();
db.chain.set('jobs', jobs).value();
db.write();
};
exports.getJob = (jobId) => {
const job = db
export const getJob = (jobId) => {
const job = db.chain
.get('jobs')
.find((job) => job.id === jobId)
.value();
if (job == null) {
return null;
}
return {
...job,
numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
};
};
exports.setJobStatus = ({ jobId, status }) => {
db.get('jobs')
export const setJobStatus = ({ jobId, status }) => {
db.chain
.get('jobs')
.find((job) => job.id === jobId)
.assign({ enabled: status })
.write();
.value();
db.write();
};
exports.removeJob = (jobId) => {
export const removeJob = (jobId) => {
listingStorage.removeListings(jobId);
db.get('jobs')
db.chain
.get('jobs')
.remove((job) => job.id === jobId)
.write();
.value();
db.write();
};
exports.removeJobsByUserId = (userId) => {
db.get('jobs')
.value()
export const removeJobsByUserId = (userId) => {
db.chain
.get('jobs')
.filter((job) => job.userId === userId)
.forEach((job) => listingStorage.removeListings(job.id));
db.get('jobs')
.remove((job) => job.userId === userId)
.write();
};
exports.getJobs = () => {
return db
db.chain
.get('jobs')
.remove((job) => job.userId === userId)
.value();
db.write();
};
export const getJobs = () => {
return db.chain
.get('jobs')
.value()
.map((job) => ({
...job,
numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
}));
}))
.value();
};

View File

@@ -1,10 +1,15 @@
const path = require('path');
import { JSONFileSync } from 'lowdb/node';
import { getDirName } from '../../utils.js';
import path from 'path';
import LowdashAdapter from './LowDashAdapter.js';
const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync(DB_PATH);
const low = require('lowdb');
const db = low(adapter);
const file = path.join(getDirName(), '../', 'db/jobListingData.json');
const adapter = new JSONFileSync(file);
const db = new LowdashAdapter(adapter);
db.read();
db.data ||= {};
const buildKey = (jobKey, providerId, endpoint) => {
let key = `${jobKey}`;
@@ -19,35 +24,31 @@ const buildKey = (jobKey, providerId, endpoint) => {
}
return key;
};
exports.getNumberOfAllKnownListings = (jobId) => {
const data = db.get(`${jobId}.providerData`).value() || {};
export const getNumberOfAllKnownListings = (jobId) => {
const data = db.chain.get(`${jobId}.providerData`).value() || {};
return Object.values(data)
.map((values) => Object.keys(values).length)
.reduce((accumulator, currentValue) => accumulator + currentValue, 0);
};
exports.getListingProviderDataForAnalytics = (jobId) => {
export const getListingProviderDataForAnalytics = (jobId) => {
const key = buildKey(jobId, 'providerData');
return db.get(key).value() || {};
return db.chain.get(key).value() || {};
};
exports.getKnownListings = (jobId, providerId) => {
export const getKnownListings = (jobId, providerId) => {
const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
return db.get(providerListingsKey).value() || {};
return db.chain.get(providerListingsKey).value() || {};
};
exports.setKnownListings = (jobId, providerId, listings) => {
export const setKnownListings = (jobId, providerId, listings) => {
const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
return db.set(providerListingsKey, listings).write();
db.chain.set(providerListingsKey, listings).value();
return db.write();
};
exports.setLastJobExecution = (jobId) => {
export const setLastJobExecution = (jobId) => {
const key = buildKey(jobId, null, 'lastExecution');
return db.set(key, Date.now()).write();
db.chain.set(key, Date.now()).value();
return db.write();
};
exports.removeListings = (jobId) => {
db.unset(jobId).write();
export const removeListings = (jobId) => {
db.chain.unset(jobId).value();
db.write();
};

View File

@@ -1,14 +1,17 @@
const path = require('path');
const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
const FileSync = require('lowdb/adapters/FileSync');
const adapter = new FileSync(DB_PATH);
const low = require('lowdb');
const db = low(adapter);
const hasher = require('../security/hash');
const { nanoid } = require('nanoid');
const jobStorage = require('./jobStorage');
import { JSONFileSync } from 'lowdb/node';
import { getDirName } from '../../utils.js';
import * as hasher from '../security/hash.js';
import { nanoid } from 'nanoid';
import * as jobStorage from './jobStorage.js';
import path from 'path';
import LowdashAdapter from './LowDashAdapter.js';
db.defaults({
const file = path.join(getDirName(), '../', 'db/users.json');
const adapter = new JSONFileSync(file);
const db = new LowdashAdapter(adapter);
db.read();
db.data ||= {
user: [
//you probably want to change the default password ;)
{
@@ -20,11 +23,11 @@ db.defaults({
isDemo: false,
},
],
}).write();
};
exports.getUsers = (withPassword) => {
export const getUsers = (withPassword) => {
const jobs = jobStorage.getJobs();
return db
return db.chain
.get('user')
.value()
.map((user) => ({
@@ -34,13 +37,12 @@ exports.getUsers = (withPassword) => {
numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
}));
};
exports.getUser = (id) => {
export const getUser = (id) => {
const jobs = jobStorage.getJobs();
const user = db
const user = db.chain
.get('user')
.value()
.find((user) => user.id === id);
.find((user) => user.id === id)
.value();
if (user == null) {
return null;
}
@@ -49,13 +51,11 @@ exports.getUser = (id) => {
numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
};
};
exports.upsertUser = ({ username, password, userId, isAdmin }) => {
const user = db
export const upsertUser = ({ username, password, userId, isAdmin }) => {
const user = db.chain
.get('user')
.value()
.filter((u) => u.id !== userId);
.filter((u) => u.id !== userId)
.value();
user.push({
id: userId || nanoid(),
username,
@@ -63,21 +63,24 @@ exports.upsertUser = ({ username, password, userId, isAdmin }) => {
password: hasher.hash(password),
isAdmin,
});
db.set('user', user).write();
db.chain.set('user', user).value();
db.write();
};
exports.setLastLoginToNow = ({ userId }) => {
db.get('user')
export const setLastLoginToNow = ({ userId }) => {
db.chain
.get('user')
.find((u) => u.id === userId)
.assign({ lastLogin: Date.now() })
.write();
.value();
db.write();
};
exports.removeUser = (userId) => {
const user = db.get('user').value();
db.set(
'user',
user.filter((u) => u.id !== userId)
).write();
export const removeUser = (userId) => {
const user = db.chain.get('user').value();
db.chain
.set(
'user',
user.filter((u) => u.id !== userId)
)
.value();
db.write();
};

View File

@@ -1,17 +1,18 @@
import { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { readFile } from 'fs/promises';
function isOneOf(word, arr) {
if (arr == null || arr.length === 0) {
return false;
}
const expression = String.raw`\b(${arr.join('|')})\b`;
const blacklist = new RegExp(expression, 'ig');
return blacklist.test(word);
}
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function timeStringToMs(timeString, now) {
const d = new Date(now);
const parts = timeString.split(':');
@@ -20,17 +21,31 @@ function timeStringToMs(timeString, now) {
d.setSeconds(0);
return d.getTime();
}
function duringWorkingHoursOrNotSet(config, now) {
const { workingHours } = config;
if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
return true;
}
const toDate = timeStringToMs(workingHours.to, now);
const fromDate = timeStringToMs(workingHours.from, now);
return fromDate <= now && toDate >= now;
}
module.exports = { isOneOf, nullOrEmpty, duringWorkingHoursOrNotSet };
function getDirName() {
return dirname(fileURLToPath(import.meta.url));
}
const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
export { isOneOf };
export { nullOrEmpty };
export { duringWorkingHoursOrNotSet };
export { getDirName };
export { config };
export default {
isOneOf,
nullOrEmpty,
duringWorkingHoursOrNotSet,
getDirName,
config,
};

View File

@@ -1,20 +1,22 @@
{
"name": "fredy",
"version": "5.4.8",
"version": "7.1.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"start": "node index.js",
"dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
"prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
"format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
"test": "mocha --timeout 20000 test/**/*.test.js",
"lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
"dev": "yarn && rm -rf ./ui/public/* && vite",
"ui": "rm -rf ./ui/public/* && vite",
"prod": "yarn && vite build --emptyOutDir",
"format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
"test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
"lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
},
"husky": {
"hooks": {
"pre-commit": "lint-staged"
}
},
"type": "module",
"lint-staged": {
"*.js": [
"eslint ./index.js ./lib/**/*.js ./test/**/*.js",
@@ -43,7 +45,7 @@
},
"license": "MIT",
"engines": {
"node": ">=14.0.0",
"node": ">=16.0.0",
"npm": ">=7.0.0"
},
"browserslist": [
@@ -53,65 +55,54 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-ui": "2.31.0",
"@rematch/core": "2.2.0",
"@rematch/loading": "2.1.2",
"@sendgrid/mail": "7.6.2",
"axios": "0.26.1",
"axios-retry": "^3.2.4",
"better-sqlite3": "^7.5.0",
"body-parser": "1.19.2",
"@sendgrid/mail": "7.7.0",
"@vitejs/plugin-react": "3.1.0",
"better-sqlite3": "8.2.0",
"body-parser": "1.20.2",
"cookie-session": "2.0.0",
"handlebars": "4.7.7",
"highcharts": "10.0.0",
"highcharts-react-official": "3.1.0",
"lowdb": "1.0.0",
"highcharts": "10.3.3",
"highcharts-react-official": "3.2.0",
"lodash": "4.17.21",
"lowdb": "5.1.0",
"markdown": "^0.5.0",
"nanoid": "3.3.1",
"node-mailjet": "3.3.7",
"query-string": "7.1.1",
"react": "17.0.2",
"react-dom": "17.0.2",
"react-redux": "7.2.6",
"nanoid": "4.0.1",
"node-fetch": "3.3.1",
"node-mailjet": "6.0.2",
"query-string": "8.1.0",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-redux": "8.0.5",
"react-router": "5.2.1",
"react-router-dom": "5.3.0",
"react-switch": "^6.0.0",
"redux": "4.1.2",
"redux-thunk": "2.4.1",
"restana": "4.9.3",
"semantic-ui-react": "2.1.2",
"redux": "4.2.1",
"redux-thunk": "2.4.2",
"restana": "4.9.7",
"serve-static": "1.15.0",
"slack": "11.0.2",
"string-similarity": "^4.0.4",
"vite": "4.2.0",
"x-ray": "2.3.4"
},
"devDependencies": {
"@babel/core": "7.17.8",
"@babel/preset-env": "7.16.11",
"@babel/preset-react": "7.16.7",
"babel-eslint": "10.1.0",
"babel-loader": "8.2.4",
"chai": "4.3.6",
"clean-webpack-plugin": "4.0.0",
"copy-webpack-plugin": "10.2.4",
"css-loader": "6.7.1",
"eslint": "7.32.0",
"eslint-config-prettier": "8.5.0",
"eslint-plugin-react": "7.29.3",
"file-loader": "6.2.0",
"@babel/core": "7.21.3",
"@babel/eslint-parser": "7.21.3",
"@babel/preset-env": "7.20.2",
"@babel/preset-react": "7.18.6",
"chai": "4.3.7",
"eslint": "8.36.0",
"eslint-config-prettier": "8.7.0",
"eslint-plugin-react": "7.32.2",
"esmock": "2.1.0",
"history": "5.3.0",
"husky": "4.3.8",
"less": "4.1.2",
"less-loader": "10.2.0",
"lint-staged": "12.3.7",
"mocha": "9.2.2",
"prettier": "2.6.1",
"proxyquire": "2.1.3",
"redux-logger": "3.0.6",
"style-loader": "3.3.1",
"url-loader": "4.1.1",
"webpack": "5.70.0",
"webpack-cli": "4.9.2",
"webpack-dev-server": "3.11.2",
"webpack-merge": "5.8.0"
"less": "4.1.3",
"lint-staged": "13.2.0",
"mocha": "10.2.0",
"prettier": "2.8.5",
"redux-logger": "3.0.6"
}
}

View File

@@ -1,12 +1,10 @@
module.exports = {
_tmpStore: {},
let tmpStore = {};
send: (serviceName, payload) => {
this._tmpStore = { serviceName, payload };
return [Promise.resolve()];
},
get: () => {
return this._tmpStore;
},
export const send = (serviceName, payload) => {
tmpStore = { serviceName, payload };
return [Promise.resolve()];
};
export const get = () => {
return tmpStore;
};

View File

@@ -1,11 +1,8 @@
const db = {};
exports.setKnownListings = (jobKey, providerId, listings) => {
export const setKnownListings = (jobKey, providerId, listings) => {
if (!Array.isArray(listings)) throw Error('Not a valid array');
db[providerId] = listings;
};
exports.getKnownListings = (jobKey, providerId) => {
export const getKnownListings = (jobKey, providerId) => {
return db[providerId] || [];
};

View File

@@ -1,35 +1,25 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/einsAImmobilien');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { providerConfig, mockFredy } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/einsAImmobilien.js';
const expect = chai.expect;
describe('#einsAImmobilien testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.einsAImmobilien, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
it('should test einsAImmobilien provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
fredy.execute().then((listings) => {
expect(listings).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('number');
@@ -37,9 +27,7 @@ describe('#einsAImmobilien testsuite()', () => {
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
/** check the values if possible **/
expect(notify.price).that.does.include('EUR');
expect(notify.size).to.be.not.empty;
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');

View File

@@ -0,0 +1,40 @@
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { providerConfig, mockFredy } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/immobilienDe.js';
const expect = chai.expect;
describe('#immobilien.de testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immobilienDe, [], []);
it('should test immobilien.de provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immobilienDe');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.immobilien.de');
expect(notify.address).to.be.not.empty;
});
resolve();
});
});
});
});

View File

@@ -1,34 +1,23 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/immonet');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/immonet.js';
const expect = chai.expect;
describe('#immonet testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immonet, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
it('should test immonet provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('number');
@@ -37,7 +26,6 @@ describe('#immonet testsuite()', () => {
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');

View File

@@ -1,25 +1,17 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/immoscout');
const scrapingAnt = require('../../lib/services/scrapingAnt');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/immoscout.js';
import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
const expect = chai.expect;
describe('#immoscout testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immoscout, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
it('should test immoscout provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
if (!scrapingAnt.isScrapingAntApiKeySet()) {
/* eslint-disable no-console */
@@ -28,15 +20,12 @@ describe('#immoscout testsuite()', () => {
resolve();
return;
}
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immoscout');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('number');
@@ -45,7 +34,6 @@ describe('#immoscout testsuite()', () => {
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.price).that.does.include('€');
expect(notify.size).that.does.include('m²');

View File

@@ -1,34 +1,23 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/immoswp');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/immoswp.js';
const expect = chai.expect;
describe('#immoswp testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immoswp, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
it('should test immoswp provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immoswp');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
@@ -37,7 +26,6 @@ describe('#immoswp testsuite()', () => {
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.price).that.does.include('€');
expect(notify.title).to.be.not.empty;

View File

@@ -1,42 +1,30 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/immowelt');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/immowelt.js';
const expect = chai.expect;
describe('#immowelt testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test immowelt provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.immowelt, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immowelt');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
expect(notify.size).that.does.include('m²');

View File

@@ -1,42 +1,29 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/kleinanzeigen');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/kleinanzeigen.js';
const expect = chai.expect;
describe('#kleinanzeigen testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test kleinanzeigen provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.kleinanzeigen, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('number');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');

View File

@@ -1,41 +1,29 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/neubauKompass');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/neubauKompass.js';
const expect = chai.expect;
describe('#neubauKompass testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.neubauKompass, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
it('should test neubauKompass provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj.serviceName).to.equal('neubauKompass');
notificationObj.payload.forEach((notify) => {
expect(notify).to.be.a('object');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.neubaukompass.de');

View File

@@ -4,6 +4,10 @@
"enabled": true,
"id": "einsAImmobilien"
},
"immobilienDe": {
"url": "https://www.immobilien.de/Wohnen/Suchergebnisse-51797.html?search._digest=true&search._filter=wohnen&search.flaeche_von=50&search.objektart=wohnung&search.preis_bis=1200&search.typ=mieten&search.umkreis=15&search.wo=district%3A2434%2C2695%2C2621%2C2700%2C2967%2C2734%2C2909%2C2955%2C2392%2C2746%2C2767%2C2982%2C2904%2C2612%2C2892%2C2587%2C2871%2C2975%2C2591%2C2887%2C2569%2C2640%2C2735&sort_col=*created_ts&sort_dir=desc",
"enabled": true
},
"immonet": {
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
"enabled": true
@@ -36,4 +40,4 @@
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
"enabled": true
}
}
}

View File

@@ -1,14 +1,13 @@
const utils = require('../../lib/utils');
const assert = require('assert');
const expect = require('chai').expect;
import utils from '../../lib/utils.js';
import assert from 'assert';
import chai from 'chai';
const expect = chai.expect;
const fakeWorkingHoursConfig = (from, to) => ({
workingHours: {
to,
from,
},
});
describe('utils', () => {
describe('#isOneOf()', () => {
it('should be false', () => {
@@ -18,7 +17,6 @@ describe('utils', () => {
assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
});
});
describe('#duringWorkingHoursOrNotSet()', () => {
it('should be false', () => {
expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;

View File

@@ -1,35 +1,25 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore');
const proxyquire = require('proxyquire').noCallThru();
const expect = require('chai').expect;
const provider = require('../../lib/provider/wgGesucht');
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import chai from 'chai';
import * as provider from '../../lib/provider/wgGesucht.js';
const expect = chai.expect;
describe('#wgGesucht testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.wgGesucht, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': {
...mockStore,
},
'./notification/notify': mockNotification,
});
it('should test wgGesucht provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = mockNotification.get();
const notificationObj = get();
expect(notificationObj.serviceName).to.equal('wgGesucht');
notificationObj.payload.forEach((notify) => {
expect(notify).to.be.a('object');
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.details).to.be.a('string');

View File

@@ -1,9 +1,17 @@
const testData = require('./testData.json');
const expect = require('chai').expect;
const fs = require('fs');
import fs from 'fs';
import chai from 'chai';
import { readFile } from 'fs/promises';
import mutator from '../../lib/services/queryStringMutator.js';
import queryString from 'query-string';
const expect = chai.expect;
const mutator = require('../../lib/services/queryStringMutator.js');
const queryString = require('query-string');
const data = await readFile(new URL('./testData.json', import.meta.url));
const testData = JSON.parse(data);
let _provider = await Promise.all(
fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
);
/**
* Test test might look a bit weird at first, but listen stranger...
@@ -12,18 +20,14 @@ const queryString = require('query-string');
*/
describe('queryStringMutator', () => {
it('should fix all urls', () => {
let _provider = fs.readdirSync('./lib/provider/').map((integPath) => require(`../../lib/provider/${integPath}`));
for (let test of testData) {
const provider = _provider.find((p) => p.metaInformation.id === test.id);
if (provider == null) {
throw new Error(`Cannot find provider for given id: ${test.id}`);
}
const fixedUrl = mutator(test.url, provider.config.sortByDateParam);
const expectedParams = queryString.parseUrl(test.shouldBecome);
const actualParams = queryString.parseUrl(fixedUrl);
//check if all new params are existing
expect(Object.keys(expectedParams.query)).to.include.members(Object.keys(actualParams.query));
expect(Object.values(expectedParams.query)).to.include.members(Object.values(actualParams.query));

View File

@@ -1,6 +1,6 @@
const SimilarityCacheEntry = require('../../lib/services/similarity-check/SimilarityCacheEntry');
const expect = require('chai').expect;
import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
import chai from 'chai';
const expect = chai.expect;
describe('similarityCheck', () => {
describe('#similarityCheck()', () => {
it('should be false', () => {

17
test/utils.js Normal file
View File

@@ -0,0 +1,17 @@
import { readFile } from 'fs/promises';
import esmock from 'esmock';
import * as mockStore from './mocks/mockStore.js';
import { send } from './mocks/mockNotification.js';
export const providerConfig = JSON.parse(await readFile(new URL('./provider/testProvider.json', import.meta.url)));
export const mockFredy = async () => {
return await esmock('../lib/FredyRuntime', {
'../lib/services/storage/listingsStorage.js': {
...mockStore,
},
'../lib/notification/notify.js': {
send,
},
});
};

View File

@@ -1,97 +0,0 @@
import React, { useEffect } from 'react';
import InsufficientPermission from './components/permission/InsufficientPermission';
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
import GeneralSettings from './views/generalSettings/GeneralSettings';
import ToastsContainer from './components/toasts/ToastContainer';
import JobMutation from './views/jobs/mutation/JobMutation';
import UserMutator from './views/user/mutation/UserMutator';
import ToastContext from './components/toasts/ToastContext';
import JobInsight from './views/jobs/insights/JobInsight';
import { useDispatch, useSelector } from 'react-redux';
import useToast from './components/toasts/useToast';
import { Switch, Redirect } from 'react-router-dom';
import Logout from './components/logout/Logout';
import Logo from './components/logo/Logo';
import Menu from './components/menu/Menu';
import Login from './views/login/Login';
import Users from './views/user/Users';
import Jobs from './views/jobs/Jobs';
import { Route } from 'react-router';
import './App.less';
export default function FredyApp() {
const dispatch = useDispatch();
const [showToast, onToastFinished, toasts] = useToast();
const [loading, setLoading] = React.useState(true);
const currentUser = useSelector((state) => state.user.currentUser);
useEffect(async () => {
await dispatch.provider.getProvider();
await dispatch.jobs.getJobs();
await dispatch.jobs.getProcessingTimes();
await dispatch.notificationAdapter.getAdapter();
await dispatch.user.getCurrentUser();
setLoading(false);
}, [currentUser?.userId]);
const needsLogin = () => {
return currentUser == null || Object.keys(currentUser).length === 0;
};
const isAdmin = () => currentUser != null && currentUser.isAdmin;
const login = () => (
<Switch>
<Route name="Login" path={'/login'} component={Login} />
<Redirect from="*" to={'/login'} />
</Switch>
);
return loading ? null : needsLogin() ? (
login()
) : (
<ToastContext.Provider value={{ showToast }}>
<div className="app">
<div className="app__container">
<Logout />
<Logo width={190} white />
<Menu isAdmin={isAdmin()} />
<ToastsContainer toasts={toasts} onToastFinished={onToastFinished} />
<Switch>
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
<Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
<Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
<Route name="Job overview" path={'/jobs'} component={Jobs} />
<PermissionAwareRoute
name="Create new User"
path="/users/new"
component={<UserMutator />}
currentUser={currentUser}
/>
<PermissionAwareRoute
name="Edit a user"
path="/users/edit/:userId"
component={<UserMutator />}
currentUser={currentUser}
/>
<PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
<PermissionAwareRoute
name="General Settings"
path="/generalSettings"
component={<GeneralSettings />}
currentUser={currentUser}
/>
<Redirect from="/" to={'/jobs'} />
</Switch>
</div>
</div>
</ToastContext.Provider>
);
}
FredyApp.displayName = 'FredyApp';

95
ui/src/App.jsx Normal file
View File

@@ -0,0 +1,95 @@
import React, { useEffect } from 'react';
import InsufficientPermission from './components/permission/InsufficientPermission';
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
import GeneralSettings from './views/generalSettings/GeneralSettings';
import JobMutation from './views/jobs/mutation/JobMutation';
import UserMutator from './views/user/mutation/UserMutator';
import JobInsight from './views/jobs/insights/JobInsight.jsx';
import { useDispatch, useSelector } from 'react-redux';
import { Switch, Redirect } from 'react-router-dom';
import Logout from './components/logout/Logout';
import Logo from './components/logo/Logo';
import Menu from './components/menu/Menu';
import Login from './views/login/Login';
import Users from './views/user/Users';
import Jobs from './views/jobs/Jobs';
import { Route } from 'react-router';
import './App.less';
export default function FredyApp() {
const dispatch = useDispatch();
const [loading, setLoading] = React.useState(true);
const currentUser = useSelector((state) => state.user.currentUser);
useEffect(() => {
async function init() {
await dispatch.user.getCurrentUser();
if (!needsLogin()) {
await dispatch.provider.getProvider();
await dispatch.jobs.getJobs();
await dispatch.jobs.getProcessingTimes();
await dispatch.notificationAdapter.getAdapter();
}
setLoading(false);
}
init();
}, [currentUser?.userId]);
const needsLogin = () => {
return currentUser == null || Object.keys(currentUser).length === 0;
};
const isAdmin = () => currentUser != null && currentUser.isAdmin;
const login = () => (
<Switch>
<Route name="Login" path={'/login'} component={Login} />
<Redirect from="*" to={'/login'} />
</Switch>
);
return loading ? null : needsLogin() ? (
login()
) : (
<div className="app">
<div className="app__container">
<Logout />
<Logo width={190} white />
<Menu isAdmin={isAdmin()} />
<Switch>
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
<Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
<Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
<Route name="Job overview" path={'/jobs'} component={Jobs} />
<PermissionAwareRoute
name="Create new User"
path="/users/new"
component={<UserMutator />}
currentUser={currentUser}
/>
<PermissionAwareRoute
name="Edit a user"
path="/users/edit/:userId"
component={<UserMutator />}
currentUser={currentUser}
/>
<PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
<PermissionAwareRoute
name="General Settings"
path="/generalSettings"
component={<GeneralSettings />}
currentUser={currentUser}
/>
<Redirect from="/" to={'/jobs'} />
</Switch>
</div>
</div>
);
}
FredyApp.displayName = 'FredyApp';

View File

@@ -4,10 +4,40 @@
width:100%;
&__container {
width: 100%;
padding: 1rem 1rem;
background-color: #3f3e3ef5;
color: #f1f1f1;
color: var(--semi-color-text-0);
background-color: #232429;
}
}
.ui.inverted.segment{
background: #31303078!important;
}
.ui.black.label, .ui.black.labels .label {
background-color: #31303078!important;
}
a:link {
color: #54a9ff;
background-color: transparent;
text-decoration: none;
}
a:visited {
color: #54a9ff;
background-color: transparent;
text-decoration: none;
}
a:hover {
color: #54a9ff;
background-color: transparent;
text-decoration: underline;
}
a:active {
color: #54a9ff;
background-color: transparent;
text-decoration: underline;
}

View File

@@ -4,19 +4,24 @@ import { reduxStore } from './services/rematch/store';
import { HashRouter } from 'react-router-dom';
import { createHashHistory } from 'history';
import { Provider } from 'react-redux';
import ReactDOM from 'react-dom';
import { createRoot } from 'react-dom/client';
import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
import { LocaleProvider } from '@douyinfe/semi-ui';
const container = document.getElementById('fredy');
const root = createRoot(container);
const history = createHashHistory();
import App from './App';
import './Index.less';
ReactDOM.render(
root.render(
<Provider store={reduxStore}>
<HashRouter history={history}>
<App />
<LocaleProvider locale={en_US}>
<App />
</LocaleProvider>
</HashRouter>
</Provider>,
document.getElementById('fredy')
</Provider>
);

View File

@@ -2,5 +2,14 @@ body, html {
margin: 0;
height: 100%;
width: 100%;
background-color: #3f3e3ef5;
background-color: #232429;
}
.semi-table-row-head{
background-color: #2b2b2b !important;
color: #fff !important;
}
.semi-table-row-cell {
background-color: #333333 !important;
}

View File

@@ -1,12 +0,0 @@
import React from 'react';
import { Header } from 'semantic-ui-react';
import './Headline.less';
export default function Headline({ text, size = 'medium', className = '' } = {}) {
return (
<Header className={`headline ${className}`} size={size}>
{text}
</Header>
);
}

View File

@@ -0,0 +1,11 @@
import React from 'react';
import { Typography } from '@douyinfe/semi-ui';
export default function Headline({ text, size = 3 } = {}) {
const { Title } = Typography;
return (
<Title heading={size} style={{ marginBottom: '1rem' }}>
{text}
</Title>
);
}

View File

@@ -1,3 +0,0 @@
.headline{
color: #f1f1f1 !important;
}

View File

@@ -1,20 +1,20 @@
import React from 'react';
import { Button } from 'semantic-ui-react';
import { Button } from '@douyinfe/semi-ui';
import { xhrPost } from '../../services/xhr';
import { IconUser } from '@douyinfe/semi-icons';
const Logout = function Logout() {
return (
<Button
content="Logout"
labelPosition="left"
icon="user"
size="mini"
icon={<IconUser />}
type="danger"
theme="solid"
onClick={async () => {
await xhrPost('/api/login/logout');
location.reload();
}}
negative
/>
>
Logout
</Button>
);
};

View File

@@ -1,50 +0,0 @@
import React from 'react';
import { useHistory } from 'react-router-dom';
import { Icon, Menu } from 'semantic-ui-react';
import './Menu.less';
import { useLocation } from 'react-router';
const TopMenu = function TopMenu({ isAdmin }) {
const history = useHistory();
const location = useLocation();
const isActiveRoute = (name) => location.pathname.indexOf(name) !== -1;
return (
<Menu pointing secondary className="topMenu">
<Menu.Item
name="jobs"
active={isActiveRoute('jobs')}
className={isActiveRoute('jobs') ? 'topMenu__active' : 'topMenu__item'}
onClick={() => history.push('/jobs')}
>
<Icon name="search" /> Job Configuration
</Menu.Item>
{isAdmin && (
<Menu.Item
name="user"
active={isActiveRoute('users')}
className={isActiveRoute('users') ? 'topMenu__active' : 'topMenu__item'}
onClick={() => history.push('/users')}
>
<Icon name="user" /> User configuration
</Menu.Item>
)}
{isAdmin && (
<Menu.Item
name="general"
active={isActiveRoute('general')}
className={isActiveRoute('general') ? 'topMenu__active' : 'topMenu__item'}
onClick={() => history.push('/generalSettings')}
>
<Icon name="cog" /> General Settings
</Menu.Item>
)}
</Menu>
);
};
export default TopMenu;

View File

@@ -0,0 +1,55 @@
import React from 'react';
import { useHistory } from 'react-router-dom';
import { Tabs, TabPane } from '@douyinfe/semi-ui';
import { useLocation } from 'react-router';
import { IconUser, IconTerminal, IconSetting } from '@douyinfe/semi-icons';
function parsePathName(name) {
const split = name.split('/').filter((s) => s.length !== 0);
return '/' + split[0];
}
const TopMenu = function TopMenu({ isAdmin }) {
const history = useHistory();
const location = useLocation();
return (
<Tabs type="line" activeKey={parsePathName(location.pathname)} onTabClick={(key) => history.push(key)}>
<TabPane
itemKey="/jobs"
tab={
<span>
<IconTerminal />
Jobs
</span>
}
/>
{isAdmin && (
<TabPane
itemKey="/users"
tab={
<span>
<IconUser />
User
</span>
}
/>
)}
{isAdmin && (
<TabPane
itemKey="/generalSettings"
tab={
<span>
<IconSetting />
General
</span>
}
/>
)}
</Tabs>
);
};
export default TopMenu;

View File

@@ -1,15 +0,0 @@
.topMenu {
border-bottom: 1px solid #b7b7b7f2 !important;
&__active {
border-bottom: 1px solid #06dcfff2 !important;
font-weight: 550 !important;
color: #78e5ff !important;
margin: 0 0 -1px !important;
}
&__item {
color: #fffffff2 !important;
border-color: transparent !important;
}
}

View File

@@ -1,5 +1,4 @@
import React from 'react';
import { Header } from 'semantic-ui-react';
import insufficientPermission from '../../assets/insufficient_permission.png';
export default function InsufficientPermission() {
@@ -7,9 +6,7 @@ export default function InsufficientPermission() {
<div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center', flexDirection: 'column' }}>
<img src={insufficientPermission} height={250} />
<br />
<Header as="h4" inverted>
Insufficient permission :(
</Header>
<h4>Insufficient permission :(</h4>
</div>
);
}

View File

@@ -0,0 +1,18 @@
import React from 'react';
import { Card } from '@douyinfe/semi-ui';
import './SegmentParts.less';
export const SegmentPart = ({ name, Icon = null, children, helpText }) => {
const { Meta } = Card;
return (
<Card
title={
<Meta title={name} description={helpText} avatar={Icon == null ? null : <Icon size="extra-extra-small" />} />
}
>
{children}
</Card>
);
};

View File

@@ -0,0 +1,4 @@
.segmentParts {
border: 1px solid #323232 !important;
border-radius: 5px !important;
}

View File

@@ -1,66 +0,0 @@
import React, { Fragment } from 'react';
import { Table, Button } from 'semantic-ui-react';
import Switch from 'react-switch';
const emptyTable = () => {
return (
<Table.Row>
<Table.Cell collapsing colSpan={6} style={{ textAlign: 'center' }}>
No Data
</Table.Cell>
</Table.Row>
);
};
const content = (jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight) => {
return (
<Fragment>
{Object.keys(jobs).map((jobKey) => {
const job = jobs[jobKey];
return (
<Table.Row key={jobKey}>
<Table.Cell collapsing>
<Switch onChange={(checked) => onJobStatusChanged(job.id, checked)} checked={job.enabled} />
</Table.Cell>
<Table.Cell>{job.name}</Table.Cell>
<Table.Cell>{job.numberOfFoundListings || 0}</Table.Cell>
<Table.Cell>{job.provider.length || 0}</Table.Cell>
<Table.Cell>{job.notificationAdapter.length || 0}</Table.Cell>
<Table.Cell>
<div style={{ float: 'right' }}>
<Button circular color="teal" icon="chart line" onClick={() => onJobInsight(job.id)} />
<Button circular color="blue" icon="edit" onClick={() => onJobEdit(job.id)} />
<Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
</div>
</Table.Cell>
</Table.Row>
);
})}
</Fragment>
);
};
export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight } = {}) {
return (
<Table singleLine inverted>
<Table.Header>
<Table.Row>
<Table.HeaderCell />
<Table.HeaderCell>Job Name</Table.HeaderCell>
<Table.HeaderCell>Number of findings</Table.HeaderCell>
<Table.HeaderCell>Active provider</Table.HeaderCell>
<Table.HeaderCell>Active notification adapter</Table.HeaderCell>
<Table.HeaderCell></Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{Object.keys(jobs).length === 0
? emptyTable()
: content(jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight)}
</Table.Body>
</Table>
);
}

View File

@@ -0,0 +1,79 @@
import React from 'react';
import { Button, Empty, Table, Switch } from '@douyinfe/semi-ui';
import { IconDelete, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
const empty = (
<Empty
image={<IllustrationNoResult />}
darkModeImage={<IllustrationNoResultDark />}
description={'No jobs available'}
/>
);
export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight } = {}) {
return (
<Table
pagination={false}
empty={empty}
columns={[
{
title: '',
dataIndex: '',
render: (job) => {
return <Switch onChange={(checked) => onJobStatusChanged(job.id, checked)} checked={job.enabled} />;
},
},
{
title: 'Job Name',
dataIndex: 'name',
},
{
title: 'Number of findings',
dataIndex: 'numberOfFoundListings',
render: (value) => {
return value || 0;
},
},
{
title: 'Active provider',
dataIndex: 'provider',
render: (value) => {
return value.length || 0;
},
},
{
title: 'Active notification adapter',
dataIndex: 'notificationAdapter',
render: (value) => {
return value.length || 0;
},
},
{
title: '',
dataIndex: 'tools',
render: (_, job) => {
return (
<div style={{ float: 'right' }}>
<Button
type="primary"
icon={<IconHistogram />}
onClick={() => onJobInsight(job.id)}
style={{ marginRight: '1rem' }}
/>
<Button
type="secondary"
icon={<IconEdit />}
onClick={() => onJobEdit(job.id)}
style={{ marginRight: '1rem' }}
/>
<Button type="danger" icon={<IconDelete />} onClick={() => onJobRemoval(job.id)} />
</div>
);
},
},
]}
dataSource={jobs}
/>
);
}

View File

@@ -1,50 +0,0 @@
import React, { Fragment } from 'react';
import { Table, Button } from 'semantic-ui-react';
const emptyTable = () => {
return (
<Table.Row>
<Table.Cell collapsing colSpan="3" style={{ textAlign: 'center' }}>
No Data
</Table.Cell>
</Table.Row>
);
};
const content = (adapterData, onRemove, onEdit) => {
return (
<Fragment>
{adapterData.map((data) => {
return (
<Table.Row key={data.id}>
<Table.Cell>{data.name}</Table.Cell>
<Table.Cell>
<div style={{ float: 'right' }}>
<Button circular color="blue" icon="edit" onClick={() => onEdit(data.id)} />
<Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
</div>
</Table.Cell>
</Table.Row>
);
})}
</Fragment>
);
};
export default function NotificationAdapterTable({ notificationAdapter = [], onRemove, onEdit } = {}) {
return (
<Table singleLine inverted>
<Table.Header>
<Table.Row>
<Table.HeaderCell>Notification Adapter Name</Table.HeaderCell>
<Table.HeaderCell></Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>
{notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove, onEdit)}
</Table.Body>
</Table>
);
}

View File

@@ -0,0 +1,38 @@
import React from 'react';
import { Empty, Table, Button } from '@douyinfe/semi-ui';
import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
export default function NotificationAdapterTable({ notificationAdapter = [], onRemove, onEdit } = {}) {
return (
<Table
pagination={false}
empty={<Empty description="No Data" />}
columns={[
{
title: 'Notification Adapter Name',
dataIndex: 'name',
},
{
title: '',
dataIndex: 'tools',
render: (_, record) => {
return (
<div style={{ float: 'right' }}>
<Button
type="secondary"
icon={<IconEdit />}
onClick={() => onEdit(record.id)}
style={{ marginRight: '1rem' }}
/>
<Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
</div>
);
},
},
]}
dataSource={notificationAdapter}
/>
);
}

View File

@@ -1,53 +0,0 @@
import React, { Fragment } from 'react';
import { Table, Button } from 'semantic-ui-react';
const emptyTable = () => {
return (
<Table.Row>
<Table.Cell collapsing colSpan="3" style={{ textAlign: 'center' }}>
No Data
</Table.Cell>
</Table.Row>
);
};
const content = (providerData, onRemove) => {
return (
<Fragment>
{providerData.map((data) => {
return (
<Table.Row key={data.id}>
<Table.Cell>{data.name}</Table.Cell>
<Table.Cell>
<a href={data.url} target="_blank" rel="noopener noreferrer">
Visit site
</a>
</Table.Cell>
<Table.Cell>
<div style={{ float: 'right' }}>
<Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
</div>
</Table.Cell>
</Table.Row>
);
})}
</Fragment>
);
};
export default function ProviderTable({ providerData = [], onRemove } = {}) {
return (
<Table singleLine inverted>
<Table.Header>
<Table.Row>
<Table.HeaderCell>Provider Name</Table.HeaderCell>
<Table.HeaderCell>Url</Table.HeaderCell>
<Table.HeaderCell></Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>{providerData.length === 0 ? emptyTable() : content(providerData, onRemove)}</Table.Body>
</Table>
);
}

View File

@@ -0,0 +1,42 @@
import React from 'react';
import { Empty, Table, Button } from '@douyinfe/semi-ui';
import { IconDelete } from '@douyinfe/semi-icons';
export default function ProviderTable({ providerData = [], onRemove } = {}) {
return (
<Table
pagination={false}
empty={<Empty description="No Provider available" />}
columns={[
{
title: 'Provider Name',
dataIndex: 'name',
},
{
title: 'Provider Url',
dataIndex: 'url',
render: (_, data) => {
return (
<a href={data.url} target="_blank" rel="noopener noreferrer">
Visit site
</a>
);
},
},
{
title: '',
dataIndex: 'tools',
render: (_, record) => {
return (
<div style={{ float: 'right' }}>
<Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
</div>
);
},
},
]}
dataSource={providerData}
/>
);
}

View File

@@ -1,49 +0,0 @@
import React from 'react';
import { Table, Button } from 'semantic-ui-react';
import { format } from '../../services/time/timeService';
const emptyTable = () => {
return (
<Table.Row>
<Table.Cell collapsing colSpan={4} style={{ textAlign: 'center' }}>
No Data
</Table.Cell>
</Table.Row>
);
};
const content = (user, onUserRemoval, onUserEdit) => {
return user.map((user) => {
return (
<Table.Row key={user.id}>
<Table.Cell>{user.username}</Table.Cell>
<Table.Cell>{user.lastLogin == null ? '---' : format(user.lastLogin)}</Table.Cell>
<Table.Cell>{user.numberOfJobs}</Table.Cell>
<Table.Cell>
<div style={{ float: 'right' }}>
<Button circular color="red" icon="trash" onClick={() => onUserRemoval(user.id)} />
<Button circular color="blue" icon="edit" onClick={() => onUserEdit(user.id)} />
</div>
</Table.Cell>
</Table.Row>
);
});
};
export default function UserTable({ user = [], onUserRemoval, onUserEdit } = {}) {
return (
<Table inverted>
<Table.Header>
<Table.Row>
<Table.HeaderCell>Username</Table.HeaderCell>
<Table.HeaderCell>Last login</Table.HeaderCell>
<Table.HeaderCell>Number of jobs</Table.HeaderCell>
<Table.HeaderCell></Table.HeaderCell>
</Table.Row>
</Table.Header>
<Table.Body>{user.length === 0 ? emptyTable() : content(user, onUserRemoval, onUserEdit)}</Table.Body>
</Table>
);
}

View File

@@ -0,0 +1,58 @@
import React from 'react';
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
import { format } from '../../services/time/timeService';
import { Table, Button, Empty } from '@douyinfe/semi-ui';
import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
const empty = (
<Empty
image={<IllustrationNoResult />}
darkModeImage={<IllustrationNoResultDark />}
description={'No user available'}
/>
);
export default function UserTable({ user = [], onUserRemoval, onUserEdit } = {}) {
return (
<Table
pagination={false}
empty={empty}
columns={[
{
title: 'Username',
dataIndex: 'username',
},
{
title: 'Last login',
dataIndex: 'lastLogin',
render: (value) => {
return format(value);
},
},
{
title: 'Number of jobs',
dataIndex: 'numberOfJobs',
},
{
title: '',
dataIndex: 'tools',
render: (value, user) => {
return (
<div style={{ float: 'right' }}>
<Button
type="danger"
icon={<IconDelete />}
onClick={() => onUserRemoval(user.id)}
style={{ marginRight: '1rem' }}
/>
<Button type="primary" icon={<IconEdit />} onClick={() => onUserEdit(user.id)} />
</div>
);
},
},
]}
dataSource={user}
/>
);
}

View File

@@ -1,27 +0,0 @@
import React from 'react';
import './Toasts.css';
export default function Toast({ id, delay = 5500, message, onHide, backgroundColor, color, title }) {
const [className, setClassname] = React.useState('toast-container show-toast');
React.useEffect(() => {
let hideTimeout = null;
const timeout = setTimeout(() => {
setClassname('toast-container hide-toast');
hideTimeout = setTimeout(() => {
onHide && onHide(id);
}, 500);
}, delay);
return () => {
clearTimeout(timeout);
clearTimeout(hideTimeout);
};
}, [id, delay, onHide]);
return (
<div className={className} style={{ backgroundColor, color }}>
<h5>{title}</h5>
{message}
</div>
);
}

View File

@@ -1,12 +0,0 @@
import Toast from './Toast';
import React from 'react';
export default function ToastsContainer({ toasts, onToastFinished }) {
return (
<div className="toasts-container">
{toasts.map((toast, index) => (
<Toast key={index} {...toast} onHide={onToastFinished} />
))}
</div>
);
}

View File

@@ -1,7 +0,0 @@
import { createContext } from 'react';
const CheckoutDrawerContext = createContext({
showToast: () => {},
});
export default CheckoutDrawerContext;

Some files were not shown because too many files have changed in this diff Show More