mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
UI (#15)
Adding new Admin UI. Updating Fredy to V3.0.0 as it has been a large rewrite. Thanks for all contributions and help on the way!
This commit is contained in:
Christian Kellner
committed by
GitHub
GitHub
parent
8185bfe818
commit
b2847f6834
18
.babelrc
Normal file
18
.babelrc
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"presets": [
|
||||
[
|
||||
"@babel/preset-env",
|
||||
{
|
||||
"exclude": [
|
||||
"transform-regenerator"
|
||||
]
|
||||
}
|
||||
],
|
||||
[
|
||||
"@babel/preset-react",
|
||||
{
|
||||
"runtime": "automatic"
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
270
.eslintrc.js
270
.eslintrc.js
@@ -2,18 +2,21 @@ module.exports = {
|
||||
env: {
|
||||
commonjs: true,
|
||||
es6: true,
|
||||
node: true
|
||||
node: true,
|
||||
browser: true,
|
||||
},
|
||||
extends: 'eslint:recommended',
|
||||
parser: 'babel-eslint',
|
||||
extends: ['eslint:recommended', 'prettier', 'prettier/react'],
|
||||
plugins: ['react'],
|
||||
globals: {
|
||||
Atomics: 'readonly',
|
||||
SharedArrayBuffer: 'readonly',
|
||||
Promise: false,
|
||||
describe: true,
|
||||
it: true
|
||||
it: true,
|
||||
fetch: true,
|
||||
},
|
||||
parserOptions: {
|
||||
ecmaVersion: 2018
|
||||
ecmaVersion: 2018,
|
||||
sourceType: 'module',
|
||||
},
|
||||
rules: {
|
||||
eqeqeq: [2, 'allow-null'],
|
||||
@@ -37,6 +40,257 @@ module.exports = {
|
||||
'linebreak-style': ['error', 'unix'],
|
||||
quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }],
|
||||
semi: ['error', 'always'],
|
||||
'no-console': ['error', { allow: ['warn', 'error'] }]
|
||||
}
|
||||
'no-console': ['error', { allow: ['warn', 'error'] }],
|
||||
|
||||
// ###########################################################
|
||||
// ### React
|
||||
// ###########################################################
|
||||
// Specify whether double or single quotes should be used in JSX attributes
|
||||
// http://eslint.org/docs/rules/jsx-quotes
|
||||
'jsx-quotes': ['error', 'prefer-double'],
|
||||
|
||||
// Prevent missing displayName in a React component definition
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/display-name.md
|
||||
'react/display-name': ['off'],
|
||||
|
||||
// Forbid certain propTypes (any, array, object)
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/forbid-prop-types.md
|
||||
'react/forbid-prop-types': 'off',
|
||||
|
||||
// Validate closing bracket location in JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-closing-bracket-location.md
|
||||
'react/jsx-closing-bracket-location': ['off'],
|
||||
|
||||
// Enforce or disallow spaces inside of curly braces in JSX attributes
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-curly-spacing.md
|
||||
'react/jsx-curly-spacing': ['off'],
|
||||
|
||||
// Enforce event handler naming conventions in JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-handler-names.md
|
||||
'react/jsx-handler-names': [
|
||||
'off',
|
||||
{
|
||||
eventHandlerPrefix: 'handle',
|
||||
eventHandlerPropPrefix: 'on',
|
||||
},
|
||||
],
|
||||
|
||||
// Validate props indentation in JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-indent-props.md
|
||||
'react/jsx-indent-props': 'off',
|
||||
|
||||
// Validate JSX has key prop when in array or iterator
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-key.md
|
||||
'react/jsx-key': 'off',
|
||||
|
||||
// Limit maximum of props on a single line in JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-max-props-per-line.md
|
||||
'react/jsx-max-props-per-line': ['off'],
|
||||
|
||||
// Prevent usage of .bind() in JSX props
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-bind.md
|
||||
'react/jsx-no-bind': [
|
||||
'error',
|
||||
{
|
||||
ignoreRefs: true,
|
||||
allowArrowFunctions: true,
|
||||
allowBind: false,
|
||||
},
|
||||
],
|
||||
|
||||
// Prevent duplicate props in JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-duplicate-props.md
|
||||
'react/jsx-no-duplicate-props': ['error', { ignoreCase: true }],
|
||||
|
||||
// Prevent usage of unwrapped JSX strings
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-literals.md
|
||||
'react/jsx-no-literals': 'off',
|
||||
|
||||
// Disallow undeclared variables in JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-undef.md
|
||||
'react/jsx-no-undef': 'error',
|
||||
|
||||
// Enforce PascalCase for user-defined JSX components
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-pascal-case.md
|
||||
'react/jsx-pascal-case': [
|
||||
'error',
|
||||
{
|
||||
allowAllCaps: true,
|
||||
ignore: [],
|
||||
},
|
||||
],
|
||||
|
||||
// Enforce propTypes declarations alphabetical sorting
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-prop-types.md
|
||||
'react/sort-prop-types': [
|
||||
'off',
|
||||
{
|
||||
ignoreCase: true,
|
||||
callbacksLast: false,
|
||||
requiredFirst: false,
|
||||
},
|
||||
],
|
||||
|
||||
// Deprecated in favor of react/jsx-sort-props
|
||||
'react/jsx-sort-prop-types': 'off',
|
||||
|
||||
// Enforce props alphabetical sorting
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-sort-props.md
|
||||
'react/jsx-sort-props': 'off',
|
||||
|
||||
// Prevent React to be incorrectly marked as unused
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-uses-react.md
|
||||
'react/jsx-uses-react': 'error',
|
||||
|
||||
// Prevent variables used in JSX to be incorrectly marked as unused
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-uses-vars.md
|
||||
'react/jsx-uses-vars': 'error',
|
||||
|
||||
// Prevent usage of dangerous JSX properties
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-danger.md
|
||||
'react/no-danger': 'warn',
|
||||
|
||||
// Prevent usage of deprecated methods
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-deprecated.md
|
||||
'react/no-deprecated': ['error'],
|
||||
|
||||
// Prevent usage of setState in componentDidMount
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-did-mount-set-state.md
|
||||
'react/no-did-mount-set-state': ['error'],
|
||||
|
||||
// Prevent usage of setState in componentDidUpdate
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-did-update-set-state.md
|
||||
'react/no-did-update-set-state': ['warn'],
|
||||
|
||||
// Prevent direct mutation of this.state
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-direct-mutation-state.md
|
||||
'react/no-direct-mutation-state': 'off',
|
||||
|
||||
// Prevent usage of isMounted
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-is-mounted.md
|
||||
'react/no-is-mounted': 'error',
|
||||
|
||||
// Prevent usage of setState
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-set-state.md
|
||||
'react/no-set-state': 'off',
|
||||
|
||||
// Prevent using string references
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-string-refs.md
|
||||
'react/no-string-refs': 'warn',
|
||||
|
||||
// Prevent usage of unknown DOM property
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-unknown-property.md
|
||||
'react/no-unknown-property': 'error',
|
||||
|
||||
// Prevent missing props validation in a React component definition
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/prop-types.md
|
||||
'react/prop-types': ['error', { ignore: [], customValidators: [], skipUndeclared: true }],
|
||||
|
||||
// Prevent missing React when using JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/react-in-jsx-scope.md
|
||||
'react/react-in-jsx-scope': 'error',
|
||||
|
||||
// Restrict file extensions that may be required
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/require-extension.md
|
||||
// deprecated in favor of import/extensions
|
||||
'react/require-extension': ['off', { extensions: ['.jsx', '.js'] }],
|
||||
|
||||
// Require render() methods to return something
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/require-render-return.md
|
||||
'react/require-render-return': 'error',
|
||||
|
||||
// Prevent extra closing tags for components without children
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
|
||||
'react/self-closing-comp': 'warn',
|
||||
|
||||
// Enforce spaces before the closing bracket of self-closing JSX elements
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
|
||||
'react/jsx-space-before-closing': ['warn', 'always'],
|
||||
|
||||
// Enforce component methods order
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
|
||||
'react/sort-comp': 'off',
|
||||
|
||||
// Prevent missing parentheses around multilines JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-wrap-multilines.md
|
||||
'react/jsx-wrap-multilines': [
|
||||
'warn',
|
||||
{
|
||||
declaration: true,
|
||||
assignment: true,
|
||||
return: true,
|
||||
},
|
||||
],
|
||||
'react/wrap-multilines': 'off', // deprecated version
|
||||
|
||||
// Require that the first prop in a JSX element be on a new line when the element is multiline
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-first-prop-new-line.md
|
||||
'react/jsx-first-prop-new-line': ['off'],
|
||||
|
||||
// Enforce spacing around jsx equals signs
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-equals-spacing.md
|
||||
'react/jsx-equals-spacing': ['warn', 'never'],
|
||||
|
||||
// Disallow target="_blank" on links
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-target-blank.md
|
||||
'react/jsx-no-target-blank': 'error',
|
||||
|
||||
// only .jsx files may have JSX
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
|
||||
'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
|
||||
|
||||
// prevent accidental JS comments from being injected into JSX as text
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
|
||||
'react/jsx-no-comment-textnodes': 'error',
|
||||
'react/no-comment-textnodes': 'off', // deprecated version
|
||||
|
||||
// disallow using React.render/ReactDOM.render's return value
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-render-return-value.md
|
||||
'react/no-render-return-value': 'error',
|
||||
|
||||
// require a shouldComponentUpdate method, or PureRenderMixin
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/require-optimization.md
|
||||
'react/require-optimization': ['off', { allowDecorators: [] }],
|
||||
|
||||
// warn against using findDOMNode()
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-find-dom-node.md
|
||||
'react/no-find-dom-node': 'warn',
|
||||
|
||||
// Forbid certain props on Components
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/forbid-component-props.md
|
||||
'react/forbid-component-props': ['off', { forbid: [] }],
|
||||
|
||||
// Prevent problem with children and props.dangerouslySetInnerHTML
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-danger-with-children.md
|
||||
'react/no-danger-with-children': 'error',
|
||||
|
||||
// Prevent unused propType definitions
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-unused-prop-types.md
|
||||
'react/no-unused-prop-types': [
|
||||
'warn',
|
||||
{
|
||||
customValidators: [],
|
||||
skipShapeProps: true,
|
||||
},
|
||||
],
|
||||
|
||||
// Require style prop value be an object or var
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/style-prop-object.md
|
||||
'react/style-prop-object': 'error',
|
||||
|
||||
// Prevent passing of children as props
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
|
||||
'react/no-children-prop': 'warn',
|
||||
|
||||
// Validate whitespace in and around the JSX opening and closing brackets
|
||||
// https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
|
||||
'react/jsx-tag-spacing': [
|
||||
'warn',
|
||||
{
|
||||
closingSlash: 'never',
|
||||
beforeSelfClosing: 'always',
|
||||
afterOpening: 'never',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,5 +1,5 @@
|
||||
node_modules/
|
||||
ui/public/
|
||||
db/
|
||||
npm-debug.log
|
||||
config.json
|
||||
store.json
|
||||
.DS_Store
|
||||
|
||||
10
CHANGELOG.md
10
CHANGELOG.md
@@ -1,3 +1,10 @@
|
||||
###### [V3.0.0]
|
||||
This is basically a re-write, your old config file will not be compatible anymore. Please re-created your search jobs
|
||||
on the new ui and use the values from your previous config file if needed.
|
||||
```
|
||||
- We're getting rid of manual config changes, Fredy, now ships with a UI so that it's easy for you to create and edit jobs
|
||||
```
|
||||
|
||||
###### [V2.0.0]
|
||||
```
|
||||
- Fredy can now run multiple search job on one instance
|
||||
@@ -5,5 +12,4 @@
|
||||
[BREAKING CHANGES]
|
||||
- The config has been changed, the config of V1.x will not work any longer
|
||||
- Sources have been renamed to provider
|
||||
```
|
||||
For more info on how to upgrade from Fredy V1.x, plz check the [Upgrade Guide](./doc/upgrade-from-1-to-2.md)
|
||||
```
|
||||
103
CONTRIBUTION.md
103
CONTRIBUTION.md
@@ -5,14 +5,105 @@ If you want to contribute, please make sure you've executed the tests.
|
||||
|
||||
### How to write new provider?
|
||||
- create the provider filer under `/lib/provider`
|
||||
- create the corresponding config both in `config/config.example` and `config/config.test.json`
|
||||
- make sure the selector matches and that the needed fields are available
|
||||
- create a test under /test and make sure it is running successfully
|
||||
- create a test under /test and make sure it is running successfully
|
||||
|
||||
```javascript
|
||||
let appliedBlackList = [];
|
||||
|
||||
//normalize incoming values
|
||||
function normalize(o) {
|
||||
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
|
||||
|
||||
return Object.assign(o, { id });
|
||||
}
|
||||
|
||||
//apply blaclist if needed
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
//this is the container wrapping the search listings
|
||||
crawlContainer: '#result-list-stage .item',
|
||||
crawlFields: {
|
||||
id: '@id',
|
||||
price: 'div[id*="selPrice_"] | trim',
|
||||
size: 'div[id*="selArea_"] | trim',
|
||||
title: '.item a img@title',
|
||||
link: 'a[id*="lnkImgToDetails_"]@href',
|
||||
address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
|
||||
},
|
||||
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
//you can basically copy & paste this, as this is to initialize the provider with the values from the db
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
//ths
|
||||
exports.metaInformation = {
|
||||
name: 'your provider name',
|
||||
baseUrl: 'https://www.yourprovider.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
exports.config = config;
|
||||
|
||||
```
|
||||
|
||||
|
||||
### How to write new notification adapter?
|
||||
- create the provider filer under `/lib/notification/adapter`
|
||||
- make sure it exports a function `enabled` and a function `send`
|
||||
- create the corresponding config both in `config/config.example` and `config/config.test.json`
|
||||
- create a description of the provider under `/lib/notification/adapter/*.md`. Make sure the name of the md file is equal to the notification adapter
|
||||
|
||||
The notification adapter itself dictates how the frontend should be rendered in order to collect all necessary keys.
|
||||
|
||||
```javascript
|
||||
const Slack = require('slack');
|
||||
const msg = Slack.chat.postMessage;
|
||||
const { markdown2Html } = require('../../services/markdown');
|
||||
|
||||
|
||||
//as a parameter, you will always get the serviceName, newListings and all the values, that
|
||||
//you have defined exports.config.fields. (This is being used for rendering in the frontend)
|
||||
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
|
||||
return newListings.map((payload) => {
|
||||
//tho whatever needs to be done to send the data to the receiver, make sure the format is human readable
|
||||
});
|
||||
};
|
||||
|
||||
exports.config = {
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
name: 'someUniqueName, used in the frontend',
|
||||
//this readme is rendered in the frontend to explain how to use this
|
||||
readme: markdown2Html('lib/notification/adapter/slack.md'),
|
||||
description: 'Some description text rendered on the notification page',
|
||||
fields: {
|
||||
token: {
|
||||
//type can be text/number/boolean
|
||||
type: 'text',
|
||||
label: 'Token',
|
||||
description: 'The token needed to send notifications to slack.',
|
||||
},
|
||||
channel: {
|
||||
type: 'channel',
|
||||
label: 'Channel',
|
||||
description: 'The channel where fredy should send notifications to.',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
```
|
||||
|
||||
#### Running Tests
|
||||
If you've written a new provider you are an awesome person. You know it and I do. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
|
||||
@@ -24,7 +115,7 @@ I'm using Eslint to maintain quote style and quality. Do not skip it...
|
||||
|
||||
##### To do before merging:
|
||||
|
||||
- executed tests? (`npm run test`)
|
||||
- executed tests? (`yarn run test`)
|
||||
- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
|
||||
|
||||
_Thanks!_ :heart:
|
||||
|
||||
@@ -12,8 +12,8 @@ RUN cd /usr/src/fredy/ && npm install
|
||||
|
||||
WORKDIR /usr/src/fredy
|
||||
|
||||
EXPOSE 9876
|
||||
EXPOSE 9998
|
||||
|
||||
VOLUME [ "/conf" ]
|
||||
VOLUME [ "/conf", "/db" ]
|
||||
# --no-daemon is required for keeping Container alive
|
||||
CMD node index.js --no-daemon
|
||||
2
LICENSE
2
LICENSE
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2020 Christian Kellner
|
||||
Copyright (c) 2021 Christian Kellner
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
220
README.md
220
README.md
@@ -2,183 +2,67 @@
|
||||
|
||||
[](https://travis-ci.org/orangecoding/fredy)
|
||||
|
||||
My wife and I wanted to buy an apartment in germany. As the prices are quite high and good deals are very rare, we searched the "big 4" every morning.
|
||||
_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) as often as you want and send new listings to you once they appear. The list of available services can easily be extended. For your convenience, a ui helps you to configure your search jobs.
|
||||
|
||||
This however can get pretty frustrating. _Fredy_ will take this work away from you. It crawls the `provider`, mentioned below (Immonet, Immoscout...) every _x_ minutes. (The provider list can be extended easily...)
|
||||
|
||||
If _Fredy_ found matching results, it will send them to you via Slack. (More adapter possible.) _Fredy_ is remembering what it already has found to not send stuff twice.
|
||||
If _Fredy_ found matching results, it will send them to you via Slack, Email, Telegram etc. (More adapter possible.) As _Fredy_ will store the listings it found, new results will not be sent twice (and as a side-effect, _Fredy_ can show some statistics..)
|
||||
|
||||
## Usage
|
||||
|
||||
- Make sure to use Node 11 and above
|
||||
- Install the dependencies using `npm install` or `yarn`
|
||||
- create your configuration file. Use the example config for this. `cp conf/config.example conf/config.json`
|
||||
- configure the desired values
|
||||
- start _Fredy_ using `npm start` or `yarn run start`
|
||||
|
||||
## :point_up: Breaking Changes when updating from v1.x to v2
|
||||
See [Upgrade Guide](./doc/upgrade-from-1-to-2.md)
|
||||
|
||||
## Configuration
|
||||
|
||||
Before running _Fredy_ for the first time, you need to create a configuration file:
|
||||
|
||||
Copy the example config as a start.
|
||||
```
|
||||
cp conf/config.example conf/config.json
|
||||
```
|
||||
|
||||
### 1. Notification
|
||||
|
||||
You want to get notified when _Fredy_ found a new listing. Currently _Fredy_ support Slack and Telegram to send notification. _Fredy_ also includes a notification adapter to print to the console instead of sending to a services.
|
||||
|
||||
Adding new notification adapter is easy however. See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)
|
||||
|
||||
##### Slack
|
||||
```json
|
||||
"slack": {
|
||||
"channel": "someChannel",
|
||||
"token": "someToken",
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
##### Telegram
|
||||
```json
|
||||
"telegram": {
|
||||
"chatId": "someChannel",
|
||||
"token": "someToken",
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
|
||||
A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.
|
||||
After the user has send a message to your bot the first time, you can gather the chatId like this:
|
||||
```
|
||||
curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
|
||||
```
|
||||
|
||||
A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
|
||||
- Make sure to use NodeJs 12 and above
|
||||
- Run the following commands
|
||||
```ssh
|
||||
yarn (or npm install)
|
||||
yarn run prod
|
||||
yarn run start
|
||||
```
|
||||
_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` for username and password. (You should change the password asap when you plan to run Fredy on your server.)
|
||||
|
||||
##### SendGrid
|
||||
```json
|
||||
"sendGrid": {
|
||||
"apiKey": "",
|
||||
"from": "",
|
||||
"receiver": "",
|
||||
"templateId": "",
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
If you don't use Slack, I'd personally recommend using Sendgrid or Mailjet. Both are free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
|
||||
|
||||
To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.
|
||||
Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new template. For this new template, I recommend copying and pasting the one I provided under `/lib/notification/emailTemplate/template.hbs`.
|
||||
## Understanding the fundamentals
|
||||
There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
|
||||
|
||||
##### MailJet
|
||||
```json
|
||||
"mailJet": {
|
||||
"apiPublicKey": "",
|
||||
"apiPrivateKey": "",
|
||||
"receiver": "",
|
||||
"from": "",
|
||||
"enabled": false
|
||||
},
|
||||
```
|
||||
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.
|
||||
Feed the given piublic/private api key into Fredy's config and enable the adapter. Fredy will use the same template, it is using for SendGrid.
|
||||
#### Adapter
|
||||
_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few. Those services are called adapter within _Fredy_. When creating a new job, you can choose 1 or many adapter.
|
||||
An adapter holds the url that points to the search results for the service. If you go to immonet.de and search for something, the shown url in the browser is what the adapter needs to do it's magic.
|
||||
**It is important that you order the search results by date, so that _Fredy_ always picks the latest ones first**
|
||||
|
||||
|
||||
### 2. Multiple search jobs
|
||||
#### Provider
|
||||
_Fredy_ supports multiple provider. Slack, SendGrid, Telegram etc. A search job can have as many provider as supported by _Fredy_. Each provider needs different configuration values, which you have to provide when using it. A provider itself dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
|
||||
|
||||
Since v2.0.0, Fredy supports multiple search jobs running within the same instance of Fredy. For this to work correctly, you need to give each search job a unique name.
|
||||
See the example config...
|
||||
```json
|
||||
(...)
|
||||
"jobs": {
|
||||
"yourSearchJob": {
|
||||
"some":"config"
|
||||
},
|
||||
"yourOtherSearchJob": {
|
||||
"some":"config"
|
||||
}
|
||||
}
|
||||
(...)
|
||||
```
|
||||
|
||||
### 3. Configure the providers
|
||||
|
||||
Configure the providers like described below. To disable a provider just set to `enabled:false` within the config.
|
||||
|
||||
#### Immoscout, Immonet and more
|
||||
|
||||
These are the current provider that are already implemented within _Fredy_
|
||||
|
||||
```json
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/...", "enabled": true}
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/...", "enabled": true},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/...", "enabled": true},
|
||||
"immonet": {
|
||||
"url": "http://www.immonet.de/...", "enabled": true},
|
||||
"kalaydo": {
|
||||
"url": "http://www.kalaydo.de/...", "enabled": true},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/...", "enabled": true},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/...", "enabled": true},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/...", "enabled": true}
|
||||
```
|
||||
|
||||
Go to the respective provider page and create your custom search queries by using the provided filter options. Then just copy and paste the whole URL of
|
||||
the resulting listings page.
|
||||
|
||||
**IMPORTANT:** Make sure to always sort by newest listings! This way _Fredy_ makes sure to not accidentally report stuff twice.
|
||||
|
||||
#### Custom provider
|
||||
#### Jobs
|
||||
A Job wraps adapter and provider. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
|
||||
|
||||
## Creating your first job
|
||||
To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self explanatory, however there's one important thing.
|
||||
When configuring adapter, before copying the url from your browser make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
|
||||
|
||||
## User management
|
||||
As an administrator, you can create/edit and remove user from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, the jobs will also be removed.
|
||||
|
||||
|
||||
# Development
|
||||
|
||||
### Running Fredy in dev mode
|
||||
To run _Fredy_ in dev mode, you need to run the backend & frontend separately. Run the backend in your favorite IDE, the frontend can be started from the terminal.
|
||||
```shell
|
||||
yarn run dev
|
||||
```
|
||||
You should now be able to access _Fredy_ with your browser. Go to `http://localhost:9000`
|
||||
|
||||
### Running Tests
|
||||
To run the tests, simply run
|
||||
```shell
|
||||
yarn run test
|
||||
```
|
||||
|
||||
# Architecture
|
||||
|
||||
|
||||
|
||||
|
||||
#### Contribution guidelines
|
||||
|
||||
See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)
|
||||
|
||||
### 4. Add Filters (optional)
|
||||
|
||||
|
||||
#### Blacklist
|
||||
|
||||
```json
|
||||
"blacklist": [
|
||||
"vermietet"
|
||||
]
|
||||
```
|
||||
|
||||
Listings which contain at least on of the given terms (ignoring case, only whole words) are removed.
|
||||
|
||||
#### District blacklist
|
||||
```json
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt"
|
||||
]
|
||||
```
|
||||
Districts that are unwanted can be blacklisted here.
|
||||
|
||||
This makes sense for provider that only offer limited filter functions like Kalaydo/Ebay.
|
||||
|
||||
# API
|
||||
While Fredy is running, you can make use of the rest api provided on port `9998` to get information about the current status of Fredy.
|
||||
You can disable the api by setting `infoApi: false` within the config.
|
||||
|
||||
#### http://localhost:9998/
|
||||
Gives you an overview of running search jobs, their included enabled provider, last execution and the number of listings, found by each provider.
|
||||
|
||||
#### http://localhost:9998/ping
|
||||
Should you ever need some health checks, this returns pong ;)
|
||||
|
||||
#### http://localhost:9998/jobs/:name
|
||||
Returns specific information about the job with the given name or `404` if the job could not be found.
|
||||
|
||||
# Docker
|
||||
Use the Dockerfile in this Repository to build an image.
|
||||
|
||||
@@ -186,9 +70,9 @@ Example: `docker build -t fredy/fredy /path/to/your/Dockerfile`
|
||||
|
||||
## Create & run a container
|
||||
|
||||
Put your config.json to `/path/to/your/conf/`
|
||||
Put your config.json to `/path/to/your/conf/`
|
||||
|
||||
Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9876:9876 fredy/fredy`
|
||||
Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9988:9988 fredy/fredy`
|
||||
|
||||
## Logs
|
||||
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
{
|
||||
"interval": 1,
|
||||
"infoApi": true,
|
||||
"infoApiPort": 9998,
|
||||
"jobs": {
|
||||
"yourSearchJob": {
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "",
|
||||
"channel": "",
|
||||
"enabled": false
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"sendGrid": {
|
||||
"apiKey": "",
|
||||
"from": "",
|
||||
"receiver": "",
|
||||
"templateId": "",
|
||||
"enabled": false
|
||||
},
|
||||
"mailJet": {
|
||||
"apiPublicKey": "",
|
||||
"apiPrivateKey": "",
|
||||
"receiver": "",
|
||||
"from": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"provider": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/...",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/...",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/...",
|
||||
"enabled": true
|
||||
},
|
||||
"abInsZuHause": {
|
||||
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/...",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/...",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/...",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Friedrichstadt",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Wittlaer",
|
||||
"Lörick"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
4
conf/config.json
Executable file
4
conf/config.json
Executable file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"interval": 30,
|
||||
"port": 9998
|
||||
}
|
||||
@@ -1,167 +0,0 @@
|
||||
{
|
||||
"interval": 1,
|
||||
"jobs": {
|
||||
"test1": {
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "",
|
||||
"channel": "",
|
||||
"enabled": false
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"provider": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||
"enabled": true
|
||||
},
|
||||
"abInsZuHause": {
|
||||
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/D%C3%BCsseldorf/wohnung-kaufen/420000/90/3.5/1839/",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Dusseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Friedrichstadt",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Wittlaer",
|
||||
"Lörick"
|
||||
]
|
||||
},
|
||||
"test2": {
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "",
|
||||
"channel": "",
|
||||
"enabled": false
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"provider": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||
"enabled": true
|
||||
},
|
||||
"abInsZuHause": {
|
||||
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/D%C3%BCsseldorf/wohnung-kaufen/420000/90/3.5/1839/",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Friedrichstadt",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Wittlaer",
|
||||
"Lörick"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
BIN
doc/architecture.jpg
Normal file
BIN
doc/architecture.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 134 KiB |
BIN
doc/logo_white.png
Normal file
BIN
doc/logo_white.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
@@ -1,19 +0,0 @@
|
||||
# Upgrading from v1.x to v2.0.0
|
||||
|
||||
Fredy 2.0.0 introduced the concept of multiple jobs running within an instance of Fredy. For this to work, I had to change the config and the storage format.
|
||||
|
||||
### How to update?
|
||||
##### Store
|
||||
It's best to clear the store completely and let Fredy rewrite it. Be careful to disable all notification adapter the first time you run Fredy 2, as it will obviously treat
|
||||
everything as new.
|
||||
|
||||
##### Config
|
||||
The config format has changed. It now supports multiple jobs. It is probably easiest to simply copy the `config.example` from `/conf` and enter your urls in there.
|
||||
The new format basically wraps the config in chunks.
|
||||
|
||||
```json
|
||||
"jobs": {
|
||||
"yourSearchJob": {
|
||||
"some":"stuff"
|
||||
}
|
||||
```
|
||||
50
index.js
50
index.js
@@ -1,36 +1,48 @@
|
||||
const fs = require('fs');
|
||||
const path = './lib/provider';
|
||||
const provider = fs.readdirSync(path);
|
||||
const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
|
||||
const config = require('./conf/config.json');
|
||||
const { setLastJobExecution, init: storeInit } = require('./lib/services/store');
|
||||
|
||||
const jobStorage = require('./lib/services/storage/jobStorage');
|
||||
const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
|
||||
const FredyRuntime = require('./lib/FredyRuntime');
|
||||
|
||||
//starting the api service
|
||||
require('./lib/api/api');
|
||||
|
||||
storeInit().then(() => {
|
||||
setInterval(
|
||||
(function exec() {
|
||||
Object.keys(config.jobs).forEach(jobKey => {
|
||||
const jobConfig = config.jobs[jobKey];
|
||||
//assuming interval is always in minutes
|
||||
const INTERVAL = config.interval * 60 * 1000;
|
||||
|
||||
/* eslint-disable no-console */
|
||||
console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
|
||||
/* eslint-enable no-console */
|
||||
setInterval(
|
||||
(function exec() {
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.forEach((job) => {
|
||||
const providerIds = job.provider.map((provider) => provider.id);
|
||||
|
||||
provider
|
||||
.map(pro => require(`${path}/${pro}`))
|
||||
.forEach(pro => {
|
||||
const providerId = pro.id();
|
||||
.filter((provider) => provider.endsWith('.js'))
|
||||
.map((pro) => require(`${path}/${pro}`))
|
||||
.filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
|
||||
.forEach(async (pro) => {
|
||||
const providerId = pro.metaInformation.id;
|
||||
if (providerId == null || providerId.length === 0) {
|
||||
throw new Error('Provider id must not be empty. => ' + pro);
|
||||
}
|
||||
const providerConfig = jobConfig.provider[providerId];
|
||||
const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
|
||||
if (providerConfig == null) {
|
||||
throw new Error(`Provider Config for provider with id ${providerId} not found.`);
|
||||
}
|
||||
pro.init(providerConfig, jobConfig.blacklist, jobConfig.blacklistedDistricts);
|
||||
new FredyRuntime(pro.config, jobConfig.notification, providerId, jobKey).execute();
|
||||
setLastJobExecution(jobKey);
|
||||
pro.init(providerConfig, job.blacklist);
|
||||
await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute();
|
||||
setLastJobExecution(job.id);
|
||||
});
|
||||
});
|
||||
return exec;
|
||||
})(),
|
||||
config.interval * 60 * 1000
|
||||
);
|
||||
});
|
||||
return exec;
|
||||
})(),
|
||||
INTERVAL
|
||||
);
|
||||
|
||||
@@ -1,10 +1,5 @@
|
||||
const { NoNewListingsError } = require('./errors');
|
||||
const {
|
||||
setKnownListings,
|
||||
getKnownListings,
|
||||
setNumberOfTotalFoundProviderListings,
|
||||
getForTesting,
|
||||
} = require('./services/store');
|
||||
const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
|
||||
|
||||
const notify = require('./notification/notify');
|
||||
const xray = require('./services/scraper');
|
||||
@@ -13,7 +8,7 @@ class FredyRuntime {
|
||||
/**
|
||||
*
|
||||
* @param providerConfig the config for the specific provider, we're going to query at the moment
|
||||
* @param notificationConfig the config for all notifications (because all could be applied to a provider)
|
||||
* @param notificationConfig the config for all notifications
|
||||
* @param providerId the id of the provider currently in use
|
||||
* @param jobKey key of the job that is currently running (from within the config)
|
||||
*/
|
||||
@@ -25,8 +20,6 @@ class FredyRuntime {
|
||||
}
|
||||
|
||||
execute() {
|
||||
if (!this._providerConfig.enabled) return Promise.resolve();
|
||||
|
||||
return (
|
||||
Promise.resolve(this._providerConfig.url)
|
||||
//scraping the site and try finding new listings
|
||||
@@ -37,8 +30,6 @@ class FredyRuntime {
|
||||
.then(this._filter.bind(this))
|
||||
//check if new listings available. if so proceed
|
||||
.then(this._findNew.bind(this))
|
||||
//store update of number of found listings
|
||||
.then(this._storeStats.bind(this))
|
||||
//store everything in db
|
||||
.then(this._save.bind(this))
|
||||
//notify the user using the configured notification adapter
|
||||
@@ -52,24 +43,16 @@ class FredyRuntime {
|
||||
return new Promise((resolve, reject) => {
|
||||
let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
|
||||
|
||||
if (this._providerConfig.paginage) {
|
||||
x = x.paginate(this._providerConfig.paginage);
|
||||
}
|
||||
|
||||
x((err, listings) => {
|
||||
if (err) reject(err);
|
||||
else {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve(listings);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
_storeStats(listings) {
|
||||
setNumberOfTotalFoundProviderListings(this._jobKey, this._providerId, listings.length);
|
||||
return Promise.resolve(listings);
|
||||
}
|
||||
|
||||
_normalize(listings) {
|
||||
return listings.map(this._providerConfig.normalize);
|
||||
}
|
||||
@@ -79,7 +62,7 @@ class FredyRuntime {
|
||||
}
|
||||
|
||||
_findNew(listings) {
|
||||
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId).indexOf(o.id) === -1);
|
||||
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
|
||||
|
||||
if (newListings.length === 0) {
|
||||
throw new NoNewListingsError();
|
||||
@@ -94,25 +77,17 @@ class FredyRuntime {
|
||||
}
|
||||
|
||||
_save(newListings) {
|
||||
setKnownListings(this._jobKey, this._providerId, [
|
||||
...getKnownListings(this._jobKey, this._providerId),
|
||||
...newListings.map((l) => l.id),
|
||||
]);
|
||||
const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
|
||||
newListings.forEach((listing) => {
|
||||
currentListings[listing.id] = Date.now();
|
||||
});
|
||||
setKnownListings(this._jobKey, this._providerId, currentListings);
|
||||
return newListings;
|
||||
}
|
||||
|
||||
_handleError(err) {
|
||||
if (err.name !== 'NoNewListingsError') console.error(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* for testing purposes only
|
||||
* @returns {Store}
|
||||
* @private
|
||||
*/
|
||||
_getStore() {
|
||||
return getForTesting();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = FredyRuntime;
|
||||
|
||||
@@ -1,68 +1,42 @@
|
||||
const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
|
||||
const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
|
||||
const { analyticsRouter } = require('./routes/analyticsRouter');
|
||||
const { providerRouter } = require('./routes/providerRouter');
|
||||
const { loginRouter } = require('./routes/loginRoute');
|
||||
const config = require('../../conf/config.json');
|
||||
const { userRouter } = require('./routes/userRoute');
|
||||
const { jobRouter } = require('./routes/jobRouter');
|
||||
const bodyParser = require('body-parser');
|
||||
const config = require('../../conf/config');
|
||||
const { getLastJobExecution, getLastProviderExecution, getTotalNumberOfListings } = require('../services/store');
|
||||
const PORT = config.infoApiPort || 9998;
|
||||
const service = require('restana')();
|
||||
const enabled = config.infoApi == null ? false : config.infoApi;
|
||||
const files = require('serve-static');
|
||||
const path = require('path');
|
||||
|
||||
const staticService = files(path.join(__dirname, '../../ui/public'));
|
||||
|
||||
const PORT = config.port || 9998;
|
||||
|
||||
service.use(bodyParser.json());
|
||||
|
||||
service.get('/', async (req, res) => {
|
||||
const result = {};
|
||||
Object.keys(config.jobs).forEach((job) => {
|
||||
result[job] = {
|
||||
lastExecution: getLastJobExecution(job),
|
||||
enabledProvider: Object.keys(config.jobs[job].provider)
|
||||
.filter((providerKey) => config.jobs[job].provider[providerKey].enabled)
|
||||
.map((providerKey) => {
|
||||
return {
|
||||
name: providerKey,
|
||||
lastExecution: getLastProviderExecution(job, providerKey),
|
||||
totalFindings: getTotalNumberOfListings(job, providerKey),
|
||||
};
|
||||
}),
|
||||
};
|
||||
});
|
||||
res.body = result;
|
||||
res.send();
|
||||
});
|
||||
service.use(cookieSession());
|
||||
|
||||
service.get('/jobs/:name', async (req, res) => {
|
||||
const { name: jobKey } = req.params;
|
||||
if (Object.keys(config.jobs).indexOf(jobKey) === -1) {
|
||||
console.error(`Cannot find job with name ${jobKey}. Available Jobs are [${Object.keys(config.jobs)}]`);
|
||||
res.send(404);
|
||||
return;
|
||||
}
|
||||
res.body = {
|
||||
lastExecution: getLastJobExecution(jobKey),
|
||||
enabledProvider: Object.keys(config.jobs[jobKey].provider)
|
||||
.filter((providerKey) => config.jobs[jobKey].provider[providerKey].enabled)
|
||||
.map((providerKey) => {
|
||||
return {
|
||||
name: providerKey,
|
||||
url: config.jobs[jobKey].provider[providerKey].url,
|
||||
lastExecution: getLastProviderExecution(jobKey, providerKey),
|
||||
totalFindings: getTotalNumberOfListings(jobKey, providerKey),
|
||||
};
|
||||
}),
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
service.use(staticService);
|
||||
|
||||
service.get('/ping', function (req, res) {
|
||||
res.body = {
|
||||
pong: 'pong',
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
service.use('/api/admin', authInterceptor());
|
||||
service.use('/api/jobs', authInterceptor());
|
||||
|
||||
// /admin can only be accessed when user is having admin permissions
|
||||
service.use('/api/admin', adminInterceptor());
|
||||
|
||||
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
|
||||
service.use('/api/jobs/provider', providerRouter);
|
||||
service.use('/api/jobs/insights', analyticsRouter);
|
||||
service.use('/api/admin/users', userRouter);
|
||||
service.use('/api/jobs', jobRouter);
|
||||
|
||||
service.use('/api/login', loginRouter);
|
||||
|
||||
/* eslint-disable no-console */
|
||||
if (enabled) {
|
||||
service.start(PORT).then(() => {
|
||||
console.info(`Started API service on port ${PORT}`);
|
||||
});
|
||||
} else {
|
||||
console.info('Info Api is disabled.');
|
||||
}
|
||||
service.start(PORT).then(() => {
|
||||
console.info(`Started API service on port ${PORT}`);
|
||||
});
|
||||
/* eslint-enable no-console */
|
||||
|
||||
12
lib/api/routes/analyticsRouter.js
Normal file
12
lib/api/routes/analyticsRouter.js
Normal file
@@ -0,0 +1,12 @@
|
||||
const service = require('restana')();
|
||||
const analyticsRouter = service.newRouter();
|
||||
const listingStorage = require('../../services/storage/listingsStorage');
|
||||
|
||||
analyticsRouter.get('/:jobId', async (req, res) => {
|
||||
const { jobId } = req.params;
|
||||
|
||||
res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
|
||||
res.send();
|
||||
});
|
||||
|
||||
exports.analyticsRouter = analyticsRouter;
|
||||
84
lib/api/routes/jobRouter.js
Normal file
84
lib/api/routes/jobRouter.js
Normal file
@@ -0,0 +1,84 @@
|
||||
const service = require('restana')();
|
||||
const jobRouter = service.newRouter();
|
||||
const jobStorage = require('../../services/storage/jobStorage');
|
||||
const userStorage = require('../../services/storage/userStorage');
|
||||
const { isAdmin } = require('../security');
|
||||
|
||||
function doesJobBelongsToUser(job, req) {
|
||||
const userId = req.session.currentUser;
|
||||
if (userId == null) {
|
||||
return false;
|
||||
}
|
||||
const user = userStorage.getUser(userId);
|
||||
if (user == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return user.isAdmin || job.userId === job.userId;
|
||||
}
|
||||
|
||||
jobRouter.get('/', async (req, res) => {
|
||||
const isUserAdmin = isAdmin(req);
|
||||
|
||||
//show only the jobs which belongs to the user (or all of the user is an admin)
|
||||
res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
|
||||
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.post('/', async (req, res) => {
|
||||
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
|
||||
try {
|
||||
jobStorage.upsertJob({
|
||||
userId: req.session.currentUser,
|
||||
jobId,
|
||||
enabled,
|
||||
name,
|
||||
blacklist,
|
||||
provider,
|
||||
notificationAdapter,
|
||||
});
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
console.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.delete('', async (req, res) => {
|
||||
const { jobId } = req.body;
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send(new Error('You are trying to remove a job that is not associated to your user'));
|
||||
} else {
|
||||
jobStorage.removeJob(jobId);
|
||||
}
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
console.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.put('/:jobId/status', async (req, res) => {
|
||||
const { status } = req.body;
|
||||
const { jobId } = req.params;
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send(new Error('You are trying change a job that is not associated to your user'));
|
||||
} else {
|
||||
jobStorage.setJobStatus({
|
||||
jobId,
|
||||
status,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
console.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
exports.jobRouter = jobRouter;
|
||||
47
lib/api/routes/loginRoute.js
Normal file
47
lib/api/routes/loginRoute.js
Normal file
@@ -0,0 +1,47 @@
|
||||
const service = require('restana')();
|
||||
const loginRouter = service.newRouter();
|
||||
const userStorage = require('../../services/storage/userStorage');
|
||||
const hasher = require('../../services/security/hash');
|
||||
|
||||
loginRouter.get('/user', async (req, res) => {
|
||||
const currentUserId = req.session.currentUser;
|
||||
const isAdmin = currentUserId == null ? false : userStorage.getUser(currentUserId).isAdmin;
|
||||
if (currentUserId == null) {
|
||||
res.body = {};
|
||||
} else {
|
||||
res.body = {
|
||||
userId: currentUserId,
|
||||
isAdmin,
|
||||
};
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
loginRouter.post('/', async (req, res) => {
|
||||
const { username, password } = req.body;
|
||||
|
||||
const user = userStorage.getUsers(true).find((user) => user.username === username);
|
||||
|
||||
if (user == null) {
|
||||
res.send(401);
|
||||
return;
|
||||
}
|
||||
|
||||
if (user.password === hasher.hash(password)) {
|
||||
req.session.currentUser = user.id;
|
||||
userStorage.setLastLoginToNow({ userId: user.id });
|
||||
res.send(200);
|
||||
return;
|
||||
} else {
|
||||
console.error(`User ${username} tried to login, but password was wrong.`);
|
||||
}
|
||||
|
||||
res.send(401);
|
||||
});
|
||||
|
||||
loginRouter.post('/logout', async (req, res) => {
|
||||
req.session = null;
|
||||
res.send(200);
|
||||
});
|
||||
|
||||
exports.loginRouter = loginRouter;
|
||||
54
lib/api/routes/notificationAdapterRouter.js
Normal file
54
lib/api/routes/notificationAdapterRouter.js
Normal file
@@ -0,0 +1,54 @@
|
||||
const fs = require('fs');
|
||||
const service = require('restana')();
|
||||
const notificationAdapterRouter = service.newRouter();
|
||||
|
||||
const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
|
||||
|
||||
const notificationAdapter = notificationAdapterList.map((pro) => {
|
||||
return require(`../../notification/adapter/${pro}`);
|
||||
});
|
||||
|
||||
notificationAdapterRouter.post('/try', async (req, res) => {
|
||||
const { id, fields } = req.body;
|
||||
const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
|
||||
if (adapter == null) {
|
||||
res.send(404);
|
||||
}
|
||||
const notificationConfig = [];
|
||||
const notificationObject = {};
|
||||
Object.keys(fields).forEach((key) => {
|
||||
notificationObject[key] = fields[key].value;
|
||||
});
|
||||
notificationConfig.push({
|
||||
...notificationObject,
|
||||
enabled: true,
|
||||
id,
|
||||
});
|
||||
Promise.all(
|
||||
adapter.send({
|
||||
serviceName: 'TestCall',
|
||||
newListings: [
|
||||
{
|
||||
price: '42 €',
|
||||
title: 'This is a test listing',
|
||||
address: 'some address',
|
||||
size: '666 2m',
|
||||
link: 'https://www.orange-coding.net',
|
||||
},
|
||||
],
|
||||
notificationConfig,
|
||||
jobKey: 'TestJob',
|
||||
})
|
||||
)
|
||||
.then(() => res.send())
|
||||
.catch((error) => {
|
||||
res.send(new Error(error));
|
||||
});
|
||||
});
|
||||
|
||||
notificationAdapterRouter.get('/', async (req, res) => {
|
||||
res.body = notificationAdapter.map((adapter) => adapter.config);
|
||||
res.send();
|
||||
});
|
||||
|
||||
exports.notificationAdapterRouter = notificationAdapterRouter;
|
||||
16
lib/api/routes/providerRouter.js
Normal file
16
lib/api/routes/providerRouter.js
Normal file
@@ -0,0 +1,16 @@
|
||||
const fs = require('fs');
|
||||
const service = require('restana')();
|
||||
const providerRouter = service.newRouter();
|
||||
|
||||
const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
|
||||
|
||||
const provider = providerList.map((pro) => {
|
||||
return require(`../../provider/${pro}`).metaInformation;
|
||||
});
|
||||
|
||||
providerRouter.get('/', async (req, res) => {
|
||||
res.body = provider;
|
||||
res.send();
|
||||
});
|
||||
|
||||
exports.providerRouter = providerRouter;
|
||||
76
lib/api/routes/userRoute.js
Normal file
76
lib/api/routes/userRoute.js
Normal file
@@ -0,0 +1,76 @@
|
||||
const service = require('restana')();
|
||||
const userRouter = service.newRouter();
|
||||
const userStorage = require('../../services/storage/userStorage');
|
||||
const jobStorage = require('../../services/storage/jobStorage');
|
||||
|
||||
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
|
||||
return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
|
||||
}
|
||||
|
||||
function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
|
||||
return req.session.currentUser === userIdToBeRemoved;
|
||||
}
|
||||
|
||||
const nullOrEmpty = (str) => str == null || str.length === 0;
|
||||
|
||||
userRouter.get('/', async (req, res) => {
|
||||
res.body = userStorage.getUsers(false);
|
||||
res.send();
|
||||
});
|
||||
|
||||
userRouter.get('/:userId', async (req, res) => {
|
||||
const { userId } = req.params;
|
||||
res.body = userStorage.getUser(userId);
|
||||
res.send();
|
||||
});
|
||||
|
||||
userRouter.delete('/', async (req, res) => {
|
||||
const { userId } = req.body;
|
||||
const allUser = userStorage.getUsers(false);
|
||||
|
||||
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||
res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
|
||||
return;
|
||||
}
|
||||
if (checkIfUserToBeRemovedIsLoggedIn(userId, req)) {
|
||||
res.send(new Error('You are trying to remove yourself. This is prohibited.'));
|
||||
return;
|
||||
}
|
||||
|
||||
//TODO: Remove also analytics
|
||||
jobStorage.removeJobsByUserId(userId);
|
||||
userStorage.removeUser(userId);
|
||||
|
||||
res.send();
|
||||
});
|
||||
|
||||
userRouter.post('/', async (req, res) => {
|
||||
const { username, password, password2, isAdmin, userId } = req.body;
|
||||
if (password !== password2) {
|
||||
res.send(new Error('Passwords does not match'));
|
||||
return;
|
||||
}
|
||||
if (nullOrEmpty(username) || nullOrEmpty(password) || nullOrEmpty(password2)) {
|
||||
res.send(new Error('Username and password are mandatory.'));
|
||||
return;
|
||||
}
|
||||
const allUser = userStorage.getUsers(false);
|
||||
|
||||
if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||
res.send(
|
||||
new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
userStorage.upsertUser({
|
||||
userId,
|
||||
username,
|
||||
password,
|
||||
isAdmin,
|
||||
});
|
||||
|
||||
res.send();
|
||||
});
|
||||
|
||||
exports.userRouter = userRouter;
|
||||
53
lib/api/security.js
Normal file
53
lib/api/security.js
Normal file
@@ -0,0 +1,53 @@
|
||||
const userStorage = require('../services/storage/userStorage');
|
||||
const cookieSession = require('cookie-session');
|
||||
const { nanoid } = require('nanoid');
|
||||
|
||||
const unauthorized = (res) => {
|
||||
return res.send(401);
|
||||
};
|
||||
|
||||
const isUnauthorized = (req) => {
|
||||
return req.session.currentUser == null;
|
||||
};
|
||||
|
||||
const isAdmin = (req) => {
|
||||
if (!isUnauthorized(req)) {
|
||||
const user = userStorage.getUser(req.session.currentUser);
|
||||
return user != null && user.isAdmin;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const authInterceptor = () => {
|
||||
return (req, res, next) => {
|
||||
if (isUnauthorized(req)) {
|
||||
return unauthorized(res);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const adminInterceptor = () => {
|
||||
return (req, res, next) => {
|
||||
if (!isAdmin(req)) {
|
||||
return unauthorized(res);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
exports.cookieSession = (userId) => {
|
||||
return cookieSession({
|
||||
name: 'fredy-admin-session',
|
||||
keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
|
||||
userId,
|
||||
maxAge: 24 * 60 * 60 * 1000, // 24 hours
|
||||
});
|
||||
};
|
||||
|
||||
exports.adminInterceptor = adminInterceptor;
|
||||
exports.authInterceptor = authInterceptor;
|
||||
exports.isUnauthorized = isUnauthorized;
|
||||
exports.isAdmin = isAdmin;
|
||||
@@ -1,16 +1,21 @@
|
||||
const { markdown2Html } = require('../../services/markdown');
|
||||
|
||||
/**
|
||||
* simply prints out the found data to the console
|
||||
* @param serviceName e.g immoscout
|
||||
* @param serviceName e.g immowelt
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* @param jobKey name of the current job that is being executed
|
||||
*/
|
||||
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
const { enabled } = notificationConfig.console;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
exports.send = ({ serviceName, newListings, jobKey }) => {
|
||||
/* eslint-disable no-console */
|
||||
return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
|
||||
/* eslint-enable no-console */
|
||||
};
|
||||
|
||||
exports.config = {
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
name: 'Console',
|
||||
description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
|
||||
config: {},
|
||||
readme: markdown2Html('lib/notification/adapter/console.md'),
|
||||
};
|
||||
|
||||
4
lib/notification/adapter/console.md
Normal file
4
lib/notification/adapter/console.md
Normal file
@@ -0,0 +1,4 @@
|
||||
### Console Adapter
|
||||
|
||||
The console adapter prints everything found by Fredy into the console (not sending any notifications to you). This can be useful when you want to check if your search
|
||||
criteria meet the expectations.
|
||||
@@ -6,21 +6,20 @@ const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/t
|
||||
|
||||
const Handlebars = require('handlebars');
|
||||
const emailTemplate = Handlebars.compile(template);
|
||||
const { markdown2Html } = require('../../services/markdown');
|
||||
|
||||
/**
|
||||
* sends a new listing using MailJet
|
||||
* @param serviceName e.g immoscout
|
||||
* @param serviceName e.g immowelt
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* * @param jobKey name of the current job that is being executed
|
||||
* @returns {Promise<Chat.PostMessage.Response> | void}
|
||||
*/
|
||||
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
const { apiPublicKey, apiPrivateKey, enabled, receiver, from } = notificationConfig.mailJet;
|
||||
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
|
||||
(adapter) => adapter.id === 'mailJet'
|
||||
).fields;
|
||||
|
||||
return mailjet
|
||||
.connect(apiPublicKey, apiPrivateKey)
|
||||
@@ -47,3 +46,33 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
],
|
||||
});
|
||||
};
|
||||
|
||||
exports.config = {
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
name: 'MailJet',
|
||||
description: 'MailJet is being used to send new listings via mail.',
|
||||
readme: markdown2Html('lib/notification/adapter/mailJet.md'),
|
||||
fields: {
|
||||
apiPublicKey: {
|
||||
type: 'text',
|
||||
label: 'Public Api Key',
|
||||
description: 'The public api key needed to access this service.',
|
||||
},
|
||||
apiPrivateKey: {
|
||||
type: 'text',
|
||||
label: 'Private Api Key',
|
||||
description: 'The private api key needed to access this service.',
|
||||
},
|
||||
receiver: {
|
||||
type: 'email',
|
||||
label: 'Receiver Email',
|
||||
description: 'The email address (single one) which Fredy is using to send notifications to.',
|
||||
},
|
||||
from: {
|
||||
type: 'email',
|
||||
label: 'Sender email',
|
||||
description:
|
||||
'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
9
lib/notification/adapter/mailJet.md
Normal file
9
lib/notification/adapter/mailJet.md
Normal file
@@ -0,0 +1,9 @@
|
||||
### MailJet Adapter
|
||||
|
||||
|
||||
|
||||
SendGrid is a free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
|
||||
|
||||
To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.
|
||||
|
||||
Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new template. For this new template, I recommend copying and pasting the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
|
||||
@@ -1,18 +1,16 @@
|
||||
const sgMail = require('@sendgrid/mail');
|
||||
const { markdown2Html } = require('../../services/markdown');
|
||||
|
||||
/**
|
||||
* sends a new listing using SendGrid
|
||||
* @param serviceName e.g immoscout
|
||||
* @param serviceName e.g immowelt
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* * @param jobKey name of the current job that is being executed
|
||||
* @param jobKey name of the current job that is being executed
|
||||
* @returns {Promise<Chat.PostMessage.Response> | void}
|
||||
*/
|
||||
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
const { apiKey, enabled, receiver, from, templateId } = notificationConfig.sendGrid;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
|
||||
sgMail.setApiKey(apiKey);
|
||||
const msg = {
|
||||
templateId,
|
||||
@@ -27,3 +25,33 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
};
|
||||
return sgMail.send(msg);
|
||||
};
|
||||
|
||||
exports.config = {
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
name: 'SendGrid',
|
||||
description: 'SendGrid is being used to send new listings via mail.',
|
||||
readme: markdown2Html('lib/notification/adapter/sendGrid.md'),
|
||||
fields: {
|
||||
apiKey: {
|
||||
type: 'text',
|
||||
label: 'Api Key',
|
||||
description: 'The api key needed to access this service.',
|
||||
},
|
||||
receiver: {
|
||||
type: 'email',
|
||||
label: 'Receiver Email',
|
||||
description: 'The email address (single one) which Fredy is using to send notifications to.',
|
||||
},
|
||||
from: {
|
||||
type: 'email',
|
||||
label: 'Sender Email',
|
||||
description:
|
||||
'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
|
||||
},
|
||||
templateId: {
|
||||
type: 'text',
|
||||
label: 'Template Id',
|
||||
description: 'Sendgrid supports templates which Fredy is using to send out emails that looks awesome ;)',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
7
lib/notification/adapter/sendGrid.md
Normal file
7
lib/notification/adapter/sendGrid.md
Normal file
@@ -0,0 +1,7 @@
|
||||
### SendGrid Adapter
|
||||
|
||||
|
||||
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from.
|
||||
|
||||
E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.
|
||||
The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
|
||||
@@ -1,19 +1,17 @@
|
||||
const Slack = require('slack');
|
||||
const msg = Slack.chat.postMessage;
|
||||
const { markdown2Html } = require('../../services/markdown');
|
||||
|
||||
/**
|
||||
* sends a new listing to slack
|
||||
* @param serviceName e.g immoscout
|
||||
* @param serviceName e.g immowelt
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* * @param jobKey name of the current job that is being executed
|
||||
* @returns {Promise<Chat.PostMessage.Response> | void}
|
||||
*/
|
||||
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
const { token, channel, enabled } = notificationConfig.slack;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
|
||||
return newListings.map((payload) =>
|
||||
msg({
|
||||
token,
|
||||
@@ -49,3 +47,22 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
exports.config = {
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
name: 'Slack',
|
||||
readme: markdown2Html('lib/notification/adapter/slack.md'),
|
||||
description: 'Fredy will send new listings to the slack channel of your choice..',
|
||||
fields: {
|
||||
token: {
|
||||
type: 'text',
|
||||
label: 'Token',
|
||||
description: 'The token needed to send notifications to slack.',
|
||||
},
|
||||
channel: {
|
||||
type: 'channel',
|
||||
label: 'Channel',
|
||||
description: 'The channel where fredy should send notifications to.',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
6
lib/notification/adapter/slack.md
Normal file
6
lib/notification/adapter/slack.md
Normal file
@@ -0,0 +1,6 @@
|
||||
### Slack Adapter
|
||||
|
||||
|
||||
In order to use [Slack](https://slack.com), you need to create an account. When done, you need to create a new App in your workspace. Give it the permission `chat:write:bot` and `chat:write:user`.
|
||||
|
||||
Now you need to create a user token and a channel. Make sure the bot is installed to this channel.
|
||||
@@ -1,20 +1,17 @@
|
||||
const TelegramBot = require('tg-yarl');
|
||||
|
||||
const { markdown2Html } = require('../../services/markdown');
|
||||
const opts = { parse_mode: 'Markdown' };
|
||||
|
||||
/**
|
||||
* sends new listings to telegram
|
||||
* @param serviceName e.g immoscout
|
||||
* @param serviceName e.g immowelt
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* * @param jobKey name of the current job that is being executed
|
||||
* @returns {Promise<Void> | void}
|
||||
*/
|
||||
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
const { enabled, token, chatId } = notificationConfig.telegram;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
|
||||
|
||||
const bot = new TelegramBot(token);
|
||||
|
||||
@@ -34,3 +31,27 @@ exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
function shorten(str, len = 30) {
|
||||
return str.length > len ? str.substring(0, len) + '...' : str;
|
||||
}
|
||||
|
||||
/**
|
||||
* exported config is being used in the frontend to generate the fields
|
||||
* incoming values will be the keys (and values) of the fields
|
||||
*
|
||||
*/
|
||||
exports.config = {
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
name: 'Telegram',
|
||||
readme: markdown2Html('lib/notification/adapter/telegram.md'),
|
||||
description: 'Fredy will send new listings to your mobile, using Telegram.',
|
||||
fields: {
|
||||
token: {
|
||||
type: 'text',
|
||||
label: 'Token',
|
||||
description: 'The token needed to access this service.',
|
||||
},
|
||||
chatId: {
|
||||
type: 'chatId',
|
||||
label: 'Chat Id',
|
||||
description: 'The chat id to send messages to you.',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
13
lib/notification/adapter/telegram.md
Normal file
13
lib/notification/adapter/telegram.md
Normal file
@@ -0,0 +1,13 @@
|
||||
### Telegram Adapter
|
||||
|
||||
|
||||
For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
|
||||
|
||||
A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.
|
||||
After the user has send a message to your bot the first time, you can gather the chatId like this:
|
||||
|
||||
```
|
||||
curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
|
||||
```
|
||||
|
||||
A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
|
||||
@@ -2,13 +2,20 @@ const fs = require('fs');
|
||||
const path = './adapter';
|
||||
|
||||
/** Read every integration existing in ./adapter **/
|
||||
const adapter = fs.readdirSync('./lib/notification/adapter').map((integPath) => require(`${path}/${integPath}`));
|
||||
const adapter = fs
|
||||
.readdirSync('./lib/notification/adapter')
|
||||
.filter((file) => file.endsWith('.js'))
|
||||
.map((integPath) => require(`${path}/${integPath}`));
|
||||
|
||||
if (adapter.length === 0) {
|
||||
throw new Error('Please specify at least one notification provider');
|
||||
}
|
||||
|
||||
exports.send = (serviceName, payload, notificationConfig, jobKey) => {
|
||||
//this is not being used in tests, therefor adapter are always set
|
||||
return adapter.map((a) => a.send(serviceName, payload, notificationConfig, jobKey));
|
||||
exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
//this is not being used in tests, therefore adapter are always set
|
||||
return adapter
|
||||
.filter((notificationAdapter) => {
|
||||
return notificationConfig.find((config) => config.id === notificationAdapter.config.id);
|
||||
})
|
||||
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
|
||||
};
|
||||
|
||||
@@ -20,7 +20,6 @@ function applyBlacklist(o) {
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '.tabelle',
|
||||
crawlFields: {
|
||||
@@ -29,20 +28,23 @@ const config = {
|
||||
size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
|
||||
rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
|
||||
title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
|
||||
description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim'
|
||||
description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
|
||||
},
|
||||
paginate: '.pagination_blocks div:last a@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'einsAImmobilien';
|
||||
exports.metaInformation = {
|
||||
name: '1a Immobilien',
|
||||
baseUrl: 'https://www.1a-immobilienmarkt.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -20,7 +20,6 @@ function applyBlacklist(o) {
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#result-list-stage .item',
|
||||
crawlFields: {
|
||||
@@ -29,20 +28,23 @@ const config = {
|
||||
size: 'div[id*="selArea_"] | trim',
|
||||
title: '.item a img@title',
|
||||
link: 'a[id*="lnkImgToDetails_"]@href',
|
||||
address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim'
|
||||
address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
|
||||
},
|
||||
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'immonet';
|
||||
exports.metaInformation = {
|
||||
name: 'Immonet',
|
||||
baseUrl: 'https://www.immonet.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const title = o.title.replace('NEU', '');
|
||||
const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
||||
|
||||
return Object.assign(o, { title, address });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
return !utils.isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#resultListItems li.result-list__listing',
|
||||
crawlFields: {
|
||||
id: '.result-list-entry@data-obid | int',
|
||||
price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
|
||||
size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
|
||||
title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
|
||||
link: '.result-list-entry .result-list-entry__brand-title-container@href',
|
||||
address: '.result-list-entry .result-list-entry__map-link'
|
||||
},
|
||||
paginate: '#pager .align-right a@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'immoscout';
|
||||
|
||||
exports.config = config;
|
||||
@@ -17,7 +17,6 @@ function applyBlacklist(o) {
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '.immoliste .js-object.listitem_wrap ',
|
||||
crawlFields: {
|
||||
@@ -26,20 +25,22 @@ const config = {
|
||||
size: '.js-object.listitem_wrap .hardfacts_3 div:nth-child(2)| removeNewline | trim',
|
||||
title: '.listcontent.clear h2',
|
||||
link: 'a@href',
|
||||
address: '.listcontent .details .listlocation| removeNewline | trim'
|
||||
address: '.listcontent .details .listlocation| removeNewline | trim',
|
||||
},
|
||||
paginate: '#pnlPaging #nlbPlus@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
exports.metaInformation = {
|
||||
name: 'Immowelt',
|
||||
baseUrl: 'https://www.immowelt.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'immowelt';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
let appliedBlacklistedDistricts = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = o.id
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.reverse()[0];
|
||||
const price = o.price == null ? 'unknown' : o.price.trim().replace('Preis', '');
|
||||
let size = o.size == null ? 'unknown' : o.size.replace('Wohnfläche: ', '').replace('ca. ', '');
|
||||
size += ' / ' + o.rooms;
|
||||
const address = '---';
|
||||
|
||||
return Object.assign(o, { id, price, size, address });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
const isBlacklistedDistrict =
|
||||
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
|
||||
|
||||
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#resultList .resultitem-content-container',
|
||||
crawlFields: {
|
||||
id: '.resultitem-content-container a@href',
|
||||
price: '.description .rent | removeNewline | trim',
|
||||
title: '.resultitem-content-container a@title',
|
||||
link: '.resultitem-content-container a@href',
|
||||
rooms: '.resultitem-content-container .no-of-rooms | removeNewline | trim',
|
||||
size: '.resultitem-content-container .living-area | removeNewline | trim'
|
||||
},
|
||||
paginate: '.markt_pagination_pageLinkNext .markt_pagination_link@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlacklistedDistricts = blacklistedDistricts;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'kalaydo';
|
||||
|
||||
exports.config = config;
|
||||
@@ -13,13 +13,12 @@ function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const isBlacklistedDistrict =
|
||||
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
|
||||
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
|
||||
|
||||
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#srchrslt-adtable .ad-listitem',
|
||||
crawlFields: {
|
||||
@@ -29,22 +28,24 @@ const config = {
|
||||
title: '.aditem-main .text-module-begin a | removeNewline | trim',
|
||||
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
|
||||
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
|
||||
address: '.aditem-details | trim | removeNewline'
|
||||
address: '.aditem-details | trim | removeNewline',
|
||||
},
|
||||
paginate: '#srchrslt-pagination .pagination-next@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
exports.metaInformation = {
|
||||
name: 'Ebay Kleinanzeigen',
|
||||
baseUrl: 'https://www.ebay-kleinanzeigen.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlacklistedDistricts = blacklistedDistricts;
|
||||
appliedBlackList = blacklist;
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlacklistedDistricts = blacklistedDistricts || [];
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'kleinanzeigen';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ function applyBlacklist(o) {
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '.nbk-container >div article',
|
||||
crawlFields: {
|
||||
@@ -28,10 +27,13 @@ const config = {
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'neubauKompass';
|
||||
exports.metaInformation = {
|
||||
name: 'Neubau Kompass',
|
||||
baseUrl: 'https://www.neubaukompass.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -14,7 +14,6 @@ function applyBlacklist(o) {
|
||||
}
|
||||
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#main_column .wgg_card',
|
||||
crawlFields: {
|
||||
@@ -23,20 +22,23 @@ const config = {
|
||||
price: '.middle .col-xs-3 |removeNewline |trim',
|
||||
size: '.middle .text-right |removeNewline |trim',
|
||||
title: '.truncate_title a |removeNewline |trim',
|
||||
link: '.truncate_title a@href'
|
||||
link: '.truncate_title a@href',
|
||||
},
|
||||
paginate: '.pagination-sm:first a:last@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlackList = blacklist || [];
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'wgGesucht';
|
||||
exports.metaInformation = {
|
||||
name: 'Wg gesucht',
|
||||
baseUrl: 'https://www.wg-gesucht.de/',
|
||||
id: __filename.slice(__dirname.length + 1, -3),
|
||||
};
|
||||
|
||||
exports.config = config;
|
||||
|
||||
6
lib/services/markdown.js
Normal file
6
lib/services/markdown.js
Normal file
@@ -0,0 +1,6 @@
|
||||
const markdown = require('markdown').markdown;
|
||||
const fs = require('fs');
|
||||
|
||||
exports.markdown2Html = function markdown2Html(filePath) {
|
||||
return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
|
||||
};
|
||||
@@ -6,14 +6,16 @@ class Scraper {
|
||||
const filters = {
|
||||
removeNewline: this._removeNewline,
|
||||
trim: this._trim,
|
||||
int: this._int
|
||||
int: this._int,
|
||||
};
|
||||
|
||||
const driver = makeDriver({
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36'
|
||||
}
|
||||
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
|
||||
cookie:
|
||||
'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
|
||||
},
|
||||
});
|
||||
|
||||
const xray = Xray({ filters });
|
||||
|
||||
3
lib/services/security/hash.js
Normal file
3
lib/services/security/hash.js
Normal file
@@ -0,0 +1,3 @@
|
||||
const crypto = require('crypto');
|
||||
|
||||
exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
|
||||
83
lib/services/storage/jobStorage.js
Normal file
83
lib/services/storage/jobStorage.js
Normal file
@@ -0,0 +1,83 @@
|
||||
const path = require('path');
|
||||
const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
|
||||
const FileSync = require('lowdb/adapters/FileSync');
|
||||
const adapter = new FileSync(DB_PATH);
|
||||
const low = require('lowdb');
|
||||
const db = low(adapter);
|
||||
const { nanoid } = require('nanoid');
|
||||
const listingStorage = require('./listingsStorage');
|
||||
|
||||
db.defaults({ jobs: [] }).write();
|
||||
|
||||
exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
|
||||
const currentJob =
|
||||
jobId == null
|
||||
? null
|
||||
: db
|
||||
.get('jobs')
|
||||
.find((job) => job.id === jobId)
|
||||
.value();
|
||||
|
||||
const jobs = db
|
||||
.get('jobs')
|
||||
.value()
|
||||
.filter((job) => job.id !== jobId);
|
||||
|
||||
jobs.push({
|
||||
id: jobId || nanoid(),
|
||||
//make sure to not overwrite the user id in case an admin changes the job
|
||||
userId: currentJob == null ? userId : currentJob.userId,
|
||||
enabled,
|
||||
name,
|
||||
blacklist,
|
||||
provider,
|
||||
notificationAdapter,
|
||||
});
|
||||
|
||||
db.set('jobs', jobs).write();
|
||||
};
|
||||
|
||||
exports.getJob = (jobId) => {
|
||||
const job = db
|
||||
.get('jobs')
|
||||
.find((job) => job.id === jobId)
|
||||
.value();
|
||||
|
||||
if (job == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...job,
|
||||
numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
|
||||
};
|
||||
};
|
||||
|
||||
exports.setJobStatus = ({ jobId, status }) => {
|
||||
db.get('jobs')
|
||||
.find((job) => job.id === jobId)
|
||||
.assign({ enabled: status })
|
||||
.write();
|
||||
};
|
||||
|
||||
exports.removeJob = (jobId) => {
|
||||
db.get('jobs')
|
||||
.remove((job) => job.id === jobId)
|
||||
.write();
|
||||
};
|
||||
|
||||
exports.removeJobsByUserId = (userId) => {
|
||||
db.get('jobs')
|
||||
.remove((job) => job.userId === userId)
|
||||
.write();
|
||||
};
|
||||
|
||||
exports.getJobs = () => {
|
||||
return db
|
||||
.get('jobs')
|
||||
.value()
|
||||
.map((job) => ({
|
||||
...job,
|
||||
numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
|
||||
}));
|
||||
};
|
||||
49
lib/services/storage/listingsStorage.js
Executable file
49
lib/services/storage/listingsStorage.js
Executable file
@@ -0,0 +1,49 @@
|
||||
const path = require('path');
|
||||
|
||||
const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
|
||||
const FileSync = require('lowdb/adapters/FileSync');
|
||||
const adapter = new FileSync(DB_PATH);
|
||||
const low = require('lowdb');
|
||||
const db = low(adapter);
|
||||
|
||||
const buildKey = (jobKey, providerId, endpoint) => {
|
||||
let key = `${jobKey}`;
|
||||
if (jobKey == null && endpoint == null) {
|
||||
return key;
|
||||
}
|
||||
if (providerId != null) {
|
||||
key += `.${providerId}`;
|
||||
}
|
||||
if (endpoint != null) {
|
||||
key += `.${endpoint}`;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
exports.getNumberOfAllKnownListings = (jobId) => {
|
||||
const data = db.get(`${jobId}.providerData`).value() || {};
|
||||
return Object.values(data)
|
||||
.map((values) => Object.keys(values).length)
|
||||
.reduce((accumulator, currentValue) => accumulator + currentValue, 0);
|
||||
};
|
||||
|
||||
exports.getListingProviderDataForAnalytics = (jobId) => {
|
||||
const key = buildKey(jobId, 'providerData');
|
||||
return db.get(key).value() || {};
|
||||
};
|
||||
|
||||
exports.getKnownListings = (jobId, providerId) => {
|
||||
const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
|
||||
return db.get(providerListingsKey).value() || {};
|
||||
};
|
||||
|
||||
exports.setKnownListings = (jobId, providerId, listings) => {
|
||||
const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
|
||||
|
||||
return db.set(providerListingsKey, listings).write();
|
||||
};
|
||||
|
||||
exports.setLastJobExecution = (jobId) => {
|
||||
const key = buildKey(jobId, null, 'lastExecution');
|
||||
return db.set(key, Date.now()).write();
|
||||
};
|
||||
83
lib/services/storage/userStorage.js
Normal file
83
lib/services/storage/userStorage.js
Normal file
@@ -0,0 +1,83 @@
|
||||
const path = require('path');
|
||||
const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
|
||||
const FileSync = require('lowdb/adapters/FileSync');
|
||||
const adapter = new FileSync(DB_PATH);
|
||||
const low = require('lowdb');
|
||||
const db = low(adapter);
|
||||
const hasher = require('../security/hash');
|
||||
const { nanoid } = require('nanoid');
|
||||
const jobStorage = require('./jobStorage');
|
||||
|
||||
db.defaults({
|
||||
user: [
|
||||
//you probably want to change the default password ;)
|
||||
{
|
||||
id: nanoid(),
|
||||
lastLogin: Date.now(),
|
||||
username: 'admin',
|
||||
password: hasher.hash('admin'),
|
||||
isAdmin: true,
|
||||
isDemo: false,
|
||||
},
|
||||
],
|
||||
}).write();
|
||||
|
||||
exports.getUsers = (withPassword) => {
|
||||
const jobs = jobStorage.getJobs();
|
||||
return db
|
||||
.get('user')
|
||||
.value()
|
||||
.map((user) => ({
|
||||
//we dont want the password in the frontend, even tho it's hashed
|
||||
...user,
|
||||
password: withPassword ? user.password : null,
|
||||
numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
|
||||
}));
|
||||
};
|
||||
|
||||
exports.getUser = (id) => {
|
||||
const jobs = jobStorage.getJobs();
|
||||
const user = db
|
||||
.get('user')
|
||||
.value()
|
||||
.find((user) => user.id === id);
|
||||
if (user == null) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...user,
|
||||
numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
|
||||
};
|
||||
};
|
||||
|
||||
exports.upsertUser = ({ username, password, userId, isAdmin }) => {
|
||||
const user = db
|
||||
.get('user')
|
||||
.value()
|
||||
.filter((u) => u.id !== userId);
|
||||
|
||||
user.push({
|
||||
id: userId || nanoid(),
|
||||
username,
|
||||
lastLogin: user.lastLogin,
|
||||
password: hasher.hash(password),
|
||||
isAdmin,
|
||||
});
|
||||
|
||||
db.set('user', user).write();
|
||||
};
|
||||
|
||||
exports.setLastLoginToNow = ({ userId }) => {
|
||||
db.get('user')
|
||||
.find((u) => u.id === userId)
|
||||
.assign({ lastLogin: Date.now() })
|
||||
.write();
|
||||
};
|
||||
|
||||
exports.removeUser = (userId) => {
|
||||
const user = db.get('user').value();
|
||||
db.set(
|
||||
'user',
|
||||
user.filter((u) => u.id !== userId)
|
||||
).write();
|
||||
};
|
||||
@@ -1,85 +0,0 @@
|
||||
const path = require('path');
|
||||
const DB_PATH = path.dirname(require.main.filename) + '/conf/store.json';
|
||||
|
||||
const FileAsync = require('lowdb/adapters/FileAsync');
|
||||
const adapter = new FileAsync(DB_PATH);
|
||||
const low = require('lowdb');
|
||||
|
||||
const lowdb = low(adapter);
|
||||
|
||||
let db = null;
|
||||
|
||||
const buildKey = (jobKey, providerId, endpoint) => {
|
||||
let key = `${jobKey}`;
|
||||
if (jobKey == null && endpoint == null) {
|
||||
return key;
|
||||
}
|
||||
if (providerId != null) {
|
||||
key += `.${providerId}`;
|
||||
}
|
||||
if (endpoint != null) {
|
||||
key += `.${endpoint}`;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
exports.init = () => {
|
||||
return new Promise(resolve => {
|
||||
//warmup
|
||||
lowdb.then(database => {
|
||||
db = database;
|
||||
/* eslint-disable no-console */
|
||||
console.info('Warming up database successful');
|
||||
/* eslint-enable no-console */
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||
if (!Array.isArray(listings)) throw Error('Not a valid array');
|
||||
const providerListingsKey = buildKey(jobKey, providerId, 'listings');
|
||||
const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
|
||||
|
||||
return db
|
||||
.set(providerListingsKey, listings)
|
||||
.set(providerLastScrapeKey, Date.now())
|
||||
.write();
|
||||
};
|
||||
|
||||
exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
|
||||
if (numberOfNewListings > 0) {
|
||||
const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
|
||||
const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
|
||||
db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
|
||||
}
|
||||
};
|
||||
|
||||
exports.setLastJobExecution = jobKey => {
|
||||
const key = buildKey(jobKey, null, 'lastJobExecution');
|
||||
return db.set(key, Date.now()).write();
|
||||
};
|
||||
|
||||
exports.getKnownListings = (jobKey, providerId) => {
|
||||
const providerListingsKey = buildKey(jobKey, providerId, 'listings');
|
||||
return db.get(providerListingsKey).value() || [];
|
||||
};
|
||||
|
||||
exports.getLastProviderExecution = (jobKey, providerId) => {
|
||||
const key = buildKey(jobKey, providerId, 'lastProviderExecution');
|
||||
return db.get(key).value() || 0;
|
||||
};
|
||||
|
||||
exports.getLastJobExecution = jobKey => {
|
||||
const key = buildKey(jobKey, null, 'lastJobExecution');
|
||||
return db.get(key).value() || 0;
|
||||
};
|
||||
|
||||
exports.getTotalNumberOfListings = (jobKey, providerId) => {
|
||||
const key = buildKey(jobKey, providerId, 'foundListings');
|
||||
return db.get(key).value() || 0;
|
||||
};
|
||||
|
||||
exports.getForTesting = () => {
|
||||
return db;
|
||||
};
|
||||
72
package.json
72
package.json
@@ -1,10 +1,12 @@
|
||||
{
|
||||
"name": "fredy",
|
||||
"version": "2.0.0",
|
||||
"version": "3.0.0",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"format": "prettier --write lib/**/*.js test/**/*.js *.js --single-quote --print-width 120",
|
||||
"dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
|
||||
"prod": "export BUILD_DEV='false' && export NODE_ENV='production' && webpack --config ./webpack.prod.js",
|
||||
"format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
|
||||
"test": "mocha --timeout 20000 test/**/*.test.js"
|
||||
},
|
||||
"husky": {
|
||||
@@ -42,26 +44,70 @@
|
||||
"node": ">=11.0.0",
|
||||
"npm": ">=6.0.0"
|
||||
},
|
||||
"browserslist": [
|
||||
"> 0.5%",
|
||||
"not dead",
|
||||
"not ie 6-11",
|
||||
"Firefox ESR"
|
||||
],
|
||||
"dependencies": {
|
||||
"@sendgrid/mail": "^7.2.3",
|
||||
"@rematch/core": "^1.4.0",
|
||||
"@rematch/loading": "^1.2.1",
|
||||
"@sendgrid/mail": "7.4.2",
|
||||
"axios": "^0.21.1",
|
||||
"body-parser": "1.19.0",
|
||||
"handlebars": "^4.7.6",
|
||||
"cookie-session": "^1.4.0",
|
||||
"handlebars": "4.7.6",
|
||||
"highcharts": "^8.2.2",
|
||||
"highcharts-react-official": "^3.0.0",
|
||||
"lowdb": "1.0.0",
|
||||
"node-mailjet": "^3.3.1",
|
||||
"markdown": "^0.5.0",
|
||||
"nanoid": "^3.1.20",
|
||||
"node-mailjet": "3.3.1",
|
||||
"react": "17.0.1",
|
||||
"react-dom": "17.0.1",
|
||||
"react-redux": "7.2.2",
|
||||
"react-router": "5.2.0",
|
||||
"react-router-dom": "5.2.0",
|
||||
"react-switch": "^6.0.0",
|
||||
"redux": "4.0.5",
|
||||
"redux-thunk": "2.3.0",
|
||||
"request-x-ray": "0.1.4",
|
||||
"restana": "4.7.2",
|
||||
"restana": "4.8.1",
|
||||
"semantic-ui-react": "2.0.2",
|
||||
"serve-static": "^1.14.1",
|
||||
"slack": "11.0.2",
|
||||
"tg-yarl": "1.3.0",
|
||||
"x-ray": "2.3.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "7.12.10",
|
||||
"@babel/preset-env": "7.12.11",
|
||||
"@babel/preset-react": "7.12.10",
|
||||
"babel-eslint": "10.1.0",
|
||||
"babel-loader": "8.2.2",
|
||||
"chai": "4.2.0",
|
||||
"eslint": "7.7.0",
|
||||
"eslint-config-prettier": "6.11.0",
|
||||
"husky": "4.2.5",
|
||||
"lint-staged": "10.2.11",
|
||||
"mocha": "8.1.1",
|
||||
"prettier": "2.0.5",
|
||||
"proxyquire": "2.1.3"
|
||||
"clean-webpack-plugin": "3.0.0",
|
||||
"copy-webpack-plugin": "6.3.0",
|
||||
"css-loader": "5.0.1",
|
||||
"eslint": "7.17.0",
|
||||
"eslint-config-prettier": "7.1.0",
|
||||
"eslint-plugin-react": "7.22.0",
|
||||
"file-loader": "6.2.0",
|
||||
"history": "5.0.0",
|
||||
"husky": "4.3.8",
|
||||
"less": "4.1.0",
|
||||
"less-loader": "7.2.1",
|
||||
"lint-staged": "10.5.3",
|
||||
"mocha": "8.2.1",
|
||||
"prettier": "2.2.1",
|
||||
"proxyquire": "2.1.3",
|
||||
"redux-logger": "3.0.6",
|
||||
"style-loader": "2.0.0",
|
||||
"url-loader": "4.1.1",
|
||||
"webpack": "4.44.2",
|
||||
"webpack-cli": "3.3.12",
|
||||
"webpack-dev-server": "3.11.2",
|
||||
"webpack-merge": "5.7.3"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,5 +8,5 @@ module.exports = {
|
||||
|
||||
get: () => {
|
||||
return this._tmpStore;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
const db = {};
|
||||
|
||||
exports.init = () => {
|
||||
return new Promise(resolve => {
|
||||
resolve();
|
||||
});
|
||||
};
|
||||
|
||||
exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||
if (!Array.isArray(listings)) throw Error('Not a valid array');
|
||||
|
||||
@@ -15,38 +9,3 @@ exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||
exports.getKnownListings = (jobKey, providerId) => {
|
||||
return db[providerId] || [];
|
||||
};
|
||||
|
||||
exports.setNumberOfTotalFoundProviderListings = () => {
|
||||
/*noop*/
|
||||
};
|
||||
|
||||
exports.getForTesting = () => {
|
||||
return db;
|
||||
};
|
||||
/*
|
||||
class Store {
|
||||
constructor(name) {
|
||||
this._name = name;
|
||||
this._db = {};
|
||||
}
|
||||
|
||||
get warmup() {
|
||||
this._db = {};
|
||||
return new Promise(resolve => resolve());
|
||||
}
|
||||
|
||||
set knownListings(value) {
|
||||
if (!Array.isArray(value)) throw Error('Not a valid array');
|
||||
return new Promise(resolve => {
|
||||
this._db[this._name] = value;
|
||||
resolve(value);
|
||||
});
|
||||
}
|
||||
|
||||
get knownListings() {
|
||||
return this._db[this._name] || [];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Store;
|
||||
*/
|
||||
|
||||
@@ -1,29 +1,31 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const providerConfig = require('./testProvider.json');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/einsAImmobilien');
|
||||
|
||||
describe('#einsAImmobilien testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.einsAImmobilien, [], []);
|
||||
provider.init(providerConfig.einsAImmobilien, [], []);
|
||||
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
'./services/storage/listingsStorage': {
|
||||
...mockStore,
|
||||
},
|
||||
'./notification/notify': mockNotification,
|
||||
});
|
||||
|
||||
it('should test einsAImmobilien provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immonetDbContent = fredy._getStore();
|
||||
expect(immonetDbContent.einsAImmobilien).to.be.a('array');
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
|
||||
fredy.execute().then((listings) => {
|
||||
expect(listings).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
@@ -32,7 +34,6 @@ describe('#einsAImmobilien testsuite()', () => {
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
|
||||
expect(notify.price).that.does.include('EUR');
|
||||
expect(notify.size).to.be.not.empty;
|
||||
expect(notify.title).to.be.not.empty;
|
||||
|
||||
@@ -1,30 +1,30 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const providerConfig = require('./testProvider.json');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/immonet');
|
||||
|
||||
describe('#immonet testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.immonet, [], []);
|
||||
provider.init(providerConfig.immonet, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
'./services/storage/listingsStorage': {
|
||||
...mockStore,
|
||||
},
|
||||
'./notification/notify': mockNotification,
|
||||
});
|
||||
|
||||
it('should test immonet provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immonetDbContent = fredy._getStore();
|
||||
|
||||
expect(immonetDbContent.immonet).to.be.a('array');
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immonet');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
@@ -34,7 +34,6 @@ describe('#immonet testsuite()', () => {
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/immoscout');
|
||||
|
||||
describe('#immoscout testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.immoscout, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test immoscout provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immoscoutDbContent = fredy._getStore();
|
||||
expect(immoscoutDbContent.immoscout).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immoscout');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immoscoutDbContent.immoscout[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const providerConfig = require('./testProvider.json');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
@@ -7,23 +7,24 @@ const provider = require('../../lib/provider/immowelt');
|
||||
|
||||
describe('#immowelt testsuite()', () => {
|
||||
it('should test immowelt provider', async () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.immowelt, [], []);
|
||||
provider.init(providerConfig.immowelt, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
'./services/storage/listingsStorage': {
|
||||
...mockStore,
|
||||
},
|
||||
'./notification/notify': mockNotification,
|
||||
});
|
||||
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immoweltDbContent = fredy._getStore();
|
||||
expect(immoweltDbContent.immowelt).to.be.a('array');
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immowelt');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
@@ -33,7 +34,6 @@ describe('#immowelt testsuite()', () => {
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immoweltDbContent.immowelt[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
if (notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).that.does.include('m²');
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/kalaydo');
|
||||
|
||||
describe('#kalaydo testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.kalaydo, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test kalaydo provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const kalaydoDbContent = fredy._getStore();
|
||||
|
||||
expect(kalaydoDbContent.kalaydo).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('kalaydo');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.kalaydo.de');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const providerConfig = require('./testProvider.json');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
@@ -7,23 +7,24 @@ const provider = require('../../lib/provider/kleinanzeigen');
|
||||
|
||||
describe('#kleinanzeigen testsuite()', () => {
|
||||
it('should test kleinanzeigen provider', async () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.kleinanzeigen, [], []);
|
||||
provider.init(providerConfig.kleinanzeigen, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
'./services/storage/listingsStorage': {
|
||||
...mockStore,
|
||||
},
|
||||
'./notification/notify': mockNotification,
|
||||
});
|
||||
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const kleinanzeigenDbContent = fredy._getStore();
|
||||
expect(kleinanzeigenDbContent.kleinanzeigen).to.be.a('array');
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
@@ -33,7 +34,6 @@ describe('#kleinanzeigen testsuite()', () => {
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(kleinanzeigenDbContent.kleinanzeigen[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
|
||||
@@ -1,28 +1,29 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const providerConfig = require('./testProvider.json');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/neubauKompass');
|
||||
|
||||
describe('#neubauKompass testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.neubauKompass, [], []);
|
||||
provider.init(providerConfig.neubauKompass, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
'./services/storage/listingsStorage': {
|
||||
...mockStore,
|
||||
},
|
||||
'./notification/notify': mockNotification,
|
||||
});
|
||||
|
||||
it('should test neubauKompass provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const neubauKompassDbContent = fredy._getStore();
|
||||
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj.serviceName).to.equal('neubauKompass');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
expect(notify).to.be.a('object');
|
||||
|
||||
/** check the actual structure **/
|
||||
@@ -32,7 +33,6 @@ describe('#neubauKompass testsuite()', () => {
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.neubaukompass.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
|
||||
31
test/provider/testProvider.json
Normal file
31
test/provider/testProvider.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||
"enabled": true,
|
||||
"id": "einsAImmobilien"
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,27 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const providerConfig = require('./testProvider.json');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/wgGesucht');
|
||||
|
||||
describe('#wgGesucht testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.wgGesucht, [], []);
|
||||
provider.init(providerConfig.wgGesucht, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
'./services/storage/listingsStorage': {
|
||||
...mockStore,
|
||||
},
|
||||
'./notification/notify': mockNotification,
|
||||
});
|
||||
|
||||
it('should test wgGesucht provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const wgGesuchtDbContent = fredy._getStore();
|
||||
expect(wgGesuchtDbContent.wgGesucht).to.be.a('array');
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
|
||||
fredy.execute().then((listing) => {
|
||||
expect(listing).to.be.a('array');
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj.serviceName).to.equal('wgGesucht');
|
||||
notificationObj.payload.forEach(notify => {
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
expect(notify).to.be.a('object');
|
||||
|
||||
/** check the actual structure **/
|
||||
|
||||
89
ui/src/App.js
Normal file
89
ui/src/App.js
Normal file
@@ -0,0 +1,89 @@
|
||||
import React, { useEffect } from 'react';
|
||||
|
||||
import InsufficientPermission from './components/permission/InsufficientPermission';
|
||||
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
|
||||
import ToastsContainer from './components/toasts/ToastContainer';
|
||||
import JobMutation from './views/jobs/mutation/JobMutation';
|
||||
import UserMutator from './views/user/mutation/UserMutator';
|
||||
import ToastContext from './components/toasts/ToastContext';
|
||||
import JobInsight from './views/jobs/insights/JobInsight';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import useToast from './components/toasts/useToast';
|
||||
import { Switch, Redirect } from 'react-router-dom';
|
||||
import Logout from './components/logout/Logout';
|
||||
import Logo from './components/logo/Logo';
|
||||
import Menu from './components/menu/Menu';
|
||||
import Login from './views/login/Login';
|
||||
import Users from './views/user/Users';
|
||||
import Jobs from './views/jobs/Jobs';
|
||||
import { Route } from 'react-router';
|
||||
|
||||
import './App.less';
|
||||
|
||||
export default function FredyApp() {
|
||||
const dispatch = useDispatch();
|
||||
const [showToast, onToastFinished, toasts] = useToast();
|
||||
const [loading, setLoading] = React.useState(true);
|
||||
const currentUser = useSelector((state) => state.user.currentUser);
|
||||
|
||||
useEffect(async () => {
|
||||
await dispatch.provider.getProvider();
|
||||
await dispatch.jobs.getJobs();
|
||||
await dispatch.notificationAdapter.getAdapter();
|
||||
await dispatch.user.getCurrentUser();
|
||||
|
||||
setLoading(false);
|
||||
}, [currentUser?.userId]);
|
||||
|
||||
const needsLogin = () => {
|
||||
return currentUser == null || Object.keys(currentUser).length === 0;
|
||||
};
|
||||
|
||||
const isAdmin = () => currentUser != null && currentUser.isAdmin;
|
||||
|
||||
const login = () => (
|
||||
<Switch>
|
||||
<Route name="Login" path={'/login'} component={Login} />
|
||||
<Redirect from="*" to={'/login'} />
|
||||
</Switch>
|
||||
);
|
||||
|
||||
return loading ? null : needsLogin() ? (
|
||||
login()
|
||||
) : (
|
||||
<ToastContext.Provider value={{ showToast }}>
|
||||
<div className="app">
|
||||
<div className="app__container">
|
||||
<Logout />
|
||||
<Logo width={190} white />
|
||||
<Menu isAdmin={isAdmin()} />
|
||||
<ToastsContainer toasts={toasts} onToastFinished={onToastFinished} />
|
||||
<Switch>
|
||||
<Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
|
||||
<Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
|
||||
<Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
|
||||
<Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
|
||||
<Route name="Job overview" path={'/jobs'} component={Jobs} />
|
||||
<PermissionAwareRoute
|
||||
name="Create new User"
|
||||
path="/users/new"
|
||||
component={<UserMutator />}
|
||||
currentUser={currentUser}
|
||||
/>
|
||||
<PermissionAwareRoute
|
||||
name="Edit a user"
|
||||
path="/users/edit/:userId"
|
||||
component={<UserMutator />}
|
||||
currentUser={currentUser}
|
||||
/>
|
||||
<PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
|
||||
|
||||
<Redirect from="/" to={'/jobs'} />
|
||||
</Switch>
|
||||
</div>
|
||||
</div>
|
||||
</ToastContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
FredyApp.displayName = 'FredyApp';
|
||||
13
ui/src/App.less
Normal file
13
ui/src/App.less
Normal file
@@ -0,0 +1,13 @@
|
||||
.app {
|
||||
display:flex;
|
||||
flex-direction: column;
|
||||
width:100%;
|
||||
|
||||
&__container {
|
||||
width: 100%;
|
||||
|
||||
padding: 1rem 1rem;
|
||||
background-color: #3f3e3ef5;
|
||||
color: #f1f1f1;
|
||||
}
|
||||
}
|
||||
22
ui/src/Index.js
Normal file
22
ui/src/Index.js
Normal file
@@ -0,0 +1,22 @@
|
||||
import React from 'react';
|
||||
|
||||
import { reduxStore } from './services/rematch/store';
|
||||
import { HashRouter } from 'react-router-dom';
|
||||
import { createHashHistory } from 'history';
|
||||
import { Provider } from 'react-redux';
|
||||
import ReactDOM from 'react-dom';
|
||||
|
||||
const history = createHashHistory();
|
||||
|
||||
import App from './App';
|
||||
|
||||
import './Index.less';
|
||||
|
||||
ReactDOM.render(
|
||||
<Provider store={reduxStore}>
|
||||
<HashRouter history={history}>
|
||||
<App />
|
||||
</HashRouter>
|
||||
</Provider>,
|
||||
document.getElementById('fredy')
|
||||
);
|
||||
6
ui/src/Index.less
Normal file
6
ui/src/Index.less
Normal file
@@ -0,0 +1,6 @@
|
||||
body, html {
|
||||
margin: 0;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
background-color: #3f3e3ef5;
|
||||
}
|
||||
BIN
ui/src/assets/city_background.jpg
Normal file
BIN
ui/src/assets/city_background.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 621 KiB |
BIN
ui/src/assets/insufficient_permission.png
Normal file
BIN
ui/src/assets/insufficient_permission.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 284 KiB |
BIN
ui/src/assets/logo.png
Normal file
BIN
ui/src/assets/logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 59 KiB |
BIN
ui/src/assets/logo_white.png
Normal file
BIN
ui/src/assets/logo_white.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
12
ui/src/components/headline/Headline.js
Normal file
12
ui/src/components/headline/Headline.js
Normal file
@@ -0,0 +1,12 @@
|
||||
import React from 'react';
|
||||
import { Header } from 'semantic-ui-react';
|
||||
|
||||
import './Headline.less';
|
||||
|
||||
export default function Headline({ text, size = 'medium', className = '' } = {}) {
|
||||
return (
|
||||
<Header className={`headline ${className}`} size={size}>
|
||||
{text}
|
||||
</Header>
|
||||
);
|
||||
}
|
||||
3
ui/src/components/headline/Headline.less
Normal file
3
ui/src/components/headline/Headline.less
Normal file
@@ -0,0 +1,3 @@
|
||||
.headline{
|
||||
color: #f1f1f1 !important;
|
||||
}
|
||||
9
ui/src/components/logo/Logo.js
Normal file
9
ui/src/components/logo/Logo.js
Normal file
@@ -0,0 +1,9 @@
|
||||
import React from 'react';
|
||||
import logo from '../../assets/logo.png';
|
||||
import logoWhite from '../../assets/logo_white.png';
|
||||
|
||||
import './Logo.less';
|
||||
|
||||
export default function Logo({ width = 350, white = false } = {}) {
|
||||
return <img src={white ? logoWhite : logo} width={width} className="logo" />;
|
||||
}
|
||||
5
ui/src/components/logo/Logo.less
Normal file
5
ui/src/components/logo/Logo.less
Normal file
@@ -0,0 +1,5 @@
|
||||
.logo {
|
||||
position: absolute;
|
||||
top: .1rem;
|
||||
right: 2rem;
|
||||
}
|
||||
21
ui/src/components/logout/Logout.js
Normal file
21
ui/src/components/logout/Logout.js
Normal file
@@ -0,0 +1,21 @@
|
||||
import React from 'react';
|
||||
import { Button } from 'semantic-ui-react';
|
||||
import { xhrPost } from '../../services/xhr';
|
||||
|
||||
const Logout = function Logout() {
|
||||
return (
|
||||
<Button
|
||||
content="Logout"
|
||||
labelPosition="left"
|
||||
icon="user"
|
||||
size="mini"
|
||||
onClick={async () => {
|
||||
await xhrPost('/api/login/logout');
|
||||
location.reload();
|
||||
}}
|
||||
negative
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
export default Logout;
|
||||
39
ui/src/components/menu/Menu.js
Normal file
39
ui/src/components/menu/Menu.js
Normal file
@@ -0,0 +1,39 @@
|
||||
import React from 'react';
|
||||
import { useHistory } from 'react-router-dom';
|
||||
import { Menu } from 'semantic-ui-react';
|
||||
|
||||
import './Menu.less';
|
||||
import { useLocation } from 'react-router';
|
||||
|
||||
const TopMenu = function TopMenu({ isAdmin }) {
|
||||
const history = useHistory();
|
||||
const location = useLocation();
|
||||
|
||||
const isActiveRoute = (name) => location.pathname.indexOf(name) !== -1;
|
||||
|
||||
return (
|
||||
<Menu pointing secondary className="topMenu">
|
||||
<Menu.Item
|
||||
name="jobs"
|
||||
active={isActiveRoute('jobs')}
|
||||
className={isActiveRoute('jobs') ? 'topMenu__active' : 'topMenu__item'}
|
||||
onClick={() => history.push('/jobs')}
|
||||
>
|
||||
Job Configuration
|
||||
</Menu.Item>
|
||||
|
||||
{isAdmin && (
|
||||
<Menu.Item
|
||||
name="user"
|
||||
active={isActiveRoute('users')}
|
||||
className={isActiveRoute('users') ? 'topMenu__active' : 'topMenu__item'}
|
||||
onClick={() => history.push('/users')}
|
||||
>
|
||||
User configuration
|
||||
</Menu.Item>
|
||||
)}
|
||||
</Menu>
|
||||
);
|
||||
};
|
||||
|
||||
export default TopMenu;
|
||||
15
ui/src/components/menu/Menu.less
Normal file
15
ui/src/components/menu/Menu.less
Normal file
@@ -0,0 +1,15 @@
|
||||
.topMenu {
|
||||
border-bottom: 1px solid #b7b7b7f2 !important;
|
||||
|
||||
&__active {
|
||||
border-bottom: 1px solid #06dcfff2 !important;
|
||||
font-weight: 550 !important;
|
||||
color: #78e5ff !important;
|
||||
margin: 0 0 -1px !important;
|
||||
}
|
||||
|
||||
&__item {
|
||||
color: #fffffff2 !important;
|
||||
border-color: transparent !important;
|
||||
}
|
||||
}
|
||||
15
ui/src/components/permission/InsufficientPermission.js
Normal file
15
ui/src/components/permission/InsufficientPermission.js
Normal file
@@ -0,0 +1,15 @@
|
||||
import React from 'react';
|
||||
import { Header } from 'semantic-ui-react';
|
||||
import insufficientPermission from '../../assets/insufficient_permission.png';
|
||||
|
||||
export default function InsufficientPermission() {
|
||||
return (
|
||||
<div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center', flexDirection: 'column' }}>
|
||||
<img src={insufficientPermission} height={250} />
|
||||
<br />
|
||||
<Header as="h4" inverted>
|
||||
Insufficient permission :(
|
||||
</Header>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
21
ui/src/components/permission/PermissionAwareRoute.js
Normal file
21
ui/src/components/permission/PermissionAwareRoute.js
Normal file
@@ -0,0 +1,21 @@
|
||||
import React from 'react';
|
||||
|
||||
import { Redirect } from 'react-router-dom';
|
||||
import { Route } from 'react-router';
|
||||
|
||||
export default function PermissionAwareRoute({ currentUser, name, path, component }) {
|
||||
/**
|
||||
* Checks if given component should be rendered if current user has given permission enabled. If that's not the case,
|
||||
* user is redirected to '/403'.
|
||||
*
|
||||
* @param permission
|
||||
* @param component
|
||||
* @param path
|
||||
* @returns {*}
|
||||
*/
|
||||
const checkIfAdmin = (component, path) => {
|
||||
return currentUser != null && currentUser.isAdmin ? component : <Redirect from={path} to="/403" />;
|
||||
};
|
||||
|
||||
return <Route name={name} path={path} render={() => checkIfAdmin(component, path)} />;
|
||||
}
|
||||
29
ui/src/components/placeholder/Placeholder.js
Normal file
29
ui/src/components/placeholder/Placeholder.js
Normal file
@@ -0,0 +1,29 @@
|
||||
import React from 'react';
|
||||
|
||||
import './Placeholder.less';
|
||||
|
||||
function getPlaceholder(rowCount, className) {
|
||||
const rows = [];
|
||||
for (let i = 0; i < rowCount; i++) {
|
||||
rows.push(<div className="place__line" key={i} />);
|
||||
}
|
||||
const clazz = `place ${className == null ? '' : className}`;
|
||||
return (
|
||||
<div className={clazz}>
|
||||
<div className="place__circle" />
|
||||
<div className="place__place_lines_wrapper">{rows}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Placeholder({ rows = 3, ready = false, children, customPlaceholder, className }) {
|
||||
if (!ready) {
|
||||
if (customPlaceholder != null) {
|
||||
return customPlaceholder;
|
||||
}
|
||||
|
||||
return getPlaceholder(rows, className);
|
||||
}
|
||||
|
||||
return children;
|
||||
}
|
||||
36
ui/src/components/placeholder/Placeholder.less
Normal file
36
ui/src/components/placeholder/Placeholder.less
Normal file
@@ -0,0 +1,36 @@
|
||||
.place {
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
display:flex;
|
||||
|
||||
&__place_lines_wrapper{
|
||||
width:100%;
|
||||
}
|
||||
|
||||
&__line {
|
||||
height: 10px;
|
||||
margin: 10px;
|
||||
animation: pulse 1s infinite ease-in-out;
|
||||
}
|
||||
|
||||
&__circle {
|
||||
height: 4rem;
|
||||
width: 5rem;
|
||||
margin: 10px;
|
||||
border-radius: 360px;
|
||||
animation: pulse 1s infinite ease-in-out;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0% {
|
||||
background-color: rgba(165, 165, 165, 0.1)
|
||||
}
|
||||
50% {
|
||||
background-color: rgba(165, 165, 165, 0.3)
|
||||
}
|
||||
100% {
|
||||
background-color: rgba(165, 165, 165, 0.1)
|
||||
}
|
||||
}
|
||||
66
ui/src/components/table/JobTable.js
Normal file
66
ui/src/components/table/JobTable.js
Normal file
@@ -0,0 +1,66 @@
|
||||
import React, { Fragment } from 'react';
|
||||
|
||||
import { Table, Button } from 'semantic-ui-react';
|
||||
import Switch from 'react-switch';
|
||||
|
||||
const emptyTable = () => {
|
||||
return (
|
||||
<Table.Row>
|
||||
<Table.Cell collapsing colSpan={6} style={{ textAlign: 'center' }}>
|
||||
No Data
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
};
|
||||
|
||||
const content = (jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight) => {
|
||||
return (
|
||||
<Fragment>
|
||||
{Object.keys(jobs).map((jobKey) => {
|
||||
const job = jobs[jobKey];
|
||||
|
||||
return (
|
||||
<Table.Row key={jobKey}>
|
||||
<Table.Cell collapsing>
|
||||
<Switch onChange={(checked) => onJobStatusChanged(job.id, checked)} checked={job.enabled} />
|
||||
</Table.Cell>
|
||||
<Table.Cell>{job.name}</Table.Cell>
|
||||
<Table.Cell>{job.numberOfFoundListings || 0}</Table.Cell>
|
||||
<Table.Cell>{job.provider.length || 0}</Table.Cell>
|
||||
<Table.Cell>{job.notificationAdapter.length || 0}</Table.Cell>
|
||||
<Table.Cell>
|
||||
<div style={{ float: 'right' }}>
|
||||
<Button circular color="teal" icon="chart line" onClick={() => onJobInsight(job.id)} />
|
||||
<Button circular color="red" icon="trash" onClick={() => onJobRemoval(job.id)} />
|
||||
<Button circular color="blue" icon="edit" onClick={() => onJobEdit(job.id)} />
|
||||
</div>
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
})}
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight } = {}) {
|
||||
return (
|
||||
<Table singleLine inverted>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell />
|
||||
<Table.HeaderCell>Job Name</Table.HeaderCell>
|
||||
<Table.HeaderCell>Number of findings</Table.HeaderCell>
|
||||
<Table.HeaderCell>Active provider</Table.HeaderCell>
|
||||
<Table.HeaderCell>Active notification adapter</Table.HeaderCell>
|
||||
<Table.HeaderCell></Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
||||
<Table.Body>
|
||||
{Object.keys(jobs).length === 0
|
||||
? emptyTable()
|
||||
: content(jobs, onJobRemoval, onJobStatusChanged, onJobEdit, onJobInsight)}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
);
|
||||
}
|
||||
49
ui/src/components/table/NotificationAdapterTable.js
Normal file
49
ui/src/components/table/NotificationAdapterTable.js
Normal file
@@ -0,0 +1,49 @@
|
||||
import React, { Fragment } from 'react';
|
||||
|
||||
import { Table, Button } from 'semantic-ui-react';
|
||||
|
||||
const emptyTable = () => {
|
||||
return (
|
||||
<Table.Row>
|
||||
<Table.Cell collapsing colSpan="3" style={{ textAlign: 'center' }}>
|
||||
No Data
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
};
|
||||
|
||||
const content = (adapterData, onRemove) => {
|
||||
return (
|
||||
<Fragment>
|
||||
{adapterData.map((data) => {
|
||||
return (
|
||||
<Table.Row key={data.id}>
|
||||
<Table.Cell>{data.name}</Table.Cell>
|
||||
<Table.Cell>
|
||||
<div style={{ float: 'right' }}>
|
||||
<Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
|
||||
</div>
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
})}
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default function NotificationAdapterTable({ notificationAdapter = [], onRemove } = {}) {
|
||||
return (
|
||||
<Table singleLine inverted>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>Notification Adapter Name</Table.HeaderCell>
|
||||
<Table.HeaderCell></Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
||||
<Table.Body>
|
||||
{notificationAdapter.length === 0 ? emptyTable() : content(notificationAdapter, onRemove)}
|
||||
</Table.Body>
|
||||
</Table>
|
||||
);
|
||||
}
|
||||
53
ui/src/components/table/ProviderTable.js
Normal file
53
ui/src/components/table/ProviderTable.js
Normal file
@@ -0,0 +1,53 @@
|
||||
import React, { Fragment } from 'react';
|
||||
|
||||
import { Table, Button } from 'semantic-ui-react';
|
||||
|
||||
const emptyTable = () => {
|
||||
return (
|
||||
<Table.Row>
|
||||
<Table.Cell collapsing colSpan="3" style={{ textAlign: 'center' }}>
|
||||
No Data
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
};
|
||||
|
||||
const truncate = (str, n) => {
|
||||
return str.length > n ? str.substr(0, n - 1) + '…' : str;
|
||||
};
|
||||
|
||||
const content = (providerData, onRemove) => {
|
||||
return (
|
||||
<Fragment>
|
||||
{providerData.map((data) => {
|
||||
return (
|
||||
<Table.Row key={data.id}>
|
||||
<Table.Cell>{data.name}</Table.Cell>
|
||||
<Table.Cell>{truncate(data.url, 60)}</Table.Cell>
|
||||
<Table.Cell>
|
||||
<div style={{ float: 'right' }}>
|
||||
<Button circular color="red" icon="trash" onClick={() => onRemove(data.id)} />
|
||||
</div>
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
})}
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default function ProviderTable({ providerData = [], onRemove } = {}) {
|
||||
return (
|
||||
<Table singleLine inverted>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>Provider Name</Table.HeaderCell>
|
||||
<Table.HeaderCell>Url</Table.HeaderCell>
|
||||
<Table.HeaderCell></Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
||||
<Table.Body>{providerData.length === 0 ? emptyTable() : content(providerData, onRemove)}</Table.Body>
|
||||
</Table>
|
||||
);
|
||||
}
|
||||
49
ui/src/components/table/UserTable.js
Normal file
49
ui/src/components/table/UserTable.js
Normal file
@@ -0,0 +1,49 @@
|
||||
import React from 'react';
|
||||
|
||||
import { Table, Button } from 'semantic-ui-react';
|
||||
import { format } from '../../services/time/timeService';
|
||||
|
||||
const emptyTable = () => {
|
||||
return (
|
||||
<Table.Row>
|
||||
<Table.Cell collapsing colSpan={4} style={{ textAlign: 'center' }}>
|
||||
No Data
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
};
|
||||
|
||||
const content = (user, onUserRemoval, onUserEdit) => {
|
||||
return user.map((user) => {
|
||||
return (
|
||||
<Table.Row key={user.id}>
|
||||
<Table.Cell>{user.username}</Table.Cell>
|
||||
<Table.Cell>{user.lastLogin == null ? '---' : format(user.lastLogin)}</Table.Cell>
|
||||
<Table.Cell>{user.numberOfJobs}</Table.Cell>
|
||||
<Table.Cell>
|
||||
<div style={{ float: 'right' }}>
|
||||
<Button circular color="red" icon="trash" onClick={() => onUserRemoval(user.id)} />
|
||||
<Button circular color="blue" icon="edit" onClick={() => onUserEdit(user.id)} />
|
||||
</div>
|
||||
</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
export default function UserTable({ user = [], onUserRemoval, onUserEdit } = {}) {
|
||||
return (
|
||||
<Table inverted>
|
||||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>Username</Table.HeaderCell>
|
||||
<Table.HeaderCell>Last login</Table.HeaderCell>
|
||||
<Table.HeaderCell>Number of jobs</Table.HeaderCell>
|
||||
<Table.HeaderCell></Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
||||
<Table.Body>{user.length === 0 ? emptyTable() : content(user, onUserRemoval, onUserEdit)}</Table.Body>
|
||||
</Table>
|
||||
);
|
||||
}
|
||||
27
ui/src/components/toasts/Toast.js
Normal file
27
ui/src/components/toasts/Toast.js
Normal file
@@ -0,0 +1,27 @@
|
||||
import React from 'react';
|
||||
|
||||
import './Toasts.css';
|
||||
|
||||
export default function Toast({ id, delay = 5500, message, onHide, backgroundColor, color, title }) {
|
||||
const [className, setClassname] = React.useState('toast-container show-toast');
|
||||
|
||||
React.useEffect(() => {
|
||||
let hideTimeout = null;
|
||||
const timeout = setTimeout(() => {
|
||||
setClassname('toast-container hide-toast');
|
||||
hideTimeout = setTimeout(() => {
|
||||
onHide && onHide(id);
|
||||
}, 500);
|
||||
}, delay);
|
||||
return () => {
|
||||
clearTimeout(timeout);
|
||||
clearTimeout(hideTimeout);
|
||||
};
|
||||
}, [id, delay, onHide]);
|
||||
return (
|
||||
<div className={className} style={{ backgroundColor, color }}>
|
||||
<h5>{title}</h5>
|
||||
{message}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
12
ui/src/components/toasts/ToastContainer.js
Normal file
12
ui/src/components/toasts/ToastContainer.js
Normal file
@@ -0,0 +1,12 @@
|
||||
import Toast from './Toast';
|
||||
import React from 'react';
|
||||
|
||||
export default function ToastsContainer({ toasts, onToastFinished }) {
|
||||
return (
|
||||
<div className="toasts-container">
|
||||
{toasts.map((toast, index) => (
|
||||
<Toast key={index} {...toast} onHide={onToastFinished} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
7
ui/src/components/toasts/ToastContext.js
Normal file
7
ui/src/components/toasts/ToastContext.js
Normal file
@@ -0,0 +1,7 @@
|
||||
import { createContext } from 'react';
|
||||
|
||||
const CheckoutDrawerContext = createContext({
|
||||
showToast: () => {},
|
||||
});
|
||||
|
||||
export default CheckoutDrawerContext;
|
||||
63
ui/src/components/toasts/Toasts.css
Normal file
63
ui/src/components/toasts/Toasts.css
Normal file
@@ -0,0 +1,63 @@
|
||||
.toasts-container {
|
||||
position: fixed;
|
||||
z-index: 65535;
|
||||
right: 0;
|
||||
max-width: 250px;
|
||||
display: flex;
|
||||
flex-direction: column-reverse;
|
||||
}
|
||||
|
||||
.toasts-container > .toast-container {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.toasts-container:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.toast-container {
|
||||
visibility: hidden;
|
||||
position: relative;
|
||||
z-index: 65535;
|
||||
right: -1000px;
|
||||
|
||||
background-color: skyblue;
|
||||
border-radius: 10px;
|
||||
padding: 10px;
|
||||
|
||||
min-width: 10rem;
|
||||
min-height: 3rem;
|
||||
}
|
||||
|
||||
.toast-container.show-toast {
|
||||
visibility: visible;
|
||||
right: 24px;
|
||||
animation: slidein 0.5s;
|
||||
}
|
||||
|
||||
.toast-container.hide-toast {
|
||||
visibility: visible;
|
||||
animation: slideout 0.5s;
|
||||
}
|
||||
|
||||
@keyframes slidein {
|
||||
from {
|
||||
right: -1000px;
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
right: 24px;
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes slideout {
|
||||
from {
|
||||
right: 24px;
|
||||
opacity: 1;
|
||||
}
|
||||
to {
|
||||
right: -1000px;
|
||||
opacity: 0;
|
||||
}
|
||||
}
|
||||
23
ui/src/components/toasts/useToast.js
Normal file
23
ui/src/components/toasts/useToast.js
Normal file
@@ -0,0 +1,23 @@
|
||||
import React from 'react';
|
||||
|
||||
export default function useToast() {
|
||||
const [toasts, setToasts] = React.useState([]);
|
||||
|
||||
const showToast = ({ message, delay, color, backgroundColor, title }) => {
|
||||
const toast = {
|
||||
id: toasts.length,
|
||||
message,
|
||||
delay,
|
||||
backgroundColor,
|
||||
color,
|
||||
title,
|
||||
};
|
||||
setToasts([...toasts, toast].reverse());
|
||||
};
|
||||
|
||||
const onToastFinished = (id) => {
|
||||
setToasts(toasts.filter((toast) => toast.id !== id));
|
||||
};
|
||||
|
||||
return [showToast, onToastFinished, toasts];
|
||||
}
|
||||
17
ui/src/index.html
Normal file
17
ui/src/index.html
Normal file
@@ -0,0 +1,17 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"
|
||||
name="viewport"
|
||||
content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
|
||||
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css">
|
||||
<meta name="google" content="notranslate">
|
||||
|
||||
<title>Fredy</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
|
||||
</body>
|
||||
<script src="fredy.bundle.js"></script>
|
||||
</html>
|
||||
43
ui/src/services/rematch/models/jobs.js
Normal file
43
ui/src/services/rematch/models/jobs.js
Normal file
@@ -0,0 +1,43 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
|
||||
export const jobs = {
|
||||
state: {
|
||||
jobs: [],
|
||||
insights: {},
|
||||
},
|
||||
reducers: {
|
||||
setJobs: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
jobs: Object.freeze(payload),
|
||||
};
|
||||
},
|
||||
setJobInsights: (state, payload, jobId) => {
|
||||
return {
|
||||
...state,
|
||||
insights: {
|
||||
...state.insights,
|
||||
[jobId]: Object.freeze(payload),
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getJobs() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs');
|
||||
this.setJobs(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
async getInsightDataForJob(jobId) {
|
||||
try {
|
||||
const response = await xhrGet(`/api/jobs/insights/${jobId}`);
|
||||
this.setJobInsights(response.json, jobId);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs/insights. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
19
ui/src/services/rematch/models/notificationAdapter.js
Normal file
19
ui/src/services/rematch/models/notificationAdapter.js
Normal file
@@ -0,0 +1,19 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const notificationAdapter = {
|
||||
state: [],
|
||||
reducers: {
|
||||
setAdapter: (state, payload) => {
|
||||
return [...Object.freeze(payload)];
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getAdapter() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs/notificationAdapter');
|
||||
this.setAdapter(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs/notificationAdapter. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
19
ui/src/services/rematch/models/provider.js
Normal file
19
ui/src/services/rematch/models/provider.js
Normal file
@@ -0,0 +1,19 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
export const provider = {
|
||||
state: [],
|
||||
reducers: {
|
||||
setProvider: (state, payload) => {
|
||||
return [...Object.freeze(payload)];
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getProvider() {
|
||||
try {
|
||||
const response = await xhrGet('/api/jobs/provider');
|
||||
this.setProvider(response.json);
|
||||
} catch (Exception) {
|
||||
console.error(`Error while trying to get resource for api/jobs/provider. Error:`, Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
41
ui/src/services/rematch/models/user.js
Normal file
41
ui/src/services/rematch/models/user.js
Normal file
@@ -0,0 +1,41 @@
|
||||
import { xhrGet } from '../../xhr';
|
||||
|
||||
export const user = {
|
||||
state: {
|
||||
users: [],
|
||||
currentUser: null,
|
||||
},
|
||||
reducers: {
|
||||
//only admins
|
||||
setUsers: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
users: payload,
|
||||
};
|
||||
},
|
||||
setCurrentUser: (state, payload) => {
|
||||
return {
|
||||
...state,
|
||||
currentUser: Object.freeze(payload),
|
||||
};
|
||||
},
|
||||
},
|
||||
effects: {
|
||||
async getUsers() {
|
||||
try {
|
||||
const response = await xhrGet('/api/admin/users');
|
||||
this.setUsers(response.json);
|
||||
} catch (Exception) {
|
||||
console.error('Error while trying to get resource for api/admin/users. Error:', Exception);
|
||||
}
|
||||
},
|
||||
async getCurrentUser() {
|
||||
try {
|
||||
const response = await xhrGet('/api/login/user');
|
||||
this.setCurrentUser(response.json);
|
||||
} catch (Exception) {
|
||||
console.error('Error while trying to get resource for api/login/user. Error:', Exception);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
30
ui/src/services/rematch/store.js
Normal file
30
ui/src/services/rematch/store.js
Normal file
@@ -0,0 +1,30 @@
|
||||
import { notificationAdapter } from './models/notificationAdapter';
|
||||
import createLoadingPlugin from '@rematch/loading';
|
||||
import { provider } from './models/provider';
|
||||
import { createLogger } from 'redux-logger';
|
||||
import { jobs } from './models/jobs';
|
||||
import { user } from './models/user';
|
||||
import { init } from '@rematch/core';
|
||||
|
||||
const middleware = [];
|
||||
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
// eslint-disable-line no-redeclare
|
||||
middleware.push(createLogger({ duration: false, collapsed: (getState, action, logEntry) => !logEntry.error }));
|
||||
}
|
||||
|
||||
const store = init({
|
||||
name: 'fredy',
|
||||
models: {
|
||||
notificationAdapter,
|
||||
provider,
|
||||
jobs,
|
||||
user,
|
||||
},
|
||||
plugins: [createLoadingPlugin({})],
|
||||
redux: {
|
||||
middlewares: middleware,
|
||||
},
|
||||
});
|
||||
|
||||
export const reduxStore = store;
|
||||
12
ui/src/services/time/timeService.js
Normal file
12
ui/src/services/time/timeService.js
Normal file
@@ -0,0 +1,12 @@
|
||||
export function format(ts) {
|
||||
return new Intl.DateTimeFormat('default', {
|
||||
year: 'numeric',
|
||||
month: 'numeric',
|
||||
day: 'numeric',
|
||||
hour: 'numeric',
|
||||
minute: 'numeric',
|
||||
second: 'numeric',
|
||||
}).format(ts);
|
||||
}
|
||||
|
||||
export const roundToNext5Minute = (ts) => Math.ceil(ts / (1000 * 60 * 5)) * (1000 * 60 * 5);
|
||||
@@ -0,0 +1,13 @@
|
||||
export function transform({ id, name, fields }) {
|
||||
const fieldValues = {};
|
||||
|
||||
Object.keys(fields).map((key) => {
|
||||
fieldValues[key] = fields[key].value;
|
||||
});
|
||||
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
fields: fieldValues,
|
||||
};
|
||||
}
|
||||
8
ui/src/services/transformer/providerTransformer.js
Normal file
8
ui/src/services/transformer/providerTransformer.js
Normal file
@@ -0,0 +1,8 @@
|
||||
export function transform({ name, id, enabled, url }) {
|
||||
return {
|
||||
name,
|
||||
id,
|
||||
enabled,
|
||||
url,
|
||||
};
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user