mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Concurrent-Jobs (#7)
Fredy 2.0.0 introduces the concept of search jobs. You can now configure multiple of these jobs running in the same instance of Fredy. Also a new API has been created 🎉
This commit is contained in:
Christian Kellner
committed by
GitHub
GitHub
parent
52a32f7453
commit
770d30af95
42
.eslintrc.js
Normal file
42
.eslintrc.js
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
module.exports = {
|
||||||
|
env: {
|
||||||
|
commonjs: true,
|
||||||
|
es6: true,
|
||||||
|
node: true
|
||||||
|
},
|
||||||
|
extends: 'eslint:recommended',
|
||||||
|
globals: {
|
||||||
|
Atomics: 'readonly',
|
||||||
|
SharedArrayBuffer: 'readonly',
|
||||||
|
Promise: false,
|
||||||
|
describe: true,
|
||||||
|
it: true
|
||||||
|
},
|
||||||
|
parserOptions: {
|
||||||
|
ecmaVersion: 2018
|
||||||
|
},
|
||||||
|
rules: {
|
||||||
|
eqeqeq: [2, 'allow-null'],
|
||||||
|
|
||||||
|
// ###########################################################
|
||||||
|
// ### Semantics / Performance impacting
|
||||||
|
// ###########################################################
|
||||||
|
// babel inserts `'use strict';` for us
|
||||||
|
strict: 0,
|
||||||
|
|
||||||
|
'no-redeclare': [2, { builtinGlobals: false }],
|
||||||
|
|
||||||
|
// If a class method does not use this, it can safely be made a static function.
|
||||||
|
// http://eslint.org/docs/rules/class-methods-use-this
|
||||||
|
'class-methods-use-this': ['off'],
|
||||||
|
|
||||||
|
// ###########################################################
|
||||||
|
// ### Style
|
||||||
|
// ###########################################################
|
||||||
|
indent: ['off', 2],
|
||||||
|
'linebreak-style': ['error', 'unix'],
|
||||||
|
quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }],
|
||||||
|
semi: ['error', 'always'],
|
||||||
|
'no-console': ['error', { allow: ['warn', 'error'] }]
|
||||||
|
}
|
||||||
|
};
|
||||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,5 +1,5 @@
|
|||||||
node_modules/
|
node_modules/
|
||||||
npm-debug.log
|
npm-debug.log
|
||||||
|
config.json
|
||||||
store.json
|
store.json
|
||||||
.DS_Store
|
.DS_Store
|
||||||
config.json
|
|
||||||
|
|||||||
9
CHANGELOG.md
Normal file
9
CHANGELOG.md
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
###### [V2.0.0]
|
||||||
|
```
|
||||||
|
- Fredy can now run multiple search job on one instance
|
||||||
|
- Changed lot's of the structure of Fredy to make this happen
|
||||||
|
[BREAKING CHANGES]
|
||||||
|
- The config has been changed, the config of V1.x will not work any longer
|
||||||
|
- Sources have been renamed to provider
|
||||||
|
```
|
||||||
|
For more info on how to upgrade from Fredy V1.x, plz check the [Upgrade Guide](./doc/upgrade-from-1-to-2.md)
|
||||||
@@ -19,10 +19,12 @@ If you've written a new provider you are an awesome person. You know it and I do
|
|||||||
|
|
||||||
To write tests for provider, you need to use Node 8 as the tests are using `async / await`
|
To write tests for provider, you need to use Node 8 as the tests are using `async / await`
|
||||||
|
|
||||||
|
#### Codestyle
|
||||||
|
I'm using Eslint to maintain quote style and quality. Do not skip it...
|
||||||
|
|
||||||
##### To do before merging:
|
##### To do before merging:
|
||||||
|
|
||||||
- executed tests? (`npm run test`)
|
- executed tests? (`npm run test`)
|
||||||
- executed reformat? (`npm run format`)
|
|
||||||
- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
|
- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
|
||||||
|
|
||||||
_Thanks!_ :heart:
|
_Thanks!_ :heart:
|
||||||
|
|||||||
107
README.md
107
README.md
@@ -10,12 +10,14 @@ If _Fredy_ found matching results, it will send them to via Slack. (More adapter
|
|||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
- Make sure to use Node 8 and above
|
- Make sure to use Node 11 and above
|
||||||
- Install the dependencies using `npm install`
|
- Install the dependencies using `npm install` or `yarn`
|
||||||
- create your configuration file. Use the example config for this. `cp conf/config.example conf/config.json`
|
- create your configuration file. Use the example config for this. `cp conf/config.example conf/config.json`
|
||||||
- configure the desired values
|
- configure the desired values
|
||||||
- start _Fredy_ using `npm start`
|
- start _Fredy_ using `npm start` or `yarn run start`
|
||||||
|
|
||||||
|
## :point_up: Breaking Changes when updating from v1.x to v2
|
||||||
|
See [Upgrade Guide](./doc/upgrade-from-1-to-2.md)
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
@@ -35,19 +37,13 @@ Adding new notification adapter is easy however. See [Contribution](https://gith
|
|||||||
##### Slack
|
##### Slack
|
||||||
```json
|
```json
|
||||||
"slack": {
|
"slack": {
|
||||||
"channel": "someChannel",
|
"channel": "someChannel", "token": "someToken", "enabled": true}
|
||||||
"token": "someToken",
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
##### Telegram
|
##### Telegram
|
||||||
```json
|
```json
|
||||||
"telegram": {
|
"telegram": {
|
||||||
"chatId": "someChannel",
|
"chatId": "someChannel", "token": "someToken", "enabled": true}
|
||||||
"token": "someToken",
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
|
For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
|
||||||
@@ -59,7 +55,24 @@ curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
|
|||||||
|
|
||||||
A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
|
A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
|
||||||
|
|
||||||
### 2. Configure the providers
|
### 2. Multiple search jobs
|
||||||
|
|
||||||
|
Since v2.0.0, Fredy supports multiple search jobs running within the same instance of Fredy. For this to work correctly, you need to give each search job a unique name.
|
||||||
|
See the example config...
|
||||||
|
```json
|
||||||
|
(...)
|
||||||
|
"jobs": {
|
||||||
|
"yourSearchJob": {
|
||||||
|
"some":"config"
|
||||||
|
},
|
||||||
|
"yourOtherSearchJob": {
|
||||||
|
"some":"config"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(...)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Configure the providers
|
||||||
|
|
||||||
Configure the providers like described below. To disable a provider just remove its entry from the configuration or set it to `false`.
|
Configure the providers like described below. To disable a provider just remove its entry from the configuration or set it to `false`.
|
||||||
|
|
||||||
@@ -69,37 +82,21 @@ These are the current provider that are already implemented within _Fredy_
|
|||||||
|
|
||||||
```json
|
```json
|
||||||
"kleinanzeigen": {
|
"kleinanzeigen": {
|
||||||
"url": "https://www.ebay-kleinanzeigen.de/...",
|
"url": "https://www.ebay-kleinanzeigen.de/...", "enabled": true}
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
"immoscout": {
|
"immoscout": {
|
||||||
"url": "https://www.immobilienscout24.de/...",
|
"url": "https://www.immobilienscout24.de/...", "enabled": true},
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"immowelt": {
|
"immowelt": {
|
||||||
"url": "https://www.immowelt.de/...",
|
"url": "https://www.immowelt.de/...", "enabled": true},
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"immonet": {
|
"immonet": {
|
||||||
"url": "http://www.immonet.de/...",
|
"url": "http://www.immonet.de/...", "enabled": true},
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"kalaydo": {
|
"kalaydo": {
|
||||||
"url": "http://www.kalaydo.de/...",
|
"url": "http://www.kalaydo.de/...", "enabled": true},
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"einsAImmobilien": {
|
"einsAImmobilien": {
|
||||||
"url": "https://www.1a-immobilienmarkt.de/...",
|
"url": "https://www.1a-immobilienmarkt.de/...", "enabled": true},
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"neubauKompass": {
|
"neubauKompass": {
|
||||||
"url": "https://www.neubaukompass.de/...",
|
"url": "https://www.neubaukompass.de/...", "enabled": true},
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"wgGesucht": {
|
"wgGesucht": {
|
||||||
"url": "https://www.wg-gesucht.de/...",
|
"url": "https://www.wg-gesucht.de/...", "enabled": true}
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Go to the respective provider page and create your custom search queries by
|
Go to the respective provider page and create your custom search queries by
|
||||||
@@ -112,14 +109,14 @@ the resulting listings page.
|
|||||||
|
|
||||||
See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)
|
See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)
|
||||||
|
|
||||||
### 3. Add Filters (optional)
|
### 4. Add Filters (optional)
|
||||||
|
|
||||||
|
|
||||||
#### Blacklist
|
#### Blacklist
|
||||||
|
|
||||||
```json
|
```json
|
||||||
"blacklist": [
|
"blacklist": [
|
||||||
"vermietet"
|
"vermietet"
|
||||||
]
|
]
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -129,38 +126,36 @@ whole words) are removed.
|
|||||||
#### District blacklist
|
#### District blacklist
|
||||||
```json
|
```json
|
||||||
"blacklistedDistricts": [
|
"blacklistedDistricts": [
|
||||||
"Altstadt"
|
"Altstadt"
|
||||||
]
|
]
|
||||||
```
|
```
|
||||||
Districts that are unwanted can be blacklisted here.
|
Districts that are unwanted can be blacklisted here.
|
||||||
|
|
||||||
This makes sense for provider that only offer limited filter functions like Kalaydo/Ebay.
|
This makes sense for provider that only offer limited filter functions like Kalaydo/Ebay.
|
||||||
|
|
||||||
## Stats
|
# API
|
||||||
To monitor, what _Fredy_ is internally doing, you might want to check the current stats. These includes the `config` that is currently being used.
|
While Fredy is running, you can make use of the rest api provided on port `9988` to get information about the current status of Fredy.
|
||||||
Also it includes an array of filter results per provider.
|
#### http://localhost:9988/
|
||||||
|
Gives you an overview of running search jobs, their included enabled provider, last execution and the number of listings, found by each provider.
|
||||||
|
|
||||||
You can call the stats http endpoint like this:
|
#### http://localhost:9988/ping
|
||||||
```
|
Should you ever need some health checks, this returns pong ;)
|
||||||
curl http://localhost:9876
|
|
||||||
```
|
#### http://localhost:9988/jobs/:name
|
||||||
The ports is depending on what you've configured in your config file.
|
Returns specific information about the job with the given name or `404` if the job could not be found.
|
||||||
|
|
||||||
# Docker
|
# Docker
|
||||||
|
Use the Dockerfile in this Repository to build an image.
|
||||||
|
|
||||||
Use the Dockerfile in this Repo to build a Image
|
Example: `docker build -t fredy/fredy /path/to/your/Dockerfile`
|
||||||
|
|
||||||
Ex: docker build -t fredy/fredy /path/to/your/Dockerfile
|
## Create & run a container
|
||||||
|
|
||||||
## Create & Run a Container
|
Put your config.json to `/path/to/your/conf/`
|
||||||
|
|
||||||
Put your config.json to /path/to/your/conf/
|
Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9876:9876 fredy/fredy`
|
||||||
|
|
||||||
Ex: docker create --name fredy -v /path/to/your/conf/:/conf -p 9876:9876 fredy/fredy
|
|
||||||
|
|
||||||
## Logs
|
## Logs
|
||||||
|
|
||||||
You can browse Logs with
|
You can browse the logs with `docker logs fredy -f`
|
||||||
|
|
||||||
docker logs fredy -f
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,66 +1,87 @@
|
|||||||
{
|
{
|
||||||
"interval": 10,
|
"interval": 1,
|
||||||
"enableStats": false,
|
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36",
|
||||||
"statsPort": 9875,
|
"jobs": {
|
||||||
"notification": {
|
"yourSearchJob": {
|
||||||
"slack": {
|
"notification": {
|
||||||
"token": "someToken",
|
"slack": {
|
||||||
"channel": "someChannel",
|
"token": "",
|
||||||
"enabled": false
|
"channel": "",
|
||||||
},
|
"enabled": false
|
||||||
"console": {
|
},
|
||||||
"enabled": true
|
"telegram": {
|
||||||
},
|
|
||||||
"telegram": {
|
|
||||||
"chatId": "",
|
"chatId": "",
|
||||||
"token": "",
|
"token": "",
|
||||||
"enabled": false
|
"enabled": false
|
||||||
},
|
},
|
||||||
},
|
"console": {
|
||||||
"sources": {
|
"enabled": true
|
||||||
"immoscout": {
|
}
|
||||||
"url": "https://www.immobilienscout24.de/...",
|
},
|
||||||
"enabled": true
|
"provider": {
|
||||||
},
|
"immoscout": {
|
||||||
"immonet": {
|
"url": "https://www.immobilienscout24.de/...",
|
||||||
"url": "https://www.immonet.de/...",
|
"enabled": true
|
||||||
"enabled": true
|
},
|
||||||
},
|
"immonet": {
|
||||||
"immowelt": {
|
"url": "https://www.immonet.de/immobiliensuche/...",
|
||||||
"url": "https://www.immowelt.de/...",
|
"enabled": true
|
||||||
"enabled": true
|
},
|
||||||
},
|
"immowelt": {
|
||||||
"kleinanzeigen": {
|
"url": "https://www.immowelt.de/liste/...",
|
||||||
"url": "https://www.ebay-kleinanzeigen.de/...",
|
"enabled": true
|
||||||
"enabled": true
|
},
|
||||||
},
|
"kleinanzeigen": {
|
||||||
"kalaydo": {
|
"url": "https://www.ebay-kleinanzeigen.de/...",
|
||||||
"url": "https://www.kalaydo.de/...",
|
"enabled": true
|
||||||
"enabled": true
|
},
|
||||||
},
|
"kalaydo": {
|
||||||
"einsAImmobilien":{
|
"url": "https://www.kalaydo.de/immobilien/...",
|
||||||
"url": "https://www.1a-immobilienmarkt.de/...",
|
"enabled": true
|
||||||
"enabled": true
|
},
|
||||||
},"neubauKompass": {
|
"abInsZuHause": {
|
||||||
"url": "https://www.neubaukompass.de/...",
|
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/...",
|
||||||
"enabled": true
|
"enabled": true
|
||||||
},
|
},
|
||||||
"wgGesucht": {
|
"einsAImmobilien": {
|
||||||
"url": "https://www.wg-gesucht.de/...",
|
"url": "https://www.1a-immobilienmarkt.de/suchen/...",
|
||||||
"enabled": true
|
"enabled": true
|
||||||
|
},
|
||||||
|
"neubauKompass": {
|
||||||
|
"url": "https://www.neubaukompass.de/...",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"wgGesucht": {
|
||||||
|
"url": "https://www.wg-gesucht.de/...",
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"blacklist": [
|
||||||
|
"Vermietete",
|
||||||
|
"Vermietet",
|
||||||
|
"vermietete",
|
||||||
|
"vermietet"
|
||||||
|
],
|
||||||
|
"blacklistedDistricts": [
|
||||||
|
"Altstadt",
|
||||||
|
"Angermund",
|
||||||
|
"Carlstadt",
|
||||||
|
"Friedrichstadt",
|
||||||
|
"Heerdt",
|
||||||
|
"Hellerhof",
|
||||||
|
"Hubbelrath",
|
||||||
|
"Itter",
|
||||||
|
"Kalkum",
|
||||||
|
"Lichtenbroich",
|
||||||
|
"Lohausen",
|
||||||
|
"Niederkassel",
|
||||||
|
"Oberkassel",
|
||||||
|
"Stadtmittel",
|
||||||
|
"Stockum",
|
||||||
|
"Urdenbach",
|
||||||
|
"Wittlaer",
|
||||||
|
"Lörick"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
"blacklist": [
|
|
||||||
"swap",
|
|
||||||
"tausch",
|
|
||||||
"wg",
|
|
||||||
"Vermietete",
|
|
||||||
"Vermietet",
|
|
||||||
"vermietete",
|
|
||||||
"vermietet"
|
|
||||||
],
|
|
||||||
"blacklistedDistrics": [
|
|
||||||
"some District you don't want"
|
|
||||||
],
|
|
||||||
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36"
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
{
|
|
||||||
"interval": 10,
|
|
||||||
"enableStats": false,
|
|
||||||
"statsPort": 9875,
|
|
||||||
"notification": {
|
|
||||||
"slack": {
|
|
||||||
"token": "",
|
|
||||||
"channel": "test",
|
|
||||||
"enabled": false
|
|
||||||
},
|
|
||||||
"telegram": {
|
|
||||||
"chatId": "",
|
|
||||||
"token": "",
|
|
||||||
"enabled": false
|
|
||||||
},
|
|
||||||
"console": {
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"sources": {
|
|
||||||
"immoscout": {
|
|
||||||
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"immonet": {
|
|
||||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=25&objecttype=1&locationname=D%C3%BCsseldorf&acid=&actype=&city=100207&ajaxIsRadiusActive=false&sortby=19&suchart=2&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=1_1&parentcat=1&marketingtype=1&fromprice=&toprice=&fromarea=&toarea=&fromplotarea=&toplotarea=&fromrooms=&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=&fulltext=&absenden=Ergebnisse+anzeigen",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"immowelt": {
|
|
||||||
"url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?lat=51.22061&lon=6.80575&sort=createdate%2Bdesc",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"kleinanzeigen": {
|
|
||||||
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/wohnung/k0c196l2068r5",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"kalaydo": {
|
|
||||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"einsAImmobilien":{
|
|
||||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=fcfa4ee1e6cfaf791051a6f342eec1f8&sort_type=newest",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"neubauKompass": {
|
|
||||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"wgGesucht": {
|
|
||||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"blacklist": [
|
|
||||||
"Vermietete",
|
|
||||||
"Vermietet",
|
|
||||||
"vermietete",
|
|
||||||
"vermietet"
|
|
||||||
],
|
|
||||||
"blacklistedDistrics": [
|
|
||||||
"Altstadt",
|
|
||||||
"Angermund",
|
|
||||||
"Carlstadt",
|
|
||||||
"Flehe",
|
|
||||||
"Friedrichstadt",
|
|
||||||
"Garath",
|
|
||||||
"Hafen",
|
|
||||||
"Hamm",
|
|
||||||
"Hassels",
|
|
||||||
"Heerdt",
|
|
||||||
"Hellerhof",
|
|
||||||
"Himmelgeist",
|
|
||||||
"Hubbelrath",
|
|
||||||
"Itter",
|
|
||||||
"Kaiserswerth",
|
|
||||||
"Kalkum",
|
|
||||||
"Lichtenbroich",
|
|
||||||
"Lohausen",
|
|
||||||
"Ludenberg",
|
|
||||||
"Niederkassel",
|
|
||||||
"Oberkassel",
|
|
||||||
"Stadtmittel",
|
|
||||||
"Stockum",
|
|
||||||
"Urdenbach",
|
|
||||||
"Vennhausen",
|
|
||||||
"Volmerswerth",
|
|
||||||
"Wittlaer"
|
|
||||||
],
|
|
||||||
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36"
|
|
||||||
}
|
|
||||||
168
conf/forTesting/config.multi.test.json
Executable file
168
conf/forTesting/config.multi.test.json
Executable file
@@ -0,0 +1,168 @@
|
|||||||
|
{
|
||||||
|
"interval": 1,
|
||||||
|
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36",
|
||||||
|
"jobs": {
|
||||||
|
"test1": {
|
||||||
|
"notification": {
|
||||||
|
"slack": {
|
||||||
|
"token": "",
|
||||||
|
"channel": "",
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
|
"telegram": {
|
||||||
|
"chatId": "",
|
||||||
|
"token": "",
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
|
"console": {
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"provider": {
|
||||||
|
"immoscout": {
|
||||||
|
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"immonet": {
|
||||||
|
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"immowelt": {
|
||||||
|
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"kleinanzeigen": {
|
||||||
|
"url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"kalaydo": {
|
||||||
|
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"abInsZuHause": {
|
||||||
|
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/D%C3%BCsseldorf/wohnung-kaufen/420000/90/3.5/1839/",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"einsAImmobilien": {
|
||||||
|
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"neubauKompass": {
|
||||||
|
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"wgGesucht": {
|
||||||
|
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Dusseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"blacklist": [
|
||||||
|
"Vermietete",
|
||||||
|
"Vermietet",
|
||||||
|
"vermietete",
|
||||||
|
"vermietet"
|
||||||
|
],
|
||||||
|
"blacklistedDistricts": [
|
||||||
|
"Altstadt",
|
||||||
|
"Angermund",
|
||||||
|
"Carlstadt",
|
||||||
|
"Friedrichstadt",
|
||||||
|
"Heerdt",
|
||||||
|
"Hellerhof",
|
||||||
|
"Hubbelrath",
|
||||||
|
"Itter",
|
||||||
|
"Kalkum",
|
||||||
|
"Lichtenbroich",
|
||||||
|
"Lohausen",
|
||||||
|
"Niederkassel",
|
||||||
|
"Oberkassel",
|
||||||
|
"Stadtmittel",
|
||||||
|
"Stockum",
|
||||||
|
"Urdenbach",
|
||||||
|
"Wittlaer",
|
||||||
|
"Lörick"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"test2": {
|
||||||
|
"notification": {
|
||||||
|
"slack": {
|
||||||
|
"token": "",
|
||||||
|
"channel": "",
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
|
"telegram": {
|
||||||
|
"chatId": "",
|
||||||
|
"token": "",
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
|
"console": {
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"provider": {
|
||||||
|
"immoscout": {
|
||||||
|
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"immonet": {
|
||||||
|
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"immowelt": {
|
||||||
|
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"kleinanzeigen": {
|
||||||
|
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"kalaydo": {
|
||||||
|
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"abInsZuHause": {
|
||||||
|
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/D%C3%BCsseldorf/wohnung-kaufen/420000/90/3.5/1839/",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"einsAImmobilien": {
|
||||||
|
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"neubauKompass": {
|
||||||
|
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"wgGesucht": {
|
||||||
|
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||||
|
"enabled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"blacklist": [
|
||||||
|
"Vermietete",
|
||||||
|
"Vermietet",
|
||||||
|
"vermietete",
|
||||||
|
"vermietet"
|
||||||
|
],
|
||||||
|
"blacklistedDistricts": [
|
||||||
|
"Altstadt",
|
||||||
|
"Angermund",
|
||||||
|
"Carlstadt",
|
||||||
|
"Friedrichstadt",
|
||||||
|
"Heerdt",
|
||||||
|
"Hellerhof",
|
||||||
|
"Hubbelrath",
|
||||||
|
"Itter",
|
||||||
|
"Kalkum",
|
||||||
|
"Lichtenbroich",
|
||||||
|
"Lohausen",
|
||||||
|
"Niederkassel",
|
||||||
|
"Oberkassel",
|
||||||
|
"Stadtmittel",
|
||||||
|
"Stockum",
|
||||||
|
"Urdenbach",
|
||||||
|
"Wittlaer",
|
||||||
|
"Lörick"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
doc/upgrade-from-1-to-2.md
Normal file
19
doc/upgrade-from-1-to-2.md
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
# Upgrading from v1.x to v2.0.0
|
||||||
|
|
||||||
|
Fredy 2.0.0 introduced the concept of multiple jobs running within an instance of Fredy. For this to work, I had to change the config and the storage format.
|
||||||
|
|
||||||
|
### How to update?
|
||||||
|
##### Store
|
||||||
|
It's best to clear the store completely and let Fredy rewrite it. Be careful to disable all notification adapter the first time you run Fredy 2, as it will obviously treat
|
||||||
|
everything as new.
|
||||||
|
|
||||||
|
##### Config
|
||||||
|
The config format has changed. It now supports multiple jobs. It is probably easiest to simply copy the `config.example` from `/conf` and enter your urls in there.
|
||||||
|
The new format basically wraps the config in chunks.
|
||||||
|
|
||||||
|
```json
|
||||||
|
"jobs": {
|
||||||
|
"yourSearchJob": {
|
||||||
|
"some":"stuff"
|
||||||
|
}
|
||||||
|
```
|
||||||
41
index.js
41
index.js
@@ -1,13 +1,36 @@
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const path = './lib/provider';
|
const path = './lib/provider';
|
||||||
const sources = fs.readdirSync(path);
|
const provider = fs.readdirSync(path);
|
||||||
const config = require('./conf/config.json');
|
const config = require('./conf/config.json');
|
||||||
const stats = require('./lib/services/stats');
|
const { setLastJobExecution, init: storeInit } = require('./lib/services/store');
|
||||||
|
const FredyRuntime = require('./lib/FredyRuntime');
|
||||||
|
|
||||||
setInterval(
|
//starting the api service
|
||||||
(function exec() {
|
require('./lib/api/api');
|
||||||
sources.forEach(source => require(`${path}/${source}`).run(stats));
|
|
||||||
return exec;
|
storeInit().then(() => {
|
||||||
})(),
|
setInterval(
|
||||||
config.interval * 60 * 1000
|
(function exec() {
|
||||||
);
|
Object.keys(config.jobs).forEach(jobKey => {
|
||||||
|
const jobConfig = config.jobs[jobKey];
|
||||||
|
provider
|
||||||
|
.map(pro => require(`${path}/${pro}`))
|
||||||
|
.forEach(pro => {
|
||||||
|
const providerId = pro.id();
|
||||||
|
if (providerId == null || providerId.length === 0) {
|
||||||
|
throw new Error('Provider id must not be empty. => ' + pro);
|
||||||
|
}
|
||||||
|
const providerConfig = jobConfig.provider[providerId];
|
||||||
|
if (providerConfig == null) {
|
||||||
|
throw new Error(`Provider Config for provider with id ${providerId} not found.`);
|
||||||
|
}
|
||||||
|
pro.init(providerConfig, jobConfig.blacklist, jobConfig.blacklistedDistricts);
|
||||||
|
new FredyRuntime(pro.config, jobConfig.notification, providerId, jobKey).execute();
|
||||||
|
setLastJobExecution(jobKey);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return exec;
|
||||||
|
})(),
|
||||||
|
config.interval * 60 * 1000
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|||||||
114
lib/FredyRuntime.js
Executable file
114
lib/FredyRuntime.js
Executable file
@@ -0,0 +1,114 @@
|
|||||||
|
const { NoNewListingsError } = require('./errors');
|
||||||
|
const {
|
||||||
|
setKnownListings,
|
||||||
|
getKnownListings,
|
||||||
|
setNumberOfTotalFoundProviderListings,
|
||||||
|
getForTesting
|
||||||
|
} = require('./services/store');
|
||||||
|
|
||||||
|
const notify = require('./notification/notify');
|
||||||
|
const xray = require('./services/scraper');
|
||||||
|
|
||||||
|
class FredyRuntime {
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param providerConfig the config for the specific provider, we're going to query at the moment
|
||||||
|
* @param notificationConfig the config for all notifications (because all could be applied to a provider)
|
||||||
|
* @param providerId the id of the provider currently in use
|
||||||
|
* @param jobKey key of the job that is currently running (from within the config)
|
||||||
|
*/
|
||||||
|
constructor(providerConfig, notificationConfig, providerId, jobKey) {
|
||||||
|
this._providerConfig = providerConfig;
|
||||||
|
this._notificationConfig = notificationConfig;
|
||||||
|
this._providerId = providerId;
|
||||||
|
this._jobKey = jobKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
execute() {
|
||||||
|
if (!this._providerConfig.enabled) return Promise.resolve();
|
||||||
|
|
||||||
|
return Promise.resolve(this._providerConfig.url)
|
||||||
|
.then(this._getListings.bind(this))
|
||||||
|
.then(this._normalize.bind(this))
|
||||||
|
.then(this._filter.bind(this))
|
||||||
|
.then(this._findNew.bind(this))
|
||||||
|
.then(this._storeStats.bind(this))
|
||||||
|
.then(this._save.bind(this))
|
||||||
|
.then(this._notify.bind(this))
|
||||||
|
.then(this._updateStates.bind(this))
|
||||||
|
.catch(this._handleError.bind(this));
|
||||||
|
}
|
||||||
|
|
||||||
|
_getListings(url) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
|
||||||
|
|
||||||
|
if (this._providerConfig.paginage) {
|
||||||
|
x = x.paginate(this._providerConfig.paginage);
|
||||||
|
}
|
||||||
|
|
||||||
|
x((err, listings) => {
|
||||||
|
if (err) reject(err);
|
||||||
|
else {
|
||||||
|
resolve(listings);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
_storeStats(listings) {
|
||||||
|
setNumberOfTotalFoundProviderListings(this._jobKey, this._providerId, listings.length);
|
||||||
|
return Promise.resolve(listings);
|
||||||
|
}
|
||||||
|
|
||||||
|
_normalize(listings) {
|
||||||
|
return listings.map(this._providerConfig.normalize);
|
||||||
|
}
|
||||||
|
|
||||||
|
_filter(listings) {
|
||||||
|
return listings.filter(this._providerConfig.filter);
|
||||||
|
}
|
||||||
|
|
||||||
|
_findNew(listings) {
|
||||||
|
const newListings = listings.filter(o => getKnownListings(this._jobKey, this._providerId).indexOf(o.id) === -1);
|
||||||
|
|
||||||
|
if (newListings.length === 0) {
|
||||||
|
this._updateStates([]);
|
||||||
|
throw new NoNewListingsError();
|
||||||
|
}
|
||||||
|
|
||||||
|
return newListings;
|
||||||
|
}
|
||||||
|
|
||||||
|
_notify(newListings) {
|
||||||
|
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig);
|
||||||
|
return Promise.all(sendNotifications).then(() => newListings);
|
||||||
|
}
|
||||||
|
|
||||||
|
_updateStates(newListings) {
|
||||||
|
return newListings;
|
||||||
|
}
|
||||||
|
|
||||||
|
_save(newListings) {
|
||||||
|
setKnownListings(this._jobKey, this._providerId, [
|
||||||
|
...getKnownListings(this._jobKey, this._providerId),
|
||||||
|
...newListings.map(l => l.id)
|
||||||
|
]);
|
||||||
|
return newListings;
|
||||||
|
}
|
||||||
|
|
||||||
|
_handleError(err) {
|
||||||
|
if (err.name !== 'NoNewListingsError') console.error(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* for testing purposes only
|
||||||
|
* @returns {Store}
|
||||||
|
* @private
|
||||||
|
*/
|
||||||
|
_getStore() {
|
||||||
|
return getForTesting();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = FredyRuntime;
|
||||||
62
lib/api/api.js
Normal file
62
lib/api/api.js
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
const bodyParser = require('body-parser');
|
||||||
|
const config = require('../../conf/config');
|
||||||
|
const { getLastJobExecution, getLastProviderExecution, getTotalNumberOfListings } = require('../services/store');
|
||||||
|
const PORT = 9988;
|
||||||
|
const service = require('restana')();
|
||||||
|
service.use(bodyParser.json());
|
||||||
|
|
||||||
|
service.get('/', async (req, res) => {
|
||||||
|
const result = {};
|
||||||
|
Object.keys(config.jobs).forEach(job => {
|
||||||
|
result[job] = {
|
||||||
|
lastExecution: getLastJobExecution(job),
|
||||||
|
enabledProvider: Object.keys(config.jobs[job].provider)
|
||||||
|
.filter(providerKey => config.jobs[job].provider[providerKey].enabled)
|
||||||
|
.map(providerKey => {
|
||||||
|
return {
|
||||||
|
name: providerKey,
|
||||||
|
lastExecution: getLastProviderExecution(job, providerKey),
|
||||||
|
totalFindings: getTotalNumberOfListings(job, providerKey)
|
||||||
|
};
|
||||||
|
})
|
||||||
|
};
|
||||||
|
});
|
||||||
|
res.body = result;
|
||||||
|
res.send();
|
||||||
|
});
|
||||||
|
|
||||||
|
service.get('/jobs/:name', async (req, res) => {
|
||||||
|
const { name: jobKey } = req.params;
|
||||||
|
if (Object.keys(config.jobs).indexOf(jobKey) === -1) {
|
||||||
|
console.error(`Cannot find job with name ${jobKey}. Available Jobs are [${Object.keys(config.jobs)}]`);
|
||||||
|
res.send(404);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.body = {
|
||||||
|
lastExecution: getLastJobExecution(jobKey),
|
||||||
|
enabledProvider: Object.keys(config.jobs[jobKey].provider)
|
||||||
|
.filter(providerKey => config.jobs[jobKey].provider[providerKey].enabled)
|
||||||
|
.map(providerKey => {
|
||||||
|
return {
|
||||||
|
name: providerKey,
|
||||||
|
url: config.jobs[jobKey].provider[providerKey].url,
|
||||||
|
lastExecution: getLastProviderExecution(jobKey, providerKey),
|
||||||
|
totalFindings: getTotalNumberOfListings(jobKey, providerKey)
|
||||||
|
};
|
||||||
|
})
|
||||||
|
};
|
||||||
|
res.send();
|
||||||
|
});
|
||||||
|
|
||||||
|
service.get('/ping', function(req, res) {
|
||||||
|
res.body = {
|
||||||
|
pong: 'pong'
|
||||||
|
};
|
||||||
|
res.send();
|
||||||
|
});
|
||||||
|
|
||||||
|
service.start(PORT).then(() => {
|
||||||
|
/* eslint-disable no-console */
|
||||||
|
console.info(`Started API service on port ${PORT}`);
|
||||||
|
/* eslint-enable no-console */
|
||||||
|
});
|
||||||
@@ -1,16 +1,15 @@
|
|||||||
class ExtendableError extends Error {
|
class ExtendableError extends Error {
|
||||||
constructor(message) {
|
constructor(message) {
|
||||||
super(message);
|
super(message);
|
||||||
this.name = this.constructor.name;
|
this.name = this.constructor.name;
|
||||||
if (typeof Error.captureStackTrace === 'function') {
|
if (typeof Error.captureStackTrace === 'function') {
|
||||||
Error.captureStackTrace(this, this.constructor)
|
Error.captureStackTrace(this, this.constructor);
|
||||||
} else {
|
} else {
|
||||||
this.stack = (new Error(message)).stack
|
this.stack = new Error(message).stack;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class NoNewListingsError extends ExtendableError {
|
class NoNewListingsError extends ExtendableError {}
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {NoNewListingsError};
|
module.exports = { NoNewListingsError };
|
||||||
|
|||||||
106
lib/fredy.js
106
lib/fredy.js
@@ -1,106 +0,0 @@
|
|||||||
const {NoNewListingsError} = require('./errors');
|
|
||||||
const Store = require('./services/store');
|
|
||||||
|
|
||||||
const notify = require('./notification/notify');
|
|
||||||
const xray = require('./services/scraper');
|
|
||||||
|
|
||||||
class Fredy {
|
|
||||||
constructor(source) {
|
|
||||||
this._store = new Store(source.name);
|
|
||||||
this._fullCrawl = true;
|
|
||||||
this._source = source;
|
|
||||||
this._stats = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
run(stats) {
|
|
||||||
|
|
||||||
if(!this._stats){
|
|
||||||
this._stats = stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!this._source.enabled) return Promise.resolve();
|
|
||||||
|
|
||||||
return Promise.resolve(this._source.url)
|
|
||||||
.then(this._store.warmup)
|
|
||||||
.then(this._getListings.bind(this))
|
|
||||||
.then(this._normalize.bind(this))
|
|
||||||
.then(this._filter.bind(this))
|
|
||||||
.then(this._findNew.bind(this))
|
|
||||||
.then(this._save.bind(this))
|
|
||||||
.then(this._notify.bind(this))
|
|
||||||
.then(this._updateStates.bind(this))
|
|
||||||
.catch(this._handleError.bind(this))
|
|
||||||
}
|
|
||||||
|
|
||||||
_getListings(url) {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
let x = xray(url, this._source.crawlContainer, [this._source.crawlFields]);
|
|
||||||
|
|
||||||
if (this._source.paginage && this._fullCrawl) {
|
|
||||||
this._fullCrawl = false;
|
|
||||||
x = x.paginate(this._source.paginage)
|
|
||||||
}
|
|
||||||
|
|
||||||
x((err, listings) => {
|
|
||||||
if (err) reject(err);
|
|
||||||
else {
|
|
||||||
resolve(listings);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
_normalize(listings) {
|
|
||||||
return listings.map(this._source.normalize)
|
|
||||||
}
|
|
||||||
|
|
||||||
_filter(listings) {
|
|
||||||
return listings.filter(this._source.filter)
|
|
||||||
}
|
|
||||||
|
|
||||||
_findNew(listings) {
|
|
||||||
const newListings = listings.filter(
|
|
||||||
o => this._store.knownListings.indexOf(o.id) === -1
|
|
||||||
);
|
|
||||||
|
|
||||||
if (newListings.length === 0) {
|
|
||||||
this._updateStates([]);
|
|
||||||
throw new NoNewListingsError();
|
|
||||||
}
|
|
||||||
|
|
||||||
return newListings
|
|
||||||
}
|
|
||||||
|
|
||||||
_notify(newListings) {
|
|
||||||
const sendNotifications = notify.send(this._source.name, newListings);
|
|
||||||
return Promise.all(sendNotifications).then(() => newListings)
|
|
||||||
}
|
|
||||||
|
|
||||||
_updateStates(newListings){
|
|
||||||
this._stats.setLastScrape(this._source.name, newListings.length);
|
|
||||||
return newListings;
|
|
||||||
}
|
|
||||||
|
|
||||||
_save(newListings) {
|
|
||||||
this._store.knownListings = [
|
|
||||||
...this._store.knownListings,
|
|
||||||
...newListings.map(l => l.id)
|
|
||||||
];
|
|
||||||
return newListings;
|
|
||||||
}
|
|
||||||
|
|
||||||
_handleError(err) {
|
|
||||||
if (err.name !== 'NoNewListingsError') console.error(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* for testing purposes only
|
|
||||||
* @returns {Store}
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
_getStore(){
|
|
||||||
return this._store;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = Fredy;
|
|
||||||
@@ -1,17 +1,15 @@
|
|||||||
const config = require('../../../conf/config.json');
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* simply prints out the found data to the console
|
* simply prints out the found data to the console
|
||||||
* @param serviceName e.g immoscout
|
* @param serviceName e.g immoscout
|
||||||
* @param newListings an array with newly found listings
|
* @param newListings an array with newly found listings
|
||||||
|
* @param notificationConfig config of this notification adapter
|
||||||
*/
|
*/
|
||||||
exports.send = (serviceName, newListings) => {
|
exports.send = (serviceName, newListings, notificationConfig) => {
|
||||||
return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
|
const { enabled } = notificationConfig.console;
|
||||||
};
|
if (!enabled) {
|
||||||
|
return [Promise.resolve()];
|
||||||
/**
|
}
|
||||||
* each integration needs to implement this method
|
/* eslint-disable no-console */
|
||||||
*/
|
return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
|
||||||
exports.enabled = () => {
|
/* eslint-enable no-console */
|
||||||
return config.notification.console.enabled;
|
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,54 +1,50 @@
|
|||||||
const Slack = require('slack');
|
const Slack = require('slack');
|
||||||
const config = require('../../../conf/config.json');
|
|
||||||
const msg = Slack.chat.postMessage;
|
const msg = Slack.chat.postMessage;
|
||||||
|
|
||||||
const {token, channel} = config.notification.slack;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* sends a new listing to slack
|
* sends a new listing to slack
|
||||||
* @param serviceName e.g immoscout
|
* @param serviceName e.g immoscout
|
||||||
* @param newListings an array with newly found listings
|
* @param newListings an array with newly found listings
|
||||||
|
* @param notificationConfig config of this notification adapter
|
||||||
* @returns {Promise<Chat.PostMessage.Response> | void}
|
* @returns {Promise<Chat.PostMessage.Response> | void}
|
||||||
*/
|
*/
|
||||||
exports.send = (serviceName, newListings) => {
|
exports.send = (serviceName, newListings, notificationConfig) => {
|
||||||
return newListings.map(payload => msg({
|
const { token, channel, enabled } = notificationConfig.slack;
|
||||||
token,
|
if (!enabled) {
|
||||||
channel,
|
return [Promise.resolve()];
|
||||||
text: `*(${serviceName})* - ${payload.title}`,
|
}
|
||||||
"attachments": [
|
return newListings.map(payload =>
|
||||||
{
|
msg({
|
||||||
"fallback": payload.title,
|
token,
|
||||||
"color": "#36a64f",
|
channel,
|
||||||
"title": "Link to Exposé",
|
text: `*(${serviceName})* - ${payload.title}`,
|
||||||
"title_link": payload.link,
|
attachments: [
|
||||||
"fields": [
|
{
|
||||||
{
|
fallback: payload.title,
|
||||||
"title": "Price",
|
color: '#36a64f',
|
||||||
"value": payload.price,
|
title: 'Link to Exposé',
|
||||||
"short": false
|
title_link: payload.link,
|
||||||
},
|
fields: [
|
||||||
{
|
{
|
||||||
"title": "Size",
|
title: 'Price',
|
||||||
"value": payload.size,
|
value: payload.price,
|
||||||
"short": false
|
short: false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"title": "Address",
|
title: 'Size',
|
||||||
"value": payload.address,
|
value: payload.size,
|
||||||
"short": false
|
short: false
|
||||||
}
|
},
|
||||||
],
|
{
|
||||||
"footer": "Powered by Fredy",
|
title: 'Address',
|
||||||
ts: new Date().getTime() / 1000
|
value: payload.address,
|
||||||
}
|
short: false
|
||||||
]
|
}
|
||||||
})
|
],
|
||||||
);
|
footer: 'Powered by Fredy',
|
||||||
};
|
ts: new Date().getTime() / 1000
|
||||||
|
}
|
||||||
/**
|
]
|
||||||
* each integration needs to implement this method
|
})
|
||||||
*/
|
);
|
||||||
exports.enabled = () => {
|
|
||||||
return config.notification.slack.enabled;
|
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,35 +1,33 @@
|
|||||||
const config = require('../../../conf/config.json');
|
|
||||||
const TelegramBot = require('tg-yarl');
|
const TelegramBot = require('tg-yarl');
|
||||||
|
|
||||||
const opts = { parse_mode: 'Markdown' };
|
const opts = {parse_mode: 'Markdown'};
|
||||||
const bot = new TelegramBot(config.notification.telegram.token);
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* sends new listings to telegram
|
* sends new listings to telegram
|
||||||
* @param serviceName e.g immoscout
|
* @param serviceName e.g immoscout
|
||||||
* @param newListings an array with newly found listings
|
* @param newListings an array with newly found listings
|
||||||
|
* @param notificationConfig config of this notification adapter
|
||||||
* @returns {Promise<Void> | void}
|
* @returns {Promise<Void> | void}
|
||||||
*/
|
*/
|
||||||
exports.send = (serviceName, newListings) => {
|
exports.send = (serviceName, newListings, notificationConfig) => {
|
||||||
|
const {enabled, token, chatId} = notificationConfig.telegram;
|
||||||
|
if (!enabled) {
|
||||||
|
return [Promise.resolve()];
|
||||||
|
}
|
||||||
|
|
||||||
|
const bot = new TelegramBot(token);
|
||||||
|
|
||||||
let message = `Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
|
let message = `Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
|
||||||
|
|
||||||
message += newListings.map(o =>
|
message += newListings.map(o =>
|
||||||
`*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
|
`*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
|
||||||
[o.address, o.price, o.size].join(' | ') + '\n' +
|
[o.address, o.price, o.size].join(' | ') + '\n' +
|
||||||
`[LINK](${o.link})\n\n`);
|
`[LINK](${o.link})\n\n`);
|
||||||
|
|
||||||
return bot.sendMessage(config.notification.telegram.chatId, message, opts);
|
return bot.sendMessage(chatId, message, opts);
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
function shorten(str, len = 30) {
|
||||||
* each integration needs to implement this method
|
|
||||||
*/
|
|
||||||
exports.enabled = () => {
|
|
||||||
return config.notification.telegram.enabled;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
function shorten (str, len = 30) {
|
|
||||||
return str.length > len ? str.substring(0, len) + '...' : str;
|
return str.length > len ? str.substring(0, len) + '...' : str;
|
||||||
}
|
}
|
||||||
@@ -4,14 +4,13 @@ const path = './adapter';
|
|||||||
/** Read every integration existing in ./adapter **/
|
/** Read every integration existing in ./adapter **/
|
||||||
const adapter = fs
|
const adapter = fs
|
||||||
.readdirSync('./lib/notification/adapter')
|
.readdirSync('./lib/notification/adapter')
|
||||||
.map(integPath => require(`${path}/${integPath}`))
|
.map(integPath => require(`${path}/${integPath}`));
|
||||||
.filter(integration => integration.enabled());
|
|
||||||
|
|
||||||
if (adapter.length === 0) {
|
if (adapter.length === 0) {
|
||||||
throw new Error('Please specify at least one notification provider');
|
throw new Error('Please specify at least one notification provider');
|
||||||
}
|
}
|
||||||
|
|
||||||
exports.send = (serviceName, payload) => {
|
exports.send = (serviceName, payload, notificationConfig) => {
|
||||||
//this is not being used in tests, therefor adapter are always set
|
//this is not being used in tests, therefor adapter are always set
|
||||||
return adapter.map(a => a.send(serviceName, payload));
|
return adapter.map(a => a.send(serviceName, payload, notificationConfig));
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
const Fredy = require('../fredy');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
|
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
|
||||||
if(o.rooms != null){
|
if (o.rooms != null) {
|
||||||
size+=` / / ${o.rooms.trim()}`;
|
size += ` / / ${o.rooms.trim()}`;
|
||||||
}
|
}
|
||||||
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
|
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
|
||||||
|
|
||||||
@@ -13,16 +13,15 @@ function normalize(o) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
|
|
||||||
return titleNotBlacklisted && descNotBlacklisted;
|
return titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const einsAImmobilien = {
|
const config = {
|
||||||
name: 'einsAImmobilien',
|
enabled: null,
|
||||||
enabled: config.sources.einsAImmobilien.enabled,
|
url: null,
|
||||||
url: config.sources.einsAImmobilien.url,
|
|
||||||
crawlContainer: '.tabelle',
|
crawlContainer: '.tabelle',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
|
id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
|
||||||
@@ -37,4 +36,13 @@ const einsAImmobilien = {
|
|||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(einsAImmobilien);
|
exports.init = (sourceConfig, blacklist) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'einsAImmobilien';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
const Fredy = require('../fredy');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
|
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
|
||||||
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
|
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
|
||||||
@@ -13,16 +13,15 @@ function normalize(o) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
|
|
||||||
return titleNotBlacklisted && descNotBlacklisted;
|
return titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const immonet = {
|
const config = {
|
||||||
name: 'immonet',
|
enabled: null,
|
||||||
enabled: config.sources.immonet.enabled,
|
url: null,
|
||||||
url: config.sources.immonet.url,
|
|
||||||
crawlContainer: '#result-list-stage .item',
|
crawlContainer: '#result-list-stage .item',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '@id',
|
id: '@id',
|
||||||
@@ -37,4 +36,13 @@ const immonet = {
|
|||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(immonet);
|
exports.init = (sourceConfig, blacklist) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'immonet';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
const Fredy = require('../fredy');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const title = o.title.replace('NEU', '');
|
const title = o.title.replace('NEU', '');
|
||||||
const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
||||||
@@ -10,25 +10,33 @@ function normalize(o) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
return !utils.isOneOf(o.title, config.blacklist);
|
return !utils.isOneOf(o.title, appliedBlackList);
|
||||||
}
|
}
|
||||||
|
|
||||||
const immoscout = {
|
const config = {
|
||||||
name: 'immoscout',
|
enabled: null,
|
||||||
enabled: config.sources.immoscout.enabled,
|
url: null,
|
||||||
url: config.sources.immoscout.url,
|
|
||||||
crawlContainer: '#resultListItems li.result-list__listing',
|
crawlContainer: '#resultListItems li.result-list__listing',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '.result-list-entry@data-obid | int',
|
id: '.result-list-entry@data-obid | int',
|
||||||
price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
|
price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
|
||||||
size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
|
size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
|
||||||
title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
|
title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
|
||||||
link: '.result-list-entry .result-list-entry__brand-title-container@href',
|
link: '.result-list-entry .result-list-entry__brand-title-container@href',
|
||||||
address: '.result-list-entry .result-list-entry__map-link'
|
address: '.result-list-entry .result-list-entry__map-link'
|
||||||
},
|
},
|
||||||
paginate: '#pager .align-right a@href',
|
paginate: '#pager .align-right a@href',
|
||||||
normalize: normalize,
|
normalize: normalize,
|
||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(immoscout);
|
exports.init = (sourceConfig, blacklist) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'immoscout';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
const Fredy = require('../fredy');
|
|
||||||
const config = require('../../conf/config.json');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
|
const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
|
||||||
const address = o.address;
|
const address = o.address;
|
||||||
@@ -10,16 +10,15 @@ function normalize(o) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
|
|
||||||
return titleNotBlacklisted && descNotBlacklisted;
|
return titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const immowelt = {
|
const config = {
|
||||||
name: 'immowelt',
|
enabled: null,
|
||||||
enabled: config.sources.immowelt.enabled,
|
url: null,
|
||||||
url: config.sources.immowelt.url,
|
|
||||||
crawlContainer: '.immoliste .js-object.listitem_wrap ',
|
crawlContainer: '.immoliste .js-object.listitem_wrap ',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '@data-estateid | int',
|
id: '@data-estateid | int',
|
||||||
@@ -34,4 +33,13 @@ const immowelt = {
|
|||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(immowelt);
|
exports.init = (sourceConfig, blacklist) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'immowelt';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
const Fredy = require('../fredy');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
let appliedBlacklistedDistricts = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const id = o.id
|
const id = o.id
|
||||||
.split('/')
|
.split('/')
|
||||||
@@ -16,19 +17,18 @@ function normalize(o) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
|
|
||||||
const isBlacklistedDistrict =
|
const isBlacklistedDistrict =
|
||||||
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.title, config.blacklistedDistrics);
|
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
|
||||||
|
|
||||||
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const kalaydo = {
|
const config = {
|
||||||
name: 'kalaydo',
|
enabled: null,
|
||||||
enabled: config.sources.kalaydo.enabled,
|
url: null,
|
||||||
url: config.sources.kalaydo.url,
|
|
||||||
crawlContainer: '#resultList .resultitem-content-container',
|
crawlContainer: '#resultList .resultitem-content-container',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '.resultitem-content-container a@href',
|
id: '.resultitem-content-container a@href',
|
||||||
@@ -43,4 +43,15 @@ const kalaydo = {
|
|||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(kalaydo);
|
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
appliedBlacklistedDistricts = blacklistedDistricts;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'kalaydo';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
const Fredy = require('../fredy');
|
|
||||||
const config = require('../../conf/config.json');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
let appliedBlacklistedDistricts = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
const size = o.size || '--- m²';
|
const size = o.size || '--- m²';
|
||||||
|
|
||||||
@@ -9,18 +10,17 @@ function normalize(o) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
const isBlacklistedDistrict =
|
const isBlacklistedDistrict =
|
||||||
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
|
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
|
||||||
|
|
||||||
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const kleinanzeigen = {
|
const config = {
|
||||||
name: 'kleinanzeigen',
|
enabled: null,
|
||||||
enabled: config.sources.kleinanzeigen.enabled,
|
url: null,
|
||||||
url: config.sources.kleinanzeigen.url,
|
|
||||||
crawlContainer: '#srchrslt-adtable .ad-listitem',
|
crawlContainer: '#srchrslt-adtable .ad-listitem',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '.aditem@data-adid | int',
|
id: '.aditem@data-adid | int',
|
||||||
@@ -36,4 +36,15 @@ const kleinanzeigen = {
|
|||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(kleinanzeigen);
|
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlacklistedDistricts = blacklistedDistricts;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'kleinanzeigen';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,18 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
const Fredy = require('../fredy');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
return o;
|
return o;
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
return !utils.isOneOf(o.title, config.blacklist);
|
return !utils.isOneOf(o.title, appliedBlackList);
|
||||||
}
|
}
|
||||||
|
|
||||||
const neubauKompass = {
|
const config = {
|
||||||
name: 'neubauKompass',
|
enabled: null,
|
||||||
enabled: config.sources.neubauKompass.enabled,
|
url: null,
|
||||||
url: config.sources.neubauKompass.url,
|
|
||||||
crawlContainer: '.row article',
|
crawlContainer: '.row article',
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '@id',
|
id: '@id',
|
||||||
@@ -26,4 +25,13 @@ const neubauKompass = {
|
|||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(neubauKompass);
|
exports.init = (sourceConfig, blacklist) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'neubauKompass';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|||||||
@@ -1,34 +1,42 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
const Fredy = require('../fredy');
|
|
||||||
const utils = require('../utils');
|
const utils = require('../utils');
|
||||||
|
|
||||||
|
let appliedBlackList = [];
|
||||||
|
|
||||||
function normalize(o) {
|
function normalize(o) {
|
||||||
return o;
|
return o;
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyBlacklist(o) {
|
function applyBlacklist(o) {
|
||||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||||
|
|
||||||
|
|
||||||
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
|
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
|
||||||
}
|
}
|
||||||
|
|
||||||
const wgGesucht = {
|
const config = {
|
||||||
name: 'wgGesucht',
|
enabled: null,
|
||||||
enabled: config.sources.wgGesucht.enabled,
|
url: null,
|
||||||
url: config.sources.wgGesucht.url,
|
crawlContainer: '#main_column .wgg_card',
|
||||||
crawlContainer: '#main_column .panel:not(.display-none):not(.noprint)',
|
|
||||||
crawlFields: {
|
crawlFields: {
|
||||||
id: '@data-id',
|
id: '@data-id',
|
||||||
details: ' .list-details-costs-col |removeNewline |trim',
|
details: '.row .noprint .col-xs-11 |removeNewline |trim',
|
||||||
title: '.headline .detailansicht |removeNewline |trim',
|
price: '.middle .col-xs-3 |removeNewline |trim',
|
||||||
description: '.list-details-category-location |removeNewline |trim',
|
size: '.middle .text-right |removeNewline |trim',
|
||||||
link: '.headline .detailansicht@href'
|
title: '.truncate_title a |removeNewline |trim',
|
||||||
|
link: '.truncate_title a@href'
|
||||||
},
|
},
|
||||||
paginate: '.pagination-sm:first a:last@href',
|
paginate: '.pagination-sm:first a:last@href',
|
||||||
normalize: normalize,
|
normalize: normalize,
|
||||||
filter: applyBlacklist
|
filter: applyBlacklist
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = new Fredy(wgGesucht);
|
exports.init = (sourceConfig, blacklist) => {
|
||||||
|
config.enabled = sourceConfig.enabled;
|
||||||
|
config.url = sourceConfig.url;
|
||||||
|
appliedBlackList = blacklist;
|
||||||
|
};
|
||||||
|
|
||||||
|
//must match the id of the source given in the config!
|
||||||
|
exports.id = () => 'wgGesucht';
|
||||||
|
|
||||||
|
exports.config = config;
|
||||||
|
|||||||
@@ -1,35 +0,0 @@
|
|||||||
const config = require('../../conf/config.json');
|
|
||||||
let stats = {
|
|
||||||
lastScrape: {},
|
|
||||||
foundScrapes: {}
|
|
||||||
};
|
|
||||||
|
|
||||||
if (config.enableStats) {
|
|
||||||
const http = require('http');
|
|
||||||
http
|
|
||||||
.createServer((req, res) => {
|
|
||||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
||||||
res.end(
|
|
||||||
JSON.stringify({
|
|
||||||
config,
|
|
||||||
stats
|
|
||||||
})
|
|
||||||
);
|
|
||||||
})
|
|
||||||
.listen(config.statsPort, '127.0.0.1');
|
|
||||||
}
|
|
||||||
|
|
||||||
const datetime = date => {
|
|
||||||
return `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()} ${date.getHours()}:${date.getMinutes()}`;
|
|
||||||
};
|
|
||||||
|
|
||||||
exports.setLastScrape = (serviceName, numberOfNewListsings) => {
|
|
||||||
const d = new Date();
|
|
||||||
const dt = datetime(d);
|
|
||||||
stats.lastScrape[serviceName] = d.toString();
|
|
||||||
|
|
||||||
if (numberOfNewListsings > 0) {
|
|
||||||
stats.foundScrapes[dt] = stats.foundScrapes[dt] || {};
|
|
||||||
stats.foundScrapes[dt][serviceName] = numberOfNewListsings;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -7,30 +7,79 @@ const low = require('lowdb');
|
|||||||
|
|
||||||
const lowdb = low(adapter);
|
const lowdb = low(adapter);
|
||||||
|
|
||||||
class Store {
|
let db = null;
|
||||||
constructor(name) {
|
|
||||||
this._name = name;
|
|
||||||
this._db = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
get warmup() {
|
const buildKey = (jobKey, providerId, endpoint) => {
|
||||||
return new Promise(resolve => {
|
let key = `${jobKey}`;
|
||||||
lowdb.then(db => {
|
if (jobKey == null && endpoint == null) {
|
||||||
this._db = db;
|
return key;
|
||||||
resolve();
|
}
|
||||||
});
|
if (providerId != null) {
|
||||||
|
key += `.${providerId}`;
|
||||||
|
}
|
||||||
|
if (endpoint != null) {
|
||||||
|
key += `.${endpoint}`;
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.init = () => {
|
||||||
|
return new Promise(resolve => {
|
||||||
|
//warmup
|
||||||
|
lowdb.then(database => {
|
||||||
|
db = database;
|
||||||
|
/* eslint-disable no-console */
|
||||||
|
console.info('Warming up database successful');
|
||||||
|
/* eslint-enable no-console */
|
||||||
|
resolve();
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||||
|
if (!Array.isArray(listings)) throw Error('Not a valid array');
|
||||||
|
const providerListingsKey = buildKey(jobKey, providerId, 'listings');
|
||||||
|
const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
|
||||||
|
|
||||||
|
return db
|
||||||
|
.set(providerListingsKey, listings)
|
||||||
|
.set(providerLastScrapeKey, Date.now())
|
||||||
|
.write();
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
|
||||||
|
if (numberOfNewListings > 0) {
|
||||||
|
const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
|
||||||
|
const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
|
||||||
|
db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
set knownListings(value) {
|
exports.setLastJobExecution = jobKey => {
|
||||||
if (!Array.isArray(value)) throw Error('Not a valid array');
|
const key = buildKey(jobKey, null, 'lastJobExecution');
|
||||||
|
return db.set(key, Date.now()).write();
|
||||||
|
};
|
||||||
|
|
||||||
return this._db.set(this._name, value).write();
|
exports.getKnownListings = (jobKey, providerId) => {
|
||||||
}
|
const providerListingsKey = buildKey(jobKey, providerId, 'listings');
|
||||||
|
return db.get(providerListingsKey).value() || [];
|
||||||
|
};
|
||||||
|
|
||||||
get knownListings() {
|
exports.getLastProviderExecution = (jobKey, providerId) => {
|
||||||
return this._db.get(this._name).value() || [];
|
const key = buildKey(jobKey, providerId, 'lastProviderExecution');
|
||||||
}
|
return db.get(key).value() || 0;
|
||||||
}
|
};
|
||||||
|
|
||||||
module.exports = Store;
|
exports.getLastJobExecution = jobKey => {
|
||||||
|
const key = buildKey(jobKey, null, 'lastJobExecution');
|
||||||
|
return db.get(key).value() || 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.getTotalNumberOfListings = (jobKey, providerId) => {
|
||||||
|
const key = buildKey(jobKey, providerId, 'foundListings');
|
||||||
|
return db.get(key).value() || 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.getForTesting = () => {
|
||||||
|
return db;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
function isOneOf (word, arr) {
|
function isOneOf(word, arr) {
|
||||||
|
if (arr == null || arr.length === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
const expression = String.raw`\b(${arr.join('|')})\b`;
|
const expression = String.raw`\b(${arr.join('|')})\b`;
|
||||||
const blacklist = new RegExp(expression, 'ig');
|
const blacklist = new RegExp(expression, 'ig');
|
||||||
|
|
||||||
return blacklist.test(word)
|
return blacklist.test(word);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { isOneOf };
|
module.exports = { isOneOf };
|
||||||
|
|||||||
33
package.json
33
package.json
@@ -1,11 +1,22 @@
|
|||||||
{
|
{
|
||||||
"name": "Fredy",
|
"name": "fredy",
|
||||||
"version": "1.3.0",
|
"version": "2.0.0",
|
||||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node index.js",
|
"start": "node index.js",
|
||||||
"format": "prettier --write lib/**/*.js test/**/*.js *.js --single-quote --print-width 120",
|
"format": "prettier --write lib/**/*.js test/**/*.js *.js --single-quote --print-width 120",
|
||||||
"test": "mocha --timeout 12000"
|
"test": "mocha --timeout 15000 test/**/*.test.js"
|
||||||
|
},
|
||||||
|
"husky": {
|
||||||
|
"hooks": {
|
||||||
|
"pre-commit": "lint-staged"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"lint-staged": {
|
||||||
|
"*.js": [
|
||||||
|
"eslint ./index.js ./lib/**/*.js ./test/**/*.js",
|
||||||
|
"prettier --single-quote --print-width 120 --write"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"author": "Christian Kellner",
|
"author": "Christian Kellner",
|
||||||
@@ -27,17 +38,27 @@
|
|||||||
"url": "https://github.com/orangecoding/fredy/issues"
|
"url": "https://github.com/orangecoding/fredy/issues"
|
||||||
},
|
},
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=11.0.0",
|
||||||
|
"npm": ">=6.0.0"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"body-parser": "1.19.0",
|
||||||
"lowdb": "1.0.0",
|
"lowdb": "1.0.0",
|
||||||
"request-x-ray": "0.1.4",
|
"request-x-ray": "0.1.4",
|
||||||
|
"restana": "4.0.8",
|
||||||
"slack": "11.0.2",
|
"slack": "11.0.2",
|
||||||
"tg-yarl": "1.3.0",
|
"tg-yarl": "1.3.0",
|
||||||
"x-ray": "2.3.4"
|
"x-ray": "2.3.4"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"mocha": "7.0.1",
|
"chai": "4.2.0",
|
||||||
|
"eslint": "6.8.0",
|
||||||
|
"eslint-config-prettier": "6.10.0",
|
||||||
|
"husky": "4.2.3",
|
||||||
|
"lint-staged": "10.0.8",
|
||||||
|
"mocha": "7.1.0",
|
||||||
"prettier": "1.19.1",
|
"prettier": "1.19.1",
|
||||||
"proxyquire": "1.8.0",
|
"proxyquire": "2.1.3"
|
||||||
"chai": "4.2.0"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,49 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#einsAImmobilien testsuite()', () => {
|
|
||||||
|
|
||||||
const einsAImmobilien = proxyquire('../lib/provider/einsAImmobilien', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should test einsAImmobilien provider', async () => {
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
einsAImmobilien.run(mockStats).then(() => {
|
|
||||||
const immonetDbContent = einsAImmobilien._getStore()._db;
|
|
||||||
|
|
||||||
expect(immonetDbContent.einsAImmobilien).to.be.a('array');
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj).to.be.a('object');
|
|
||||||
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('number');
|
|
||||||
expect(notify.price).to.be.a('string');
|
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
|
|
||||||
expect(notify.price).that.does.include('EUR');
|
|
||||||
expect(notify.size).to.be.not.empty;
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#immonet testsuite()', () => {
|
|
||||||
|
|
||||||
const immonet = proxyquire('../lib/provider/immonet', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should test immonet provider', async () => {
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
immonet.run(mockStats).then(() => {
|
|
||||||
const immonetDbContent = immonet._getStore()._db;
|
|
||||||
|
|
||||||
expect(immonetDbContent.immonet).to.be.a('array');
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj).to.be.a('object');
|
|
||||||
expect(notificationObj.serviceName).to.equal('immonet');
|
|
||||||
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('number');
|
|
||||||
expect(notify.price).to.be.a('string');
|
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
expect(notify.address).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
|
|
||||||
expect(notify.price).that.does.include('€');
|
|
||||||
expect(notify.size).that.does.include('m²');
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.immonet.de');
|
|
||||||
expect(notify.address).to.be.not.empty;
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#immoscout testsuite()', () => {
|
|
||||||
|
|
||||||
const immoscout = proxyquire('../lib/provider/immoscout', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should test immoscout provider', async () => {
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
immoscout.run(mockStats).then(() => {
|
|
||||||
const immoscoutDbContent = immoscout._getStore()._db;
|
|
||||||
expect(immoscoutDbContent.immoscout).to.be.a('array');
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj).to.be.a('object');
|
|
||||||
expect(notificationObj.serviceName).to.equal('immoscout');
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('number');
|
|
||||||
expect(notify.price).to.be.a('string');
|
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
expect(notify.address).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(
|
|
||||||
immoscoutDbContent.immoscout[idx]
|
|
||||||
);
|
|
||||||
expect(notify.price).that.does.include('€');
|
|
||||||
expect(notify.size).that.does.include('m²');
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
|
|
||||||
expect(notify.address).to.be.not.empty;
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#immowelt testsuite()', () => {
|
|
||||||
|
|
||||||
it('should test immowelt provider', async () => {
|
|
||||||
|
|
||||||
const immowelt = proxyquire('../lib/provider/immowelt', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
immowelt.run(mockStats).then(() => {
|
|
||||||
const immoweltDbContent = immowelt._getStore()._db;
|
|
||||||
expect(immoweltDbContent.immowelt).to.be.a('array');
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj).to.be.a('object');
|
|
||||||
expect(notificationObj.serviceName).to.equal('immowelt');
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('number');
|
|
||||||
expect(notify.price).to.be.a('string');
|
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
expect(notify.address).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(
|
|
||||||
immoweltDbContent.immowelt[idx]
|
|
||||||
);
|
|
||||||
expect(notify.price).that.does.include('€');
|
|
||||||
if(notify.size.trim().toLowerCase() !== 'k.a.') {
|
|
||||||
expect(notify.size).that.does.include('m²');
|
|
||||||
}
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.immowelt.de');
|
|
||||||
expect(notify.address).to.be.not.empty;
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#kalaydo testsuite()', () => {
|
|
||||||
|
|
||||||
const kalaydo = proxyquire('../lib/provider/kalaydo', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should test kalaydo provider', async () => {
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
kalaydo.run(mockStats).then(() => {
|
|
||||||
const kalaydoDbContent = kalaydo._getStore()._db;
|
|
||||||
|
|
||||||
expect(kalaydoDbContent.kalaydo).to.be.a('array');
|
|
||||||
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj).to.be.a('object');
|
|
||||||
expect(notificationObj.serviceName).to.equal('kalaydo');
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('string');
|
|
||||||
expect(notify.price).to.be.a('string');
|
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.kalaydo.de');
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#kleinanzeigen testsuite()', () => {
|
|
||||||
|
|
||||||
it('should test kleinanzeigen provider', async () => {
|
|
||||||
|
|
||||||
const kleinanzeigen = proxyquire('../lib/provider/kleinanzeigen', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
kleinanzeigen.run(mockStats).then(() => {
|
|
||||||
const kleinanzeigenDbContent = kleinanzeigen._getStore()._db;
|
|
||||||
expect(kleinanzeigenDbContent.kleinanzeigen).to.be.a('array');
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj).to.be.a('object');
|
|
||||||
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('number');
|
|
||||||
expect(notify.price).to.be.a('string');
|
|
||||||
expect(notify.size).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
expect(notify.address).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(
|
|
||||||
kleinanzeigenDbContent.kleinanzeigen[idx]
|
|
||||||
);
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
|
|
||||||
expect(notify.address).to.be.not.empty;
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
setLastScrape: () => {
|
|
||||||
/*noop*/
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -1,27 +1,52 @@
|
|||||||
|
const db = {};
|
||||||
|
|
||||||
|
exports.init = () => {
|
||||||
|
return new Promise(resolve => {
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||||
|
if (!Array.isArray(listings)) throw Error('Not a valid array');
|
||||||
|
|
||||||
|
db[providerId] = listings;
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.getKnownListings = (jobKey, providerId) => {
|
||||||
|
return db[providerId] || [];
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.setNumberOfTotalFoundProviderListings = () => {
|
||||||
|
/*noop*/
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.getForTesting = () => {
|
||||||
|
return db;
|
||||||
|
};
|
||||||
|
/*
|
||||||
class Store {
|
class Store {
|
||||||
constructor(name) {
|
constructor(name) {
|
||||||
this._name = name;
|
this._name = name;
|
||||||
this._db = {};
|
this._db = {};
|
||||||
}
|
}
|
||||||
|
|
||||||
get warmup() {
|
get warmup() {
|
||||||
this._db = {};
|
this._db = {};
|
||||||
return new Promise(resolve => resolve());
|
return new Promise(resolve => resolve());
|
||||||
}
|
}
|
||||||
|
|
||||||
set knownListings(value) {
|
set knownListings(value) {
|
||||||
if (!Array.isArray(value)) throw Error('Not a valid array');
|
if (!Array.isArray(value)) throw Error('Not a valid array');
|
||||||
return new Promise(resolve => {
|
return new Promise(resolve => {
|
||||||
this._db[this._name] = value;
|
this._db[this._name] = value;
|
||||||
resolve(value);
|
resolve(value);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
get bla() {}
|
get knownListings() {
|
||||||
|
return this._db[this._name] || [];
|
||||||
get knownListings() {
|
}
|
||||||
return this._db[this._name] || [];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = Store;
|
module.exports = Store;
|
||||||
|
*/
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#neubauKompass testsuite()', () => {
|
|
||||||
|
|
||||||
const neubauKompass = proxyquire('../lib/provider/neubauKompass', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should test neubauKompass provider', async () => {
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
neubauKompass.run(mockStats).then(() => {
|
|
||||||
const neubauKompassDbContent = neubauKompass._getStore()._db;
|
|
||||||
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
|
|
||||||
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj.serviceName).to.equal('neubauKompass');
|
|
||||||
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
expect(notify).to.be.a('object');
|
|
||||||
|
|
||||||
/** check the actual structure **/
|
|
||||||
expect(notify.id).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
expect(notify.address).to.be.a('string');
|
|
||||||
|
|
||||||
/** check the values if possible **/
|
|
||||||
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
|
|
||||||
expect(notify.title).to.be.not.empty;
|
|
||||||
expect(notify.link).that.does.include('https://www.neubaukompass.de');
|
|
||||||
expect(notify.address).to.be.not.empty;
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
45
test/provider/einsAImmobilien.test.js
Normal file
45
test/provider/einsAImmobilien.test.js
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/einsAImmobilien');
|
||||||
|
|
||||||
|
describe('#einsAImmobilien testsuite()', () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.einsAImmobilien, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should test einsAImmobilien provider', async () => {
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const immonetDbContent = fredy._getStore();
|
||||||
|
expect(immonetDbContent.einsAImmobilien).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj).to.be.a('object');
|
||||||
|
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('number');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
|
||||||
|
expect(notify.price).that.does.include('EUR');
|
||||||
|
expect(notify.size).to.be.not.empty;
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
48
test/provider/immonet.test.js
Normal file
48
test/provider/immonet.test.js
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/immonet');
|
||||||
|
|
||||||
|
describe('#immonet testsuite()', () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.immonet, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should test immonet provider', async () => {
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const immonetDbContent = fredy._getStore();
|
||||||
|
|
||||||
|
expect(immonetDbContent.immonet).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj).to.be.a('object');
|
||||||
|
expect(notificationObj.serviceName).to.equal('immonet');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('number');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
|
||||||
|
expect(notify.price).that.does.include('€');
|
||||||
|
expect(notify.size).that.does.include('m²');
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.immonet.de');
|
||||||
|
expect(notify.address).to.be.not.empty;
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
47
test/provider/immoscout.test.js
Normal file
47
test/provider/immoscout.test.js
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/immoscout');
|
||||||
|
|
||||||
|
describe('#immoscout testsuite()', () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.immoscout, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should test immoscout provider', async () => {
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const immoscoutDbContent = fredy._getStore();
|
||||||
|
expect(immoscoutDbContent.immoscout).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj).to.be.a('object');
|
||||||
|
expect(notificationObj.serviceName).to.equal('immoscout');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('number');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(immoscoutDbContent.immoscout[idx]);
|
||||||
|
expect(notify.price).that.does.include('€');
|
||||||
|
expect(notify.size).that.does.include('m²');
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
|
||||||
|
expect(notify.address).to.be.not.empty;
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
49
test/provider/immowelt.test.js
Normal file
49
test/provider/immowelt.test.js
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/immowelt');
|
||||||
|
|
||||||
|
describe('#immowelt testsuite()', () => {
|
||||||
|
it('should test immowelt provider', async () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.immowelt, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const immoweltDbContent = fredy._getStore();
|
||||||
|
expect(immoweltDbContent.immowelt).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj).to.be.a('object');
|
||||||
|
expect(notificationObj.serviceName).to.equal('immowelt');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('number');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(immoweltDbContent.immowelt[idx]);
|
||||||
|
expect(notify.price).that.does.include('€');
|
||||||
|
if (notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||||
|
expect(notify.size).that.does.include('m²');
|
||||||
|
}
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.immowelt.de');
|
||||||
|
expect(notify.address).to.be.not.empty;
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
44
test/provider/kalaydo.test.js
Normal file
44
test/provider/kalaydo.test.js
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/kalaydo');
|
||||||
|
|
||||||
|
describe('#kalaydo testsuite()', () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.kalaydo, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should test kalaydo provider', async () => {
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const kalaydoDbContent = fredy._getStore();
|
||||||
|
|
||||||
|
expect(kalaydoDbContent.kalaydo).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj).to.be.a('object');
|
||||||
|
expect(notificationObj.serviceName).to.equal('kalaydo');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('string');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.kalaydo.de');
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
45
test/provider/kleinanzeigen.test.js
Normal file
45
test/provider/kleinanzeigen.test.js
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/kleinanzeigen');
|
||||||
|
|
||||||
|
describe('#kleinanzeigen testsuite()', () => {
|
||||||
|
it('should test kleinanzeigen provider', async () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.kleinanzeigen, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const kleinanzeigenDbContent = fredy._getStore();
|
||||||
|
expect(kleinanzeigenDbContent.kleinanzeigen).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj).to.be.a('object');
|
||||||
|
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('number');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(kleinanzeigenDbContent.kleinanzeigen[idx]);
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
|
||||||
|
expect(notify.address).to.be.not.empty;
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
44
test/provider/neubauKompass.test.js
Normal file
44
test/provider/neubauKompass.test.js
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/neubauKompass');
|
||||||
|
|
||||||
|
describe('#neubauKompass testsuite()', () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.neubauKompass, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should test neubauKompass provider', async () => {
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const neubauKompassDbContent = fredy._getStore();
|
||||||
|
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
|
||||||
|
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj.serviceName).to.equal('neubauKompass');
|
||||||
|
|
||||||
|
notificationObj.payload.forEach((notify, idx) => {
|
||||||
|
expect(notify).to.be.a('object');
|
||||||
|
|
||||||
|
/** check the actual structure **/
|
||||||
|
expect(notify.id).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
expect(notify.address).to.be.a('string');
|
||||||
|
|
||||||
|
/** check the values if possible **/
|
||||||
|
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
|
||||||
|
expect(notify.title).to.be.not.empty;
|
||||||
|
expect(notify.link).that.does.include('https://www.neubaukompass.de');
|
||||||
|
expect(notify.address).to.be.not.empty;
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
const utils = require('../lib/utils');
|
const utils = require('../../lib/utils');
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
|
|
||||||
describe('utils', () => {
|
describe('utils', () => {
|
||||||
39
test/provider/wgGesucht.test.js
Normal file
39
test/provider/wgGesucht.test.js
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
const mockNotification = require('../mocks/mockNotification');
|
||||||
|
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||||
|
const mockStore = require('../mocks/mockStore');
|
||||||
|
const proxyquire = require('proxyquire').noCallThru();
|
||||||
|
const expect = require('chai').expect;
|
||||||
|
const provider = require('../../lib/provider/wgGesucht');
|
||||||
|
|
||||||
|
describe('#wgGesucht testsuite()', () => {
|
||||||
|
provider.init(mockConfig.jobs.test1.provider.wgGesucht, [], []);
|
||||||
|
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||||
|
'./services/store': mockStore,
|
||||||
|
'./notification/notify': mockNotification
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should test wgGesucht provider', async () => {
|
||||||
|
return await new Promise(resolve => {
|
||||||
|
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||||
|
fredy.execute().then(() => {
|
||||||
|
const wgGesuchtDbContent = fredy._getStore();
|
||||||
|
expect(wgGesuchtDbContent.wgGesucht).to.be.a('array');
|
||||||
|
const notificationObj = mockNotification.get();
|
||||||
|
expect(notificationObj.serviceName).to.equal('wgGesucht');
|
||||||
|
notificationObj.payload.forEach(notify => {
|
||||||
|
expect(notify).to.be.a('object');
|
||||||
|
|
||||||
|
/** check the actual structure **/
|
||||||
|
|
||||||
|
expect(notify.id).to.be.a('string');
|
||||||
|
expect(notify.title).to.be.a('string');
|
||||||
|
expect(notify.details).to.be.a('string');
|
||||||
|
expect(notify.size).to.be.a('string');
|
||||||
|
expect(notify.price).to.be.a('string');
|
||||||
|
expect(notify.link).to.be.a('string');
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
const mockNotification = require('./mocks/mockNotification');
|
|
||||||
const mockConfig = require('../conf/config.test');
|
|
||||||
const mockStore = require('./mocks/mockStore');
|
|
||||||
const mockStats = require('./mocks/mockStats');
|
|
||||||
const proxyquire = require('proxyquire').noCallThru();
|
|
||||||
const expect = require('chai').expect;
|
|
||||||
|
|
||||||
describe('#wgGesucht testsuite()', () => {
|
|
||||||
|
|
||||||
const wgGesucht = proxyquire('../lib/provider/wgGesucht', {
|
|
||||||
'../../conf/config.json': mockConfig,
|
|
||||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
|
||||||
'./services/store': mockStore,
|
|
||||||
'./notification/notify': mockNotification
|
|
||||||
})
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should test wgGesucht provider', async () => {
|
|
||||||
return await new Promise(resolve => {
|
|
||||||
wgGesucht.run(mockStats).then(() => {
|
|
||||||
const wgGesuchtDbContent = wgGesucht._getStore()._db;
|
|
||||||
expect(wgGesuchtDbContent.wgGesucht).to.be.a('array');
|
|
||||||
const notificationObj = mockNotification.get();
|
|
||||||
expect(notificationObj.serviceName).to.equal('wgGesucht');
|
|
||||||
notificationObj.payload.forEach((notify, idx) => {
|
|
||||||
expect(notify).to.be.a('object');
|
|
||||||
|
|
||||||
/** check the actual structure **/
|
|
||||||
|
|
||||||
expect(notify.id).to.be.a('string');
|
|
||||||
expect(notify.title).to.be.a('string');
|
|
||||||
expect(notify.details).to.be.a('string');
|
|
||||||
expect(notify.description).to.be.a('string');
|
|
||||||
expect(notify.link).to.be.a('string');
|
|
||||||
|
|
||||||
});
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
Reference in New Issue
Block a user