mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Concurrent-Jobs (#7)
Fredy 2.0.0 introduces the concept of search jobs. You can now configure multiple of these jobs running in the same instance of Fredy. Also a new API has been created 🎉
This commit is contained in:
Christian Kellner
committed by
GitHub
GitHub
parent
52a32f7453
commit
770d30af95
42
.eslintrc.js
Normal file
42
.eslintrc.js
Normal file
@@ -0,0 +1,42 @@
|
||||
module.exports = {
|
||||
env: {
|
||||
commonjs: true,
|
||||
es6: true,
|
||||
node: true
|
||||
},
|
||||
extends: 'eslint:recommended',
|
||||
globals: {
|
||||
Atomics: 'readonly',
|
||||
SharedArrayBuffer: 'readonly',
|
||||
Promise: false,
|
||||
describe: true,
|
||||
it: true
|
||||
},
|
||||
parserOptions: {
|
||||
ecmaVersion: 2018
|
||||
},
|
||||
rules: {
|
||||
eqeqeq: [2, 'allow-null'],
|
||||
|
||||
// ###########################################################
|
||||
// ### Semantics / Performance impacting
|
||||
// ###########################################################
|
||||
// babel inserts `'use strict';` for us
|
||||
strict: 0,
|
||||
|
||||
'no-redeclare': [2, { builtinGlobals: false }],
|
||||
|
||||
// If a class method does not use this, it can safely be made a static function.
|
||||
// http://eslint.org/docs/rules/class-methods-use-this
|
||||
'class-methods-use-this': ['off'],
|
||||
|
||||
// ###########################################################
|
||||
// ### Style
|
||||
// ###########################################################
|
||||
indent: ['off', 2],
|
||||
'linebreak-style': ['error', 'unix'],
|
||||
quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }],
|
||||
semi: ['error', 'always'],
|
||||
'no-console': ['error', { allow: ['warn', 'error'] }]
|
||||
}
|
||||
};
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,5 +1,5 @@
|
||||
node_modules/
|
||||
npm-debug.log
|
||||
config.json
|
||||
store.json
|
||||
.DS_Store
|
||||
config.json
|
||||
|
||||
9
CHANGELOG.md
Normal file
9
CHANGELOG.md
Normal file
@@ -0,0 +1,9 @@
|
||||
###### [V2.0.0]
|
||||
```
|
||||
- Fredy can now run multiple search job on one instance
|
||||
- Changed lot's of the structure of Fredy to make this happen
|
||||
[BREAKING CHANGES]
|
||||
- The config has been changed, the config of V1.x will not work any longer
|
||||
- Sources have been renamed to provider
|
||||
```
|
||||
For more info on how to upgrade from Fredy V1.x, plz check the [Upgrade Guide](./doc/upgrade-from-1-to-2.md)
|
||||
@@ -19,10 +19,12 @@ If you've written a new provider you are an awesome person. You know it and I do
|
||||
|
||||
To write tests for provider, you need to use Node 8 as the tests are using `async / await`
|
||||
|
||||
#### Codestyle
|
||||
I'm using Eslint to maintain quote style and quality. Do not skip it...
|
||||
|
||||
##### To do before merging:
|
||||
|
||||
- executed tests? (`npm run test`)
|
||||
- executed reformat? (`npm run format`)
|
||||
- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
|
||||
|
||||
_Thanks!_ :heart:
|
||||
|
||||
103
README.md
103
README.md
@@ -10,12 +10,14 @@ If _Fredy_ found matching results, it will send them to via Slack. (More adapter
|
||||
|
||||
## Usage
|
||||
|
||||
- Make sure to use Node 8 and above
|
||||
- Install the dependencies using `npm install`
|
||||
- Make sure to use Node 11 and above
|
||||
- Install the dependencies using `npm install` or `yarn`
|
||||
- create your configuration file. Use the example config for this. `cp conf/config.example conf/config.json`
|
||||
- configure the desired values
|
||||
- start _Fredy_ using `npm start`
|
||||
- start _Fredy_ using `npm start` or `yarn run start`
|
||||
|
||||
## :point_up: Breaking Changes when updating from v1.x to v2
|
||||
See [Upgrade Guide](./doc/upgrade-from-1-to-2.md)
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -35,19 +37,13 @@ Adding new notification adapter is easy however. See [Contribution](https://gith
|
||||
##### Slack
|
||||
```json
|
||||
"slack": {
|
||||
"channel": "someChannel",
|
||||
"token": "someToken",
|
||||
"enabled": true
|
||||
}
|
||||
"channel": "someChannel", "token": "someToken", "enabled": true}
|
||||
```
|
||||
|
||||
##### Telegram
|
||||
```json
|
||||
"telegram": {
|
||||
"chatId": "someChannel",
|
||||
"token": "someToken",
|
||||
"enabled": true
|
||||
}
|
||||
"chatId": "someChannel", "token": "someToken", "enabled": true}
|
||||
```
|
||||
|
||||
For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
|
||||
@@ -59,7 +55,24 @@ curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
|
||||
|
||||
A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
|
||||
|
||||
### 2. Configure the providers
|
||||
### 2. Multiple search jobs
|
||||
|
||||
Since v2.0.0, Fredy supports multiple search jobs running within the same instance of Fredy. For this to work correctly, you need to give each search job a unique name.
|
||||
See the example config...
|
||||
```json
|
||||
(...)
|
||||
"jobs": {
|
||||
"yourSearchJob": {
|
||||
"some":"config"
|
||||
},
|
||||
"yourOtherSearchJob": {
|
||||
"some":"config"
|
||||
}
|
||||
}
|
||||
(...)
|
||||
```
|
||||
|
||||
### 3. Configure the providers
|
||||
|
||||
Configure the providers like described below. To disable a provider just remove its entry from the configuration or set it to `false`.
|
||||
|
||||
@@ -69,37 +82,21 @@ These are the current provider that are already implemented within _Fredy_
|
||||
|
||||
```json
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/...",
|
||||
"enabled": true
|
||||
}
|
||||
"url": "https://www.ebay-kleinanzeigen.de/...", "enabled": true}
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"url": "https://www.immobilienscout24.de/...", "enabled": true},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"url": "https://www.immowelt.de/...", "enabled": true},
|
||||
"immonet": {
|
||||
"url": "http://www.immonet.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"url": "http://www.immonet.de/...", "enabled": true},
|
||||
"kalaydo": {
|
||||
"url": "http://www.kalaydo.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"url": "http://www.kalaydo.de/...", "enabled": true},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"url": "https://www.1a-immobilienmarkt.de/...", "enabled": true},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"url": "https://www.neubaukompass.de/...", "enabled": true},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/...",
|
||||
"enabled": true
|
||||
}
|
||||
"url": "https://www.wg-gesucht.de/...", "enabled": true}
|
||||
```
|
||||
|
||||
Go to the respective provider page and create your custom search queries by
|
||||
@@ -112,7 +109,7 @@ the resulting listings page.
|
||||
|
||||
See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)
|
||||
|
||||
### 3. Add Filters (optional)
|
||||
### 4. Add Filters (optional)
|
||||
|
||||
|
||||
#### Blacklist
|
||||
@@ -136,31 +133,29 @@ Districts that are unwanted can be blacklisted here.
|
||||
|
||||
This makes sense for provider that only offer limited filter functions like Kalaydo/Ebay.
|
||||
|
||||
## Stats
|
||||
To monitor, what _Fredy_ is internally doing, you might want to check the current stats. These includes the `config` that is currently being used.
|
||||
Also it includes an array of filter results per provider.
|
||||
# API
|
||||
While Fredy is running, you can make use of the rest api provided on port `9988` to get information about the current status of Fredy.
|
||||
#### http://localhost:9988/
|
||||
Gives you an overview of running search jobs, their included enabled provider, last execution and the number of listings, found by each provider.
|
||||
|
||||
You can call the stats http endpoint like this:
|
||||
```
|
||||
curl http://localhost:9876
|
||||
```
|
||||
The ports is depending on what you've configured in your config file.
|
||||
#### http://localhost:9988/ping
|
||||
Should you ever need some health checks, this returns pong ;)
|
||||
|
||||
#### http://localhost:9988/jobs/:name
|
||||
Returns specific information about the job with the given name or `404` if the job could not be found.
|
||||
|
||||
# Docker
|
||||
Use the Dockerfile in this Repository to build an image.
|
||||
|
||||
Use the Dockerfile in this Repo to build a Image
|
||||
Example: `docker build -t fredy/fredy /path/to/your/Dockerfile`
|
||||
|
||||
Ex: docker build -t fredy/fredy /path/to/your/Dockerfile
|
||||
## Create & run a container
|
||||
|
||||
## Create & Run a Container
|
||||
Put your config.json to `/path/to/your/conf/`
|
||||
|
||||
Put your config.json to /path/to/your/conf/
|
||||
|
||||
Ex: docker create --name fredy -v /path/to/your/conf/:/conf -p 9876:9876 fredy/fredy
|
||||
Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9876:9876 fredy/fredy`
|
||||
|
||||
## Logs
|
||||
|
||||
You can browse Logs with
|
||||
|
||||
docker logs fredy -f
|
||||
You can browse the logs with `docker logs fredy -f`
|
||||
|
||||
|
||||
@@ -1,33 +1,34 @@
|
||||
{
|
||||
"interval": 10,
|
||||
"enableStats": false,
|
||||
"statsPort": 9875,
|
||||
"interval": 1,
|
||||
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36",
|
||||
"jobs": {
|
||||
"yourSearchJob": {
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "someToken",
|
||||
"channel": "someChannel",
|
||||
"token": "",
|
||||
"channel": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"sources": {
|
||||
"provider": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/...",
|
||||
"url": "https://www.immonet.de/immobiliensuche/...",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/...",
|
||||
"url": "https://www.immowelt.de/liste/...",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
@@ -35,13 +36,18 @@
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/...",
|
||||
"url": "https://www.kalaydo.de/immobilien/...",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien":{
|
||||
"url": "https://www.1a-immobilienmarkt.de/...",
|
||||
"abInsZuHause": {
|
||||
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/...",
|
||||
"enabled": true
|
||||
},"neubauKompass": {
|
||||
},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/...",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/...",
|
||||
"enabled": true
|
||||
},
|
||||
@@ -51,16 +57,31 @@
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"swap",
|
||||
"tausch",
|
||||
"wg",
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistrics": [
|
||||
"some District you don't want"
|
||||
],
|
||||
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36"
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Friedrichstadt",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Wittlaer",
|
||||
"Lörick"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
{
|
||||
"interval": 10,
|
||||
"enableStats": false,
|
||||
"statsPort": 9875,
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "",
|
||||
"channel": "test",
|
||||
"enabled": false
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"sources": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=25&objecttype=1&locationname=D%C3%BCsseldorf&acid=&actype=&city=100207&ajaxIsRadiusActive=false&sortby=19&suchart=2&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=1_1&parentcat=1&marketingtype=1&fromprice=&toprice=&fromarea=&toarea=&fromplotarea=&toplotarea=&fromrooms=&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=&fulltext=&absenden=Ergebnisse+anzeigen",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?lat=51.22061&lon=6.80575&sort=createdate%2Bdesc",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/wohnung/k0c196l2068r5",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien":{
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=fcfa4ee1e6cfaf791051a6f342eec1f8&sort_type=newest",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistrics": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Flehe",
|
||||
"Friedrichstadt",
|
||||
"Garath",
|
||||
"Hafen",
|
||||
"Hamm",
|
||||
"Hassels",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Himmelgeist",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kaiserswerth",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Ludenberg",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Vennhausen",
|
||||
"Volmerswerth",
|
||||
"Wittlaer"
|
||||
],
|
||||
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36"
|
||||
}
|
||||
168
conf/forTesting/config.multi.test.json
Executable file
168
conf/forTesting/config.multi.test.json
Executable file
@@ -0,0 +1,168 @@
|
||||
{
|
||||
"interval": 1,
|
||||
"userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36",
|
||||
"jobs": {
|
||||
"test1": {
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "",
|
||||
"channel": "",
|
||||
"enabled": false
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"provider": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||
"enabled": true
|
||||
},
|
||||
"abInsZuHause": {
|
||||
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/D%C3%BCsseldorf/wohnung-kaufen/420000/90/3.5/1839/",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Dusseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Friedrichstadt",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Wittlaer",
|
||||
"Lörick"
|
||||
]
|
||||
},
|
||||
"test2": {
|
||||
"notification": {
|
||||
"slack": {
|
||||
"token": "",
|
||||
"channel": "",
|
||||
"enabled": false
|
||||
},
|
||||
"telegram": {
|
||||
"chatId": "",
|
||||
"token": "",
|
||||
"enabled": false
|
||||
},
|
||||
"console": {
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"provider": {
|
||||
"immoscout": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/S-2/Wohnung-Kauf/Nordrhein-Westfalen/Duesseldorf/-/-/-/-/false/-/3,6,7,8,40,113,117,118,127?enteredFrom=result_list",
|
||||
"enabled": true
|
||||
},
|
||||
"immonet": {
|
||||
"url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
|
||||
"enabled": true
|
||||
},
|
||||
"immowelt": {
|
||||
"url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
|
||||
"enabled": true
|
||||
},
|
||||
"kleinanzeigen": {
|
||||
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,",
|
||||
"enabled": true
|
||||
},
|
||||
"kalaydo": {
|
||||
"url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
|
||||
"enabled": true
|
||||
},
|
||||
"abInsZuHause": {
|
||||
"url": "https://ab-ins-zuhause.de/neues-zuhause-finden/D%C3%BCsseldorf/wohnung-kaufen/420000/90/3.5/1839/",
|
||||
"enabled": true
|
||||
},
|
||||
"einsAImmobilien": {
|
||||
"url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-kaufen.html?search=yes&cfid=98b39c7e-b403-4764-8f3c-57bf590923d0&data_hash=f46f89548257740094dd708996adcd68&sort_type=newest",
|
||||
"enabled": true
|
||||
},
|
||||
"neubauKompass": {
|
||||
"url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
|
||||
"enabled": true
|
||||
},
|
||||
"wgGesucht": {
|
||||
"url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"blacklist": [
|
||||
"Vermietete",
|
||||
"Vermietet",
|
||||
"vermietete",
|
||||
"vermietet"
|
||||
],
|
||||
"blacklistedDistricts": [
|
||||
"Altstadt",
|
||||
"Angermund",
|
||||
"Carlstadt",
|
||||
"Friedrichstadt",
|
||||
"Heerdt",
|
||||
"Hellerhof",
|
||||
"Hubbelrath",
|
||||
"Itter",
|
||||
"Kalkum",
|
||||
"Lichtenbroich",
|
||||
"Lohausen",
|
||||
"Niederkassel",
|
||||
"Oberkassel",
|
||||
"Stadtmittel",
|
||||
"Stockum",
|
||||
"Urdenbach",
|
||||
"Wittlaer",
|
||||
"Lörick"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
19
doc/upgrade-from-1-to-2.md
Normal file
19
doc/upgrade-from-1-to-2.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# Upgrading from v1.x to v2.0.0
|
||||
|
||||
Fredy 2.0.0 introduced the concept of multiple jobs running within an instance of Fredy. For this to work, I had to change the config and the storage format.
|
||||
|
||||
### How to update?
|
||||
##### Store
|
||||
It's best to clear the store completely and let Fredy rewrite it. Be careful to disable all notification adapter the first time you run Fredy 2, as it will obviously treat
|
||||
everything as new.
|
||||
|
||||
##### Config
|
||||
The config format has changed. It now supports multiple jobs. It is probably easiest to simply copy the `config.example` from `/conf` and enter your urls in there.
|
||||
The new format basically wraps the config in chunks.
|
||||
|
||||
```json
|
||||
"jobs": {
|
||||
"yourSearchJob": {
|
||||
"some":"stuff"
|
||||
}
|
||||
```
|
||||
33
index.js
33
index.js
@@ -1,13 +1,36 @@
|
||||
const fs = require('fs');
|
||||
const path = './lib/provider';
|
||||
const sources = fs.readdirSync(path);
|
||||
const provider = fs.readdirSync(path);
|
||||
const config = require('./conf/config.json');
|
||||
const stats = require('./lib/services/stats');
|
||||
const { setLastJobExecution, init: storeInit } = require('./lib/services/store');
|
||||
const FredyRuntime = require('./lib/FredyRuntime');
|
||||
|
||||
setInterval(
|
||||
//starting the api service
|
||||
require('./lib/api/api');
|
||||
|
||||
storeInit().then(() => {
|
||||
setInterval(
|
||||
(function exec() {
|
||||
sources.forEach(source => require(`${path}/${source}`).run(stats));
|
||||
Object.keys(config.jobs).forEach(jobKey => {
|
||||
const jobConfig = config.jobs[jobKey];
|
||||
provider
|
||||
.map(pro => require(`${path}/${pro}`))
|
||||
.forEach(pro => {
|
||||
const providerId = pro.id();
|
||||
if (providerId == null || providerId.length === 0) {
|
||||
throw new Error('Provider id must not be empty. => ' + pro);
|
||||
}
|
||||
const providerConfig = jobConfig.provider[providerId];
|
||||
if (providerConfig == null) {
|
||||
throw new Error(`Provider Config for provider with id ${providerId} not found.`);
|
||||
}
|
||||
pro.init(providerConfig, jobConfig.blacklist, jobConfig.blacklistedDistricts);
|
||||
new FredyRuntime(pro.config, jobConfig.notification, providerId, jobKey).execute();
|
||||
setLastJobExecution(jobKey);
|
||||
});
|
||||
});
|
||||
return exec;
|
||||
})(),
|
||||
config.interval * 60 * 1000
|
||||
);
|
||||
);
|
||||
});
|
||||
|
||||
114
lib/FredyRuntime.js
Executable file
114
lib/FredyRuntime.js
Executable file
@@ -0,0 +1,114 @@
|
||||
const { NoNewListingsError } = require('./errors');
|
||||
const {
|
||||
setKnownListings,
|
||||
getKnownListings,
|
||||
setNumberOfTotalFoundProviderListings,
|
||||
getForTesting
|
||||
} = require('./services/store');
|
||||
|
||||
const notify = require('./notification/notify');
|
||||
const xray = require('./services/scraper');
|
||||
|
||||
class FredyRuntime {
|
||||
/**
|
||||
*
|
||||
* @param providerConfig the config for the specific provider, we're going to query at the moment
|
||||
* @param notificationConfig the config for all notifications (because all could be applied to a provider)
|
||||
* @param providerId the id of the provider currently in use
|
||||
* @param jobKey key of the job that is currently running (from within the config)
|
||||
*/
|
||||
constructor(providerConfig, notificationConfig, providerId, jobKey) {
|
||||
this._providerConfig = providerConfig;
|
||||
this._notificationConfig = notificationConfig;
|
||||
this._providerId = providerId;
|
||||
this._jobKey = jobKey;
|
||||
}
|
||||
|
||||
execute() {
|
||||
if (!this._providerConfig.enabled) return Promise.resolve();
|
||||
|
||||
return Promise.resolve(this._providerConfig.url)
|
||||
.then(this._getListings.bind(this))
|
||||
.then(this._normalize.bind(this))
|
||||
.then(this._filter.bind(this))
|
||||
.then(this._findNew.bind(this))
|
||||
.then(this._storeStats.bind(this))
|
||||
.then(this._save.bind(this))
|
||||
.then(this._notify.bind(this))
|
||||
.then(this._updateStates.bind(this))
|
||||
.catch(this._handleError.bind(this));
|
||||
}
|
||||
|
||||
_getListings(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
|
||||
|
||||
if (this._providerConfig.paginage) {
|
||||
x = x.paginate(this._providerConfig.paginage);
|
||||
}
|
||||
|
||||
x((err, listings) => {
|
||||
if (err) reject(err);
|
||||
else {
|
||||
resolve(listings);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
_storeStats(listings) {
|
||||
setNumberOfTotalFoundProviderListings(this._jobKey, this._providerId, listings.length);
|
||||
return Promise.resolve(listings);
|
||||
}
|
||||
|
||||
_normalize(listings) {
|
||||
return listings.map(this._providerConfig.normalize);
|
||||
}
|
||||
|
||||
_filter(listings) {
|
||||
return listings.filter(this._providerConfig.filter);
|
||||
}
|
||||
|
||||
_findNew(listings) {
|
||||
const newListings = listings.filter(o => getKnownListings(this._jobKey, this._providerId).indexOf(o.id) === -1);
|
||||
|
||||
if (newListings.length === 0) {
|
||||
this._updateStates([]);
|
||||
throw new NoNewListingsError();
|
||||
}
|
||||
|
||||
return newListings;
|
||||
}
|
||||
|
||||
_notify(newListings) {
|
||||
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig);
|
||||
return Promise.all(sendNotifications).then(() => newListings);
|
||||
}
|
||||
|
||||
_updateStates(newListings) {
|
||||
return newListings;
|
||||
}
|
||||
|
||||
_save(newListings) {
|
||||
setKnownListings(this._jobKey, this._providerId, [
|
||||
...getKnownListings(this._jobKey, this._providerId),
|
||||
...newListings.map(l => l.id)
|
||||
]);
|
||||
return newListings;
|
||||
}
|
||||
|
||||
_handleError(err) {
|
||||
if (err.name !== 'NoNewListingsError') console.error(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* for testing purposes only
|
||||
* @returns {Store}
|
||||
* @private
|
||||
*/
|
||||
_getStore() {
|
||||
return getForTesting();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = FredyRuntime;
|
||||
62
lib/api/api.js
Normal file
62
lib/api/api.js
Normal file
@@ -0,0 +1,62 @@
|
||||
const bodyParser = require('body-parser');
|
||||
const config = require('../../conf/config');
|
||||
const { getLastJobExecution, getLastProviderExecution, getTotalNumberOfListings } = require('../services/store');
|
||||
const PORT = 9988;
|
||||
const service = require('restana')();
|
||||
service.use(bodyParser.json());
|
||||
|
||||
service.get('/', async (req, res) => {
|
||||
const result = {};
|
||||
Object.keys(config.jobs).forEach(job => {
|
||||
result[job] = {
|
||||
lastExecution: getLastJobExecution(job),
|
||||
enabledProvider: Object.keys(config.jobs[job].provider)
|
||||
.filter(providerKey => config.jobs[job].provider[providerKey].enabled)
|
||||
.map(providerKey => {
|
||||
return {
|
||||
name: providerKey,
|
||||
lastExecution: getLastProviderExecution(job, providerKey),
|
||||
totalFindings: getTotalNumberOfListings(job, providerKey)
|
||||
};
|
||||
})
|
||||
};
|
||||
});
|
||||
res.body = result;
|
||||
res.send();
|
||||
});
|
||||
|
||||
service.get('/jobs/:name', async (req, res) => {
|
||||
const { name: jobKey } = req.params;
|
||||
if (Object.keys(config.jobs).indexOf(jobKey) === -1) {
|
||||
console.error(`Cannot find job with name ${jobKey}. Available Jobs are [${Object.keys(config.jobs)}]`);
|
||||
res.send(404);
|
||||
return;
|
||||
}
|
||||
res.body = {
|
||||
lastExecution: getLastJobExecution(jobKey),
|
||||
enabledProvider: Object.keys(config.jobs[jobKey].provider)
|
||||
.filter(providerKey => config.jobs[jobKey].provider[providerKey].enabled)
|
||||
.map(providerKey => {
|
||||
return {
|
||||
name: providerKey,
|
||||
url: config.jobs[jobKey].provider[providerKey].url,
|
||||
lastExecution: getLastProviderExecution(jobKey, providerKey),
|
||||
totalFindings: getTotalNumberOfListings(jobKey, providerKey)
|
||||
};
|
||||
})
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
|
||||
service.get('/ping', function(req, res) {
|
||||
res.body = {
|
||||
pong: 'pong'
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
|
||||
service.start(PORT).then(() => {
|
||||
/* eslint-disable no-console */
|
||||
console.info(`Started API service on port ${PORT}`);
|
||||
/* eslint-enable no-console */
|
||||
});
|
||||
@@ -3,14 +3,13 @@ class ExtendableError extends Error {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
if (typeof Error.captureStackTrace === 'function') {
|
||||
Error.captureStackTrace(this, this.constructor)
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
} else {
|
||||
this.stack = (new Error(message)).stack
|
||||
this.stack = new Error(message).stack;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class NoNewListingsError extends ExtendableError {
|
||||
}
|
||||
class NoNewListingsError extends ExtendableError {}
|
||||
|
||||
module.exports = {NoNewListingsError};
|
||||
module.exports = { NoNewListingsError };
|
||||
|
||||
106
lib/fredy.js
106
lib/fredy.js
@@ -1,106 +0,0 @@
|
||||
const {NoNewListingsError} = require('./errors');
|
||||
const Store = require('./services/store');
|
||||
|
||||
const notify = require('./notification/notify');
|
||||
const xray = require('./services/scraper');
|
||||
|
||||
class Fredy {
|
||||
constructor(source) {
|
||||
this._store = new Store(source.name);
|
||||
this._fullCrawl = true;
|
||||
this._source = source;
|
||||
this._stats = null;
|
||||
}
|
||||
|
||||
run(stats) {
|
||||
|
||||
if(!this._stats){
|
||||
this._stats = stats;
|
||||
}
|
||||
|
||||
if (!this._source.enabled) return Promise.resolve();
|
||||
|
||||
return Promise.resolve(this._source.url)
|
||||
.then(this._store.warmup)
|
||||
.then(this._getListings.bind(this))
|
||||
.then(this._normalize.bind(this))
|
||||
.then(this._filter.bind(this))
|
||||
.then(this._findNew.bind(this))
|
||||
.then(this._save.bind(this))
|
||||
.then(this._notify.bind(this))
|
||||
.then(this._updateStates.bind(this))
|
||||
.catch(this._handleError.bind(this))
|
||||
}
|
||||
|
||||
_getListings(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let x = xray(url, this._source.crawlContainer, [this._source.crawlFields]);
|
||||
|
||||
if (this._source.paginage && this._fullCrawl) {
|
||||
this._fullCrawl = false;
|
||||
x = x.paginate(this._source.paginage)
|
||||
}
|
||||
|
||||
x((err, listings) => {
|
||||
if (err) reject(err);
|
||||
else {
|
||||
resolve(listings);
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
_normalize(listings) {
|
||||
return listings.map(this._source.normalize)
|
||||
}
|
||||
|
||||
_filter(listings) {
|
||||
return listings.filter(this._source.filter)
|
||||
}
|
||||
|
||||
_findNew(listings) {
|
||||
const newListings = listings.filter(
|
||||
o => this._store.knownListings.indexOf(o.id) === -1
|
||||
);
|
||||
|
||||
if (newListings.length === 0) {
|
||||
this._updateStates([]);
|
||||
throw new NoNewListingsError();
|
||||
}
|
||||
|
||||
return newListings
|
||||
}
|
||||
|
||||
_notify(newListings) {
|
||||
const sendNotifications = notify.send(this._source.name, newListings);
|
||||
return Promise.all(sendNotifications).then(() => newListings)
|
||||
}
|
||||
|
||||
_updateStates(newListings){
|
||||
this._stats.setLastScrape(this._source.name, newListings.length);
|
||||
return newListings;
|
||||
}
|
||||
|
||||
_save(newListings) {
|
||||
this._store.knownListings = [
|
||||
...this._store.knownListings,
|
||||
...newListings.map(l => l.id)
|
||||
];
|
||||
return newListings;
|
||||
}
|
||||
|
||||
_handleError(err) {
|
||||
if (err.name !== 'NoNewListingsError') console.error(err)
|
||||
}
|
||||
|
||||
/**
|
||||
* for testing purposes only
|
||||
* @returns {Store}
|
||||
* @private
|
||||
*/
|
||||
_getStore(){
|
||||
return this._store;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Fredy;
|
||||
@@ -1,17 +1,15 @@
|
||||
const config = require('../../../conf/config.json');
|
||||
|
||||
/**
|
||||
* simply prints out the found data to the console
|
||||
* @param serviceName e.g immoscout
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
*/
|
||||
exports.send = (serviceName, newListings) => {
|
||||
exports.send = (serviceName, newListings, notificationConfig) => {
|
||||
const { enabled } = notificationConfig.console;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
/* eslint-disable no-console */
|
||||
return [Promise.resolve(console.info(`Found entry from service ${serviceName}:`, newListings))];
|
||||
};
|
||||
|
||||
/**
|
||||
* each integration needs to implement this method
|
||||
*/
|
||||
exports.enabled = () => {
|
||||
return config.notification.console.enabled;
|
||||
/* eslint-enable no-console */
|
||||
};
|
||||
|
||||
@@ -1,54 +1,50 @@
|
||||
const Slack = require('slack');
|
||||
const config = require('../../../conf/config.json');
|
||||
const msg = Slack.chat.postMessage;
|
||||
|
||||
const {token, channel} = config.notification.slack;
|
||||
|
||||
/**
|
||||
* sends a new listing to slack
|
||||
* @param serviceName e.g immoscout
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* @returns {Promise<Chat.PostMessage.Response> | void}
|
||||
*/
|
||||
exports.send = (serviceName, newListings) => {
|
||||
return newListings.map(payload => msg({
|
||||
exports.send = (serviceName, newListings, notificationConfig) => {
|
||||
const { token, channel, enabled } = notificationConfig.slack;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
return newListings.map(payload =>
|
||||
msg({
|
||||
token,
|
||||
channel,
|
||||
text: `*(${serviceName})* - ${payload.title}`,
|
||||
"attachments": [
|
||||
attachments: [
|
||||
{
|
||||
"fallback": payload.title,
|
||||
"color": "#36a64f",
|
||||
"title": "Link to Exposé",
|
||||
"title_link": payload.link,
|
||||
"fields": [
|
||||
fallback: payload.title,
|
||||
color: '#36a64f',
|
||||
title: 'Link to Exposé',
|
||||
title_link: payload.link,
|
||||
fields: [
|
||||
{
|
||||
"title": "Price",
|
||||
"value": payload.price,
|
||||
"short": false
|
||||
title: 'Price',
|
||||
value: payload.price,
|
||||
short: false
|
||||
},
|
||||
{
|
||||
"title": "Size",
|
||||
"value": payload.size,
|
||||
"short": false
|
||||
title: 'Size',
|
||||
value: payload.size,
|
||||
short: false
|
||||
},
|
||||
{
|
||||
"title": "Address",
|
||||
"value": payload.address,
|
||||
"short": false
|
||||
title: 'Address',
|
||||
value: payload.address,
|
||||
short: false
|
||||
}
|
||||
],
|
||||
"footer": "Powered by Fredy",
|
||||
footer: 'Powered by Fredy',
|
||||
ts: new Date().getTime() / 1000
|
||||
}
|
||||
]
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* each integration needs to implement this method
|
||||
*/
|
||||
exports.enabled = () => {
|
||||
return config.notification.slack.enabled;
|
||||
};
|
||||
|
||||
@@ -1,17 +1,23 @@
|
||||
const config = require('../../../conf/config.json');
|
||||
const TelegramBot = require('tg-yarl');
|
||||
|
||||
const opts = { parse_mode: 'Markdown' };
|
||||
const bot = new TelegramBot(config.notification.telegram.token);
|
||||
const opts = {parse_mode: 'Markdown'};
|
||||
|
||||
|
||||
/**
|
||||
* sends new listings to telegram
|
||||
* @param serviceName e.g immoscout
|
||||
* @param newListings an array with newly found listings
|
||||
* @param notificationConfig config of this notification adapter
|
||||
* @returns {Promise<Void> | void}
|
||||
*/
|
||||
exports.send = (serviceName, newListings) => {
|
||||
exports.send = (serviceName, newListings, notificationConfig) => {
|
||||
const {enabled, token, chatId} = notificationConfig.telegram;
|
||||
if (!enabled) {
|
||||
return [Promise.resolve()];
|
||||
}
|
||||
|
||||
const bot = new TelegramBot(token);
|
||||
|
||||
let message = `Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
|
||||
|
||||
message += newListings.map(o =>
|
||||
@@ -19,17 +25,9 @@ exports.send = (serviceName, newListings) => {
|
||||
[o.address, o.price, o.size].join(' | ') + '\n' +
|
||||
`[LINK](${o.link})\n\n`);
|
||||
|
||||
return bot.sendMessage(config.notification.telegram.chatId, message, opts);
|
||||
return bot.sendMessage(chatId, message, opts);
|
||||
};
|
||||
|
||||
/**
|
||||
* each integration needs to implement this method
|
||||
*/
|
||||
exports.enabled = () => {
|
||||
return config.notification.telegram.enabled;
|
||||
};
|
||||
|
||||
|
||||
function shorten (str, len = 30) {
|
||||
function shorten(str, len = 30) {
|
||||
return str.length > len ? str.substring(0, len) + '...' : str;
|
||||
}
|
||||
@@ -4,14 +4,13 @@ const path = './adapter';
|
||||
/** Read every integration existing in ./adapter **/
|
||||
const adapter = fs
|
||||
.readdirSync('./lib/notification/adapter')
|
||||
.map(integPath => require(`${path}/${integPath}`))
|
||||
.filter(integration => integration.enabled());
|
||||
.map(integPath => require(`${path}/${integPath}`));
|
||||
|
||||
if (adapter.length === 0) {
|
||||
throw new Error('Please specify at least one notification provider');
|
||||
}
|
||||
|
||||
exports.send = (serviceName, payload) => {
|
||||
exports.send = (serviceName, payload, notificationConfig) => {
|
||||
//this is not being used in tests, therefor adapter are always set
|
||||
return adapter.map(a => a.send(serviceName, payload));
|
||||
return adapter.map(a => a.send(serviceName, payload, notificationConfig));
|
||||
};
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
const config = require('../../conf/config.json');
|
||||
const Fredy = require('../fredy');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
|
||||
if(o.rooms != null){
|
||||
size+=` / / ${o.rooms.trim()}`;
|
||||
if (o.rooms != null) {
|
||||
size += ` / / ${o.rooms.trim()}`;
|
||||
}
|
||||
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
|
||||
|
||||
@@ -13,16 +13,15 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const einsAImmobilien = {
|
||||
name: 'einsAImmobilien',
|
||||
enabled: config.sources.einsAImmobilien.enabled,
|
||||
url: config.sources.einsAImmobilien.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '.tabelle',
|
||||
crawlFields: {
|
||||
id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
|
||||
@@ -37,4 +36,13 @@ const einsAImmobilien = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(einsAImmobilien);
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'einsAImmobilien';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const config = require('../../conf/config.json');
|
||||
const Fredy = require('../fredy');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
|
||||
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
|
||||
@@ -13,16 +13,15 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const immonet = {
|
||||
name: 'immonet',
|
||||
enabled: config.sources.immonet.enabled,
|
||||
url: config.sources.immonet.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#result-list-stage .item',
|
||||
crawlFields: {
|
||||
id: '@id',
|
||||
@@ -37,4 +36,13 @@ const immonet = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(immonet);
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'immonet';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const config = require('../../conf/config.json');
|
||||
const Fredy = require('../fredy');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const title = o.title.replace('NEU', '');
|
||||
const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
||||
@@ -10,13 +10,12 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
return !utils.isOneOf(o.title, config.blacklist);
|
||||
return !utils.isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
|
||||
const immoscout = {
|
||||
name: 'immoscout',
|
||||
enabled: config.sources.immoscout.enabled,
|
||||
url: config.sources.immoscout.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#resultListItems li.result-list__listing',
|
||||
crawlFields: {
|
||||
id: '.result-list-entry@data-obid | int',
|
||||
@@ -31,4 +30,13 @@ const immoscout = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(immoscout);
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'immoscout';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const Fredy = require('../fredy');
|
||||
const config = require('../../conf/config.json');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
|
||||
const address = o.address;
|
||||
@@ -10,16 +10,15 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const immowelt = {
|
||||
name: 'immowelt',
|
||||
enabled: config.sources.immowelt.enabled,
|
||||
url: config.sources.immowelt.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '.immoliste .js-object.listitem_wrap ',
|
||||
crawlFields: {
|
||||
id: '@data-estateid | int',
|
||||
@@ -34,4 +33,13 @@ const immowelt = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(immowelt);
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'immowelt';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
const config = require('../../conf/config.json');
|
||||
const Fredy = require('../fredy');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
let appliedBlacklistedDistricts = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = o.id
|
||||
.split('/')
|
||||
@@ -16,19 +17,18 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
const isBlacklistedDistrict =
|
||||
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.title, config.blacklistedDistrics);
|
||||
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
|
||||
|
||||
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const kalaydo = {
|
||||
name: 'kalaydo',
|
||||
enabled: config.sources.kalaydo.enabled,
|
||||
url: config.sources.kalaydo.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#resultList .resultitem-content-container',
|
||||
crawlFields: {
|
||||
id: '.resultitem-content-container a@href',
|
||||
@@ -43,4 +43,15 @@ const kalaydo = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(kalaydo);
|
||||
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
appliedBlacklistedDistricts = blacklistedDistricts;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'kalaydo';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
const Fredy = require('../fredy');
|
||||
const config = require('../../conf/config.json');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
let appliedBlacklistedDistricts = [];
|
||||
|
||||
function normalize(o) {
|
||||
const size = o.size || '--- m²';
|
||||
|
||||
@@ -9,18 +10,17 @@ function normalize(o) {
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
const isBlacklistedDistrict =
|
||||
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
|
||||
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
|
||||
|
||||
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const kleinanzeigen = {
|
||||
name: 'kleinanzeigen',
|
||||
enabled: config.sources.kleinanzeigen.enabled,
|
||||
url: config.sources.kleinanzeigen.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#srchrslt-adtable .ad-listitem',
|
||||
crawlFields: {
|
||||
id: '.aditem@data-adid | int',
|
||||
@@ -36,4 +36,15 @@ const kleinanzeigen = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(kleinanzeigen);
|
||||
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlacklistedDistricts = blacklistedDistricts;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'kleinanzeigen';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
|
||||
@@ -1,19 +1,18 @@
|
||||
const config = require('../../conf/config.json');
|
||||
const Fredy = require('../fredy');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
return o;
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
return !utils.isOneOf(o.title, config.blacklist);
|
||||
return !utils.isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
|
||||
const neubauKompass = {
|
||||
name: 'neubauKompass',
|
||||
enabled: config.sources.neubauKompass.enabled,
|
||||
url: config.sources.neubauKompass.url,
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '.row article',
|
||||
crawlFields: {
|
||||
id: '@id',
|
||||
@@ -26,4 +25,13 @@ const neubauKompass = {
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(neubauKompass);
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'neubauKompass';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,34 +1,42 @@
|
||||
const config = require('../../conf/config.json');
|
||||
const Fredy = require('../fredy');
|
||||
const utils = require('../utils');
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
return o;
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
|
||||
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
|
||||
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
const wgGesucht = {
|
||||
name: 'wgGesucht',
|
||||
enabled: config.sources.wgGesucht.enabled,
|
||||
url: config.sources.wgGesucht.url,
|
||||
crawlContainer: '#main_column .panel:not(.display-none):not(.noprint)',
|
||||
const config = {
|
||||
enabled: null,
|
||||
url: null,
|
||||
crawlContainer: '#main_column .wgg_card',
|
||||
crawlFields: {
|
||||
id: '@data-id',
|
||||
details: ' .list-details-costs-col |removeNewline |trim',
|
||||
title: '.headline .detailansicht |removeNewline |trim',
|
||||
description: '.list-details-category-location |removeNewline |trim',
|
||||
link: '.headline .detailansicht@href'
|
||||
details: '.row .noprint .col-xs-11 |removeNewline |trim',
|
||||
price: '.middle .col-xs-3 |removeNewline |trim',
|
||||
size: '.middle .text-right |removeNewline |trim',
|
||||
title: '.truncate_title a |removeNewline |trim',
|
||||
link: '.truncate_title a@href'
|
||||
},
|
||||
paginate: '.pagination-sm:first a:last@href',
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist
|
||||
};
|
||||
|
||||
module.exports = new Fredy(wgGesucht);
|
||||
exports.init = (sourceConfig, blacklist) => {
|
||||
config.enabled = sourceConfig.enabled;
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklist;
|
||||
};
|
||||
|
||||
//must match the id of the source given in the config!
|
||||
exports.id = () => 'wgGesucht';
|
||||
|
||||
exports.config = config;
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
const config = require('../../conf/config.json');
|
||||
let stats = {
|
||||
lastScrape: {},
|
||||
foundScrapes: {}
|
||||
};
|
||||
|
||||
if (config.enableStats) {
|
||||
const http = require('http');
|
||||
http
|
||||
.createServer((req, res) => {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
config,
|
||||
stats
|
||||
})
|
||||
);
|
||||
})
|
||||
.listen(config.statsPort, '127.0.0.1');
|
||||
}
|
||||
|
||||
const datetime = date => {
|
||||
return `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()} ${date.getHours()}:${date.getMinutes()}`;
|
||||
};
|
||||
|
||||
exports.setLastScrape = (serviceName, numberOfNewListsings) => {
|
||||
const d = new Date();
|
||||
const dt = datetime(d);
|
||||
stats.lastScrape[serviceName] = d.toString();
|
||||
|
||||
if (numberOfNewListsings > 0) {
|
||||
stats.foundScrapes[dt] = stats.foundScrapes[dt] || {};
|
||||
stats.foundScrapes[dt][serviceName] = numberOfNewListsings;
|
||||
}
|
||||
};
|
||||
@@ -7,30 +7,79 @@ const low = require('lowdb');
|
||||
|
||||
const lowdb = low(adapter);
|
||||
|
||||
class Store {
|
||||
constructor(name) {
|
||||
this._name = name;
|
||||
this._db = null;
|
||||
}
|
||||
let db = null;
|
||||
|
||||
get warmup() {
|
||||
const buildKey = (jobKey, providerId, endpoint) => {
|
||||
let key = `${jobKey}`;
|
||||
if (jobKey == null && endpoint == null) {
|
||||
return key;
|
||||
}
|
||||
if (providerId != null) {
|
||||
key += `.${providerId}`;
|
||||
}
|
||||
if (endpoint != null) {
|
||||
key += `.${endpoint}`;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
exports.init = () => {
|
||||
return new Promise(resolve => {
|
||||
lowdb.then(db => {
|
||||
this._db = db;
|
||||
//warmup
|
||||
lowdb.then(database => {
|
||||
db = database;
|
||||
/* eslint-disable no-console */
|
||||
console.info('Warming up database successful');
|
||||
/* eslint-enable no-console */
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||
if (!Array.isArray(listings)) throw Error('Not a valid array');
|
||||
const providerListingsKey = buildKey(jobKey, providerId, 'listings');
|
||||
const providerLastScrapeKey = buildKey(jobKey, providerId, 'lastProviderExecution');
|
||||
|
||||
return db
|
||||
.set(providerListingsKey, listings)
|
||||
.set(providerLastScrapeKey, Date.now())
|
||||
.write();
|
||||
};
|
||||
|
||||
exports.setNumberOfTotalFoundProviderListings = (jobKey, providerId, numberOfNewListings) => {
|
||||
if (numberOfNewListings > 0) {
|
||||
const numberOfFoundListingsKey = buildKey(jobKey, providerId, 'foundListings');
|
||||
const currentNumber = db.get(numberOfFoundListingsKey).value() || 0;
|
||||
db.set(numberOfFoundListingsKey, currentNumber + numberOfNewListings).write();
|
||||
}
|
||||
};
|
||||
|
||||
set knownListings(value) {
|
||||
if (!Array.isArray(value)) throw Error('Not a valid array');
|
||||
exports.setLastJobExecution = jobKey => {
|
||||
const key = buildKey(jobKey, null, 'lastJobExecution');
|
||||
return db.set(key, Date.now()).write();
|
||||
};
|
||||
|
||||
return this._db.set(this._name, value).write();
|
||||
}
|
||||
exports.getKnownListings = (jobKey, providerId) => {
|
||||
const providerListingsKey = buildKey(jobKey, providerId, 'listings');
|
||||
return db.get(providerListingsKey).value() || [];
|
||||
};
|
||||
|
||||
get knownListings() {
|
||||
return this._db.get(this._name).value() || [];
|
||||
}
|
||||
}
|
||||
exports.getLastProviderExecution = (jobKey, providerId) => {
|
||||
const key = buildKey(jobKey, providerId, 'lastProviderExecution');
|
||||
return db.get(key).value() || 0;
|
||||
};
|
||||
|
||||
module.exports = Store;
|
||||
exports.getLastJobExecution = jobKey => {
|
||||
const key = buildKey(jobKey, null, 'lastJobExecution');
|
||||
return db.get(key).value() || 0;
|
||||
};
|
||||
|
||||
exports.getTotalNumberOfListings = (jobKey, providerId) => {
|
||||
const key = buildKey(jobKey, providerId, 'foundListings');
|
||||
return db.get(key).value() || 0;
|
||||
};
|
||||
|
||||
exports.getForTesting = () => {
|
||||
return db;
|
||||
};
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
function isOneOf (word, arr) {
|
||||
function isOneOf(word, arr) {
|
||||
if (arr == null || arr.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const expression = String.raw`\b(${arr.join('|')})\b`;
|
||||
const blacklist = new RegExp(expression, 'ig');
|
||||
|
||||
return blacklist.test(word)
|
||||
return blacklist.test(word);
|
||||
}
|
||||
|
||||
module.exports = { isOneOf };
|
||||
|
||||
33
package.json
33
package.json
@@ -1,11 +1,22 @@
|
||||
{
|
||||
"name": "Fredy",
|
||||
"version": "1.3.0",
|
||||
"name": "fredy",
|
||||
"version": "2.0.0",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"format": "prettier --write lib/**/*.js test/**/*.js *.js --single-quote --print-width 120",
|
||||
"test": "mocha --timeout 12000"
|
||||
"test": "mocha --timeout 15000 test/**/*.test.js"
|
||||
},
|
||||
"husky": {
|
||||
"hooks": {
|
||||
"pre-commit": "lint-staged"
|
||||
}
|
||||
},
|
||||
"lint-staged": {
|
||||
"*.js": [
|
||||
"eslint ./index.js ./lib/**/*.js ./test/**/*.js",
|
||||
"prettier --single-quote --print-width 120 --write"
|
||||
]
|
||||
},
|
||||
"main": "index.js",
|
||||
"author": "Christian Kellner",
|
||||
@@ -27,17 +38,27 @@
|
||||
"url": "https://github.com/orangecoding/fredy/issues"
|
||||
},
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=11.0.0",
|
||||
"npm": ">=6.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"body-parser": "1.19.0",
|
||||
"lowdb": "1.0.0",
|
||||
"request-x-ray": "0.1.4",
|
||||
"restana": "4.0.8",
|
||||
"slack": "11.0.2",
|
||||
"tg-yarl": "1.3.0",
|
||||
"x-ray": "2.3.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"mocha": "7.0.1",
|
||||
"chai": "4.2.0",
|
||||
"eslint": "6.8.0",
|
||||
"eslint-config-prettier": "6.10.0",
|
||||
"husky": "4.2.3",
|
||||
"lint-staged": "10.0.8",
|
||||
"mocha": "7.1.0",
|
||||
"prettier": "1.19.1",
|
||||
"proxyquire": "1.8.0",
|
||||
"chai": "4.2.0"
|
||||
"proxyquire": "2.1.3"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#einsAImmobilien testsuite()', () => {
|
||||
|
||||
const einsAImmobilien = proxyquire('../lib/provider/einsAImmobilien', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
it('should test einsAImmobilien provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
einsAImmobilien.run(mockStats).then(() => {
|
||||
const immonetDbContent = einsAImmobilien._getStore()._db;
|
||||
|
||||
expect(immonetDbContent.einsAImmobilien).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
|
||||
expect(notify.price).that.does.include('EUR');
|
||||
expect(notify.size).to.be.not.empty;
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,51 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#immonet testsuite()', () => {
|
||||
|
||||
const immonet = proxyquire('../lib/provider/immonet', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
it('should test immonet provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
immonet.run(mockStats).then(() => {
|
||||
const immonetDbContent = immonet._getStore()._db;
|
||||
|
||||
expect(immonetDbContent.immonet).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immonet');
|
||||
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immonet.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,51 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#immoscout testsuite()', () => {
|
||||
|
||||
const immoscout = proxyquire('../lib/provider/immoscout', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
it('should test immoscout provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
immoscout.run(mockStats).then(() => {
|
||||
const immoscoutDbContent = immoscout._getStore()._db;
|
||||
expect(immoscoutDbContent.immoscout).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immoscout');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(
|
||||
immoscoutDbContent.immoscout[idx]
|
||||
);
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,55 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#immowelt testsuite()', () => {
|
||||
|
||||
it('should test immowelt provider', async () => {
|
||||
|
||||
const immowelt = proxyquire('../lib/provider/immowelt', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
return await new Promise(resolve => {
|
||||
immowelt.run(mockStats).then(() => {
|
||||
const immoweltDbContent = immowelt._getStore()._db;
|
||||
expect(immoweltDbContent.immowelt).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immowelt');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(
|
||||
immoweltDbContent.immowelt[idx]
|
||||
);
|
||||
expect(notify.price).that.does.include('€');
|
||||
if(notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).that.does.include('m²');
|
||||
}
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immowelt.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,48 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#kalaydo testsuite()', () => {
|
||||
|
||||
const kalaydo = proxyquire('../lib/provider/kalaydo', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
it('should test kalaydo provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
kalaydo.run(mockStats).then(() => {
|
||||
const kalaydoDbContent = kalaydo._getStore()._db;
|
||||
|
||||
expect(kalaydoDbContent.kalaydo).to.be.a('array');
|
||||
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('kalaydo');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.kalaydo.de');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,51 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#kleinanzeigen testsuite()', () => {
|
||||
|
||||
it('should test kleinanzeigen provider', async () => {
|
||||
|
||||
const kleinanzeigen = proxyquire('../lib/provider/kleinanzeigen', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
return await new Promise(resolve => {
|
||||
kleinanzeigen.run(mockStats).then(() => {
|
||||
const kleinanzeigenDbContent = kleinanzeigen._getStore()._db;
|
||||
expect(kleinanzeigenDbContent.kleinanzeigen).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(
|
||||
kleinanzeigenDbContent.kleinanzeigen[idx]
|
||||
);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +0,0 @@
|
||||
module.exports = {
|
||||
setLastScrape: () => {
|
||||
/*noop*/
|
||||
}
|
||||
};
|
||||
@@ -1,3 +1,29 @@
|
||||
const db = {};
|
||||
|
||||
exports.init = () => {
|
||||
return new Promise(resolve => {
|
||||
resolve();
|
||||
});
|
||||
};
|
||||
|
||||
exports.setKnownListings = (jobKey, providerId, listings) => {
|
||||
if (!Array.isArray(listings)) throw Error('Not a valid array');
|
||||
|
||||
db[providerId] = listings;
|
||||
};
|
||||
|
||||
exports.getKnownListings = (jobKey, providerId) => {
|
||||
return db[providerId] || [];
|
||||
};
|
||||
|
||||
exports.setNumberOfTotalFoundProviderListings = () => {
|
||||
/*noop*/
|
||||
};
|
||||
|
||||
exports.getForTesting = () => {
|
||||
return db;
|
||||
};
|
||||
/*
|
||||
class Store {
|
||||
constructor(name) {
|
||||
this._name = name;
|
||||
@@ -17,11 +43,10 @@ class Store {
|
||||
});
|
||||
}
|
||||
|
||||
get bla() {}
|
||||
|
||||
get knownListings() {
|
||||
return this._db[this._name] || [];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Store;
|
||||
*/
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#neubauKompass testsuite()', () => {
|
||||
|
||||
const neubauKompass = proxyquire('../lib/provider/neubauKompass', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
it('should test neubauKompass provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
neubauKompass.run(mockStats).then(() => {
|
||||
const neubauKompassDbContent = neubauKompass._getStore()._db;
|
||||
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj.serviceName).to.equal('neubauKompass');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
expect(notify).to.be.a('object');
|
||||
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.neubaukompass.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
45
test/provider/einsAImmobilien.test.js
Normal file
45
test/provider/einsAImmobilien.test.js
Normal file
@@ -0,0 +1,45 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/einsAImmobilien');
|
||||
|
||||
describe('#einsAImmobilien testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.einsAImmobilien, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test einsAImmobilien provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immonetDbContent = fredy._getStore();
|
||||
expect(immonetDbContent.einsAImmobilien).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('einsAImmobilien');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immonetDbContent.einsAImmobilien[idx]);
|
||||
expect(notify.price).that.does.include('EUR');
|
||||
expect(notify.size).to.be.not.empty;
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
48
test/provider/immonet.test.js
Normal file
48
test/provider/immonet.test.js
Normal file
@@ -0,0 +1,48 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/immonet');
|
||||
|
||||
describe('#immonet testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.immonet, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test immonet provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immonetDbContent = fredy._getStore();
|
||||
|
||||
expect(immonetDbContent.immonet).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immonet');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immonetDbContent.immonet[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immonet.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
47
test/provider/immoscout.test.js
Normal file
47
test/provider/immoscout.test.js
Normal file
@@ -0,0 +1,47 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/immoscout');
|
||||
|
||||
describe('#immoscout testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.immoscout, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test immoscout provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immoscoutDbContent = fredy._getStore();
|
||||
expect(immoscoutDbContent.immoscout).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immoscout');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immoscoutDbContent.immoscout[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
expect(notify.size).that.does.include('m²');
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immobilienscout24.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
49
test/provider/immowelt.test.js
Normal file
49
test/provider/immowelt.test.js
Normal file
@@ -0,0 +1,49 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/immowelt');
|
||||
|
||||
describe('#immowelt testsuite()', () => {
|
||||
it('should test immowelt provider', async () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.immowelt, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const immoweltDbContent = fredy._getStore();
|
||||
expect(immoweltDbContent.immowelt).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('immowelt');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(immoweltDbContent.immowelt[idx]);
|
||||
expect(notify.price).that.does.include('€');
|
||||
if (notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).that.does.include('m²');
|
||||
}
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.immowelt.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
44
test/provider/kalaydo.test.js
Normal file
44
test/provider/kalaydo.test.js
Normal file
@@ -0,0 +1,44 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/kalaydo');
|
||||
|
||||
describe('#kalaydo testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.kalaydo, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test kalaydo provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const kalaydoDbContent = fredy._getStore();
|
||||
|
||||
expect(kalaydoDbContent.kalaydo).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('kalaydo');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(kalaydoDbContent.kalaydo[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.kalaydo.de');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
45
test/provider/kleinanzeigen.test.js
Normal file
45
test/provider/kleinanzeigen.test.js
Normal file
@@ -0,0 +1,45 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/kleinanzeigen');
|
||||
|
||||
describe('#kleinanzeigen testsuite()', () => {
|
||||
it('should test kleinanzeigen provider', async () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.kleinanzeigen, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const kleinanzeigenDbContent = fredy._getStore();
|
||||
expect(kleinanzeigenDbContent.kleinanzeigen).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj).to.be.a('object');
|
||||
expect(notificationObj.serviceName).to.equal('kleinanzeigen');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('number');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(kleinanzeigenDbContent.kleinanzeigen[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
44
test/provider/neubauKompass.test.js
Normal file
44
test/provider/neubauKompass.test.js
Normal file
@@ -0,0 +1,44 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/neubauKompass');
|
||||
|
||||
describe('#neubauKompass testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.neubauKompass, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test neubauKompass provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const neubauKompassDbContent = fredy._getStore();
|
||||
expect(neubauKompassDbContent.neubauKompass).to.be.a('array');
|
||||
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj.serviceName).to.equal('neubauKompass');
|
||||
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
expect(notify).to.be.a('object');
|
||||
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
expect(notify.address).to.be.a('string');
|
||||
|
||||
/** check the values if possible **/
|
||||
expect(notify.id).to.equal(neubauKompassDbContent.neubauKompass[idx]);
|
||||
expect(notify.title).to.be.not.empty;
|
||||
expect(notify.link).that.does.include('https://www.neubaukompass.de');
|
||||
expect(notify.address).to.be.not.empty;
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
const utils = require('../lib/utils');
|
||||
const utils = require('../../lib/utils');
|
||||
const assert = require('assert');
|
||||
|
||||
describe('utils', () => {
|
||||
39
test/provider/wgGesucht.test.js
Normal file
39
test/provider/wgGesucht.test.js
Normal file
@@ -0,0 +1,39 @@
|
||||
const mockNotification = require('../mocks/mockNotification');
|
||||
const mockConfig = require('../../conf/forTesting/config.multi.test');
|
||||
const mockStore = require('../mocks/mockStore');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
const provider = require('../../lib/provider/wgGesucht');
|
||||
|
||||
describe('#wgGesucht testsuite()', () => {
|
||||
provider.init(mockConfig.jobs.test1.provider.wgGesucht, [], []);
|
||||
const Fredy = proxyquire('../../lib/FredyRuntime', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
});
|
||||
|
||||
it('should test wgGesucht provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
const fredy = new Fredy(provider.config, null, provider.id(), 'test1');
|
||||
fredy.execute().then(() => {
|
||||
const wgGesuchtDbContent = fredy._getStore();
|
||||
expect(wgGesuchtDbContent.wgGesucht).to.be.a('array');
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj.serviceName).to.equal('wgGesucht');
|
||||
notificationObj.payload.forEach(notify => {
|
||||
expect(notify).to.be.a('object');
|
||||
|
||||
/** check the actual structure **/
|
||||
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.details).to.be.a('string');
|
||||
expect(notify.size).to.be.a('string');
|
||||
expect(notify.price).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,41 +0,0 @@
|
||||
const mockNotification = require('./mocks/mockNotification');
|
||||
const mockConfig = require('../conf/config.test');
|
||||
const mockStore = require('./mocks/mockStore');
|
||||
const mockStats = require('./mocks/mockStats');
|
||||
const proxyquire = require('proxyquire').noCallThru();
|
||||
const expect = require('chai').expect;
|
||||
|
||||
describe('#wgGesucht testsuite()', () => {
|
||||
|
||||
const wgGesucht = proxyquire('../lib/provider/wgGesucht', {
|
||||
'../../conf/config.json': mockConfig,
|
||||
'../lib/fredy': proxyquire('../lib/fredy', {
|
||||
'./services/store': mockStore,
|
||||
'./notification/notify': mockNotification
|
||||
})
|
||||
});
|
||||
|
||||
it('should test wgGesucht provider', async () => {
|
||||
return await new Promise(resolve => {
|
||||
wgGesucht.run(mockStats).then(() => {
|
||||
const wgGesuchtDbContent = wgGesucht._getStore()._db;
|
||||
expect(wgGesuchtDbContent.wgGesucht).to.be.a('array');
|
||||
const notificationObj = mockNotification.get();
|
||||
expect(notificationObj.serviceName).to.equal('wgGesucht');
|
||||
notificationObj.payload.forEach((notify, idx) => {
|
||||
expect(notify).to.be.a('object');
|
||||
|
||||
/** check the actual structure **/
|
||||
|
||||
expect(notify.id).to.be.a('string');
|
||||
expect(notify.title).to.be.a('string');
|
||||
expect(notify.details).to.be.a('string');
|
||||
expect(notify.description).to.be.a('string');
|
||||
expect(notify.link).to.be.a('string');
|
||||
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user