upgrading dependencies

Modernizing ui

.dockerignore

@@ -1,3 +1,7 @@
-node_modules
+node_modules/
 npm-debug.log
-test
+test/
+conf/
+db/
+.git/
+.github/

.eslintrc.cjs

@@ -1,21 +1,21 @@
 module.exports = {
   env: {
-    commonjs: true,
+    es2021: true,
-    es6: true,
     node: true,
     browser: true,
+    mocha: true,
   },
-  parser: 'babel-eslint',
+  parser: '@babel/eslint-parser',
-  extends: ['eslint:recommended', 'prettier', 'prettier/react'],
+  extends: ['eslint:recommended', 'prettier'],
   plugins: ['react'],
   globals: {
     Promise: false,
     describe: true,
+    after: true,
     it: true,
     fetch: true,
   },
   parserOptions: {
-    ecmaVersion: 2018,
     sourceType: 'module',
   },
   rules: {
@@ -203,10 +203,6 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
     'react/self-closing-comp': 'warn',
 
-    // Enforce spaces before the closing bracket of self-closing JSX elements
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
-    'react/jsx-space-before-closing': ['warn', 'always'],
-
     // Enforce component methods order
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
     'react/sort-comp': 'off',
@@ -237,7 +233,7 @@ module.exports = {
 
     // only .jsx files may have JSX
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
-    'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
+    'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
 
     // prevent accidental JS comments from being injected into JSX as text
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
@@ -282,15 +278,5 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
     'react/no-children-prop': 'warn',
 
-    // Validate whitespace in and around the JSX opening and closing brackets
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
-    'react/jsx-tag-spacing': [
-      'warn',
-      {
-        closingSlash: 'never',
-        beforeSelfClosing: 'always',
-        afterOpening: 'never',
-      },
-    ],
   },
 };

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/docker.yml

@@ -0,0 +1,46 @@
+name: Create and publish Docker image
+on:
+  push:
+    branches:
+      - 'master'
+    tags:
+      - '*'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setup node
+        uses: actions/setup-node@v2.5.1
+        with:
+          node-version: 16
+          cache: 'yarn'
+      - run: yarn install
+      - run: yarn run test

.gitignore

@@ -3,3 +3,4 @@ ui/public/
 db/
 npm-debug.log
 .DS_Store
+.idea

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

CHANGELOG.md

@@ -1,3 +1,66 @@
+Newer release changelog see https://github.com/orangecoding/fredy/releases
+
+------------
+
+###### [V5.5.0]
+- Upgrading dependencies
+- fixing provider
+- allow multiple instances of 1 provider  
+- __BREAKING__: Minimum node version is now 16
+
+###### [V5.4.6]
+- Adding Instana node.js monitoring
+- 
+###### [V5.4.5]
+- Adding Instana node.js monitoring 
+
+###### [V5.4.4]
+- Add support for Immo Südwest Presse (immo.swp.de)
+- Telegram: Use job name instead of ID and link in title
+- Fix race condition if user ID is in session but not in user store
+- Allow visiting the original provider URL
+
+###### [V5.4.3]
+- re-writing readme
+- improving docker build
+- using github's actions to build docker and test automatically
+
+###### [V5.4.2]
+- Fixing prod build
+
+###### [V5.4.1]
+- Upgrading dependencies
+- Provider urls are now automagically been changed to include the correct sort order for search results
+
+```
+Note: It has been an point of confusion since the very beginning of Fredy, that people simply copied the url, but
+did not take care of sorting the search results by date. If this is not done, Fredy will most likely not see the latest
+results, thus cannot report them. This release fixes it by adding the necessary params (or replaces them).
+```
+
+###### [V5.3.0]
+- Upgrading dependencies
+- It's now possible to send mails to multiple receiver using comma separation for MailJet & Sendgrid
+- Fixing Immowelt scraping
+
+###### [V5.2.0]
+- Upgrading dependencies
+- Adding new similarity check layer (Duplicates are being removed now)
+- Adding paging for search results
+
+###### [V5.1.0]
+- Upgrading dependencies
+- NodeJS 12.13 is now the minimum supported version
+- Adding general settings as new configuration page to ui
+- Adding new feature working hours
+
+###### [V5.0.0]
+- Upgrading dependencies
+- NodeJS 12 is now the minimum supported version
+
+###### [V4.0.0]
+Bringing back Immoscout :tada:
+
 ###### [V3.0.0]
 This is basically a re-write, your old config file will not be compatible anymore. Please re-created your search jobs
 on the new ui and use the values from your previous config file if needed.
@@ -12,4 +75,4 @@ on the new ui and use the values from your previous config file if needed.
 [BREAKING CHANGES]
 - The config has been changed, the config of V1.x will not work any longer
 - Sources have been renamed to provider
-```
+```

CONTRIBUTING.md

@@ -17,7 +17,7 @@ function normalize(o) {
   return Object.assign(o, { id });
 }
 
-//apply blaclist if needed
+//apply blacklist if needed
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);

Dockerfile

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1.3
+FROM node:16-alpine AS builder
+COPY --chown=1000:1000 . /fredy
+WORKDIR  /fredy
+USER 1000
+RUN yarn install
+RUN yarn run prod
+
+FROM node:16-alpine
+COPY --from=builder --chown=1000:1000 /fredy /fredy
+RUN mkdir /db /conf && \
+  chown 1000:1000 /db /conf && \
+  ln -s /db /fredy/db && ln -s /conf /fredy/conf
+EXPOSE 9998
+USER 1000
+VOLUME [ "/conf", "/db" ]
+WORKDIR  /fredy
+CMD node index.js --no-daemon

README.md

@@ -1,55 +1,79 @@
 <img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">  
 
-[![Build Status](https://travis-ci.org/orangecoding/fredy.svg?branch=master)](https://travis-ci.org/orangecoding/fredy)
+![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
 
-_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) as often as you want and send new listings to you once they appear. The list of available services can easily be extended. For your convenience, a ui helps you to configure your search jobs.   
+Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
 
-If _Fredy_ found matching results, it will send them to you via Slack, Email, Telegram etc. (More adapter possible.) As _Fredy_ will store the listings it found, new results will not be sent twice (and as a side-effect, _Fredy_ can show some statistics..)  
+_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings to you once they become available. The list of available services can easily be extended. For your convenience, _Fredy_ has a UI to help you configure your search jobs.   
+
+If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
+
+# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
+If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
+
+<img src="https://github.com/orangecoding/fredy/blob/master/doc/jetbrains.png" width="200">   
+
+_Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use NodeJs 12 and above
+- Make sure to use Node.js 16 or above
-- Run the following commands
+- Run the following commands:
 ```ssh
 yarn (or npm install)
 yarn run prod
 yarn run start
 ```
-_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` for username and password. (You should change the password asap when you plan to run Fredy on your server.)
+_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
+
+<p align="center">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot1.png" width="30%">
+&nbsp; &nbsp; &nbsp; &nbsp;
+  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_2.png" width="30%">
+&nbsp; &nbsp; &nbsp; &nbsp;  
+  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
+</p>
+<p align="center">
+
+</p>
 
 ## Understanding the fundamentals
 There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
 
-#### Adapter
-_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few. Those services are called adapter within _Fredy_. When creating a new job, you can choose 1 or many adapter.    
-An adapter holds the url that points to the search results for the service. If you go to immonet.de and search for something, the shown url in the browser is what the adapter needs to do it's magic.   
-**It is important that you order the search results by date, so that _Fredy_ always picks the latest ones first**
-
 #### Provider
-_Fredy_ supports multiple provider. Slack, SendGrid, Telegram etc. A search job can have as many provider as supported by _Fredy_. Each provider needs different configuration values, which you have to provide when using it. A provider itself dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few examples. Those services are called providers within _Fredy_. When creating a new job, you can choose one or more providers.
+A provider contains the URL that points to the search results for the respective service. If you go to immonet.de and search for something, the displayed URL in the browser is what the provider needs to do its magic.
+**It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
+
+#### Adapter
+_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
 
 #### Jobs
-A Job wraps adapter and provider. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
+A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
 
 ## Creating your first job
-To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self explanatory, however there's one important thing.
+To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self-explanatory, however there is one important thing.
-When configuring adapter, before copying the url from your browser make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
+When configuring providers, before copying the URL from your browser, make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
 
 ## User management
-As an administrator, you can create/edit and remove user from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, the jobs will also be removed.
+As an administrator, you can create, edit and remove users from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, their jobs will also be removed.
-
 
 # Development
 
-### Running Fredy in dev mode
+### Running Fredy in development mode
-To run _Fredy_ in dev mode, you need to run the backend & frontend separately. Run the backend in your favorite IDE, the frontend can be started from the terminal.
+To run _Fredy_ in development mode, you need to run the backend & frontend separately.
+Start the backend with:
+```shell
+yarn run start
+```
+For the frontend, run:
 ```shell
 yarn run dev
 ```
-You should now be able to access _Fredy_ with your browser. Go to `http://localhost:9000`
+You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
 
 ### Running Tests
-To run the tests, simply run
+To run the tests, run
 ```shell
 yarn run test
 ```
@@ -57,15 +81,20 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-## Why is Immoscout missing
+### Immoscout / Immonet
-Immoscout decided to add "robot protection" to their service. Meaning if Fredy tries to check for listings, it will be recognized as a bot. I haven't found a way around it (yet) ;)
+I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
-#### Contribution guidelines  
+To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
+The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)
 
-See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)  
+If you need more than the 1000 API calls allowed per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefore they have decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (Disclaimer: I do not earn any money for recommending their service).
+
+### Contribution guidelines  
+
+See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
 # Docker   
-Use the Dockerfile in this Repository to build an image.  
+Use the Dockerfile in this repository to build an image.  
 
 Example: `docker build -t fredy/fredy /path/to/your/Dockerfile`  
 
@@ -73,12 +102,16 @@ Or use docker-compose:
 
 Example `docker-compose build`
 
+Or use the container that will be built automatically.
+
+`docker pull ghcr.io/orangecoding/fredy:master`
+
 ## Create & run a container  
 
-Put your config.json to `/path/to/your/conf/`
+Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
 ## Logs  
 
-You can browse the logs with  `docker logs fredy -f`  
+You can browse the logs with  `docker logs fredy -f`.

build/Dockerfile

@@ -1,33 +0,0 @@
-FROM alpine:latest AS build
-# use given repository, default below:
-ARG repo=https://github.com/orangecoding/fredy.git
-
-RUN mkdir -p /usr/src/
-#Install Software
-RUN apk add --update nodejs npm git
-
-# Output used repository
-RUN echo "Cloning from $repo"
-
-RUN cd /usr/src && git clone $repo
-
-RUN ln -s /usr/src/fredy/conf/ /conf
-
-# create db folder
-RUN mkdir /usr/src/fredy/db/
-
-RUN ln -s /usr/src/fredy/db/ /db
-
-RUN npm i -g yarn
-
-RUN cd /usr/src/fredy/ && yarn
-
-WORKDIR  /usr/src/fredy
-
-RUN yarn run prod
-
-EXPOSE 9998
-
-VOLUME [ "/conf", "/db" ]
-# --no-daemon is required for keeping Container alive
-CMD node index.js --no-daemon

conf/config.json

@@ -1,4 +1 @@
-{
+{"interval":"60","port":9998,"scrapingAnt":{"apiKey":"","proxy":"datacenter"},"workingHours":{"from":"","to":""}}
-  "interval": 30,
-  "port": 9998
-}

doc/Untitled Diagram.drawio

@@ -0,0 +1,84 @@
+<mxfile host="app.diagrams.net" modified="2022-01-29T18:34:51.211Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" etag="W0jmvptvMSkuHq89hwUy" version="16.5.2" type="github">
+  <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
+    <mxGraphModel dx="850" dy="907" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-3" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="Job1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="100" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-2">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-3">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-4">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-7" value="FredyRuntime" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="110" y="120" width="360" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-0" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-0" value="Job2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-7" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-1" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-1" value="Job3" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-2" target="4kAlOAlRylSy7JMoHAEd-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-2" value="Provider1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="100" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-3">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="290" y="290" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-3" value="Provider2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-4" target="4kAlOAlRylSy7JMoHAEd-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-4" value="Provider3" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-12" target="4kAlOAlRylSy7JMoHAEd-16">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-12" value="Similarity check" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="110" y="290" width="360" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-16" target="4kAlOAlRylSy7JMoHAEd-18">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-16" value="Found similarity" style="rhombus;whiteSpace=wrap;html=1;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="250" y="360" width="80" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-18" target="4kAlOAlRylSy7JMoHAEd-19">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-18" value="Notification Adapter1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="460" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-19" value="Notification Adapter2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="520" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-22" value="No" style="text;html=1;resizable=0;autosize=1;align=center;verticalAlign=middle;points=[];fillColor=none;strokeColor=none;rounded=0;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="300" y="440" width="30" height="20" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

docker-compose.yml

@@ -4,10 +4,8 @@ services:
     container_name: fredy
     # build from empty build folder to reduce size of image
     build:
-      context: ./build
+      context: .
       dockerfile: Dockerfile
-      args:
-        repo: https://github.com/orangecoding/fredy.git
     image: fredy/fredy
     # map existing config and database
     volumes:

index.html

@@ -4,14 +4,13 @@
     <meta charset="UTF-8"
           name="viewport"
           content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
-    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css">
     <meta name="google" content="notranslate">
 
     <title>Fredy</title>
 </head>
-<body>
+<body theme-mode="dark">
 
 <div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
 </body>
-<script src="fredy.bundle.js"></script>
+<script type="module" src="/ui/src/Index.jsx"></script>
 </html>

index.js

@@ -1,53 +1,49 @@
-const fs = require('fs');
+import fs from 'fs';
-
+import { config } from './lib/utils.js';
+import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
+import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
+import * as jobStorage from './lib/services/storage/jobStorage.js';
+import FredyRuntime from './lib/FredyRuntime.js';
+import { duringWorkingHoursOrNotSet } from './lib/utils.js';
+import './lib/api/api.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
 }
-
 const path = './lib/provider';
 const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
-const config = require('./conf/config.json');
-
-const jobStorage = require('./lib/services/storage/jobStorage');
-const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
-const FredyRuntime = require('./lib/FredyRuntime');
-
-//starting the api service
-require('./lib/api/api');
-
 //assuming interval is always in minutes
 const INTERVAL = config.interval * 60 * 1000;
-
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
 /* eslint-enable no-console */
+const fetchedProvider = await Promise.all(
+  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
+);
+
 setInterval(
   (function exec() {
-    jobStorage
+    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-      .getJobs()
+    if (isDuringWorkingHoursOrNotSet) {
-      .filter((job) => job.enabled)
+      config.lastRun = Date.now();
-      .forEach((job) => {
+      jobStorage
-        const providerIds = job.provider.map((provider) => provider.id);
+        .getJobs()
-
+        .filter((job) => job.enabled)
-        provider
+        .forEach((job) => {
-          .filter((provider) => provider.endsWith('.js'))
+          job.provider
-          .map((pro) => require(`${path}/${pro}`))
+            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-          .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
+            .forEach(async (prov) => {
-          .forEach(async (pro) => {
+              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-            const providerId = pro.metaInformation.id;
+              pro.init(prov, job.blacklist);
-            if (providerId == null || providerId.length === 0) {
+              await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
-              throw new Error('Provider id must not be empty. => ' + pro);
+              setLastJobExecution(job.id);
-            }
+            });
-            const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
+        });
-            if (providerConfig == null) {
+    } else {
-              throw new Error(`Provider Config for provider with id ${providerId} not found.`);
+      /* eslint-disable no-console */
-            }
+      console.debug('Working hours set. Skipping as outside of working hours.');
-            pro.init(providerConfig, job.blacklist);
+      /* eslint-enable no-console */
-            await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute();
+    }
-            setLastJobExecution(job.id);
-          });
-      });
     return exec;
   })(),
   INTERVAL

lib/FredyRuntime.js

@@ -1,9 +1,9 @@
-const { NoNewListingsError } = require('./errors');
+import { NoNewListingsWarning } from './errors.js';
-const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
+import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
-
+import * as notify from './notification/notify.js';
-const notify = require('./notification/notify');
+import xray from './services/scraper.js';
-const xray = require('./services/scraper');
+import * as scrapingAnt from './services/scrapingAnt.js';
-
+import urlModifier from './services/queryStringMutator.js';
 class FredyRuntime {
   /**
    *
@@ -11,17 +11,19 @@ class FredyRuntime {
    * @param notificationConfig the config for all notifications
    * @param providerId the id of the provider currently in use
    * @param jobKey key of the job that is currently running (from within the config)
+   * @param similarityCache cache instance holding values to check for similarity of entries
    */
-  constructor(providerConfig, notificationConfig, providerId, jobKey) {
+  constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
     this._providerConfig = providerConfig;
     this._notificationConfig = notificationConfig;
     this._providerId = providerId;
     this._jobKey = jobKey;
+    this._similarityCache = similarityCache;
   }
-
   execute() {
     return (
-      Promise.resolve(this._providerConfig.url)
+      //modify the url to make sure search order is correctly set
+      Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
         //scraping the site and try finding new listings
         .then(this._getListings.bind(this))
         //bring them in a proper form (dictated by the provider)
@@ -32,50 +34,75 @@ class FredyRuntime {
         .then(this._findNew.bind(this))
         //store everything in db
         .then(this._save.bind(this))
+        //check for similar listings. if found, remove them before notifying
+        .then(this._filterBySimilarListings.bind(this))
         //notify the user using the configured notification adapter
         .then(this._notify.bind(this))
         //if an error occurred on the way, handle it here.
         .catch(this._handleError.bind(this))
     );
   }
-
   _getListings(url) {
     return new Promise((resolve, reject) => {
-      let x = xray(url, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]);
+      const id = this._providerId;
-
+      if (scrapingAnt.needScrapingAnt(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
-      x((err, listings) => {
+        const error = 'Immoscout or Immonet can only be used with if you have set an apikey for scrapingAnt.';
-        if (err) {
+        /* eslint-disable no-console */
-          reject(err);
+        console.log(error);
+        /* eslint-enable no-console */
+        reject(error);
+        return;
+      }
+      const u = scrapingAnt.needScrapingAnt(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
+      try {
+        if (this._providerConfig.paginate != null) {
+          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
+            //the first 2 pages should be enough here
+            .limit(2)
+            .paginate(this._providerConfig.paginate)
+            .then((listings) => {
+              resolve(listings == null ? [] : listings);
+            })
+            .catch((err) => {
+              reject(err);
+              console.error(err);
+            });
         } else {
-          resolve(listings);
+          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
+            .then((listings) => {
+              resolve(listings == null ? [] : listings);
+            })
+            .catch((err) => {
+              reject(err);
+              console.error(err);
+            });
         }
-      });
+      } catch (error) {
+        reject(error);
+        console.error(error);
+      }
     });
   }
-
   _normalize(listings) {
     return listings.map(this._providerConfig.normalize);
   }
-
   _filter(listings) {
     return listings.filter(this._providerConfig.filter);
   }
-
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
-
     if (newListings.length === 0) {
-      throw new NoNewListingsError();
+      throw new NoNewListingsWarning();
     }
-
     return newListings;
   }
-
   _notify(newListings) {
+    if (newListings.length === 0) {
+      throw new NoNewListingsWarning();
+    }
     const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
     return Promise.all(sendNotifications).then(() => newListings);
   }
-
   _save(newListings) {
     const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
     newListings.forEach((listing) => {
@@ -84,10 +111,21 @@ class FredyRuntime {
     setKnownListings(this._jobKey, this._providerId, currentListings);
     return newListings;
   }
-
+  _filterBySimilarListings(listings) {
+    const filteredList = listings.filter((listing) => {
+      const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
+      if (similar) {
+        /* eslint-disable no-console */
+        console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
+        /* eslint-enable no-console */
+      }
+      return !similar;
+    });
+    filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
+    return filteredList;
+  }
   _handleError(err) {
-    if (err.name !== 'NoNewListingsError') console.error(err);
+    if (err.name !== 'NoNewListingsWarning') console.error(err);
   }
 }
-
+export default FredyRuntime;
-module.exports = FredyRuntime;

lib/api/api.js

@@ -1,42 +1,36 @@
-const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
+import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
-const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
+import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
-const { analyticsRouter } = require('./routes/analyticsRouter');
+import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
-const { providerRouter } = require('./routes/providerRouter');
+import { analyticsRouter } from './routes/analyticsRouter.js';
-const { loginRouter } = require('./routes/loginRoute');
+import { providerRouter } from './routes/providerRouter.js';
-const config = require('../../conf/config.json');
+import { loginRouter } from './routes/loginRoute.js';
-const { userRouter } = require('./routes/userRoute');
+import { config } from '../utils.js';
-const { jobRouter } = require('./routes/jobRouter');
+import { userRouter } from './routes/userRoute.js';
-const bodyParser = require('body-parser');
+import { jobRouter } from './routes/jobRouter.js';
-const service = require('restana')();
+import bodyParser from 'body-parser';
-const files = require('serve-static');
+import restana from 'restana';
-const path = require('path');
+import files from 'serve-static';
-
+import path from 'path';
-const staticService = files(path.join(__dirname, '../../ui/public'));
+import { getDirName } from '../utils.js';
-
+const service = restana();
+const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
 
 service.use(bodyParser.json());
-
 service.use(cookieSession());
-
 service.use(staticService);
-
 service.use('/api/admin', authInterceptor());
 service.use('/api/jobs', authInterceptor());
-
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
-
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
+service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
 service.use('/api/jobs', jobRouter);
-
 service.use('/api/login', loginRouter);
-
 /* eslint-disable no-console */
 service.start(PORT).then(() => {
   console.info(`Started API service on port ${PORT}`);
 });
-/* eslint-enable no-console */

lib/api/routes/analyticsRouter.js

@@ -1,12 +1,10 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as listingStorage from '../../services/storage/listingsStorage.js';
+const service = restana();
 const analyticsRouter = service.newRouter();
-const listingStorage = require('../../services/storage/listingsStorage');
-
 analyticsRouter.get('/:jobId', async (req, res) => {
   const { jobId } = req.params;
-
   res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
   res.send();
 });
-
+export { analyticsRouter };
-exports.analyticsRouter = analyticsRouter;

lib/api/routes/generalSettingsRoute.js

@@ -0,0 +1,21 @@
+import restana from 'restana';
+import { config, getDirName } from '../../utils.js';
+import fs from 'fs';
+const service = restana();
+const generalSettingsRouter = service.newRouter();
+generalSettingsRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, config);
+  res.send();
+});
+generalSettingsRouter.post('/', async (req, res) => {
+  const settings = req.body;
+  try {
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify(settings));
+  } catch (err) {
+    console.error(err);
+    res.send(new Error('Error while trying to write settings.'));
+    return;
+  }
+  res.send();
+});
+export { generalSettingsRouter };

lib/api/routes/jobRouter.js

@@ -1,9 +1,12 @@
-const service = require('restana')();
+import restana from 'restana';
+import fetch from 'node-fetch';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as immoscoutProvider from '../../provider/immoscout.js';
+import { config } from '../../utils.js';
+import { isAdmin } from '../security.js';
+const service = restana();
 const jobRouter = service.newRouter();
-const jobStorage = require('../../services/storage/jobStorage');
-const userStorage = require('../../services/storage/userStorage');
-const { isAdmin } = require('../security');
-
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -13,21 +16,42 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-
   return user.isAdmin || job.userId === job.userId;
 }
-
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
-
   //show only the jobs which belongs to the user (or all of the user is an admin)
   res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
-
   res.send();
 });
-
+jobRouter.get('/processingTimes', async (req, res) => {
+  let scrapingAntData = null;
+  if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
+    try {
+      const response = await fetch(`https://api.scrapingant.com/v2/usage?x-api-key=${config.scrapingAnt.apiKey}`);
+      scrapingAntData = await response.json();
+    } catch (Exception) {
+      console.error('Could not query plan data from scraping ant.', Exception);
+    }
+  }
+  res.body = {
+    interval: config.interval,
+    lastRun: config.lastRun || null,
+    scrapingAntData,
+  };
+  res.send();
+});
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
+  if (
+    provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
+    (config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
+  ) {
+    res.send(
+      new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
+    );
+    return;
+  }
   try {
     jobStorage.upsertJob({
       userId: req.session.currentUser,
@@ -44,7 +68,6 @@ jobRouter.post('/', async (req, res) => {
   }
   res.send();
 });
-
 jobRouter.delete('', async (req, res) => {
   const { jobId } = req.body;
   try {
@@ -60,7 +83,6 @@ jobRouter.delete('', async (req, res) => {
   }
   res.send();
 });
-
 jobRouter.put('/:jobId/status', async (req, res) => {
   const { status } = req.body;
   const { jobId } = req.params;
@@ -80,5 +102,4 @@ jobRouter.put('/:jobId/status', async (req, res) => {
   }
   res.send();
 });
-
+export { jobRouter };
-exports.jobRouter = jobRouter;

lib/api/routes/loginRoute.js

@@ -1,32 +1,28 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as hasher from '../../services/security/hash.js';
+const service = restana();
 const loginRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const hasher = require('../../services/security/hash');
-
 loginRouter.get('/user', async (req, res) => {
   const currentUserId = req.session.currentUser;
-  const isAdmin = currentUserId == null ? false : userStorage.getUser(currentUserId).isAdmin;
+  const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
-  if (currentUserId == null) {
+  if (currentUser == null) {
     res.body = {};
   } else {
     res.body = {
-      userId: currentUserId,
+      userId: currentUser.id,
-      isAdmin,
+      isAdmin: currentUser.isAdmin,
     };
   }
   res.send();
 });
-
 loginRouter.post('/', async (req, res) => {
   const { username, password } = req.body;
-
   const user = userStorage.getUsers(true).find((user) => user.username === username);
-
   if (user == null) {
     res.send(401);
     return;
   }
-
   if (user.password === hasher.hash(password)) {
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
@@ -35,13 +31,10 @@ loginRouter.post('/', async (req, res) => {
   } else {
     console.error(`User ${username} tried to login, but password was wrong.`);
   }
-
   res.send(401);
 });
-
 loginRouter.post('/logout', async (req, res) => {
   req.session = null;
   res.send(200);
 });
-
+export { loginRouter };
-exports.loginRouter = loginRouter;

lib/api/routes/notificationAdapterRouter.js

@@ -1,13 +1,13 @@
-const fs = require('fs');
+import fs from 'fs';
-const service = require('restana')();
+import restana from 'restana';
+const service = restana();
 const notificationAdapterRouter = service.newRouter();
-
 const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
-
+const notificationAdapter = await Promise.all(
-const notificationAdapter = notificationAdapterList.map((pro) => {
+  notificationAdapterList.map(async (pro) => {
-  return require(`../../notification/adapter/${pro}`);
+    return await import(`../../notification/adapter/${pro}`);
-});
+  })
-
+);
 notificationAdapterRouter.post('/try', async (req, res) => {
   const { id, fields } = req.body;
   const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
@@ -24,7 +24,6 @@ notificationAdapterRouter.post('/try', async (req, res) => {
     enabled: true,
     id,
   });
-
   try {
     await adapter.send({
       serviceName: 'TestCall',
@@ -40,16 +39,13 @@ notificationAdapterRouter.post('/try', async (req, res) => {
       notificationConfig,
       jobKey: 'TestJob',
     });
-
     res.send();
   } catch (Exception) {
     res.send(new Error(Exception));
   }
 });
-
 notificationAdapterRouter.get('/', async (req, res) => {
   res.body = notificationAdapter.map((adapter) => adapter.config);
   res.send();
 });
-
+export { notificationAdapterRouter };
-exports.notificationAdapterRouter = notificationAdapterRouter;

lib/api/routes/providerRouter.js

@@ -1,16 +1,15 @@
-const fs = require('fs');
+import fs from 'fs';
-const service = require('restana')();
+import restana from 'restana';
+const service = restana();
 const providerRouter = service.newRouter();
-
 const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
-
+const provider = await Promise.all(
-const provider = providerList.map((pro) => {
+  providerList.map(async (pro) => {
-  return require(`../../provider/${pro}`).metaInformation;
+    return await import(`../../provider/${pro}`);
-});
+  })
-
+);
 providerRouter.get('/', async (req, res) => {
-  res.body = provider;
+  res.body = provider.map((p) => p.metaInformation);
   res.send();
 });
-
+export { providerRouter };
-exports.providerRouter = providerRouter;

lib/api/routes/userRoute.js

@@ -1,33 +1,27 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+const service = restana();
 const userRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const jobStorage = require('../../services/storage/jobStorage');
-
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
   return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
 }
-
 function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
   return req.session.currentUser === userIdToBeRemoved;
 }
-
 const nullOrEmpty = (str) => str == null || str.length === 0;
-
 userRouter.get('/', async (req, res) => {
   res.body = userStorage.getUsers(false);
   res.send();
 });
-
 userRouter.get('/:userId', async (req, res) => {
   const { userId } = req.params;
   res.body = userStorage.getUser(userId);
   res.send();
 });
-
 userRouter.delete('/', async (req, res) => {
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
-
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
     return;
@@ -36,14 +30,11 @@ userRouter.delete('/', async (req, res) => {
     res.send(new Error('You are trying to remove yourself. This is prohibited.'));
     return;
   }
-
   //TODO: Remove also analytics
   jobStorage.removeJobsByUserId(userId);
   userStorage.removeUser(userId);
-
   res.send();
 });
-
 userRouter.post('/', async (req, res) => {
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
@@ -55,22 +46,18 @@ userRouter.post('/', async (req, res) => {
     return;
   }
   const allUser = userStorage.getUsers(false);
-
   if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(
       new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
     );
     return;
   }
-
   userStorage.upsertUser({
     userId,
     username,
     password,
     isAdmin,
   });
-
   res.send();
 });
-
+export { userRouter };
-exports.userRouter = userRouter;

lib/api/security.js

@@ -1,15 +1,12 @@
-const userStorage = require('../services/storage/userStorage');
+import * as userStorage from '../services/storage/userStorage.js';
-const cookieSession = require('cookie-session');
+import cookieSession from 'cookie-session';
-const { nanoid } = require('nanoid');
+import { nanoid } from 'nanoid';
-
 const unauthorized = (res) => {
   return res.send(401);
 };
-
 const isUnauthorized = (req) => {
   return req.session.currentUser == null;
 };
-
 const isAdmin = (req) => {
   if (!isUnauthorized(req)) {
     const user = userStorage.getUser(req.session.currentUser);
@@ -17,7 +14,6 @@ const isAdmin = (req) => {
   }
   return false;
 };
-
 const authInterceptor = () => {
   return (req, res, next) => {
     if (isUnauthorized(req)) {
@@ -27,7 +23,6 @@ const authInterceptor = () => {
     }
   };
 };
-
 const adminInterceptor = () => {
   return (req, res, next) => {
     if (!isAdmin(req)) {
@@ -37,8 +32,7 @@ const adminInterceptor = () => {
     }
   };
 };
-
+const cookieSession$0 = (userId) => {
-exports.cookieSession = (userId) => {
   return cookieSession({
     name: 'fredy-admin-session',
     keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
@@ -46,8 +40,8 @@ exports.cookieSession = (userId) => {
     maxAge: 8 * 60 * 60 * 1000, // 8 hours
   });
 };
-
+export { cookieSession$0 as cookieSession };
-exports.adminInterceptor = adminInterceptor;
+export { adminInterceptor };
-exports.authInterceptor = authInterceptor;
+export { authInterceptor };
-exports.isUnauthorized = isUnauthorized;
+export { isUnauthorized };
-exports.isAdmin = isAdmin;
+export { isAdmin };

lib/errors.js

@@ -9,7 +9,8 @@ class ExtendableError extends Error {
     }
   }
 }
-
+class NoNewListingsWarning extends ExtendableError {}
-class NoNewListingsError extends ExtendableError {}
+export { NoNewListingsWarning };
-
+export default {
-module.exports = { NoNewListingsError };
+  NoNewListingsWarning,
+};

lib/notification/adapter/console.js

@@ -1,19 +1,12 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
 
-/**
+export const send = ({ serviceName, newListings, jobKey }) => {
- * simply prints out the found data to the console
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
   /* eslint-disable no-console */
   return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
   /* eslint-enable no-console */
 };
-
+export const config = {
-exports.config = {
+  id: 'console',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Console',
   description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
   config: {},

lib/notification/adapter/mailJet.js

@@ -1,28 +1,24 @@
-const mailjet = require('node-mailjet');
+import mailjet from 'node-mailjet';
-
+import path from 'path';
-const path = require('path');
+import fs from 'fs';
-const fs = require('fs');
+import Handlebars from 'handlebars';
-const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/template.hbs'), 'utf8');
+import { markdown2Html } from '../../services/markdown.js';
-
+import { getDirName } from '../../utils.js';
-const Handlebars = require('handlebars');
+const __dirname = getDirName();
+const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
 const emailTemplate = Handlebars.compile(template);
-const { markdown2Html } = require('../../services/markdown');
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-
-/**
- * sends a new listing using MailJet
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailJet'
+    (adapter) => adapter.id === 'mailjet'
   ).fields;
-
+  const to = receiver
+    .trim()
+    .split(',')
+    .map((r) => ({
+      Email: r.trim(),
+    }));
   return mailjet
-    .connect(apiPublicKey, apiPrivateKey)
+    .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [
@@ -31,11 +27,7 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
             Email: from,
             Name: 'Fredy',
           },
-          To: [
+          To: to,
-            {
-              Email: receiver,
-            },
-          ],
           Subject: `Fredy found ${newListings.length} new listings for ${serviceName}`,
           HTMLPart: emailTemplate({
             serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
@@ -46,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
       ],
     });
 };
-
+export const config = {
-exports.config = {
+  id: 'mailjet',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'MailJet',
   description: 'MailJet is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/mailJet.md'),

lib/notification/adapter/mailJet.md

@@ -4,3 +4,5 @@ To use [MailJet](https://mailjet.com), you need to create an account. You'll nee
  
 E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.     
 The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
+
+If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com). 

lib/notification/adapter/mattermost.js

@@ -0,0 +1,39 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
+  message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
+  message += newListings.map(
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+  );
+  return fetch(webhook, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: {
+      channel: channel,
+      text: message,
+    },
+  });
+};
+export const config = {
+  id: 'mattermost',
+  name: 'Mattermost',
+  readme: markdown2Html('lib/notification/adapter/mattermost.md'),
+  description: 'Fredy will send new listings to your mattermost team chat.',
+  fields: {
+    webhook: {
+      type: 'text',
+      label: 'Webhook-URL',
+      description: 'The incoming webhook url',
+    },
+    channel: {
+      type: 'text',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/mattermost.md

@@ -0,0 +1,5 @@
+### Mattermost Adapter
+
+For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the  [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+
+As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.

lib/notification/adapter/ntfy.js

@@ -0,0 +1,51 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const message = `Address: ${newListing.address} Size: ${newListing.size.replace(/2m/g, '$m^2$')} Price: ${
+      newListing.price
+    }`;
+    return fetch(server, {
+      method: 'POST',
+      body: JSON.stringify({
+        topic: topic,
+        message: message,
+        title: newListing.title,
+        tags: [serviceName, jobName],
+        priority: parseInt(priority),
+        click: newListing.link,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'ntfy',
+  name: 'ntfy',
+  readme: markdown2Html('lib/notification/adapter/ntfy.md'),
+  description: 'Fredy will send new listings to your ntfy.',
+  fields: {
+    priority: {
+      type: 'number',
+      label: 'Priority',
+      description: 'The priority of the send notification.',
+    },
+    server: {
+      type: 'text',
+      label: 'Server-URL',
+      description: 'The server url to the send the notification to.',
+    },
+    topic: {
+      type: 'text',
+      label: 'topic',
+      description:
+        'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
+    },
+  },
+};

lib/notification/adapter/ntfy.md

@@ -0,0 +1,5 @@
+### ntfy Adapter
+
+For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the  [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+
+As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.

lib/notification/adapter/sendGrid.js

@@ -1,20 +1,14 @@
-const sgMail = require('@sendgrid/mail');
+import sgMail from '@sendgrid/mail';
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
-
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-/**
- * sends a new listing using SendGrid
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,
-    to: receiver,
+    to: receiver
+      .trim()
+      .split(',')
+      .map((r) => r.trim()),
     from,
     subject: `Job ${jobKey} | Service ${serviceName} found ${newListings.length} new listing(s)`,
     dynamic_template_data: {
@@ -25,9 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   };
   return sgMail.send(msg);
 };
-
+export const config = {
-exports.config = {
+  id: 'sendgrid',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'SendGrid',
   description: 'SendGrid is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/sendGrid.md'),

lib/notification/adapter/sendGrid.md

@@ -6,3 +6,5 @@ SendGrid is a free email service (free as in "you cannot send more than 100(Send
 To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.     
 
 Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new dynamic template. For this new template, I recommend copying and pasting the code from the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
+
+If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).

lib/notification/adapter/slack.js

@@ -1,16 +1,7 @@
-const Slack = require('slack');
+import Slack from 'slack';
+import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
-const { markdown2Html } = require('../../services/markdown');
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-
-/**
- * sends a new listing to slack
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
   return newListings.map((payload) =>
     msg({
@@ -47,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     })
   );
 };
-
+export const config = {
-exports.config = {
+  id: 'slack',
-  id: __filename.slice(__dirname.length + 1, -3),
   name: 'Slack',
   readme: markdown2Html('lib/notification/adapter/slack.md'),
   description: 'Fredy will send new listings to the slack channel of your choice..',

lib/notification/adapter/sqlite.js

@@ -0,0 +1,25 @@
+import { markdown2Html } from '../../services/markdown.js';
+import Database from 'better-sqlite3';
+export const send = ({ serviceName, newListings, jobKey }) => {
+  const db = new Database('db/listings.db');
+  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+export const config = {
+  id: 'sqlite',
+  name: 'Sqlite',
+  description: 'This adapter stores listings in a local sqlite3 database.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,3 @@
+### Sqlite Adapter
+
+This adapter stores search results in an sqlite database in db/listings.db

lib/notification/adapter/telegram.js

@@ -1,44 +1,65 @@
-const TelegramBot = require('tg-yarl');
+import { markdown2Html } from '../../services/markdown.js';
-const { markdown2Html } = require('../../services/markdown');
+import { getJob } from '../../services/storage/jobStorage.js';
-const opts = { parse_mode: 'Markdown' };
+import fetch from 'node-fetch';
-
+const MAX_ENTITIES_PER_CHUNK = 8;
+const RATE_LIMIT_INTERVAL = 1010;
 /**
- * sends new listings to telegram
+ * splitting an array into chunks because Telegram only allows for messages up to
- * @param serviceName e.g immowelt
+ * 4096 chars, thus we have to split messages into chunks
- * @param newListings an array with newly found listings
+ * @param inputArray
- * @param notificationConfig config of this notification adapter
+ * @param perChunk
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
  */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+const arrayChunks = (inputArray, perChunk) =>
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
+  inputArray.reduce((all, one, i) => {
-
+    const ch = Math.floor(i / perChunk);
-  const bot = new TelegramBot(token);
+    all[ch] = [].concat(all[ch] || [], one);
-
+    return all;
-  let message = `Job: ${jobKey} | Service _${serviceName}_ found _${newListings.length}_ new listings:\n\n`;
+  }, []);
-
-  message += newListings.map(
-    (o) =>
-      `*${shorten(o.title.replace(/\*/g, ''), 45)}*\n` +
-      [o.address, o.price, o.size].join(' | ') +
-      '\n' +
-      `[LINK](${o.link})\n\n`
-  );
-
-  return bot.sendMessage(chatId, message, opts);
-};
-
 function shorten(str, len = 30) {
   return str.length > len ? str.substring(0, len) + '...' : str;
 }
-
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-/**
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
- * exported config is being used in the frontend to generate the fields
+  const job = getJob(jobKey);
- * incoming values will be the keys (and values) of the fields
+  const jobName = job == null ? jobKey : job.name;
- *
+  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
- */
+  const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
-exports.config = {
+  const promises = chunks.map((chunk) => {
-  id: __filename.slice(__dirname.length + 1, -3),
+    let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
+    message += chunk.map(
+      (o) =>
+        `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
+        [o.address, o.price, o.size].join(' | ') +
+        '\n\n'
+    );
+    /**
+     * This is to not break the rate limit. It is to only send 1 message per second
+     */
+    return new Promise((resolve, reject) => {
+      setTimeout(() => {
+        fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+          method: 'post',
+          body: JSON.stringify({
+            chat_id: chatId,
+            text: message,
+            parse_mode: 'HTML',
+            disable_web_page_preview: true,
+          }),
+          headers: { 'Content-Type': 'application/json' },
+        })
+          .then(() => {
+            resolve();
+          })
+          .catch(() => {
+            reject();
+          });
+      }, RATE_LIMIT_INTERVAL);
+    });
+  });
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'telegram',
   name: 'Telegram',
   readme: markdown2Html('lib/notification/adapter/telegram.md'),
   description: 'Fredy will send new listings to your mobile, using Telegram.',

lib/notification/notify.js

@@ -1,24 +1,24 @@
-const fs = require('fs');
+import fs from 'fs';
 const path = './adapter';
 
 /** Read every integration existing in ./adapter **/
-const adapter = fs
+const adapter = await Promise.all(
-  .readdirSync('./lib/notification/adapter')
+  fs
-  .filter((file) => file.endsWith('.js'))
+    .readdirSync('./lib/notification/adapter')
-  .map((integPath) => require(`${path}/${integPath}`));
+    .filter((file) => file.endsWith('.js'))
+    .map(async (integPath) => await import(`${path}/${integPath}`))
+);
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
-
+const findAdapter = (notificationAdapter) => {
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
+  return adapter.find((a) => a.config.id === notificationAdapter.id);
+};
+export const send = (serviceName, newListings, notificationConfig, jobKey) => {
   //this is not being used in tests, therefore adapter are always set
   return notificationConfig
     .filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
     .map((notificationAdapter) => findAdapter(notificationAdapter))
     .map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
 };
-
-const findAdapter = (notificationAdapter) => {
-  return adapter.find((a) => a.config.id === notificationAdapter.id);
-};

lib/provider/einsAImmobilien.js

@@ -1,27 +1,22 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
   if (o.rooms != null) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-
   return Object.assign(o, { size, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '.tabelle',
+  sortByDateParam: 'sort_type=newest',
   crawlFields: {
     id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
     price: '.tabelle .inner_object_data .single_data_price | removeNewline | trim',
@@ -30,21 +25,17 @@ const config = {
     title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
     description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
-  paginate: '.pagination_blocks div:last a@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: '1a Immobilien',
   baseUrl: 'https://www.1a-immobilienmarkt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'einsAImmobilien',
 };
-
+export { config };
-exports.config = config;

lib/provider/immobilienDe.js

@@ -0,0 +1,49 @@
+import utils from '../utils.js';
+let appliedBlackList = [];
+function shortenLink(link) {
+  return link.substring(0, link.indexOf('?'));
+}
+function parseId(shortenedLink) {
+  return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
+}
+function normalize(o) {
+  const id = parseId(shortenLink(o.link));
+  const size = o.size || 'N/A m²';
+  const price = o.price || 'N/A €';
+  const title = o.title || 'No title available';
+  const address = o.address || 'No address available';
+  const link = shortenLink(o.link);
+  return Object.assign(o, { id, price, size, title, address, link });
+}
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+const config = {
+  url: null,
+  crawlContainer: '.estates_list .list_immo a._ref',
+  sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
+  crawlFields: {
+    price: '.list_entry .immo_preis .label_info',
+    size: '.list_entry .flaeche .label_info | removeNewline | trim',
+    title: '.list_entry .part_text h3 span',
+    description: '.list_entry .description | trim',
+    link: '@href',
+    address: '.list_entry .place',
+  },
+  paginate: '.list_immo .blocknav .blocknav_list li.next a@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immobilien.de',
+  baseUrl: 'https://www.immobilien.de/',
+  id: 'immobilienDe',
+};
+export { config };

lib/provider/immonet.js

@@ -1,51 +1,42 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
-  const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
+  const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
-  const address = o.address.split(' • ')[1];
+  const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
-  //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
+  const link = o.id;
-  //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
-  const link = `https://www.immonet.de/angebot/${id}`;
   return Object.assign(o, { id, address, price, size, title, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '#result-list-stage .item',
+  crawlContainer: '.content-wrapper-tiles .ng-star-inserted',
+  sortByDateParam: 'sortby=19',
   crawlFields: {
-    id: '@id',
+    id: '.card a@href',
-    price: 'div[id*="selPrice_"] | trim',
+    title: '.card h3 |trim',
-    size: 'div[id*="selArea_"] | trim',
+    price: '.card .has-font-300 .is-bold | trim',
-    title: '.item a img@title',
+    size: '.card .has-font-300 .ml-100 | trim',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
+    address: '.card span:nth-child(2) | trim',
   },
   paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Immonet',
   baseUrl: 'https://www.immonet.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immonet',
 };
-
+export { config };
-exports.config = config;

lib/provider/immoscout.js

@@ -0,0 +1,41 @@
+import utils from '../utils.js';
+let appliedBlackList = [];
+function nullOrEmpty(val) {
+  return val == null || val.length === 0;
+}
+function normalize(o) {
+  const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
+  const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
+  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  return Object.assign(o, { title, address, link });
+}
+function applyBlacklist(o) {
+  return !utils.isOneOf(o.title, appliedBlackList);
+}
+const config = {
+  url: null,
+  crawlContainer: '#resultListItems li.result-list__listing',
+  sortByDateParam: 'sorting=2',
+  crawlFields: {
+    id: '.result-list-entry@data-obid | int',
+    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
+    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
+    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
+    link: '.result-list-entry .result-list-entry__brand-title-container@href',
+    address: '.result-list-entry .result-list-entry__map-link',
+  },
+  paginate: '#pager .align-right a@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immoscout',
+  baseUrl: 'https://www.immobilienscout24.de/',
+  id: 'immoscout',
+};
+export { config };

lib/provider/immoswp.js

@@ -0,0 +1,44 @@
+import utils from '../utils.js';
+let appliedBlackList = [];
+function normalize(o) {
+  const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+  const size = o.size || 'N/A m²';
+  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+  const address = o.address || 'No address available';
+  const title = o.title || 'No title available';
+  const link = `https://immo.swp.de/immobilien/${id}`;
+  const description = o.description;
+  return Object.assign(o, { id, address, price, size, title, link, description });
+}
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+const config = {
+  url: null,
+  crawlContainer: '.js-serp-item',
+  sortByDateParam: 's=most_recently_updated_first',
+  crawlFields: {
+    id: '@id',
+    price: 'div.item__spec.item-spec-price | trim',
+    size: 'div.item__spec.item-spec-area | trim',
+    title: 'a.js-item-title-link@title',
+    address: 'div.item__locality | removeNewline | trim',
+    description: 'div.item__main-info-points.clearfix p small | removeNewline | trim',
+  },
+  paginate: 'li.page-item.pagination__item a.page-link@href',
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immo Südwest Presse',
+  baseUrl: 'https://immo.swp.de/',
+  id: 'immoswp',
+};
+export { config };

lib/provider/immowelt.js

@@ -1,46 +1,37 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
-  const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
+  return o;
-  const address = o.address;
-
-  return Object.assign(o, { size, address });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '.immoliste .js-object.listitem_wrap ',
+  crawlContainer: "div[class^='EstateItem-']",
+  sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
   crawlFields: {
-    id: '@data-estateid | int',
+    id: 'a@id',
-    price: '.hardfacts_3 strong | removeNewline | trim',
+    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
-    size: '.js-object.listitem_wrap .hardfacts_3 div:nth-child(2)| removeNewline | trim',
+    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
-    title: '.listcontent.clear h2',
+    title: "div[class^='FactsMain-'] h2",
     link: 'a@href',
-    address: '.listcontent .details .listlocation| removeNewline | trim',
+    address: "div[class^='estateFacts-'] span | removeNewline | trim",
   },
   paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immowelt',
   baseUrl: 'https://www.immowelt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immowelt',
 };
-
+export { config };
-exports.config = config;

lib/provider/kleinanzeigen.js

@@ -1,26 +1,22 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
-
 function normalize(o) {
   const size = o.size || '--- m²';
-
   return Object.assign(o, { size });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   const isBlacklistedDistrict =
     appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-
   return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '#srchrslt-adtable .ad-listitem',
+  crawlContainer: '#srchrslt-adtable .ad-listitem ',
+  //sort by date is standard oO
+  sortByDateParam: null,
   crawlFields: {
     id: '.aditem@data-adid | int',
     price: '.aditem-main--middle--price | removeNewline | trim',
@@ -34,18 +30,15 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Ebay Kleinanzeigen',
-  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
+  baseUrl: 'https://www.kleinanzeigen.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'kleinanzeigen',
 };
-
+export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlacklistedDistricts = blacklistedDistricts || [];
   appliedBlackList = blacklist || [];
 };
-
+export { config };
-exports.config = config;

lib/provider/neubauKompass.js

@@ -1,18 +1,15 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   return o;
 }
-
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
-
 const config = {
   url: null,
   crawlContainer: '.nbk-container >div article',
+  sortByDateParam: 'Sortierung=Id&Richtung=DESC',
   crawlFields: {
     id: '@id',
     title: 'a.nbk-truncate@title | removeNewline | trim',
@@ -24,17 +21,14 @@ const config = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Neubau Kompass',
   baseUrl: 'https://www.neubaukompass.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'neubauKompass',
 };
-
+export { config };
-exports.config = config;

lib/provider/wgGesucht.js

@@ -1,21 +1,17 @@
-const utils = require('../utils');
+import utils from '../utils.js';
-
 let appliedBlackList = [];
-
 function normalize(o) {
   return o;
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
   crawlContainer: '#main_column .wgg_card',
+  sortByDateParam: 'sort_column=0&sort_order=0',
   crawlFields: {
     id: '@data-id',
     details: '.row .noprint .col-xs-11 |removeNewline |trim',
@@ -24,21 +20,17 @@ const config = {
     title: '.truncate_title a |removeNewline |trim',
     link: '.truncate_title a@href',
   },
-  paginate: '.pagination-sm:first a:last@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
+export const init = (sourceConfig, blacklist) => {
-exports.init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
+export const metaInformation = {
-exports.metaInformation = {
   name: 'Wg gesucht',
   baseUrl: 'https://www.wg-gesucht.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'wgGesucht',
 };
-
+export { config };
-exports.config = config;

lib/services/markdown.js

@@ -1,6 +1,6 @@
-const markdown = require('markdown').markdown;
+import markdown$0 from 'markdown';
-const fs = require('fs');
+import fs from 'fs';
-
+const markdown = markdown$0.markdown;
-exports.markdown2Html = function markdown2Html(filePath) {
+export function markdown2Html(filePath) {
   return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
-};
+}

lib/services/queryStringMutator.js

@@ -0,0 +1,10 @@
+import queryString from 'query-string';
+export default (_url, sortByDateParam) => {
+  //if no mutation is necessary, just return the original url
+  if (sortByDateParam == null) {
+    return _url;
+  }
+  const original = queryString.parseUrl(_url);
+  const mutate = queryString.parse(sortByDateParam);
+  return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;
+};

lib/services/requestDriver.js

@@ -0,0 +1,74 @@
+import fetch from 'node-fetch';
+import { config } from '../utils.js';
+import { makeUrlResidential } from './scrapingAnt.js';
+import https from 'https';
+//if ScrapingAnt got blocked, this http status is returned
+const BLOCKED_HTTP_STATUS = 423;
+const NOT_FOUND_HTTP_STATUS = 404;
+const MAX_RETRIES_SCRAPING_ANT = 10;
+const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
+const agent = new https.Agent({
+  rejectUnauthorized: false,
+});
+
+function makeDriver(headers = {}) {
+  let cookies = '';
+  async function scrapingAntDriver(context, callback, retryCounter = 0) {
+    const proxyType = config.scrapingAnt?.proxy || 'datacenter';
+    try {
+      const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
+      const response = await fetch(url, {
+        headers: {
+          ...headers,
+          cookie: cookies,
+        },
+      });
+      const result = await response.text();
+      if (cookies.length === 0) {
+        cookies = response.headers.raw()['set-cookie'] || [];
+      }
+      callback(null, result);
+    } catch (exception) {
+      /* eslint-disable no-console */
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
+        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
+        callback(null, []);
+        return;
+      }
+      if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
+        retryCounter++;
+        console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
+        await scrapingAntDriver(context, callback, retryCounter);
+      } else {
+        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
+        callback(null, []);
+      }
+      /* eslint-enable no-console */
+    }
+  }
+
+  /**
+   * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
+   * everything != Immoscout & Immonet as of writing this)
+   */
+  return async function driver(context, callback) {
+    if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
+      return scrapingAntDriver(context, callback);
+    }
+    try {
+      const response = await fetch(context.url, {
+        headers: {
+          ...headers,
+          Cookie: cookies,
+        },
+        agent,
+      });
+      const result = await response.text();
+      callback(null, result);
+    } catch (exception) {
+      console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
+      callback(null, []);
+    }
+  };
+}
+export default makeDriver;

lib/services/scraper.js

@@ -1,6 +1,6 @@
-const makeDriver = require('request-x-ray');
+import { config } from '../utils.js';
-const Xray = require('x-ray');
+import makeDriver from './requestDriver.js';
-
+import Xray from 'x-ray';
 class Scraper {
   constructor() {
     const filters = {
@@ -8,37 +8,29 @@ class Scraper {
       trim: this._trim,
       int: this._int,
     };
-
+    const headers = {
-    const driver = makeDriver({
+      'User-Agent':
-      headers: {
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-        'User-Agent':
+    };
-          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
+    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
-        cookie:
+      headers['x-api-key'] = config.scrapingAnt.apiKey;
-          'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
+    }
-      },
+    const driver = makeDriver(headers);
-    });
-
     const xray = Xray({ filters });
     xray.driver(driver);
-
     this.xray = xray;
   }
-
   get x() {
     return this.xray;
   }
-
   _removeNewline(value) {
     return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
   }
-
   _trim(value) {
     return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
   }
-
   _int(value) {
     return typeof value === 'string' ? parseInt(value, 10) : value;
   }
 }
-
+export default new Scraper().x;
-module.exports = new Scraper().x;

lib/services/scrapingAnt.js

@@ -0,0 +1,29 @@
+import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
+import { metaInformation as immoNetInfo } from '../provider/immonet.js';
+import { config } from '../utils.js';
+
+const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
+  'window.scrollTo(0,document.body.scrollHeight);'
+).toString('base64')}`;
+
+const needScrapingAnt = (id) => {
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
+};
+export const transformUrlForScrapingAnt = (url, id) => {
+  let urlParams = '';
+  if (needScrapingAnt(id)) {
+    if (id.toLowerCase() === immoNetInfo.id) {
+      urlParams = additionalImmonetUrlParams;
+    }
+    //only do calls to scrapingAnt when dealing with Immoscout/Immonet
+    url = `https://api.scrapingant.com/v2/general?url=${encodeURIComponent(url)}&proxy_type=datacenter${urlParams}`;
+  }
+  return url;
+};
+export const isScrapingAntApiKeySet = () => {
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+};
+export const makeUrlResidential = (url) => {
+  return url.replace('datacenter', 'residential');
+};
+export { needScrapingAnt };

lib/services/security/hash.js

@@ -1,3 +1,2 @@
-const crypto = require('crypto');
+import crypto from 'crypto';
-
+export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
-exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/similarity-check/SimilarityCacheEntry.js

@@ -0,0 +1,26 @@
+import stringSimilarity from 'string-similarity';
+//if the score is higher than this, it will be considered a match
+const MAX_DICE_INDEX = 0.7;
+export default (class SimilarityCacheEntry {
+  constructor(time) {
+    this.time = time;
+    this.values = [];
+  }
+  setCacheEntry = (entry) => {
+    this.values.push(entry);
+  };
+  getTime = () => {
+    return this.time;
+  };
+  hasSimilarEntries = (value) => {
+    if (this.values.length > 0) {
+      for (let i = 0; i < this.values.length; i++) {
+        const index = stringSimilarity.compareTwoStrings(value, this.values[i]);
+        if (index >= MAX_DICE_INDEX) {
+          return true;
+        }
+      }
+    }
+    return false;
+  };
+});

lib/services/similarity-check/similarityCache.js

@@ -0,0 +1,40 @@
+import SimilarityCacheEntry from './SimilarityCacheEntry.js';
+import { config } from '../../utils.js';
+//5 minutes
+let retention = 5 * 60 * 1000;
+const intervalInMs = config.interval * 60 * 1000;
+//an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
+if (intervalInMs <= retention) {
+  retention = Math.floor(intervalInMs / 2);
+}
+//jobid -> SimilarityCacheEntry
+const cache = {};
+let intervalId;
+/**
+ * cleanup
+ */
+intervalId = setInterval(() => {
+  const keysToBeRemoved = [];
+  const now = Date.now();
+  Object.keys(cache).forEach((key) => {
+    if (cache[key].getTime() + retention < now) {
+      keysToBeRemoved.push(key);
+    }
+  });
+  if (keysToBeRemoved.length > 0) {
+    keysToBeRemoved.forEach((key) => delete cache[key]);
+  }
+}, 10000);
+export const addCacheEntry = (jobId, value) => {
+  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
+  cache[jobId].setCacheEntry(value);
+};
+export const hasSimilarEntries = (jobId, value) => {
+  if (cache[jobId] == null) {
+    return false;
+  }
+  return cache[jobId].hasSimilarEntries(value);
+};
+export const stopCacheCleanup = () => {
+  clearInterval(intervalId);
+};

lib/services/storage/LowDashAdapter.js

@@ -0,0 +1,8 @@
+import lodash from 'lodash';
+import { LowSync } from 'lowdb';
+export default class LowdashAdapter extends LowSync {
+  constructor(adapter, defaultData = {}) {
+    super(adapter, defaultData);
+    this.chain = lodash.chain(this).get('data');
+  }
+}

lib/services/storage/jobStorage.js

@@ -1,28 +1,29 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
+import { nanoid } from 'nanoid';
-const FileSync = require('lowdb/adapters/FileSync');
+import * as listingStorage from './listingsStorage.js';
-const adapter = new FileSync(DB_PATH);
+import { getDirName } from '../../utils.js';
-const low = require('lowdb');
+import path from 'path';
-const db = low(adapter);
+import LowdashAdapter from './LowDashAdapter.js';
-const { nanoid } = require('nanoid');
-const listingStorage = require('./listingsStorage');
 
-db.defaults({ jobs: [] }).write();
+const file = path.join(getDirName(), '../', 'db/jobs.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, { jobs: [] });
 
-exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+db.read();
+
+
+export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
       ? null
-      : db
+      : db.chain
           .get('jobs')
           .find((job) => job.id === jobId)
           .value();
-
+  const jobs = db.chain
-  const jobs = db
     .get('jobs')
-    .value()
+    .filter((job) => job.id !== jobId)
-    .filter((job) => job.id !== jobId);
+    .value();
-
   jobs.push({
     id: jobId || nanoid(),
     //make sure to not overwrite the user id in case an admin changes the job
@@ -33,51 +34,55 @@ exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, no
     provider,
     notificationAdapter,
   });
-
+  db.chain.set('jobs', jobs).value();
-  db.set('jobs', jobs).write();
+  db.write();
 };
-
+export const getJob = (jobId) => {
-exports.getJob = (jobId) => {
+  const job = db.chain
-  const job = db
     .get('jobs')
     .find((job) => job.id === jobId)
     .value();
-
   if (job == null) {
     return null;
   }
-
   return {
     ...job,
     numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
   };
 };
-
+export const setJobStatus = ({ jobId, status }) => {
-exports.setJobStatus = ({ jobId, status }) => {
+  db.chain
-  db.get('jobs')
+    .get('jobs')
     .find((job) => job.id === jobId)
     .assign({ enabled: status })
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeJob = (jobId) => {
-exports.removeJob = (jobId) => {
+  listingStorage.removeListings(jobId);
-  db.get('jobs')
+  db.chain
-    .remove((job) => job.id === jobId)
+    .get('jobs')
-    .write();
+    .remove((job) => job.id === jobId)
-};
+    .value();
-
+  db.write();
-exports.removeJobsByUserId = (userId) => {
+};
-  db.get('jobs')
+export const removeJobsByUserId = (userId) => {
-    .remove((job) => job.userId === userId)
+  db.chain
-    .write();
+    .get('jobs')
-};
+    .filter((job) => job.userId === userId)
-
+    .forEach((job) => listingStorage.removeListings(job.id));
-exports.getJobs = () => {
+  db.chain
-  return db
+    .get('jobs')
+    .remove((job) => job.userId === userId)
+    .value();
+  db.write();
+};
+export const getJobs = () => {
+  return db.chain
     .get('jobs')
-    .value()
     .map((job) => ({
       ...job,
       numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
-    }));
+    }))
+    .value();
 };

lib/services/storage/listingsStorage.js

@@ -1,10 +1,13 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
+const file = path.join(getDirName(), '../', 'db/jobListingData.json');
-const FileSync = require('lowdb/adapters/FileSync');
+const adapter = new JSONFileSync(file);
-const adapter = new FileSync(DB_PATH);
+const db = new LowdashAdapter(adapter, {});
-const low = require('lowdb');
+
-const db = low(adapter);
+db.read();
 
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
@@ -19,31 +22,31 @@ const buildKey = (jobKey, providerId, endpoint) => {
   }
   return key;
 };
-
+export const getNumberOfAllKnownListings = (jobId) => {
-exports.getNumberOfAllKnownListings = (jobId) => {
+  const data = db.chain.get(`${jobId}.providerData`).value() || {};
-  const data = db.get(`${jobId}.providerData`).value() || {};
   return Object.values(data)
     .map((values) => Object.keys(values).length)
     .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
 };
-
+export const getListingProviderDataForAnalytics = (jobId) => {
-exports.getListingProviderDataForAnalytics = (jobId) => {
   const key = buildKey(jobId, 'providerData');
-  return db.get(key).value() || {};
+  return db.chain.get(key).value() || {};
 };
-
+export const getKnownListings = (jobId, providerId) => {
-exports.getKnownListings = (jobId, providerId) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-  return db.get(providerListingsKey).value() || {};
+  return db.chain.get(providerListingsKey).value() || {};
 };
-
+export const setKnownListings = (jobId, providerId, listings) => {
-exports.setKnownListings = (jobId, providerId, listings) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-
+  db.chain.set(providerListingsKey, listings).value();
-  return db.set(providerListingsKey, listings).write();
+  return db.write();
 };
-
+export const setLastJobExecution = (jobId) => {
-exports.setLastJobExecution = (jobId) => {
   const key = buildKey(jobId, null, 'lastExecution');
-  return db.set(key, Date.now()).write();
+  db.chain.set(key, Date.now()).value();
+  return db.write();
+};
+export const removeListings = (jobId) => {
+  db.chain.unset(jobId).value();
+  db.write();
 };

lib/services/storage/userStorage.js

@@ -1,30 +1,33 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
-const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
+import { getDirName } from '../../utils.js';
-const FileSync = require('lowdb/adapters/FileSync');
+import * as hasher from '../security/hash.js';
-const adapter = new FileSync(DB_PATH);
+import { nanoid } from 'nanoid';
-const low = require('lowdb');
+import * as jobStorage from './jobStorage.js';
-const db = low(adapter);
+import path from 'path';
-const hasher = require('../security/hash');
+import LowdashAdapter from './LowDashAdapter.js';
-const { nanoid } = require('nanoid');
-const jobStorage = require('./jobStorage');
 
-db.defaults({
+const defaultData = {
-  user: [
+    user: [
-    //you probably want to change the default password ;)
+        //you probably want to change the default password ;)
-    {
+        {
-      id: nanoid(),
+            id: nanoid(),
-      lastLogin: Date.now(),
+            lastLogin: Date.now(),
-      username: 'admin',
+            username: 'admin',
-      password: hasher.hash('admin'),
+            password: hasher.hash('admin'),
-      isAdmin: true,
+            isAdmin: true,
-      isDemo: false,
+        },
-    },
+    ],
-  ],
+};
-}).write();
 
-exports.getUsers = (withPassword) => {
+const file = path.join(getDirName(), '../', 'db/users.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, defaultData);
+
+db.read();
+
+export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();
-  return db
+  return db.chain
     .get('user')
     .value()
     .map((user) => ({
@@ -34,13 +37,12 @@ exports.getUsers = (withPassword) => {
       numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
     }));
 };
-
+export const getUser = (id) => {
-exports.getUser = (id) => {
   const jobs = jobStorage.getJobs();
-  const user = db
+  const user = db.chain
     .get('user')
-    .value()
+    .find((user) => user.id === id)
-    .find((user) => user.id === id);
+    .value();
   if (user == null) {
     return null;
   }
@@ -49,13 +51,11 @@ exports.getUser = (id) => {
     numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
   };
 };
-
+export const upsertUser = ({ username, password, userId, isAdmin }) => {
-exports.upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db.chain
-  const user = db
     .get('user')
-    .value()
+    .filter((u) => u.id !== userId)
-    .filter((u) => u.id !== userId);
+    .value();
-
   user.push({
     id: userId || nanoid(),
     username,
@@ -63,21 +63,24 @@ exports.upsertUser = ({ username, password, userId, isAdmin }) => {
     password: hasher.hash(password),
     isAdmin,
   });
-
+  db.chain.set('user', user).value();
-  db.set('user', user).write();
+  db.write();
 };
-
+export const setLastLoginToNow = ({ userId }) => {
-exports.setLastLoginToNow = ({ userId }) => {
+  db.chain
-  db.get('user')
+    .get('user')
     .find((u) => u.id === userId)
     .assign({ lastLogin: Date.now() })
-    .write();
+    .value();
+  db.write();
 };
-
+export const removeUser = (userId) => {
-exports.removeUser = (userId) => {
+  const user = db.chain.get('user').value();
-  const user = db.get('user').value();
+  db.chain
-  db.set(
+    .set(
-    'user',
+      'user',
-    user.filter((u) => u.id !== userId)
+      user.filter((u) => u.id !== userId)
-  ).write();
+    )
+    .value();
+  db.write();
 };

lib/utils.js

@@ -1,11 +1,51 @@
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { readFile } from 'fs/promises';
+
 function isOneOf(word, arr) {
   if (arr == null || arr.length === 0) {
     return false;
   }
   const expression = String.raw`\b(${arr.join('|')})\b`;
   const blacklist = new RegExp(expression, 'ig');
-
   return blacklist.test(word);
 }
+function nullOrEmpty(val) {
+  return val == null || val.length === 0;
+}
+function timeStringToMs(timeString, now) {
+  const d = new Date(now);
+  const parts = timeString.split(':');
+  d.setHours(parts[0]);
+  d.setMinutes(parts[1]);
+  d.setSeconds(0);
+  return d.getTime();
+}
+function duringWorkingHoursOrNotSet(config, now) {
+  const { workingHours } = config;
+  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+    return true;
+  }
+  const toDate = timeStringToMs(workingHours.to, now);
+  const fromDate = timeStringToMs(workingHours.from, now);
+  return fromDate <= now && toDate >= now;
+}
 
-module.exports = { isOneOf };
+function getDirName() {
+  return dirname(fileURLToPath(import.meta.url));
+}
+
+const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+
+export { isOneOf };
+export { nullOrEmpty };
+export { duringWorkingHoursOrNotSet };
+export { getDirName };
+export { config };
+export default {
+  isOneOf,
+  nullOrEmpty,
+  duringWorkingHoursOrNotSet,
+  getDirName,
+  config,
+};

package.json

@@ -1,19 +1,22 @@
 {
   "name": "fredy",
-  "version": "3.0.0",
+  "version": "7.3.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
+    "dev": "yarn && rm -rf ./ui/public/* && vite",
-    "prod": "export BUILD_DEV='false' && export NODE_ENV='production' && webpack --config ./webpack.prod.js",
+    "ui": "rm -rf ./ui/public/* && vite",
-    "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
+    "prod": "yarn && vite build --emptyOutDir",
-    "test": "mocha --timeout 20000 test/**/*.test.js"
+    "format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
+    "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
+    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
   },
   "husky": {
     "hooks": {
       "pre-commit": "lint-staged"
     }
   },
+  "type": "module",
   "lint-staged": {
     "*.js": [
       "eslint ./index.js ./lib/**/*.js ./test/**/*.js",
@@ -32,6 +35,7 @@
     "house",
     "rent",
     "immoscout",
+    "scraper",
     "immonet",
     "immowelt",
     "immobilienscout24"
@@ -41,8 +45,8 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=11.0.0",
+    "node": ">=16.0.0",
-    "npm": ">=6.0.0"
+    "npm": ">=7.0.0"
   },
   "browserslist": [
     "> 0.5%",
@@ -51,63 +55,54 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@rematch/core": "2.0.1",
+    "@douyinfe/semi-ui": "2.42.4",
-    "@rematch/loading": "2.0.1",
+    "@rematch/core": "2.2.0",
-    "@sendgrid/mail": "7.4.2",
+    "@rematch/loading": "2.1.2",
-    "axios": "^0.21.1",
+    "@sendgrid/mail": "7.7.0",
-    "body-parser": "1.19.0",
+    "@vitejs/plugin-react": "4.0.4",
-    "cookie-session": "^1.4.0",
+    "better-sqlite3": "8.6.0",
-    "handlebars": "4.7.7",
+    "body-parser": "1.20.2",
-    "highcharts": "9.0.1",
+    "cookie-session": "2.0.0",
-    "highcharts-react-official": "^3.0.0",
+    "handlebars": "4.7.8",
-    "lowdb": "1.0.0",
+    "highcharts": "11.1.0",
+    "highcharts-react-official": "3.2.1",
+    "lodash": "4.17.21",
+    "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "^3.1.22",
+    "nanoid": "4.0.2",
-    "node-mailjet": "3.3.1",
+    "node-fetch": "3.3.2",
-    "react": "17.0.2",
+    "node-mailjet": "6.0.4",
-    "react-dom": "17.0.2",
+    "query-string": "8.1.0",
-    "react-redux": "7.2.3",
+    "react": "18.2.0",
-    "react-router": "5.2.0",
+    "react-dom": "18.2.0",
-    "react-router-dom": "5.2.0",
+    "react-redux": "8.1.2",
-    "react-switch": "^6.0.0",
+    "react-router": "5.2.1",
-    "redux": "4.0.5",
+    "react-router-dom": "5.3.0",
-    "redux-thunk": "2.3.0",
+    "redux": "4.2.1",
-    "request-x-ray": "0.1.4",
+    "redux-thunk": "2.4.2",
-    "restana": "4.8.1",
+    "restana": "4.9.7",
-    "semantic-ui-react": "2.0.3",
+    "serve-static": "1.15.0",
-    "serve-static": "^1.14.1",
     "slack": "11.0.2",
-    "tg-yarl": "1.3.0",
+    "string-similarity": "^4.0.4",
+    "vite": "4.4.9",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.13.13",
+    "@babel/core": "7.22.15",
-    "@babel/preset-env": "7.13.12",
+    "@babel/eslint-parser": "7.22.15",
-    "@babel/preset-react": "7.13.13",
+    "@babel/preset-env": "7.22.15",
-    "babel-eslint": "10.1.0",
+    "@babel/preset-react": "7.22.15",
-    "babel-loader": "8.2.2",
+    "chai": "4.3.8",
-    "chai": "4.3.4",
+    "eslint": "8.48.0",
-    "clean-webpack-plugin": "3.0.0",
+    "eslint-config-prettier": "8.8.0",
-    "copy-webpack-plugin": "6.3.0",
+    "eslint-plugin-react": "7.33.2",
-    "css-loader": "5.0.1",
+    "esmock": "2.4.0",
-    "eslint": "7.23.0",
+    "history": "5.3.0",
-    "eslint-config-prettier": "7.1.0",
-    "eslint-plugin-react": "7.23.1",
-    "file-loader": "6.2.0",
-    "history": "5.0.0",
     "husky": "4.3.8",
-    "less": "4.1.1",
+    "less": "4.2.0",
-    "less-loader": "7.2.1",
+    "lint-staged": "13.2.2",
-    "lint-staged": "10.5.4",
+    "mocha": "10.2.0",
-    "mocha": "8.3.2",
+    "prettier": "2.8.8",
-    "prettier": "2.2.1",
+    "redux-logger": "3.0.6"
-    "proxyquire": "2.1.3",
-    "redux-logger": "3.0.6",
-    "style-loader": "2.0.0",
-    "url-loader": "4.1.1",
-    "webpack": "4.44.2",
-    "webpack-cli": "3.3.12",
-    "webpack-dev-server": "3.11.2",
-    "webpack-merge": "5.7.3"
   }
 }

test/mocks/mockNotification.js

@@ -1,12 +1,10 @@
-module.exports = {
+let tmpStore = {};
-  _tmpStore: {},
 
-  send: (serviceName, payload) => {
+export const send = (serviceName, payload) => {
-    this._tmpStore = { serviceName, payload };
+  tmpStore = { serviceName, payload };
-    return [Promise.resolve()];
+  return [Promise.resolve()];
-  },
+};
-
+
-  get: () => {
+export const get = () => {
-    return this._tmpStore;
+  return tmpStore;
-  },
 };

test/mocks/mockStore.js

@@ -1,11 +1,8 @@
 const db = {};
-
+export const setKnownListings = (jobKey, providerId, listings) => {
-exports.setKnownListings = (jobKey, providerId, listings) => {
   if (!Array.isArray(listings)) throw Error('Not a valid array');
-
   db[providerId] = listings;
 };
-
+export const getKnownListings = (jobKey, providerId) => {
-exports.getKnownListings = (jobKey, providerId) => {
   return db[providerId] || [];
 };

test/provider/einsAImmobilien.test.js

@@ -1,30 +1,25 @@
-const mockNotification = require('../mocks/mockNotification');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const providerConfig = require('./testProvider.json');
+import { get } from '../mocks/mockNotification.js';
-const mockStore = require('../mocks/mockStore');
+import { providerConfig, mockFredy } from '../utils.js';
-const proxyquire = require('proxyquire').noCallThru();
+import chai from 'chai';
-const expect = require('chai').expect;
+import * as provider from '../../lib/provider/einsAImmobilien.js';
-const provider = require('../../lib/provider/einsAImmobilien');
+
+const expect = chai.expect;
 
 describe('#einsAImmobilien testsuite()', () => {
-  provider.init(providerConfig.einsAImmobilien, [], []);
+  after(() => {
-
+    similarityCache.stopCacheCleanup();
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
   });
-
+  provider.init(providerConfig.einsAImmobilien, [], []);
   it('should test einsAImmobilien provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
       fredy.execute().then((listings) => {
         expect(listings).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
@@ -32,9 +27,7 @@ describe('#einsAImmobilien testsuite()', () => {
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-
           /** check the values if possible **/
-          expect(notify.price).that.does.include('EUR');
           expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');

test/provider/immobilienDe.test.js

@@ -0,0 +1,40 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { providerConfig, mockFredy } from '../utils.js';
+import chai from 'chai';
+import * as provider from '../../lib/provider/immobilienDe.js';
+const expect = chai.expect;
+describe('#immobilien.de testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+  provider.init(providerConfig.immobilienDe, [], []);
+  it('should test immobilien.de provider', async () => {
+    const Fredy = await mockFredy();
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immobilienDe');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.size).that.does.include('m²');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilien.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/immonet.test.js

@@ -1,32 +1,34 @@
-const mockNotification = require('../mocks/mockNotification');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const providerConfig = require('./testProvider.json');
+import { get } from '../mocks/mockNotification.js';
-const mockStore = require('../mocks/mockStore');
+import { mockFredy, providerConfig } from '../utils.js';
-const proxyquire = require('proxyquire').noCallThru();
+import chai from 'chai';
-const expect = require('chai').expect;
+import * as provider from '../../lib/provider/immonet.js';
-const provider = require('../../lib/provider/immonet');
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-
+const expect = chai.expect;
 describe('#immonet testsuite()', () => {
-  provider.init(providerConfig.immonet, [], []);
+  after(() => {
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    similarityCache.stopCacheCleanup();
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
   });
-
+  provider.init(providerConfig.immonet, [], []);
   it('should test immonet provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+        /* eslint-disable no-console */
+        console.info('Skipping Immonet test as ScrapingAnt Api Key is not set.');
+        /* eslint-enable no-console */
+        resolve();
+        return;
+      }
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immonet');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
@@ -37,7 +39,6 @@ describe('#immonet testsuite()', () => {
           expect(notify.price).that.does.include('€');
           expect(notify.size).that.does.include('m²');
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immonet.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/immoscout.test.js

@@ -0,0 +1,48 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import chai from 'chai';
+import * as provider from '../../lib/provider/immoscout.js';
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
+const expect = chai.expect;
+describe('#immoscout testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+  provider.init(providerConfig.immoscout, [], []);
+  it('should test immoscout provider', async () => {
+    const Fredy = await mockFredy();
+    return await new Promise((resolve) => {
+      if (!scrapingAnt.isScrapingAntApiKeySet()) {
+        /* eslint-disable no-console */
+        console.info('Skipping Immoscout test as ScrapingAnt Api Key is not set.');
+        /* eslint-enable no-console */
+        resolve();
+        return;
+      }
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immoscout');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('number');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.size).that.does.include('m²');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilienscout24.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/immoswp.test.js

@@ -0,0 +1,39 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import chai from 'chai';
+import * as provider from '../../lib/provider/immoswp.js';
+const expect = chai.expect;
+describe('#immoswp testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+  provider.init(providerConfig.immoswp, [], []);
+  it('should test immoswp provider', async () => {
+    const Fredy = await mockFredy();
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immoswp');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://immo.swp.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/immowelt.test.js

@@ -1,41 +1,32 @@
-const mockNotification = require('../mocks/mockNotification');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const providerConfig = require('./testProvider.json');
+import { get } from '../mocks/mockNotification.js';
-const mockStore = require('../mocks/mockStore');
+import { mockFredy, providerConfig } from '../utils.js';
-const proxyquire = require('proxyquire').noCallThru();
+import chai from 'chai';
-const expect = require('chai').expect;
+import * as provider from '../../lib/provider/immowelt.js';
-const provider = require('../../lib/provider/immowelt');
+const expect = chai.expect;
-
 describe('#immowelt testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
   it('should test immowelt provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immowelt');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
-          expect(notify.price).that.does.include('€');
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
-          if (notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');
           }
           expect(notify.title).to.be.not.empty;

test/provider/kleinanzeigen.test.js

@@ -1,41 +1,32 @@
-const mockNotification = require('../mocks/mockNotification');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const providerConfig = require('./testProvider.json');
+import { get } from '../mocks/mockNotification.js';
-const mockStore = require('../mocks/mockStore');
+import { mockFredy, providerConfig } from '../utils.js';
-const proxyquire = require('proxyquire').noCallThru();
+import chai from 'chai';
-const expect = require('chai').expect;
+import * as provider from '../../lib/provider/kleinanzeigen.js';
-const provider = require('../../lib/provider/kleinanzeigen');
+const expect = chai.expect;
-
 describe('#kleinanzeigen testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
   it('should test kleinanzeigen provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.kleinanzeigen, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('number');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.price).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
+          expect(notify.link).that.does.include('https://www.kleinanzeigen.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/neubauKompass.test.js

@@ -1,37 +1,29 @@
-const mockNotification = require('../mocks/mockNotification');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const providerConfig = require('./testProvider.json');
+import { get } from '../mocks/mockNotification.js';
-const mockStore = require('../mocks/mockStore');
+import { mockFredy, providerConfig } from '../utils.js';
-const proxyquire = require('proxyquire').noCallThru();
+import chai from 'chai';
-const expect = require('chai').expect;
+import * as provider from '../../lib/provider/neubauKompass.js';
-const provider = require('../../lib/provider/neubauKompass');
+const expect = chai.expect;
-
 describe('#neubauKompass testsuite()', () => {
-  provider.init(providerConfig.neubauKompass, [], []);
+  after(() => {
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    similarityCache.stopCacheCleanup();
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
   });
-
+  provider.init(providerConfig.neubauKompass, [], []);
   it('should test neubauKompass provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
+        const notificationObj = get();
-        const notificationObj = mockNotification.get();
         expect(notificationObj.serviceName).to.equal('neubauKompass');
-
         notificationObj.payload.forEach((notify) => {
           expect(notify).to.be.a('object');
-
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.neubaukompass.de');

test/provider/testProvider.json

@@ -4,12 +4,24 @@
     "enabled": true,
     "id": "einsAImmobilien"
   },
+  "immobilienDe": {
+    "url": "https://www.immobilien.de/Wohnen/Suchergebnisse-51797.html?search._digest=true&search._filter=wohnen&search.flaeche_von=50&search.objektart=wohnung&search.preis_bis=1200&search.typ=mieten&search.umkreis=15&search.wo=district%3A2434%2C2695%2C2621%2C2700%2C2967%2C2734%2C2909%2C2955%2C2392%2C2746%2C2767%2C2982%2C2904%2C2612%2C2892%2C2587%2C2871%2C2975%2C2591%2C2887%2C2569%2C2640%2C2735&sort_col=*created_ts&sort_dir=desc",
+    "enabled": true
+  },
   "immonet": {
-    "url": "https://www.immonet.de/immobiliensuche/sel.do?pageoffset=1&listsize=100&objecttype=1&locationname=Düsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
+    "url": "https://www.immonet.de/immobiliensuche/beta?pageoffset=1&listsize=100&objecttype=1&locationname=D%C3%BCsseldorf&acid=&actype=&district=8717&district=8718&district=8719&district=8720&district=8721&district=8723&district=8724&district=8725&district=8727&district=8728&district=8729&district=8730&district=8731&district=8732&district=8733&district=8737&district=8738&district=8741&district=8745&district=8747&district=8750&district=8752&district=8754&district=8755&district=8756&district=8759&district=8760&district=8761&district=8763&district=8764&district=8765&ajaxIsRadiusActive=false&sortby=19&suchart=1&radius=0&pcatmtypes=1_1&pCatMTypeStoragefield=&parentcat=1&marketingtype=1&fromprice=&toprice=420000&fromarea=90&toarea=&fromplotarea=&toplotarea=&fromrooms=3&torooms=&objectcat=225&objectcat=18&objectcat=17&objectcat=12&objectcat=16&objectcat=181&objectcat=14&objectcat=15&objectcat=226&objectcat=13&wbs=-1&fromyear=&toyear=",
     "enabled": true
   },
   "immowelt": {
-    "url": "https://www.immowelt.de/liste/duesseldorf-benrath/wohnungen/kaufen?geoid=10805111000004%2C10805111000005%2C10805111000006%2C10805111000007%2C10805111000009%2C10805111000010%2C10805111000011%2C10805111000013%2C10805111000014%2C10805111000015%2C10805111000016%2C10805111000017%2C10805111000018%2C10805111000019%2C10805111000023%2C10805111000024%2C10805111000027%2C10805111000032%2C10805111000034%2C10805111000035%2C10805111000039%2C10805111000041%2C10805111000042%2C10805111000043%2C10805111000047%2C10805111000048%2C10805111000049%2C10805111000051%2C10805111000052%2C10805111000053&roomi=3&prima=420000&wflmi=90&sort=createdate%2Bdesc",
+    "url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?d=true&rmi=3&sd=DESC&sf=TIMESTAMP&sp=1",
+    "enabled": true
+  },
+  "immoscout": {
+    "url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search",
+    "enabled": true
+  },
+  "immoswp": {
+    "url": "https://immo.swp.de/suchergebnisse?l=M%C3%BCnchen&r=0km&_multiselect_r=0km&ut=private&t=apartment%3Arental&a=de.muenchen&pf=&pt=&rf=0&rt=0&sf=50&st=&yf=&yt=&ff=&ft=&s=most_recently_updated_first&pa=&o=&ad=&u=",
     "enabled": true
   },
   "kalaydo": {
@@ -17,7 +29,7 @@
     "enabled": true
   },
   "kleinanzeigen": {
-    "url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,",
+    "url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
     "enabled": true
   },
   "neubauKompass": {
@@ -28,4 +40,4 @@
     "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
     "enabled": true
   }
-}
+}

test/provider/utils.test.js

@@ -1,6 +1,13 @@
-const utils = require('../../lib/utils');
+import utils from '../../lib/utils.js';
-const assert = require('assert');
+import assert from 'assert';
-
+import chai from 'chai';
+const expect = chai.expect;
+const fakeWorkingHoursConfig = (from, to) => ({
+  workingHours: {
+    to,
+    from,
+  },
+});
 describe('utils', () => {
   describe('#isOneOf()', () => {
     it('should be false', () => {
@@ -10,4 +17,21 @@ describe('utils', () => {
       assert.equal(utils.isOneOf('bla blub blubber', ['bla']), true);
     });
   });
+  describe('#duringWorkingHoursOrNotSet()', () => {
+    it('should be false', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', '13:00'), 0)).to.be.false;
+    });
+    it('should be true', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('10:00', '16:00'), 1622026740000)).to.be.true;
+    });
+    it('should be true if nothing set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, null), 1622026740000)).to.be.true;
+    });
+    it('should be true if only to is set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig(null, '13:00'), 1622026740000)).to.be.true;
+    });
+    it('should be true if only from is set', () => {
+      expect(utils.duringWorkingHoursOrNotSet(fakeWorkingHoursConfig('12:00', null), 1622026740000)).to.be.true;
+    });
+  });
 });

test/provider/wgGesucht.test.js

@@ -1,35 +1,28 @@
-const mockNotification = require('../mocks/mockNotification');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-const providerConfig = require('./testProvider.json');
+import { get } from '../mocks/mockNotification.js';
-const mockStore = require('../mocks/mockStore');
+import { mockFredy, providerConfig } from '../utils.js';
-const proxyquire = require('proxyquire').noCallThru();
+import chai from 'chai';
-const expect = require('chai').expect;
+import * as provider from '../../lib/provider/wgGesucht.js';
-const provider = require('../../lib/provider/wgGesucht');
+const expect = chai.expect;
-
 describe('#wgGesucht testsuite()', () => {
-  provider.init(providerConfig.wgGesucht, [], []);
+  after(() => {
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
+    similarityCache.stopCacheCleanup();
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
   });
-
+  provider.init(providerConfig.wgGesucht, [], []);
   it('should test wgGesucht provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1');
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-        const notificationObj = mockNotification.get();
+        const notificationObj = get();
         expect(notificationObj.serviceName).to.equal('wgGesucht');
         notificationObj.payload.forEach((notify) => {
           expect(notify).to.be.a('object');
-
           /** check the actual structure **/
-
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.details).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.link).to.be.a('string');
         });

test/queryStringMutator/queryStringMutator.test.js

@@ -0,0 +1,36 @@
+import fs from 'fs';
+import chai from 'chai';
+import { readFile } from 'fs/promises';
+import mutator from '../../lib/services/queryStringMutator.js';
+import queryString from 'query-string';
+const expect = chai.expect;
+
+const data = await readFile(new URL('./testData.json', import.meta.url));
+
+const testData = JSON.parse(data);
+
+let _provider = await Promise.all(
+  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
+);
+
+/**
+ * Test test might look a bit weird at first, but listen stranger...
+ * It's not wise to compare 2 urls, as this means all url params must be in the expected order. This is however not
+ * guaranteed, as params (and their order) are totally variable.
+ */
+describe('queryStringMutator', () => {
+  it('should fix all urls', () => {
+    for (let test of testData) {
+      const provider = _provider.find((p) => p.metaInformation.id === test.id);
+      if (provider == null) {
+        throw new Error(`Cannot find provider for given id: ${test.id}`);
+      }
+      const fixedUrl = mutator(test.url, provider.config.sortByDateParam);
+      const expectedParams = queryString.parseUrl(test.shouldBecome);
+      const actualParams = queryString.parseUrl(fixedUrl);
+      //check if all new params are existing
+      expect(Object.keys(expectedParams.query)).to.include.members(Object.keys(actualParams.query));
+      expect(Object.values(expectedParams.query)).to.include.members(Object.values(actualParams.query));
+    }
+  });
+});

test/queryStringMutator/testData.json

@@ -0,0 +1,33 @@
+[
+  {
+    "url": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=PRIMARY_PRICE_AMOUNT&sp=1",
+    "shouldBecome": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=TIMESTAMP&sp=1",
+    "id": "immowelt"
+  },
+  {
+    "url": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-mieten.html?search=yes",
+    "shouldBecome": "https://www.1a-immobilienmarkt.de/suchen/duesseldorf/wohnung-mieten.html?search=yes&sort_type=newest",
+    "id": "einsAImmobilien"
+  },
+  {
+    "url": "https://www.wg-gesucht.de/1-zimmer-wohnungen-in-Dusseldorf.30.1.1.0.html?sort_column=1&sort_order=0",
+    "shouldBecome": "https://www.wg-gesucht.de/1-zimmer-wohnungen-in-Dusseldorf.30.1.1.0.html?sort_column=0&sort_order=0",
+    "id": "wgGesucht"
+  },
+
+  {
+    "url": "https://www.immonet.de/immobiliensuche/sel.do?sortby=0&suchart=1&objecttype=1&marketingtype=2&parentcat=1&locationname=d%C3%BCsseldorf",
+    "shouldBecome": "https://www.immonet.de/immobiliensuche/sel.do?sortby=19&suchart=1&objecttype=1&marketingtype=2&parentcat=1&locationname=d%C3%BCsseldorf",
+    "id": "immonet"
+  },
+  {
+    "url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten",
+    "shouldBecome": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?sorting=2",
+    "id": "immoscout"
+  },
+  {
+    "url": "https://www.neubaukompass.de/neubau-immobilien/berlin-region/",
+    "shouldBecome": "https://www.neubaukompass.de/neubau-immobilien/berlin-region/?Sortierung=Id&Richtung=DESC",
+    "id": "neubauKompass"
+  }
+]

test/similarity/similarity.test.js

@@ -0,0 +1,39 @@
+import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
+import chai from 'chai';
+const expect = chai.expect;
+describe('similarityCheck', () => {
+  describe('#similarityCheck()', () => {
+    it('should be false', () => {
+      const check = new SimilarityCacheEntry(0);
+      check.setCacheEntry('Hallo');
+      expect(check.hasSimilarEntries('Welt')).to.be.false;
+    });
+    it('should be true', () => {
+      const check = new SimilarityCacheEntry(0);
+      check.setCacheEntry('Hallo');
+      expect(check.hasSimilarEntries('hallo')).to.be.true;
+    });
+    it('should be true', () => {
+      const check = new SimilarityCacheEntry(0);
+      check.setCacheEntry('Selling an incredible house in san francisco');
+      expect(check.hasSimilarEntries('incredible house in san francisco for sale')).to.be.true;
+    });
+    it('should be true', () => {
+      const check = new SimilarityCacheEntry(0);
+      check.setCacheEntry('a');
+      check.setCacheEntry('b');
+      check.setCacheEntry('c');
+      check.setCacheEntry('d');
+      expect(check.hasSimilarEntries('b')).to.be.true;
+    });
+    it('should be false', () => {
+      const check = new SimilarityCacheEntry(0);
+      check.setCacheEntry(
+        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.'
+      );
+      check.setCacheEntry(
+        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
+      );
+    });
+  });
+});

test/utils.js

@@ -0,0 +1,17 @@
+import { readFile } from 'fs/promises';
+import esmock from 'esmock';
+import * as mockStore from './mocks/mockStore.js';
+import { send } from './mocks/mockNotification.js';
+
+export const providerConfig = JSON.parse(await readFile(new URL('./provider/testProvider.json', import.meta.url)));
+
+export const mockFredy = async () => {
+  return await esmock('../lib/FredyRuntime', {
+    '../lib/services/storage/listingsStorage.js': {
+      ...mockStore,
+    },
+    '../lib/notification/notify.js': {
+      send,
+    },
+  });
+};

ui/src/App.js

@@ -1,89 +0,0 @@
-import React, { useEffect } from 'react';
-
-import InsufficientPermission from './components/permission/InsufficientPermission';
-import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
-import ToastsContainer from './components/toasts/ToastContainer';
-import JobMutation from './views/jobs/mutation/JobMutation';
-import UserMutator from './views/user/mutation/UserMutator';
-import ToastContext from './components/toasts/ToastContext';
-import JobInsight from './views/jobs/insights/JobInsight';
-import { useDispatch, useSelector } from 'react-redux';
-import useToast from './components/toasts/useToast';
-import { Switch, Redirect } from 'react-router-dom';
-import Logout from './components/logout/Logout';
-import Logo from './components/logo/Logo';
-import Menu from './components/menu/Menu';
-import Login from './views/login/Login';
-import Users from './views/user/Users';
-import Jobs from './views/jobs/Jobs';
-import { Route } from 'react-router';
-
-import './App.less';
-
-export default function FredyApp() {
-  const dispatch = useDispatch();
-  const [showToast, onToastFinished, toasts] = useToast();
-  const [loading, setLoading] = React.useState(true);
-  const currentUser = useSelector((state) => state.user.currentUser);
-
-  useEffect(async () => {
-    await dispatch.provider.getProvider();
-    await dispatch.jobs.getJobs();
-    await dispatch.notificationAdapter.getAdapter();
-    await dispatch.user.getCurrentUser();
-
-    setLoading(false);
-  }, [currentUser?.userId]);
-
-  const needsLogin = () => {
-    return currentUser == null || Object.keys(currentUser).length === 0;
-  };
-
-  const isAdmin = () => currentUser != null && currentUser.isAdmin;
-
-  const login = () => (
-    <Switch>
-      <Route name="Login" path={'/login'} component={Login} />
-      <Redirect from="*" to={'/login'} />
-    </Switch>
-  );
-
-  return loading ? null : needsLogin() ? (
-    login()
-  ) : (
-    <ToastContext.Provider value={{ showToast }}>
-      <div className="app">
-        <div className="app__container">
-          <Logout />
-          <Logo width={190} white />
-          <Menu isAdmin={isAdmin()} />
-          <ToastsContainer toasts={toasts} onToastFinished={onToastFinished} />
-          <Switch>
-            <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
-            <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
-            <Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
-            <Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
-            <Route name="Job overview" path={'/jobs'} component={Jobs} />
-            <PermissionAwareRoute
-              name="Create new User"
-              path="/users/new"
-              component={<UserMutator />}
-              currentUser={currentUser}
-            />
-            <PermissionAwareRoute
-              name="Edit a user"
-              path="/users/edit/:userId"
-              component={<UserMutator />}
-              currentUser={currentUser}
-            />
-            <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
-
-            <Redirect from="/" to={'/jobs'} />
-          </Switch>
-        </div>
-      </div>
-    </ToastContext.Provider>
-  );
-}
-
-FredyApp.displayName = 'FredyApp';

ui/src/App.jsx

@@ -0,0 +1,95 @@
+import React, { useEffect } from 'react';
+
+import InsufficientPermission from './components/permission/InsufficientPermission';
+import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
+import GeneralSettings from './views/generalSettings/GeneralSettings';
+import JobMutation from './views/jobs/mutation/JobMutation';
+import UserMutator from './views/user/mutation/UserMutator';
+import JobInsight from './views/jobs/insights/JobInsight.jsx';
+import { useDispatch, useSelector } from 'react-redux';
+import { Switch, Redirect } from 'react-router-dom';
+import Logout from './components/logout/Logout';
+import Logo from './components/logo/Logo';
+import Menu from './components/menu/Menu';
+import Login from './views/login/Login';
+import Users from './views/user/Users';
+import Jobs from './views/jobs/Jobs';
+import { Route } from 'react-router';
+
+import './App.less';
+
+export default function FredyApp() {
+  const dispatch = useDispatch();
+  const [loading, setLoading] = React.useState(true);
+  const currentUser = useSelector((state) => state.user.currentUser);
+
+  useEffect(() => {
+    async function init() {
+      await dispatch.user.getCurrentUser();
+      if (!needsLogin()) {
+        await dispatch.provider.getProvider();
+        await dispatch.jobs.getJobs();
+        await dispatch.jobs.getProcessingTimes();
+        await dispatch.notificationAdapter.getAdapter();
+      }
+      setLoading(false);
+    }
+
+    init();
+  }, [currentUser?.userId]);
+
+  const needsLogin = () => {
+    return currentUser == null || Object.keys(currentUser).length === 0;
+  };
+
+  const isAdmin = () => currentUser != null && currentUser.isAdmin;
+
+  const login = () => (
+    <Switch>
+      <Route name="Login" path={'/login'} component={Login} />
+      <Redirect from="*" to={'/login'} />
+    </Switch>
+  );
+
+  return loading ? null : needsLogin() ? (
+    login()
+  ) : (
+    <div className="app">
+      <div className="app__container">
+        <Logout />
+        <Logo width={190} white />
+        <Menu isAdmin={isAdmin()} />
+        <Switch>
+          <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
+          <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />
+          <Route name="Edit a Job" path={'/jobs/edit/:jobId'} component={JobMutation} />
+          <Route name="Insights into a Job" path={'/jobs/insights/:jobId'} component={JobInsight} />
+          <Route name="Job overview" path={'/jobs'} component={Jobs} />
+          <PermissionAwareRoute
+            name="Create new User"
+            path="/users/new"
+            component={<UserMutator />}
+            currentUser={currentUser}
+          />
+          <PermissionAwareRoute
+            name="Edit a user"
+            path="/users/edit/:userId"
+            component={<UserMutator />}
+            currentUser={currentUser}
+          />
+          <PermissionAwareRoute name="Users" path="/users" component={<Users />} currentUser={currentUser} />
+          <PermissionAwareRoute
+            name="General Settings"
+            path="/generalSettings"
+            component={<GeneralSettings />}
+            currentUser={currentUser}
+          />
+
+          <Redirect from="/" to={'/jobs'} />
+        </Switch>
+      </div>
+    </div>
+  );
+}
+
+FredyApp.displayName = 'FredyApp';

ui/src/App.less

@@ -4,10 +4,40 @@
   width:100%;
 
   &__container {
-    width: 100%;
-
     padding: 1rem 1rem;
-    background-color: #3f3e3ef5;
+    color: var(--semi-color-text-0);
-    color: #f1f1f1;
+    background-color: #232429;
   }
+}
+
+.ui.inverted.segment{
+  background: #31303078!important;
+}
+
+.ui.black.label, .ui.black.labels .label {
+  background-color: #31303078!important;
+}
+
+a:link {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: none;
+}
+
+a:visited {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: none;
+}
+
+a:hover {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: underline;
+}
+
+a:active {
+  color: #54a9ff;
+  background-color: transparent;
+  text-decoration: underline;
 }

ui/src/Index.jsx

@@ -4,19 +4,24 @@ import { reduxStore } from './services/rematch/store';
 import { HashRouter } from 'react-router-dom';
 import { createHashHistory } from 'history';
 import { Provider } from 'react-redux';
-import ReactDOM from 'react-dom';
+import { createRoot } from 'react-dom/client';
+import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
+import { LocaleProvider } from '@douyinfe/semi-ui';
 
+const container = document.getElementById('fredy');
+const root = createRoot(container);
 const history = createHashHistory();
 
 import App from './App';
 
 import './Index.less';
 
-ReactDOM.render(
+root.render(
   <Provider store={reduxStore}>
     <HashRouter history={history}>
-      <App />
+      <LocaleProvider locale={en_US}>
+        <App />
+      </LocaleProvider>
     </HashRouter>
-  </Provider>,
+  </Provider>
-  document.getElementById('fredy')
 );

ui/src/Index.less

@@ -2,5 +2,14 @@ body, html {
   margin: 0;
   height: 100%;
   width: 100%;
-  background-color: #3f3e3ef5;
+  background-color: #232429;
+}
+
+.semi-table-row-head{
+  background-color: #2b2b2b !important;
+  color: #fff !important;
+}
+
+.semi-table-row-cell {
+  background-color: #333333 !important;
 }

ui/src/components/headline/Headline.js

@@ -1,12 +0,0 @@
-import React from 'react';
-import { Header } from 'semantic-ui-react';
-
-import './Headline.less';
-
-export default function Headline({ text, size = 'medium', className = '' } = {}) {
-  return (
-    <Header className={`headline ${className}`} size={size}>
-      {text}
-    </Header>
-  );
-}

ui/src/components/headline/Headline.jsx

@@ -0,0 +1,11 @@
+import React from 'react';
+import { Typography } from '@douyinfe/semi-ui';
+
+export default function Headline({ text, size = 3 } = {}) {
+  const { Title } = Typography;
+  return (
+    <Title heading={size} style={{ marginBottom: '1rem' }}>
+      {text}
+    </Title>
+  );
+}

ui/src/components/headline/Headline.less

@@ -1,3 +0,0 @@
-.headline{
-  color: #f1f1f1 !important;
-}

ui/src/components/logout/Logout.jsx

@@ -1,20 +1,20 @@
 import React from 'react';
-import { Button } from 'semantic-ui-react';
+import { Button } from '@douyinfe/semi-ui';
 import { xhrPost } from '../../services/xhr';
-
+import { IconUser } from '@douyinfe/semi-icons';
 const Logout = function Logout() {
   return (
     <Button
-      content="Logout"
+      icon={<IconUser />}
-      labelPosition="left"
+      type="danger"
-      icon="user"
+      theme="solid"
-      size="mini"
       onClick={async () => {
         await xhrPost('/api/login/logout');
         location.reload();
       }}
-      negative
+    >
-    />
+      Logout
+    </Button>
   );
 };
 

ui/src/components/menu/Menu.js

@@ -1,39 +0,0 @@
-import React from 'react';
-import { useHistory } from 'react-router-dom';
-import { Menu } from 'semantic-ui-react';
-
-import './Menu.less';
-import { useLocation } from 'react-router';
-
-const TopMenu = function TopMenu({ isAdmin }) {
-  const history = useHistory();
-  const location = useLocation();
-
-  const isActiveRoute = (name) => location.pathname.indexOf(name) !== -1;
-
-  return (
-    <Menu pointing secondary className="topMenu">
-      <Menu.Item
-        name="jobs"
-        active={isActiveRoute('jobs')}
-        className={isActiveRoute('jobs') ? 'topMenu__active' : 'topMenu__item'}
-        onClick={() => history.push('/jobs')}
-      >
-        Job Configuration
-      </Menu.Item>
-
-      {isAdmin && (
-        <Menu.Item
-          name="user"
-          active={isActiveRoute('users')}
-          className={isActiveRoute('users') ? 'topMenu__active' : 'topMenu__item'}
-          onClick={() => history.push('/users')}
-        >
-          User configuration
-        </Menu.Item>
-      )}
-    </Menu>
-  );
-};
-
-export default TopMenu;

ui/src/components/menu/Menu.jsx

@@ -0,0 +1,55 @@
+import React from 'react';
+import { useHistory } from 'react-router-dom';
+import { Tabs, TabPane } from '@douyinfe/semi-ui';
+
+import { useLocation } from 'react-router';
+import { IconUser, IconTerminal, IconSetting } from '@douyinfe/semi-icons';
+
+function parsePathName(name) {
+  const split = name.split('/').filter((s) => s.length !== 0);
+  return '/' + split[0];
+}
+
+const TopMenu = function TopMenu({ isAdmin }) {
+  const history = useHistory();
+  const location = useLocation();
+  return (
+    <Tabs type="line" activeKey={parsePathName(location.pathname)} onTabClick={(key) => history.push(key)}>
+      <TabPane
+        itemKey="/jobs"
+        tab={
+          <span>
+            <IconTerminal />
+            Jobs
+          </span>
+        }
+      />
+
+      {isAdmin && (
+        <TabPane
+          itemKey="/users"
+          tab={
+            <span>
+              <IconUser />
+              User
+            </span>
+          }
+        />
+      )}
+
+      {isAdmin && (
+        <TabPane
+          itemKey="/generalSettings"
+          tab={
+            <span>
+              <IconSetting />
+              General
+            </span>
+          }
+        />
+      )}
+    </Tabs>
+  );
+};
+
+export default TopMenu;