next version

* starting docu on reverse engineering immoscout api

* improving immoscout reverse engineering and adding support for most other types

.dockerignore

@@ -1,3 +1,7 @@
-node_modules
+node_modules/
 npm-debug.log
-test
+test/
+conf/
+db/
+.git/
+.github/

.eslintrc.cjs

@@ -1,12 +1,11 @@
 module.exports = {
   env: {
-    commonjs: true,
-    es6: true,
+    es2021: true,
     node: true,
     browser: true,
     mocha: true,
   },
-  parser: 'babel-eslint',
+  parser: '@babel/eslint-parser',
   extends: ['eslint:recommended', 'prettier'],
   plugins: ['react'],
   globals: {
@@ -17,7 +16,6 @@ module.exports = {
     fetch: true,
   },
   parserOptions: {
-    ecmaVersion: 2018,
     sourceType: 'module',
   },
   rules: {
@@ -205,10 +203,6 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
     'react/self-closing-comp': 'warn',
 
-    // Enforce spaces before the closing bracket of self-closing JSX elements
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
-    'react/jsx-space-before-closing': ['warn', 'always'],
-
     // Enforce component methods order
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
     'react/sort-comp': 'off',
@@ -239,7 +233,7 @@ module.exports = {
 
     // only .jsx files may have JSX
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
-    'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
+    'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
 
     // prevent accidental JS comments from being injected into JSX as text
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
@@ -284,15 +278,5 @@ module.exports = {
     // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
     'react/no-children-prop': 'warn',
 
-    // Validate whitespace in and around the JSX opening and closing brackets
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
-    'react/jsx-tag-spacing': [
-      'warn',
-      {
-        closingSlash: 'never',
-        beforeSelfClosing: 'always',
-        afterOpening: 'never',
-      },
-    ],
   },
 };

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/docker.yml

@@ -0,0 +1,47 @@
+name: Create and publish Docker image
+on:
+  push:
+    branches:
+      - 'master'
+    tags:
+      - '*'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64

.github/workflows/stales.yml

@@ -0,0 +1,21 @@
+name: "Close stale issues and PRs"
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          days-before-stale: 30
+          days-before-close: 7
+          stale-issue-message: "This issue has been automatically marked as stale due to inactivity."
+          stale-pr-message: "This PR has been automatically marked as stale due to inactivity."
+          close-issue-message: "Closing this issue due to prolonged inactivity."
+          close-pr-message: "Closing this PR due to prolonged inactivity."
+          exempt-issue-labels: "keep-open"
+          exempt-pr-labels: "keep-open"
+          only: "pulls"

.github/workflows/test.yml

@@ -0,0 +1,23 @@
+name: Test
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  schedule:
+    - cron: '0 12 * * *'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'yarn'
+
+      - run: yarn install
+      - run: yarn test

.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

.travis.yml

@@ -1,2 +0,0 @@
-sudo: false
-language: node_js

CHANGELOG.md

@@ -1,4 +1,34 @@
-###### [V5.4.0]
+Newer release changelog see https://github.com/orangecoding/fredy/releases
+
+------------
+
+###### [V5.5.0]
+- Upgrading dependencies
+- fixing provider
+- allow multiple instances of 1 provider  
+- __BREAKING__: Minimum node version is now 16
+
+###### [V5.4.6]
+- Adding Instana node.js monitoring
+- 
+###### [V5.4.5]
+- Adding Instana node.js monitoring 
+
+###### [V5.4.4]
+- Add support for Immo Südwest Presse (immo.swp.de)
+- Telegram: Use job name instead of ID and link in title
+- Fix race condition if user ID is in session but not in user store
+- Allow visiting the original provider URL
+
+###### [V5.4.3]
+- re-writing readme
+- improving docker build
+- using github's actions to build docker and test automatically
+
+###### [V5.4.2]
+- Fixing prod build
+
+###### [V5.4.1]
 - Upgrading dependencies
 - Provider urls are now automagically been changed to include the correct sort order for search results
 
@@ -45,4 +75,4 @@ on the new ui and use the values from your previous config file if needed.
 [BREAKING CHANGES]
 - The config has been changed, the config of V1.x will not work any longer
 - Sources have been renamed to provider
-```
+```

CONTRIBUTING.md

@@ -17,7 +17,7 @@ function normalize(o) {
   return Object.assign(o, { id });
 }
 
-//apply blaclist if needed
+//apply blacklist if needed
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
@@ -106,9 +106,7 @@ exports.config = {
 ```
 
 #### Running Tests
-If you've written a new provider you are an awesome person. You know it and I do. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
-
-To write tests for provider, you need to use Node 8 as the tests are using `async / await`
+If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
 
 #### Codestyle
 I'm using Eslint to maintain quote style and quality. Do not skip it...

Dockerfile

@@ -0,0 +1,25 @@
+FROM node:20
+
+WORKDIR  /fredy
+
+COPY . /fredy
+
+RUN apt-get update && apt-get install -y chromium
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+
+RUN yarn install
+
+RUN yarn global add pm2
+
+RUN yarn run prod
+
+RUN mkdir /db /conf && \
+  chown 1000:1000 /db /conf && \
+  chmod 777 -R /db/ && \
+  ln -s /db /fredy/db && ln -s /conf /fredy/conf
+
+EXPOSE 9998
+
+CMD pm2-runtime index.js

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Christian Kellner
+Copyright (c) 2025 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,77 +1,79 @@
 <img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">  
 
-[![Build Status](https://travis-ci.org/orangecoding/fredy.svg?branch=master)](https://travis-ci.org/orangecoding/fredy)
+![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
 
-Searching an apartment in Germany can be quite frustrating. Not any longer as Fredy will take over and only notifies you once new listings have been found that matches your requirements.
+Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
 
-_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings to you once they appear. The list of available services can easily be extended. For your convenience, a ui helps you to configure your search jobs.   
+_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings to you once they become available. The list of available services can easily be extended. For your convenience, _Fredy_ has a UI to help you configure your search jobs.   
 
-If _Fredy_ found matching results, it will send them to you via Slack, Email, Telegram etc. (More adapter possible.) As _Fredy_ will store the listings it has found, new results will not be sent twice (and as a side-effect, _Fredy_ can show some statistics..). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent when posted on various platforms. (Happens more often than one might think)
+If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
+
+# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
+If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
+
+[![JetBrains logo.](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg)](https://jb.gg/OpenSourceSupport)
+
+_Fredy_ is supported by JetBrains under Open Source Support Program
+
+## Demo
+If you want to try out _Fredy_, you can access the demo version [here](https://fredy.orange-coding.net) 🤘
 
 ## Usage  
 
-- Make sure to use NodeJs 12 and above
-- Run the following commands
+- Make sure to use Node.js 20 or above
+- Run the following commands:
 ```ssh
 yarn (or npm install)
 yarn run prod
 yarn run start
 ```
-_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` both for username and password. (You should change the password asap when you plan to run Fredy on your server.)
+_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
 
 <p align="center">
-  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
+  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot1.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;
-  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
+  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_2.png" width="30%">
 &nbsp; &nbsp; &nbsp; &nbsp;  
-  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
+  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
 </p>
-<p align="center">
-
-</p>
-
-## Immoscout
-I have added **experimental** support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successful validated) re-send the cookies each time.
-
-To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the ApiKey in the "General Settings" tab (visible when logged in as administrator).
-The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time it works pretty good :)
-
-If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefor they've decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (No I don't get any money for recommending good services...)
-
 
 ## Understanding the fundamentals
-There are 3 important parts in Fredy, that you need to understand leveraging the full power of _Fredy_.
-
-#### Adapter
-_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few. Those services are called adapter within _Fredy_. When creating a new job, you can choose 1 or many adapter.    
-An adapter holds the url that points to the search results for the service. If you go to immonet.de and search for something, the shown url in the browser is what the adapter needs to do it's magic.   
-**It is important that you order the search results by date, so that _Fredy_ always picks the latest ones first**
+There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.
 
 #### Provider
-_Fredy_ supports multiple provider. Slack, SendGrid, Telegram etc. A search job can have as many provider as supported by _Fredy_. Each provider needs different configuration values, which you have to provide when using it. A provider itself dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few examples. Those services are called providers within _Fredy_. When creating a new job, you can choose one or more providers.
+A provider contains the URL that points to the search results for the respective service. If you go to immonet.de and search for something, the displayed URL in the browser is what the provider needs to do its magic.
+**It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
+
+#### Adapter
+_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
 
 #### Jobs
-A Job wraps adapter and provider. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
+A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
 
 ## Creating your first job
-To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self explanatory, however there's one important thing.
-When configuring adapter, before copying the url from your browser make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
+To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self-explanatory, however there is one important thing.
+When configuring providers, before copying the URL from your browser, make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
 
 ## User management
-As an administrator, you can create/edit and remove user from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, the jobs will also be removed.
-
+As an administrator, you can create, edit and remove users from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, their jobs will also be removed.
 
 # Development
 
-### Running Fredy in dev mode
-To run _Fredy_ in dev mode, you need to run the backend & frontend separately. Run the backend in your favorite IDE, the frontend can be started from the terminal.
+### Running Fredy in development mode
+To run _Fredy_ in development mode, you need to run the backend & frontend separately.
+Start the backend with:
+```shell
+yarn run start
+```
+For the frontend, run:
 ```shell
 yarn run dev
 ```
-You should now be able to access _Fredy_ with your browser. Go to `http://localhost:9000`
+You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
 
 ### Running Tests
-To run the tests, simply run
+To run the tests, run
 ```shell
 yarn run test
 ```
@@ -79,22 +81,18 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-## Immoscout 
-I have added EXPERIMENTAL support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way
-around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/).
+### Immoscout
+Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
 
-To be able to use Immoscout, you need to create an account and copy the apiKey into the config file under /conf/config.json.
-The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time
-it works pretty good :)
-
-If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... (No I don't get any money for recommending good service)
-
-#### Contribution guidelines  
-
-See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)  
+# Analytics
+Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 
+Before you freak out, let me explain...         
+If you agree, Fredy will send a ping to my Mixpanel project each time it runs.   
+The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.</p>
+**Thanks**🤘
 
 # Docker   
-Use the Dockerfile in this Repository to build an image.  
+Use the Dockerfile in this repository to build an image.  
 
 Example: `docker build -t fredy/fredy /path/to/your/Dockerfile`  
 
@@ -102,12 +100,25 @@ Or use docker-compose:
 
 Example `docker-compose build`
 
+Or use the container that will be built automatically.
+
+`docker pull ghcr.io/orangecoding/fredy:master`
+
 ## Create & run a container  
 
-Put your config.json to `/path/to/your/conf/`
+Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+### 👐 Contributing
+Thanks to all the people who already contributed!
+
+<a href="https://github.com/orangecoding/fredy/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=orangecoding/fredy" />
+</a>
+
+See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
+
 ## Logs  
 
-You can browse the logs with  `docker logs fredy -f`  
+You can browse the logs with  `docker logs fredy -f`.

build/Dockerfile

@@ -1,33 +0,0 @@
-FROM alpine:latest AS build
-# use given repository, default below:
-ARG repo=https://github.com/orangecoding/fredy.git
-
-RUN mkdir -p /usr/src/
-#Install Software
-RUN apk add --update nodejs npm git
-
-# Output used repository
-RUN echo "Cloning from $repo"
-
-RUN cd /usr/src && git clone $repo
-
-RUN ln -s /usr/src/fredy/conf/ /conf
-
-# create db folder
-RUN mkdir /usr/src/fredy/db/
-
-RUN ln -s /usr/src/fredy/db/ /db
-
-RUN npm i -g yarn
-
-RUN cd /usr/src/fredy/ && yarn
-
-WORKDIR  /usr/src/fredy
-
-RUN yarn run prod
-
-EXPOSE 9998
-
-VOLUME [ "/conf", "/db" ]
-# --no-daemon is required for keeping Container alive
-CMD node index.js --no-daemon

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null}

doc/Untitled Diagram.drawio

@@ -0,0 +1,84 @@
+<mxfile host="app.diagrams.net" modified="2022-01-29T18:34:51.211Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" etag="W0jmvptvMSkuHq89hwUy" version="16.5.2" type="github">
+  <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
+    <mxGraphModel dx="850" dy="907" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-5" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-3" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="Job1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="100" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-2">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-3">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="4kAlOAlRylSy7JMoHAEd-4">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-7" value="FredyRuntime" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="110" y="120" width="360" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-6" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-0" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-0" value="Job2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-7" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-1" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-1" value="Job3" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="50" width="120" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-2" target="4kAlOAlRylSy7JMoHAEd-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-2" value="Provider1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="100" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-3">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="290" y="290" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-3" value="Provider2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-4" target="4kAlOAlRylSy7JMoHAEd-12">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-4" value="Provider3" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="210" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-12" target="4kAlOAlRylSy7JMoHAEd-16">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-12" value="Similarity check" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="110" y="290" width="360" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-16" target="4kAlOAlRylSy7JMoHAEd-18">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-16" value="Found similarity" style="rhombus;whiteSpace=wrap;html=1;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="250" y="360" width="80" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="4kAlOAlRylSy7JMoHAEd-18" target="4kAlOAlRylSy7JMoHAEd-19">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-18" value="Notification Adapter1" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="460" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-19" value="Notification Adapter2" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="230" y="520" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="4kAlOAlRylSy7JMoHAEd-22" value="No" style="text;html=1;resizable=0;autosize=1;align=center;verticalAlign=middle;points=[];fillColor=none;strokeColor=none;rounded=0;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="300" y="440" width="30" height="20" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

docker-compose.yml

@@ -1,13 +1,11 @@
-version: '3.3'
+version: '3.8'
 services:
   fredy:
     container_name: fredy
     # build from empty build folder to reduce size of image
     build:
-      context: ./build
+      context: .
       dockerfile: Dockerfile
-      args:
-        repo: https://github.com/orangecoding/fredy.git
     image: fredy/fredy
     # map existing config and database
     volumes:
@@ -15,3 +13,4 @@ services:
       - ./db:/db
     ports:
         - 9998:9998
+    restart: unless-stopped

index.html

@@ -4,14 +4,13 @@
     <meta charset="UTF-8"
           name="viewport"
           content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
-    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css">
     <meta name="google" content="notranslate">
 
     <title>Fredy</title>
 </head>
-<body>
+<body theme-mode="dark">
 
 <div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
 </body>
-<script src="fredy.bundle.js"></script>
+<script type="module" src="/ui/src/Index.jsx"></script>
 </html>

index.js

@@ -1,71 +1,61 @@
-const fs = require('fs');
-
+import fs from 'fs';
+import {config} from './lib/utils.js';
+import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
+import { setLastJobExecution } from './lib/services/storage/listingsStorage.js';
+import * as jobStorage from './lib/services/storage/jobStorage.js';
+import FredyRuntime from './lib/FredyRuntime.js';
+import { duringWorkingHoursOrNotSet } from './lib/utils.js';
+import './lib/api/api.js';
+import {track} from './lib/services/tracking/Tracker.js';
+import {handleDemoUser} from './lib/services/storage/userStorage.js';
+import {cleanupDemoAtMidnight} from './lib/services/demoCleanup.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
 }
-
 const path = './lib/provider';
 const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
-const config = require('./conf/config.json');
-
-const similarityCache = require('./lib/services/similarity-check/similarityCache');
-const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
-const jobStorage = require('./lib/services/storage/jobStorage');
-const FredyRuntime = require('./lib/FredyRuntime');
-
-const { duringWorkingHoursOrNotSet } = require('./lib/utils');
-
-//starting the api service
-require('./lib/api/api');
-
 //assuming interval is always in minutes
 const INTERVAL = config.interval * 60 * 1000;
-
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+if(config.demoMode){
+    console.info('Running in demo mode');
+    cleanupDemoAtMidnight();
+}
 /* eslint-enable no-console */
+const fetchedProvider = await Promise.all(
+  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
+);
+
+handleDemoUser();
+
 setInterval(
   (function exec() {
     const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
-
-    if (isDuringWorkingHoursOrNotSet) {
-      config.lastRun = Date.now();
-      jobStorage
-        .getJobs()
-        .filter((job) => job.enabled)
-        .forEach((job) => {
-          const providerIds = job.provider.map((provider) => provider.id);
-
-          provider
-            .filter((provider) => provider.endsWith('.js'))
-            .map((pro) => require(`${path}/${pro}`))
-            .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
-            .forEach(async (pro) => {
-              const providerId = pro.metaInformation.id;
-              if (providerId == null || providerId.length === 0) {
-                throw new Error('Provider id must not be empty. => ' + pro);
-              }
-              const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
-              if (providerConfig == null) {
-                throw new Error(`Provider Config for provider with id ${providerId} not found.`);
-              }
-              pro.init(providerConfig, job.blacklist);
-              await new FredyRuntime(
-                pro.config,
-                job.notificationAdapter,
-                providerId,
-                job.id,
-                similarityCache
-              ).execute();
-              setLastJobExecution(job.id);
-            });
-        });
-    } else {
-      /* eslint-disable no-console */
-      console.debug('Working hours set. Skipping as outside of working hours.');
-      /* eslint-enable no-console */
-    }
+        if(!config.demoMode) {
+            if (isDuringWorkingHoursOrNotSet) {
+                track();
+                config.lastRun = Date.now();
+                jobStorage
+                    .getJobs()
+                    .filter((job) => job.enabled)
+                    .forEach((job) => {
+                        job.provider
+                            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
+                            .forEach(async (prov) => {
+                                const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
+                                pro.init(prov, job.blacklist);
+                                await new FredyRuntime(pro.config, job.notificationAdapter, prov.id, job.id, similarityCache).execute();
+                                setLastJobExecution(job.id);
+                            });
+                    });
+            } else {
+                /* eslint-disable no-console */
+                console.debug('Working hours set. Skipping as outside of working hours.');
+                /* eslint-enable no-console */
+            }
+        }
     return exec;
   })(),
   INTERVAL

lib/FredyRuntime.js

@@ -1,10 +1,8 @@
-const { NoNewListingsWarning } = require('./errors');
-const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
-
-const notify = require('./notification/notify');
-const xray = require('./services/scraper');
-const scrapingAnt = require('./services/scrapingAnt');
-const urlModifier = require('./services/queryStringMutator');
+import { NoNewListingsWarning } from './errors.js';
+import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
+import * as notify from './notification/notify.js';
+import Extractor from './services/extractor/extractor.js';
+import urlModifier from './services/queryStringMutator.js';
 
 class FredyRuntime {
   /**
@@ -28,7 +26,7 @@ class FredyRuntime {
       //modify the url to make sure search order is correctly set
       Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
         //scraping the site and try finding new listings
-        .then(this._getListings.bind(this))
+        .then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
         //bring them in a proper form (dictated by the provider)
         .then(this._normalize.bind(this))
         //filter listings with stuff tagged by the blacklist of the provider
@@ -47,45 +45,24 @@ class FredyRuntime {
   }
 
   _getListings(url) {
+    const extractor = new Extractor();
     return new Promise((resolve, reject) => {
-      const id = this._providerId;
-      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
-        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
-        /* eslint-disable no-console */
-        console.log(error);
-        /* eslint-enable no-console */
-        reject(error);
-        return;
-      }
-      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
-      try {
-        if (this._providerConfig.paginate != null) {
-          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
-            //the first 2 pages should be enough here
-            //TODO: Think about automagically sort by date
-            .limit(2)
-            .paginate(this._providerConfig.paginate)
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        } else {
-          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        }
-      } catch (error) {
-        reject(error);
-        console.error(error);
-      }
+      extractor
+        .execute(url, this._providerConfig.waitForSelector)
+        .then(() => {
+          const listings = extractor.parseResponseText(
+            this._providerConfig.crawlContainer,
+            this._providerConfig.crawlFields,
+            url,
+          );
+          resolve(listings == null ? [] : listings);
+        })
+        .catch((err) => {
+          reject(err);
+          /* eslint-disable no-console */
+          console.error(err);
+          /* eslint-enable no-console */
+        });
     });
   }
 
@@ -94,16 +71,17 @@ class FredyRuntime {
   }
 
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
 
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
-
     if (newListings.length === 0) {
       throw new NoNewListingsWarning();
     }
-
     return newListings;
   }
 
@@ -143,4 +121,4 @@ class FredyRuntime {
   }
 }
 
-module.exports = FredyRuntime;
+export default FredyRuntime;

lib/api/api.js

@@ -1,44 +1,40 @@
-const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
-const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
-const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
-const { analyticsRouter } = require('./routes/analyticsRouter');
-const { providerRouter } = require('./routes/providerRouter');
-const { loginRouter } = require('./routes/loginRoute');
-const config = require('../../conf/config.json');
-const { userRouter } = require('./routes/userRoute');
-const { jobRouter } = require('./routes/jobRouter');
-const bodyParser = require('body-parser');
-const service = require('restana')();
-const files = require('serve-static');
-const path = require('path');
-
-const staticService = files(path.join(__dirname, '../../ui/public'));
-
+import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
+import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
+import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
+import { analyticsRouter } from './routes/analyticsRouter.js';
+import { providerRouter } from './routes/providerRouter.js';
+import { loginRouter } from './routes/loginRoute.js';
+import { config } from '../utils.js';
+import { userRouter } from './routes/userRoute.js';
+import { jobRouter } from './routes/jobRouter.js';
+import bodyParser from 'body-parser';
+import restana from 'restana';
+import files from 'serve-static';
+import path from 'path';
+import { getDirName } from '../utils.js';
+import {demoRouter} from './routes/demoRouter.js';
+const service = restana();
+const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
 
 service.use(bodyParser.json());
-
 service.use(cookieSession());
-
 service.use(staticService);
-
 service.use('/api/admin', authInterceptor());
 service.use('/api/jobs', authInterceptor());
-
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
-
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
 service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
 service.use('/api/jobs', jobRouter);
-
 service.use('/api/login', loginRouter);
+//this route is unsecured intentionally as it is being queried from the login page
+service.use('/api/demo', demoRouter);
 
 /* eslint-disable no-console */
 service.start(PORT).then(() => {
   console.info(`Started API service on port ${PORT}`);
 });
-/* eslint-enable no-console */

lib/api/routes/analyticsRouter.js

@@ -1,12 +1,10 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as listingStorage from '../../services/storage/listingsStorage.js';
+const service = restana();
 const analyticsRouter = service.newRouter();
-const listingStorage = require('../../services/storage/listingsStorage');
-
 analyticsRouter.get('/:jobId', async (req, res) => {
   const { jobId } = req.params;
-
   res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
   res.send();
 });
-
-exports.analyticsRouter = analyticsRouter;
+export { analyticsRouter };

lib/api/routes/demoRouter.js

@@ -0,0 +1,11 @@
+import restana from 'restana';
+import {config} from '../../utils.js';
+const service = restana();
+const demoRouter = service.newRouter();
+
+demoRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, {demoMode: config.demoMode});
+  res.send();
+});
+
+export { demoRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,18 +1,24 @@
-const service = require('restana')();
+import restana from 'restana';
+import {config, getDirName, readConfigFromStorage, refreshConfig} from '../../utils.js';
+import fs from 'fs';
+import {handleDemoUser} from '../../services/storage/userStorage.js';
+const service = restana();
 const generalSettingsRouter = service.newRouter();
-const config = require('../../../conf/config.json');
-const fs = require('fs');
-
 generalSettingsRouter.get('/', async (req, res) => {
   res.body = Object.assign({}, config);
   res.send();
 });
-
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
-
   try {
-    fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
+    if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+      return;
+    }
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({...currentConfig, ...settings}));
+    await refreshConfig();
+    handleDemoUser();
   } catch (err) {
     console.error(err);
     res.send(new Error('Error while trying to write settings.'));
@@ -20,5 +26,4 @@ generalSettingsRouter.post('/', async (req, res) => {
   }
   res.send();
 });
-
-exports.generalSettingsRouter = generalSettingsRouter;
+export { generalSettingsRouter };

lib/api/routes/jobRouter.js

@@ -1,12 +1,11 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import * as userStorage from '../../services/storage/userStorage.js';
+import { config } from '../../utils.js';
+import { isAdmin } from '../security.js';
+import { trackDemoJobCreated } from '../../services/tracking/Tracker.js';
+const service = restana();
 const jobRouter = service.newRouter();
-const jobStorage = require('../../services/storage/jobStorage');
-const userStorage = require('../../services/storage/userStorage');
-const immoscoutProvider = require('../../provider/immoscout');
-const config = require('../../../conf/config.json');
-
-const { isAdmin } = require('../security');
-
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
   if (userId == null) {
@@ -16,39 +15,23 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-
   return user.isAdmin || job.userId === job.userId;
 }
-
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
-
   //show only the jobs which belongs to the user (or all of the user is an admin)
   res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
-
   res.send();
 });
-
 jobRouter.get('/processingTimes', async (req, res) => {
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
   };
-
   res.send();
 });
-
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
-  if (
-    provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
-    (config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
-  ) {
-    res.send(
-      new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
-    );
-    return;
-  }
   try {
     jobStorage.upsertJob({
       userId: req.session.currentUser,
@@ -63,9 +46,13 @@ jobRouter.post('/', async (req, res) => {
     res.send(new Error(error));
     console.error(error);
   }
+  trackDemoJobCreated({
+    name,
+    provider,
+    adapter: notificationAdapter,
+  });
   res.send();
 });
-
 jobRouter.delete('', async (req, res) => {
   const { jobId } = req.body;
   try {
@@ -81,7 +68,6 @@ jobRouter.delete('', async (req, res) => {
   }
   res.send();
 });
-
 jobRouter.put('/:jobId/status', async (req, res) => {
   const { status } = req.body;
   const { jobId } = req.params;
@@ -101,5 +87,4 @@ jobRouter.put('/:jobId/status', async (req, res) => {
   }
   res.send();
 });
-
-exports.jobRouter = jobRouter;
+export { jobRouter };

lib/api/routes/loginRoute.js

@@ -1,33 +1,36 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as hasher from '../../services/security/hash.js';
+import {config} from '../../utils.js';
+import {trackDemoAccessed} from '../../services/tracking/Tracker.js';
+const service = restana();
 const loginRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const hasher = require('../../services/security/hash');
-
 loginRouter.get('/user', async (req, res) => {
   const currentUserId = req.session.currentUser;
-  const isAdmin = currentUserId == null ? false : userStorage.getUser(currentUserId).isAdmin;
-  if (currentUserId == null) {
+  const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
+  if (currentUser == null) {
     res.body = {};
   } else {
     res.body = {
-      userId: currentUserId,
-      isAdmin,
+      userId: currentUser.id,
+      isAdmin: currentUser.isAdmin,
     };
   }
   res.send();
 });
-
 loginRouter.post('/', async (req, res) => {
   const { username, password } = req.body;
-
   const user = userStorage.getUsers(true).find((user) => user.username === username);
-
   if (user == null) {
     res.send(401);
     return;
   }
-
   if (user.password === hasher.hash(password)) {
+
+    if(config.demoMode){
+      trackDemoAccessed();
+    }
+
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
     res.send(200);
@@ -35,13 +38,10 @@ loginRouter.post('/', async (req, res) => {
   } else {
     console.error(`User ${username} tried to login, but password was wrong.`);
   }
-
   res.send(401);
 });
-
 loginRouter.post('/logout', async (req, res) => {
   req.session = null;
   res.send(200);
 });
-
-exports.loginRouter = loginRouter;
+export { loginRouter };

lib/api/routes/notificationAdapterRouter.js

@@ -1,13 +1,13 @@
-const fs = require('fs');
-const service = require('restana')();
+import fs from 'fs';
+import restana from 'restana';
+const service = restana();
 const notificationAdapterRouter = service.newRouter();
-
 const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
-
-const notificationAdapter = notificationAdapterList.map((pro) => {
-  return require(`../../notification/adapter/${pro}`);
-});
-
+const notificationAdapter = await Promise.all(
+  notificationAdapterList.map(async (pro) => {
+    return await import(`../../notification/adapter/${pro}`);
+  })
+);
 notificationAdapterRouter.post('/try', async (req, res) => {
   const { id, fields } = req.body;
   const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
@@ -24,7 +24,6 @@ notificationAdapterRouter.post('/try', async (req, res) => {
     enabled: true,
     id,
   });
-
   try {
     await adapter.send({
       serviceName: 'TestCall',
@@ -40,16 +39,13 @@ notificationAdapterRouter.post('/try', async (req, res) => {
       notificationConfig,
       jobKey: 'TestJob',
     });
-
     res.send();
   } catch (Exception) {
     res.send(new Error(Exception));
   }
 });
-
 notificationAdapterRouter.get('/', async (req, res) => {
   res.body = notificationAdapter.map((adapter) => adapter.config);
   res.send();
 });
-
-exports.notificationAdapterRouter = notificationAdapterRouter;
+export { notificationAdapterRouter };

lib/api/routes/providerRouter.js

@@ -1,16 +1,15 @@
-const fs = require('fs');
-const service = require('restana')();
+import fs from 'fs';
+import restana from 'restana';
+const service = restana();
 const providerRouter = service.newRouter();
-
 const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
-
-const provider = providerList.map((pro) => {
-  return require(`../../provider/${pro}`).metaInformation;
-});
-
+const provider = await Promise.all(
+  providerList.map(async (pro) => {
+    return await import(`../../provider/${pro}`);
+  })
+);
 providerRouter.get('/', async (req, res) => {
-  res.body = provider;
+  res.body = provider.map((p) => p.metaInformation);
   res.send();
 });
-
-exports.providerRouter = providerRouter;
+export { providerRouter };

lib/api/routes/userRoute.js

@@ -1,33 +1,33 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import {config} from '../../utils.js';
+const service = restana();
 const userRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const jobStorage = require('../../services/storage/jobStorage');
-
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
   return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
 }
-
 function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
   return req.session.currentUser === userIdToBeRemoved;
 }
-
 const nullOrEmpty = (str) => str == null || str.length === 0;
-
 userRouter.get('/', async (req, res) => {
   res.body = userStorage.getUsers(false);
   res.send();
 });
-
 userRouter.get('/:userId', async (req, res) => {
   const { userId } = req.params;
   res.body = userStorage.getUser(userId);
   res.send();
 });
-
 userRouter.delete('/', async (req, res) => {
+  if(config.demoMode){
+    res.send(new Error('In demo mode, it is not allowed to remove user.'));
+    return;
+  }
+
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
-
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
     return;
@@ -36,15 +36,18 @@ userRouter.delete('/', async (req, res) => {
     res.send(new Error('You are trying to remove yourself. This is prohibited.'));
     return;
   }
-
   //TODO: Remove also analytics
   jobStorage.removeJobsByUserId(userId);
   userStorage.removeUser(userId);
-
   res.send();
 });
-
 userRouter.post('/', async (req, res) => {
+
+  if(config.demoMode){
+      res.send(new Error('In demo mode, it is not allowed to change or add user.'));
+      return;
+  }
+
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
     res.send(new Error('Passwords does not match'));
@@ -55,22 +58,18 @@ userRouter.post('/', async (req, res) => {
     return;
   }
   const allUser = userStorage.getUsers(false);
-
   if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(
       new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
     );
     return;
   }
-
   userStorage.upsertUser({
     userId,
     username,
     password,
     isAdmin,
   });
-
   res.send();
 });
-
-exports.userRouter = userRouter;
+export { userRouter };

lib/api/security.js

@@ -1,15 +1,12 @@
-const userStorage = require('../services/storage/userStorage');
-const cookieSession = require('cookie-session');
-const { nanoid } = require('nanoid');
-
+import * as userStorage from '../services/storage/userStorage.js';
+import cookieSession from 'cookie-session';
+import { nanoid } from 'nanoid';
 const unauthorized = (res) => {
   return res.send(401);
 };
-
 const isUnauthorized = (req) => {
   return req.session.currentUser == null;
 };
-
 const isAdmin = (req) => {
   if (!isUnauthorized(req)) {
     const user = userStorage.getUser(req.session.currentUser);
@@ -17,7 +14,6 @@ const isAdmin = (req) => {
   }
   return false;
 };
-
 const authInterceptor = () => {
   return (req, res, next) => {
     if (isUnauthorized(req)) {
@@ -27,7 +23,6 @@ const authInterceptor = () => {
     }
   };
 };
-
 const adminInterceptor = () => {
   return (req, res, next) => {
     if (!isAdmin(req)) {
@@ -37,8 +32,7 @@ const adminInterceptor = () => {
     }
   };
 };
-
-exports.cookieSession = (userId) => {
+const cookieSession$0 = (userId) => {
   return cookieSession({
     name: 'fredy-admin-session',
     keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
@@ -46,8 +40,8 @@ exports.cookieSession = (userId) => {
     maxAge: 8 * 60 * 60 * 1000, // 8 hours
   });
 };
-
-exports.adminInterceptor = adminInterceptor;
-exports.authInterceptor = authInterceptor;
-exports.isUnauthorized = isUnauthorized;
-exports.isAdmin = isAdmin;
+export { cookieSession$0 as cookieSession };
+export { adminInterceptor };
+export { authInterceptor };
+export { isUnauthorized };
+export { isAdmin };

lib/defaultConfig.js

@@ -0,0 +1,7 @@
+export const DEFAULT_CONFIG = {
+            'interval': '60',
+            'port': 9998,
+            'workingHours': {'from': '', 'to': ''},
+            'demoMode': false,
+            'analyticsEnabled': null
+    };

lib/errors.js

@@ -9,7 +9,8 @@ class ExtendableError extends Error {
     }
   }
 }
-
 class NoNewListingsWarning extends ExtendableError {}
-
-module.exports = { NoNewListingsWarning };
+export { NoNewListingsWarning };
+export default {
+  NoNewListingsWarning,
+};

lib/notification/adapter/apprise.js

@@ -0,0 +1,36 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        body: message,
+        title: title,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'apprise',
+  name: 'Apprise',
+  readme: markdown2Html('lib/notification/adapter/apprise.md'),
+  description: 'Fredy will send new listings to your Apprise instance.',
+  fields: {
+    server: {
+      type: 'text',
+      label: 'Server',
+      description: 'The server URL to send the notification to.',
+    },
+  },
+};

lib/notification/adapter/apprise.md

@@ -0,0 +1,3 @@
+### Apprise Adapter
+
+Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.

lib/notification/adapter/console.js

@@ -1,19 +1,12 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
 
-/**
- * simply prints out the found data to the console
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
+export const send = ({ serviceName, newListings, jobKey }) => {
   /* eslint-disable no-console */
   return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
   /* eslint-enable no-console */
 };
-
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'console',
   name: 'Console',
   description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
   config: {},

lib/notification/adapter/mailJet.js

@@ -1,35 +1,24 @@
-const mailjet = require('node-mailjet');
-
-const path = require('path');
-const fs = require('fs');
-const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/template.hbs'), 'utf8');
-
-const Handlebars = require('handlebars');
+import mailjet from 'node-mailjet';
+import path from 'path';
+import fs from 'fs';
+import Handlebars from 'handlebars';
+import { markdown2Html } from '../../services/markdown.js';
+import { getDirName } from '../../utils.js';
+const __dirname = getDirName();
+const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
 const emailTemplate = Handlebars.compile(template);
-const { markdown2Html } = require('../../services/markdown');
-
-/**
- * sends a new listing using MailJet
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailJet'
+    (adapter) => adapter.id === config.id,
   ).fields;
-
   const to = receiver
     .trim()
     .split(',')
     .map((r) => ({
       Email: r.trim(),
     }));
-
   return mailjet
-    .connect(apiPublicKey, apiPrivateKey)
+    .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [
@@ -49,9 +38,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
       ],
     });
 };
-
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'mailjet',
   name: 'MailJet',
   description: 'MailJet is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/mailJet.md'),

lib/notification/adapter/mattermost.js

@@ -0,0 +1,39 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
+  message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
+  message += newListings.map(
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
+  );
+  return fetch(webhook, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: {
+      channel: channel,
+      text: message,
+    },
+  });
+};
+export const config = {
+  id: 'mattermost',
+  name: 'Mattermost',
+  readme: markdown2Html('lib/notification/adapter/mattermost.md'),
+  description: 'Fredy will send new listings to your mattermost team chat.',
+  fields: {
+    webhook: {
+      type: 'text',
+      label: 'Webhook-URL',
+      description: 'The incoming webhook url',
+    },
+    channel: {
+      type: 'text',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/mattermost.md

@@ -0,0 +1,5 @@
+### Mattermost Adapter
+
+For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the  [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+
+As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.

lib/notification/adapter/ntfy.js

@@ -0,0 +1,53 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const message = `
+    Address: ${newListing.address} 
+    Size: ${newListing.size.replace(/2m/g, '$m^2$')} 
+    Price: ${newListing.price}
+    Link: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      body: JSON.stringify({
+        topic: topic,
+        message: message,
+        title: newListing.title,
+        tags: [serviceName, jobName],
+        priority: parseInt(priority),
+        click: newListing.link,
+      }),
+    });
+  });
+
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'ntfy',
+  name: 'ntfy',
+  readme: markdown2Html('lib/notification/adapter/ntfy.md'),
+  description: 'Fredy will send new listings to your ntfy.',
+  fields: {
+    priority: {
+      type: 'number',
+      label: 'Priority',
+      description: 'The priority of the send notification.',
+    },
+    server: {
+      type: 'text',
+      label: 'Server-URL',
+      description: 'The server url to the send the notification to.',
+    },
+    topic: {
+      type: 'text',
+      label: 'topic',
+      description:
+        'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
+    },
+  },
+};

lib/notification/adapter/ntfy.md

@@ -0,0 +1,5 @@
+### ntfy Adapter
+
+For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the  [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+
+As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.

lib/notification/adapter/pushover.js

@@ -0,0 +1,73 @@
+import {markdown2Html} from '../../services/markdown.js';
+import {getJob} from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({serviceName, newListings, notificationConfig, jobKey}) => {
+    const {token, user, device} = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+    const job = getJob(jobKey);
+    const jobName = job == null ? jobKey : job.name;
+    const promises = newListings.map((newListing) => {
+        const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+        const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+        return fetch('https://api.pushover.net/1/messages.json', {
+            method: 'POST',
+            headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify({
+                token: token,
+                user: user,
+                message: message,
+                device: device,
+                title: title,
+            }),
+        });
+    });
+
+    return Promise.all(promises)
+        .then((responses) => {
+            // Convert all responses to JSON
+            return Promise.all(responses.map((response) => response.json()));
+        })
+        .then((data) => {
+            // Check for errors in the data
+            const error = data
+                .map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
+                .filter((err) => err !== null);
+
+            if (error.length > 0) {
+                // Reject with the combined error messages
+                return Promise.reject(error.join('; '));
+            }
+
+            return data;
+        })
+        .then(() => {
+            return Promise.resolve();
+        })
+        .catch((error) => {
+            return Promise.reject(error);
+        });
+};
+
+export const config = {
+    id: 'pushover',
+    name: 'Pushover',
+    readme: markdown2Html('lib/notification/adapter/pushover.md'),
+    description: 'Fredy will send new listings to your mobile using Pushover.',
+    fields: {
+        token: {
+            type: 'text',
+            label: 'API token',
+            description: 'Your application\'s API token.',
+        },
+        user: {
+            type: 'text',
+            label: 'User key',
+            description: 'Your user/group key.',
+        },
+        device: {
+            type: 'text',
+            label: 'Device name',
+            description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
+        },
+    },
+};

lib/notification/adapter/pushover.md

@@ -0,0 +1,5 @@
+### Pushover Adapter
+
+Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+
+After setting up the application, please enter both your newly created User key and API token.

lib/notification/adapter/sendGrid.js

@@ -1,16 +1,7 @@
-const sgMail = require('@sendgrid/mail');
-const { markdown2Html } = require('../../services/markdown');
-
-/**
- * sends a new listing using SendGrid
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
+import sgMail from '@sendgrid/mail';
+import { markdown2Html } from '../../services/markdown.js';
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,
@@ -28,9 +19,8 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   };
   return sgMail.send(msg);
 };
-
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'sendgrid',
   name: 'SendGrid',
   description: 'SendGrid is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/sendGrid.md'),

lib/notification/adapter/slack.js

@@ -1,17 +1,8 @@
-const Slack = require('slack');
+import Slack from 'slack';
+import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
-const { markdown2Html } = require('../../services/markdown');
-
-/**
- * sends a new listing to slack
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   return newListings.map((payload) =>
     msg({
       token,
@@ -44,12 +35,11 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
           ts: new Date().getTime() / 1000,
         },
       ],
-    })
+    }),
   );
 };
-
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'slack',
   name: 'Slack',
   readme: markdown2Html('lib/notification/adapter/slack.md'),
   description: 'Fredy will send new listings to the slack channel of your choice..',

lib/notification/adapter/sqlite.js

@@ -0,0 +1,25 @@
+import { markdown2Html } from '../../services/markdown.js';
+import Database from 'better-sqlite3';
+export const send = ({ serviceName, newListings, jobKey }) => {
+  const db = new Database('db/listings.db');
+  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+export const config = {
+  id: 'sqlite',
+  name: 'Sqlite',
+  description: 'This adapter stores listings in a local sqlite3 database.',
+  config: {},
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,7 @@
+### Sqlite Adapter
+This adapter stores search results in a sqlite database located in db/listings.db. This file can be used for further analysis later on.   
+
+Fields are:
+```
+['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description']
+```

lib/notification/adapter/telegram.js

@@ -1,6 +1,8 @@
-const { markdown2Html } = require('../../services/markdown');
-const axios = require('axios');
-
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+const MAX_ENTITIES_PER_CHUNK = 8;
+const RATE_LIMIT_INTERVAL = 1010;
 /**
  * splitting an array into chunks because Telegram only allows for messages up to
  * 4096 chars, thus we have to split messages into chunks
@@ -13,53 +15,51 @@ const arrayChunks = (inputArray, perChunk) =>
     all[ch] = [].concat(all[ch] || [], one);
     return all;
   }, []);
-
-/**
- * sends new listings to telegram
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
-
-  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
-  const chunks = arrayChunks(newListings, 3);
-
-  const promises = chunks.map((chunk) => {
-    let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
-    message += chunk.map(
-      (o) =>
-        `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
-        [o.address, o.price, o.size].join(' | ') +
-        '\n' +
-        `<a href="${o.link}">${o.link}</a>\n\n`
-    );
-
-    return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
-      chat_id: chatId,
-      text: message,
-      parse_mode: 'HTML',
-      disable_web_page_preview: true,
-    });
-  });
-
-  return Promise.all(promises);
-};
-
 function shorten(str, len = 30) {
   return str.length > len ? str.substring(0, len) + '...' : str;
 }
-
-/**
- * exported config is being used in the frontend to generate the fields
- * incoming values will be the keys (and values) of the fields
- *
- */
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
+  const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
+  const promises = chunks.map((chunk) => {
+    let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
+    message += chunk.map(
+      (o) =>
+        `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
+        [o.address, o.price, o.size].join(' | ') +
+        '\n\n',
+    );
+    /**
+     * This is to not break the rate limit. It is to only send 1 message per second
+     */
+    return new Promise((resolve, reject) => {
+      setTimeout(() => {
+        fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+          method: 'post',
+          body: JSON.stringify({
+            chat_id: chatId,
+            text: message,
+            parse_mode: 'HTML',
+            disable_web_page_preview: true,
+          }),
+          headers: { 'Content-Type': 'application/json' },
+        })
+          .then(() => {
+            resolve();
+          })
+          .catch(() => {
+            reject();
+          });
+      }, RATE_LIMIT_INTERVAL);
+    });
+  });
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'telegram',
   name: 'Telegram',
   readme: markdown2Html('lib/notification/adapter/telegram.md'),
   description: 'Fredy will send new listings to your mobile, using Telegram.',

lib/notification/notify.js

@@ -1,24 +1,24 @@
-const fs = require('fs');
+import fs from 'fs';
 const path = './adapter';
 
 /** Read every integration existing in ./adapter **/
-const adapter = fs
-  .readdirSync('./lib/notification/adapter')
-  .filter((file) => file.endsWith('.js'))
-  .map((integPath) => require(`${path}/${integPath}`));
+const adapter = await Promise.all(
+  fs
+    .readdirSync('./lib/notification/adapter')
+    .filter((file) => file.endsWith('.js'))
+    .map(async (integPath) => await import(`${path}/${integPath}`))
+);
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
-
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
+const findAdapter = (notificationAdapter) => {
+  return adapter.find((a) => a.config.id === notificationAdapter.id);
+};
+export const send = (serviceName, newListings, notificationConfig, jobKey) => {
   //this is not being used in tests, therefore adapter are always set
   return notificationConfig
     .filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
     .map((notificationAdapter) => findAdapter(notificationAdapter))
     .map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
 };
-
-const findAdapter = (notificationAdapter) => {
-  return adapter.find((a) => a.config.id === notificationAdapter.id);
-};

lib/provider/einsAImmobilien.js

@@ -1,21 +1,33 @@
-const utils = require('../utils');
-
+import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
 
 function normalize(o) {
-  let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
-  if (o.rooms != null) {
-    size += ` / / ${o.rooms.trim()}`;
-  }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-
-  return Object.assign(o, { size, link });
+  const price = normalizePrice(o.price);
+  const id = buildHash(o.id, price);
+  return Object.assign(o, { id, price, link });
 }
 
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
+}
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
@@ -23,28 +35,24 @@ const config = {
   url: null,
   crawlContainer: '.tabelle',
   sortByDateParam: 'sort_type=newest',
+  waitForSelector: 'body',
   crawlFields: {
     id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
-    price: '.tabelle .inner_object_data .single_data_price | removeNewline | trim',
-    size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
-    rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
-    title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
+    price: '.inner_object_data .single_data_price | removeNewline | trim',
+    size: '.tabelle .tabelle_inhalt_infos .single_data_box  | removeNewline | trim',
+    title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: '1a Immobilien',
   baseUrl: 'https://www.1a-immobilienmarkt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'einsAImmobilien',
 };
-
-exports.config = config;
+export { config };

lib/provider/immobilienDe.js

@@ -0,0 +1,51 @@
+import utils, {buildHash} from '../utils.js';
+let appliedBlackList = [];
+function shortenLink(link) {
+  return link.substring(0, link.indexOf('?'));
+}
+function parseId(shortenedLink) {
+  return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
+}
+function normalize(o) {
+  const size = o.size || 'N/A m²';
+  const price = o.price || 'N/A €';
+  const title = o.title || 'No title available';
+  const address = o.address || 'No address available';
+  const shortLink = shortenLink(o.link);
+  const link = `https://www.immobilien.de/${shortLink}`;
+  const id = buildHash(parseId(shortLink), o.price);
+  return Object.assign(o, { id, price, size, title, address, link });
+}
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+const config = {
+  url: null,
+  crawlContainer: '._ref',
+  sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
+  waitForSelector: 'body',
+  crawlFields: {
+    id: '@href', //will be transformed later
+    price: '.list_entry .immo_preis .label_info',
+    size: '.list_entry .flaeche .label_info | removeNewline | trim',
+    title: '.list_entry .part_text h3 span',
+    description: '.list_entry .description | trim',
+    link: '@href',
+    address: '.list_entry .place',
+  },
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immobilien.de',
+  baseUrl: 'https://www.immobilien.de/',
+  id: 'immobilienDe',
+};
+export { config };

lib/provider/immonet.js

@@ -1,52 +1,50 @@
-const utils = require('../utils');
-
+import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
 
+/**
+ * Note, Immonet is rly a piece of sh*t. It is using a weird combination of React and some buttons (instead of links),
+ * so that if somebody clicks the listing, a new page will open with the actual link to the listing. Of course, a scraper
+ * cannot do this (which is why I always just return the link to the whole list of listings).
+ * This is not only bad for us, but also bad for ppl with disabilities...
+ */
+
 function normalize(o) {
-  const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
-  const address = o.address.split(' • ')[1];
+  const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
-  //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
-  //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
-  const link = `https://www.immonet.de/angebot/${id}`;
+  const link = config.url;
+  const id = buildHash(title, price);
   return Object.assign(o, { id, address, price, size, title, link });
 }
-
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '#result-list-stage .item',
+  crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
   sortByDateParam: 'sortby=19',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
-    id: '@id',
-    price: 'div[id*="selPrice_"] | trim',
-    size: 'div[id*="selArea_"] | trim',
-    title: '.item a img@title',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
+    id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
+    title: 'button@title |trim',
+    price: 'div[data-testid="cardmfe-price-testid"] | trim',
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
+    address: 'div[data-testid="cardmfe-description-box-address"] | trim',
   },
-  paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immonet',
   baseUrl: 'https://www.immonet.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immonet',
 };
-
-exports.config = config;
+export { config };

lib/provider/immoscout.js

@@ -1,49 +1,115 @@
-const utils = require('../utils');
+/**
+ * ImmoScout provider using the mobile API to retrieve listings.
+ *
+ * The mobile API provides the following endpoints:
+ * - GET /search/total?{search parameters}: Returns the total number of listings for the given query
+ *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
+ *
+ * - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
+ *   data specifying additional results (advertisements) to return. The format is as follows:
+ *   ```
+ *   {
+ *   "supportedResultListTypes": [],
+ *   "userData": {}
+ *   }
+ *   ```
+ *   It is not necessary to provide data for the specified keys.
+ *
+ *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
 
+ * - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
+ *   listing response.
+ *
+ *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
+ *
+ *
+ * It is necessary to set the correct User Agent (see `getListings`) in the request header.
+ *
+ * Note that the mobile API is not publicly documented. I've reverse-engineered
+ * it by intercepting traffic from an android emulator running the immoscout app.
+ * Moreover, the search parameters differ slightly from the web API. I've mapped them
+ * to the web API parameters by comparing a search request with all parameters set between
+ * the web and mobile API. The mobile API actually seems to be a superset of the web API,
+ * but I have decided not to include new parameters as I wanted to keep the existing UX (i.e.,
+ * users only have to provide a link to an existing search).
+ *
+ */
+
+import utils, { buildHash } from '../utils.js';
+import { convertWebToMobile } from '../services/immoscout/immoscout-web-translater.js';
 let appliedBlackList = [];
 
+async function getListings(url) {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'User-Agent': 'ImmoScout24_1410_30_._',
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      supportedResultListTypes: [],
+      userData: {},
+    }),
+  });
+  if (!response.ok) {
+    console.error('Error fetching data from ImmoScout Mobile API:', response.statusText);
+    return [];
+  }
+
+  const responseBody = await response.json();
+  return responseBody.resultListItems
+    .filter((item) => item.type === 'EXPOSE_RESULT')
+    .map((expose) => {
+      const item = expose.item;
+      const [price, size] = item.attributes;
+      return {
+        id: item.id,
+        price: price?.value,
+        size: size?.value,
+        title: item.title,
+        link: `${metaInformation.baseUrl}expose/${item.id}`,
+        address: item.address?.line,
+      };
+    });
+}
+
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
-
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
-  return Object.assign(o, { title, address, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, title, address });
 }
-
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
-
 const config = {
   url: null,
-  crawlContainer: '#resultListItems li.result-list__listing',
-  sortByDateParam: 'sorting=2',
   crawlFields: {
-    id: '.result-list-entry@data-obid | int',
-    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
-    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
-    link: '.result-list-entry .result-list-entry__brand-title-container@href',
-    address: '.result-list-entry .result-list-entry__map-link',
+    id: 'id',
+    title: 'title',
+    price: 'price',
+    size: 'size',
+    link: 'link',
+    address: 'address',
   },
-  paginate: '#pager .align-right a@href',
+  // Not required - used by filter to remove and listings that failed to parse
+  sortByDateParam: 'sorting=-firstactivation',
   normalize: normalize,
   filter: applyBlacklist,
+  getListings: getListings,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+  config.url = convertWebToMobile(sourceConfig.url);
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immoscout',
   baseUrl: 'https://www.immobilienscout24.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immoscout',
 };
 
-exports.config = config;
+export { config };

lib/provider/immoswp.js

@@ -0,0 +1,48 @@
+import utils, { buildHash } from '../utils.js';
+
+let appliedBlackList = [];
+
+function normalize(o) {
+  const size = o.size || 'N/A m²';
+  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+  const title = o.title || 'No title available';
+  const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+  const link = `https://immo.swp.de/immobilien/${immoId}`;
+  const description = o.description;
+  const id = buildHash(immoId, price);
+  return Object.assign(o, { id, price, size, title, link, description });
+}
+
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+
+const config = {
+  url: null,
+  crawlContainer: '.js-serp-item',
+  sortByDateParam: 's=most_recently_updated_first',
+  waitForSelector: 'body',
+  crawlFields: {
+    id: '.js-bookmark-btn@data-id',
+    price: 'div.align-items-start div:first-child | trim',
+    size: 'div.align-items-start div:nth-child(3) | trim',
+    title: '.js-item-title-link@title | trim',
+    link: '.ci-search-result__link@href',
+    description: '.js-show-more-item-sm | removeNewline | trim',
+  },
+  normalize: normalize,
+  filter: applyBlacklist,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immo Südwest Presse',
+  baseUrl: 'https://immo.swp.de/',
+  id: 'immoswp',
+};
+export { config };

lib/provider/immowelt.js

@@ -1,44 +1,43 @@
-const utils = require('../utils');
+import utils, { buildHash } from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  return o;
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id });
 }
 
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
   url: null,
-  crawlContainer: "div[class^='EstateItem-']",
-  sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
+  crawlContainer:
+    'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
+  sortByDateParam: 'order=DateDesc',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
-    id: 'a@id',
-    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
-    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
-    title: "div[class^='FactsMain-'] h2",
+    id: 'a@href',
+    price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
+    title: '.css-jv3zx6',
     link: 'a@href',
-    address: "div[class^='estateFacts-'] span | removeNewline | trim",
+    address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },
-  paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,
   filter: applyBlacklist,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immowelt',
   baseUrl: 'https://www.immowelt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immowelt',
 };
-
-exports.config = config;
+export { config };

lib/provider/kleinanzeigen.js

@@ -1,53 +1,50 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
 
 function normalize(o) {
-  const size = o.size || '--- m²';
-
-  return Object.assign(o, { size });
+    const size = o.size || '--- m²';
+    const id = buildHash(o.id, o.price);
+    const link = `https://www.kleinanzeigen.de${o.link}`;
+    return Object.assign(o, {id, size, link});
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-  const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    const isBlacklistedDistrict =
+        appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
+    return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-  url: null,
-  crawlContainer: '#srchrslt-adtable .ad-listitem ',
-  //sort by date is standard oO
-  sortByDateParam: null,
-  crawlFields: {
-    id: '.aditem@data-adid | int',
-    price: '.aditem-main--middle--price | removeNewline | trim',
-    size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
-    title: '.aditem-main .text-module-begin a | removeNewline | trim',
-    link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
-    description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
-    address: '.aditem-main--top--left | trim | removeNewline',
-  },
-  paginate: '#srchrslt-pagination .pagination-next@href',
-  normalize: normalize,
-  filter: applyBlacklist,
+    url: null,
+    crawlContainer: '#srchrslt-adtable .ad-listitem ',
+    //sort by date is standard oO
+    sortByDateParam: null,
+    waitForSelector: 'body',
+    crawlFields: {
+        id: '.aditem@data-adid | int',
+        price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
+        size: '.aditem-main .text-module-end | removeNewline | trim',
+        title: '.aditem-main .text-module-begin a | removeNewline | trim',
+        link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
+        description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
+        address: '.aditem-main--top--left | trim | removeNewline',
+    },
+    normalize: normalize,
+    filter: applyBlacklist,
 };
-
-exports.metaInformation = {
-  name: 'Ebay Kleinanzeigen',
-  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+export const metaInformation = {
+    name: 'Ebay Kleinanzeigen',
+    baseUrl: 'https://www.kleinanzeigen.de/',
+    id: 'kleinanzeigen',
 };
-
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlacklistedDistricts = blacklistedDistricts || [];
-  appliedBlackList = blacklist || [];
+export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
+    config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
+    appliedBlacklistedDistricts = blacklistedDistricts || [];
+    appliedBlackList = blacklist || [];
 };
-
-exports.config = config;
+export {config};

lib/provider/neubauKompass.js

@@ -1,41 +1,44 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 
+function nullOrEmpty(val) {
+    return val == null || val.length === 0;
+}
+
 function normalize(o) {
-  return o;
+    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+    const id = buildHash(o.link, o.price);
+    return Object.assign(o, {id, link});
 }
 
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
+    return !utils.isOneOf(o.title, appliedBlackList);
 }
 
 const config = {
-  url: null,
-  crawlContainer: '.nbk-container >div article',
-  sortByDateParam: 'Sortierung=Id&Richtung=DESC',
-  crawlFields: {
-    id: '@id',
-    title: 'a.nbk-truncate@title | removeNewline | trim',
-    link: 'a.nbk-truncate@href',
-    address: 'p.nbk-truncate | removeNewline | trim',
-    price: 'p.nbk-mb-0 | removeNewline | trim',
-  },
-  paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
-  normalize: normalize,
-  filter: applyBlacklist,
+    url: null,
+    crawlContainer: '.col-12.mb-4',
+    sortByDateParam: 'Sortierung=Id&Richtung=DESC',
+    waitForSelector: '.nbk-section',
+    crawlFields: {
+        id: 'a@href',
+        title: 'a@title | removeNewline | trim',
+        link: 'a@href',
+        address: '.nbk-project-card__description | removeNewline | trim',
+        price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
+    },
+    normalize: normalize,
+    filter: applyBlacklist,
 };
-
-exports.init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+export const init = (sourceConfig, blacklist) => {
+    config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
+    appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
-  name: 'Neubau Kompass',
-  baseUrl: 'https://www.neubaukompass.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+export const metaInformation = {
+    name: 'Neubau Kompass',
+    baseUrl: 'https://www.neubaukompass.de/',
+    id: 'neubauKompass',
 };
-
-exports.config = config;
+export {config};

lib/provider/wgGesucht.js

@@ -1,44 +1,43 @@
-const utils = require('../utils');
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  return o;
+    const id = buildHash(o.id, o.price);
+    const link = `https://www.wg-gesucht.de${o.link}`;
+    return Object.assign(o, { id, link });
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
-  return o.id != null && titleNotBlacklisted && descNotBlacklisted;
+    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+    return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-  url: null,
-  crawlContainer: '#main_column .wgg_card',
-  sortByDateParam: 'sort_column=0&sort_order=0',
-  crawlFields: {
-    id: '@data-id',
-    details: '.row .noprint .col-xs-11 |removeNewline |trim',
-    price: '.middle .col-xs-3 |removeNewline |trim',
-    size: '.middle .text-right |removeNewline |trim',
-    title: '.truncate_title a |removeNewline |trim',
-    link: '.truncate_title a@href',
-  },
-  normalize: normalize,
-  filter: applyBlacklist,
+    url: null,
+    crawlContainer: '#main_column .wgg_card',
+    sortByDateParam: 'sort_column=0&sort_order=0',
+    waitForSelector: 'body',
+    crawlFields: {
+        id: '@data-id',
+        details: '.row .noprint .col-xs-11 |removeNewline |trim',
+        price: '.middle .col-xs-3 |removeNewline |trim',
+        size: '.middle .text-right |removeNewline |trim',
+        title: '.truncate_title a |removeNewline |trim',
+        link: '.truncate_title a@href',
+    },
+    normalize: normalize,
+    filter: applyBlacklist,
 };
-
-exports.init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
+export const init = (sourceConfig, blacklist) => {
+    config.enabled = sourceConfig.enabled;
+    config.url = sourceConfig.url;
+    appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
-  name: 'Wg gesucht',
-  baseUrl: 'https://www.wg-gesucht.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+export const metaInformation = {
+    name: 'Wg gesucht',
+    baseUrl: 'https://www.wg-gesucht.de/',
+    id: 'wgGesucht',
 };
-
-exports.config = config;
+export {config};

lib/services/demoCleanup.js

@@ -0,0 +1,29 @@
+import { setInterval } from 'node:timers';
+import {removeJobsByUserName} from './storage/jobStorage.js';
+import {config} from '../utils.js';
+
+/**
+ * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
+ */
+export function cleanupDemoAtMidnight() {
+    const now = new Date();
+    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+        - now.getUTCMinutes() * 60 * 1000
+        - now.getUTCSeconds() * 1000
+        - now.getUTCMilliseconds();
+
+    setTimeout(() => {
+        cleanup();
+
+        setInterval(() => {
+            cleanup();
+        }, 24 * 60 * 60 * 1000);
+
+    }, millisUntilMidnightUTC);
+}
+
+function cleanup(){
+    if(config.demoMode){
+        removeJobsByUserName('demo');
+    }
+}

lib/services/extractor/extractor.js

@@ -0,0 +1,43 @@
+import { setDebug } from './utils.js';
+import puppeteerExtractor from './puppeteerExtractor.js';
+import { loadParser, parse } from './parser/parser.js';
+
+const DEFAULT_OPTIONS = {
+  debug: false,
+  puppeteerTimeout: 60_000,
+  puppeteerHeadless: true,
+};
+
+export default class Extractor {
+  constructor(options) {
+    this.options = {
+      ...DEFAULT_OPTIONS,
+      ...options,
+    };
+    this.responseText = null;
+    setDebug(this.options);
+  }
+
+  /**
+   * if you are extracting data from a SPA, you must provide a selector, otherwise
+   * your response will never contain what you are really looking for
+   * @param url
+   * @param waitForSelector
+   */
+  execute = async (url, waitForSelector = null) => {
+    this.responseText = null;
+    try {
+      this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+      if (this.responseText != null) {
+        loadParser(this.responseText);
+      }
+    } catch (error) {
+      console.error('Error trying to load page.', error);
+    }
+    return this;
+  };
+
+  parseResponseText = (crawlContainer, crawlFields, url) => {
+    return parse(crawlContainer, crawlFields, this.responseText, url);
+  };
+}

lib/services/extractor/parser/parser.js

@@ -0,0 +1,97 @@
+import * as cheerio from 'cheerio';
+
+let $ = null;
+
+export function loadParser(text) {
+  $ = cheerio.load(text);
+}
+
+export function parse(crawlContainer, crawlFields, text, url) {
+  if (!text) {
+    console.warn('Cannot parse, text was empty for url ', url);
+    return null;
+  }
+
+  if (!crawlContainer || !crawlFields) {
+    console.warn('Cannot parse, selector was empty for url ', url);
+    return null;
+  }
+
+  const result = [];
+
+  if ($(crawlContainer).length === 0) {
+    console.warn('No elements in crawl container found for url ', url);
+    return null;
+  }
+
+  $(crawlContainer).each((_, element) => {
+    const container = $(element);
+    const parsedObject = {};
+
+    // Parse fields based on crawlFields
+    for (const [key, fieldSelector] of Object.entries(crawlFields)) {
+      let value;
+
+      try {
+        const selector = fieldSelector.includes('|')
+          ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim()
+          : fieldSelector;
+
+        if (selector.includes('@')) {
+          const [sel, attr] = selector.split('@');
+          if (sel.length === 0) {
+            value = container.attr(attr.trim());
+          } else {
+            value = container.find(sel.trim()).attr(attr.trim());
+          }
+        } else {
+          value = container.find(selector.trim()).text();
+        }
+
+        // Apply modifiers if specified
+        if (fieldSelector.includes('|')) {
+          /* eslint-disable no-unused-vars */
+          const [_, ...modifiers] = fieldSelector.split('|').map((s) => s.trim());
+          /* eslint-disable no-unused-vars */
+          value = applyModifiers(value, modifiers);
+        }
+
+        parsedObject[key] = value || null;
+      } catch (error) {
+        console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
+        parsedObject[key] = null;
+      }
+    }
+
+    if (parsedObject.id != null) {
+      result.push(parsedObject);
+    } else {
+      console.warn('ID not found. Not relaying object.');
+    }
+  });
+
+  return result;
+}
+
+// Helper function to apply modifiers
+function applyModifiers(value, modifiers) {
+  if (!value) return value;
+
+  modifiers.forEach((modifier) => {
+    switch (modifier) {
+      case 'int':
+        value = parseInt(value, 10);
+        break;
+      case 'trim':
+        value = value.replace(/\s+/g, ' ').trim();
+        break;
+      case 'removeNewline':
+        value = value.replace(/\n/g, ' ');
+        break;
+      default:
+        console.warn(`Unknown modifier: ${modifier}`);
+    }
+  });
+
+  return value;
+}

lib/services/extractor/puppeteerExtractor.js

@@ -0,0 +1,49 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
+
+puppeteer.use(StealthPlugin());
+
+export default async function execute(url, waitForSelector, options) {
+  let browser;
+  try {
+    debug(`Sending request to ${url} using Puppeteer.`);
+
+    browser = await puppeteer.launch({
+      headless: options.puppeteerHeadless ?? true,
+      args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox'],
+      timeout: options.puppeteerTimeout || 30_000,
+    });
+    let page = await browser.newPage();
+    await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+    const response = await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+    });
+    let pageSource;
+    //if we're extracting data from a spa, we must wait for the selector
+    if (waitForSelector != null) {
+      await page.waitForSelector(waitForSelector);
+      pageSource = await page.evaluate((selector) => {
+        return document.querySelector(selector).innerHTML;
+      }, waitForSelector);
+    } else {
+      pageSource = await page.content();
+    }
+
+    const statusCode = response.status();
+
+    if (botDetected(pageSource, statusCode)) {
+      console.warn('We have been detected as a bot :-/ Tried url: => ', url);
+      return null;
+    }
+
+    return await page.content();
+  } catch (error) {
+    console.error('Error executing with puppeteer executor', error);
+    return null;
+  } finally {
+    if (browser != null) {
+      await browser.close();
+    }
+  }
+}

lib/services/extractor/utils.js

@@ -0,0 +1,32 @@
+let debuggingOn = false;
+
+export const DEFAULT_HEADER = {
+  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.5',
+  Connection: 'keep-alive',
+  'Upgrade-Insecure-Requests': '1',
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+};
+
+export const setDebug = (options) => {
+  debuggingOn = !!options?.debug;
+};
+
+export const debug = (message) => {
+  if (debuggingOn) {
+    /* eslint-disable no-console */
+    console.debug(message);
+    /* eslint-enable no-console */
+  }
+};
+
+export const botDetected = (pageSource, statusCode) => {
+  const suspiciousStatusCodes = [403, 429];
+  const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
+
+  const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
+  const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
+
+  return detectedInSource || detectedByStatus;
+};

lib/services/immoscout/immoscout-web-translater.js

@@ -0,0 +1,157 @@
+/*
+Rent a flat
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?numberofrooms=1.0-10000.0&price=1.0-10000.0&livingspace=10.0-10000.0&pricetype=rentpermonth&enteredFrom=result_list
+*/
+
+/*
+Rent a flat:
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search
+Mobile:
+https://api.mobile.immobilienscout24.de/search/list?numberofrooms=1.5-&searchId=d7c127d8-6630-49e8-a1dd-5ae04dad454d&sorting=standard&pagesize=20&livingspace=10-500&pagenumber=1&realestatetype=apartmentrent&priceType=calculatedtotalrent&price=1-10000&publishedafter=2025-05-14T09:11:54&channel=is24&searchType=region&geocodes=/de/nordrhein-westfalen/duesseldorf&features=adKeysAndStringValues,virtualTour,contactDetails,viareporting,nextgen,calculatedTotalRent,listingsInListFirstSummary,xxlListingType,quickfilters,grouping,projectsInAllRealestateTypes,fairPrice
+*/
+
+/*
+Rent a house:
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-mieten?enteredFrom=one_step_search
+Mobile:
+https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T09:12:49&pagenumber=1&searchType=region&geocodes=/de/nordrhein-westfalen/duesseldorf&realEstateType=houserent&pagesize=300&features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&sorting=standard
+*/
+
+/*
+ buy a flat
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0&livingspace=1.0-10000.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&sorting=standard&realEstateType=apartmentbuy&pagesize=300&pagenumber=1&geocodes=/de/nordrhein-westfalen/duesseldorf&publishedafter=2025-05-14T09:14:43&searchType=region
+ */
+
+/*
+ Buy a house
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0E7&livingspace=1.0-10000.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?geocodes=/de/nordrhein-westfalen/duesseldorf&features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&searchType=region&realEstateType=housebuy&pagenumber=1&pagesize=300&sorting=standard&publishedafter=2025-05-14T09:16:28
+ */
+
+/*
+ Buy a house only in parts of a city
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/haus-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0E7&livingspace=1.0-10000.0&geocodes=1276010037,1276010014,1276010012&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/list?pagesize=20&pagenumber=1&features=adKeysAndStringValues,virtualTour,contactDetails,viareporting,grouping,nextgen,listingsInListFirstSummary,xxlListingType,quickfilters,fairPrice&sorting=standard&channel=is24&geocodes=/de/nordrhein-westfalen/duesseldorf/stadtbezirk-1&searchType=region&realestatetype=housebuy&publishedafter=2025-05-14T09:17:23
+ */
+
+/*
+ Buy a house with radius
+ Web:
+ https://www.immobilienscout24.de/Suche/radius/haus-kaufen?centerofsearchaddress=D%C3%BCsseldorf%3B%3B%3B%3B%3B%3B&numberofrooms=1.0-10000.0&price=1.0-1.0E7&livingspace=1.0-10000.0&geocoordinates=51.22496%3B6.77567%3B5.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/home/search/total?pagenumber=1&pagesize=1&geocoordinates=51.224960;6.775670;4.0&sorting=standard&searchType=radius&features=adKeysAndStringValues,virtualTour,contactDetails,grouping,nextgen,listingsInListFirstSummary,xxlListingType,fairPrice&channel=is24&realestatetype=housebuy&publishedafter=2025-05-14T09:19:43
+ */
+
+/*
+ Buy a house with shape
+ Web:
+ https://www.immobilienscout24.de/Suche/shape/haus-kaufen?shape=eW1yd0hpZGloQGBJa1NfQWFsQG9Uc1ZvVmlDbHdAZ2BAaEBjfEB5U3NWY2NCa0RvWmpwQG1KYGdCeldqU3Z4QGBAbENvQmJWaGtA&numberofrooms=1.0-100000.0&price=1.0-1.0E7&livingspace=1.0-100000.0&enteredFrom=result_list#/
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC?
+ */
+import queryString from 'query-string';
+
+const PARAM_NAME_MAP = {
+  heatingtypes: 'heatingtypes',
+  haspromotion: 'haspromotion',
+  numberofrooms: 'numberofrooms',
+  livingspace: 'livingspace',
+  energyefficiencyclasses: 'energyefficiencyclasses',
+  exclusioncriteria: 'exclusioncriteria',
+  equipment: 'equipment',
+  petsallowedtypes: 'petsallowedtypes',
+  price: 'price',
+  constructionyear: 'constructionyear',
+  apartmenttypes: 'apartmenttypes',
+  pricetype: 'pricetype',
+  floor: 'floor',
+  geocodes: 'geocodes',
+  geocoordinates: 'geocoordinates',
+  shape: 'shape',
+  sorting: 'sorting',
+};
+
+const EQUIPMENT_MAP = {
+  parking: 'parking',
+  cellar: 'cellar',
+  builtinkitchen: 'builtInKitchen',
+  lift: 'lift',
+  garden: 'garden',
+  guesttoilet: 'guestToilet',
+  balcony: 'balcony',
+};
+
+const REAL_ESTATE_TYPE = {
+  'haus-mieten': 'houserent',
+  'wohnung-mieten': 'apartmentrent',
+  'wohnung-kaufen': 'apartmentbuy',
+  'haus-kaufen': 'housebuy',
+};
+
+export function convertWebToMobile(webUrl) {
+  let url;
+  try {
+    url = new URL(webUrl);
+  } catch {
+    throw new Error(`Invalid URL: ${webUrl}`);
+  }
+
+  const segments = url.pathname.split('/');
+  if (segments[1] !== 'Suche') {
+    throw new Error(`Unexpected path format: ${url.pathname}. We're expecting to see "/Suche" in the path.`);
+  }
+
+  const realTypeKey = segments.at(-1);
+  const realType = REAL_ESTATE_TYPE[realTypeKey];
+  if (!realType) {
+    throw new Error(`Real estate type not found: ${realTypeKey}`);
+  }
+
+  if (segments.includes('shape')) {
+    throw new Error('Shape is currently not supported using Immoscout');
+  }
+
+  const { query: rawParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
+  const webParams = Object.fromEntries(
+    Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
+  );
+
+  const geocodes = `/${segments.slice(2, 5).join('/')}`;
+  const isRadius = segments.includes('radius');
+  const mobileParams = {
+    searchType: isRadius ? 'radius' : 'region',
+    realestatetype: realType,
+    ...(isRadius ? {} : { geocodes }),
+  };
+
+  if (webParams.geocoordinates) {
+    mobileParams.geocoordinates = webParams.geocoordinates;
+  }
+
+  for (const [key, val] of Object.entries(webParams)) {
+    if (key === 'equipment') {
+      const items = [].concat(val).flatMap((v) => `${v}`.split(','));
+      mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean);
+    } else {
+      mobileParams[PARAM_NAME_MAP[key]] = val;
+    }
+  }
+
+  const mobileQuery = queryString.stringify(mobileParams, {
+    arrayFormat: 'comma',
+    encode: true,
+    skipEmptyString: true,
+  });
+
+  return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
+}

lib/services/markdown.js

@@ -1,6 +1,6 @@
-const markdown = require('markdown').markdown;
-const fs = require('fs');
-
-exports.markdown2Html = function markdown2Html(filePath) {
+import markdown$0 from 'markdown';
+import fs from 'fs';
+const markdown = markdown$0.markdown;
+export function markdown2Html(filePath) {
   return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
-};
+}

lib/services/queryStringMutator.js

@@ -1,21 +1,9 @@
-const queryString = require('query-string');
-
-/**
- * for Fredy, it is important to sort search results by date, starting with the latest listing. if it is not sorted, we
- * might never actually find the newest results, no matter how many pages we crawl.
- * It has been written in the documentation, but obviously nobody reads docu theses days which is why it's been done
- * automagically now.
- *
- * @param _url actual provider url containing the searchParams
- * @param sortByDateParam param(s) indicating the correct sort order
- * @returns {`${string}?${string}`} correctly formatted url
- */
-module.exports = (_url, sortByDateParam) => {
+import queryString from 'query-string';
+export default (_url, sortByDateParam) => {
   //if no mutation is necessary, just return the original url
   if (sortByDateParam == null) {
     return _url;
   }
-
   const original = queryString.parseUrl(_url);
   const mutate = queryString.parse(sortByDateParam);
   return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;

lib/services/requestDriver.js

@@ -1,37 +0,0 @@
-const axios = require('axios');
-const axiosRetry = require('axios-retry');
-
-axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
-
-function makeDriver(headers = {}) {
-  let cookies = '';
-
-  return async function driver(context, callback) {
-    const url = context.url;
-    let result;
-    try {
-      result = await axios({
-        url,
-        headers: {
-          ...headers,
-          Cookie: cookies,
-        },
-      });
-    } catch (exception) {
-      console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
-      callback(null, []);
-    }
-
-    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
-      //assume we have gotten a response from scrapingAnt
-      if (cookies.length === 0) {
-        cookies = result.data.cookies;
-      }
-      callback(null, result.data.content);
-    } else {
-      callback(null, result.data);
-    }
-  };
-}
-
-module.exports = makeDriver;

lib/services/scraper.js

@@ -1,46 +0,0 @@
-const config = require('../../conf/config.json');
-const makeDriver = require('./requestDriver');
-const Xray = require('x-ray');
-
-class Scraper {
-  constructor() {
-    const filters = {
-      removeNewline: this._removeNewline,
-      trim: this._trim,
-      int: this._int,
-    };
-
-    const headers = {
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-    };
-
-    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
-      headers['x-api-key'] = config.scrapingAnt.apiKey;
-    }
-    const driver = makeDriver(headers);
-
-    const xray = Xray({ filters });
-    xray.driver(driver);
-
-    this.xray = xray;
-  }
-
-  get x() {
-    return this.xray;
-  }
-
-  _removeNewline(value) {
-    return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
-  }
-
-  _trim(value) {
-    return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
-  }
-
-  _int(value) {
-    return typeof value === 'string' ? parseInt(value, 10) : value;
-  }
-}
-
-module.exports = new Scraper().x;

lib/services/scrapingAnt.js

@@ -1,24 +0,0 @@
-const { metaInformation } = require('../provider/immoscout');
-//to better confure re-capture chose a random proxy each time we do a call
-const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
-const config = require('../../conf/config.json');
-
-const isImmoscout = (id) => {
-  return id.toLowerCase() === metaInformation.id;
-};
-
-exports.transformUrlForScrapingAnt = (url, id) => {
-  const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
-
-  if (isImmoscout(id)) {
-    //only do calls to scrapingAnt when dealing with Immoscout
-    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
-  }
-  return url;
-};
-
-exports.isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
-};
-
-exports.isImmoscout = isImmoscout;

lib/services/security/hash.js

@@ -1,3 +1,2 @@
-const crypto = require('crypto');
-
-exports.hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');
+import crypto from 'crypto';
+export const hash = (x) => crypto.createHash('sha256').update(x, 'utf8').digest('hex');

lib/services/similarity-check/SimilarityCacheEntry.js

@@ -1,27 +1,17 @@
-const stringSimilarity = require('string-similarity');
-
+import stringSimilarity from 'string-similarity';
 //if the score is higher than this, it will be considered a match
 const MAX_DICE_INDEX = 0.7;
-
-/**
- * The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
- *
- * @type {module.SimilarityCacheEntry}
- */
-module.exports = class SimilarityCacheEntry {
+export default (class SimilarityCacheEntry {
   constructor(time) {
     this.time = time;
     this.values = [];
   }
-
   setCacheEntry = (entry) => {
     this.values.push(entry);
   };
-
   getTime = () => {
     return this.time;
   };
-
   hasSimilarEntries = (value) => {
     if (this.values.length > 0) {
       for (let i = 0; i < this.values.length; i++) {
@@ -33,4 +23,4 @@ module.exports = class SimilarityCacheEntry {
     }
     return false;
   };
-};
+});

lib/services/similarity-check/similarityCache.js

@@ -1,63 +1,40 @@
-/**
- * each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
- * a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
- * the user won't see any duplicates
- *
- * The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
- *
- * @type {module.SimilarityCacheEntry|{}}
- */
-const SimilarityCacheEntry = require('./SimilarityCacheEntry');
-const config = require('../../../conf/config.json');
-
+import SimilarityCacheEntry from './SimilarityCacheEntry.js';
+import { config } from '../../utils.js';
 //5 minutes
 let retention = 5 * 60 * 1000;
-
 const intervalInMs = config.interval * 60 * 1000;
 //an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
 if (intervalInMs <= retention) {
   retention = Math.floor(intervalInMs / 2);
 }
-
 //jobid -> SimilarityCacheEntry
 const cache = {};
-
 let intervalId;
-
-exports.addCacheEntry = (jobId, value) => {
-  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
-  cache[jobId].setCacheEntry(value);
-};
-
-exports.hasSimilarEntries = (jobId, value) => {
-  if (cache[jobId] == null) {
-    return false;
-  }
-
-  return cache[jobId].hasSimilarEntries(value);
-};
-
 /**
  * cleanup
  */
 intervalId = setInterval(() => {
   const keysToBeRemoved = [];
   const now = Date.now();
-
   Object.keys(cache).forEach((key) => {
     if (cache[key].getTime() + retention < now) {
       keysToBeRemoved.push(key);
     }
   });
-
   if (keysToBeRemoved.length > 0) {
     keysToBeRemoved.forEach((key) => delete cache[key]);
   }
 }, 10000);
-
-/**
- * mostly used for tests
- */
-exports.stopCacheCleanup = () => {
+export const addCacheEntry = (jobId, value) => {
+  cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
+  cache[jobId].setCacheEntry(value);
+};
+export const hasSimilarEntries = (jobId, value) => {
+  if (cache[jobId] == null) {
+    return false;
+  }
+  return cache[jobId].hasSimilarEntries(value);
+};
+export const stopCacheCleanup = () => {
   clearInterval(intervalId);
 };

lib/services/storage/LowDashAdapter.js

@@ -0,0 +1,8 @@
+import lodash from 'lodash';
+import { LowSync } from 'lowdb';
+export default class LowdashAdapter extends LowSync {
+  constructor(adapter, defaultData = {}) {
+    super(adapter, defaultData);
+    this.chain = lodash.chain(this).get('data');
+  }
+}

lib/services/storage/jobStorage.js

@@ -1,28 +1,29 @@
-const path = require('path');
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobs.json';
-const FileSync = require('lowdb/adapters/FileSync');
-const adapter = new FileSync(DB_PATH);
-const low = require('lowdb');
-const db = low(adapter);
-const { nanoid } = require('nanoid');
-const listingStorage = require('./listingsStorage');
+import { JSONFileSync } from 'lowdb/node';
+import { nanoid } from 'nanoid';
+import * as listingStorage from './listingsStorage.js';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-db.defaults({ jobs: [] }).write();
+const file = path.join(getDirName(), '../', 'db/jobs.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, { jobs: [] });
 
-exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
+db.read();
+
+
+export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
       ? null
-      : db
+      : db.chain
           .get('jobs')
           .find((job) => job.id === jobId)
           .value();
-
-  const jobs = db
+  const jobs = db.chain
     .get('jobs')
-    .value()
-    .filter((job) => job.id !== jobId);
-
+    .filter((job) => job.id !== jobId)
+    .value();
   jobs.push({
     id: jobId || nanoid(),
     //make sure to not overwrite the user id in case an admin changes the job
@@ -33,57 +34,66 @@ exports.upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, no
     provider,
     notificationAdapter,
   });
-
-  db.set('jobs', jobs).write();
+  db.chain.set('jobs', jobs).value();
+  db.write();
 };
-
-exports.getJob = (jobId) => {
-  const job = db
+export const getJob = (jobId) => {
+  const job = db.chain
     .get('jobs')
     .find((job) => job.id === jobId)
     .value();
-
   if (job == null) {
     return null;
   }
-
   return {
     ...job,
     numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id).length,
   };
 };
-
-exports.setJobStatus = ({ jobId, status }) => {
-  db.get('jobs')
+export const setJobStatus = ({ jobId, status }) => {
+  db.chain
+    .get('jobs')
     .find((job) => job.id === jobId)
     .assign({ enabled: status })
-    .write();
+    .value();
+  db.write();
 };
-
-exports.removeJob = (jobId) => {
+export const removeJob = (jobId) => {
   listingStorage.removeListings(jobId);
-  db.get('jobs')
+  db.chain
+    .get('jobs')
     .remove((job) => job.id === jobId)
-    .write();
+    .value();
+  db.write();
 };
-
-exports.removeJobsByUserId = (userId) => {
-  db.get('jobs')
-    .value()
+export const removeJobsByUserId = (userId) => {
+  db.chain
+    .get('jobs')
     .filter((job) => job.userId === userId)
     .forEach((job) => listingStorage.removeListings(job.id));
-
-  db.get('jobs')
-    .remove((job) => job.userId === userId)
-    .write();
-};
-
-exports.getJobs = () => {
-  return db
+  db.chain
+    .get('jobs')
+    .remove((job) => job.userId === userId)
+    .value();
+  db.write();
+};
+export const removeJobsByUserName = (userName) => {
+  db.chain
+      .get('jobs')
+      .filter((job) => job.username === userName)
+      .forEach((job) => listingStorage.removeListings(job.id));
+  db.chain
+      .get('jobs')
+      .remove((job) => job.username === userName)
+      .value();
+  db.write();
+};
+export const getJobs = () => {
+  return db.chain
     .get('jobs')
-    .value()
     .map((job) => ({
       ...job,
       numberOfFoundListings: listingStorage.getNumberOfAllKnownListings(job.id),
-    }));
+    }))
+    .value();
 };

lib/services/storage/listingsStorage.js

@@ -1,10 +1,13 @@
-const path = require('path');
+import { JSONFileSync } from 'lowdb/node';
+import { getDirName } from '../../utils.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-const DB_PATH = path.dirname(require.main.filename) + '/db/jobListingData.json';
-const FileSync = require('lowdb/adapters/FileSync');
-const adapter = new FileSync(DB_PATH);
-const low = require('lowdb');
-const db = low(adapter);
+const file = path.join(getDirName(), '../', 'db/jobListingData.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, {});
+
+db.read();
 
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
@@ -19,35 +22,31 @@ const buildKey = (jobKey, providerId, endpoint) => {
   }
   return key;
 };
-
-exports.getNumberOfAllKnownListings = (jobId) => {
-  const data = db.get(`${jobId}.providerData`).value() || {};
+export const getNumberOfAllKnownListings = (jobId) => {
+  const data = db.chain.get(`${jobId}.providerData`).value() || {};
   return Object.values(data)
     .map((values) => Object.keys(values).length)
     .reduce((accumulator, currentValue) => accumulator + currentValue, 0);
 };
-
-exports.getListingProviderDataForAnalytics = (jobId) => {
+export const getListingProviderDataForAnalytics = (jobId) => {
   const key = buildKey(jobId, 'providerData');
-  return db.get(key).value() || {};
+  return db.chain.get(key).value() || {};
 };
-
-exports.getKnownListings = (jobId, providerId) => {
+export const getKnownListings = (jobId, providerId) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-  return db.get(providerListingsKey).value() || {};
+  return db.chain.get(providerListingsKey).value() || {};
 };
-
-exports.setKnownListings = (jobId, providerId, listings) => {
+export const setKnownListings = (jobId, providerId, listings) => {
   const providerListingsKey = buildKey(jobId, 'providerData', providerId, 'listings');
-
-  return db.set(providerListingsKey, listings).write();
+  db.chain.set(providerListingsKey, listings).value();
+  return db.write();
 };
-
-exports.setLastJobExecution = (jobId) => {
+export const setLastJobExecution = (jobId) => {
   const key = buildKey(jobId, null, 'lastExecution');
-  return db.set(key, Date.now()).write();
+  db.chain.set(key, Date.now()).value();
+  return db.write();
 };
-
-exports.removeListings = (jobId) => {
-  db.unset(jobId).write();
+export const removeListings = (jobId) => {
+  db.chain.unset(jobId).value();
+  db.write();
 };

lib/services/storage/userStorage.js

@@ -1,30 +1,40 @@
-const path = require('path');
-const DB_PATH = path.dirname(require.main.filename) + '/db/users.json';
-const FileSync = require('lowdb/adapters/FileSync');
-const adapter = new FileSync(DB_PATH);
-const low = require('lowdb');
-const db = low(adapter);
-const hasher = require('../security/hash');
-const { nanoid } = require('nanoid');
-const jobStorage = require('./jobStorage');
+import { JSONFileSync } from 'lowdb/node';
+import {config, getDirName} from '../../utils.js';
+import * as hasher from '../security/hash.js';
+import { nanoid } from 'nanoid';
+import * as jobStorage from './jobStorage.js';
+import path from 'path';
+import LowdashAdapter from './LowDashAdapter.js';
 
-db.defaults({
-  user: [
-    //you probably want to change the default password ;)
-    {
-      id: nanoid(),
-      lastLogin: Date.now(),
-      username: 'admin',
-      password: hasher.hash('admin'),
-      isAdmin: true,
-      isDemo: false,
-    },
-  ],
-}).write();
+const defaultData = {
+    user: [
+        //you probably want to change the default password ;)
+        {
+            id: nanoid(),
+            lastLogin: Date.now(),
+            username: 'admin',
+            password: hasher.hash('admin'),
+            isAdmin: true,
+        },
+        {
+            id: nanoid(),
+            lastLogin: Date.now(),
+            username: 'demo',
+            password: hasher.hash('demo'),
+            isAdmin: true,
+        },
+    ],
+};
 
-exports.getUsers = (withPassword) => {
+const file = path.join(getDirName(), '../', 'db/users.json');
+const adapter = new JSONFileSync(file);
+const db = new LowdashAdapter(adapter, defaultData);
+
+db.read();
+
+export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();
-  return db
+  return db.chain
     .get('user')
     .value()
     .map((user) => ({
@@ -34,13 +44,12 @@ exports.getUsers = (withPassword) => {
       numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
     }));
 };
-
-exports.getUser = (id) => {
+export const getUser = (id) => {
   const jobs = jobStorage.getJobs();
-  const user = db
+  const user = db.chain
     .get('user')
-    .value()
-    .find((user) => user.id === id);
+    .find((user) => user.id === id)
+    .value();
   if (user == null) {
     return null;
   }
@@ -49,13 +58,11 @@ exports.getUser = (id) => {
     numberOfJobs: jobs.filter((job) => job.userId === user.id).length,
   };
 };
-
-exports.upsertUser = ({ username, password, userId, isAdmin }) => {
-  const user = db
+export const upsertUser = ({ username, password, userId, isAdmin }) => {
+  const user = db.chain
     .get('user')
-    .value()
-    .filter((u) => u.id !== userId);
-
+    .filter((u) => u.id !== userId)
+    .value();
   user.push({
     id: userId || nanoid(),
     username,
@@ -63,21 +70,50 @@ exports.upsertUser = ({ username, password, userId, isAdmin }) => {
     password: hasher.hash(password),
     isAdmin,
   });
-
-  db.set('user', user).write();
+  db.chain.set('user', user).value();
+  db.write();
 };
-
-exports.setLastLoginToNow = ({ userId }) => {
-  db.get('user')
+export const setLastLoginToNow = ({ userId }) => {
+  db.chain
+    .get('user')
     .find((u) => u.id === userId)
     .assign({ lastLogin: Date.now() })
-    .write();
+    .value();
+  db.write();
+};
+export const removeUser = (userId) => {
+  const user = db.chain.get('user').value();
+  db.chain
+    .set(
+      'user',
+      user.filter((u) => u.id !== userId)
+    )
+    .value();
+  db.write();
 };
 
-exports.removeUser = (userId) => {
-  const user = db.get('user').value();
-  db.set(
-    'user',
-    user.filter((u) => u.id !== userId)
-  ).write();
+export const handleDemoUser = () => {
+    if(!config.demoMode){
+        const user = db.chain.get('user').value();
+        db.chain.get('user').value();
+        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+        db.write();
+    }else {
+        const demoUser = db.chain
+            .get('user')
+            .filter((u) => u.username === 'demo')
+            .value();
+        if (demoUser == null || demoUser.length === 0) {
+            db.chain.get('user')
+                .value()
+                .push({
+                    id: nanoid(),
+                    username: 'demo',
+                    password: hasher.hash('demo'),
+                    isAdmin: true,
+                });
+            db.write();
+        }
+    }
 };
+

lib/services/tracking/Tracker.js

@@ -0,0 +1,90 @@
+import Mixpanel from 'mixpanel';
+import {getJobs} from '../storage/jobStorage.js';
+import {getUniqueId} from './uniqueId.js';
+import {config, inDevMode} from '../../utils.js';
+import os from 'os';
+import {readFileSync} from 'fs';
+import {packageUp} from 'package-up';
+
+const mixpanelTracker = Mixpanel.init('718670ef1c58c0208256c1e408a3d75e');
+const distinct_id = getUniqueId() || 'N/A';
+const version = await getPackageVersion();
+
+export const track = function () {
+    //only send tracking information if the user allowed to do so.
+    if (config.analyticsEnabled && !inDevMode()) {
+        const activeProvider = new Set();
+        const activeAdapter = new Set();
+
+        const jobs = getJobs();
+
+        if (jobs != null && jobs.length > 0) {
+            jobs.forEach((job) => {
+                job.provider.forEach((provider) => {
+                    activeProvider.add(provider.id);
+                });
+                job.notificationAdapter.forEach((adapter) => {
+                    activeAdapter.add(adapter.id);
+                });
+            });
+
+            mixpanelTracker.track(
+                'fredy_tracking',
+                enrichTrackingObject({
+                    adapter: Array.from(activeAdapter),
+                    provider: Array.from(activeProvider),
+                }),
+            );
+        }
+    }
+};
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoJobCreated(jobData) {
+    if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+        mixpanelTracker.track('demoJobCreated', enrichTrackingObject(jobData));
+    }
+}
+
+/**
+ * Note, this will only be used when Fredy runs in demo mode
+ */
+export function trackDemoAccessed() {
+    if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+        mixpanelTracker.track('demoAccessed', enrichTrackingObject({}));
+    }
+}
+
+function enrichTrackingObject(trackingObject) {
+    const operating_system = os.platform();
+    const os_version = os.release();
+    const arch = process.arch;
+    const language = process.env.LANG || 'en';
+    const nodeVersion = process.version || 'N/A';
+
+    return {
+        ...trackingObject,
+        isDemo: config.demoMode,
+        operating_system,
+        os_version,
+        arch,
+        nodeVersion,
+        language,
+        distinct_id,
+        fredy_version: version
+    };
+}
+
+async function getPackageVersion() {
+    try {
+        const packagePath = await packageUp();
+        const packageJson = readFileSync(packagePath, 'utf8');
+        const json = JSON.parse(packageJson);
+        return json.version;
+    } catch (error) {
+        console.error('Error reading version from package.json', error);
+    }
+    return 'N/A';
+}

lib/services/tracking/uniqueId.js

@@ -0,0 +1,19 @@
+import { hostname, arch, cpus, platform } from 'os';
+import { createHash } from 'crypto';
+
+/**
+ * Don't worry, we are not evil ;) We however need a unique id per running instance
+ * @returns {string}
+ */
+export const getUniqueId = () => {
+  const systemInfo = {
+    hostname: hostname(),
+    architecture: arch(),
+    cpuCount: cpus().length,
+    platform: platform(),
+  };
+
+  const baseData = JSON.stringify(systemInfo);
+
+  return createHash('sha256').update(baseData).digest('hex');
+};

lib/utils.js

@@ -1,36 +1,91 @@
-function isOneOf(word, arr) {
-  if (arr == null || arr.length === 0) {
-    return false;
-  }
-  const expression = String.raw`\b(${arr.join('|')})\b`;
-  const blacklist = new RegExp(expression, 'ig');
+import {dirname} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {readFile} from 'fs/promises';
+import {createHash} from 'crypto';
+import {DEFAULT_CONFIG} from './defaultConfig.js';
 
-  return blacklist.test(word);
+function inDevMode(){
+    return process.env.NODE_ENV == null || process.env.NODE_ENV !== 'production';
+}
+
+function isOneOf(word, arr) {
+    if (arr == null || arr.length === 0) {
+        return false;
+    }
+    const expression = String.raw`\b(${arr.join('|')})\b`;
+    const blacklist = new RegExp(expression, 'ig');
+    return blacklist.test(word);
 }
 
 function nullOrEmpty(val) {
-  return val == null || val.length === 0;
+    return val == null || val.length === 0;
 }
 
 function timeStringToMs(timeString, now) {
-  const d = new Date(now);
-  const parts = timeString.split(':');
-  d.setHours(parts[0]);
-  d.setMinutes(parts[1]);
-  d.setSeconds(0);
-  return d.getTime();
+    const d = new Date(now);
+    const parts = timeString.split(':');
+    d.setHours(parts[0]);
+    d.setMinutes(parts[1]);
+    d.setSeconds(0);
+    return d.getTime();
 }
 
 function duringWorkingHoursOrNotSet(config, now) {
-  const { workingHours } = config;
-  if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
-    return true;
-  }
-
-  const toDate = timeStringToMs(workingHours.to, now);
-  const fromDate = timeStringToMs(workingHours.from, now);
-
-  return fromDate <= now && toDate >= now;
+    const {workingHours} = config;
+    if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
+        return true;
+    }
+    const toDate = timeStringToMs(workingHours.to, now);
+    const fromDate = timeStringToMs(workingHours.from, now);
+    return fromDate <= now && toDate >= now;
 }
 
-module.exports = { isOneOf, nullOrEmpty, duringWorkingHoursOrNotSet };
+function getDirName() {
+    return dirname(fileURLToPath(import.meta.url));
+}
+
+function buildHash(...inputs) {
+    if (inputs == null) {
+        return null;
+    }
+    const cleaned = inputs.filter(i => i != null && i.length > 0);
+    if (cleaned.length === 0) {
+        return null;
+    }
+    return createHash('sha256')
+        .update(cleaned.join(','))
+        .digest('hex');
+}
+
+let config = {};
+export async function readConfigFromStorage(){
+    return JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
+}
+
+export async function refreshConfig(){
+    try {
+        config = await readConfigFromStorage();
+        //backwards compatability...
+        config.analyticsEnabled ??= null;
+        config.demoMode ??= false;
+    } catch (error) {
+        config = {...DEFAULT_CONFIG};
+        console.error('Error reading config file', error);
+    }
+}
+await refreshConfig();
+
+export {isOneOf};
+export {inDevMode};
+export {nullOrEmpty};
+export {duringWorkingHoursOrNotSet};
+export {getDirName};
+export {config};
+export {buildHash};
+export default {
+    isOneOf,
+    nullOrEmpty,
+    duringWorkingHoursOrNotSet,
+    getDirName,
+    config,
+};

package.json

@@ -1,20 +1,17 @@
 {
   "name": "fredy",
-  "version": "5.4.2",
+  "version": "11.2.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
-    "start": "node index.js",
-    "dev": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack-dev-server --progress --colors --watch --config ./webpack.dev.js",
-    "prod": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
-    "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
-    "test": "mocha --timeout 20000 test/**/*.test.js",
-    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged"
-    }
+    "start": "node prod.js",
+    "dev": "yarn && rm -rf ./ui/public/* && vite",
+    "ui": "rm -rf ./ui/public/* && vite",
+    "prod": "yarn && vite build --emptyOutDir",
+    "format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
+    "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
+    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
   },
+  "type": "module",
   "lint-staged": {
     "*.js": [
       "eslint ./index.js ./lib/**/*.js ./test/**/*.js",
@@ -43,8 +40,8 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=12.13.0",
-    "npm": ">=6.0.0"
+    "node": ">=20.0.0",
+    "npm": ">=7.0.0"
   },
   "browserslist": [
     "> 0.5%",
@@ -53,64 +50,59 @@
     "Firefox ESR"
   ],
   "dependencies": {
+    "@douyinfe/semi-ui": "2.79.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.6.0",
-    "axios": "0.24.0",
-    "axios-retry": "^3.2.4",
-    "body-parser": "1.19.0",
-    "cookie-session": "1.4.0",
-    "handlebars": "4.7.7",
-    "highcharts": "9.3.1",
-    "highcharts-react-official": "3.1.0",
-    "lowdb": "1.0.0",
+    "@sendgrid/mail": "8.1.5",
+    "@vitejs/plugin-react": "4.4.1",
+    "better-sqlite3": "^11.10.0",
+    "body-parser": "2.2.0",
+    "cheerio": "^1.0.0",
+    "cookie-session": "2.1.0",
+    "handlebars": "4.7.8",
+    "highcharts": "12.2.0",
+    "highcharts-react-official": "3.2.2",
+    "lodash": "4.17.21",
+    "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "3.1.30",
-    "node-mailjet": "3.3.4",
-    "query-string": "^7.0.1",
-    "react": "17.0.2",
-    "react-dom": "17.0.2",
-    "react-redux": "7.2.6",
+    "mixpanel": "^0.18.1",
+    "nanoid": "5.1.5",
+    "node-fetch": "3.3.2",
+    "node-mailjet": "6.0.8",
+    "package-up": "^5.0.0",
+    "puppeteer": "^24.8.2",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "query-string": "9.1.2",
+    "react": "18.3.1",
+    "react-dom": "18.3.1",
+    "react-redux": "9.2.0",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "react-switch": "^6.0.0",
-    "redux": "4.1.2",
-    "redux-thunk": "2.4.0",
-    "restana": "4.9.2",
-    "semantic-ui-react": "2.0.4",
-    "serve-static": "^1.14.1",
+    "redux": "5.0.1",
+    "redux-thunk": "3.1.0",
+    "restana": "4.9.9",
+    "serve-static": "1.16.2",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "x-ray": "2.3.4"
+    "vite": "5.4.11"
   },
   "devDependencies": {
-    "@babel/core": "7.16.0",
-    "@babel/preset-env": "7.16.4",
-    "@babel/preset-react": "7.16.0",
-    "babel-eslint": "10.1.0",
-    "babel-loader": "8.2.3",
-    "chai": "4.3.4",
-    "clean-webpack-plugin": "4.0.0",
-    "copy-webpack-plugin": "10.0.0",
-    "css-loader": "6.5.1",
-    "eslint": "7.32.0",
-    "eslint-config-prettier": "8.3.0",
-    "eslint-plugin-react": "7.27.1",
-    "file-loader": "6.2.0",
-    "history": "5.1.0",
-    "husky": "4.3.8",
-    "less": "4.1.2",
-    "less-loader": "10.2.0",
-    "lint-staged": "12.1.2",
-    "mocha": "9.1.3",
-    "prettier": "2.5.0",
-    "proxyquire": "2.1.3",
-    "redux-logger": "3.0.6",
-    "style-loader": "3.3.1",
-    "url-loader": "4.1.1",
-    "webpack": "5.64.4",
-    "webpack-cli": "4.9.1",
-    "webpack-dev-server": "3.11.2",
-    "webpack-merge": "5.8.0"
+    "@babel/core": "7.27.1",
+    "@babel/eslint-parser": "7.27.1",
+    "@babel/preset-env": "7.27.2",
+    "@babel/preset-react": "7.27.1",
+    "chai": "5.2.0",
+    "eslint": "8.56.0",
+    "eslint-config-prettier": "8.8.0",
+    "eslint-plugin-react": "7.37.4",
+    "esmock": "2.7.0",
+    "history": "5.3.0",
+    "husky": "9.1.7",
+    "less": "4.3.0",
+    "lint-staged": "15.5.2",
+    "mocha": "10.8.2",
+    "prettier": "3.5.3",
+    "redux-logger": "3.0.6"
   }
 }

prod.js

@@ -0,0 +1,2 @@
+process.env.NODE_ENV = 'production';
+import('./index.js');

reverse-engineered-immoscout.md

@@ -0,0 +1,80 @@
+# Reverse Engineered Immoscout24's Mobile API
+
+## What is Immoscout24?
+
+Immobilienscout24 (commonly known as Immoscout) is one of Germany's largest and most popular real estate platforms. It serves as a marketplace where property owners, real estate agents, and property management companies can list apartments, houses, and commercial properties for rent or sale. For people searching for a new home in Germany, Immoscout is often one of the first platforms they check.
+
+The platform allows users to filter properties based on various criteria such as location, price, size, number of rooms, and additional features like balconies or built-in kitchens. Immoscout24 is available both as a website and as a mobile application, making it accessible across different devices.
+
+## Why do we do this?
+
+Crawling Immoscout24 the oldschool way has become virtually impossible due to their extensive bot detection mechanisms. Immoscout has implemented various anti-scraping measures to prevent automated access to their platform. These measures can include:
+
+1. IP-based rate limiting
+2. Browser fingerprinting
+3. CAPTCHA challenges
+4. Behavior analysis to detect non-human patterns
+5. JavaScript-based challenges that must be solved before content is displayed
+
+These protections make it extremely difficult to reliably extract data from Immoscout using conventional web scraping approaches. Even with techniques like rotating proxies or mimicking human behavior, the bot detection systems have become increasingly effective at identifying and blocking automated access attempts.
+
+## Mobile API Reverse Engineering
+
+To work around these limitations, we are in the progress of reverse-engineering Immoscout24's mobile API. The mobile applications need to communicate with Immoscout's servers to retrieve listing data, and these API endpoints typically have fewer anti-bot protections than the web interface.
+
+The mobile API provides several key endpoints:    
+- Search total endpoint: Returns the total number of listings for a given query
+- Search list endpoint: Retrieves the actual listings with details
+- Expose endpoint: Returns detailed information about a specific listing
+
+Challenges:    
+1. Identifying the necessary endpoints and parameters required to perform searches
+2. Mapping the mobile API parameters to their web counterparts to maintain compatibility with existing search URLs
+
+
+## Api Specs
+
+#### Search for Listings
+
+`GET /search/total?{search parameters}`    
+*Returns the total number of listings for the given query.*
+```
+curl -H "User-Agent: ImmoScout24_1410_30_._" \
+     -H "Accept: application/json" \
+     "https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
+```
+
+---
+
+#### Retrieve the listings
+`POST /search/list?{search parameters}`   
+*The body is json encoded and contains data specifying additional results (advertisements) to return. The format is as follows (It is not necessary to provide data for the specified keys.)*
+  ```
+  {
+  "supportedResultListTypes": [],
+  "userData": {}
+  }
+  ```
+```
+curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
+  -H "Connection: keep-alive" \
+  -H "User-Agent: ImmoScout24_1410_30_._" \
+  -H "Accept: application/json" \
+  -H "Content-Type: application/json" \
+  -d '{"supportedResultListType":[],"userData":{}}'
+
+```
+
+---
+#### Get details of listings
+`GET /expose/{id}`
+The response contains additional details not included in the listing response.
+```
+curl -H "User-Agent: ImmoScout24_1410_30_._" \
+     -H "Accept: application/json" \
+     "https://api.mobile.immobilienscout24.de/expose/158382494"
+```
+
+
+## Parameters 
+The parameters between web and mobile are very different which is why we have to translate them. Please see `immoscout-web-translator.js`.

test/mocks/mockNotification.js

@@ -1,12 +1,10 @@
-module.exports = {
-  _tmpStore: {},
+let tmpStore = {};
 
-  send: (serviceName, payload) => {
-    this._tmpStore = { serviceName, payload };
-    return [Promise.resolve()];
-  },
-
-  get: () => {
-    return this._tmpStore;
-  },
+export const send = (serviceName, payload) => {
+  tmpStore = { serviceName, payload };
+  return [Promise.resolve()];
+};
+
+export const get = () => {
+  return tmpStore;
 };

test/mocks/mockStore.js

@@ -1,11 +1,8 @@
 const db = {};
-
-exports.setKnownListings = (jobKey, providerId, listings) => {
+export const setKnownListings = (jobKey, providerId, listings) => {
   if (!Array.isArray(listings)) throw Error('Not a valid array');
-
   db[providerId] = listings;
 };
-
-exports.getKnownListings = (jobKey, providerId) => {
+export const getKnownListings = (jobKey, providerId) => {
   return db[providerId] || [];
 };

test/provider/einsAImmobilien.test.js

@@ -1,45 +1,31 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
-const mockNotification = require('../mocks/mockNotification');
-const providerConfig = require('./testProvider.json');
-const mockStore = require('../mocks/mockStore');
-const proxyquire = require('proxyquire').noCallThru();
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/einsAImmobilien');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { providerConfig, mockFredy } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/einsAImmobilien.js';
 
 describe('#einsAImmobilien testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   provider.init(providerConfig.einsAImmobilien, [], []);
-
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test einsAImmobilien provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
       fredy.execute().then((listings) => {
         expect(listings).to.be.a('array');
-
-        const notificationObj = mockNotification.get();
+        const notificationObj = get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-
           /** check the values if possible **/
-          expect(notify.price).that.does.include('EUR');
           expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;
           expect(notify.link).that.does.include('https://www.1a-immobilienmarkt.de');

test/provider/immobilienDe.test.js

@@ -0,0 +1,40 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { providerConfig, mockFredy } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/immobilienDe.js';
+
+describe('#immobilien.de testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+  provider.init(providerConfig.immobilienDe, [], []);
+  it('should test immobilien.de provider', async () => {
+    const Fredy = await mockFredy();
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immobilienDe');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.size).that.does.include('m²');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilien.de');
+          expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/immonet.test.js

@@ -1,52 +1,38 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
-const mockNotification = require('../mocks/mockNotification');
-const providerConfig = require('./testProvider.json');
-const mockStore = require('../mocks/mockStore');
-const proxyquire = require('proxyquire').noCallThru();
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immonet');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import {get} from '../mocks/mockNotification.js';
+import {mockFredy, providerConfig} from '../utils.js';
+import {expect} from 'chai';
+import * as provider from '../../lib/provider/immonet.js';
 
 describe('#immonet testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
-  provider.init(providerConfig.immonet, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
-  it('should test immonet provider', async () => {
-    return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
-      fredy.execute().then((listing) => {
-        expect(listing).to.be.a('array');
-
-        const notificationObj = mockNotification.get();
-        expect(notificationObj).to.be.a('object');
-        expect(notificationObj.serviceName).to.equal('immonet');
-
-        notificationObj.payload.forEach((notify) => {
-          /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
-          expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
-          expect(notify.title).to.be.a('string');
-          expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
-
-          /** check the values if possible **/
-          expect(notify.price).that.does.include('€');
-          expect(notify.size).that.does.include('m²');
-          expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immonet.de');
-          expect(notify.address).to.be.not.empty;
-        });
-        resolve();
-      });
+    after(() => {
+        similarityCache.stopCacheCleanup();
+    });
+    provider.init(providerConfig.immonet, [], []);
+    it('should test immonet provider', async () => {
+        const Fredy = await mockFredy();
+        return await new Promise((resolve) => {
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
+            fredy.execute().then((listing) => {
+                expect(listing).to.be.a('array');
+                const notificationObj = get();
+                expect(notificationObj).to.be.a('object');
+                expect(notificationObj.serviceName).to.equal('immonet');
+                notificationObj.payload.forEach((notify) => {
+                    /** check the actual structure **/
+                    expect(notify.id).to.be.a('string');
+                    expect(notify.price).to.be.a('string');
+                    expect(notify.size).to.be.a('string');
+                    expect(notify.title).to.be.a('string');
+                    expect(notify.link).to.be.a('string');
+                    expect(notify.address).to.be.a('string');
+
+                    expect(notify.size).that.does.include('m²');
+                    expect(notify.title).to.be.not.empty;
+                    expect(notify.address).to.be.not.empty;
+                });
+                resolve();
+            });
+        });
     });
-  });
 });

test/provider/immoscout.test.js

@@ -1,57 +1,36 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
-const mockNotification = require('../mocks/mockNotification');
-const providerConfig = require('./testProvider.json');
-const mockStore = require('../mocks/mockStore');
-const proxyquire = require('proxyquire').noCallThru();
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immoscout');
-const scrapingAnt = require('../../lib/services/scrapingAnt');
+import { expect } from 'chai';
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { get } from '../mocks/mockNotification.js';
+import * as provider from '../../lib/provider/immoscout.js';
 
-describe('#immoscout testsuite()', () => {
+describe('#immoscout provider testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
+
   provider.init(providerConfig.immoscout, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
   it('should test immoscout provider', async () => {
+    const Fredy = await mockFredy();
     return await new Promise((resolve) => {
-      if (!scrapingAnt.isScrapingAntApiKeySet()) {
-        /* eslint-disable no-console */
-        console.info('Skipping Immoscout test as ScrapingAnt Api Key is not set.');
-        /* eslint-enable no-console */
-        resolve();
-        return;
-      }
-
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
-      fredy.execute().then((listing) => {
-        expect(listing).to.be.a('array');
-
-        const notificationObj = mockNotification.get();
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, '', similarityCache);
+      fredy.execute().then((listings) => {
+        expect(listings).to.be.a('array');
+        const notificationObj = get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immoscout');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
-          expect(notify.price).that.does.include('€');
-          expect(notify.size).that.does.include('m²');
+          expect(notify.size).to.be.not.empty;
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.immobilienscout24.de');
-          expect(notify.address).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immobilienscout24.de/');
         });
         resolve();
       });

test/provider/immoswp.test.js

@@ -0,0 +1,37 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/immoswp.js';
+
+describe('#immoswp testsuite()', () => {
+  after(() => {
+    similarityCache.stopCacheCleanup();
+  });
+  provider.init(providerConfig.immoswp, [], []);
+  it('should test immoswp provider', async () => {
+    const Fredy = await mockFredy();
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immoswp');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          /** check the values if possible **/
+          expect(notify.price).that.does.include('€');
+          expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://immo.swp.de');
+        });
+        resolve();
+      });
+    });
+  });
+});

test/provider/immowelt.test.js

@@ -1,45 +1,32 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
-const mockNotification = require('../mocks/mockNotification');
-const providerConfig = require('./testProvider.json');
-const mockStore = require('../mocks/mockStore');
-const proxyquire = require('proxyquire').noCallThru();
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/immowelt');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/immowelt.js';
 
 describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   it('should test immowelt provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
-        const notificationObj = mockNotification.get();
+        const notificationObj = get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('immowelt');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
           expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
-          expect(notify.price).that.does.include('€');
-          if (notify.size.trim().toLowerCase() !== 'k.a.') {
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
             expect(notify.size).that.does.include('m²');
           }
           expect(notify.title).to.be.not.empty;

test/provider/kleinanzeigen.test.js

@@ -1,45 +1,32 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
-const mockNotification = require('../mocks/mockNotification');
-const providerConfig = require('./testProvider.json');
-const mockStore = require('../mocks/mockStore');
-const proxyquire = require('proxyquire').noCallThru();
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/kleinanzeigen');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/kleinanzeigen.js';
 
 describe('#kleinanzeigen testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
   it('should test kleinanzeigen provider', async () => {
+    const Fredy = await mockFredy();
     provider.init(providerConfig.kleinanzeigen, [], []);
-    const Fredy = proxyquire('../../lib/FredyRuntime', {
-      './services/storage/listingsStorage': {
-        ...mockStore,
-      },
-      './notification/notify': mockNotification,
-    });
-
     return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
       fredy.execute().then((listing) => {
         expect(listing).to.be.a('array');
-
-        const notificationObj = mockNotification.get();
+        const notificationObj = get();
         expect(notificationObj).to.be.a('object');
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
-
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
-          expect(notify.size).to.be.a('string');
+          expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
-          expect(notify.price).to.be.a('string');
           expect(notify.address).to.be.a('string');
-
           /** check the values if possible **/
           expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
+          expect(notify.link).that.does.include('https://www.kleinanzeigen.de');
           expect(notify.address).to.be.not.empty;
         });
         resolve();

test/provider/neubauKompass.test.js

@@ -1,48 +1,36 @@
-const similarityCache = require('../../lib/services/similarity-check/similarityCache');
-const mockNotification = require('../mocks/mockNotification');
-const providerConfig = require('./testProvider.json');
-const mockStore = require('../mocks/mockStore');
-const proxyquire = require('proxyquire').noCallThru();
-const expect = require('chai').expect;
-const provider = require('../../lib/provider/neubauKompass');
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import {get} from '../mocks/mockNotification.js';
+import {mockFredy, providerConfig} from '../utils.js';
+import {expect} from 'chai';
+import * as provider from '../../lib/provider/neubauKompass.js';
 
 describe('#neubauKompass testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-  provider.init(providerConfig.neubauKompass, [], []);
-  const Fredy = proxyquire('../../lib/FredyRuntime', {
-    './services/storage/listingsStorage': {
-      ...mockStore,
-    },
-    './notification/notify': mockNotification,
-  });
-
-  it('should test neubauKompass provider', async () => {
-    return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
-      fredy.execute().then((listing) => {
-        expect(listing).to.be.a('array');
-
-        const notificationObj = mockNotification.get();
-        expect(notificationObj.serviceName).to.equal('neubauKompass');
-
-        notificationObj.payload.forEach((notify) => {
-          expect(notify).to.be.a('object');
-
-          /** check the actual structure **/
-          expect(notify.id).to.be.a('string');
-          expect(notify.title).to.be.a('string');
-          expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
-
-          /** check the values if possible **/
-          expect(notify.title).to.be.not.empty;
-          expect(notify.link).that.does.include('https://www.neubaukompass.de');
-          expect(notify.address).to.be.not.empty;
-        });
-        resolve();
-      });
+    after(() => {
+        similarityCache.stopCacheCleanup();
+    });
+    provider.init(providerConfig.neubauKompass, [], []);
+    it('should test neubauKompass provider', async () => {
+        const Fredy = await mockFredy();
+        return await new Promise((resolve) => {
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
+            fredy.execute().then((listing) => {
+                expect(listing).to.be.a('array');
+                const notificationObj = get();
+                expect(notificationObj.serviceName).to.equal('neubauKompass');
+                notificationObj.payload.forEach((notify) => {
+                    expect(notify).to.be.a('object');
+                    /** check the actual structure **/
+                    expect(notify.id).to.be.a('string');
+                    expect(notify.title).to.be.a('string');
+                    expect(notify.link).to.be.a('string');
+                    expect(notify.address).to.be.a('string');
+                    /** check the values if possible **/
+                    expect(notify.title).to.be.not.empty;
+                    expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    expect(notify.address).to.be.not.empty;
+                });
+                resolve();
+            });
+        });
     });
-  });
 });