Improvements (#193)

* improving release banner

* renaming general to settings

* fixing working hours if they go to next day

* fixing comparing versions

* upgrade dependencies

.babelrc

@@ -3,9 +3,7 @@
     [
       "@babel/preset-env",
       {
-        "exclude": [
-          "transform-regenerator"
-        ]
+        "exclude": ["transform-regenerator"]
       }
     ],
     [
@@ -15,4 +13,4 @@
       }
     ]
   ]
-}
+}

.dockerignore

@@ -1,3 +1,7 @@
-node_modules
+node_modules/
 npm-debug.log
-test
+test/
+db/
+conf/
+.git/
+.github/

.eslintrc.js

@@ -1,298 +0,0 @@
-module.exports = {
-  env: {
-    commonjs: true,
-    es6: true,
-    node: true,
-    browser: true,
-    mocha: true,
-  },
-  parser: 'babel-eslint',
-  extends: ['eslint:recommended', 'prettier'],
-  plugins: ['react'],
-  globals: {
-    Promise: false,
-    describe: true,
-    after: true,
-    it: true,
-    fetch: true,
-  },
-  parserOptions: {
-    ecmaVersion: 2018,
-    sourceType: 'module',
-  },
-  rules: {
-    eqeqeq: [2, 'allow-null'],
-
-    // ###########################################################
-    // ### Semantics / Performance impacting
-    // ###########################################################
-    // babel inserts `'use strict';` for us
-    strict: 0,
-
-    'no-redeclare': [2, { builtinGlobals: false }],
-
-    // If a class method does not use this, it can safely be made a static function.
-    // http://eslint.org/docs/rules/class-methods-use-this
-    'class-methods-use-this': ['off'],
-
-    // ###########################################################
-    // ### Style
-    // ###########################################################
-    indent: ['off', 2],
-    'linebreak-style': ['error', 'unix'],
-    quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }],
-    semi: ['error', 'always'],
-    'no-console': ['error', { allow: ['warn', 'error'] }],
-
-    // ###########################################################
-    // ### React
-    // ###########################################################
-    // Specify whether double or single quotes should be used in JSX attributes
-    // http://eslint.org/docs/rules/jsx-quotes
-    'jsx-quotes': ['error', 'prefer-double'],
-
-    // Prevent missing displayName in a React component definition
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/display-name.md
-    'react/display-name': ['off'],
-
-    // Forbid certain propTypes (any, array, object)
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/forbid-prop-types.md
-    'react/forbid-prop-types': 'off',
-
-    // Validate closing bracket location in JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-closing-bracket-location.md
-    'react/jsx-closing-bracket-location': ['off'],
-
-    // Enforce or disallow spaces inside of curly braces in JSX attributes
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-curly-spacing.md
-    'react/jsx-curly-spacing': ['off'],
-
-    // Enforce event handler naming conventions in JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-handler-names.md
-    'react/jsx-handler-names': [
-      'off',
-      {
-        eventHandlerPrefix: 'handle',
-        eventHandlerPropPrefix: 'on',
-      },
-    ],
-
-    // Validate props indentation in JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-indent-props.md
-    'react/jsx-indent-props': 'off',
-
-    // Validate JSX has key prop when in array or iterator
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-key.md
-    'react/jsx-key': 'off',
-
-    // Limit maximum of props on a single line in JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-max-props-per-line.md
-    'react/jsx-max-props-per-line': ['off'],
-
-    // Prevent usage of .bind() in JSX props
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-bind.md
-    'react/jsx-no-bind': [
-      'error',
-      {
-        ignoreRefs: true,
-        allowArrowFunctions: true,
-        allowBind: false,
-      },
-    ],
-
-    // Prevent duplicate props in JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-duplicate-props.md
-    'react/jsx-no-duplicate-props': ['error', { ignoreCase: true }],
-
-    // Prevent usage of unwrapped JSX strings
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-literals.md
-    'react/jsx-no-literals': 'off',
-
-    // Disallow undeclared variables in JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-undef.md
-    'react/jsx-no-undef': 'error',
-
-    // Enforce PascalCase for user-defined JSX components
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-pascal-case.md
-    'react/jsx-pascal-case': [
-      'error',
-      {
-        allowAllCaps: true,
-        ignore: [],
-      },
-    ],
-
-    // Enforce propTypes declarations alphabetical sorting
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-prop-types.md
-    'react/sort-prop-types': [
-      'off',
-      {
-        ignoreCase: true,
-        callbacksLast: false,
-        requiredFirst: false,
-      },
-    ],
-
-    // Deprecated in favor of react/jsx-sort-props
-    'react/jsx-sort-prop-types': 'off',
-
-    // Enforce props alphabetical sorting
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-sort-props.md
-    'react/jsx-sort-props': 'off',
-
-    // Prevent React to be incorrectly marked as unused
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-uses-react.md
-    'react/jsx-uses-react': 'error',
-
-    // Prevent variables used in JSX to be incorrectly marked as unused
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-uses-vars.md
-    'react/jsx-uses-vars': 'error',
-
-    // Prevent usage of dangerous JSX properties
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-danger.md
-    'react/no-danger': 'warn',
-
-    // Prevent usage of deprecated methods
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-deprecated.md
-    'react/no-deprecated': ['error'],
-
-    // Prevent usage of setState in componentDidMount
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-did-mount-set-state.md
-    'react/no-did-mount-set-state': ['error'],
-
-    // Prevent usage of setState in componentDidUpdate
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-did-update-set-state.md
-    'react/no-did-update-set-state': ['warn'],
-
-    // Prevent direct mutation of this.state
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-direct-mutation-state.md
-    'react/no-direct-mutation-state': 'off',
-
-    // Prevent usage of isMounted
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-is-mounted.md
-    'react/no-is-mounted': 'error',
-
-    // Prevent usage of setState
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-set-state.md
-    'react/no-set-state': 'off',
-
-    // Prevent using string references
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-string-refs.md
-    'react/no-string-refs': 'warn',
-
-    // Prevent usage of unknown DOM property
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-unknown-property.md
-    'react/no-unknown-property': 'error',
-
-    // Prevent missing props validation in a React component definition
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/prop-types.md
-    'react/prop-types': ['error', { ignore: [], customValidators: [], skipUndeclared: true }],
-
-    // Prevent missing React when using JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/react-in-jsx-scope.md
-    'react/react-in-jsx-scope': 'error',
-
-    // Restrict file extensions that may be required
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/require-extension.md
-    // deprecated in favor of import/extensions
-    'react/require-extension': ['off', { extensions: ['.jsx', '.js'] }],
-
-    // Require render() methods to return something
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/require-render-return.md
-    'react/require-render-return': 'error',
-
-    // Prevent extra closing tags for components without children
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/self-closing-comp.md
-    'react/self-closing-comp': 'warn',
-
-    // Enforce spaces before the closing bracket of self-closing JSX elements
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-space-before-closing.md
-    'react/jsx-space-before-closing': ['warn', 'always'],
-
-    // Enforce component methods order
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/sort-comp.md
-    'react/sort-comp': 'off',
-
-    // Prevent missing parentheses around multilines JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-wrap-multilines.md
-    'react/jsx-wrap-multilines': [
-      'warn',
-      {
-        declaration: true,
-        assignment: true,
-        return: true,
-      },
-    ],
-    'react/wrap-multilines': 'off', // deprecated version
-
-    // Require that the first prop in a JSX element be on a new line when the element is multiline
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-first-prop-new-line.md
-    'react/jsx-first-prop-new-line': ['off'],
-
-    // Enforce spacing around jsx equals signs
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-equals-spacing.md
-    'react/jsx-equals-spacing': ['warn', 'never'],
-
-    // Disallow target="_blank" on links
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-target-blank.md
-    'react/jsx-no-target-blank': 'error',
-
-    // only .jsx files may have JSX
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-filename-extension.md
-    'react/jsx-filename-extension': ['error', { extensions: ['.js'] }],
-
-    // prevent accidental JS comments from being injected into JSX as text
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-no-comment-textnodes.md
-    'react/jsx-no-comment-textnodes': 'error',
-    'react/no-comment-textnodes': 'off', // deprecated version
-
-    // disallow using React.render/ReactDOM.render's return value
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-render-return-value.md
-    'react/no-render-return-value': 'error',
-
-    // require a shouldComponentUpdate method, or PureRenderMixin
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/require-optimization.md
-    'react/require-optimization': ['off', { allowDecorators: [] }],
-
-    // warn against using findDOMNode()
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-find-dom-node.md
-    'react/no-find-dom-node': 'warn',
-
-    // Forbid certain props on Components
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/forbid-component-props.md
-    'react/forbid-component-props': ['off', { forbid: [] }],
-
-    // Prevent problem with children and props.dangerouslySetInnerHTML
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-danger-with-children.md
-    'react/no-danger-with-children': 'error',
-
-    // Prevent unused propType definitions
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-unused-prop-types.md
-    'react/no-unused-prop-types': [
-      'warn',
-      {
-        customValidators: [],
-        skipShapeProps: true,
-      },
-    ],
-
-    // Require style prop value be an object or var
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/style-prop-object.md
-    'react/style-prop-object': 'error',
-
-    // Prevent passing of children as props
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/no-children-prop.md
-    'react/no-children-prop': 'warn',
-
-    // Validate whitespace in and around the JSX opening and closing brackets
-    // https://github.com/yannickcr/eslint-plugin-react/blob/master/docs/rules/jsx-tag-spacing.md
-    'react/jsx-tag-spacing': [
-      'warn',
-      {
-        closingSlash: 'never',
-        beforeSelfClosing: 'always',
-        afterOpening: 'never',
-      },
-    ],
-  },
-};

.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,73 @@
+name: Bug Report
+description: Help us improve Fredy by reporting a bug
+title: "[Bug]: "
+labels: [bug]
+assignees: []
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Bug Description
+      description: Provide a clear and concise description of the bug.
+      placeholder: e.g. "Fredy crashes when I click on Save."
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to Reproduce
+      description: List the steps to reproduce the issue.
+      placeholder: |
+        1. Go to '...'
+        2. Click on '...'
+        3. Scroll down to '...'
+        4. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+      placeholder: "It should save without errors."
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+      placeholder: "Fredy crashed with error XYZ."
+    validations:
+      required: true
+
+  - type: textarea
+    id: screenshots
+    attributes:
+      label: Screenshots / Logs
+      description: Add screenshots or paste log output to help explain the problem.
+      placeholder: "Drag and drop screenshots here, or paste logs."
+    validations:
+      required: false
+
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: Provide details about your environment.
+      placeholder: "OS: macOS 15, Browser: Chrome 124, App version: 1.2.3"
+    validations:
+      required: true
+
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional Context
+      description: Add any other context about the problem here.
+      placeholder: "Any other information that might help..."
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,51 @@
+name: Feature Request
+description: Suggest an improvement or new idea for Fredy
+title: "[Feature]: "
+labels: [enhancement]
+assignees: []
+
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Related Problem
+      description: Is your feature request related to a problem? Describe it clearly.
+      placeholder: "Example: It’s difficult to do X when Y happens..."
+    validations:
+      required: false
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed Feature
+      description: Describe the feature you would like to see.
+      placeholder: "I would like Fredy to automatically..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives Considered
+      description: List any alternative solutions or workarounds you’ve tried or thought about.
+      placeholder: "Instead of this, I also considered..."
+    validations:
+      required: false
+
+  - type: textarea
+    id: benefits
+    attributes:
+      label: Benefits
+      description: Explain how this feature would improve Fredy or it's user experience.
+      placeholder: "This would save users time by..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional Context
+      description: Add any other context, examples, or screenshots that might help clarify your idea.
+      placeholder: "Any other relevant information..."
+    validations:
+      required: false

.github/workflows/check_source.yml

@@ -0,0 +1,26 @@
+name: Check the source code
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  check_source_code:
+    name: Check the source code
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Check formatting
+        run: yarn format:check
+
+      - name: Lint
+        run: yarn lint

.github/workflows/docker.yml

@@ -0,0 +1,97 @@
+name: Create and publish Docker image
+
+on:
+  push:
+    branches:
+      - 'master'
+    tags:
+      - '*'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Test container health with docker compose
+      - name: Test container with docker compose
+        run: |
+          echo "Starting container with docker compose..."
+          docker compose up --build -d
+          echo "Waiting for container to be ready (60 seconds for start_period)..."
+          sleep 60
+
+          echo "Monitoring container health for 30 seconds..."
+          SECONDS_ELAPSED=0
+          HEALTH_CHECK_INTERVAL=5
+          TOTAL_DURATION=30
+          
+          while [ $SECONDS_ELAPSED -lt $TOTAL_DURATION ]; do
+            HEALTH_STATUS=$(docker inspect --format='{{.State.Health.Status}}' fredy 2>/dev/null || echo "not_found")
+            CONTAINER_STATUS=$(docker inspect --format='{{.State.Status}}' fredy 2>/dev/null || echo "not_found")
+            echo "[$SECONDS_ELAPSED/$TOTAL_DURATION sec] Container: $CONTAINER_STATUS, Health: $HEALTH_STATUS"
+            
+            # Check if container is not running or unhealthy
+            if [ "$CONTAINER_STATUS" != "running" ]; then
+              echo "Container stopped running! Status: $CONTAINER_STATUS"
+              docker compose logs fredy
+              exit 1
+            fi
+            
+            if [ "$HEALTH_STATUS" = "unhealthy" ]; then
+              echo "Container is unhealthy!"
+              docker compose logs fredy
+              docker inspect --format='{{json .State.Health}}' fredy | jq
+              exit 1
+            fi
+            
+            sleep $HEALTH_CHECK_INTERVAL
+            SECONDS_ELAPSED=$((SECONDS_ELAPSED + HEALTH_CHECK_INTERVAL))
+          done
+
+          docker compose down

.github/workflows/stales.yml

@@ -0,0 +1,21 @@
+name: Close stale issues and PRs
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Daily
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v7
+        with:
+          days-before-stale: 30
+          days-before-close: 7
+          stale-issue-message: 'This issue has been automatically marked as stale due to inactivity.'
+          stale-pr-message: 'This PR has been automatically marked as stale due to inactivity.'
+          close-issue-message: 'Closing this issue due to prolonged inactivity.'
+          close-pr-message: 'Closing this PR due to prolonged inactivity.'
+          exempt-issue-labels: 'keep-open'
+          exempt-pr-labels: 'keep-open'
+          only: 'pulls'

.github/workflows/test.yml

@@ -0,0 +1,22 @@
+name: Test
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  schedule:
+    - cron: '0 12 * * *'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'yarn'
+
+      - run: yarn install
+      - run: yarn test

.gitignore

@@ -1,6 +1,7 @@
 node_modules/
 ui/public/
-db/
+db/*.json
+db/*.db*
 npm-debug.log
 .DS_Store
 .idea

.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

.nvmrc

@@ -1 +1 @@
-12.18.3
+16.14.0

.prettierignore

@@ -0,0 +1,6 @@
+/ui/public
+/db/
+/conf/
+
+# TODO re-write from scratch or fix all html structure issues
+/lib/notification/emailTemplate/template.hbs

.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "printWidth": 120
+}

.travis.yml

@@ -1,2 +0,0 @@
-sudo: false
-language: node_js

CHANGELOG.md

@@ -1,4 +1,42 @@
-###### [V5.4.0]
+Newer release changelog see https://github.com/orangecoding/fredy/releases
+
+---
+
+###### [V5.5.0]
+
+- Upgrading dependencies
+- fixing provider
+- allow multiple instances of 1 provider
+- **BREAKING**: Minimum node version is now 16
+
+###### [V5.4.6]
+
+- Adding Instana node.js monitoring
+-
+
+###### [V5.4.5]
+
+- Adding Instana node.js monitoring
+
+###### [V5.4.4]
+
+- Add support for Immo Südwest Presse (immo.swp.de)
+- Telegram: Use job name instead of ID and link in title
+- Fix race condition if user ID is in session but not in user store
+- Allow visiting the original provider URL
+
+###### [V5.4.3]
+
+- re-writing readme
+- improving docker build
+- using github's actions to build docker and test automatically
+
+###### [V5.4.2]
+
+- Fixing prod build
+
+###### [V5.4.1]
+
 - Upgrading dependencies
 - Provider urls are now automagically been changed to include the correct sort order for search results
 
@@ -9,40 +47,48 @@ results, thus cannot report them. This release fixes it by adding the necessary
 ```
 
 ###### [V5.3.0]
+
 - Upgrading dependencies
 - It's now possible to send mails to multiple receiver using comma separation for MailJet & Sendgrid
 - Fixing Immowelt scraping
 
 ###### [V5.2.0]
+
 - Upgrading dependencies
 - Adding new similarity check layer (Duplicates are being removed now)
 - Adding paging for search results
 
 ###### [V5.1.0]
+
 - Upgrading dependencies
 - NodeJS 12.13 is now the minimum supported version
 - Adding general settings as new configuration page to ui
 - Adding new feature working hours
 
 ###### [V5.0.0]
+
 - Upgrading dependencies
 - NodeJS 12 is now the minimum supported version
 
 ###### [V4.0.0]
+
 Bringing back Immoscout :tada:
 
 ###### [V3.0.0]
+
 This is basically a re-write, your old config file will not be compatible anymore. Please re-created your search jobs
 on the new ui and use the values from your previous config file if needed.
+
 ```
 - We're getting rid of manual config changes, Fredy, now ships with a UI so that it's easy for you to create and edit jobs
 ```
 
 ###### [V2.0.0]
+
 ```
 - Fredy can now run multiple search job on one instance
 - Changed lot's of the structure of Fredy to make this happen
 [BREAKING CHANGES]
 - The config has been changed, the config of V1.x will not work any longer
 - Sources have been renamed to provider
-```
+```

CONTRIBUTING.md

@@ -2,8 +2,8 @@
 
 If you want to contribute, please make sure you've executed the tests.
 
-
 ### How to write new provider?
+
 - create the provider filer under `/lib/provider`
 - create a test under /test and make sure it is running successfully
 
@@ -13,11 +13,11 @@ let appliedBlackList = [];
 //normalize incoming values
 function normalize(o) {
   const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
-  
+
   return Object.assign(o, { id });
 }
 
-//apply blaclist if needed
+//apply blacklist if needed
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
@@ -27,7 +27,7 @@ function applyBlacklist(o) {
 
 const config = {
   url: null,
-    //this is the container wrapping the search listings
+  //this is the container wrapping the search listings
   crawlContainer: '#result-list-stage .item',
   crawlFields: {
     id: '@id',
@@ -49,7 +49,7 @@ exports.init = (sourceConfig, blacklist) => {
   appliedBlackList = blacklist || [];
 };
 
-//ths 
+//ths
 exports.metaInformation = {
   name: 'your provider name',
   baseUrl: 'https://www.yourprovider.de/',
@@ -57,11 +57,10 @@ exports.metaInformation = {
 };
 
 exports.config = config;
-
 ```
- 
 
 ### How to write new notification adapter?
+
 - create the provider filer under `/lib/notification/adapter`
 - create a description of the provider under `/lib/notification/adapter/*.md`. Make sure the name of the md file is equal to the notification adapter
 
@@ -72,50 +71,48 @@ const Slack = require('slack');
 const msg = Slack.chat.postMessage;
 const { markdown2Html } = require('../../services/markdown');
 
-
 //as a parameter, you will always get the serviceName, newListings and all the values, that
 //you have defined exports.config.fields. (This is being used for rendering in the frontend)
 exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-    const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
-    return newListings.map((payload) => {
-        //tho whatever needs to be done to send the data to the receiver, make sure the format is human readable
-    });
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
+  return newListings.map((payload) => {
+    //tho whatever needs to be done to send the data to the receiver, make sure the format is human readable
+  });
 };
 
 exports.config = {
-    id: __filename.slice(__dirname.length + 1, -3),
-    name: 'someUniqueName, used in the frontend',
-    //this readme is rendered in the frontend to explain how to use this
-    readme: markdown2Html('lib/notification/adapter/slack.md'),
-    description: 'Some description text rendered on the notification page',
-    fields: {
-        token: {
-            //type can be text/number/boolean
-            type: 'text',
-            label: 'Token',
-            description: 'The token needed to send notifications to slack.',
-        },
-        channel: {
-            type: 'channel',
-            label: 'Channel',
-            description: 'The channel where fredy should send notifications to.',
-        },
+  id: __filename.slice(__dirname.length + 1, -3),
+  name: 'someUniqueName, used in the frontend',
+  //this readme is rendered in the frontend to explain how to use this
+  readme: markdown2Html('lib/notification/adapter/slack.md'),
+  description: 'Some description text rendered on the notification page',
+  fields: {
+    token: {
+      //type can be text/number/boolean
+      type: 'text',
+      label: 'Token',
+      description: 'The token needed to send notifications to slack.',
     },
+    channel: {
+      type: 'channel',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
 };
-
 ```
 
 #### Running Tests
-If you've written a new provider you are an awesome person. You know it and I do. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
 
-To write tests for provider, you need to use Node 8 as the tests are using `async / await`
+If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome, right?
 
 #### Codestyle
-I'm using Eslint to maintain quote style and quality. Do not skip it...
 
-##### To do before merging:
+I'm using ESLint to maintain quote style and quality. Do not skip it...
 
-- executed tests? (`yarn run test`)
-- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
+##### To-do before merging:
+
+- Have you executed the tests? (`yarn test`)
+- Are you sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
 
 _Thanks!_ :heart:

Dockerfile

@@ -0,0 +1,36 @@
+FROM node:22-slim
+
+WORKDIR /fredy
+
+# Install Chromium and curl without extra recommended packages and clean apt cache
+# curl is needed for the health check
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends chromium curl \
+  && rm -rf /var/lib/apt/lists/*
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+
+# Copy lockfiles first to leverage cache for dependencies
+COPY package.json yarn.lock .
+
+# Set Yarn timeout, install dependencies and PM2 globally
+RUN yarn config set network-timeout 600000 \
+  && yarn --frozen-lockfile \
+  && yarn global add pm2
+
+# Copy application source and build production assets
+COPY . .
+RUN yarn build:frontend
+
+# Prepare runtime directories and symlinks for data and config
+RUN mkdir -p /db /conf \
+  && chown 1000:1000 /db /conf \
+  && chmod 777 /db /conf \
+  && ln -s /db /fredy/db \
+  && ln -s /conf /fredy/conf
+
+EXPOSE 9998
+
+# Start application using PM2 runtime
+CMD ["pm2-runtime", "index.js"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Christian Kellner
+Copyright (c) 2025 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,113 +1,225 @@
-<img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">  
+<p align="center">
 
-[![Build Status](https://travis-ci.org/orangecoding/fredy.svg?branch=master)](https://travis-ci.org/orangecoding/fredy)
+<a href="https://fredy.orange-coding.net/">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/orangecoding/fredy/blob/master/doc/logo_white.png" width="400">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">
+  <img alt="Jetbrains Open Source" src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png">
+</picture>
+</a>
+</p>
 
-Searching an apartment in Germany can be quite frustrating. Not any longer as Fredy will take over and only notifies you once new listings have been found that matches your requirements.
+![Tests](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
+[![Docker](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
+![Source](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
+![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fghcr-badge.elias.eu.org%2Fapi%2Forangecoding%2Ffredy%2Ffredy&query=%24.downloadCount&label=Docker%20Pulls)
 
-_Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings to you once they appear. The list of available services can easily be extended. For your convenience, a ui helps you to configure your search jobs.   
 
-If _Fredy_ found matching results, it will send them to you via Slack, Email, Telegram etc. (More adapter possible.) As _Fredy_ will store the listings it has found, new results will not be sent twice (and as a side-effect, _Fredy_ can show some statistics..). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent when posted on various platforms. (Happens more often than one might think)
+# Fredy 🏡 – Your Self-Hosted Real Estate Finder for Germany
 
-## Usage  
+Finding an apartment or house in Germany can be stressful and
+time-consuming.\
+**Fredy** makes it easier: it automatically scrapes **ImmoScout24,
+Immowelt, Immonet, eBay Kleinanzeigen, and WG-Gesucht** and notifies you
+instantly via **Slack, Telegram, Email, ntfy, and more** when new
+listings appear.
 
-- Make sure to use NodeJs 12 and above
-- Run the following commands
-```ssh
-yarn (or npm install)
-yarn run prod
-yarn run start
+With a modern architecture, Fredy provides a **clean Web UI**, removes
+duplicates across platforms, and stores results so you never see the
+same listing twice.
+
+------------------------------------------------------------------------
+
+## ✨ Key Features
+
+-   🏠 Scrapes **ImmoScout24, Immowelt, Immonet, eBay Kleinanzeigen,
+    WG-Gesucht**
+-   ⚡ Instant notifications: Slack, Telegram, Email (SendGrid,
+    Mailjet), ntfy
+-   🔎 Uses the **ImmoScout Mobile API** (reverse engineered)
+-   🌍 Runs anywhere: Docker, Node.js, self-hosted
+-   🖥️ Intuitive **Web UI** to manage searches
+-   🎯 Easy to use thanks to a user-friendly Web UI
+-   🔄 Deduplication across platforms
+-   ⏱️ Customizable search intervals
+
+------------------------------------------------------------------------
+
+## 🤝 Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=❤&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)
+
+I maintain Fredy and other open-source projects in my free time.\
+If you find it useful, consider supporting the project 💙
+
+Fredy is proudly backed by the **JetBrains Open Source Support Program**.   
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://www.jetbrains.com/company/brand/img/logo_jb_dos_3.svg">
+  <source media="(prefers-color-scheme: light)" srcset="https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg">
+  <img alt="Jetbrains Open Source" src="https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg">
+</picture>
+
+------------------------------------------------------------------------
+
+## 👨‍🏫 Demo
+You can try out Fredy here: [Fredy Demo](https://fredy-demo.orange-coding.net/)
+
+------------------------------------------------------------------------
+
+## 🚀 Quick Start
+
+### With Docker
+
+> [!NOTE]
+> In order to start Fredy, you must provide a config.json. As a start, use the one in this repo: https://github.com/orangecoding/fredy/blob/master/conf/config.json
+
+``` bash
+docker run -d --name fredy \
+  -v fredy_conf:/conf \
+  -v fredy_db:/db \
+  -p 9998:9998 \
+  ghcr.io/orangecoding/fredy:master
 ```
-_Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening a browser `http://localhost:9998`. The default login is `admin` both for username and password. (You should change the password asap when you plan to run Fredy on your server.)
 
-<p align="center">
-  <img alt="Job Configuration" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot__1.png" width="30%">
-&nbsp; &nbsp; &nbsp; &nbsp;
-  <img alt="Job Analytics" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot2.png" width="30%">
-&nbsp; &nbsp; &nbsp; &nbsp;  
-  <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot3.png">
-</p>
-<p align="center">
+Logs:
 
-</p>
+``` bash
+docker logs fredy -f
+```
+
+### Manual (Node.js)
+
+-   Requirement: **Node.js 22 or higher**
+-   Install dependencies and start:
+
+``` bash
+yarn
+yarn run start:backend   # in one terminal
+yarn run start:frontend  # in another terminal
+```
+
+👉 Open <http://localhost:9998>
+
+**Default Login:**
+- Username: `admin`
+- Password: `admin`
+
+------------------------------------------------------------------------
+
+## 📸 Screenshots
+
+| Job Configuration | Job Analytics | Job Overview |
+|-------------------|--------------|--------------|
+| ![Screenshot showing job configuration in Fredy](doc/screenshot1.png) | ![Screenshot showing job analytics in Fredy](doc/screenshot_2.png) | ![Screenshot showing job overview in Fredy](doc/screenshot_3.png) |
+
+------------------------------------------------------------------------
+
+## 🧩 Core Concepts
+
+Fredy is built around three simple concepts:
+
+### Provider 🌐
+
+A **provider** is a real-estate platform (e.g. ImmoScout24, Immowelt,
+Immonet, eBay Kleinanzeigen, WG-Gesucht).\
+When you create a job, you paste the search URL from the platform into
+Fredy.\
+⚠️ Always make sure the search results are sorted by **date**, so Fredy
+picks up the newest listings first.
+
+### Adapter 📡
+
+An **adapter** is the channel through which Fredy notifies you (Slack,
+Telegram, Email, ntfy, ...).\
+Each adapter has its own configuration (e.g. API keys, webhook URLs).\
+You can use multiple adapters at once --- Fredy will send new listings
+through all of them.
+
+### Job 📅
+
+A **job** combines providers and adapters.\
+Example: "Search apartments on ImmoScout24 + Immowelt and send results
+to Slack + Telegram."\
+Jobs run automatically at the interval you configure (see
+`/conf/config.json`).
+
+------------------------------------------------------------------------
 
 ## Immoscout
-I have added **experimental** support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successful validated) re-send the cookies each time.
 
-To be able to use Immoscout, you need to create an account at ScrapingAnt. Configure the ApiKey in the "General Settings" tab (visible when logged in as administrator).
-The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time it works pretty good :)
+Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
 
-If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... ScrapingAnt loves OpenSource, therefor they've decided to give all _Fredy_ users a 10% discount by using the code **FREDY10** (No I don't get any money for recommending good services...)
+## Analytics
 
+Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data.
+Before you freak out, let me explain...  
+If you agree, Fredy will send a ping once every 6 hours to my internal tracking project (Will be open sourced soon).  
+The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The information is entirely anonymous and helps me understand which adapters/providers are most frequently used.</p>
 
-## Understanding the fundamentals
-There are 3 important parts in Fredy, that you need to understand leveraging the full power of _Fredy_.
+**Thanks**🤘
 
-#### Adapter
-_Fredy_ supports multiple services. Immonet, Immowelt and Ebay are just a few. Those services are called adapter within _Fredy_. When creating a new job, you can choose 1 or many adapter.    
-An adapter holds the url that points to the search results for the service. If you go to immonet.de and search for something, the shown url in the browser is what the adapter needs to do it's magic.   
-**It is important that you order the search results by date, so that _Fredy_ always picks the latest ones first**
+## 🛠️ Development
 
-#### Provider
-_Fredy_ supports multiple provider. Slack, SendGrid, Telegram etc. A search job can have as many provider as supported by _Fredy_. Each provider needs different configuration values, which you have to provide when using it. A provider itself dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+### Development Mode
 
-#### Jobs
-A Job wraps adapter and provider. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
-
-## Creating your first job
-To create your first job, click on the button "Create New Job" on the job table. The job creation dialog should be self explanatory, however there's one important thing.
-When configuring adapter, before copying the url from your browser make sure that you have sorted the results by date to make sure _Fredy_ always picks the latest results first.
-
-## User management
-As an administrator, you can create/edit and remove user from _Fredy_. Be careful, each job is connected to the user that has created the job. If you remove the user, the jobs will also be removed.
-
-
-# Development
-
-### Running Fredy in dev mode
-To run _Fredy_ in dev mode, you need to run the backend & frontend separately. Run the backend in your favorite IDE, the frontend can be started from the terminal.
-```shell
-yarn run dev
+``` bash
+yarn run start:backend:dev
+yarn run start:frontend:dev
 ```
-You should now be able to access _Fredy_ with your browser. Go to `http://localhost:9000`
+You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
 
-### Running Tests
-To run the tests, simply run
-```shell
+### Run Tests
+
+``` bash
 yarn run test
 ```
 
-# Architecture
-![Architecture](/doc/architecture.jpg "Architecture")
+------------------------------------------------------------------------
 
-## Immoscout 
-I have added EXPERIMENTAL support for Immoscout. Immoscout is somewhat special, coz they have decided to secure their service from bots using Re-Capture. Finding a way
-around this is barely possible. For _Fredy_ to be able to bypass the check, I'm using a service called [ScrapingAnt](https://scrapingant.com/).
+## 📐 Architecture
 
-To be able to use Immoscout, you need to create an account and copy the apiKey into the config file under /conf/config.json.
-The rest should be done by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always get pass the re-capture check, but most of the time
-it works pretty good :)
+``` mermaid
+flowchart TD
+ subgraph Jobs["Jobs"]
+        A1["Job 1"]
+        A2["Job 2"]
+        A3["Job 3"]
+  end
+ subgraph Providers["Providers"]
+        C1["Provider 1"]
+        C2["Provider 2"]
+        C3["Provider 3"]
+  end
+ subgraph NotificationAdapters["Notification Adapters"]
+        F1["Adapter 1"]
+        F2["Adapter 2"]
+  end
 
-If you need more that the 1000 api calls you can do per month, I'd suggest opting for a paid account... (No I don't get any money for recommending good service)
+    A1 --> B["FredyRuntime"]
+    A2 --> B
+    A3 --> B
+    B --> C1 & C2 & C3
+    C1 --> D["Similarity Check"]
+    C2 --> D
+    C3 --> D
+    D --> E{"Duplicate?"}
+    E -- No --> F1
+    F1 --> F2
+```
 
-#### Contribution guidelines  
+------------------------------------------------------------------------
 
-See [Contribution](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTION.md)  
+## 👐 Contributing
 
-# Docker   
-Use the Dockerfile in this Repository to build an image.  
+Thanks to everyone who has contributed!
 
-Example: `docker build -t fredy/fredy /path/to/your/Dockerfile`  
+<a href="https://github.com/orangecoding/fredy/graphs/contributors"><img src="https://contrib.rocks/image?repo=orangecoding/fredy" /></a>
 
-Or use docker-compose:
+See the [Contributing
+Guide](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md).
 
-Example `docker-compose build`
+------------------------------------------------------------------------
 
-## Create & run a container  
+## ⭐ Star History
 
-Put your config.json to `/path/to/your/conf/`
-
-Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
-
-## Logs  
-
-You can browse the logs with  `docker logs fredy -f`  
+[![Star History
+Chart](https://api.star-history.com/svg?repos=orangecoding/fredy&type=Date)](https://www.star-history.com/#orangecoding/fredy&Date)

build/Dockerfile

@@ -1,33 +0,0 @@
-FROM alpine:latest AS build
-# use given repository, default below:
-ARG repo=https://github.com/orangecoding/fredy.git
-
-RUN mkdir -p /usr/src/
-#Install Software
-RUN apk add --update nodejs npm git
-
-# Output used repository
-RUN echo "Cloning from $repo"
-
-RUN cd /usr/src && git clone $repo
-
-RUN ln -s /usr/src/fredy/conf/ /conf
-
-# create db folder
-RUN mkdir /usr/src/fredy/db/
-
-RUN ln -s /usr/src/fredy/db/ /db
-
-RUN npm i -g yarn
-
-RUN cd /usr/src/fredy/ && yarn
-
-WORKDIR  /usr/src/fredy
-
-RUN yarn run prod
-
-EXPOSE 9998
-
-VOLUME [ "/conf", "/db" ]
-# --no-daemon is required for keeping Container alive
-CMD node index.js --no-daemon

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}
+{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}

docker-compose.yml

@@ -1,17 +1,22 @@
-version: '3.3'
 services:
   fredy:
     container_name: fredy
     # build from empty build folder to reduce size of image
     build:
-      context: ./build
+      context: .
       dockerfile: Dockerfile
-      args:
-        repo: https://github.com/orangecoding/fredy.git
     image: fredy/fredy
     # map existing config and database
     volumes:
       - ./conf:/conf
       - ./db:/db
     ports:
-        - 9998:9998
+      - "9998:9998"
+    restart: unless-stopped
+    healthcheck:
+      # The container will immediately stop when health check fails after retries
+      test: ["CMD-SHELL", "curl --fail --silent --show-error --max-time 5 http://localhost:9998/ || exit 1"]
+      interval: 120s
+      timeout: 10s
+      retries: 1
+      start_period: 10s

eslint.config.js

@@ -0,0 +1,96 @@
+// eslint.config.js
+import js from '@eslint/js';
+import prettier from 'eslint-config-prettier';
+import globals from 'globals';
+import react from 'eslint-plugin-react';
+import babelParser from '@babel/eslint-parser';
+
+export default [
+  {
+    files: ['**/*.{js,jsx,ts,tsx}'],
+    ignores: ['**/node_modules/**', '**/dist/**', '**/build/**', '**/public/**', 'db/**', 'conf/**'],
+  },
+  js.configs.recommended,
+  prettier,
+  {
+    languageOptions: {
+      parser: babelParser,
+      sourceType: 'module',
+      ecmaVersion: 2021,
+      globals: {
+        ...globals.browser,
+        ...globals.node,
+        ...globals.mocha,
+        Promise: 'readonly',
+        fetch: 'readonly',
+        describe: 'readonly',
+        after: 'readonly',
+        it: 'readonly',
+      },
+      parserOptions: { requireConfigFile: false },
+    },
+    plugins: { react },
+    rules: {
+      eqeqeq: [2, 'allow-null'],
+      strict: 0,
+      'no-redeclare': [2, { builtinGlobals: false }],
+      'class-methods-use-this': 'off',
+      indent: ['off', 2],
+      'linebreak-style': ['error', 'unix'],
+      quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }],
+      semi: ['error', 'always'],
+      'no-console': ['error', { allow: ['warn', 'error'] }],
+      'jsx-quotes': ['error', 'prefer-double'],
+      'react/display-name': 'off',
+      'react/forbid-prop-types': 'off',
+      'react/jsx-closing-bracket-location': 'off',
+      'react/jsx-curly-spacing': 'off',
+      'react/jsx-handler-names': ['off', { eventHandlerPrefix: 'handle', eventHandlerPropPrefix: 'on' }],
+      'react/jsx-indent-props': 'off',
+      'react/jsx-key': 'off',
+      'react/jsx-max-props-per-line': 'off',
+      'react/jsx-no-bind': ['error', { ignoreRefs: true, allowArrowFunctions: true, allowBind: false }],
+      'react/jsx-no-duplicate-props': ['error', { ignoreCase: true }],
+      'react/jsx-no-literals': 'off',
+      'react/jsx-no-undef': 'error',
+      'react/jsx-pascal-case': ['error', { allowAllCaps: true, ignore: [] }],
+      'react/sort-prop-types': ['off', { ignoreCase: true, callbacksLast: false, requiredFirst: false }],
+      'react/jsx-sort-prop-types': 'off',
+      'react/jsx-sort-props': 'off',
+      'react/jsx-uses-react': 'error',
+      'react/jsx-uses-vars': 'error',
+      'react/no-danger': 'warn',
+      'react/no-deprecated': 'error',
+      'react/no-did-mount-set-state': 'error',
+      'react/no-did-update-set-state': 'warn',
+      'react/no-direct-mutation-state': 'off',
+      'react/no-is-mounted': 'error',
+      'react/no-set-state': 'off',
+      'react/no-string-refs': 'warn',
+      'react/no-unknown-property': 'error',
+      'react/prop-types': ['error', { ignore: [], customValidators: [], skipUndeclared: true }],
+      'react/react-in-jsx-scope': 'error',
+      'react/require-extension': 'off',
+      'react/require-render-return': 'error',
+      'react/self-closing-comp': 'warn',
+      'react/sort-comp': 'off',
+      'react/jsx-wrap-multilines': ['warn', { declaration: true, assignment: true, return: true }],
+      'react/wrap-multilines': 'off',
+      'react/jsx-first-prop-new-line': 'off',
+      'react/jsx-equals-spacing': ['warn', 'never'],
+      'react/jsx-no-target-blank': 'error',
+      'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
+      'react/jsx-no-comment-textnodes': 'error',
+      'react/no-comment-textnodes': 'off',
+      'react/no-render-return-value': 'error',
+      'react/require-optimization': ['off', { allowDecorators: [] }],
+      'react/no-find-dom-node': 'warn',
+      'react/forbid-component-props': ['off', { forbid: [] }],
+      'react/no-danger-with-children': 'error',
+      'react/no-unused-prop-types': ['warn', { customValidators: [], skipShapeProps: true }],
+      'react/style-prop-object': 'error',
+      'react/no-children-prop': 'warn',
+    },
+    settings: { react: { version: 'detect' } },
+  },
+];

index.html

@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta
+      charset="UTF-8"
+      name="viewport"
+      content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"
+    />
+    <meta name="google" content="notranslate" />
+
+    <title>Fredy</title>
+  </head>
+  <body theme-mode="dark">
+    <div id="fredy" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0"></div>
+  </body>
+  <script type="module" src="/ui/src/Index.jsx"></script>
+</html>

index.js

@@ -1,72 +1,88 @@
-const fs = require('fs');
+import fs from 'fs';
+import path from 'path';
+import { config, getProviders, refreshConfig } from './lib/utils.js';
+import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
+import * as jobStorage from './lib/services/storage/jobStorage.js';
+import FredyRuntime from './lib/FredyRuntime.js';
+import { duringWorkingHoursOrNotSet } from './lib/utils.js';
+import { runMigrations } from './lib/services/storage/migrations/migrate.js';
+import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
+import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
+import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
+import logger from './lib/services/logger.js';
+import { bus } from './lib/services/events/event-bus.js';
+import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
 
-//if db folder does not exist, ensure to create it before loading anything else
-if (!fs.existsSync('./db')) {
-  fs.mkdirSync('./db');
+// Load configuration before any other startup steps
+await refreshConfig();
+
+// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
+const rawDir = config.sqlitepath || '/db';
+const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
+const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
+if (!fs.existsSync(absDir)) {
+  fs.mkdirSync(absDir, { recursive: true });
 }
 
-const path = './lib/provider';
-const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
-const config = require('./conf/config.json');
+// Run DB migrations once at startup and block until finished
+await runMigrations();
 
-const similarityCache = require('./lib/services/similarity-check/similarityCache');
-const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
-const jobStorage = require('./lib/services/storage/jobStorage');
-const FredyRuntime = require('./lib/FredyRuntime');
-
-const { duringWorkingHoursOrNotSet } = require('./lib/utils');
-
-//starting the api service
-require('./lib/api/api');
+// Load provider modules once at startup
+const providers = await getProviders();
 
 //assuming interval is always in minutes
 const INTERVAL = config.interval * 60 * 1000;
 
-/* eslint-disable no-console */
-console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
-/* eslint-enable no-console */
-setInterval(
-  (function exec() {
-    const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+// Initialize API only after migrations completed
+await import('./lib/api/api.js');
 
+if (config.demoMode) {
+  logger.info('Running in demo mode');
+  cleanupDemoAtMidnight();
+}
+
+logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+
+ensureAdminUserExists();
+ensureDemoUserExists();
+await initTrackerCron();
+//do not wait for this to finish, let it run in the background
+initActiveCheckerCron();
+
+bus.on('jobs:runAll', () => {
+  logger.debug('Running Fredy Job manually');
+  execute();
+});
+
+const execute = () => {
+  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+  if (!config.demoMode) {
     if (isDuringWorkingHoursOrNotSet) {
       config.lastRun = Date.now();
       jobStorage
         .getJobs()
         .filter((job) => job.enabled)
         .forEach((job) => {
-          const providerIds = job.provider.map((provider) => provider.id);
-
-          provider
-            .filter((provider) => provider.endsWith('.js'))
-            .map((pro) => require(`${path}/${pro}`))
-            .filter((provider) => providerIds.indexOf(provider.metaInformation.id) !== -1)
-            .forEach(async (pro) => {
-              const providerId = pro.metaInformation.id;
-              if (providerId == null || providerId.length === 0) {
-                throw new Error('Provider id must not be empty. => ' + pro);
-              }
-              const providerConfig = job.provider.find((jobProvider) => jobProvider.id === providerId);
-              if (providerConfig == null) {
-                throw new Error(`Provider Config for provider with id ${providerId} not found.`);
-              }
-              pro.init(providerConfig, job.blacklist);
+          job.provider
+            .filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
+            .forEach(async (prov) => {
+              const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
+              matchedProvider.init(prov, job.blacklist);
               await new FredyRuntime(
-                pro.config,
+                matchedProvider.config,
                 job.notificationAdapter,
-                providerId,
+                prov.id,
                 job.id,
-                similarityCache
+                similarityCache,
               ).execute();
-              setLastJobExecution(job.id);
             });
         });
     } else {
-      /* eslint-disable no-console */
-      console.debug('Working hours set. Skipping as outside of working hours.');
-      /* eslint-enable no-console */
+      logger.debug('Working hours set. Skipping as outside of working hours.');
     }
-    return exec;
-  })(),
-  INTERVAL
-);
+  }
+};
+
+setInterval(execute, INTERVAL);
+//start once at startup
+execute();

lib/FredyRuntime.js

@@ -1,10 +1,9 @@
-const { NoNewListingsWarning } = require('./errors');
-const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
-
-const notify = require('./notification/notify');
-const xray = require('./services/scraper');
-const scrapingAnt = require('./services/scrapingAnt');
-const urlModifier = require('./services/queryStringMutator');
+import { NoNewListingsWarning } from './errors.js';
+import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
+import * as notify from './notification/notify.js';
+import Extractor from './services/extractor/extractor.js';
+import urlModifier from './services/queryStringMutator.js';
+import logger from './services/logger.js';
 
 class FredyRuntime {
   /**
@@ -28,7 +27,7 @@ class FredyRuntime {
       //modify the url to make sure search order is correctly set
       Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
         //scraping the site and try finding new listings
-        .then(this._getListings.bind(this))
+        .then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
         //bring them in a proper form (dictated by the provider)
         .then(this._normalize.bind(this))
         //filter listings with stuff tagged by the blacklist of the provider
@@ -47,45 +46,22 @@ class FredyRuntime {
   }
 
   _getListings(url) {
+    const extractor = new Extractor();
     return new Promise((resolve, reject) => {
-      const id = this._providerId;
-      if (scrapingAnt.isImmoscout(id) && !scrapingAnt.isScrapingAntApiKeySet()) {
-        const error = 'Immoscout can only be used with if you have set an apikey for scrapingAnt.';
-        /* eslint-disable no-console */
-        console.log(error);
-        /* eslint-enable no-console */
-        reject(error);
-        return;
-      }
-      const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
-      try {
-        if (this._providerConfig.paginate != null) {
-          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
-            //the first 2 pages should be enough here
-            //TODO: Think about automagically sort by date
-            .limit(2)
-            .paginate(this._providerConfig.paginate)
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        } else {
-          xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        }
-      } catch (error) {
-        reject(error);
-        console.error(error);
-      }
+      extractor
+        .execute(url, this._providerConfig.waitForSelector)
+        .then(() => {
+          const listings = extractor.parseResponseText(
+            this._providerConfig.crawlContainer,
+            this._providerConfig.crawlFields,
+            url,
+          );
+          resolve(listings == null ? [] : listings);
+        })
+        .catch((err) => {
+          reject(err);
+          logger.error(err);
+        });
     });
   }
 
@@ -94,16 +70,19 @@ class FredyRuntime {
   }
 
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
 
   _findNew(listings) {
-    const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
+    const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
 
+    const newListings = listings.filter((o) => !hashes.includes(o.id));
     if (newListings.length === 0) {
       throw new NoNewListingsWarning();
     }
-
     return newListings;
   }
 
@@ -116,31 +95,25 @@ class FredyRuntime {
   }
 
   _save(newListings) {
-    const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
-    newListings.forEach((listing) => {
-      currentListings[listing.id] = Date.now();
-    });
-    setKnownListings(this._jobKey, this._providerId, currentListings);
+    storeListings(this._jobKey, this._providerId, newListings);
     return newListings;
   }
 
   _filterBySimilarListings(listings) {
     const filteredList = listings.filter((listing) => {
-      const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
+      const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
       if (similar) {
-        /* eslint-disable no-console */
-        console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
-        /* eslint-enable no-console */
+        logger.debug(`Filtering similar entry for title: ${listing.title} and address ${listing.address}`);
       }
       return !similar;
     });
-    filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
+    filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
     return filteredList;
   }
 
   _handleError(err) {
-    if (err.name !== 'NoNewListingsWarning') console.error(err);
+    if (err.name !== 'NoNewListingsWarning') logger.error(err);
   }
 }
 
-module.exports = FredyRuntime;
+export default FredyRuntime;

lib/api/api.js

@@ -1,44 +1,47 @@
-const { notificationAdapterRouter } = require('./routes/notificationAdapterRouter');
-const { authInterceptor, cookieSession, adminInterceptor } = require('./security');
-const { generalSettingsRouter } = require('./routes/generalSettingsRoute');
-const { analyticsRouter } = require('./routes/analyticsRouter');
-const { providerRouter } = require('./routes/providerRouter');
-const { loginRouter } = require('./routes/loginRoute');
-const config = require('../../conf/config.json');
-const { userRouter } = require('./routes/userRoute');
-const { jobRouter } = require('./routes/jobRouter');
-const bodyParser = require('body-parser');
-const service = require('restana')();
-const files = require('serve-static');
-const path = require('path');
-
-const staticService = files(path.join(__dirname, '../../ui/public'));
-
+import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
+import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
+import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
+import { analyticsRouter } from './routes/analyticsRouter.js';
+import { providerRouter } from './routes/providerRouter.js';
+import { versionRouter } from './routes/versionRouter.js';
+import { loginRouter } from './routes/loginRoute.js';
+import { userRouter } from './routes/userRoute.js';
+import { jobRouter } from './routes/jobRouter.js';
+import { config } from '../utils.js';
+import bodyParser from 'body-parser';
+import restana from 'restana';
+import files from 'serve-static';
+import path from 'path';
+import { getDirName } from '../utils.js';
+import { demoRouter } from './routes/demoRouter.js';
+import logger from '../services/logger.js';
+import { listingsRouter } from './routes/listingsRouter.js';
+const service = restana();
+const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
 
 service.use(bodyParser.json());
-
 service.use(cookieSession());
-
 service.use(staticService);
-
 service.use('/api/admin', authInterceptor());
 service.use('/api/jobs', authInterceptor());
+service.use('/api/version', authInterceptor());
+service.use('/api/listings', authInterceptor());
 
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
-
 service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
 service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/jobs/insights', analyticsRouter);
 service.use('/api/admin/users', userRouter);
+service.use('/api/version', versionRouter);
 service.use('/api/jobs', jobRouter);
-
 service.use('/api/login', loginRouter);
+service.use('/api/listings', listingsRouter);
+//this route is unsecured intentionally as it is being queried from the login page
+service.use('/api/demo', demoRouter);
 
-/* eslint-disable no-console */
 service.start(PORT).then(() => {
-  console.info(`Started API service on port ${PORT}`);
+  logger.debug(`Started API service on port ${PORT}`);
 });
-/* eslint-enable no-console */

lib/api/routes/analyticsRouter.js

@@ -1,12 +1,10 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as listingStorage from '../../services/storage/listingsStorage.js';
+const service = restana();
 const analyticsRouter = service.newRouter();
-const listingStorage = require('../../services/storage/listingsStorage');
-
 analyticsRouter.get('/:jobId', async (req, res) => {
   const { jobId } = req.params;
-
   res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
   res.send();
 });
-
-exports.analyticsRouter = analyticsRouter;
+export { analyticsRouter };

lib/api/routes/demoRouter.js

@@ -0,0 +1,11 @@
+import restana from 'restana';
+import { config } from '../../utils.js';
+const service = restana();
+const demoRouter = service.newRouter();
+
+demoRouter.get('/', async (req, res) => {
+  res.body = Object.assign({}, { demoMode: config.demoMode });
+  res.send();
+});
+
+export { demoRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,24 +1,30 @@
-const service = require('restana')();
+import restana from 'restana';
+import { config, getDirName, readConfigFromStorage, refreshConfig } from '../../utils.js';
+import fs from 'fs';
+import { ensureDemoUserExists } from '../../services/storage/userStorage.js';
+import logger from '../../services/logger.js';
+const service = restana();
 const generalSettingsRouter = service.newRouter();
-const config = require('../../../conf/config.json');
-const fs = require('fs');
-
 generalSettingsRouter.get('/', async (req, res) => {
   res.body = Object.assign({}, config);
   res.send();
 });
-
 generalSettingsRouter.post('/', async (req, res) => {
   const settings = req.body;
-
   try {
-    fs.writeFileSync(`${__dirname}/../../../conf/config.json`, JSON.stringify(settings));
+    if (config.demoMode) {
+      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+      return;
+    }
+    const currentConfig = await readConfigFromStorage();
+    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ ...currentConfig, ...settings }));
+    await refreshConfig();
+    ensureDemoUserExists();
   } catch (err) {
-    console.error(err);
+    logger.error(err);
     res.send(new Error('Error while trying to write settings.'));
     return;
   }
   res.send();
 });
-
-exports.generalSettingsRouter = generalSettingsRouter;
+export { generalSettingsRouter };

lib/api/routes/jobRouter.js

@@ -1,11 +1,13 @@
-const service = require('restana')();
-const jobRouter = service.newRouter();
-const jobStorage = require('../../services/storage/jobStorage');
-const userStorage = require('../../services/storage/userStorage');
-const immoscoutProvider = require('../../provider/immoscout');
-const config = require('../../../conf/config.json');
+import restana from 'restana';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import * as userStorage from '../../services/storage/userStorage.js';
+import { config } from '../../utils.js';
+import { isAdmin } from '../security.js';
+import logger from '../../services/logger.js';
+import { bus } from '../../services/events/event-bus.js';
 
-const { isAdmin } = require('../security');
+const service = restana();
+const jobRouter = service.newRouter();
 
 function doesJobBelongsToUser(job, req) {
   const userId = req.session.currentUser;
@@ -16,39 +18,30 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-
-  return user.isAdmin || job.userId === job.userId;
+  return user.isAdmin || job.userId === user.id;
 }
 
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);
-
   //show only the jobs which belongs to the user (or all of the user is an admin)
   res.body = jobStorage.getJobs().filter((job) => isUserAdmin || job.userId === req.session.currentUser);
-
   res.send();
 });
-
 jobRouter.get('/processingTimes', async (req, res) => {
   res.body = {
     interval: config.interval,
     lastRun: config.lastRun || null,
   };
+  res.send();
+});
 
+jobRouter.post('/startAll', async (req, res) => {
+  bus.emit('jobs:runAll');
   res.send();
 });
 
 jobRouter.post('/', async (req, res) => {
   const { provider, notificationAdapter, name, blacklist = [], jobId, enabled } = req.body;
-  if (
-    provider.find((p) => p.id === immoscoutProvider.metaInformation.id) != null &&
-    (config.scrapingAnt.apiKey == null || config.scrapingAnt.apiKey.length === 0)
-  ) {
-    res.send(
-      new Error('To use Immoscout as provider, you need to configure ScrapingAnt first. Please check the readme.')
-    );
-    return;
-  }
   try {
     jobStorage.upsertJob({
       userId: req.session.currentUser,
@@ -61,11 +54,10 @@ jobRouter.post('/', async (req, res) => {
     });
   } catch (error) {
     res.send(new Error(error));
-    console.error(error);
+    logger.error(error);
   }
   res.send();
 });
-
 jobRouter.delete('', async (req, res) => {
   const { jobId } = req.body;
   try {
@@ -77,11 +69,10 @@ jobRouter.delete('', async (req, res) => {
     }
   } catch (error) {
     res.send(new Error(error));
-    console.error(error);
+    logger.error(error);
   }
   res.send();
 });
-
 jobRouter.put('/:jobId/status', async (req, res) => {
   const { status } = req.body;
   const { jobId } = req.params;
@@ -97,9 +88,8 @@ jobRouter.put('/:jobId/status', async (req, res) => {
     }
   } catch (error) {
     res.send(new Error(error));
-    console.error(error);
+    logger.error(error);
   }
   res.send();
 });
-
-exports.jobRouter = jobRouter;
+export { jobRouter };

lib/api/routes/listingsRouter.js

@@ -0,0 +1,23 @@
+import restana from 'restana';
+import * as listingStorage from '../../services/storage/listingsStorage.js';
+import { isAdmin as isAdminFn } from '../security.js';
+const service = restana();
+
+const listingsRouter = service.newRouter();
+
+listingsRouter.get('/table', async (req, res) => {
+  const { page, pageSize = 50, filter, sortfield = null, sortdir = 'asc' } = req.query || {};
+
+  const result = listingStorage.queryListings({
+    page: page ? parseInt(page, 10) : 1,
+    pageSize: pageSize ? parseInt(pageSize, 10) : 50,
+    filter: filter || undefined,
+    sortField: sortfield || null,
+    sortDir: sortdir === 'desc' ? 'desc' : 'asc',
+    userId: req.session.currentUser,
+    isAdmin: isAdminFn(req),
+  });
+  res.body = result;
+  res.send();
+});
+export { listingsRouter };

lib/api/routes/loginRoute.js

@@ -1,47 +1,47 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as hasher from '../../services/security/hash.js';
+import { config } from '../../utils.js';
+import { trackDemoAccessed } from '../../services/tracking/Tracker.js';
+import logger from '../../services/logger.js';
+const service = restana();
 const loginRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const hasher = require('../../services/security/hash');
-
 loginRouter.get('/user', async (req, res) => {
   const currentUserId = req.session.currentUser;
-  const isAdmin = currentUserId == null ? false : userStorage.getUser(currentUserId).isAdmin;
-  if (currentUserId == null) {
+  const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
+  if (currentUser == null) {
     res.body = {};
   } else {
     res.body = {
-      userId: currentUserId,
-      isAdmin,
+      userId: currentUser.id,
+      isAdmin: currentUser.isAdmin,
     };
   }
   res.send();
 });
-
 loginRouter.post('/', async (req, res) => {
   const { username, password } = req.body;
-
   const user = userStorage.getUsers(true).find((user) => user.username === username);
-
   if (user == null) {
     res.send(401);
     return;
   }
-
   if (user.password === hasher.hash(password)) {
+    if (config.demoMode) {
+      await trackDemoAccessed();
+    }
+
     req.session.currentUser = user.id;
     userStorage.setLastLoginToNow({ userId: user.id });
     res.send(200);
     return;
   } else {
-    console.error(`User ${username} tried to login, but password was wrong.`);
+    logger.error(`User ${username} tried to login, but password was wrong.`);
   }
-
   res.send(401);
 });
-
 loginRouter.post('/logout', async (req, res) => {
   req.session = null;
   res.send(200);
 });
-
-exports.loginRouter = loginRouter;
+export { loginRouter };

lib/api/routes/notificationAdapterRouter.js

@@ -1,13 +1,13 @@
-const fs = require('fs');
-const service = require('restana')();
+import fs from 'fs';
+import restana from 'restana';
+const service = restana();
 const notificationAdapterRouter = service.newRouter();
-
 const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
-
-const notificationAdapter = notificationAdapterList.map((pro) => {
-  return require(`../../notification/adapter/${pro}`);
-});
-
+const notificationAdapter = await Promise.all(
+  notificationAdapterList.map(async (pro) => {
+    return await import(`../../notification/adapter/${pro}`);
+  }),
+);
 notificationAdapterRouter.post('/try', async (req, res) => {
   const { id, fields } = req.body;
   const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
@@ -24,7 +24,6 @@ notificationAdapterRouter.post('/try', async (req, res) => {
     enabled: true,
     id,
   });
-
   try {
     await adapter.send({
       serviceName: 'TestCall',
@@ -40,16 +39,13 @@ notificationAdapterRouter.post('/try', async (req, res) => {
       notificationConfig,
       jobKey: 'TestJob',
     });
-
     res.send();
   } catch (Exception) {
     res.send(new Error(Exception));
   }
 });
-
 notificationAdapterRouter.get('/', async (req, res) => {
   res.body = notificationAdapter.map((adapter) => adapter.config);
   res.send();
 });
-
-exports.notificationAdapterRouter = notificationAdapterRouter;
+export { notificationAdapterRouter };

lib/api/routes/providerRouter.js

@@ -1,16 +1,15 @@
-const fs = require('fs');
-const service = require('restana')();
+import fs from 'fs';
+import restana from 'restana';
+const service = restana();
 const providerRouter = service.newRouter();
-
 const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
-
-const provider = providerList.map((pro) => {
-  return require(`../../provider/${pro}`).metaInformation;
-});
-
+const provider = await Promise.all(
+  providerList.map(async (pro) => {
+    return await import(`../../provider/${pro}`);
+  }),
+);
 providerRouter.get('/', async (req, res) => {
-  res.body = provider;
+  res.body = provider.map((p) => p.metaInformation);
   res.send();
 });
-
-exports.providerRouter = providerRouter;
+export { providerRouter };

lib/api/routes/userRoute.js

@@ -1,33 +1,33 @@
-const service = require('restana')();
+import restana from 'restana';
+import * as userStorage from '../../services/storage/userStorage.js';
+import * as jobStorage from '../../services/storage/jobStorage.js';
+import { config } from '../../utils.js';
+const service = restana();
 const userRouter = service.newRouter();
-const userStorage = require('../../services/storage/userStorage');
-const jobStorage = require('../../services/storage/jobStorage');
-
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
   return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
 }
-
 function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
   return req.session.currentUser === userIdToBeRemoved;
 }
-
 const nullOrEmpty = (str) => str == null || str.length === 0;
-
 userRouter.get('/', async (req, res) => {
   res.body = userStorage.getUsers(false);
   res.send();
 });
-
 userRouter.get('/:userId', async (req, res) => {
   const { userId } = req.params;
   res.body = userStorage.getUser(userId);
   res.send();
 });
-
 userRouter.delete('/', async (req, res) => {
+  if (config.demoMode) {
+    res.send(new Error('In demo mode, it is not allowed to remove user.'));
+    return;
+  }
+
   const { userId } = req.body;
   const allUser = userStorage.getUsers(false);
-
   if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
     return;
@@ -36,15 +36,17 @@ userRouter.delete('/', async (req, res) => {
     res.send(new Error('You are trying to remove yourself. This is prohibited.'));
     return;
   }
-
   //TODO: Remove also analytics
   jobStorage.removeJobsByUserId(userId);
   userStorage.removeUser(userId);
-
   res.send();
 });
-
 userRouter.post('/', async (req, res) => {
+  if (config.demoMode) {
+    res.send(new Error('In demo mode, it is not allowed to change or add user.'));
+    return;
+  }
+
   const { username, password, password2, isAdmin, userId } = req.body;
   if (password !== password2) {
     res.send(new Error('Passwords does not match'));
@@ -55,22 +57,18 @@ userRouter.post('/', async (req, res) => {
     return;
   }
   const allUser = userStorage.getUsers(false);
-
   if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
     res.send(
-      new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system')
+      new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system'),
     );
     return;
   }
-
   userStorage.upsertUser({
     userId,
     username,
     password,
     isAdmin,
   });
-
   res.send();
 });
-
-exports.userRouter = userRouter;
+export { userRouter };

lib/api/routes/versionRouter.js

@@ -0,0 +1,31 @@
+import restana from 'restana';
+import fetch from 'node-fetch';
+import { getPackageVersion } from '../../utils.js';
+import semver from 'semver';
+
+const service = restana();
+const versionRouter = service.newRouter();
+
+versionRouter.get('/', async (req, res) => {
+  const versionPayload = await getCurrentVersionFromGithub();
+  res.body = versionPayload == null ? { newVersion: false } : versionPayload;
+  res.send();
+});
+
+async function getCurrentVersionFromGithub() {
+  const raw = await fetch('https://api.github.com/repos/orangecoding/fredy/releases/latest');
+  const data = await raw.json();
+  const localFredyVersion = await getPackageVersion();
+  if (data.tag_name == null || semver.gte(localFredyVersion, data.tag_name)) {
+    return null;
+  }
+  return {
+    newVersion: true,
+    version: data.tag_name,
+    url: data.html_url,
+    body: data.body,
+    localFredyVersion,
+  };
+}
+
+export { versionRouter };

lib/api/security.js

@@ -1,15 +1,12 @@
-const userStorage = require('../services/storage/userStorage');
-const cookieSession = require('cookie-session');
-const { nanoid } = require('nanoid');
-
+import * as userStorage from '../services/storage/userStorage.js';
+import cookieSession from 'cookie-session';
+import { nanoid } from 'nanoid';
 const unauthorized = (res) => {
   return res.send(401);
 };
-
 const isUnauthorized = (req) => {
   return req.session.currentUser == null;
 };
-
 const isAdmin = (req) => {
   if (!isUnauthorized(req)) {
     const user = userStorage.getUser(req.session.currentUser);
@@ -17,7 +14,6 @@ const isAdmin = (req) => {
   }
   return false;
 };
-
 const authInterceptor = () => {
   return (req, res, next) => {
     if (isUnauthorized(req)) {
@@ -27,7 +23,6 @@ const authInterceptor = () => {
     }
   };
 };
-
 const adminInterceptor = () => {
   return (req, res, next) => {
     if (!isAdmin(req)) {
@@ -37,17 +32,16 @@ const adminInterceptor = () => {
     }
   };
 };
-
-exports.cookieSession = (userId) => {
+const cookieSession$0 = (userId) => {
   return cookieSession({
     name: 'fredy-admin-session',
     keys: ['fredy', 'super', 'fancy', 'key', nanoid()],
     userId,
-    maxAge: 8 * 60 * 60 * 1000, // 8 hours
+    maxAge: 2 * 60 * 60 * 1000, // 2 hours
   });
 };
-
-exports.adminInterceptor = adminInterceptor;
-exports.authInterceptor = authInterceptor;
-exports.isUnauthorized = isUnauthorized;
-exports.isAdmin = isAdmin;
+export { cookieSession$0 as cookieSession };
+export { adminInterceptor };
+export { authInterceptor };
+export { isUnauthorized };
+export { isAdmin };

lib/defaultConfig.js

@@ -0,0 +1,9 @@
+export const DEFAULT_CONFIG = {
+  interval: '60',
+  port: 9998,
+  workingHours: { from: '', to: '' },
+  demoMode: false,
+  analyticsEnabled: null,
+  // Default path for sqlite storage directory. Interpreted relative to project root.
+  sqlitepath: '/db',
+};

lib/errors.js

@@ -9,7 +9,8 @@ class ExtendableError extends Error {
     }
   }
 }
-
 class NoNewListingsWarning extends ExtendableError {}
-
-module.exports = { NoNewListingsWarning };
+export { NoNewListingsWarning };
+export default {
+  NoNewListingsWarning,
+};

lib/notification/adapter/apprise.js

@@ -0,0 +1,35 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  const promises = newListings.map((newListing) => {
+    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
+    return fetch(server, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        body: message,
+        title: title,
+      }),
+    });
+  });
+  return Promise.all(promises);
+};
+export const config = {
+  id: 'apprise',
+  name: 'Apprise',
+  readme: markdown2Html('lib/notification/adapter/apprise.md'),
+  description: 'Fredy will send new listings to your Apprise instance.',
+  fields: {
+    server: {
+      type: 'text',
+      label: 'Server',
+      description: 'The server URL to send the notification to.',
+    },
+  },
+};

lib/notification/adapter/apprise.md

@@ -0,0 +1,3 @@
+### Apprise Adapter
+
+Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.

lib/notification/adapter/console.js

@@ -1,19 +1,12 @@
-const { markdown2Html } = require('../../services/markdown');
+import { markdown2Html } from '../../services/markdown.js';
 
-/**
- * simply prints out the found data to the console
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param jobKey name of the current job that is being executed
- */
-exports.send = ({ serviceName, newListings, jobKey }) => {
+export const send = ({ serviceName, newListings, jobKey }) => {
   /* eslint-disable no-console */
   return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
   /* eslint-enable no-console */
 };
-
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'console',
   name: 'Console',
   description: 'This adapter sends new listings to the console. It is mostly useful for debugging.',
   config: {},

lib/notification/adapter/console.md

@@ -1,4 +1,4 @@
 ### Console Adapter
 
 The console adapter prints everything found by Fredy into the console (not sending any notifications to you). This can be useful when you want to check if your search
-criteria meet the expectations.
+criteria meet the expectations.

lib/notification/adapter/mailJet.js

@@ -1,57 +1,115 @@
-const mailjet = require('node-mailjet');
+import mailjet from 'node-mailjet';
+import path from 'path';
+import fs from 'fs';
+import Handlebars from 'handlebars';
+import fetch from 'node-fetch';
+import { markdown2Html } from '../../services/markdown.js';
+import { getDirName, normalizeImageUrl } from '../../utils.js';
+import logger from '../../services/logger.js';
 
-const path = require('path');
-const fs = require('fs');
-const template = fs.readFileSync(path.resolve(__dirname, '../', 'emailTemplate/template.hbs'), 'utf8');
-
-const Handlebars = require('handlebars');
+const __dirname = getDirName();
+const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
 const emailTemplate = Handlebars.compile(template);
-const { markdown2Html } = require('../../services/markdown');
 
-/**
- * sends a new listing using MailJet
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+const guessMime = (url) => {
+  const lower = url.split('?')[0].toLowerCase();
+  if (lower.endsWith('.png')) return 'image/png';
+  if (lower.endsWith('.gif')) return 'image/gif';
+  return 'image/jpeg';
+};
+
+const toBase64 = async (url) => {
+  try {
+    const res = await fetch(url);
+    if (!res.ok) throw new Error(`Fetch failed with status ${res.status} for URL: ${url}`);
+    const ab = await res.arrayBuffer();
+    return Buffer.from(ab).toString('base64');
+  } catch (error) {
+    logger.error(`Error fetching image from ${url}:`, error.message);
+    throw error;
+  }
+};
+
+const mapListingsWithCid = async (serviceName, jobKey, listings) => {
+  const out = [];
+  const attachments = [];
+
+  for (let i = 0; i < listings.length; i++) {
+    const l = listings[i] || {};
+    const imgUrl = normalizeImageUrl(l.image);
+
+    const item = {
+      title: l.title || '',
+      link: l.link || '',
+      address: l.address || '',
+      size: l.size || '',
+      price: l.price || '',
+      serviceName,
+      jobKey,
+      hasImage: false,
+      imageCid: '',
+    };
+
+    if (imgUrl) {
+      try {
+        const base64 = await toBase64(imgUrl);
+        const cid = `listing-${i}`;
+        attachments.push({
+          ContentType: guessMime(imgUrl),
+          Filename: `listing-${i}.${imgUrl.split('.').pop().split('?')[0] || 'jpg'}`,
+          Base64Content: base64,
+          ContentID: cid,
+        });
+        item.hasImage = true;
+        item.imageCid = cid;
+      } catch (error) {
+        logger.warn(`Skipping image for listing ${i} due to error: ${error.message}`);
+      }
+    }
+
+    out.push(item);
+  }
+
+  return { listings: out, attachments };
+};
+
+export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailJet'
+    (adapter) => adapter.id === config.id,
   ).fields;
 
   const to = receiver
     .trim()
     .split(',')
-    .map((r) => ({
-      Email: r.trim(),
-    }));
+    .map((r) => ({ Email: r.trim() }))
+    .filter((r) => r.Email.length > 0);
+
+  const { listings, attachments } = await mapListingsWithCid(serviceName, jobKey, newListings);
+
+  const html = emailTemplate({
+    serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
+    numberOfListings: listings.length,
+    listings,
+  });
 
   return mailjet
-    .connect(apiPublicKey, apiPrivateKey)
+    .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [
         {
-          From: {
-            Email: from,
-            Name: 'Fredy',
-          },
+          From: { Email: from, Name: 'Fredy' },
           To: to,
-          Subject: `Fredy found ${newListings.length} new listings for ${serviceName}`,
-          HTMLPart: emailTemplate({
-            serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
-            numberOfListings: newListings.length,
-            listings: newListings,
-          }),
+          Subject: `Fredy found ${listings.length} new listing(s) for ${serviceName}`,
+          HTMLPart: html,
+          InlinedAttachments: attachments,
         },
       ],
     });
 };
 
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'mailjet',
   name: 'MailJet',
   description: 'MailJet is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/mailJet.md'),

lib/notification/adapter/mailJet.md

@@ -1,8 +1,8 @@
 ### MailJet Adapter
- 
-To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from.  
- 
-E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.     
+
+To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
+
+E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.  
 The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
 
-If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com). 
+If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).

lib/notification/adapter/mattermost.js

@@ -0,0 +1,39 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+  let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
+  message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
+  message += newListings.map(
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
+  );
+  return fetch(webhook, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: {
+      channel: channel,
+      text: message,
+    },
+  });
+};
+export const config = {
+  id: 'mattermost',
+  name: 'Mattermost',
+  readme: markdown2Html('lib/notification/adapter/mattermost.md'),
+  description: 'Fredy will send new listings to your mattermost team chat.',
+  fields: {
+    webhook: {
+      type: 'text',
+      label: 'Webhook-URL',
+      description: 'The incoming webhook url',
+    },
+    channel: {
+      type: 'text',
+      label: 'Channel',
+      description: 'The channel where fredy should send notifications to.',
+    },
+  },
+};

lib/notification/adapter/mattermost.md

@@ -0,0 +1,5 @@
+### Mattermost Adapter
+
+For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+
+As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.

lib/notification/adapter/ntfy.js

@@ -0,0 +1,62 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+import { normalizeImageUrl } from '../../utils.js';
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+
+  const promises = newListings.map((newListing) => {
+    const message = `
+Address: ${newListing.address}
+Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
+Price: ${newListing.price}
+Link: ${newListing.link}`;
+
+    const headers = {
+      Title: newListing.title,
+      Priority: String(priority),
+      Tags: `${serviceName},${jobName}`,
+      Click: newListing.link,
+    };
+
+    if (newListing.image && typeof newListing.image === 'string') {
+      headers.Attach = normalizeImageUrl(newListing.image);
+    }
+
+    return fetch(`${server}/${topic}`, {
+      method: 'POST',
+      headers,
+      body: message,
+    });
+  });
+
+  return Promise.all(promises);
+};
+
+export const config = {
+  id: 'ntfy',
+  name: 'ntfy',
+  readme: markdown2Html('lib/notification/adapter/ntfy.md'),
+  description: 'Fredy will send new listings to your ntfy.',
+  fields: {
+    priority: {
+      type: 'number',
+      label: 'Priority',
+      description: 'The priority of the send notification.',
+    },
+    server: {
+      type: 'text',
+      label: 'Server-URL',
+      description: 'The server url to the send the notification to.',
+    },
+    topic: {
+      type: 'text',
+      label: 'topic',
+      description:
+        'The topic where fredy should send notifications to. The topic is a secret, only known to you, make sure it is something not generic.',
+    },
+  },
+};

lib/notification/adapter/ntfy.md

@@ -0,0 +1,5 @@
+### ntfy Adapter
+
+For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+
+As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.

lib/notification/adapter/pushover.js

@@ -0,0 +1,79 @@
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+
+export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+
+  const results = await Promise.all(
+    newListings.map(async (newListing) => {
+      const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+      const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+
+      const form = new FormData();
+      form.append('token', token);
+      form.append('user', user);
+      form.append('title', title);
+      form.append('message', message);
+      if (device) form.append('device', device);
+
+      // Try to attach image if available
+      if (newListing.image && typeof newListing.image === 'string') {
+        try {
+          const imgRes = await fetch(newListing.image);
+          if (imgRes.ok) {
+            const ab = await imgRes.arrayBuffer();
+            form.append('attachment', new Blob([ab]), 'image.jpg');
+          }
+        } catch {
+          // fail silently, just skip the image
+        }
+      }
+
+      const res = await fetch('https://api.pushover.net/1/messages.json', {
+        method: 'POST',
+        body: form,
+      });
+
+      return res.json();
+    }),
+  );
+
+  // Collect errors
+  const errors = results
+    .map((r) => (r.errors && r.errors.length > 0 ? r.errors.join(', ') : null))
+    .filter((e) => e !== null);
+
+  if (errors.length > 0) {
+    return Promise.reject(errors.join('; '));
+  }
+
+  return results;
+};
+
+export const config = {
+  id: 'pushover',
+  name: 'Pushover',
+  readme: markdown2Html('lib/notification/adapter/pushover.md'),
+  description: 'Fredy will send new listings to your mobile using Pushover.',
+  fields: {
+    token: {
+      type: 'text',
+      label: 'API token',
+      description: "Your application's API token.",
+    },
+    user: {
+      type: 'text',
+      label: 'User key',
+      description: 'Your user/group key.',
+    },
+    device: {
+      type: 'text',
+      label: 'Device name',
+      description:
+        'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
+    },
+  },
+};

lib/notification/adapter/pushover.md

@@ -0,0 +1,5 @@
+### Pushover Adapter
+
+Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+
+After setting up the application, please enter both your newly created User key and API token.

lib/notification/adapter/sendGrid.js

@@ -1,36 +1,55 @@
-const sgMail = require('@sendgrid/mail');
-const { markdown2Html } = require('../../services/markdown');
+import sgMail from '@sendgrid/mail';
+import { markdown2Html } from '../../services/markdown.js';
+import { normalizeImageUrl } from '../../utils.js';
+
+const mapListings = (serviceName, jobKey, listings) =>
+  listings.map((l) => {
+    const image = normalizeImageUrl(l.image);
+    return {
+      title: l.title || '',
+      link: l.link || '',
+      address: l.address || '',
+      size: l.size || '',
+      price: l.price || '',
+      image,
+      hasImage: Boolean(image),
+      // optional plain text snippet
+      snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
+      serviceName,
+      jobKey,
+    };
+  });
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
 
-/**
- * sends a new listing using SendGrid
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
   sgMail.setApiKey(apiKey);
+
+  const to = receiver
+    .trim()
+    .split(',')
+    .map((r) => r.trim())
+    .filter(Boolean);
+
+  const listings = mapListings(serviceName, jobKey, newListings);
+
   const msg = {
     templateId,
-    to: receiver
-      .trim()
-      .split(',')
-      .map((r) => r.trim()),
+    to,
     from,
     subject: `Job ${jobKey} | Service ${serviceName} found ${newListings.length} new listing(s)`,
     dynamic_template_data: {
       serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
       numberOfListings: newListings.length,
-      listings: newListings,
+      listings,
     },
   };
+
   return sgMail.send(msg);
 };
 
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'sendgrid',
   name: 'SendGrid',
   description: 'SendGrid is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/sendGrid.md'),

lib/notification/adapter/sendGrid.md

@@ -1,9 +1,8 @@
 ### SendGrid Adapter
 
- 
 SendGrid is a free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
-   
-To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.     
+
+To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.
 
 Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new dynamic template. For this new template, I recommend copying and pasting the code from the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
 

lib/notification/adapter/slack.js

@@ -1,55 +1,63 @@
-const Slack = require('slack');
-const msg = Slack.chat.postMessage;
-const { markdown2Html } = require('../../services/markdown');
+import Slack from 'slack';
+import { markdown2Html } from '../../services/markdown.js';
+import { normalizeImageUrl } from '../../utils.js';
 
-/**
- * sends a new listing to slack
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Chat.PostMessage.Response> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
-  return newListings.map((payload) =>
-    msg({
-      token,
-      channel,
-      text: `*(${serviceName} - ${jobKey})* - ${payload.title}`,
-      attachments: [
-        {
-          fallback: payload.title,
-          color: '#36a64f',
-          title: 'Link to Exposé',
-          title_link: payload.link,
-          fields: [
-            {
-              title: 'Price',
-              value: payload.price,
-              short: false,
-            },
-            {
-              title: 'Size',
-              value: payload.size,
-              short: false,
-            },
-            {
-              title: 'Address',
-              value: payload.address,
-              short: false,
-            },
-          ],
-          footer: 'Powered by Fredy',
-          ts: new Date().getTime() / 1000,
-        },
+const buildBlocks = (serviceName, jobKey, p) => {
+  const blocks = [
+    {
+      type: 'header',
+      text: { type: 'plain_text', text: `New Listing from ${serviceName} (${jobKey})`, emoji: false },
+    },
+    {
+      type: 'section',
+      text: { type: 'mrkdwn', text: `*<${p.link}|${p.title}>*` },
+    },
+    {
+      type: 'section',
+      fields: [
+        { type: 'mrkdwn', text: `*Price*\n${p.price ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Size*\n${p.size ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Address*\n${p.address ?? 'n/a'}` },
       ],
-    })
+    },
+  ];
+
+  const img = normalizeImageUrl(p.image);
+  if (img) {
+    blocks.push({
+      type: 'image',
+      image_url: img,
+      alt_text: p.title || 'listing image',
+    });
+  }
+
+  blocks.push({
+    type: 'context',
+    elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
+  });
+
+  return blocks;
+};
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, channel } = notificationConfig.find((a) => a.id === config.id).fields;
+
+  return Promise.allSettled(
+    newListings.map((p) =>
+      Slack.chat.postMessage({
+        token,
+        channel,
+        text: `${serviceName} ${jobKey}: ${p.title}`,
+        blocks: buildBlocks(serviceName, jobKey, p),
+        unfurl_links: false,
+        unfurl_media: false,
+      }),
+    ),
   );
 };
 
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'slack',
   name: 'Slack',
   readme: markdown2Html('lib/notification/adapter/slack.md'),
   description: 'Fredy will send new listings to the slack channel of your choice..',

lib/notification/adapter/slack.md

@@ -1,6 +1,4 @@
 ### Slack Adapter
+IMPORTANT:
+Don't use this adapter anymore, it is outdated and only here for backwards compatability reasons. Use the new Slack Adapter with webhooks!
 
- 
-In order to use [Slack](https://slack.com), you need to create an account. When done, you need to create a new App in your workspace. Give it the permission `chat:write:bot` and `chat:write:user`.  
- 
-Now you need to create a user token and a channel. Make sure the bot is installed to this channel. 

lib/notification/adapter/slack_with_webhooks.js

@@ -0,0 +1,79 @@
+import fetch from 'node-fetch';
+import { markdown2Html } from '../../services/markdown.js';
+import { normalizeImageUrl } from '../../utils.js';
+
+const buildBlocks = (serviceName, jobKey, p) => {
+  const blocks = [
+    {
+      type: 'header',
+      text: { type: 'plain_text', text: `New Listing from ${serviceName} (${jobKey})`, emoji: false },
+    },
+    {
+      type: 'section',
+      text: { type: 'mrkdwn', text: `*<${p.link}|${p.title}>*` },
+    },
+    {
+      type: 'section',
+      fields: [
+        { type: 'mrkdwn', text: `*Price*\n${p.price ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Size*\n${p.size ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Address*\n${p.address ?? 'n/a'}` },
+      ],
+    },
+  ];
+
+  const img = normalizeImageUrl(p.image);
+  if (img) {
+    blocks.push({
+      type: 'image',
+      image_url: img,
+      alt_text: p.title || 'listing image',
+    });
+  }
+
+  blocks.push({
+    type: 'context',
+    elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
+  });
+
+  return blocks;
+};
+
+const postJson = (url, body) =>
+  fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body,
+  });
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const adapter = notificationConfig.find((a) => a.id === config.id);
+  const webhookUrl = adapter?.fields?.webhookUrl;
+  if (!webhookUrl) return Promise.resolve([]);
+
+  const promises = newListings.map((p) => {
+    const body = JSON.stringify({
+      text: `${serviceName} ${jobKey}: ${p.title}`,
+      blocks: buildBlocks(serviceName, jobKey, p),
+      unfurl_links: false,
+      unfurl_media: false,
+    });
+    return postJson(webhookUrl, body);
+  });
+
+  return Promise.allSettled(promises);
+};
+
+export const config = {
+  id: 'slack_with_webhooks',
+  name: 'Slack with Webhooks',
+  readme: markdown2Html('lib/notification/adapter/slack_with_webhooks.md'),
+  description: 'Fredy will send new listings to the slack channel of your choice..',
+  fields: {
+    webhookUrl: {
+      type: 'text',
+      label: 'Webhook-Url',
+      description: 'The Url of the Webhook to send messages to.',
+    },
+  },
+};

lib/notification/adapter/slack_with_webhooks.md

@@ -0,0 +1,6 @@
+### Slack Adapter
+
+IMPORTANT:
+This is the new version of the Slack adapter. I strongly encourage you to use it, the old version is now unmaintained and only kept due to backwards compatability reasons.
+
+In order to use [Slack](https://slack.com), you need to create an account. When done, create a new channel and add the Webhook integration to that channel. Copy the webhook url. That's it.

lib/notification/adapter/sqlite.js

@@ -0,0 +1,56 @@
+import { markdown2Html } from '../../services/markdown.js';
+import Database from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs';
+
+export const send = ({ serviceName, newListings, jobKey, notificationConfig }) => {
+  const sqliteConfig = notificationConfig.find((adapter) => adapter.id === config.id);
+  const dbPath = sqliteConfig?.fields?.dbPath || 'db/listings.db';
+
+  const dbDir = path.dirname(dbPath);
+  if (!fs.existsSync(dbDir)) {
+    fs.mkdirSync(dbDir, { recursive: true });
+  }
+
+  const db = new Database(dbPath);
+  const fields = [
+    'serviceName',
+    'jobKey',
+    'id',
+    'size',
+    'rooms',
+    'price',
+    'address',
+    'title',
+    'link',
+    'description',
+    'image',
+  ];
+  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
+  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
+  newListings.map((listing) => {
+    let insertListing = {};
+    fields.map((field) => {
+      insertListing[field] = listing[field];
+    });
+    insertListing.serviceName = serviceName;
+    insertListing.jobKey = jobKey;
+    insert.run(insertListing);
+  });
+  return Promise.resolve();
+};
+export const config = {
+  id: 'sqlite',
+  name: 'SQLite',
+  description: 'This adapter stores listings in a local SQLite 3 database.',
+  fields: {
+    dbPath: {
+      type: 'text',
+      label: 'Database Path',
+      description:
+        'Path to the SQLite database file (e.g., db/listings.db). If not specified, defaults to db/listings.db',
+      placeholder: 'db/listings.db',
+    },
+  },
+  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
+};

lib/notification/adapter/sqlite.md

@@ -0,0 +1,9 @@
+### SQLite Adapter
+
+This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. This file can be used for further analysis later.
+
+The database table contains the following columns (all stored as `TEXT` type):
+
+```
+['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description', 'image']
+```

lib/notification/adapter/telegram.js

@@ -1,65 +1,110 @@
-const { markdown2Html } = require('../../services/markdown');
-const axios = require('axios');
+import { markdown2Html } from '../../services/markdown.js';
+import { getJob } from '../../services/storage/jobStorage.js';
+import fetch from 'node-fetch';
+import pThrottle from 'p-throttle';
+import { normalizeImageUrl } from '../../utils.js';
 
-/**
- * splitting an array into chunks because Telegram only allows for messages up to
- * 4096 chars, thus we have to split messages into chunks
- * @param inputArray
- * @param perChunk
- */
-const arrayChunks = (inputArray, perChunk) =>
-  inputArray.reduce((all, one, i) => {
-    const ch = Math.floor(i / perChunk);
-    all[ch] = [].concat(all[ch] || [], one);
-    return all;
-  }, []);
+const RATE_LIMIT_INTERVAL = 1000;
+const chatThrottleMap = new Map();
 
-/**
- * sends new listings to telegram
- * @param serviceName e.g immowelt
- * @param newListings an array with newly found listings
- * @param notificationConfig config of this notification adapter
- * * @param jobKey name of the current job that is being executed
- * @returns {Promise<Void> | void}
- */
-exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
+function cleanupOldThrottles() {
+  const now = Date.now();
+  const maxAge = RATE_LIMIT_INTERVAL + 1000;
+  const toBeDeleted = [];
+  for (const [chatId, chatThrottle] of chatThrottleMap.entries()) {
+    if (now - chatThrottle.lastUsedAt > maxAge) toBeDeleted.push(chatId);
+  }
+  for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId);
+}
 
-  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
-  const chunks = arrayChunks(newListings, 3);
+function getThrottled(chatId, call) {
+  cleanupOldThrottles();
+  const now = Date.now();
+  const chatThrottle = chatThrottleMap.get(chatId);
+  if (chatThrottle) {
+    chatThrottle.lastUsedAt = now;
+    return chatThrottle.throttled;
+  }
+  const throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call);
+  chatThrottleMap.set(chatId, { lastUsedAt: now, throttled });
+  return throttled;
+}
 
-  const promises = chunks.map((chunk) => {
-    let message = `Job: ${jobKey} | Service <b>${serviceName}</b> found <b>${newListings.length}</b> new listings:\n\n`;
-    message += chunk.map(
-      (o) =>
-        `<b>${shorten(o.title.replace(/\*/g, ''), 45)}</b>\n` +
-        [o.address, o.price, o.size].join(' | ') +
-        '\n' +
-        `<a href="${o.link}">${o.link}</a>\n\n`
-    );
+function shorten(str, len = 90) {
+  if (!str) return '';
+  return str.length > len ? str.substring(0, len).trim() + '...' : str;
+}
 
-    return axios.post(`https://api.telegram.org/bot${token}/sendMessage`, {
+function escapeHtml(s = '') {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function buildCaption(jobName, serviceName, o) {
+  const title = shorten((o.title || '').replace(/\*/g, ''), 90);
+  const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
+  return `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n<a href='${escapeHtml(
+    o.link || '',
+  )}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}`.slice(0, 1024);
+}
+
+function buildText(jobName, serviceName, o) {
+  const title = shorten((o.title || '').replace(/\*/g, ''), 90);
+  const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
+  return (
+    `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n` +
+    `<a href='${escapeHtml(o.link || '')}'><b>${escapeHtml(title)}</b></a>\n` +
+    `${escapeHtml(meta)}`
+  );
+}
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+  const job = getJob(jobKey);
+  const jobName = job == null ? jobKey : job.name;
+
+  const throttledCall = getThrottled(chatId, async function (endpoint, body) {
+    const res = await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, {
+      method: 'post',
+      body: JSON.stringify(body),
+      headers: { 'Content-Type': 'application/json' },
+    });
+    return res;
+  });
+
+  const promises = newListings.map(async (o) => {
+    const img = normalizeImageUrl(o.image);
+    const textPayload = {
       chat_id: chatId,
-      text: message,
+      text: buildText(jobName, serviceName, o),
       parse_mode: 'HTML',
       disable_web_page_preview: true,
-    });
+    };
+
+    if (!img) {
+      return throttledCall('sendMessage', textPayload);
+    }
+
+    try {
+      return await throttledCall('sendPhoto', {
+        chat_id: chatId,
+        photo: img,
+        caption: buildCaption(jobName, serviceName, o),
+        parse_mode: 'HTML',
+      });
+    } catch (e) {
+      // If we see a timeout due to sending an image, try sending it without
+      if (e && (e.code === 'ETIMEDOUT' || e.errno === 'ETIMEDOUT')) {
+        return throttledCall('sendMessage', textPayload);
+      }
+      throw e;
+    }
   });
 
   return Promise.all(promises);
 };
 
-function shorten(str, len = 30) {
-  return str.length > len ? str.substring(0, len) + '...' : str;
-}
-
-/**
- * exported config is being used in the frontend to generate the fields
- * incoming values will be the keys (and values) of the fields
- *
- */
-exports.config = {
-  id: __filename.slice(__dirname.length + 1, -3),
+export const config = {
+  id: 'telegram',
   name: 'Telegram',
   readme: markdown2Html('lib/notification/adapter/telegram.md'),
   description: 'Fredy will send new listings to your mobile, using Telegram.',

lib/notification/adapter/telegram.md

@@ -1,13 +1,12 @@
 ### Telegram Adapter
 
- 
 For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
-  
-A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.        
-After the user has send a message to your bot the first time, you can gather the chatId like this:   
 
-```  
-curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates  
-```  
-  
+A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.  
+After the user has send a message to your bot the first time, you can gather the chatId like this:
+
+```
+curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
+```
+
 A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)

lib/notification/emailTemplate/mailjet.hbs

@@ -0,0 +1,123 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <title>Listings</title>
+    <style type="text/css">
+        body { margin:0; padding:0; background:#000000; }
+        table { border-collapse:collapse; }
+        img { border:0; outline:none; text-decoration:none; display:block; }
+        a { text-decoration:none; }
+        .container { width:100%; max-width:640px; margin:0 auto; }
+        .card { background:#111111; border:1px solid #222222; border-radius:8px; overflow:hidden; }
+        .divider { height:2px; line-height:2px; font-size:0; background:#00dc73; }
+        .h1 { font:700 18px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .h2 { font:700 16px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .p { font:400 14px/1.6 Arial, Helvetica, sans-serif; color:#d9d9d9; margin:0; }
+        .meta { font:400 13px/1.5 Arial, Helvetica, sans-serif; color:#bfbfbf; }
+        .btn { background:#00dc73; color:#0b0b0b; font:700 14px/1 Arial, Helvetica, sans-serif; padding:12px 18px; border-radius:6px; display:inline-block; }
+        .sp-8 { height:8px; line-height:8px; font-size:0; }
+        .sp-12 { height:12px; line-height:12px; font-size:0; }
+        .sp-16 { height:16px; line-height:16px; font-size:0; }
+        .sp-20 { height:20px; line-height:20px; font-size:0; }
+        .sp-24 { height:24px; line-height:24px; font-size:0; }
+        @media screen and (max-width:480px){
+            .container { width:100% !important; }
+            .stack { display:block !important; width:100% !important; }
+        }
+    </style>
+</head>
+<body style="background:#000000;">
+<table role="presentation" width="100%" bgcolor="#000000">
+    <tr>
+        <td align="center">
+            <table role="presentation" class="container" width="640">
+                <tr><td class="sp-20"></td></tr>
+
+                <tr>
+                    <td align="center">
+                        <h1 class="h1" style="text-align:center;">
+                            Service {{serviceName}} found {{numberOfListings}} new listings
+                        </h1>
+                    </td>
+                </tr>
+
+                <tr><td class="sp-12"></td></tr>
+                <tr><td class="divider"></td></tr>
+                <tr><td class="sp-16"></td></tr>
+
+                {{#each listings}}
+                    <tr>
+                        <td>
+                            <table role="presentation" class="card" width="100%">
+                                {{#if this.hasImage}}
+                                    <tr>
+                                        <td>
+                                            <a href="{{this.link}}" target="_blank">
+                                                <img src="cid:{{this.imageCid}}" alt="{{this.title}}" width="640"
+                                                     style="width:100%; height:auto; background:#1a1a1a;" />
+                                            </a>
+                                        </td>
+                                    </tr>
+                                {{/if}}
+
+                                <tr>
+                                    <td style="padding:16px 18px 0 18px;">
+                                        <a href="{{this.link}}" target="_blank" style="color:#ffffff;">
+                                            <h2 class="h2">{{this.title}}</h2>
+                                        </a>
+                                    </td>
+                                </tr>
+
+                                <tr><td class="sp-8"></td></tr>
+
+                                <tr>
+                                    <td style="padding:0 18px;">
+                                        <table role="presentation" width="100%">
+                                            <tr>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-right:8px;">
+                                                    <p class="meta"><strong>Price</strong><br/>{{#if this.price}}{{this.price}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-left:8px;">
+                                                    <p class="meta"><strong>Size</strong><br/>{{#if this.size}}{{this.size}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                            <tr><td class="sp-8"></td><td class="sp-8"></td></tr>
+                                            <tr>
+                                                <td colspan="2">
+                                                    <p class="meta"><strong>Address</strong><br/>{{#if this.address}}{{this.address}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                        </table>
+                                    </td>
+                                </tr>
+
+                                <tr><td class="sp-16"></td></tr>
+
+                                <tr>
+                                    <td align="left" style="padding:0 18px 18px 18px;">
+                                        <a href="{{this.link}}" class="btn" target="_blank">View Listing</a>
+                                    </td>
+                                </tr>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr><td class="sp-24"></td></tr>
+                {{/each}}
+
+                <tr><td class="divider"></td></tr>
+                <tr><td class="sp-16"></td></tr>
+                <tr>
+                    <td align="center">
+                        <p class="p" style="color:#9f9f9f;">Powered by Fredy</p>
+                    </td>
+                </tr>
+                <tr><td class="sp-20"></td></tr>
+            </table>
+        </td>
+    </tr>
+</table>
+</body>
+</html>

lib/notification/emailTemplate/template.hbs

@@ -1,237 +1,131 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html data-editor-version="2" class="sg-campaigns" xmlns="http://www.w3.org/1999/xhtml"><head>
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1, maximum-scale=1">
-    <!--[if !mso]><!-->
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
-    <!--<![endif]-->
-    <!--[if (gte mso 9)|(IE)]>
-    <xml>
-        <o:OfficeDocumentSettings>
-            <o:AllowPNG/>
-            <o:PixelsPerInch>96</o:PixelsPerInch>
-        </o:OfficeDocumentSettings>
-    </xml>
-    <![endif]-->
-    <!--[if (gte mso 9)|(IE)]>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <title>Fredy found some new listings</title>
     <style type="text/css">
-        body {width: 600px;margin: 0 auto;}
-        table {border-collapse: collapse;}
-        table, td {mso-table-lspace: 0pt;mso-table-rspace: 0pt;}
-        img {-ms-interpolation-mode: bicubic;}
-    </style>
-    <![endif]-->
-    <style type="text/css">
-        body, p, div {
-            font-family: arial,helvetica,sans-serif;
-            font-size: 14px;
-        }
-        body {
-            color: #000000;
-        }
-        body a {
-            color: #42ee99;
-            text-decoration: none;
-        }
-        p { margin: 0; padding: 0; }
-        table.wrapper {
-            width:100% !important;
-            table-layout: fixed;
-            -webkit-font-smoothing: antialiased;
-            -webkit-text-size-adjust: 100%;
-            -moz-text-size-adjust: 100%;
-            -ms-text-size-adjust: 100%;
-        }
-        img.max-width {
-            max-width: 100% !important;
-        }
-        .column.of-2 {
-            width: 50%;
-        }
-        .column.of-3 {
-            width: 33.333%;
-        }
-        .column.of-4 {
-            width: 25%;
-        }
-        @media screen and (max-width:480px) {
-            .preheader .rightColumnContent,
-            .footer .rightColumnContent {
-                text-align: left !important;
-            }
-            .preheader .rightColumnContent div,
-            .preheader .rightColumnContent span,
-            .footer .rightColumnContent div,
-            .footer .rightColumnContent span {
-                text-align: left !important;
-            }
-            .preheader .rightColumnContent,
-            .preheader .leftColumnContent {
-                font-size: 80% !important;
-                padding: 5px 0;
-            }
-            table.wrapper-mobile {
-                width: 100% !important;
-                table-layout: fixed;
-            }
-            img.max-width {
-                height: auto !important;
-                max-width: 100% !important;
-            }
-            a.bulletproof-button {
-                display: block !important;
-                width: auto !important;
-                font-size: 80%;
-                padding-left: 0 !important;
-                padding-right: 0 !important;
-            }
-            .columns {
-                width: 100% !important;
-            }
-            .column {
-                display: block !important;
-                width: 100% !important;
-                padding-left: 0 !important;
-                padding-right: 0 !important;
-                margin-left: 0 !important;
-                margin-right: 0 !important;
-            }
+        body { margin:0; padding:0; background:#000000; }
+        table { border-collapse:collapse; }
+        img { border:0; outline:none; text-decoration:none; display:block; }
+        a { text-decoration:none; }
+        .container { width:100%; max-width:640px; margin:0 auto; }
+        .card { background:#111111; border:1px solid #222222; border-radius:8px; overflow:hidden; }
+        .divider { height:2px; line-height:2px; font-size:0; background:#00dc73; }
+        .h1 { font:700 18px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .h2 { font:700 16px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .p { font:400 14px/1.6 Arial, Helvetica, sans-serif; color:#d9d9d9; margin:0; }
+        .meta { font:400 13px/1.5 Arial, Helvetica, sans-serif; color:#bfbfbf; }
+        .btn { background:#00dc73; color:#0b0b0b; font:700 14px/1 Arial, Helvetica, sans-serif; padding:12px 18px; border-radius:6px; display:inline-block; }
+        .sp-8 { height:8px; line-height:8px; font-size:0; }
+        .sp-12 { height:12px; line-height:12px; font-size:0; }
+        .sp-16 { height:16px; line-height:16px; font-size:0; }
+        .sp-20 { height:20px; line-height:20px; font-size:0; }
+        .sp-24 { height:24px; line-height:24px; font-size:0; }
+        @media screen and (max-width:480px){
+            .container { width:100% !important; }
+            .stack { display:block !important; width:100% !important; }
         }
     </style>
-    <!--user entered Head Start-->
-
-    <!--End Head user entered-->
 </head>
-<body>
-<center class="wrapper" data-link-color="#42ee99" data-body-style="font-size:14px; font-family:arial,helvetica,sans-serif; color:#000000; background-color:#000000;">
-    <div class="webkit">
-        <table cellpadding="0" cellspacing="0" border="0" width="100%" class="wrapper" bgcolor="#000000">
-            <tbody><tr>
-                <td valign="top" bgcolor="#000000" width="100%">
-                    <table width="100%" role="content-container" class="outer" align="center" cellpadding="0" cellspacing="0" border="0">
-                        <tbody><tr>
-                            <td width="100%">
-                                <table width="100%" cellpadding="0" cellspacing="0" border="0">
-                                    <tbody><tr>
+<body style="background:#000000;">
+<table role="presentation" width="100%" bgcolor="#000000">
+    <tr>
+        <td align="center">
+            <table role="presentation" class="container" width="640">
+                <tr><td class="sp-20"></td></tr>
+                <tr>
+                    <td align="center">
+                        <table role="presentation" width="100%">
+                            <tr>
+                                <td align="center">
+                                    <h1 class="h1" style="text-align:center;">
+                                        Service {{serviceName}} found {{numberOfListings}} new listings
+                                    </h1>
+                                </td>
+                            </tr>
+                            <tr><td class="sp-12"></td></tr>
+                            <tr><td class="divider"></td></tr>
+                            <tr><td class="sp-16"></td></tr>
+                        </table>
+                    </td>
+                </tr>
+
+                {{#each listings}}
+                    <tr>
+                        <td>
+                            <table role="presentation" class="card" width="100%">
+                                {{#if this.hasImage}}
+                                    <tr>
                                         <td>
-                                            <!--[if mso]>
-                                            <center>
-                                                <table><tr><td width="600">
-                                            <![endif]-->
-                                            <table width="100%" cellpadding="0" cellspacing="0" border="0" style="width:100%; max-width:600px;" align="center">
-                                                <tbody><tr>
-                                                    <td role="modules-container" style="padding:0px 0px 0px 0px; color:#000000; text-align:left;" bgcolor="#FFFFFF" width="100%" align="left"><table class="module preheader preheader-hide" role="module" data-type="preheader" border="0" cellpadding="0" cellspacing="0" width="100%" style="display: none !important; mso-hide: all; visibility: hidden; opacity: 0; color: transparent; height: 0; width: 0;">
-                                                    </table><table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="vB9TDziyvx65CC2nx3oyRH">
-                                                        <tbody><tr>
-                                                            <td style="padding:0px 0px 20px 0px;" role="module-content" bgcolor="#000000">
-                                                            </td>
-                                                        </tr>
-                                                        </tbody></table><table class="wrapper" role="module" data-type="image" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="uXsDxMnn1bRMmDcX8NB6rW">
-                                                    </table><table class="module" role="module" data-type="text" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="hL6wjQ2qknNd5qDwT1p7Up">
-                                                        <tbody><tr>
-                                                            <td style="background-color:#000000; padding:10px 20px 10px 20px; line-height:40px; text-align:justify;" height="100%" valign="top" bgcolor="#000000"><div><h1 style="text-align: center"><span style="color: #ffffff; font-size: 14px; font-family: verdana,geneva,sans-serif"><strong>Service {{serviceName}} found {{numberOfListings}} new listing(s)!</strong></span></h1><div></div></div></td>
-                                                        </tr>
-                                                        </tbody></table>
-
-                                                        <table border="0" cellpadding="0" cellspacing="0" align="center" width="100%" height="1px" style="line-height:3px; font-size:3px;">
-                                                            <tbody><tr>
-                                                                <td style="padding:0px 0px 1px 0px;" bgcolor="#42ee99"></td>
-                                                            </tr>
-                                                            </tbody></table>
-                                                    </td>
-                                                </tr>
-                                                </tbody>
-                                            </table>
-
-                                            <table class="module" role="module" data-type="text" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="qk51Jjn4bm3rn2Yb31Dxzb">
-                                                <tbody>
-                                                {{#each listings}}
-                                                    <tr>
-                                                        <td style="padding:50px 50px 10px 50px; line-height:22px; text-align:center; color:white" bgcolor="#000000" height="100%" valign="top">
-                                                            <div>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;"><b>{{this.title}}</b></span>
-                                                                <br/>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;">Size: {{#if this.size}}{{this.size}}{{else}}unknown{{/if}}</span>
-                                                                <br/>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;">Price: {{#if this.price}}{{this.price}}{{else}}unknown{{/if}}</span>
-                                                                <br/>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;">{{#if this.address}}{{this.address}}{{else}}unknown{{/if}}</span>
-                                                                <br/>
-                                                                <a href="{{this.link}}" target="_blank" style="color:#00dc73; font-size:13px">{{this.link}}</a>
-                                                                <br/>
-                                                                <span style="color: white;">---------------------------</span>
-                                                            </div>
-                                                        </td>
-                                                    </tr>
-                                                {{/each}}
-                                                </tbody>
-                                            </table>
-
-                                            <table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="2ga5f7koD5ApvUfnqUK6aT">
-                                                <tbody><tr>
-                                                    <td style="padding:0px 0px 30px 0px;" role="module-content" bgcolor="#000000">
-                                                    </td>
-                                                </tr>
-                                                </tbody></table>
-                                            <table class="module" role="module" data-type="divider" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="c3nRrjMndqXf1snYDFPSF9">
-                                                <tbody><tr>
-                                                    <td style="padding:0px 0px 0px 0px;" role="module-content" height="100%" valign="top" bgcolor="#000000">
-                                                        <table border="0" cellpadding="0" cellspacing="0" align="center" width="100%" height="2px" style="line-height:1px; font-size:2px;">
-                                                            <tbody><tr>
-                                                                <td style="padding:0px 0px 2px 0px;" bgcolor="#42ee99"></td>
-                                                            </tr>
-                                                            </tbody></table>
-                                                    </td>
-                                                </tr>
-                                                </tbody>
-                                            </table>
-                                            <table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="pa9PeYjCEFyByuP5878Sd2">
-                                                <tbody><tr>
-                                                    <td style="padding:0px 0px 30px 0px;" role="module-content" bgcolor="#000000">
-                                                    </td>
-                                                </tr>
-                                                </tbody></table>
-
-                                            <table class="module" role="module" data-type="social" align="center" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="n7FceQWVnLmounEt32B1gj">
-                                                <tbody>
-                                                <tr>
-                                                    <td valign="top" style="padding:0px 0px 0px 0px; font-size:6px; line-height:10px; background-color:#000000;" align="center">
-                                                        <table align="center">
-                                                            <tbody>
-                                                            <tr><td style="padding: 0px 5px;">
-                                                                <a href="https://github.com/orangecoding/fredy" target="_blank" alt="Fredy" title="Powered by Fredy" style="color:#00dc73; font-size:17px">
-                                                                    Powered by Fredy
-                                                                </a>
-                                                            </td></tr>
-                                                            </tbody>
-                                                        </table>
-                                                    </td>
-                                                </tr>
-                                                </tbody>
-                                            </table></table><table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="35xFa9abxGTBYt9yR9BeQ2">
-                                <tbody><tr>
-                                    <td style="padding:0px 0px 30px 0px;" role="module-content" bgcolor="#000000">
+                                            <a href="{{this.link}}" target="_blank">
+                                                <img src="{{this.image}}" alt="{{this.title}}" width="640" style="width:100%;height:auto;background:#1a1a1a;" />
+                                            </a>
+                                        </td>
+                                    </tr>
+                                {{/if}}
+                                <tr>
+                                    <td style="padding:16px 18px 0 18px;">
+                                        <a href="{{this.link}}" target="_blank" style="color:#ffffff;">
+                                            <h2 class="h2">{{this.title}}</h2>
+                                        </a>
                                     </td>
                                 </tr>
-                                </tbody></table></td>
-                        </tr>
-                        </tbody></table>
-                    <!--[if mso]>
-                    </td>
-                    </tr>
-                    </table>
-                    </center>
+                                <tr><td class="sp-8"></td></tr>
+                                <tr>
+                                    <td style="padding:0 18px;">
+                                        <table role="presentation" width="100%">
+                                            <tr>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-right:8px;">
+                                                    <p class="meta"><strong>Price</strong><br/>{{#if this.price}}{{this.price}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-left:8px;">
+                                                    <p class="meta"><strong>Size</strong><br/>{{#if this.size}}{{this.size}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                            <tr><td class="sp-8"></td><td class="sp-8"></td></tr>
+                                            <tr>
+                                                <td colspan="2">
+                                                    <p class="meta"><strong>Address</strong><br/>{{#if this.address}}{{this.address}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                        </table>
+                                    </td>
+                                </tr>
+                                <tr><td class="sp-16"></td></tr>
+                                <tr>
+                                    <td align="left" style="padding:0 18px 18px 18px;">
+                                        <!--[if mso]>
+                    <v:roundrect xmlns:v="urn:schemas-microsoft-com:vml" href="{{this.link}}" arcsize="8%" strokecolor="#00dc73" strokeweight="0" fillcolor="#00dc73" style="height:40px;v-text-anchor:middle;width:180px;">
+                      <w:anchorlock/>
+                      <center style="color:#0b0b0b;font-family:Arial;font-size:14px;font-weight:bold;">
+                        View Listing
+                      </center>
+                    </v:roundrect>
                     <![endif]-->
-                </td>
-            </tr>
-            </tbody></table>
-        </td>
-        </tr>
-        </tbody></table>
-        </td>
-        </tr>
-        </tbody></table>
-    </div>
-</center>
+                                        <!--[if !mso]><!-- -->
+                                        <a href="{{this.link}}" class="btn" target="_blank">View Listing</a>
+                                        <!--<![endif]-->
+                                    </td>
+                                </tr>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr><td class="sp-24"></td></tr>
+                {{/each}}
 
-
-</body></html>
+                <tr><td class="divider"></td></tr>
+                <tr><td class="sp-16"></td></tr>
+                <tr>
+                    <td align="center">
+                        <p class="p" style="color:#9f9f9f;">Powered by Fredy</p>
+                    </td>
+                </tr>
+                <tr><td class="sp-20"></td></tr>
+            </table>
+        </td>
+    </tr>
+</table>
+</body>
+</html>

lib/notification/notify.js

@@ -1,24 +1,24 @@
-const fs = require('fs');
+import fs from 'fs';
 const path = './adapter';
 
 /** Read every integration existing in ./adapter **/
-const adapter = fs
-  .readdirSync('./lib/notification/adapter')
-  .filter((file) => file.endsWith('.js'))
-  .map((integPath) => require(`${path}/${integPath}`));
+const adapter = await Promise.all(
+  fs
+    .readdirSync('./lib/notification/adapter')
+    .filter((file) => file.endsWith('.js'))
+    .map(async (integPath) => await import(`${path}/${integPath}`)),
+);
 
 if (adapter.length === 0) {
   throw new Error('Please specify at least one notification provider');
 }
-
-exports.send = (serviceName, newListings, notificationConfig, jobKey) => {
+const findAdapter = (notificationAdapter) => {
+  return adapter.find((a) => a.config.id === notificationAdapter.id);
+};
+export const send = (serviceName, newListings, notificationConfig, jobKey) => {
   //this is not being used in tests, therefore adapter are always set
   return notificationConfig
     .filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
     .map((notificationAdapter) => findAdapter(notificationAdapter))
     .map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
 };
-
-const findAdapter = (notificationAdapter) => {
-  return adapter.find((a) => a.config.id === notificationAdapter.id);
-};

lib/provider/einsAImmobilien.js

@@ -1,21 +1,37 @@
-const utils = require('../utils');
-
+import { buildHash, isOneOf } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
 let appliedBlackList = [];
 
 function normalize(o) {
-  let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
-  if (o.rooms != null) {
-    size += ` / / ${o.rooms.trim()}`;
-  }
-  const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-
-  return Object.assign(o, { size, link });
+  const baseUrl = 'https://www.1a-immobilienmarkt.de';
+  const link = `${baseUrl}/expose/${o.id}.html`;
+  const price = normalizePrice(o.price);
+  const id = buildHash(o.id, price);
+  const image = baseUrl + o.image;
+  const address = o.address == null ? null : o.address.trim().replaceAll('/', ',');
+  return Object.assign(o, { id, price, link, image, address });
 }
 
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
+}
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
@@ -23,28 +39,27 @@ const config = {
   url: null,
   crawlContainer: '.tabelle',
   sortByDateParam: 'sort_type=newest',
+  waitForSelector: 'body',
   crawlFields: {
     id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
-    price: '.tabelle .inner_object_data .single_data_price | removeNewline | trim',
-    size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
-    rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
-    title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
+    price: '.inner_object_data .single_data_price | removeNewline | trim',
+    size: '.tabelle .tabelle_inhalt_infos .single_data_box  | removeNewline | trim',
+    title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
+    image: '.inner_object_pic img@src',
+    address: '.tabelle .tabelle_inhalt_infos .left_information > div:nth-child(2) | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: '1a Immobilien',
   baseUrl: 'https://www.1a-immobilienmarkt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'einsAImmobilien',
 };
-
-exports.config = config;
+export { config };

lib/provider/immobilienDe.js

@@ -0,0 +1,62 @@
+import { buildHash, isOneOf } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
+
+let appliedBlackList = [];
+
+function shortenLink(link) {
+  return link.substring(0, link.indexOf('?'));
+}
+
+function parseId(shortenedLink) {
+  return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
+}
+
+function normalize(o) {
+  const baseUrl = 'https://www.immobilien.de';
+  const size = o.size || null;
+  const price = o.price || null;
+  const title = o.title || 'No title available';
+  const address = o.address || null;
+  const shortLink = shortenLink(o.link);
+  const link = `${baseUrl}/${shortLink}`;
+  const image = baseUrl + o.image;
+  const id = buildHash(parseId(shortLink), o.price);
+  return Object.assign(o, { id, price, size, title, address, link, image });
+}
+
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+
+const config = {
+  url: null,
+  crawlContainer: '._ref',
+  sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
+  waitForSelector: 'body',
+  crawlFields: {
+    id: '@href', //will be transformed later
+    price: '.list_entry .immo_preis .label_info',
+    size: '.list_entry .flaeche .label_info | removeNewline | trim',
+    title: '.list_entry .part_text h3 span',
+    description: '.list_entry .description | trim',
+    link: '@href',
+    address: '.list_entry .place',
+    image: '.list_entry img@src',
+  },
+  normalize: normalize,
+  filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immobilien.de',
+  baseUrl: 'https://www.immobilien.de/',
+  id: 'immobilienDe',
+};
+export { config };

lib/provider/immonet.js

@@ -1,52 +1,47 @@
-const utils = require('../utils');
-
+import { isOneOf, buildHash } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
 let appliedBlackList = [];
 
 function normalize(o) {
-  const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
-  const address = o.address.split(' • ')[1];
+  const address = o.address?.split(' • ')?.pop() ?? null;
   const title = o.title || 'No title available';
-  //normally we would just read the link from the source, but immonet decided to trick user by adding a click listener instead of
-  //a href to do some weird reporting. (Very user friendly for handicaped ppl... not)
-  const link = `https://www.immonet.de/angebot/${id}`;
+  const link = o.link != null ? decodeURIComponent(o.link) : config.url;
+  const id = buildHash(title, price);
   return Object.assign(o, { id, address, price, size, title, link });
 }
-
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
-
 const config = {
   url: null,
-  crawlContainer: '#result-list-stage .item',
+  crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
   sortByDateParam: 'sortby=19',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
-    id: '@id',
-    price: 'div[id*="selPrice_"] | trim',
-    size: 'div[id*="selArea_"] | trim',
-    title: '.item a img@title',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
+    id: 'button@title |trim',
+    title: 'button@title |trim',
+    price: 'div[data-testid="cardmfe-price-testid"] | trim',
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
+    address: 'div[data-testid="cardmfe-description-box-address"] | trim',
+    image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
+    link: 'button@data-base',
   },
-  paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
   normalize: normalize,
   filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immonet',
   baseUrl: 'https://www.immonet.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immonet',
 };
-
-exports.config = config;
+export { config };

lib/provider/immoscout.js

@@ -1,49 +1,142 @@
-const utils = require('../utils');
+/**
+ * ImmoScout provider using the mobile API to retrieve listings.
+ *
+ * The mobile API provides the following endpoints:
+ * - GET /search/total?{search parameters}: Returns the total number of listings for the given query
+ *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
+ *
+ * - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
+ *   data specifying additional results (advertisements) to return. The format is as follows:
+ *   ```
+ *   {
+ *   "supportedResultListTypes": [],
+ *   "userData": {}
+ *   }
+ *   ```
+ *   It is not necessary to provide data for the specified keys.
+ *
+ *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
 
+ * - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
+ *   listing response.
+ *
+ *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
+ *
+ *
+ * It is necessary to set the correct User Agent (see `getListings`) in the request header.
+ *
+ * Note that the mobile API is not publicly documented. I've reverse-engineered
+ * it by intercepting traffic from an android emulator running the immoscout app.
+ * Moreover, the search parameters differ slightly from the web API. I've mapped them
+ * to the web API parameters by comparing a search request with all parameters set between
+ * the web and mobile API. The mobile API actually seems to be a superset of the web API,
+ * but I have decided not to include new parameters as I wanted to keep the existing UX (i.e.,
+ * users only have to provide a link to an existing search).
+ *
+ */
+
+import { buildHash, isOneOf } from '../utils.js';
+import {
+  convertImmoscoutListingToMobileListing,
+  convertWebToMobile,
+} from '../services/immoscout/immoscout-web-translator.js';
+import logger from '../services/logger.js';
 let appliedBlackList = [];
 
+async function getListings(url) {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'User-Agent': 'ImmoScout_27.3_26.0_._',
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      supportedResultListTypes: [],
+      userData: {},
+    }),
+  });
+  if (!response.ok) {
+    logger.error('Error fetching data from ImmoScout Mobile API:', response.statusText);
+    return [];
+  }
+
+  const responseBody = await response.json();
+  return responseBody.resultListItems
+    .filter((item) => item.type === 'EXPOSE_RESULT')
+    .map((expose) => {
+      const item = expose.item;
+      const [price, size] = item.attributes;
+      const image = item?.titlePicture?.preview ?? null;
+      return {
+        id: item.id,
+        price: price?.value,
+        size: size?.value,
+        title: item.title,
+        description: item.description,
+        link: `${metaInformation.baseUrl}expose/${item.id}`,
+        address: item.address?.line,
+        image,
+      };
+    });
+}
+
+async function isListingActive(link) {
+  const result = await fetch(convertImmoscoutListingToMobileListing(link), {
+    headers: {
+      'User-Agent': 'ImmoScout_27.3_26.0_._',
+    },
+  });
+
+  if (result.status === 200) {
+    return 1;
+  }
+
+  if (result.status === 404) {
+    return 0;
+  }
+
+  logger.warn('Unknown status for immoscout listing', link);
+  return -1;
+}
+
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
-
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
-  return Object.assign(o, { title, address, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, title, address });
 }
-
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
+  return !isOneOf(o.title, appliedBlackList);
 }
-
 const config = {
   url: null,
-  crawlContainer: '#resultListItems li.result-list__listing',
-  sortByDateParam: 'sorting=2',
   crawlFields: {
-    id: '.result-list-entry@data-obid | int',
-    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
-    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
-    link: '.result-list-entry .result-list-entry__brand-title-container@href',
-    address: '.result-list-entry .result-list-entry__map-link',
+    id: 'id',
+    title: 'title',
+    price: 'price',
+    size: 'size',
+    link: 'link',
+    address: 'address',
   },
-  paginate: '#pager .align-right a@href',
+  // Not required - used by filter to remove and listings that failed to parse
+  sortByDateParam: 'sorting=-firstactivation',
   normalize: normalize,
   filter: applyBlacklist,
+  getListings: getListings,
+  activeTester: isListingActive,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+  config.url = convertWebToMobile(sourceConfig.url);
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immoscout',
   baseUrl: 'https://www.immobilienscout24.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immoscout',
 };
 
-exports.config = config;
+export { config };

lib/provider/immoswp.js

@@ -0,0 +1,51 @@
+import { isOneOf, buildHash } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
+
+let appliedBlackList = [];
+
+function normalize(o) {
+  const size = o.size || 'N/A m²';
+  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+  const title = o.title || 'No title available';
+  const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+  const link = `https://immo.swp.de/immobilien/${immoId}`;
+  const description = o.description;
+  const id = buildHash(immoId, price);
+  return Object.assign(o, { id, price, size, title, link, description });
+}
+
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+
+const config = {
+  url: null,
+  crawlContainer: '.js-serp-item',
+  sortByDateParam: 's=most_recently_updated_first',
+  waitForSelector: 'body',
+  crawlFields: {
+    id: '.js-bookmark-btn@data-id',
+    price: 'div.align-items-start div:first-child | trim',
+    size: 'div.align-items-start div:nth-child(3) | trim',
+    title: '.js-item-title-link@title | trim',
+    link: '.ci-search-result__link@href',
+    description: '.js-show-more-item-sm | removeNewline | trim',
+    image: 'img@src',
+  },
+  normalize: normalize,
+  filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
+};
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+export const metaInformation = {
+  name: 'Immo Südwest Presse',
+  baseUrl: 'https://immo.swp.de/',
+  id: 'immoswp',
+};
+export { config };

lib/provider/immowelt.js

@@ -1,44 +1,46 @@
-const utils = require('../utils');
+import { buildHash, isOneOf } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  return o;
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id });
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
   url: null,
-  crawlContainer: "div[class^='EstateItem-']",
-  sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
+  crawlContainer:
+    'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
+  sortByDateParam: 'order=DateDesc',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
-    id: 'a@id',
-    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
-    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
-    title: "div[class^='FactsMain-'] h2",
+    id: 'a@href',
+    price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
+    title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
-    address: "div[class^='estateFacts-'] span | removeNewline | trim",
+    address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
+    image: 'div[data-testid="cardMfe-card-pictureBox-opacity"] img@src',
   },
-  paginate: '#pnlPaging #nlbPlus@href',
   normalize: normalize,
   filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Immowelt',
   baseUrl: 'https://www.immowelt.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'immowelt',
 };
-
-exports.config = config;
+export { config };

lib/provider/kleinanzeigen.js

@@ -1,21 +1,22 @@
-const utils = require('../utils');
+import { buildHash, isOneOf } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
 
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
 
 function normalize(o) {
   const size = o.size || '--- m²';
-
-  return Object.assign(o, { size });
+  const id = buildHash(o.id, o.price);
+  const link = `https://www.kleinanzeigen.de${o.link}`;
+  return Object.assign(o, { id, size, link });
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
   const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
+    appliedBlacklistedDistricts.length === 0 ? false : isOneOf(o.description, appliedBlacklistedDistricts);
+  return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
@@ -23,31 +24,30 @@ const config = {
   crawlContainer: '#srchrslt-adtable .ad-listitem ',
   //sort by date is standard oO
   sortByDateParam: null,
+  waitForSelector: 'body',
   crawlFields: {
     id: '.aditem@data-adid | int',
-    price: '.aditem-main--middle--price | removeNewline | trim',
-    size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
+    price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
+    size: '.aditem-main .text-module-end | removeNewline | trim',
     title: '.aditem-main .text-module-begin a | removeNewline | trim',
     link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
-    description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
+    description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
     address: '.aditem-main--top--left | trim | removeNewline',
+    image: 'img@src',
   },
-  paginate: '#srchrslt-pagination .pagination-next@href',
   normalize: normalize,
   filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Ebay Kleinanzeigen',
-  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  baseUrl: 'https://www.kleinanzeigen.de/',
+  id: 'kleinanzeigen',
 };
-
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
+export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlacklistedDistricts = blacklistedDistricts || [];
   appliedBlackList = blacklist || [];
 };
-
-exports.config = config;
+export { config };

lib/provider/neubauKompass.js

@@ -1,41 +1,49 @@
-const utils = require('../utils');
+import { isOneOf, buildHash } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
 
 let appliedBlackList = [];
 
+function nullOrEmpty(val) {
+  return val == null || val.length === 0;
+}
+
 function normalize(o) {
-  return o;
+  const link = nullOrEmpty(o.link)
+    ? 'NO LINK'
+    : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+  const id = buildHash(o.link, o.price);
+  return Object.assign(o, { id, link });
 }
 
 function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
+  return !isOneOf(o.title, appliedBlackList);
 }
 
 const config = {
   url: null,
-  crawlContainer: '.nbk-container >div article',
+  crawlContainer: '.col-12.mb-4',
   sortByDateParam: 'Sortierung=Id&Richtung=DESC',
+  waitForSelector: 'div[data-live-name-value="SearchList"]',
   crawlFields: {
-    id: '@id',
-    title: 'a.nbk-truncate@title | removeNewline | trim',
-    link: 'a.nbk-truncate@href',
-    address: 'p.nbk-truncate | removeNewline | trim',
-    price: 'p.nbk-mb-0 | removeNewline | trim',
+    id: 'a@href',
+    title: 'a@title | removeNewline | trim',
+    link: 'a@href',
+    address: '.nbk-project-card__description | removeNewline | trim',
+    price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
+    image: '.nbk-project-card__image@src',
   },
-  paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
   normalize: normalize,
   filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Neubau Kompass',
   baseUrl: 'https://www.neubaukompass.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'neubauKompass',
 };
-
-exports.config = config;
+export { config };

lib/provider/wgGesucht.js

@@ -1,15 +1,18 @@
-const utils = require('../utils');
+import { isOneOf, buildHash } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-  return o;
+  const id = buildHash(o.id, o.price);
+  const link = `https://www.wg-gesucht.de${o.link}`;
+  const image = o.image != null ? o.image.replace('small', 'large') : null;
+  return Object.assign(o, { id, link, image });
 }
 
 function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
   return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
 
@@ -17,6 +20,7 @@ const config = {
   url: null,
   crawlContainer: '#main_column .wgg_card',
   sortByDateParam: 'sort_column=0&sort_order=0',
+  waitForSelector: 'body',
   crawlFields: {
     id: '@data-id',
     details: '.row .noprint .col-xs-11 |removeNewline |trim',
@@ -24,21 +28,20 @@ const config = {
     size: '.middle .text-right |removeNewline |trim',
     title: '.truncate_title a |removeNewline |trim',
     link: '.truncate_title a@href',
+    image: '.img-responsive@src',
   },
   normalize: normalize,
   filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
 };
-
-exports.init = (sourceConfig, blacklist) => {
+export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };
-
-exports.metaInformation = {
+export const metaInformation = {
   name: 'Wg gesucht',
   baseUrl: 'https://www.wg-gesucht.de/',
-  id: __filename.slice(__dirname.length + 1, -3),
+  id: 'wgGesucht',
 };
-
-exports.config = config;
+export { config };

lib/services/crons/demoCleanup-cron.js

@@ -0,0 +1,23 @@
+import { removeJobsByUserId } from '../storage/jobStorage.js';
+import { config } from '../../utils.js';
+import { getUsers } from '../storage/userStorage.js';
+import logger from '../logger.js';
+import cron from 'node-cron';
+
+/**
+ * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
+ */
+export function cleanupDemoAtMidnight() {
+  cron.schedule('0 0 * * *', cleanup);
+}
+
+function cleanup() {
+  if (config.demoMode) {
+    const demoUser = getUsers(false).find((user) => user.username === 'demo');
+    if (demoUser == null) {
+      logger.error('Demo user not found, cannot remove Jobs');
+      return;
+    }
+    removeJobsByUserId(demoUser.id);
+  }
+}

lib/services/crons/listing-alive-cron.js

@@ -0,0 +1,13 @@
+import cron from 'node-cron';
+import runActiveChecker from '../listings/listingActiveService.js';
+
+async function runTask() {
+  await runActiveChecker();
+}
+
+export async function initActiveCheckerCron() {
+  //run directly on start
+  await runTask();
+  // then every day at 1 am
+  cron.schedule('0 1 * * *', runTask);
+}

lib/services/crons/tracker-cron.js

@@ -0,0 +1,17 @@
+import cron from 'node-cron';
+import { config, inDevMode } from '../../utils.js';
+import { trackMainEvent } from '../tracking/Tracker.js';
+
+async function runTask() {
+  //make sure to only send tracking events if the user gave us the green light and we are not in dev mode
+  if (config.analyticsEnabled && !inDevMode()) {
+    await trackMainEvent();
+  }
+}
+
+export async function initTrackerCron() {
+  //run directly on start
+  await runTask();
+  // then every 6 hours
+  cron.schedule('0 */6 * * *', runTask);
+}

lib/services/events/event-bus.js

@@ -0,0 +1,2 @@
+import { EventEmitter } from 'node:events';
+export const bus = new EventEmitter();

lib/services/extractor/extractor.js

@@ -0,0 +1,44 @@
+import { setDebug } from './utils.js';
+import puppeteerExtractor from './puppeteerExtractor.js';
+import { loadParser, parse } from './parser/parser.js';
+import logger from '../logger.js';
+
+const DEFAULT_OPTIONS = {
+  debug: false,
+  puppeteerTimeout: 60_000,
+  puppeteerHeadless: true,
+};
+
+export default class Extractor {
+  constructor(options) {
+    this.options = {
+      ...DEFAULT_OPTIONS,
+      ...options,
+    };
+    this.responseText = null;
+    setDebug(this.options);
+  }
+
+  /**
+   * if you are extracting data from a SPA, you must provide a selector, otherwise
+   * your response will never contain what you are really looking for
+   * @param url
+   * @param waitForSelector
+   */
+  execute = async (url, waitForSelector = null) => {
+    this.responseText = null;
+    try {
+      this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+      if (this.responseText != null) {
+        loadParser(this.responseText);
+      }
+    } catch (error) {
+      logger.error('Error trying to load page.', error);
+    }
+    return this;
+  };
+
+  parseResponseText = (crawlContainer, crawlFields, url) => {
+    return parse(crawlContainer, crawlFields, this.responseText, url);
+  };
+}

lib/services/extractor/parser/parser.js

@@ -0,0 +1,98 @@
+import * as cheerio from 'cheerio';
+import logger from '../../logger.js';
+
+let $ = null;
+
+export function loadParser(text) {
+  $ = cheerio.load(text);
+}
+
+export function parse(crawlContainer, crawlFields, text, url) {
+  if (!text) {
+    logger.debug('No content found for ', url);
+    return null;
+  }
+
+  if (!crawlContainer || !crawlFields) {
+    logger.debug('Cannot parse, selector was empty for url ', url);
+    return null;
+  }
+
+  const result = [];
+
+  if ($(crawlContainer).length === 0) {
+    logger.debug('No elements in crawl container found for url ', url);
+    return null;
+  }
+
+  $(crawlContainer).each((_, element) => {
+    const container = $(element);
+    const parsedObject = {};
+
+    // Parse fields based on crawlFields
+    for (const [key, fieldSelector] of Object.entries(crawlFields)) {
+      let value;
+
+      try {
+        const selector = fieldSelector.includes('|')
+          ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim()
+          : fieldSelector;
+
+        if (selector.includes('@')) {
+          const [sel, attr] = selector.split('@');
+          if (sel.length === 0) {
+            value = container.attr(attr.trim());
+          } else {
+            value = container.find(sel.trim()).attr(attr.trim());
+          }
+        } else {
+          value = container.find(selector.trim()).text();
+        }
+
+        // Apply modifiers if specified
+        if (fieldSelector.includes('|')) {
+          /* eslint-disable no-unused-vars */
+          const [_, ...modifiers] = fieldSelector.split('|').map((s) => s.trim());
+          /* eslint-enable no-unused-vars */
+          value = applyModifiers(value, modifiers);
+        }
+
+        parsedObject[key] = value || null;
+      } catch (error) {
+        logger.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
+        parsedObject[key] = null;
+      }
+    }
+
+    if (parsedObject.id != null) {
+      result.push(parsedObject);
+    } else {
+      logger.debug('ID not found. Not relaying object.');
+    }
+  });
+
+  return result;
+}
+
+// Helper function to apply modifiers
+function applyModifiers(value, modifiers) {
+  if (!value) return value;
+
+  modifiers.forEach((modifier) => {
+    switch (modifier) {
+      case 'int':
+        value = parseInt(value, 10);
+        break;
+      case 'trim':
+        value = value.replace(/\s+/g, ' ').trim();
+        break;
+      case 'removeNewline':
+        value = value.replace(/\n/g, ' ');
+        break;
+      default:
+        logger.warn(`Unknown modifier: ${modifier}`);
+    }
+  });
+
+  return value;
+}

lib/services/extractor/puppeteerExtractor.js

@@ -0,0 +1,95 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
+import logger from '../logger.js';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+puppeteer.use(StealthPlugin());
+
+export default async function execute(url, waitForSelector, options) {
+  let browser;
+  let page;
+  let result = null;
+  let userDataDir;
+  let removeUserDataDir = false;
+  try {
+    debug(`Sending request to ${url} using Puppeteer.`);
+
+    // Prepare a dedicated temporary userDataDir to avoid leaking /tmp/.org.chromium.* dirs
+    if (options && options.userDataDir) {
+      userDataDir = options.userDataDir;
+      removeUserDataDir = !!options.cleanupUserDataDir;
+    } else {
+      const prefix = path.join(os.tmpdir(), 'puppeteer-fredy-');
+      userDataDir = fs.mkdtempSync(prefix);
+      removeUserDataDir = true;
+    }
+
+    browser = await puppeteer.launch({
+      headless: options.puppeteerHeadless ?? true,
+      args: [
+        '--no-sandbox',
+        '--disable-gpu',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-crash-reporter',
+      ],
+      timeout: options.puppeteerTimeout || 30_000,
+      userDataDir,
+    });
+    page = await browser.newPage();
+    await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+    const response = await page.goto(url, {
+      waitUntil: 'domcontentloaded',
+    });
+    let pageSource;
+    // if we're extracting data from a SPA, we must wait for the selector
+    if (waitForSelector != null) {
+      const selectorTimeout = options?.puppeteerSelectorTimeout ?? options?.puppeteerTimeout ?? 30_000;
+      await page.waitForSelector(waitForSelector, { timeout: selectorTimeout });
+      pageSource = await page.evaluate((selector) => {
+        const el = document.querySelector(selector);
+        return el ? el.innerHTML : '';
+      }, waitForSelector);
+    } else {
+      pageSource = await page.content();
+    }
+
+    const statusCode = response.status();
+
+    if (botDetected(pageSource, statusCode)) {
+      logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
+      result = null;
+    } else {
+      result = pageSource || (await page.content());
+    }
+  } catch (error) {
+    logger.error('Error executing with puppeteer executor', error);
+    result = null;
+  } finally {
+    try {
+      if (page) {
+        await page.close();
+      }
+    } catch {
+      // ignore
+    }
+    try {
+      if (browser != null) {
+        await browser.close();
+      }
+    } catch {
+      // ignore
+    }
+    try {
+      if (removeUserDataDir && userDataDir) {
+        await fs.promises.rm(userDataDir, { recursive: true, force: true });
+      }
+    } catch {
+      // ignore
+    }
+  }
+  return result;
+}

lib/services/extractor/utils.js

@@ -0,0 +1,32 @@
+import logger from '../logger.js';
+
+let debuggingOn = false;
+
+export const DEFAULT_HEADER = {
+  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.5',
+  Connection: 'keep-alive',
+  'Upgrade-Insecure-Requests': '1',
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+};
+
+export const setDebug = (options) => {
+  debuggingOn = !!options?.debug;
+};
+
+export const debug = (message) => {
+  if (debuggingOn) {
+    logger.debug(message);
+  }
+};
+
+export const botDetected = (pageSource, statusCode) => {
+  const suspiciousStatusCodes = [403, 429];
+  const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
+
+  const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
+  const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
+
+  return detectedInSource || detectedByStatus;
+};

lib/services/immoscout/immoscout-web-translator.js

@@ -0,0 +1,207 @@
+/*
+Rent a flat
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?numberofrooms=1.0-10000.0&price=1.0-10000.0&livingspace=10.0-10000.0&pricetype=rentpermonth&enteredFrom=result_list
+*/
+
+/*
+Rent a flat:
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search
+Mobile:
+https://api.mobile.immobilienscout24.de/search/list?numberofrooms=1.5-&searchId=d7c127d8-6630-49e8-a1dd-5ae04dad454d&sorting=standard&pagesize=20&livingspace=10-500&pagenumber=1&realestatetype=apartmentrent&priceType=calculatedtotalrent&price=1-10000&publishedafter=2025-05-14T09:11:54&channel=is24&searchType=region&geocodes=/de/nordrhein-westfalen/duesseldorf&features=adKeysAndStringValues,virtualTour,contactDetails,viareporting,nextgen,calculatedTotalRent,listingsInListFirstSummary,xxlListingType,quickfilters,grouping,projectsInAllRealestateTypes,fairPrice
+*/
+
+/*
+Rent a house:
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-mieten?enteredFrom=one_step_search
+Mobile:
+https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T09:12:49&pagenumber=1&searchType=region&geocodes=/de/nordrhein-westfalen/duesseldorf&realEstateType=houserent&pagesize=300&features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&sorting=standard
+*/
+
+/*
+ buy a flat
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0&livingspace=1.0-10000.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&sorting=standard&realEstateType=apartmentbuy&pagesize=300&pagenumber=1&geocodes=/de/nordrhein-westfalen/duesseldorf&publishedafter=2025-05-14T09:14:43&searchType=region
+ */
+
+/*
+ Buy a house
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0E7&livingspace=1.0-10000.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?geocodes=/de/nordrhein-westfalen/duesseldorf&features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&searchType=region&realEstateType=housebuy&pagenumber=1&pagesize=300&sorting=standard&publishedafter=2025-05-14T09:16:28
+ */
+
+/*
+ Buy a house only in parts of a city
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/haus-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0E7&livingspace=1.0-10000.0&geocodes=1276010037,1276010014,1276010012&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/list?pagesize=20&pagenumber=1&features=adKeysAndStringValues,virtualTour,contactDetails,viareporting,grouping,nextgen,listingsInListFirstSummary,xxlListingType,quickfilters,fairPrice&sorting=standard&channel=is24&geocodes=/de/nordrhein-westfalen/duesseldorf/stadtbezirk-1&searchType=region&realestatetype=housebuy&publishedafter=2025-05-14T09:17:23
+ */
+
+/*
+ Buy a house with radius
+ Web:
+ https://www.immobilienscout24.de/Suche/radius/haus-kaufen?centerofsearchaddress=D%C3%BCsseldorf%3B%3B%3B%3B%3B%3B&numberofrooms=1.0-10000.0&price=1.0-1.0E7&livingspace=1.0-10000.0&geocoordinates=51.22496%3B6.77567%3B5.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/home/search/total?pagenumber=1&pagesize=1&geocoordinates=51.224960;6.775670;4.0&sorting=standard&searchType=radius&features=adKeysAndStringValues,virtualTour,contactDetails,grouping,nextgen,listingsInListFirstSummary,xxlListingType,fairPrice&channel=is24&realestatetype=housebuy&publishedafter=2025-05-14T09:19:43
+ */
+
+/*
+ Buy a house with shape
+ Web:
+ https://www.immobilienscout24.de/Suche/shape/haus-kaufen?shape=eW1yd0hpZGloQGBJa1NfQWFsQG9Uc1ZvVmlDbHdAZ2BAaEBjfEB5U3NWY2NCa0RvWmpwQG1KYGdCeldqU3Z4QGBAbENvQmJWaGtA&numberofrooms=1.0-100000.0&price=1.0-1.0E7&livingspace=1.0-100000.0&enteredFrom=result_list#/
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC?
+ */
+import queryString from 'query-string';
+import { nullOrEmpty } from '../../utils.js';
+
+const PARAM_NAME_MAP = {
+  heatingtypes: 'heatingtypes',
+  haspromotion: 'haspromotion',
+  numberofrooms: 'numberofrooms',
+  livingspace: 'livingspace',
+  energyefficiencyclasses: 'energyefficiencyclasses',
+  exclusioncriteria: 'exclusioncriteria',
+  equipment: 'equipment',
+  petsallowedtypes: 'petsallowedtypes',
+  price: 'price',
+  constructionyear: 'constructionyear',
+  apartmenttypes: 'apartmenttypes',
+  pricetype: 'pricetype',
+  floor: 'floor',
+  geocodes: 'geocodes',
+  geocoordinates: 'geocoordinates',
+  shape: 'shape',
+  sorting: 'sorting',
+  newbuilding: 'newbuilding',
+};
+
+const EQUIPMENT_MAP = {
+  parking: 'parking',
+  cellar: 'cellar',
+  builtinkitchen: 'builtInKitchen',
+  lift: 'lift',
+  garden: 'garden',
+  guesttoilet: 'guestToilet',
+  balcony: 'balcony',
+  handicappedaccessible: 'handicappedAccessible',
+};
+
+const REAL_ESTATE_TYPE = {
+  'haus-mieten': 'houserent',
+  'wohnung-mieten': 'apartmentrent',
+  'wohnung-kaufen': 'apartmentbuy',
+  'haus-kaufen': 'housebuy',
+};
+
+const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
+  // Category "Balkon/Terrasse"
+  'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
+  'wohnung-mit-garten-mieten': { equipment: ['garden'] },
+  // Category "Wohnungstyp"
+  'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
+  'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
+  'hochparterrewohnung-mieten': { apartmenttypes: ['raisedgroundfloor'] },
+  'etagenwohnung-mieten': { apartmenttypes: ['apartment'] },
+  'loft-mieten': { apartmenttypes: ['loft'] },
+  'maisonette-mieten': { apartmenttypes: ['maisonette'] },
+  'terrassenwohnung-mieten': { apartmenttypes: ['terracedflat'] },
+  'penthouse-mieten': { apartmenttypes: ['penthouse'] },
+  'dachgeschosswohnung-mieten': { apartmenttypes: ['roofstorey'] },
+  // Category "Ausstattung"
+  'wohnung-mit-garage-mieten': { equipment: ['parking'] },
+  'wohnung-mit-einbaukueche-mieten': { equipment: ['builtinkitchen'] },
+  'wohnung-mit-keller-mieten': { equipment: ['cellar'] },
+  // Category "Merkmale"
+  'neubauwohnung-mieten': { newbuilding: true },
+  'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
+};
+
+export function convertWebToMobile(webUrl) {
+  let url;
+  try {
+    url = new URL(webUrl);
+  } catch {
+    throw new Error(`Invalid URL: ${webUrl}`);
+  }
+
+  const segments = url.pathname.split('/');
+  if (segments[1] !== 'Suche') {
+    throw new Error(`Unexpected path format: ${url.pathname}. We're expecting to see "/Suche" in the path.`);
+  }
+
+  const realTypeKey = segments.at(-1);
+  let realType = REAL_ESTATE_TYPE[realTypeKey];
+  let additionalParamsFromWebPath;
+
+  if (!realType) {
+    // Test for seo optimized apartment path (only used on the ImmoScout web app)
+    if (WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey]) {
+      additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey];
+      realType = REAL_ESTATE_TYPE['wohnung-mieten'];
+    } else {
+      throw new Error(`Real estate type not found: ${realTypeKey}`);
+    }
+  }
+
+  if (segments.includes('shape')) {
+    throw new Error('Shape is currently not supported using Immoscout');
+  }
+
+  const { query: rawParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
+  const webParams = Object.fromEntries(
+    Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
+  );
+
+  const geocodes = `/${segments.slice(2, 5).join('/')}`;
+  const isRadius = segments.includes('radius');
+  const mobileParams = {
+    searchType: isRadius ? 'radius' : 'region',
+    realestatetype: realType,
+    ...(isRadius ? {} : { geocodes }),
+    ...additionalParamsFromWebPath,
+  };
+
+  if (webParams.geocoordinates) {
+    mobileParams.geocoordinates = webParams.geocoordinates;
+  }
+
+  for (const [key, val] of Object.entries(webParams)) {
+    if (key === 'equipment') {
+      const items = [].concat(val).flatMap((v) => `${v}`.split(','));
+      const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]];
+      mobileParams[PARAM_NAME_MAP[key]] = [
+        ...(currentEquipmentParams ?? []),
+        ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
+      ];
+    } else {
+      mobileParams[PARAM_NAME_MAP[key]] = val;
+    }
+  }
+
+  const mobileQuery = queryString.stringify(mobileParams, {
+    arrayFormat: 'comma',
+    encode: true,
+    skipEmptyString: true,
+  });
+
+  return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
+}
+
+export function convertImmoscoutListingToMobileListing(url) {
+  if (nullOrEmpty(url)) {
+    return null;
+  }
+
+  return url.replace(
+    /^https:\/\/www\.immobilienscout24\.de\/expose\//,
+    'https://api.mobile.immobilienscout24.de/expose/',
+  );
+}

lib/services/listings/listingActiveService.js

@@ -0,0 +1,104 @@
+import { deactivateListings, getActiveOrUnknownListings } from '../storage/listingsStorage.js';
+import { getProviders } from '../../utils.js';
+import logger from '../../services/logger.js';
+
+/**
+ * Runs the active-listing checker:
+ * 1) Loads all listings with unknown or active status.
+ * 2) Resolves each listing's provider and calls its `activeTester(link)`.
+ * 3) Collects listings that are no longer active and deactivates them in one batch.
+ *
+ * Concurrency: network-bound checks are executed with a configurable concurrency limit.
+ *
+ * @param {object} [opts]
+ * @param {number} [opts.concurrency=8] Max number of parallel activeTester calls.
+ * @returns {Promise<void>}
+ */
+export default async function runActiveChecker(opts = {}) {
+  const { concurrency = 4 } = opts;
+
+  const listings = getActiveOrUnknownListings();
+  if (!Array.isArray(listings) || listings.length === 0) {
+    logger.debug('No listings to check.');
+    return;
+  }
+
+  const providers = await getProviders();
+  if (!Array.isArray(providers) || providers.length === 0) {
+    logger.warn('No providers available. Skipping active checks.');
+    return;
+  }
+
+  // Build a map for O(1) provider lookup by id
+  /** @type {Record<string, any>} */
+  const providerById = Object.create(null);
+  for (const p of providers) {
+    const id = p?.metaInformation?.id;
+    if (id) providerById[id] = p;
+  }
+
+  // Small generic mapLimit to cap concurrency without extra deps
+  /**
+   * @template T, R
+   * @param {T[]} items
+   * @param {number} limit
+   * @param {(item: T, index: number) => Promise<R>} worker
+   * @returns {Promise<R[]>}
+   */
+  async function mapLimit(items, limit, worker) {
+    const results = new Array(items.length);
+    let next = 0;
+
+    async function runOne() {
+      while (next < items.length) {
+        const i = next++;
+        try {
+          results[i] = await worker(items[i], i);
+        } catch (err) {
+          results[i] = /** @type {any} */ (err);
+        }
+      }
+    }
+
+    const runners = Array.from({ length: Math.min(limit, items.length) }, runOne);
+    await Promise.all(runners);
+    return results;
+  }
+
+  /** @type {string[]} */
+  const listingsSetToInactive = [];
+
+  await mapLimit(listings, concurrency, async (listing) => {
+    const { provider: listingProviderId, link, id } = listing || {};
+
+    const matchedProvider = providerById[listingProviderId];
+    if (!matchedProvider) {
+      logger.warn('Could not find matching provider for', listingProviderId);
+      return;
+    }
+    const tester = matchedProvider?.config?.activeTester;
+    if (typeof tester !== 'function') {
+      logger.warn('No activeTester configured for', listingProviderId);
+      return;
+    }
+
+    // Contract: activeTester(link) returns 1 if active, 0 if inactive
+    let result;
+    try {
+      result = await tester(link);
+    } catch {
+      result = -1;
+    }
+
+    if (result === 0 && id) {
+      listingsSetToInactive.push(id);
+    }
+  });
+
+  if (listingsSetToInactive.length > 0) {
+    logger.info(`Setting ${listingsSetToInactive.length} listings to inactive.`);
+    deactivateListings(listingsSetToInactive);
+  } else {
+    logger.debug('No listings need to be set inactive.');
+  }
+}

lib/services/listings/listingActiveTester.js

@@ -0,0 +1,68 @@
+import fetch from 'node-fetch';
+import { randomBetween, sleep } from '../../utils.js';
+
+const maxAttempts = 3;
+
+/**
+ * Check if a listing is still active with up to 3 attempts and exponential backoff.
+ * Backoff waits are capped and the last wait is at most 2000 ms.
+ *
+ * Rules:
+ * - HTTP 200 => return 1
+ * - HTTP 401/403 => return -1 (most certainly detected as a bot)
+ * - HTTP 404 => return 0
+ * - Other statuses or network errors => retry until attempts are exhausted
+ *
+ * @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
+ */
+export default async function checkIfListingIsActive(link) {
+  await sleep(randomBetween(50, 100));
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const res = await fetch(link, {
+        headers: {
+          'User-Agent':
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
+          'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
+        },
+      });
+
+      if (res.status === 200) {
+        return 1;
+      }
+      if (res.status === 401) return -1;
+      if (res.status === 403) return -1;
+      if (res.status === 404) return 0;
+
+      // For any other status, only retry if attempts remain
+      if (attempt < maxAttempts) {
+        await sleep(backoffDelay(attempt));
+        continue;
+      }
+
+      return 0;
+    } catch {
+      // Network error: retry if attempts remain
+      if (attempt < maxAttempts) {
+        await sleep(backoffDelay(attempt));
+        continue;
+      }
+      return 0;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Exponential backoff delay with cap.
+ * attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
+ * @param {number} attempt 1-based attempt index
+ * @returns {number} delay in ms
+ */
+function backoffDelay(attempt) {
+  const base = 500;
+  const cap = 2000;
+  return Math.min(base * 2 ** (attempt - 1), cap);
+}

lib/services/logger.js

@@ -0,0 +1,59 @@
+const COLORS = {
+  debug: '\x1b[36m',
+  info: '\x1b[32m',
+  warn: '\x1b[33m',
+  error: '\x1b[31m',
+  reset: '\x1b[0m',
+};
+
+const env = process.env.NODE_ENV || 'development';
+const useColor = process.stdout.isTTY || process.stderr.isTTY;
+
+function ts() {
+  const d = new Date();
+  const yyyy = d.getFullYear();
+  const mm = String(d.getMonth() + 1).padStart(2, '0');
+  const dd = String(d.getDate()).padStart(2, '0');
+  const hh = String(d.getHours()).padStart(2, '0');
+  const mi = String(d.getMinutes()).padStart(2, '0');
+  const ss = String(d.getSeconds()).padStart(2, '0');
+  return `${yyyy}-${mm}-${dd} ${hh}:${mi}:${ss}`;
+}
+
+function lvl(level) {
+  const upper = level.toUpperCase();
+  if (!useColor) return upper;
+  return `${COLORS[level] || ''}${upper}${COLORS.reset}`;
+}
+
+/* eslint-disable no-console */
+function log(level, ...args) {
+  if (level === 'debug' && env !== 'development') {
+    return; // Skip debug logs in non-development environments
+  }
+
+  const prefix = `[${ts()}] ${lvl(level)}:`;
+  switch (level) {
+    case 'debug':
+      console.debug(prefix, ...args);
+      break;
+    case 'info':
+      console.info(prefix, ...args);
+      break;
+    case 'warn':
+      console.warn(prefix, ...args);
+      break;
+    case 'error':
+      console.error(prefix, ...args);
+      break;
+    default:
+      console.log(prefix, ...args);
+  }
+}
+
+export default {
+  debug: (...a) => log('debug', ...a),
+  info: (...a) => log('info', ...a),
+  warn: (...a) => log('warn', ...a),
+  error: (...a) => log('error', ...a),
+};

lib/services/markdown.js

@@ -1,6 +1,6 @@
-const markdown = require('markdown').markdown;
-const fs = require('fs');
-
-exports.markdown2Html = function markdown2Html(filePath) {
+import markdown$0 from 'markdown';
+import fs from 'fs';
+const markdown = markdown$0.markdown;
+export function markdown2Html(filePath) {
   return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
-};
+}

lib/services/queryStringMutator.js

@@ -1,21 +1,9 @@
-const queryString = require('query-string');
-
-/**
- * for Fredy, it is important to sort search results by date, starting with the latest listing. if it is not sorted, we
- * might never actually find the newest results, no matter how many pages we crawl.
- * It has been written in the documentation, but obviously nobody reads docu theses days which is why it's been done
- * automagically now.
- *
- * @param _url actual provider url containing the searchParams
- * @param sortByDateParam param(s) indicating the correct sort order
- * @returns {`${string}?${string}`} correctly formatted url
- */
-module.exports = (_url, sortByDateParam) => {
+import queryString from 'query-string';
+export default (_url, sortByDateParam) => {
   //if no mutation is necessary, just return the original url
   if (sortByDateParam == null) {
     return _url;
   }
-
   const original = queryString.parseUrl(_url);
   const mutate = queryString.parse(sortByDateParam);
   return `${original.url}?${queryString.stringify({ ...original.query, ...mutate })}`;

lib/services/requestDriver.js

@@ -1,37 +0,0 @@
-const axios = require('axios');
-const axiosRetry = require('axios-retry');
-
-axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
-
-function makeDriver(headers = {}) {
-  let cookies = '';
-
-  return async function driver(context, callback) {
-    const url = context.url;
-    let result;
-    try {
-      result = await axios({
-        url,
-        headers: {
-          ...headers,
-          Cookie: cookies,
-        },
-      });
-    } catch (exception) {
-      console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
-      callback(null, []);
-    }
-
-    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
-      //assume we have gotten a response from scrapingAnt
-      if (cookies.length === 0) {
-        cookies = result.data.cookies;
-      }
-      callback(null, result.data.content);
-    } else {
-      callback(null, result.data);
-    }
-  };
-}
-
-module.exports = makeDriver;