next version

* Add provider that uses the immoscout mobile API to avoid failing bot checks.

.github/workflows/test.yml

@@ -1,23 +1,23 @@
 name: Test
 on:
   push:
-    branches:
+    branches: [master]
-      - master
   pull_request:
-    branches:
+    branches: [master]
-      - master
   schedule:
     - cron: '0 12 * * *'
+
 jobs:
   test:
-    name: Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
-      - name: Setup node
+
-        uses: actions/setup-node@v2.5.1
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'yarn'
+
       - run: yarn install
-      - run: yarn run test
+      - run: yarn test

README.md

@@ -11,7 +11,7 @@ If _Fredy_ finds matching results, it will send them to you via Slack, Email, Te
 # Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
 If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
 
-<img src="https://github.com/orangecoding/fredy/blob/master/doc/jetbrains.png" width="200">   
+[![JetBrains logo.](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg)](https://jb.gg/OpenSourceSupport)
 
 _Fredy_ is supported by JetBrains under Open Source Support Program
 
@@ -82,7 +82,7 @@ yarn run test
 ![Architecture](/doc/architecture.jpg "Architecture")
 
 ### Immoscout
-Immoscout has implemented advanced bot detection. I’m actively working on bypassing these measures, but until then, selecting Immoscout as a provider will not return any results. I apologize for the inconvenience. 😉
+Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. For now, only real estate rentals are supported. Purchases will be supported at a later point in time.
 
 # Analytics
 Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 

lib/FredyRuntime.js

@@ -26,7 +26,7 @@ class FredyRuntime {
       //modify the url to make sure search order is correctly set
       Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
         //scraping the site and try finding new listings
-        .then(this._getListings.bind(this))
+        .then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
         //bring them in a proper form (dictated by the provider)
         .then(this._normalize.bind(this))
         //filter listings with stuff tagged by the blacklist of the provider

lib/notification/adapter/ntfy.js

@@ -7,9 +7,11 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   const promises = newListings.map((newListing) => {
-    const message = `Address: ${newListing.address} Size: ${newListing.size.replace(/2m/g, '$m^2$')} Price: ${
+    const message = `
-      newListing.price
+    Address: ${newListing.address} 
-    }`;
+    Size: ${newListing.size.replace(/2m/g, '$m^2$')} 
+    Price: ${newListing.price}
+    Link: ${newListing.link}`;
     return fetch(server, {
       method: 'POST',
       body: JSON.stringify({

lib/notification/adapter/pushover.js

@@ -1,50 +1,73 @@
-import { markdown2Html } from '../../services/markdown.js';
+import {markdown2Html} from '../../services/markdown.js';
-import { getJob } from '../../services/storage/jobStorage.js';
+import {getJob} from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+export const send = ({serviceName, newListings, notificationConfig, jobKey}) => {
-  const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+    const {token, user, device} = notificationConfig.find((adapter) => adapter.id === config.id).fields;
-  const job = getJob(jobKey);
+    const job = getJob(jobKey);
-  const jobName = job == null ? jobKey : job.name;
+    const jobName = job == null ? jobKey : job.name;
-  const promises = newListings.map((newListing) => {
+    const promises = newListings.map((newListing) => {
-    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+        const title = `${jobName} at ${serviceName}: ${newListing.title}`;
-    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+        const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
-    return fetch('https://api.pushover.net/1/messages.json', {
+        return fetch('https://api.pushover.net/1/messages.json', {
-      method: 'POST',
+            method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
+            headers: {'Content-Type': 'application/json'},
-      body: JSON.stringify({
+            body: JSON.stringify({
-        token: token,
+                token: token,
-        user: user,
+                user: user,
-        message: message,
+                message: message,
-        device: device,
+                device: device,
-        title: title,
+                title: title,
-      }),
+            }),
+        });
     });
-  });
 
-  return Promise.all(promises);
+    return Promise.all(promises)
+        .then((responses) => {
+            // Convert all responses to JSON
+            return Promise.all(responses.map((response) => response.json()));
+        })
+        .then((data) => {
+            // Check for errors in the data
+            const error = data
+                .map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
+                .filter((err) => err !== null);
+
+            if (error.length > 0) {
+                // Reject with the combined error messages
+                return Promise.reject(error.join('; '));
+            }
+
+            return data;
+        })
+        .then(() => {
+            return Promise.resolve();
+        })
+        .catch((error) => {
+            return Promise.reject(error);
+        });
 };
 
 export const config = {
-  id: 'pushover',
+    id: 'pushover',
-  name: 'Pushover',
+    name: 'Pushover',
-  readme: markdown2Html('lib/notification/adapter/pushover.md'),
+    readme: markdown2Html('lib/notification/adapter/pushover.md'),
-  description: 'Fredy will send new listings to your mobile using Pushover.',
+    description: 'Fredy will send new listings to your mobile using Pushover.',
-  fields: {
+    fields: {
-    token: {
+        token: {
-      type: 'text',
+            type: 'text',
-      label: 'API token',
+            label: 'API token',
-      description: 'Your application\'s API token.',
+            description: 'Your application\'s API token.',
+        },
+        user: {
+            type: 'text',
+            label: 'User key',
+            description: 'Your user/group key.',
+        },
+        device: {
+            type: 'text',
+            label: 'Device name',
+            description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
+        },
     },
-    user: {
-      type: 'text',
-      label: 'User key',
-      description: 'Your user/group key.',
-    },
-    device: {
-      type: 'text',
-      label: 'Device name',
-      description: 'The device name to send your notification to. Messages may be addressed to multiple specific devices by joining them with a comma.',
-    },
-  },
 };

lib/provider/immoscout.js

@@ -1,37 +1,114 @@
-import utils, {buildHash} from '../utils.js';
+/**
+ * ImmoScout provider using the mobile API to retrieve listings.
+ *
+ * The mobile API provides the following endpoints:
+ * - GET /search/total?{search parameters}: Returns the total number of listings for the given query
+ *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
+ *
+ * - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
+ *   data specifying additional results (advertisements) to return. The format is as follows:
+ *   ```
+ *   {
+ *   "supportedResultListTypes": [],
+ *   "userData": {}
+ *   }
+ *   ```
+ *   It is not necessary to provide data for the specified keys.
+ *
+ *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
+
+ * - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
+ *   listing response.
+ *
+ *   Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
+ *
+ *
+ * It is necessary to set the correct User Agent (see `getListings`) in the request header.
+ *
+ * Note that the mobile API is not publicly documented. I've reverse-engineered
+ * it by intercepting traffic from an android emulator running the immoscout app.
+ * Moreover, the search parameters differ slightly from the web API. I've mapped them
+ * to the web API parameters by comparing a search request with all parameters set between
+ * the web and mobile API. The mobile API actually seems to be a superset of the web API,
+ * but I have decided not to include new parameters as I wanted to keep the existing UX (i.e.,
+ * users only have to provide a link to an existing search).
+ *
+ * Limitations:
+ * - The current implementation of this provider *does not* support non-rental properties,
+ *   although the same approach can be used to implement support. It's just a matter of
+ *   mapping the web search URL to the corresponding mobile API URL.
+ * - Pagination support is not implemented.
+ */
+
+import utils, { buildHash } from '../utils.js';
+import queryString from 'query-string';
 let appliedBlackList = [];
+
+async function getListings(url) {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'User-Agent': 'ImmoScout24_1410_30_._',
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      supportedResultListTypes: [],
+      userData: {},
+    }),
+  });
+  if (!response.ok) {
+    console.error('Error fetching data from ImmoScout Mobile API:', response.statusText);
+    return [];
+  }
+
+  const responseBody = await response.json();
+  return responseBody.resultListItems
+    .filter((item) => item.type === 'EXPOSE_RESULT')
+    .map((expose) => {
+      const item = expose.item;
+      const [price, size] = item.attributes;
+      return {
+        id: item.id,
+        price: price?.value,
+        size: size?.value,
+        title: item.title,
+        link: `${metaInformation.baseUrl}expose/${item.id}`,
+        address: item.address?.line,
+      };
+    });
+}
+
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
   const id = buildHash(o.id, o.price);
-  return Object.assign(o, { id, title, address, link });
+  return Object.assign(o, { id, title, address });
 }
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
 const config = {
   url: null,
-  crawlContainer: '#resultListItems li.result-list__listing',
+  sortByDateParam: 'sorting=-firstactivation',
-  sortByDateParam: 'sorting=2',
+  // Not actually required - used by filter to remove and listings that failed to parse
-  waitForSelector: 'body',
   crawlFields: {
-    id: '.result-list-entry@data-obid | int',
+    id: 'id',
-    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
+    title: 'title',
-    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
+    price: 'price',
-    title: '.result-list-entry .result-list-entry__brand-title-container h2 | removeNewline | trim',
+    size: 'size',
-    link: '.result-list-entry .result-list-entry__brand-title-container@href',
+    link: 'link',
-    address: '.result-list-entry .result-list-entry__map-link',
+    address: 'address',
   },
   normalize: normalize,
   filter: applyBlacklist,
+  getListings: getListings,
 };
 export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
+  config.url = convertWebToMobile(sourceConfig.url);
   appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
@@ -39,4 +116,90 @@ export const metaInformation = {
   baseUrl: 'https://www.immobilienscout24.de/',
   id: 'immoscout',
 };
+
+export function convertWebToMobile(webUrl) {
+  let url;
+  try {
+    url = new URL(webUrl);
+  } catch (err) {
+    throw new Error(`Invalid URL: ${webUrl}`);
+  }
+  const segments = url.pathname.split('/');
+  if (segments.length < 6 || segments[1] !== 'Suche') {
+    throw new Error(`Unexpected path format: ${url.pathname}`);
+  }
+  const geocodes = `/${segments[2]}/${segments[3]}/${segments[4]}`;
+
+  const paramNameMap = {
+    heatingtypes: 'heatingtypes',
+    haspromotion: 'haspromotion',
+    numberofrooms: 'numberofrooms',
+    livingspace: 'livingspace',
+    energyefficiencyclasses: 'energyefficiencyclasses',
+    exclusioncriteria: 'exclusioncriteria',
+    equipment: 'equipment',
+    petsallowedtypes: 'petsallowedtypes',
+    price: 'price',
+    constructionyear: 'constructionyear',
+    apartmenttypes: 'apartmenttypes',
+    pricetype: 'pricetype',
+    floor: 'floor',
+  };
+
+  const equipmentValueMap = {
+    parking: 'parking',
+    cellar: 'cellar',
+    builtinkitchen: 'builtInKitchen',
+    lift: 'lift',
+    garden: 'garden',
+    guesttoilet: 'guestToilet',
+    balcony: 'balcony',
+  };
+
+  const { query: webParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
+  delete webParams['enteredFrom'];
+
+  // Remove unsupported parameters
+  Object.keys(webParams).forEach((key) => {
+    if (!paramNameMap[key]) {
+      delete webParams[key];
+    }
+  });
+
+  // Build mobile params
+  const mobileParams = {
+    searchType: 'region',
+    geocodes,
+    realestatetype: 'apartmentrent',
+  };
+
+  Object.entries(webParams).forEach(([webKey, webVal]) => {
+    let value = webVal;
+
+    if (webKey === 'equipment') {
+      // Map equipment list to camelCase values
+      if (!Array.isArray(value)) {
+        value = ('' + value).split(',');
+      }
+      value = value.map((token) => {
+        const lower = token.toLowerCase();
+        if (!equipmentValueMap[lower]) {
+          throw new Error(`Unknown equipment type: "${token}"`);
+        }
+        return equipmentValueMap[lower];
+      });
+    }
+
+    mobileParams[paramNameMap[webKey]] = value;
+  });
+
+  const mobileQuery = queryString.stringify(mobileParams, {
+    arrayFormat: 'comma',
+    encode: true,
+    skipEmptyString: true,
+  });
+
+  return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
+}
+
 export { config };

lib/provider/immoswp.js

@@ -1,48 +1,48 @@
-import utils, {buildHash} from '../utils.js';
+import utils, { buildHash } from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-    const size = o.size || 'N/A m²';
+  const size = o.size || 'N/A m²';
-    const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
+  const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
-    const title = o.title || 'No title available';
+  const title = o.title || 'No title available';
-    const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+  const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
-    const link = `https://immo.swp.de/immobilien/${immoId}`;
+  const link = `https://immo.swp.de/immobilien/${immoId}`;
-    const description = o.description;
+  const description = o.description;
-    const id = buildHash(immoId, price);
+  const id = buildHash(immoId, price);
-    return Object.assign(o, {id, price, size, title, link, description});
+  return Object.assign(o, { id, price, size, title, link, description });
 }
 
 function applyBlacklist(o) {
-    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-    return titleNotBlacklisted && descNotBlacklisted;
+  return titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-    url: null,
+  url: null,
-    crawlContainer: '.js-serp-item',
+  crawlContainer: '.js-serp-item',
-    sortByDateParam: 's=most_recently_updated_first',
+  sortByDateParam: 's=most_recently_updated_first',
-    waitForSelector: 'body',
+  waitForSelector: 'body',
-    crawlFields: {
+  crawlFields: {
-        id: '.js-bookmark-btn@data-id',
+    id: '.js-bookmark-btn@data-id',
-        price: 'div.align-items-start div:first-child | trim',
+    price: 'div.align-items-start div:first-child | trim',
-        size: 'div.align-items-start div:nth-child(3) | trim',
+    size: 'div.align-items-start div:nth-child(3) | trim',
-        title: '.card-title h2 | trim',
+    title: '.js-item-title-link@title | trim',
-        link: '.ci-search-result__link@href',
+    link: '.ci-search-result__link@href',
-        description: '.js-show-more-item-sm | removeNewline | trim',
+    description: '.js-show-more-item-sm | removeNewline | trim',
-    },
+  },
-    normalize: normalize,
+  normalize: normalize,
-    filter: applyBlacklist,
+  filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-    config.enabled = sourceConfig.enabled;
+  config.enabled = sourceConfig.enabled;
-    config.url = sourceConfig.url;
+  config.url = sourceConfig.url;
-    appliedBlackList = blacklist || [];
+  appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-    name: 'Immo Südwest Presse',
+  name: 'Immo Südwest Presse',
-    baseUrl: 'https://immo.swp.de/',
+  baseUrl: 'https://immo.swp.de/',
-    id: 'immoswp',
+  id: 'immoswp',
 };
-export {config};
+export { config };

lib/provider/immowelt.js

@@ -23,7 +23,7 @@ const config = {
     id: 'a@href',
     price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: '.css-1cbj9xw',
+    title: '.css-jv3zx6',
     link: 'a@href',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },

lib/provider/wgGesucht.js

@@ -4,7 +4,8 @@ let appliedBlackList = [];
 
 function normalize(o) {
     const id = buildHash(o.id, o.price);
-    return Object.assign(o, {id});
+    const link = `https://www.wg-gesucht.de${o.link}`;
+    return Object.assign(o, { id, link });
 }
 
 function applyBlacklist(o) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.0.1",
+  "version": "11.1.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node prod.js",
@@ -50,12 +50,12 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.72.3",
+    "@douyinfe/semi-ui": "2.75.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.4",
     "@vitejs/plugin-react": "4.3.4",
-    "better-sqlite3": "^11.7.2",
+    "better-sqlite3": "^11.8.1",
     "body-parser": "1.20.3",
     "cheerio": "^1.0.0",
     "cookie-session": "2.1.0",
@@ -66,11 +66,11 @@
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
     "mixpanel": "^0.18.0",
-    "nanoid": "5.0.9",
+    "nanoid": "5.1.2",
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.6",
     "package-up": "^5.0.0",
-    "puppeteer": "^23.11.1",
+    "puppeteer": "^24.2.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "query-string": "9.1.1",
@@ -88,21 +88,21 @@
     "vite": "5.4.11"
   },
   "devDependencies": {
-    "@babel/core": "7.26.0",
+    "@babel/core": "7.26.9",
-    "@babel/eslint-parser": "7.25.9",
+    "@babel/eslint-parser": "7.26.8",
-    "@babel/preset-env": "7.26.0",
+    "@babel/preset-env": "7.26.9",
     "@babel/preset-react": "7.26.3",
-    "chai": "5.1.2",
+    "chai": "5.2.0",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.37.3",
+    "eslint-plugin-react": "7.37.4",
-    "esmock": "2.6.9",
+    "esmock": "2.7.0",
     "history": "5.3.0",
     "husky": "9.1.7",
-    "less": "4.2.1",
+    "less": "4.2.2",
-    "lint-staged": "15.3.0",
+    "lint-staged": "15.4.3",
     "mocha": "10.8.2",
-    "prettier": "3.4.2",
+    "prettier": "3.5.2",
     "redux-logger": "3.0.6"
   }
 }

test/provider/immonet.test.js

@@ -1,40 +1,38 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
+import {get} from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
+import {mockFredy, providerConfig} from '../utils.js';
-import { expect } from 'chai';
+import {expect} from 'chai';
 import * as provider from '../../lib/provider/immonet.js';
 
 describe('#immonet testsuite()', () => {
-  after(() => {
+    after(() => {
-    similarityCache.stopCacheCleanup();
+        similarityCache.stopCacheCleanup();
-  });
+    });
-  provider.init(providerConfig.immonet, [], []);
+    provider.init(providerConfig.immonet, [], []);
-  it('should test immonet provider', async () => {
+    it('should test immonet provider', async () => {
-    const Fredy = await mockFredy();
+        const Fredy = await mockFredy();
-    return await new Promise((resolve) => {
+        return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
-      fredy.execute().then((listing) => {
+            fredy.execute().then((listing) => {
-        expect(listing).to.be.a('array');
+                expect(listing).to.be.a('array');
-        const notificationObj = get();
+                const notificationObj = get();
-        expect(notificationObj).to.be.a('object');
+                expect(notificationObj).to.be.a('object');
-        expect(notificationObj.serviceName).to.equal('immonet');
+                expect(notificationObj.serviceName).to.equal('immonet');
-        notificationObj.payload.forEach((notify) => {
+                notificationObj.payload.forEach((notify) => {
-          /** check the actual structure **/
+                    /** check the actual structure **/
-          expect(notify.id).to.be.a('string');
+                    expect(notify.id).to.be.a('string');
-          expect(notify.price).to.be.a('string');
+                    expect(notify.price).to.be.a('string');
-          expect(notify.size).to.be.a('string');
+                    expect(notify.size).to.be.a('string');
-          expect(notify.title).to.be.a('string');
+                    expect(notify.title).to.be.a('string');
-          expect(notify.link).to.be.a('string');
+                    expect(notify.link).to.be.a('string');
-          expect(notify.address).to.be.a('string');
+                    expect(notify.address).to.be.a('string');
-
+
-          /** check the values if possible **/
+                    expect(notify.size).that.does.include('m²');
-          expect(notify.price).that.does.include('€');
+                    expect(notify.title).to.be.not.empty;
-          expect(notify.size).that.does.include('m²');
+                    expect(notify.address).to.be.not.empty;
-          expect(notify.title).to.be.not.empty;
+                });
-          expect(notify.address).to.be.not.empty;
+                resolve();
-        });
+            });
-        resolve();
+        });
-      });
     });
-  });
 });

test/provider/immoscout.test.js

@@ -1,43 +1,73 @@
+import { expect } from 'chai';
+import { convertWebToMobile } from '../../lib/provider/immoscout.js';
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-//import {get} from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
-import {/*mockFredy, */providerConfig} from '../utils.js';
+import { get } from '../mocks/mockNotification.js';
-//import {expect} from 'chai';
 import * as provider from '../../lib/provider/immoscout.js';
 
-describe('#immoscout testsuite()', () => {
+describe('#immoscout provider testsuite()', () => {
-    after(() => {
+  after(() => {
-        similarityCache.stopCacheCleanup();
+    similarityCache.stopCacheCleanup();
-    });
+  });
-    provider.init(providerConfig.immoscout, [], []);
+
-    it('should test immoscout provider', async () => {
+  provider.init(providerConfig.immoscout, [], []);
-        //const Fredy = await mockFredy();
+  it('should test immoscout provider', async () => {
-        return await new Promise((resolve) => {
+    const Fredy = await mockFredy();
-            /* eslint-disable no-console */
+    return await new Promise((resolve) => {
-            console.info('Skipping Immoscout test for now until we figured out how to surpass bot detection.');
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, '', similarityCache);
-            /* eslint-enable no-console */
+      fredy.execute().then((listings) => {
-            resolve();
+        expect(listings).to.be.a('array');
-            /*
+        const notificationObj = get();
-            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoscout', similarityCache);
+        expect(notificationObj).to.be.a('object');
-            fredy.execute().then((listing) => {
+        expect(notificationObj.serviceName).to.equal('immoscout');
-                expect(listing).to.be.a('array');
+        notificationObj.payload.forEach((notify) => {
-                const notificationObj = get();
+          /** check the actual structure **/
-                expect(notificationObj).to.be.a('object');
+          expect(notify.id).to.be.a('string');
-                expect(notificationObj.serviceName).to.equal('immoscout');
+          expect(notify.price).to.be.a('string');
-                notificationObj.payload.forEach((notify) => {
+          expect(notify.size).to.be.a('string');
-                    expect(notify.id).to.be.a('number');
+          expect(notify.title).to.be.a('string');
-                    expect(notify.price).to.be.a('string');
+          expect(notify.link).to.be.a('string');
-                    expect(notify.size).to.be.a('string');
+          expect(notify.address).to.be.a('string');
-                    expect(notify.title).to.be.a('string');
+          /** check the values if possible **/
-                    expect(notify.link).to.be.a('string');
+          expect(notify.size).to.be.not.empty;
-                    expect(notify.address).to.be.a('string');
+          expect(notify.title).to.be.not.empty;
-                    expect(notify.price).that.does.include('€');
+          expect(notify.link).that.does.include('https://www.immobilienscout24.de/');
-                    expect(notify.size).that.does.include('m²');
-                    expect(notify.title).to.be.not.empty;
-                    expect(notify.link).that.does.include('https://www.immobilienscout24.de');
-                    expect(notify.address).to.be.not.empty;
-                });
-                resolve();
-            });*/
         });
+        resolve();
+      });
     });
+  });
+});
+
+describe('#immoscout-mobile URL conversion', () => {
+  // Test URL conversion
+  it('should convert a full web URL to mobile URL', () => {
+    const webUrl =
+      'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?heatingtypes=central,selfcontainedcentral&haspromotion=false&numberofrooms=2.0-5.0&livingspace=10.0-25.0&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&exclusioncriteria=projectlisting,swapflat&equipment=parking,cellar,builtinkitchen,lift,garden,guesttoilet,balcony&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&constructionyear=1920-2026&apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&pricetype=calculatedtotalrent&floor=2-7&enteredFrom=result_list';
+    const expectedMobileUrl =
+      'https://api.mobile.immobilienscout24.de/search/list?apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&constructionyear=1920-2026&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&equipment=parking,cellar,builtInKitchen,lift,garden,guestToilet,balcony&exclusioncriteria=projectlisting,swapflat&floor=2-7&geocodes=%2Fde%2Fberlin%2Fberlin&haspromotion=false&heatingtypes=central,selfcontainedcentral&livingspace=10.0-25.0&numberofrooms=2.0-5.0&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region';
+
+    const actualMobileUrl = convertWebToMobile(webUrl);
+    expect(actualMobileUrl).to.equal(expectedMobileUrl);
+  });
+
+  // Test URL conversion with unsupported query parameters
+  it('should remove unsupported query parameters', () => {
+    const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';
+    const converted = convertWebToMobile(webUrl);
+    expect(converted).that.does.not.include('minimuminternetspeed');
+  });
+
+  // Test URL conversion with invalid URL
+  it('should throw an error for invalid URL', () => {
+    const invalidUrl = 'invalid-url';
+
+    expect(() => convertWebToMobile(invalidUrl)).to.throw('Invalid URL: invalid-url');
+  });
+
+  // Test URL conversion with unexpected path format
+  it('should throw an error for unexpected path format', () => {
+    const webUrl = 'https://www.immobilienscout24.de/invalid/path/format';
+    expect(() => convertWebToMobile(webUrl)).to.throw('Unexpected path format: /invalid/path/format');
+  });
 });

test/queryStringMutator/testData.json

@@ -20,11 +20,6 @@
     "shouldBecome": "https://www.immonet.de/immobiliensuche/sel.do?sortby=19&suchart=1&objecttype=1&marketingtype=2&parentcat=1&locationname=d%C3%BCsseldorf",
     "id": "immonet"
   },
-  {
-    "url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten",
-    "shouldBecome": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?sorting=2",
-    "id": "immoscout"
-  },
   {
     "url": "https://www.neubaukompass.de/neubau-immobilien/berlin-region/",
     "shouldBecome": "https://www.neubaukompass.de/neubau-immobilien/berlin-region/?Sortierung=Id&Richtung=DESC",

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,10 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              Immoscout will not work at the moment due to advanced bot detection. I'm currently working on a fix.
+              Currently, Immoscout only works for real estate rentals. Purchases are not yet supported.
-            </p>
-            <p>
-              Until a fix has been released, Immoscout won't yield any results.
             </p>
           </div>
         }