starting docu on reverse engineering immoscout api (#127)

* starting docu on reverse engineering immoscout api * improving immoscout reverse engineering and adding support for most other types
2026-06-16 12:31:07 +00:00 · 2025-05-14 13:58:58 +02:00
parent 3aae81ca19
commit 030e0ca169
12 changed files with 1244 additions and 1119 deletions
--- a/lib/provider/immonet.js
+++ b/lib/provider/immonet.js
@@ -26,7 +26,7 @@ const config = {
  url: null,
  crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
  sortByDateParam: 'sortby=19',
-  waitForSelector: 'div[data-testid="serp-resultscount-testid"]',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
  crawlFields: {
    id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
    title: 'button@title |trim',
--- a/lib/provider/immoscout.js
+++ b/lib/provider/immoscout.js
@@ -33,15 +33,10 @@
 * but I have decided not to include new parameters as I wanted to keep the existing UX (i.e.,
 * users only have to provide a link to an existing search).
 *
- * Limitations:
- * - The current implementation of this provider *does not* support non-rental properties,
- *   although the same approach can be used to implement support. It's just a matter of
- *   mapping the web search URL to the corresponding mobile API URL.
- * - Pagination support is not implemented.
 */

 import utils, { buildHash } from '../utils.js';
-import queryString from 'query-string';
+import { convertWebToMobile } from '../services/immoscout/immoscout-web-translater.js';
 let appliedBlackList = [];

 async function getListings(url) {
@@ -92,8 +87,6 @@ function applyBlacklist(o) {
 }
 const config = {
  url: null,
-  sortByDateParam: 'sorting=-firstactivation',
-  // Not actually required - used by filter to remove and listings that failed to parse
  crawlFields: {
    id: 'id',
    title: 'title',
@@ -102,6 +95,8 @@ const config = {
    link: 'link',
    address: 'address',
  },
+  // Not required - used by filter to remove and listings that failed to parse
+  sortByDateParam: 'sorting=-firstactivation',
  normalize: normalize,
  filter: applyBlacklist,
  getListings: getListings,
@@ -117,89 +112,4 @@ export const metaInformation = {
  id: 'immoscout',
 };

-export function convertWebToMobile(webUrl) {
-  let url;
-  try {
-    url = new URL(webUrl);
-  } catch (err) {
-    throw new Error(`Invalid URL: ${webUrl}`);
-  }
-  const segments = url.pathname.split('/');
-  if (segments.length < 6 || segments[1] !== 'Suche') {
-    throw new Error(`Unexpected path format: ${url.pathname}`);
-  }
-  const geocodes = `/${segments[2]}/${segments[3]}/${segments[4]}`;
-
-  const paramNameMap = {
-    heatingtypes: 'heatingtypes',
-    haspromotion: 'haspromotion',
-    numberofrooms: 'numberofrooms',
-    livingspace: 'livingspace',
-    energyefficiencyclasses: 'energyefficiencyclasses',
-    exclusioncriteria: 'exclusioncriteria',
-    equipment: 'equipment',
-    petsallowedtypes: 'petsallowedtypes',
-    price: 'price',
-    constructionyear: 'constructionyear',
-    apartmenttypes: 'apartmenttypes',
-    pricetype: 'pricetype',
-    floor: 'floor',
-  };
-
-  const equipmentValueMap = {
-    parking: 'parking',
-    cellar: 'cellar',
-    builtinkitchen: 'builtInKitchen',
-    lift: 'lift',
-    garden: 'garden',
-    guesttoilet: 'guestToilet',
-    balcony: 'balcony',
-  };
-
-  const { query: webParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
-  delete webParams['enteredFrom'];
-
-  // Remove unsupported parameters
-  Object.keys(webParams).forEach((key) => {
-    if (!paramNameMap[key]) {
-      delete webParams[key];
-    }
-  });
-
-  // Build mobile params
-  const mobileParams = {
-    searchType: 'region',
-    geocodes,
-    realestatetype: 'apartmentrent',
-  };
-
-  Object.entries(webParams).forEach(([webKey, webVal]) => {
-    let value = webVal;
-
-    if (webKey === 'equipment') {
-      // Map equipment list to camelCase values
-      if (!Array.isArray(value)) {
-        value = ('' + value).split(',');
-      }
-      value = value.map((token) => {
-        const lower = token.toLowerCase();
-        if (!equipmentValueMap[lower]) {
-          throw new Error(`Unknown equipment type: "${token}"`);
-        }
-        return equipmentValueMap[lower];
-      });
-    }
-
-    mobileParams[paramNameMap[webKey]] = value;
-  });
-
-  const mobileQuery = queryString.stringify(mobileParams, {
-    arrayFormat: 'comma',
-    encode: true,
-    skipEmptyString: true,
-  });
-
-  return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
-}
-
 export { config };
--- a/lib/services/immoscout/immoscout-web-translater.js
+++ b/lib/services/immoscout/immoscout-web-translater.js
@@ -0,0 +1,157 @@
+/*
+Rent a flat
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?numberofrooms=1.0-10000.0&price=1.0-10000.0&livingspace=10.0-10000.0&pricetype=rentpermonth&enteredFrom=result_list
+*/
+
+/*
+Rent a flat:
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-mieten?enteredFrom=one_step_search
+Mobile:
+https://api.mobile.immobilienscout24.de/search/list?numberofrooms=1.5-&searchId=d7c127d8-6630-49e8-a1dd-5ae04dad454d&sorting=standard&pagesize=20&livingspace=10-500&pagenumber=1&realestatetype=apartmentrent&priceType=calculatedtotalrent&price=1-10000&publishedafter=2025-05-14T09:11:54&channel=is24&searchType=region&geocodes=/de/nordrhein-westfalen/duesseldorf&features=adKeysAndStringValues,virtualTour,contactDetails,viareporting,nextgen,calculatedTotalRent,listingsInListFirstSummary,xxlListingType,quickfilters,grouping,projectsInAllRealestateTypes,fairPrice
+*/
+
+/*
+Rent a house:
+Web:
+https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-mieten?enteredFrom=one_step_search
+Mobile:
+https://api.mobile.immobilienscout24.de/search/map/v3?publishedafter=2025-05-14T09:12:49&pagenumber=1&searchType=region&geocodes=/de/nordrhein-westfalen/duesseldorf&realEstateType=houserent&pagesize=300&features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&sorting=standard
+*/
+
+/*
+ buy a flat
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/wohnung-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0&livingspace=1.0-10000.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&sorting=standard&realEstateType=apartmentbuy&pagesize=300&pagenumber=1&geocodes=/de/nordrhein-westfalen/duesseldorf&publishedafter=2025-05-14T09:14:43&searchType=region
+ */
+
+/*
+ Buy a house
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0E7&livingspace=1.0-10000.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?geocodes=/de/nordrhein-westfalen/duesseldorf&features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&searchType=region&realEstateType=housebuy&pagenumber=1&pagesize=300&sorting=standard&publishedafter=2025-05-14T09:16:28
+ */
+
+/*
+ Buy a house only in parts of a city
+ Web:
+ https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/haus-kaufen?numberofrooms=1.0-10000.0&price=1.0-10000.0E7&livingspace=1.0-10000.0&geocodes=1276010037,1276010014,1276010012&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/list?pagesize=20&pagenumber=1&features=adKeysAndStringValues,virtualTour,contactDetails,viareporting,grouping,nextgen,listingsInListFirstSummary,xxlListingType,quickfilters,fairPrice&sorting=standard&channel=is24&geocodes=/de/nordrhein-westfalen/duesseldorf/stadtbezirk-1&searchType=region&realestatetype=housebuy&publishedafter=2025-05-14T09:17:23
+ */
+
+/*
+ Buy a house with radius
+ Web:
+ https://www.immobilienscout24.de/Suche/radius/haus-kaufen?centerofsearchaddress=D%C3%BCsseldorf%3B%3B%3B%3B%3B%3B&numberofrooms=1.0-10000.0&price=1.0-1.0E7&livingspace=1.0-10000.0&geocoordinates=51.22496%3B6.77567%3B5.0&enteredFrom=result_list
+ Mobile:
+ https://api.mobile.immobilienscout24.de/home/search/total?pagenumber=1&pagesize=1&geocoordinates=51.224960;6.775670;4.0&sorting=standard&searchType=radius&features=adKeysAndStringValues,virtualTour,contactDetails,grouping,nextgen,listingsInListFirstSummary,xxlListingType,fairPrice&channel=is24&realestatetype=housebuy&publishedafter=2025-05-14T09:19:43
+ */
+
+/*
+ Buy a house with shape
+ Web:
+ https://www.immobilienscout24.de/Suche/shape/haus-kaufen?shape=eW1yd0hpZGloQGBJa1NfQWFsQG9Uc1ZvVmlDbHdAZ2BAaEBjfEB5U3NWY2NCa0RvWmpwQG1KYGdCeldqU3Z4QGBAbENvQmJWaGtA&numberofrooms=1.0-100000.0&price=1.0-1.0E7&livingspace=1.0-100000.0&enteredFrom=result_list#/
+ Mobile:
+ https://api.mobile.immobilienscout24.de/search/map/v3?features=disableNHBGrouping,nextGen,fairPrice,listingsInListFirstSummary,xxlListingType,contactDetails&publishedafter=2025-05-14T09:19:43&sorting=standard&pagesize=300&searchType=shape&realEstateType=housebuy&pagenumber=1&shape=%7D%7BjwHy%7Cqh@jCKdCgAvB_BdB%7DBzAaCjAqCfAqC~@uCt@iCh@eCZkCLyC?_EO%7DEa@%7DEa@iE_@%7BD%5DaDe@gDi@gDo@uCu@kBcB_AeDOiE?iDCgCMuBOkDCkG?yFRgD%60@cB%5C%7BA%60@eBx@aB%7C@kAbAy@rAe@bBUxCAhE?dFh@fGlAzGbBbHlBxGdB%60FrAhDz@xBh@nAf@l@RNNXkCkMJR~B%7CEnCpErCnDtClCvC~ApCh@rCJpC?
+ */
+import queryString from 'query-string';
+
+const PARAM_NAME_MAP = {
+  heatingtypes: 'heatingtypes',
+  haspromotion: 'haspromotion',
+  numberofrooms: 'numberofrooms',
+  livingspace: 'livingspace',
+  energyefficiencyclasses: 'energyefficiencyclasses',
+  exclusioncriteria: 'exclusioncriteria',
+  equipment: 'equipment',
+  petsallowedtypes: 'petsallowedtypes',
+  price: 'price',
+  constructionyear: 'constructionyear',
+  apartmenttypes: 'apartmenttypes',
+  pricetype: 'pricetype',
+  floor: 'floor',
+  geocodes: 'geocodes',
+  geocoordinates: 'geocoordinates',
+  shape: 'shape',
+  sorting: 'sorting',
+};
+
+const EQUIPMENT_MAP = {
+  parking: 'parking',
+  cellar: 'cellar',
+  builtinkitchen: 'builtInKitchen',
+  lift: 'lift',
+  garden: 'garden',
+  guesttoilet: 'guestToilet',
+  balcony: 'balcony',
+};
+
+const REAL_ESTATE_TYPE = {
+  'haus-mieten': 'houserent',
+  'wohnung-mieten': 'apartmentrent',
+  'wohnung-kaufen': 'apartmentbuy',
+  'haus-kaufen': 'housebuy',
+};
+
+export function convertWebToMobile(webUrl) {
+  let url;
+  try {
+    url = new URL(webUrl);
+  } catch {
+    throw new Error(`Invalid URL: ${webUrl}`);
+  }
+
+  const segments = url.pathname.split('/');
+  if (segments[1] !== 'Suche') {
+    throw new Error(`Unexpected path format: ${url.pathname}. We're expecting to see "/Suche" in the path.`);
+  }
+
+  const realTypeKey = segments.at(-1);
+  const realType = REAL_ESTATE_TYPE[realTypeKey];
+  if (!realType) {
+    throw new Error(`Real estate type not found: ${realTypeKey}`);
+  }
+
+  if (segments.includes('shape')) {
+    throw new Error('Shape is currently not supported using Immoscout');
+  }
+
+  const { query: rawParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
+  const webParams = Object.fromEntries(
+    Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
+  );
+
+  const geocodes = `/${segments.slice(2, 5).join('/')}`;
+  const isRadius = segments.includes('radius');
+  const mobileParams = {
+    searchType: isRadius ? 'radius' : 'region',
+    realestatetype: realType,
+    ...(isRadius ? {} : { geocodes }),
+  };
+
+  if (webParams.geocoordinates) {
+    mobileParams.geocoordinates = webParams.geocoordinates;
+  }
+
+  for (const [key, val] of Object.entries(webParams)) {
+    if (key === 'equipment') {
+      const items = [].concat(val).flatMap((v) => `${v}`.split(','));
+      mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean);
+    } else {
+      mobileParams[PARAM_NAME_MAP[key]] = val;
+    }
+  }
+
+  const mobileQuery = queryString.stringify(mobileParams, {
+    arrayFormat: 'comma',
+    encode: true,
+    skipEmptyString: true,
+  });
+
+  return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
+}