starting docu on reverse engineering immoscout api (#127)

* starting docu on reverse engineering immoscout api

* improving immoscout reverse engineering and adding support for most other types

lib/provider/immonet.js

@@ -26,7 +26,7 @@ const config = {
   url: null,
   crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
   sortByDateParam: 'sortby=19',
-  waitForSelector: 'div[data-testid="serp-resultscount-testid"]',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
     id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
     title: 'button@title |trim',

lib/provider/immoscout.js

@@ -33,15 +33,10 @@
  * but I have decided not to include new parameters as I wanted to keep the existing UX (i.e.,
  * users only have to provide a link to an existing search).
  *
- * Limitations:
- * - The current implementation of this provider *does not* support non-rental properties,
- *   although the same approach can be used to implement support. It's just a matter of
- *   mapping the web search URL to the corresponding mobile API URL.
- * - Pagination support is not implemented.
  */
 
 import utils, { buildHash } from '../utils.js';
-import queryString from 'query-string';
+import { convertWebToMobile } from '../services/immoscout/immoscout-web-translater.js';
 let appliedBlackList = [];
 
 async function getListings(url) {
@@ -92,8 +87,6 @@ function applyBlacklist(o) {
 }
 const config = {
   url: null,
-  sortByDateParam: 'sorting=-firstactivation',
-  // Not actually required - used by filter to remove and listings that failed to parse
   crawlFields: {
     id: 'id',
     title: 'title',
@@ -102,6 +95,8 @@ const config = {
     link: 'link',
     address: 'address',
   },
+  // Not required - used by filter to remove and listings that failed to parse
+  sortByDateParam: 'sorting=-firstactivation',
   normalize: normalize,
   filter: applyBlacklist,
   getListings: getListings,
@@ -117,89 +112,4 @@ export const metaInformation = {
   id: 'immoscout',
 };
 
-export function convertWebToMobile(webUrl) {
-  let url;
-  try {
-    url = new URL(webUrl);
-  } catch (err) {
-    throw new Error(`Invalid URL: ${webUrl}`);
-  }
-  const segments = url.pathname.split('/');
-  if (segments.length < 6 || segments[1] !== 'Suche') {
-    throw new Error(`Unexpected path format: ${url.pathname}`);
-  }
-  const geocodes = `/${segments[2]}/${segments[3]}/${segments[4]}`;
-
-  const paramNameMap = {
-    heatingtypes: 'heatingtypes',
-    haspromotion: 'haspromotion',
-    numberofrooms: 'numberofrooms',
-    livingspace: 'livingspace',
-    energyefficiencyclasses: 'energyefficiencyclasses',
-    exclusioncriteria: 'exclusioncriteria',
-    equipment: 'equipment',
-    petsallowedtypes: 'petsallowedtypes',
-    price: 'price',
-    constructionyear: 'constructionyear',
-    apartmenttypes: 'apartmenttypes',
-    pricetype: 'pricetype',
-    floor: 'floor',
-  };
-
-  const equipmentValueMap = {
-    parking: 'parking',
-    cellar: 'cellar',
-    builtinkitchen: 'builtInKitchen',
-    lift: 'lift',
-    garden: 'garden',
-    guesttoilet: 'guestToilet',
-    balcony: 'balcony',
-  };
-
-  const { query: webParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
-  delete webParams['enteredFrom'];
-
-  // Remove unsupported parameters
-  Object.keys(webParams).forEach((key) => {
-    if (!paramNameMap[key]) {
-      delete webParams[key];
-    }
-  });
-
-  // Build mobile params
-  const mobileParams = {
-    searchType: 'region',
-    geocodes,
-    realestatetype: 'apartmentrent',
-  };
-
-  Object.entries(webParams).forEach(([webKey, webVal]) => {
-    let value = webVal;
-
-    if (webKey === 'equipment') {
-      // Map equipment list to camelCase values
-      if (!Array.isArray(value)) {
-        value = ('' + value).split(',');
-      }
-      value = value.map((token) => {
-        const lower = token.toLowerCase();
-        if (!equipmentValueMap[lower]) {
-          throw new Error(`Unknown equipment type: "${token}"`);
-        }
-        return equipmentValueMap[lower];
-      });
-    }
-
-    mobileParams[paramNameMap[webKey]] = value;
-  });
-
-  const mobileQuery = queryString.stringify(mobileParams, {
-    arrayFormat: 'comma',
-    encode: true,
-    skipEmptyString: true,
-  });
-
-  return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
-}
-
 export { config };