fixing immoscout scraper

lib/provider/immoscout.js

@@ -8,7 +8,7 @@
  *
  * The mobile API provides the following endpoints:
  * - GET /search/total?{search parameters}: Returns the total number of listings for the given query
- *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
+ *   Example: `curl -H "User-Agent: ImmoScout_27.12_26.2_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
  *
  * - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
  *   data specifying additional results (advertisements) to return. The format is as follows:
@@ -20,12 +20,12 @@
  *   ```
  *   It is not necessary to provide data for the specified keys.
  *
- *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
+ *   Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.12_26.2_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
 
  * - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
  *   listing response.
  *
- *   Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
+ *   Example: `curl -H "User-Agent: ImmoScout_27.12_26.2_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
  *
  *
  * It is necessary to set the correct User Agent (see `getListings`) in the request header.
@@ -52,7 +52,7 @@ async function getListings(url) {
   const response = await fetch(url, {
     method: 'POST',
     headers: {
-      'User-Agent': 'ImmoScout_27.3_26.0_._',
+      'User-Agent': 'ImmoScout_27.12_26.2_._',
       'Content-Type': 'application/json',
     },
     body: JSON.stringify({
@@ -88,7 +88,7 @@ async function getListings(url) {
 async function isListingActive(link) {
   const result = await fetch(convertImmoscoutListingToMobileListing(link), {
     headers: {
-      'User-Agent': 'ImmoScout_27.3_26.0_._',
+      'User-Agent': 'ImmoScout_27.12_26.2_._',
     },
   });
 

lib/provider/sparkasse.js

@@ -36,7 +36,7 @@ const config = {
   },
   normalize: normalize,
   filter: applyBlacklist,
-  activeTester: checkIfListingIsActive,
+  activeTester: (url) => checkIfListingIsActive(url, 'Angebot nicht gefunden'),
 };
 export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;

lib/services/immoscout/immoscout-web-translator.js

@@ -103,6 +103,8 @@ const REAL_ESTATE_TYPE = {
   'haus-mieten': 'houserent',
   'wohnung-mieten': 'apartmentrent',
   'wohnung-kaufen': 'apartmentbuy',
+  'wohnung-kaufen-mit-balkon': 'apartmentbuy',
+  'eigentumswohnung-mit-garten': 'apartmentbuy',
   'haus-kaufen': 'housebuy',
 };
 
@@ -146,7 +148,7 @@ export function convertWebToMobile(webUrl) {
 
   const realTypeKey = segments.at(-1);
   let realType = REAL_ESTATE_TYPE[realTypeKey];
-  let additionalParamsFromWebPath;
+  let additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey] || null;
 
   if (!realType) {
     // Test for seo optimized apartment path (only used on the ImmoScout web app)
@@ -167,7 +169,7 @@ export function convertWebToMobile(webUrl) {
     Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
   );
 
-  const geocodes = `/${segments.slice(2, 5).join('/')}`;
+  const geocodes = `/${segments.slice(2, segments.length - 1).join('/')}`;
   const isRadius = segments.includes('radius');
   const mobileParams = {
     searchType: isRadius ? 'radius' : 'region',

lib/services/listings/listingActiveTester.js

@@ -8,38 +8,71 @@ import { randomBetween, sleep } from '../../utils.js';
 
 const maxAttempts = 3;
 
+const userAgents = [
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
+  'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
+];
+
 /**
- * Check if a listing is still active with up to 3 attempts and exponential backoff.
- * Backoff waits are capped and the last wait is at most 2000 ms.
+ * Check if a listing is still active with up to 5 attempts and exponential backoff.
+ * Backoff waits are randomized and capped.
  *
  * Rules:
- * - HTTP 200 => return 1
+ * - HTTP 200 => return 1 (if checkForText is provided and found, returns 0)
  * - HTTP 401/403 => return -1 (most certainly detected as a bot)
  * - HTTP 404 => return 0
  * - Other statuses or network errors => retry until attempts are exhausted
  *
- * @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
+ * @returns {Promise<Integer>} 1 if active, 0 if not active and -1 if detected as bot
  */
-export default async function checkIfListingIsActive(link) {
+export default async function checkIfListingIsActive(link, checkForText = null) {
   await sleep(randomBetween(50, 100));
 
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
+      const userAgent = userAgents[Math.floor(Math.random() * userAgents.length)];
       const res = await fetch(link, {
         redirect: 'manual',
         headers: {
-          'User-Agent':
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
-          'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
+          'User-Agent': userAgent,
+          Accept:
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+          'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
+          'Accept-Encoding': 'gzip, deflate, br',
+          'Cache-Control': 'max-age=0',
+          'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+          'Sec-Ch-Ua-Mobile': '?0',
+          'Sec-Ch-Ua-Platform': '"macOS"',
+          'Sec-Fetch-Dest': 'document',
+          'Sec-Fetch-Mode': 'navigate',
+          'Sec-Fetch-Site': 'none',
+          'Sec-Fetch-User': '?1',
+          'Upgrade-Insecure-Requests': '1',
+          Referer: 'https://www.google.com/',
         },
       });
 
       if (res.status === 200) {
+        if (checkForText) {
+          const htmText = await res.text();
+          if (htmText.includes(checkForText)) {
+            return 0;
+          }
+        }
+
         return 1;
       }
-      if (res.status === 401) return -1;
-      if (res.status === 403) return -1;
-      if (res.status === 404) return 0;
+      if (res.status === 401 || res.status === 403) {
+        if (attempt < maxAttempts) {
+          await sleep(backoffDelay(attempt));
+          continue;
+        }
+        return -1;
+      }
+      if (res.status === 404 || res.status === 410) return 0;
 
       // For any other status, only retry if attempts remain
       if (attempt < maxAttempts) {
@@ -62,13 +95,13 @@ export default async function checkIfListingIsActive(link) {
 }
 
 /**
- * Exponential backoff delay with cap.
- * attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
+ * Exponential backoff delay with cap and jitter.
  * @param {number} attempt 1-based attempt index
  * @returns {number} delay in ms
  */
 function backoffDelay(attempt) {
   const base = 500;
   const cap = 2000;
-  return Math.min(base * 2 ** (attempt - 1), cap);
+  const delay = Math.min(base * 2 ** (attempt - 1), cap);
+  return delay + randomBetween(0, 1000);
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "19.2.0",
+  "version": "19.2.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",

reverse-engineered-immoscout.md

@@ -41,7 +41,7 @@ Challenges:
 _Returns the total number of listings for the given query._
 
 ```
-curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
+curl -H "User-Agent: ImmoScout_27.12_26.2_._" \
      -H "Accept: application/json" \
      "https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
 ```
@@ -63,7 +63,7 @@ _The body is json encoded and contains data specifying additional results (adver
 ```
 curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
   -H "Connection: keep-alive" \
-  -H "User-Agent: ImmoScout_27.3_26.0_._" \
+  -H "User-Agent: ImmoScout_27.12_26.2_._" \
   -H "Accept: application/json" \
   -H "Content-Type: application/json" \
   -d '{"supportedResultListType":[],"userData":{}}'
@@ -78,7 +78,7 @@ curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calc
 The response contains additional details not included in the listing response.
 
 ```
-curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
+curl -H "User-Agent: ImmoScout_27.12_26.2_._" \
      -H "Accept: application/json" \
      "https://api.mobile.immobilienscout24.de/expose/158382494"
 ```

test/services/immoscout/immoscout-web-translator.test.js

@@ -58,7 +58,7 @@ describe('#immoscout-mobile URL conversion', () => {
       const response = await fetch(url, {
         method: 'POST',
         headers: {
-          'User-Agent': 'ImmoScout_27.3_26.0_._',
+          'User-Agent': 'ImmoScout_27.12_26.2_._',
           'Content-Type': 'application/json',
         },
         body: JSON.stringify({
@@ -75,7 +75,9 @@ describe('#immoscout-mobile URL conversion', () => {
       expect(responseBody.totalResults).to.be.greaterThan(0);
       expect(responseBody.totalResults).to.be.greaterThan(0);
       expect(responseBody.resultListItems.length).to.greaterThan(0);
-      expect(responseBody.resultListItems[0].item.realEstateType).to.equal(type);
+      expect(responseBody.resultListItems.filter((r) => r.type === 'EXPOSE_RESULT')[0].item.realEstateType).to.equal(
+        type,
+      );
     }
   });
 });