Puppeteer rewrite (#119)

* Moving to puppeteer | removing scrapingAnt

lib/services/extractor/extractor.js

@@ -0,0 +1,45 @@
+import {setDebug} from './utils.js';
+import puppeteerExtractor from './puppeteerExtractor.js';
+import {loadParser, parse} from './parser/parser.js';
+
+const DEFAULT_OPTIONS = {
+    debug: false,
+    puppeteerTimeout: 20_000,
+    puppeteerHeadless: true
+
+};
+
+export default class Extractor {
+    constructor(options) {
+        this.options = {
+            ...DEFAULT_OPTIONS,
+            ...options
+        };
+        this.responseText = null;
+        setDebug(this.options);
+    }
+
+    /**
+     * if you are extracting data from a SPA, you must provide a selector, otherwise
+     * your response will never contain what you are really looking for
+     * @param url
+     * @param waitForSelector
+     */
+    execute = async (url, waitForSelector = null) => {
+        this.responseText = null;
+        try {
+            this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+            if(this.responseText != null) {
+                loadParser(this.responseText);
+            }
+        } catch (error) {
+            console.error('Error trying to load page.', error);
+        }
+        return this;
+    };
+
+
+    parseResponseText = (crawlContainer, crawlFields) => {
+        return parse(crawlContainer, crawlFields, this.responseText);
+    };
+}

lib/services/extractor/parser/parser.js

@@ -0,0 +1,94 @@
+import * as cheerio from 'cheerio';
+
+let $ = null;
+
+export function loadParser(text) {
+    $ = cheerio.load(text);
+}
+
+export function parse(crawlContainer, crawlFields, text) {
+    if (!text) {
+        console.warn('Cannot parse, text was empty.');
+        return null;
+    }
+
+    if (!crawlContainer || !crawlFields) {
+        console.warn('Cannot parse, selector was empty.');
+        return null;
+    }
+
+    const result = [];
+
+    if ($(crawlContainer).length === 0) {
+        console.error('No elements in crawl container found!');
+    }
+
+    $(crawlContainer).each((_, element) => {
+        const container = $(element);
+        const parsedObject = {};
+
+        // Parse fields based on crawlFields
+        for (const [key, fieldSelector] of Object.entries(crawlFields)) {
+            let value;
+
+            try {
+
+                const selector = fieldSelector.includes('|') ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim() : fieldSelector;
+
+                if (selector.includes('@')) {
+                    const [sel, attr] = selector.split('@');
+                    if (sel.length === 0) {
+                        value = container.attr(attr.trim());
+                    } else {
+                        value = container.find(sel.trim()).attr(attr.trim());
+                    }
+                } else {
+                    value = container.find(selector.trim()).text();
+                }
+
+                // Apply modifiers if specified
+                if (fieldSelector.includes('|')) {
+                    const [_, ...modifiers] = fieldSelector.split('|').map(s => s.trim());
+                    value = applyModifiers(value, modifiers);
+                }
+
+                parsedObject[key] = value || null;
+            } catch (error) {
+                console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
+                parsedObject[key] = null;
+            }
+        }
+
+        if (parsedObject.id != null) {
+            result.push(parsedObject);
+        } else {
+            console.warn('ID not found. Not relaying object.');
+        }
+    });
+
+    return result;
+}
+
+// Helper function to apply modifiers
+function applyModifiers(value, modifiers) {
+    if (!value) return value;
+
+    modifiers.forEach(modifier => {
+        switch (modifier) {
+            case 'int':
+                value = parseInt(value, 10);
+                break;
+            case 'trim':
+                value = value.replace(/\s+/g, ' ').trim();
+                break;
+            case 'removeNewline':
+                value = value.replace(/\n/g, ' ');
+                break;
+            default:
+                console.warn(`Unknown modifier: ${modifier}`);
+        }
+    });
+
+    return value;
+}
+

lib/services/extractor/puppeteerExtractor.js

@@ -0,0 +1,48 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import {debug, DEFAULT_HEADER, botDetected} from './utils.js';
+
+puppeteer.use(StealthPlugin());
+
+export default async function execute(url, waitForSelector, options) {
+    let browser;
+    try {
+        debug(`Sending request to ${url} using Puppeteer.`);
+
+        browser = await puppeteer.launch({
+            headless: options.puppeteerHeadless ?? true,
+            args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox']
+        });
+        let page = await browser.newPage();
+        await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+        const response = await page.goto(url, {
+            waitUntil: 'domcontentloaded'
+        });
+        let pageSource;
+        //if we're extracting data from a spa, we must wait for the selector
+        if (waitForSelector != null) {
+            await page.waitForSelector(waitForSelector);
+            pageSource = await page.evaluate(selector => {
+                return document.querySelector(selector).innerHTML;
+            }, waitForSelector);
+        } else {
+            pageSource = await page.content();
+        }
+
+        const statusCode = response.status();
+
+        if (botDetected(pageSource, statusCode)) {
+            console.warn('We have been detected as a bot :-/ Tried url: => ', url);
+            return null;
+        }
+
+        return await page.content();
+    } catch (error) {
+        console.error('Error executing with puppeteer executor', error);
+        return null;
+    } finally {
+        if (browser != null) {
+            await browser.close();
+        }
+    }
+}

lib/services/extractor/utils.js

@@ -0,0 +1,35 @@
+let debuggingOn = false;
+
+export const DEFAULT_HEADER = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
+};
+
+export const setDebug = options => {
+    debuggingOn = !!options?.debug;
+};
+
+export const debug = (message) => {
+    if(debuggingOn) {
+        console.debug(message);
+    }
+};
+
+export const botDetected = (pageSource, statusCode) => {
+    const suspiciousStatusCodes = [
+        403, 429
+    ];
+    const botDetectionPatterns = [
+        /verify you are human/i,
+        /access denied/i,
+        /x-amz-cf-id/i,
+    ];
+
+    const detectedInSource = botDetectionPatterns.some(pattern => pattern.test(pageSource));
+    const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
+
+    return detectedInSource || detectedByStatus;
+};

lib/services/requestDriver.js

@@ -1,77 +0,0 @@
-import fetch from 'node-fetch';
-import { config } from '../utils.js';
-import { makeUrlResidential } from './scrapingAnt.js';
-import https from 'https';
-//if ScrapingAnt got blocked, this http status is returned
-const BLOCKED_HTTP_STATUS = 423;
-const NOT_FOUND_HTTP_STATUS = 404;
-const MAX_RETRIES_SCRAPING_ANT = 10;
-const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
-const agent = new https.Agent({
-  rejectUnauthorized: false,
-});
-
-function makeDriver(headers = {}) {
-  let cookies = '';
-  async function scrapingAntDriver(context, callback, retryCounter = 0) {
-    const proxyType = config.scrapingAnt?.proxy || 'datacenter';
-    try {
-      const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
-      const response = await fetch(url, {
-        headers: {
-          ...headers,
-          cookie: cookies,
-        },
-      });
-      const result = await response.text();
-      if (EXPECTED_STATUS_CODES.includes(response.status)) {
-        throw new Error(`${response.status}`);
-      }
-      if (cookies.length === 0) {
-        cookies = response.headers.raw()['set-cookie'] || [];
-      }
-      callback(null, result);
-    } catch (exception) {
-      /* eslint-disable no-console */
-      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status) && !EXPECTED_STATUS_CODES.includes(Number(exception.message))) {
-        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
-        callback(null, []);
-        return;
-      }
-      if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
-        retryCounter++;
-        console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
-        await scrapingAntDriver(context, callback, retryCounter);
-      } else {
-        console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
-        callback(null, []);
-      }
-      /* eslint-enable no-console */
-    }
-  }
-
-  /**
-   * The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
-   * everything != Immoscout & Immonet as of writing this)
-   */
-  return async function driver(context, callback) {
-    if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
-      return scrapingAntDriver(context, callback);
-    }
-    try {
-      const response = await fetch(context.url, {
-        headers: {
-          ...headers,
-          Cookie: cookies,
-        },
-        agent,
-      });
-      const result = await response.text();
-      callback(null, result);
-    } catch (exception) {
-      console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
-      callback(null, []);
-    }
-  };
-}
-export default makeDriver;

lib/services/scraper.js

@@ -1,36 +0,0 @@
-import { config } from '../utils.js';
-import makeDriver from './requestDriver.js';
-import Xray from 'x-ray';
-class Scraper {
-  constructor() {
-    const filters = {
-      removeNewline: this._removeNewline,
-      trim: this._trim,
-      int: this._int,
-    };
-    const headers = {
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-    };
-    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
-      headers['x-api-key'] = config.scrapingAnt.apiKey;
-    }
-    const driver = makeDriver(headers);
-    const xray = Xray({ filters });
-    xray.driver(driver);
-    this.xray = xray;
-  }
-  get x() {
-    return this.xray;
-  }
-  _removeNewline(value) {
-    return typeof value === 'string' ? value.replace(/\\n/g, '') : value;
-  }
-  _trim(value) {
-    return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : value;
-  }
-  _int(value) {
-    return typeof value === 'string' ? parseInt(value, 10) : value;
-  }
-}
-export default new Scraper().x;

lib/services/scrapingAnt.js

@@ -1,30 +0,0 @@
-import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
-import { metaInformation as immoNetInfo } from '../provider/immonet.js';
-import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
-import { config } from '../utils.js';
-
-const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
-  'window.scrollTo(0,document.body.scrollHeight);'
-).toString('base64')}`;
-
-const needScrapingAnt = (id) => {
-  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
-};
-export const transformUrlForScrapingAnt = (url, id) => {
-  let urlParams = '';
-  if (needScrapingAnt(id)) {
-    if (id.toLowerCase() === immoNetInfo.id) {
-      urlParams = additionalImmonetUrlParams;
-    }
-    //only do calls to scrapingAnt when dealing with Immoscout/Immonet
-    url = `https://api.scrapingant.com/v2/general?url=${encodeURIComponent(url)}&proxy_type=datacenter${urlParams}`;
-  }
-  return url;
-};
-export const isScrapingAntApiKeySet = () => {
-  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 8;
-};
-export const makeUrlResidential = (url) => {
-  return url.replace('datacenter', 'residential');
-};
-export { needScrapingAnt };