Puppeteer rewrite (#119)

* Moving to puppeteer | removing scrapingAnt

lib/services/extractor/extractor.js

@@ -0,0 +1,45 @@
+import {setDebug} from './utils.js';
+import puppeteerExtractor from './puppeteerExtractor.js';
+import {loadParser, parse} from './parser/parser.js';
+
+const DEFAULT_OPTIONS = {
+    debug: false,
+    puppeteerTimeout: 20_000,
+    puppeteerHeadless: true
+
+};
+
+export default class Extractor {
+    constructor(options) {
+        this.options = {
+            ...DEFAULT_OPTIONS,
+            ...options
+        };
+        this.responseText = null;
+        setDebug(this.options);
+    }
+
+    /**
+     * if you are extracting data from a SPA, you must provide a selector, otherwise
+     * your response will never contain what you are really looking for
+     * @param url
+     * @param waitForSelector
+     */
+    execute = async (url, waitForSelector = null) => {
+        this.responseText = null;
+        try {
+            this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+            if(this.responseText != null) {
+                loadParser(this.responseText);
+            }
+        } catch (error) {
+            console.error('Error trying to load page.', error);
+        }
+        return this;
+    };
+
+
+    parseResponseText = (crawlContainer, crawlFields) => {
+        return parse(crawlContainer, crawlFields, this.responseText);
+    };
+}

lib/services/extractor/parser/parser.js

@@ -0,0 +1,94 @@
+import * as cheerio from 'cheerio';
+
+let $ = null;
+
+export function loadParser(text) {
+    $ = cheerio.load(text);
+}
+
+export function parse(crawlContainer, crawlFields, text) {
+    if (!text) {
+        console.warn('Cannot parse, text was empty.');
+        return null;
+    }
+
+    if (!crawlContainer || !crawlFields) {
+        console.warn('Cannot parse, selector was empty.');
+        return null;
+    }
+
+    const result = [];
+
+    if ($(crawlContainer).length === 0) {
+        console.error('No elements in crawl container found!');
+    }
+
+    $(crawlContainer).each((_, element) => {
+        const container = $(element);
+        const parsedObject = {};
+
+        // Parse fields based on crawlFields
+        for (const [key, fieldSelector] of Object.entries(crawlFields)) {
+            let value;
+
+            try {
+
+                const selector = fieldSelector.includes('|') ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim() : fieldSelector;
+
+                if (selector.includes('@')) {
+                    const [sel, attr] = selector.split('@');
+                    if (sel.length === 0) {
+                        value = container.attr(attr.trim());
+                    } else {
+                        value = container.find(sel.trim()).attr(attr.trim());
+                    }
+                } else {
+                    value = container.find(selector.trim()).text();
+                }
+
+                // Apply modifiers if specified
+                if (fieldSelector.includes('|')) {
+                    const [_, ...modifiers] = fieldSelector.split('|').map(s => s.trim());
+                    value = applyModifiers(value, modifiers);
+                }
+
+                parsedObject[key] = value || null;
+            } catch (error) {
+                console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
+                parsedObject[key] = null;
+            }
+        }
+
+        if (parsedObject.id != null) {
+            result.push(parsedObject);
+        } else {
+            console.warn('ID not found. Not relaying object.');
+        }
+    });
+
+    return result;
+}
+
+// Helper function to apply modifiers
+function applyModifiers(value, modifiers) {
+    if (!value) return value;
+
+    modifiers.forEach(modifier => {
+        switch (modifier) {
+            case 'int':
+                value = parseInt(value, 10);
+                break;
+            case 'trim':
+                value = value.replace(/\s+/g, ' ').trim();
+                break;
+            case 'removeNewline':
+                value = value.replace(/\n/g, ' ');
+                break;
+            default:
+                console.warn(`Unknown modifier: ${modifier}`);
+        }
+    });
+
+    return value;
+}
+

lib/services/extractor/puppeteerExtractor.js

@@ -0,0 +1,48 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import {debug, DEFAULT_HEADER, botDetected} from './utils.js';
+
+puppeteer.use(StealthPlugin());
+
+export default async function execute(url, waitForSelector, options) {
+    let browser;
+    try {
+        debug(`Sending request to ${url} using Puppeteer.`);
+
+        browser = await puppeteer.launch({
+            headless: options.puppeteerHeadless ?? true,
+            args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox']
+        });
+        let page = await browser.newPage();
+        await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+        const response = await page.goto(url, {
+            waitUntil: 'domcontentloaded'
+        });
+        let pageSource;
+        //if we're extracting data from a spa, we must wait for the selector
+        if (waitForSelector != null) {
+            await page.waitForSelector(waitForSelector);
+            pageSource = await page.evaluate(selector => {
+                return document.querySelector(selector).innerHTML;
+            }, waitForSelector);
+        } else {
+            pageSource = await page.content();
+        }
+
+        const statusCode = response.status();
+
+        if (botDetected(pageSource, statusCode)) {
+            console.warn('We have been detected as a bot :-/ Tried url: => ', url);
+            return null;
+        }
+
+        return await page.content();
+    } catch (error) {
+        console.error('Error executing with puppeteer executor', error);
+        return null;
+    } finally {
+        if (browser != null) {
+            await browser.close();
+        }
+    }
+}

lib/services/extractor/utils.js

@@ -0,0 +1,35 @@
+let debuggingOn = false;
+
+export const DEFAULT_HEADER = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
+};
+
+export const setDebug = options => {
+    debuggingOn = !!options?.debug;
+};
+
+export const debug = (message) => {
+    if(debuggingOn) {
+        console.debug(message);
+    }
+};
+
+export const botDetected = (pageSource, statusCode) => {
+    const suspiciousStatusCodes = [
+        403, 429
+    ];
+    const botDetectionPatterns = [
+        /verify you are human/i,
+        /access denied/i,
+        /x-amz-cf-id/i,
+    ];
+
+    const detectedInSource = botDetectionPatterns.some(pattern => pattern.test(pageSource));
+    const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
+
+    return detectedInSource || detectedByStatus;
+};