Bringing back immoscout (#21)

Bringing back immoscout support 🎉

lib/services/requestDriver.js

@@ -0,0 +1,33 @@
+const axios = require('axios');
+
+function makeDriver(headers = {}) {
+  let cookies = '';
+
+  return async function driver(context, callback) {
+    const url = context.url;
+    let result;
+    try {
+      result = await axios({
+        url,
+        headers: {
+          ...headers,
+          Cookie: cookies,
+        },
+      });
+    } catch (exception) {
+      callback(exception, null);
+    }
+
+    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
+      //assume we have gotten a response from scrapingAnt
+      if (cookies.length === 0) {
+        cookies = result.data.cookies;
+      }
+      callback(null, result.data.content);
+    } else {
+      callback(null, result.data);
+    }
+  };
+}
+
+module.exports = makeDriver;

lib/services/scraper.js

@@ -1,4 +1,5 @@
-const makeDriver = require('request-x-ray');
+const config = require('../../conf/config.json');
+const makeDriver = require('./requestDriver');
 const Xray = require('x-ray');
 
 class Scraper {
@@ -9,14 +10,16 @@ class Scraper {
       int: this._int,
     };
 
-    const driver = makeDriver({
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
-        cookie:
-          'longUnreliableState="dWlkcg==:YS1kZDViMzVhZWRhMTk0MDdmYWRjNDNkY2VmYTcxZmVkOQ=="; eveD=eyJldnRfZ2FfYWN0aW9uIjpbInNlYXJjaCJdLCJldnRfZ2FfY2F0ZWdvcnkiOlsicmVzdWx0bGlzdCJdLCJnZW9fYmxuIjpbIm5vcmRyaGVpbl93ZXN0ZmFsZW4iXSwiZXZ0X2dhX2xhYmVsIjpbImRpc3RyaWN0Il0sIm9ial9pdHlwIjpbIndvaG51bmdfa2F1ZiJdLCJnZW9fa3JzIjpbImTDvHNzZWxkb3JmIl0sImdlb19sYW5kIjpbImRldXRzY2hsYW5kIl0sIm9ial9yZXN1bHRsaXN0X2NvdW50IjpbIjI4NCJdLCJvYmpfY3Jvc3N0eXBlIjpbImxpdl9hcGFydG1lbnRfYnV5Il19; ABNTEST=9526230109; is24_experiment_visitor_id=d568590b-951b-45c3-b890-13feef6ee472; reese84=3:Xf3JwcTIC3yeubDXqWBTfg==:oqnDVs58wBxZRMfpzPnlzLzscVQhboRBffkM4caxNe+vLBdozdtdrCwpcTKyvIuhB9MOMCAinb2qnSTL4D9kLpqL72gl+jtl7QdiNAEn2erDKLqX4b9/K5wFU7j6qzxFWdfcMUm295qU3o3s7O8CM8HdghKYOVtoif+qTkeztphyYMfmAePYkfYRhZXZaFwHwxUfkRVUEX2VKoepkTf9TudCHsTYXWqvnpUt/CT+yrFHlUdTgdTWfD5tQJvn3inPqKERAB8TTKoHIvM4duBJV/5fZDax07CHNqHcKhrws0pq4y2ssKfdxLxCE0OIpnMSOtmn7O0koDoV6RzRjNUC+UZ7mhPFH+YSPHTb+6VJsZQDnRufEIz4B1WWIORV+jvHzfIli9OHsmOPnskA6mnCpFwEvQAfJu9R+jI9dccjFno=:Oc7c2wwYiNMBJnvZeDCIKLP0LuVVPWJ4kzd5MPlsoTg=',
-      },
-    });
+    const headers = {
+      'User-Agent':
+        'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
+      useQueryString: true,
+    };
+
+    if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
+      headers['x-api-key'] = config.scrapingAnt.apiKey;
+    }
+    const driver = makeDriver(headers);
 
     const xray = Xray({ filters });
     xray.driver(driver);

lib/services/scrapingAnt.js

@@ -0,0 +1,24 @@
+const { metaInformation } = require('../provider/immoscout');
+//to better confure re-capture chose a random proxy each time we do a call
+const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
+const config = require('../../conf/config.json');
+
+const isImmoscout = (id) => {
+  return id.toLowerCase() === metaInformation.id;
+};
+
+exports.transformUrlForScrapingAnt = (url, id) => {
+  const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
+
+  if (isImmoscout(id)) {
+    //only do calls to scrapingAnt when dealing with Immoscout
+    url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
+  }
+  return url;
+};
+
+exports.isScrapingAntApiKeySet = () => {
+  return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
+};
+
+exports.isImmoscout = isImmoscout;