diff --git a/lib/services/requestDriver.js b/lib/services/requestDriver.js index 89ccf44..655f3fd 100644 --- a/lib/services/requestDriver.js +++ b/lib/services/requestDriver.js @@ -7,30 +7,29 @@ function makeDriver(headers = {}) { let cookies = ''; return async function driver(context, callback) { - const url = context.url; - let result; try { - result = await axios({ + const url = context.url; + const result = await axios({ url, headers: { ...headers, Cookie: cookies, }, }); + + if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) { + //assume we have gotten a response from scrapingAnt + if (cookies.length === 0) { + cookies = result.data.cookies; + } + callback(null, result.data.content); + } else { + callback(null, result.data); + } } catch (exception) { console.error(`Error while trying to scrape data. Received error: ${exception.message}`); callback(null, []); } - - if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) { - //assume we have gotten a response from scrapingAnt - if (cookies.length === 0) { - cookies = result.data.cookies; - } - callback(null, result.data.content); - } else { - callback(null, result.data); - } }; }