moving to node-fetch coz axios is causing issues with scrapingAnt

This commit is contained in:
weakmap@gmail.com
2022-12-20 10:21:15 +01:00
parent 60bb75da57
commit 8a5fbcdf71
6 changed files with 59 additions and 39 deletions

View File

@@ -1,4 +1,4 @@
const axios = require('axios');
const fetch = require('node-fetch');
const config = require('../../conf/config.json');
const { makeUrlResidential } = require('./scrapingAnt');
@@ -16,19 +16,19 @@ function makeDriver(headers = {}) {
try {
const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
const result = await axios({
url,
const response = await fetch(url, {
headers: {
...headers,
Cookie: cookies,
cookie: cookies,
},
});
const result = await response.text();
if (cookies.length === 0) {
cookies = result.data.cookies;
cookies = response.headers.raw()['set-cookie'] || [];
}
callback(null, result.data.content);
callback(null, result);
} catch (exception) {
/* eslint-disable no-console */
if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
@@ -59,15 +59,15 @@ function makeDriver(headers = {}) {
}
try {
const result = await axios({
url: context.url,
const response = await fetch(context.url, {
headers: {
...headers,
Cookie: cookies,
},
});
callback(null, result.data);
const result = await response.text();
callback(null, result);
} catch (exception) {
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
callback(null, []);