mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
moving to node-fetch coz axios is causing issues with scrapingAnt
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
const axios = require('axios');
|
||||
const fetch = require('node-fetch');
|
||||
const config = require('../../conf/config.json');
|
||||
|
||||
const { makeUrlResidential } = require('./scrapingAnt');
|
||||
@@ -16,19 +16,19 @@ function makeDriver(headers = {}) {
|
||||
|
||||
try {
|
||||
const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
|
||||
const result = await axios({
|
||||
url,
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
...headers,
|
||||
Cookie: cookies,
|
||||
cookie: cookies,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await response.text();
|
||||
if (cookies.length === 0) {
|
||||
cookies = result.data.cookies;
|
||||
cookies = response.headers.raw()['set-cookie'] || [];
|
||||
}
|
||||
|
||||
callback(null, result.data.content);
|
||||
callback(null, result);
|
||||
} catch (exception) {
|
||||
/* eslint-disable no-console */
|
||||
if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
|
||||
@@ -59,15 +59,15 @@ function makeDriver(headers = {}) {
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await axios({
|
||||
url: context.url,
|
||||
const response = await fetch(context.url, {
|
||||
headers: {
|
||||
...headers,
|
||||
Cookie: cookies,
|
||||
},
|
||||
});
|
||||
|
||||
callback(null, result.data);
|
||||
const result = await response.text();
|
||||
callback(null, result);
|
||||
} catch (exception) {
|
||||
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
|
||||
callback(null, []);
|
||||
|
||||
Reference in New Issue
Block a user