mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
b6b8d6814c
If the async request performed in `requestDriver.makeDriver()` fails, it would call the `callback` function with empty parameters but then continue the execution which can lead to the following error and crash of Fredy:
```
Error while trying to scrape data. Received error: Request failed with status code 504
/fredy/lib/services/requestDriver.js:25
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
^
TypeError: Cannot read properties of undefined (reading 'data')
at driver (/fredy/lib/services/requestDriver.js:25:23)
at runMicrotasks (<anonymous>)
at processTicksAndRejections (node:internal/process/task_queues:96:5)
```
37 lines
980 B
JavaScript
37 lines
980 B
JavaScript
const axios = require('axios');
|
|
const axiosRetry = require('axios-retry');
|
|
|
|
axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
|
|
|
|
function makeDriver(headers = {}) {
|
|
let cookies = '';
|
|
|
|
return async function driver(context, callback) {
|
|
try {
|
|
const url = context.url;
|
|
const result = await axios({
|
|
url,
|
|
headers: {
|
|
...headers,
|
|
Cookie: cookies,
|
|
},
|
|
});
|
|
|
|
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
|
|
//assume we have gotten a response from scrapingAnt
|
|
if (cookies.length === 0) {
|
|
cookies = result.data.cookies;
|
|
}
|
|
callback(null, result.data.content);
|
|
} else {
|
|
callback(null, result.data);
|
|
}
|
|
} catch (exception) {
|
|
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
|
|
callback(null, []);
|
|
}
|
|
};
|
|
}
|
|
|
|
module.exports = makeDriver;
|