Make requestDriver more resilient to errors (#46)

If the async request performed in `requestDriver.makeDriver()` fails, it would call the `callback` function with empty parameters but then continue the execution which can lead to the following error and crash of Fredy:
```
Error while trying to scrape data. Received error: Request failed with status code 504
/fredy/lib/services/requestDriver.js:25
    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
                      ^

TypeError: Cannot read properties of undefined (reading 'data')
    at driver (/fredy/lib/services/requestDriver.js:25:23)
    at runMicrotasks (<anonymous>)
    at processTicksAndRejections (node:internal/process/task_queues:96:5)
```
This commit is contained in:
Jochen Schalanda
2022-01-31 10:48:11 +01:00
committed by GitHub
parent b8d658a948
commit b6b8d6814c

View File

@@ -7,20 +7,15 @@ function makeDriver(headers = {}) {
let cookies = ''; let cookies = '';
return async function driver(context, callback) { return async function driver(context, callback) {
const url = context.url;
let result;
try { try {
result = await axios({ const url = context.url;
const result = await axios({
url, url,
headers: { headers: {
...headers, ...headers,
Cookie: cookies, Cookie: cookies,
}, },
}); });
} catch (exception) {
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
callback(null, []);
}
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) { if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
//assume we have gotten a response from scrapingAnt //assume we have gotten a response from scrapingAnt
@@ -31,6 +26,10 @@ function makeDriver(headers = {}) {
} else { } else {
callback(null, result.data); callback(null, result.data);
} }
} catch (exception) {
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
callback(null, []);
}
}; };
} }