External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make requestDriver more resilient to errors (#46)

Browse Source 
If the async request performed in `requestDriver.makeDriver()` fails, it would call the `callback` function with empty parameters but then continue the execution which can lead to the following error and crash of Fredy:
```
Error while trying to scrape data. Received error: Request failed with status code 504
/fredy/lib/services/requestDriver.js:25
    if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
                      ^

TypeError: Cannot read properties of undefined (reading 'data')
    at driver (/fredy/lib/services/requestDriver.js:25:23)
    at runMicrotasks (<anonymous>)
    at processTicksAndRejections (node:internal/process/task_queues:96:5)
```
This commit is contained in:
Jochen Schalanda
2022-01-31 10:48:11 +01:00
committed by GitHub
parent b8d658a948
commit b6b8d6814c
1 changed files with 12 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
lib/services/requestDriver.js
View File

@@ -7,30 +7,29 @@ function makeDriver(headers = {}) {
let cookies = '';
return async function driver(context, callback) {
const url = context.url;
let result;
try {
result = await axios({
const url = context.url;
const result = await axios({
url,
headers: {
...headers,
Cookie: cookies,
},
});
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
//assume we have gotten a response from scrapingAnt
if (cookies.length === 0) {
cookies = result.data.cookies;
}
callback(null, result.data.content);
} else {
callback(null, result.data);
}
} catch (exception) {
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
callback(null, []);
}
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {
//assume we have gotten a response from scrapingAnt
if (cookies.length === 0) {
cookies = result.data.cookies;
}
callback(null, result.data.content);
} else {
callback(null, result.data);
}
};
}