making sure fredy is not crashing if scrapingant has issues

This commit is contained in:
orangecoding
2021-05-13 20:51:15 +02:00
parent 1f6e2d3618
commit 1b3a95b325
2 changed files with 13 additions and 9 deletions

View File

@@ -52,14 +52,19 @@ class FredyRuntime {
return;
}
const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
.then((listings) => {
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
try {
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
.then((listings) => {
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
} catch (error) {
reject(error);
console.error(error);
}
});
}

View File

@@ -13,7 +13,6 @@ class Scraper {
const headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
useQueryString: true,
};
if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {