mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
making sure fredy is not crashing if scrapingant has issues
This commit is contained in:
@@ -52,14 +52,19 @@ class FredyRuntime {
|
||||
return;
|
||||
}
|
||||
const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
|
||||
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
|
||||
.then((listings) => {
|
||||
resolve(listings == null ? [] : listings);
|
||||
})
|
||||
.catch((err) => {
|
||||
reject(err);
|
||||
console.error(err);
|
||||
});
|
||||
try {
|
||||
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
|
||||
.then((listings) => {
|
||||
resolve(listings == null ? [] : listings);
|
||||
})
|
||||
.catch((err) => {
|
||||
reject(err);
|
||||
console.error(err);
|
||||
});
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
console.error(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ class Scraper {
|
||||
const headers = {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
|
||||
useQueryString: true,
|
||||
};
|
||||
|
||||
if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {
|
||||
|
||||
Reference in New Issue
Block a user