External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

making sure fredy is not crashing if scrapingant has issues

Browse Source
This commit is contained in:
orangecoding
2021-05-13 20:51:15 +02:00
parent 1f6e2d3618
commit 1b3a95b325
2 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
lib/FredyRuntime.js
View File

@@ -52,14 +52,19 @@ class FredyRuntime {
return; return;
} }
const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url; const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]) try {
.then((listings) => { xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
resolve(listings == null ? [] : listings); .then((listings) => {
}) resolve(listings == null ? [] : listings);
.catch((err) => { })
reject(err); .catch((err) => {
console.error(err); reject(err);
}); console.error(err);
});
} catch (error) {
reject(error);
console.error(error);
}
}); });
} }

1
lib/services/scraper.js
View File

@@ -13,7 +13,6 @@ class Scraper {
const headers = { const headers = {
'User-Agent': 'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36',
useQueryString: true,
}; };
if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) { if (config.scrapingAnt != null && config.scrapingAnt.apiKey != null) {