External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

scraping ant retries

Browse Source
This commit is contained in:
Christian Kellner
2022-04-29 13:22:39 +02:00
parent 1a37773a40
commit 676d48807a
4 changed files with 343 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lib/services/scrapingAnt.js
View File

@@ -1,6 +1,5 @@
const { metaInformation } = require('../provider/immoscout');
//to better configure re-capture chose a random proxy each time we do a call
const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
const config = require('../../conf/config.json');
const isImmoscout = (id) => {
@@ -8,13 +7,9 @@ const isImmoscout = (id) => {
};
exports.transformUrlForScrapingAnt = (url, id) => {
const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
if (isImmoscout(id)) {
//only do calls to scrapingAnt when dealing with Immoscout
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(
url
)}&proxy_country=${randomProxy}&proxy_type=residential`;
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=residential`;
}
return url;
};
@@ -24,3 +19,7 @@ exports.isScrapingAntApiKeySet = () => {
};
exports.isImmoscout = isImmoscout;
exports.makeUrlResidential = (url) => {
return url.replace('datacenter', 'residential');
};