mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
committed by
GitHub
parent
9726079f62
commit
70cab66651
24
lib/services/scrapingAnt.js
Normal file
24
lib/services/scrapingAnt.js
Normal file
@@ -0,0 +1,24 @@
|
||||
const { metaInformation } = require('../provider/immoscout');
|
||||
//to better confure re-capture chose a random proxy each time we do a call
|
||||
const proxies = ['ae', 'br', 'cn', 'de', 'es', 'fr', 'gb', 'hk', 'in', 'it', 'il', 'jp', 'nl', 'ru', 'sa', 'us', 'cz'];
|
||||
const config = require('../../conf/config.json');
|
||||
|
||||
const isImmoscout = (id) => {
|
||||
return id.toLowerCase() === metaInformation.id;
|
||||
};
|
||||
|
||||
exports.transformUrlForScrapingAnt = (url, id) => {
|
||||
const randomProxy = proxies[Math.floor(Math.random() * proxies.length)];
|
||||
|
||||
if (isImmoscout(id)) {
|
||||
//only do calls to scrapingAnt when dealing with Immoscout
|
||||
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_country=${randomProxy}`;
|
||||
}
|
||||
return url;
|
||||
};
|
||||
|
||||
exports.isScrapingAntApiKeySet = () => {
|
||||
return config.scrapingAnt != null && config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0;
|
||||
};
|
||||
|
||||
exports.isImmoscout = isImmoscout;
|
||||
Reference in New Issue
Block a user