External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

bringing back immonet by using scrapingant

Browse Source
This commit is contained in:
weakmap@gmail.com
2023-04-15 18:24:51 +02:00
parent 6fbee3e7c6
commit 4ba098e0b6
10 changed files with 47 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/services/requestDriver.js
View File

@@ -46,9 +46,10 @@ function makeDriver(headers = {}) {
/* eslint-enable no-console */
}
}
/**
* The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
* everything != Immoscout as of writing this)
* everything != Immoscout & Immonet as of writing this)
*/
return async function driver(context, callback) {
if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {

24
lib/services/scrapingAnt.js
View File

@@ -1,12 +1,22 @@
import { metaInformation } from '../provider/immoscout.js';
import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
import { metaInformation as immoNetInfo } from '../provider/immonet.js';
import { config } from '../utils.js';
const isImmoscout = (id) => {
return id.toLowerCase() === metaInformation.id;
const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${new Buffer(
'window.scrollTo(0,document.body.scrollHeight);'
).toString('base64')}`;
const needScrapingAnt = (id) => {
return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
};
export const transformUrlForScrapingAnt = (url, id) => {
if (isImmoscout(id)) {
//only do calls to scrapingAnt when dealing with Immoscout
url = `https://api.scrapingant.com/v1/general?url=${encodeURIComponent(url)}&proxy_type=datacenter`;
let urlParams = '';
if (needScrapingAnt(id)) {
if (id.toLowerCase() === immoNetInfo.id) {
urlParams = additionalImmonetUrlParams;
}
//only do calls to scrapingAnt when dealing with Immoscout/Immonet
url = `https://api.scrapingant.com/v2/general?url=${encodeURIComponent(url)}&proxy_type=datacenter${urlParams}`;
}
return url;
};
@@ -16,4 +26,4 @@ export const isScrapingAntApiKeySet = () => {
export const makeUrlResidential = (url) => {
return url.replace('datacenter', 'residential');
};
export { isImmoscout };
export { needScrapingAnt };