External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Making Fredy an ESM project (#70)

Browse Source 
Making Fredy an ESM project
This commit is contained in:
Christian Kellner
2023-03-13 13:42:43 +01:00
committed by GitHub
parent 7d0ec72a0c
commit 2c5eceb0c1
69 changed files with 862 additions and 1567 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
lib/services/requestDriver.js
View File

@@ -1,19 +1,15 @@
const fetch = require('node-fetch');
const config = require('../../conf/config.json');
const { makeUrlResidential } = require('./scrapingAnt');
import fetch from 'node-fetch';
import { config } from '../utils.js';
import { makeUrlResidential } from './scrapingAnt.js';
//if ScrapingAnt got blocked, this http status is returned
const BLOCKED_HTTP_STATUS = 423;
const NOT_FOUND_HTTP_STATUS = 404;
const MAX_RETRIES_SCRAPING_ANT = 10;
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
function makeDriver(headers = {}) {
let cookies = '';
async function scrapingAntDriver(context, callback, retryCounter = 0) {
const proxyType = config.scrapingAnt?.proxy || 'datacenter';
try {
const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
const response = await fetch(url, {
@@ -22,12 +18,10 @@ function makeDriver(headers = {}) {
cookie: cookies,
},
});
const result = await response.text();
if (cookies.length === 0) {
cookies = response.headers.raw()['set-cookie'] || [];
}
callback(null, result);
} catch (exception) {
/* eslint-disable no-console */
@@ -36,7 +30,6 @@ function makeDriver(headers = {}) {
callback(null, []);
return;
}
if (retryCounter <= MAX_RETRIES_SCRAPING_ANT) {
retryCounter++;
console.debug(`ScrapingAnt got blocked. Retrying ${retryCounter} / ${MAX_RETRIES_SCRAPING_ANT}`);
@@ -48,7 +41,6 @@ function makeDriver(headers = {}) {
/* eslint-enable no-console */
}
}
/**
* The regular request driver is taking care of everyting, that doesn't need to be scraped by ScrapingAnt (which is
* everything != Immoscout as of writing this)
@@ -57,7 +49,6 @@ function makeDriver(headers = {}) {
if (context.url.toLowerCase().indexOf('scrapingant') !== -1) {
return scrapingAntDriver(context, callback);
}
try {
const response = await fetch(context.url, {
headers: {
@@ -65,7 +56,6 @@ function makeDriver(headers = {}) {
Cookie: cookies,
},
});
const result = await response.text();
callback(null, result);
} catch (exception) {
@@ -74,5 +64,4 @@ function makeDriver(headers = {}) {
}
};
}
module.exports = makeDriver;
export default makeDriver;