mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
ignoring expired ssl certificate o0
This commit is contained in:
@@ -1,11 +1,16 @@
|
||||
import fetch from 'node-fetch';
|
||||
import { config } from '../utils.js';
|
||||
import { makeUrlResidential } from './scrapingAnt.js';
|
||||
import https from 'https';
|
||||
//if ScrapingAnt got blocked, this http status is returned
|
||||
const BLOCKED_HTTP_STATUS = 423;
|
||||
const NOT_FOUND_HTTP_STATUS = 404;
|
||||
const MAX_RETRIES_SCRAPING_ANT = 10;
|
||||
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
|
||||
const agent = new https.Agent({
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
|
||||
function makeDriver(headers = {}) {
|
||||
let cookies = '';
|
||||
async function scrapingAntDriver(context, callback, retryCounter = 0) {
|
||||
@@ -55,6 +60,7 @@ function makeDriver(headers = {}) {
|
||||
...headers,
|
||||
Cookie: cookies,
|
||||
},
|
||||
agent,
|
||||
});
|
||||
const result = await response.text();
|
||||
callback(null, result);
|
||||
|
||||
Reference in New Issue
Block a user