ignoring expired ssl certificate o0

This commit is contained in:
weakmap@gmail.com
2023-04-07 19:44:59 +02:00
parent 75a536d5ab
commit 3ec9ed3b2a
2 changed files with 7 additions and 1 deletions

View File

@@ -1,11 +1,16 @@
import fetch from 'node-fetch';
import { config } from '../utils.js';
import { makeUrlResidential } from './scrapingAnt.js';
import https from 'https';
//if ScrapingAnt got blocked, this http status is returned
const BLOCKED_HTTP_STATUS = 423;
const NOT_FOUND_HTTP_STATUS = 404;
const MAX_RETRIES_SCRAPING_ANT = 10;
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
const agent = new https.Agent({
rejectUnauthorized: false,
});
function makeDriver(headers = {}) {
let cookies = '';
async function scrapingAntDriver(context, callback, retryCounter = 0) {
@@ -55,6 +60,7 @@ function makeDriver(headers = {}) {
...headers,
Cookie: cookies,
},
agent,
});
const result = await response.text();
callback(null, result);

View File

@@ -9,7 +9,7 @@ describe('#immobilien.de testsuite()', () => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immobilienDe, [], []);
it('should test immobilien.de provider', async () => {
it.only('should test immobilien.de provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);