mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
ignoring expired ssl certificate o0
This commit is contained in:
@@ -1,11 +1,16 @@
|
||||
import fetch from 'node-fetch';
|
||||
import { config } from '../utils.js';
|
||||
import { makeUrlResidential } from './scrapingAnt.js';
|
||||
import https from 'https';
|
||||
//if ScrapingAnt got blocked, this http status is returned
|
||||
const BLOCKED_HTTP_STATUS = 423;
|
||||
const NOT_FOUND_HTTP_STATUS = 404;
|
||||
const MAX_RETRIES_SCRAPING_ANT = 10;
|
||||
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
|
||||
const agent = new https.Agent({
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
|
||||
function makeDriver(headers = {}) {
|
||||
let cookies = '';
|
||||
async function scrapingAntDriver(context, callback, retryCounter = 0) {
|
||||
@@ -55,6 +60,7 @@ function makeDriver(headers = {}) {
|
||||
...headers,
|
||||
Cookie: cookies,
|
||||
},
|
||||
agent,
|
||||
});
|
||||
const result = await response.text();
|
||||
callback(null, result);
|
||||
|
||||
@@ -9,7 +9,7 @@ describe('#immobilien.de testsuite()', () => {
|
||||
similarityCache.stopCacheCleanup();
|
||||
});
|
||||
provider.init(providerConfig.immobilienDe, [], []);
|
||||
it('should test immobilien.de provider', async () => {
|
||||
it.only('should test immobilien.de provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
return await new Promise((resolve) => {
|
||||
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
|
||||
|
||||
Reference in New Issue
Block a user