External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

ignoring expired ssl certificate o0

Browse Source
This commit is contained in:
weakmap@gmail.com
2023-04-07 19:44:59 +02:00
parent 75a536d5ab
commit 3ec9ed3b2a
2 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/services/requestDriver.js
View File

@@ -1,11 +1,16 @@
import fetch from 'node-fetch'; import fetch from 'node-fetch';
import { config } from '../utils.js'; import { config } from '../utils.js';
import { makeUrlResidential } from './scrapingAnt.js'; import { makeUrlResidential } from './scrapingAnt.js';
import https from 'https';
//if ScrapingAnt got blocked, this http status is returned //if ScrapingAnt got blocked, this http status is returned
const BLOCKED_HTTP_STATUS = 423; const BLOCKED_HTTP_STATUS = 423;
const NOT_FOUND_HTTP_STATUS = 404; const NOT_FOUND_HTTP_STATUS = 404;
const MAX_RETRIES_SCRAPING_ANT = 10; const MAX_RETRIES_SCRAPING_ANT = 10;
const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS]; const EXPECTED_STATUS_CODES = [BLOCKED_HTTP_STATUS, NOT_FOUND_HTTP_STATUS];
const agent = new https.Agent({
rejectUnauthorized: false,
});
function makeDriver(headers = {}) { function makeDriver(headers = {}) {
let cookies = ''; let cookies = '';
async function scrapingAntDriver(context, callback, retryCounter = 0) { async function scrapingAntDriver(context, callback, retryCounter = 0) {
@@ -55,6 +60,7 @@ function makeDriver(headers = {}) {
...headers, ...headers,
Cookie: cookies, Cookie: cookies,
}, },
agent,
}); });
const result = await response.text(); const result = await response.text();
callback(null, result); callback(null, result);

2
test/provider/immobilienDe.test.js
View File

@@ -9,7 +9,7 @@ describe('#immobilien.de testsuite()', () => {
similarityCache.stopCacheCleanup(); similarityCache.stopCacheCleanup();
}); });
provider.init(providerConfig.immobilienDe, [], []); provider.init(providerConfig.immobilienDe, [], []);
it('should test immobilien.de provider', async () => { it.only('should test immobilien.de provider', async () => {
const Fredy = await mockFredy(); const Fredy = await mockFredy();
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);