External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: node would crash if axios throw an error

Browse Source
This commit is contained in:
Christian Kellner
2021-11-11 15:31:53 +01:00
parent ec986e4b18
commit c97b323b35
4 changed files with 38 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/provider/immoscout.js
View File

@@ -2,9 +2,13 @@ const utils = require('../utils');
let appliedBlackList = [];
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function normalize(o) {
const title = o.title.replace('NEU', '');
const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
return Object.assign(o, { title, address, link });
}

6
lib/services/requestDriver.js
View File

@@ -1,4 +1,7 @@
const axios = require('axios');
const axiosRetry = require('axios-retry');
axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 });
function makeDriver(headers = {}) {
let cookies = '';
@@ -15,7 +18,8 @@ function makeDriver(headers = {}) {
},
});
} catch (exception) {
callback(exception, null);
console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
callback(null, []);
}
if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) {