diff --git a/lib/provider/immoscout.js b/lib/provider/immoscout.js index ddf8364..f7a52a4 100644 --- a/lib/provider/immoscout.js +++ b/lib/provider/immoscout.js @@ -2,9 +2,13 @@ const utils = require('../utils'); let appliedBlackList = []; +function nullOrEmpty(val) { + return val == null || val.length === 0; +} + function normalize(o) { - const title = o.title.replace('NEU', ''); - const address = (o.address || '').replace(/\(.*\),.*$/, '').trim(); + const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', ''); + const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim(); const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`; return Object.assign(o, { title, address, link }); } diff --git a/lib/services/requestDriver.js b/lib/services/requestDriver.js index 39d8ebd..89ccf44 100644 --- a/lib/services/requestDriver.js +++ b/lib/services/requestDriver.js @@ -1,4 +1,7 @@ const axios = require('axios'); +const axiosRetry = require('axios-retry'); + +axiosRetry(axios, { retryDelay: axiosRetry.exponentialDelay, retries: 3 }); function makeDriver(headers = {}) { let cookies = ''; @@ -15,7 +18,8 @@ function makeDriver(headers = {}) { }, }); } catch (exception) { - callback(exception, null); + console.error(`Error while trying to scrape data. Received error: ${exception.message}`); + callback(null, []); } if (typeof result.data === 'object' && url.toLowerCase().indexOf('scrapingant') !== -1) { diff --git a/package.json b/package.json index 8af8ae4..38841bc 100755 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "house", "rent", "immoscout", + "scraper", "immonet", "immowelt", "immobilienscout24" @@ -54,7 +55,8 @@ "@rematch/core": "2.1.0", "@rematch/loading": "2.1.0", "@sendgrid/mail": "7.4.7", - "axios": "0.22.0", + "axios": "0.24.0", + "axios-retry": "^3.2.4", "body-parser": "1.19.0", "cookie-session": "1.4.0", "handlebars": "4.7.7", diff --git a/yarn.lock b/yarn.lock index 5d5c9fe..7fc97e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1276,6 +1276,13 @@ dependencies: regenerator-runtime "^0.13.4" +"@babel/runtime@^7.15.4": + version "7.16.3" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.16.3.tgz#b86f0db02a04187a3c17caa77de69840165d42d5" + integrity sha512-WBwekcqacdY2e9AF/Q7WLFUWmdJGJTkbjqTjoMDgXkVZ3ZRUvOPsLb5KdwISoQVsbP+DQzVZW4Zhci0DvpbNTQ== + dependencies: + regenerator-runtime "^0.13.4" + "@babel/runtime@^7.9.2": version "7.14.0" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.14.0.tgz#46794bc20b612c5f75e62dd071e24dfd95f1cbe6" @@ -2092,10 +2099,18 @@ atob@^2.1.2: resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== -axios@0.22.0: - version "0.22.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.22.0.tgz#bf702c41fb50fbca4539589d839a077117b79b25" - integrity sha512-Z0U3uhqQeg1oNcihswf4ZD57O3NrR1+ZXhxaROaWpDmsDTx7T2HNBV2ulBtie2hwJptu8UvgnJoK+BIqdzh/1w== +axios-retry@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.2.4.tgz#f447a53c3456f5bfeca18f20c3a3272207d082ae" + integrity sha512-Co3UXiv4npi6lM963mfnuH90/YFLKWWDmoBYfxkHT5xtkSSWNqK9zdG3fw5/CP/dsoKB5aMMJCsgab+tp1OxLQ== + dependencies: + "@babel/runtime" "^7.15.4" + is-retry-allowed "^2.2.0" + +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== dependencies: follow-redirects "^1.14.4" @@ -4986,6 +5001,11 @@ is-regexp@^1.0.0: resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" integrity sha1-/S2INUXEa6xaYz57mgnof6LLUGk= +is-retry-allowed@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz#88f34cbd236e043e71b6932d09b0c65fb7b4d71d" + integrity sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg== + is-shared-array-buffer@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz#97b0c85fbdacb59c9c446fe653b82cf2b5b7cfe6"