From 8a5fbcdf714c4b65c3d3e28278961d533c22f04f Mon Sep 17 00:00:00 2001 From: "weakmap@gmail.com" Date: Tue, 20 Dec 2022 10:21:15 +0100 Subject: [PATCH] moving to node-fetch coz axios is causing issues with scrapingAnt --- lib/api/routes/jobRouter.js | 8 ++--- lib/notification/adapter/mattermost.js | 12 +++++--- lib/notification/adapter/telegram.js | 17 +++++++---- lib/services/requestDriver.js | 18 +++++------ package.json | 2 +- yarn.lock | 41 ++++++++++++++++---------- 6 files changed, 59 insertions(+), 39 deletions(-) diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js index 052ca49..5ad3762 100644 --- a/lib/api/routes/jobRouter.js +++ b/lib/api/routes/jobRouter.js @@ -1,6 +1,6 @@ const service = require('restana')(); const jobRouter = service.newRouter(); -const axios = require('axios'); +const fetch = require('node-fetch'); const jobStorage = require('../../services/storage/jobStorage'); const userStorage = require('../../services/storage/userStorage'); const immoscoutProvider = require('../../provider/immoscout'); @@ -34,10 +34,8 @@ jobRouter.get('/processingTimes', async (req, res) => { if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) { try { - const result = await axios({ - url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`, - }); - scrapingAntData = result.data; + const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`); + scrapingAntData = await response.json(); } catch (Exception) { console.error('Could not query plan data from scraping ant.', Exception); } diff --git a/lib/notification/adapter/mattermost.js b/lib/notification/adapter/mattermost.js index b3ed3ff..a1cb217 100644 --- a/lib/notification/adapter/mattermost.js +++ b/lib/notification/adapter/mattermost.js @@ -1,6 +1,6 @@ const { markdown2Html } = require('../../services/markdown'); const { getJob } = require('../../services/storage/jobStorage'); -const axios = require('axios'); +const fetch = require('node-fetch'); /** * sends new listings to mattermost @@ -21,9 +21,13 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => { (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n' ); - return axios.post(`${webhook}`, { - channel: channel, - text: message, + return fetch(webhook, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: { + channel: channel, + text: message, + }, }); }; diff --git a/lib/notification/adapter/telegram.js b/lib/notification/adapter/telegram.js index 094dc62..b233a4d 100644 --- a/lib/notification/adapter/telegram.js +++ b/lib/notification/adapter/telegram.js @@ -1,6 +1,6 @@ const { markdown2Html } = require('../../services/markdown'); const { getJob } = require('../../services/storage/jobStorage'); -const axios = require('axios'); +const fetch = require('node-fetch'); const MAX_ENTITIES_PER_CHUNK = 8; const RATE_LIMIT_INTERVAL = 1010; @@ -47,15 +47,22 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => { */ return new Promise((resolve, reject) => { setTimeout(() => { - axios - .post(`https://api.telegram.org/bot${token}/sendMessage`, { + fetch(`https://api.telegram.org/bot${token}/sendMessage`, { + method: 'post', + body: JSON.stringify({ chat_id: chatId, text: message, parse_mode: 'HTML', disable_web_page_preview: true, + }), + headers: { 'Content-Type': 'application/json' }, + }) + .then(() => { + resolve(); }) - .then(() => resolve()) - .catch(() => reject()); + .catch(() => { + reject(); + }); }, RATE_LIMIT_INTERVAL); }); }); diff --git a/lib/services/requestDriver.js b/lib/services/requestDriver.js index dfff63c..44808f7 100644 --- a/lib/services/requestDriver.js +++ b/lib/services/requestDriver.js @@ -1,4 +1,4 @@ -const axios = require('axios'); +const fetch = require('node-fetch'); const config = require('../../conf/config.json'); const { makeUrlResidential } = require('./scrapingAnt'); @@ -16,19 +16,19 @@ function makeDriver(headers = {}) { try { const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url; - const result = await axios({ - url, + const response = await fetch(url, { headers: { ...headers, - Cookie: cookies, + cookie: cookies, }, }); + const result = await response.text(); if (cookies.length === 0) { - cookies = result.data.cookies; + cookies = response.headers.raw()['set-cookie'] || []; } - callback(null, result.data.content); + callback(null, result); } catch (exception) { /* eslint-disable no-console */ if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) { @@ -59,15 +59,15 @@ function makeDriver(headers = {}) { } try { - const result = await axios({ - url: context.url, + const response = await fetch(context.url, { headers: { ...headers, Cookie: cookies, }, }); - callback(null, result.data); + const result = await response.text(); + callback(null, result); } catch (exception) { console.error(`Error while trying to scrape data. Received error: ${exception.message}`); callback(null, []); diff --git a/package.json b/package.json index 6d6f7f2..60be2ab 100755 --- a/package.json +++ b/package.json @@ -58,7 +58,6 @@ "@rematch/loading": "2.1.2", "@sendgrid/mail": "7.7.0", "@vitejs/plugin-react": "^3.0.0", - "axios": "1.2.1", "better-sqlite3": "8.0.1", "body-parser": "1.20.1", "cookie-session": "2.0.0", @@ -68,6 +67,7 @@ "lowdb": "1.0.0", "markdown": "^0.5.0", "nanoid": "3.3.3", + "node-fetch": "2.6.7", "node-mailjet": "3.3.13", "query-string": "7.1.3", "react": "18.2.0", diff --git a/yarn.lock b/yarn.lock index 09bf509..18f3bba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1565,15 +1565,6 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= -axios@1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.2.1.tgz#44cf04a3c9f0c2252ebd85975361c026cb9f864a" - integrity sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - axios@^0.26.0: version "0.26.1" resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" @@ -2854,11 +2845,6 @@ follow-redirects@^1.14.8: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== -follow-redirects@^1.15.0: - version "1.15.2" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" - integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== - form-data@^2.3.1: version "2.5.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" @@ -4187,6 +4173,13 @@ node-abi@^3.3.0: dependencies: semver "^7.3.5" +node-fetch@2.6.7: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + node-mailjet@3.3.13: version "3.3.13" resolved "https://registry.yarnpkg.com/node-mailjet/-/node-mailjet-3.3.13.tgz#2fc38d76b3c6c1a9296d25102eedecb74346e203" @@ -4664,7 +4657,7 @@ proxy-agent@^5.0.0: proxy-from-env "^1.0.0" socks-proxy-agent "^5.0.0" -proxy-from-env@^1.0.0, proxy-from-env@^1.1.0: +proxy-from-env@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== @@ -5694,6 +5687,11 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + trouter@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/trouter/-/trouter-3.2.0.tgz#a9c510fce21b8e659a28732c9de9d82871efe8df" @@ -5887,6 +5885,19 @@ warning@^4.0.2: dependencies: loose-envify "^1.0.0" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"