moving to node-fetch coz axios is causing issues with scrapingAnt

lib/api/routes/jobRouter.js

@@ -1,6 +1,6 @@
 const service = require('restana')();
 const jobRouter = service.newRouter();
-const axios = require('axios');
+const fetch = require('node-fetch');
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
 const immoscoutProvider = require('../../provider/immoscout');
@@ -34,10 +34,8 @@ jobRouter.get('/processingTimes', async (req, res) => {
 
   if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
     try {
-      const result = await axios({
-        url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
-      });
-      scrapingAntData = result.data;
+      const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
+      scrapingAntData = await response.json();
     } catch (Exception) {
       console.error('Could not query plan data from scraping ant.', Exception);
     }

lib/notification/adapter/mattermost.js

@@ -1,6 +1,6 @@
 const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
-const axios = require('axios');
+const fetch = require('node-fetch');
 
 /**
  * sends new listings to mattermost
@@ -21,9 +21,13 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
   );
 
-  return axios.post(`${webhook}`, {
-    channel: channel,
-    text: message,
+  return fetch(webhook, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: {
+      channel: channel,
+      text: message,
+    },
   });
 };
 

lib/notification/adapter/telegram.js

@@ -1,6 +1,6 @@
 const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
-const axios = require('axios');
+const fetch = require('node-fetch');
 
 const MAX_ENTITIES_PER_CHUNK = 8;
 const RATE_LIMIT_INTERVAL = 1010;
@@ -47,15 +47,22 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
      */
     return new Promise((resolve, reject) => {
       setTimeout(() => {
-        axios
-          .post(`https://api.telegram.org/bot${token}/sendMessage`, {
+        fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+          method: 'post',
+          body: JSON.stringify({
             chat_id: chatId,
             text: message,
             parse_mode: 'HTML',
             disable_web_page_preview: true,
+          }),
+          headers: { 'Content-Type': 'application/json' },
+        })
+          .then(() => {
+            resolve();
           })
-          .then(() => resolve())
-          .catch(() => reject());
+          .catch(() => {
+            reject();
+          });
       }, RATE_LIMIT_INTERVAL);
     });
   });

lib/services/requestDriver.js

@@ -1,4 +1,4 @@
-const axios = require('axios');
+const fetch = require('node-fetch');
 const config = require('../../conf/config.json');
 
 const { makeUrlResidential } = require('./scrapingAnt');
@@ -16,19 +16,19 @@ function makeDriver(headers = {}) {
 
     try {
       const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
-      const result = await axios({
-        url,
+      const response = await fetch(url, {
         headers: {
           ...headers,
-          Cookie: cookies,
+          cookie: cookies,
         },
       });
 
+      const result = await response.text();
       if (cookies.length === 0) {
-        cookies = result.data.cookies;
+        cookies = response.headers.raw()['set-cookie'] || [];
       }
 
-      callback(null, result.data.content);
+      callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
       if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
@@ -59,15 +59,15 @@ function makeDriver(headers = {}) {
     }
 
     try {
-      const result = await axios({
-        url: context.url,
+      const response = await fetch(context.url, {
         headers: {
           ...headers,
           Cookie: cookies,
         },
       });
 
-      callback(null, result.data);
+      const result = await response.text();
+      callback(null, result);
     } catch (exception) {
       console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
       callback(null, []);

package.json

@@ -58,7 +58,6 @@
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "7.7.0",
     "@vitejs/plugin-react": "^3.0.0",
-    "axios": "1.2.1",
     "better-sqlite3": "8.0.1",
     "body-parser": "1.20.1",
     "cookie-session": "2.0.0",
@@ -68,6 +67,7 @@
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
     "nanoid": "3.3.3",
+    "node-fetch": "2.6.7",
     "node-mailjet": "3.3.13",
     "query-string": "7.1.3",
     "react": "18.2.0",

yarn.lock

@@ -1565,15 +1565,6 @@ asynckit@^0.4.0:
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
   integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
 
-axios@1.2.1:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.2.1.tgz#44cf04a3c9f0c2252ebd85975361c026cb9f864a"
-  integrity sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==
-  dependencies:
-    follow-redirects "^1.15.0"
-    form-data "^4.0.0"
-    proxy-from-env "^1.1.0"
-
 axios@^0.26.0:
   version "0.26.1"
   resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9"
@@ -2854,11 +2845,6 @@ follow-redirects@^1.14.8:
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7"
   integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==
 
-follow-redirects@^1.15.0:
-  version "1.15.2"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
-  integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
-
 form-data@^2.3.1:
   version "2.5.1"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4"
@@ -4187,6 +4173,13 @@ node-abi@^3.3.0:
   dependencies:
     semver "^7.3.5"
 
+node-fetch@2.6.7:
+  version "2.6.7"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
+  integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-mailjet@3.3.13:
   version "3.3.13"
   resolved "https://registry.yarnpkg.com/node-mailjet/-/node-mailjet-3.3.13.tgz#2fc38d76b3c6c1a9296d25102eedecb74346e203"
@@ -4664,7 +4657,7 @@ proxy-agent@^5.0.0:
     proxy-from-env "^1.0.0"
     socks-proxy-agent "^5.0.0"
 
-proxy-from-env@^1.0.0, proxy-from-env@^1.1.0:
+proxy-from-env@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
   integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
@@ -5694,6 +5687,11 @@ toidentifier@1.0.1:
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
   integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==
 
+tr46@~0.0.3:
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
+  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
+
 trouter@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/trouter/-/trouter-3.2.0.tgz#a9c510fce21b8e659a28732c9de9d82871efe8df"
@@ -5887,6 +5885,19 @@ warning@^4.0.2:
   dependencies:
     loose-envify "^1.0.0"
 
+webidl-conversions@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
+  integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
+
+whatwg-url@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
+  integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
+  dependencies:
+    tr46 "~0.0.3"
+    webidl-conversions "^3.0.0"
+
 which-boxed-primitive@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"