fixing end of file issue by upgrading node-fetch

README.md

@@ -8,8 +8,12 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
 
 If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
 
-[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
+# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
-If you like my work, consider sponsoring this or other projects. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of these projects in my spare time :) Thanks.
+If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
+
+<img src="https://github.com/orangecoding/fredy/blob/master/doc/jetbrains.png" width="200">   
+
+_Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 

index.js

@@ -20,6 +20,7 @@ const { duringWorkingHoursOrNotSet } = require('./lib/utils');
 require('./lib/api/api');
 
 //assuming interval is always in minutes
+
 const INTERVAL = config.interval * 60 * 1000;
 
 /* eslint-disable no-console */

lib/api/routes/jobRouter.js

@@ -1,6 +1,6 @@
 const service = require('restana')();
 const jobRouter = service.newRouter();
-const axios = require('axios');
+const fetch = require('node-fetch');
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
 const immoscoutProvider = require('../../provider/immoscout');
@@ -34,10 +34,8 @@ jobRouter.get('/processingTimes', async (req, res) => {
 
   if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
     try {
-      const result = await axios({
+      const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
-        url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
+      scrapingAntData = await response.json();
-      });
-      scrapingAntData = result.data;
     } catch (Exception) {
       console.error('Could not query plan data from scraping ant.', Exception);
     }

lib/notification/adapter/mattermost.js

@@ -1,6 +1,6 @@
 const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
-const axios = require('axios');
+const fetch = require('node-fetch');
 
 /**
  * sends new listings to mattermost
@@ -21,9 +21,13 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
   );
 
-  return axios.post(`${webhook}`, {
+  return fetch(webhook, {
-    channel: channel,
+    method: 'POST',
-    text: message,
+    headers: { 'Content-Type': 'application/json' },
+    body: {
+      channel: channel,
+      text: message,
+    },
   });
 };
 

lib/notification/adapter/telegram.js

@@ -1,6 +1,6 @@
 const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
-const axios = require('axios');
+const fetch = require('node-fetch');
 
 const MAX_ENTITIES_PER_CHUNK = 8;
 const RATE_LIMIT_INTERVAL = 1010;
@@ -47,15 +47,22 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
      */
     return new Promise((resolve, reject) => {
       setTimeout(() => {
-        axios
+        fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
-          .post(`https://api.telegram.org/bot${token}/sendMessage`, {
+          method: 'post',
+          body: JSON.stringify({
             chat_id: chatId,
             text: message,
             parse_mode: 'HTML',
             disable_web_page_preview: true,
+          }),
+          headers: { 'Content-Type': 'application/json' },
+        })
+          .then(() => {
+            resolve();
           })
-          .then(() => resolve())
+          .catch(() => {
-          .catch(() => reject());
+            reject();
+          });
       }, RATE_LIMIT_INTERVAL);
     });
   });

lib/services/requestDriver.js

@@ -1,4 +1,4 @@
-const axios = require('axios');
+const fetch = require('node-fetch');
 const config = require('../../conf/config.json');
 
 const { makeUrlResidential } = require('./scrapingAnt');
@@ -16,19 +16,19 @@ function makeDriver(headers = {}) {
 
     try {
       const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
-      const result = await axios({
+      const response = await fetch(url, {
-        url,
         headers: {
           ...headers,
-          Cookie: cookies,
+          cookie: cookies,
         },
       });
 
+      const result = await response.text();
       if (cookies.length === 0) {
-        cookies = result.data.cookies;
+        cookies = response.headers.raw()['set-cookie'] || [];
       }
 
-      callback(null, result.data.content);
+      callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
       if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
@@ -59,15 +59,15 @@ function makeDriver(headers = {}) {
     }
 
     try {
-      const result = await axios({
+      const response = await fetch(context.url, {
-        url: context.url,
         headers: {
           ...headers,
           Cookie: cookies,
         },
       });
 
-      callback(null, result.data);
+      const result = await response.text();
+      callback(null, result);
     } catch (exception) {
       console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
       callback(null, []);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "6.0.0",
+  "version": "6.0.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -57,17 +57,17 @@
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "7.7.0",
-    "@vitejs/plugin-react": "^3.0.0",
+    "@vitejs/plugin-react": "3.1.0",
-    "axios": "1.2.1",
+    "better-sqlite3": "8.1.0",
-    "better-sqlite3": "8.0.1",
     "body-parser": "1.20.1",
     "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "10.3.2",
+    "highcharts": "10.3.3",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
     "nanoid": "3.3.3",
+    "node-fetch": "2.6.9",
     "node-mailjet": "3.3.13",
     "query-string": "7.1.3",
     "react": "18.2.0",
@@ -76,37 +76,32 @@
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
     "react-switch": "7.0.0",
-    "redux": "4.2.0",
+    "redux": "4.2.1",
     "redux-thunk": "2.4.2",
     "restana": "4.9.7",
     "semantic-ui-react": "2.1.4",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "^4.0.2",
+    "vite": "4.1.1",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
+    "@babel/core": "7.20.12",
     "@babel/eslint-parser": "^7.19.1",
     "@babel/preset-env": "7.20.2",
     "@babel/preset-react": "7.18.6",
-    "babel-loader": "9.1.0",
     "chai": "4.3.7",
-    "css-loader": "6.7.3",
+    "eslint": "8.34.0",
-    "eslint": "^8.30.0",
+    "eslint-config-prettier": "8.6.0",
-    "eslint-config-prettier": "8.5.0",
+    "eslint-plugin-react": "7.32.2",
-    "eslint-plugin-react": "7.31.11",
-    "file-loader": "6.2.0",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.3",
-    "less-loader": "11.1.0",
+    "lint-staged": "13.1.2",
-    "lint-staged": "13.1.0",
     "mocha": "10.2.0",
-    "prettier": "2.8.1",
+    "prettier": "2.8.4",
     "proxyquire": "2.1.3",
-    "redux-logger": "3.0.6",
+    "redux-logger": "3.0.6"
-    "style-loader": "3.3.1",
-    "url-loader": "4.1.1"
   }
 }

ui/src/App.less

@@ -7,7 +7,7 @@
     width: 100%;
 
     padding: 1rem 1rem;
-    background-color: #3f3e3ef5;
+    background-color: #595959f5;
     color: #f1f1f1;
   }
 }

ui/src/Index.less

@@ -2,5 +2,5 @@ body, html {
   margin: 0;
   height: 100%;
   width: 100%;
-  background-color: #3f3e3ef5;
+  background-color: #595959f5;
 }

ui/src/views/generalSettings/GeneralSettings.less

@@ -15,7 +15,7 @@
   }
 
   &__message{
-    background: #60c5df!important;
+    background: #8fe8ff!important;
   }
 
 }