fixing end of file issue by upgrading node-fetch

README.md

@@ -8,8 +8,12 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
 
 If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
 
-[![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
+# Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
-If you like my work, consider sponsoring this or other projects. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of these projects in my spare time :) Thanks.
+If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
+
+<img src="https://github.com/orangecoding/fredy/blob/master/doc/jetbrains.png" width="200">   
+
+_Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
@@ -66,7 +70,7 @@ For the frontend, run:
 ```shell
 yarn run dev
 ```
-You should now be able to access _Fredy_ from your browser. Go to `http://localhost:9000`.
+You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
 
 ### Running Tests
 To run the tests, run

index.html

@@ -4,7 +4,7 @@
     <meta charset="UTF-8"
           name="viewport"
           content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1">
-    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.4.1/semantic.min.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui@2/dist/semantic.min.css">
     <meta name="google" content="notranslate">
 
     <title>Fredy</title>
@@ -13,5 +13,5 @@
 
 <div id="fredy" style="position: absolute;top: 0;left: 0;right: 0;bottom: 0;"></div>
 </body>
-<script src="fredy.bundle.js"></script>
+<script type="module" src="/ui/src/Index.jsx"></script>
 </html>

index.js

@@ -20,6 +20,7 @@ const { duringWorkingHoursOrNotSet } = require('./lib/utils');
 require('./lib/api/api');
 
 //assuming interval is always in minutes
+
 const INTERVAL = config.interval * 60 * 1000;
 
 /* eslint-disable no-console */

lib/api/routes/jobRouter.js

@@ -1,6 +1,6 @@
 const service = require('restana')();
 const jobRouter = service.newRouter();
-const axios = require('axios');
+const fetch = require('node-fetch');
 const jobStorage = require('../../services/storage/jobStorage');
 const userStorage = require('../../services/storage/userStorage');
 const immoscoutProvider = require('../../provider/immoscout');
@@ -34,10 +34,8 @@ jobRouter.get('/processingTimes', async (req, res) => {
 
   if (config.scrapingAnt.apiKey != null && config.scrapingAnt.apiKey.length > 0) {
     try {
-      const result = await axios({
+      const response = await fetch(`https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`);
-        url: `https://api.scrapingant.com/v1/usage?x-api-key=${config.scrapingAnt.apiKey}`,
+      scrapingAntData = await response.json();
-      });
-      scrapingAntData = result.data;
     } catch (Exception) {
       console.error('Could not query plan data from scraping ant.', Exception);
     }

lib/notification/adapter/mattermost.js

@@ -1,6 +1,6 @@
 const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
-const axios = require('axios');
+const fetch = require('node-fetch');
 
 /**
  * sends new listings to mattermost
@@ -21,9 +21,13 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
     (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
   );
 
-  return axios.post(`${webhook}`, {
+  return fetch(webhook, {
-    channel: channel,
+    method: 'POST',
-    text: message,
+    headers: { 'Content-Type': 'application/json' },
+    body: {
+      channel: channel,
+      text: message,
+    },
   });
 };
 

lib/notification/adapter/telegram.js

@@ -1,6 +1,6 @@
 const { markdown2Html } = require('../../services/markdown');
 const { getJob } = require('../../services/storage/jobStorage');
-const axios = require('axios');
+const fetch = require('node-fetch');
 
 const MAX_ENTITIES_PER_CHUNK = 8;
 const RATE_LIMIT_INTERVAL = 1010;
@@ -47,15 +47,22 @@ exports.send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
      */
     return new Promise((resolve, reject) => {
       setTimeout(() => {
-        axios
+        fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
-          .post(`https://api.telegram.org/bot${token}/sendMessage`, {
+          method: 'post',
+          body: JSON.stringify({
             chat_id: chatId,
             text: message,
             parse_mode: 'HTML',
             disable_web_page_preview: true,
+          }),
+          headers: { 'Content-Type': 'application/json' },
+        })
+          .then(() => {
+            resolve();
           })
-          .then(() => resolve())
+          .catch(() => {
-          .catch(() => reject());
+            reject();
+          });
       }, RATE_LIMIT_INTERVAL);
     });
   });

lib/services/requestDriver.js

@@ -1,4 +1,4 @@
-const axios = require('axios');
+const fetch = require('node-fetch');
 const config = require('../../conf/config.json');
 
 const { makeUrlResidential } = require('./scrapingAnt');
@@ -16,19 +16,19 @@ function makeDriver(headers = {}) {
 
     try {
       const url = proxyType === 'residential' ? makeUrlResidential(context.url) : context.url;
-      const result = await axios({
+      const response = await fetch(url, {
-        url,
         headers: {
           ...headers,
-          Cookie: cookies,
+          cookie: cookies,
         },
       });
 
+      const result = await response.text();
       if (cookies.length === 0) {
-        cookies = result.data.cookies;
+        cookies = response.headers.raw()['set-cookie'] || [];
       }
 
-      callback(null, result.data.content);
+      callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
       if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
@@ -59,15 +59,15 @@ function makeDriver(headers = {}) {
     }
 
     try {
-      const result = await axios({
+      const response = await fetch(context.url, {
-        url: context.url,
         headers: {
           ...headers,
           Cookie: cookies,
         },
       });
 
-      callback(null, result.data);
+      const result = await response.text();
+      callback(null, result);
     } catch (exception) {
       console.error(`Error while trying to scrape data. Received error: ${exception.message}`);
       callback(null, []);

package.json

@@ -1,15 +1,12 @@
 {
   "name": "fredy",
-  "version": "5.9.0",
+  "version": "6.0.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
-    "dev": "run-script-os",
+    "dev": "yarn && rm -rf ./ui/public/* && vite",
-    "dev:win32": "yarn && set BUILD_DEV='true' && set NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
+    "ui": "rm -rf ./ui/public/* && vite",
-    "dev:default": "yarn && export BUILD_DEV='true' && export NODE_ENV='development' && webpack serve --progress --color --config ./webpack.dev.js",
+    "prod": "yarn && vite build --emptyOutDir",
-    "prod": "run-script-os",
-    "prod:win32": "set BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
-    "prod:default": "export BUILD_DEV='false' && webpack --node-env=production --config ./webpack.prod.js",
     "format": "prettier --write lib/**/*.js ui/src/**/*.js test/**/*.js *.js --single-quote --print-width 120",
     "test": "mocha --timeout 3000000 test/**/*.test.js",
     "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js"
@@ -60,16 +57,17 @@
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "7.7.0",
-    "axios": "1.2.1",
+    "@vitejs/plugin-react": "3.1.0",
-    "better-sqlite3": "8.0.1",
+    "better-sqlite3": "8.1.0",
     "body-parser": "1.20.1",
     "cookie-session": "2.0.0",
     "handlebars": "4.7.7",
-    "highcharts": "10.3.2",
+    "highcharts": "10.3.3",
     "highcharts-react-official": "3.1.0",
     "lowdb": "1.0.0",
     "markdown": "^0.5.0",
     "nanoid": "3.3.3",
+    "node-fetch": "2.6.9",
     "node-mailjet": "3.3.13",
     "query-string": "7.1.3",
     "react": "18.2.0",
@@ -78,44 +76,32 @@
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
     "react-switch": "7.0.0",
-    "redux": "4.2.0",
+    "redux": "4.2.1",
     "redux-thunk": "2.4.2",
     "restana": "4.9.7",
     "semantic-ui-react": "2.1.4",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
+    "vite": "4.1.1",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.5",
+    "@babel/core": "7.20.12",
     "@babel/eslint-parser": "^7.19.1",
     "@babel/preset-env": "7.20.2",
     "@babel/preset-react": "7.18.6",
-    "babel-loader": "9.1.0",
     "chai": "4.3.7",
-    "clean-webpack-plugin": "4.0.0",
+    "eslint": "8.34.0",
-    "copy-webpack-plugin": "11.0.0",
+    "eslint-config-prettier": "8.6.0",
-    "css-loader": "6.7.3",
+    "eslint-plugin-react": "7.32.2",
-    "eslint": "^8.30.0",
-    "eslint-config-prettier": "8.5.0",
-    "eslint-plugin-react": "7.31.11",
-    "file-loader": "6.2.0",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.1.3",
-    "less-loader": "11.1.0",
+    "lint-staged": "13.1.2",
-    "lint-staged": "13.1.0",
     "mocha": "10.2.0",
-    "prettier": "2.8.1",
+    "prettier": "2.8.4",
     "proxyquire": "2.1.3",
-    "redux-logger": "3.0.6",
+    "redux-logger": "3.0.6"
-    "run-script-os": "^1.1.6",
-    "style-loader": "3.3.1",
-    "url-loader": "4.1.1",
-    "webpack": "5.75.0",
-    "webpack-cli": "5.0.1",
-    "webpack-dev-server": "4.11.1",
-    "webpack-merge": "5.8.0"
   }
 }

ui/src/App.jsx

@@ -7,7 +7,7 @@ import ToastsContainer from './components/toasts/ToastContainer';
 import JobMutation from './views/jobs/mutation/JobMutation';
 import UserMutator from './views/user/mutation/UserMutator';
 import ToastContext from './components/toasts/ToastContext';
-import JobInsight from './views/jobs/insights/JobInsight';
+import JobInsight from './views/jobs/insights/JobInsight.jsx';
 import { useDispatch, useSelector } from 'react-redux';
 import useToast from './components/toasts/useToast';
 import { Switch, Redirect } from 'react-router-dom';

ui/src/App.less

@@ -7,7 +7,7 @@
     width: 100%;
 
     padding: 1rem 1rem;
-    background-color: #3f3e3ef5;
+    background-color: #595959f5;
     color: #f1f1f1;
   }
 }

ui/src/Index.less

@@ -2,5 +2,5 @@ body, html {
   margin: 0;
   height: 100%;
   width: 100%;
-  background-color: #3f3e3ef5;
+  background-color: #595959f5;
 }

ui/src/views/generalSettings/GeneralSettings.less

@@ -15,7 +15,7 @@
   }
 
   &__message{
-    background: #60c5df!important;
+    background: #8fe8ff!important;
   }
 
 }

vite.config.js

@@ -0,0 +1,23 @@
+import react from '@vitejs/plugin-react';
+import { defineConfig } from 'vite';
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  base: '',
+  build: {
+    chunkSizeWarningLimit: 9999999,
+    outDir: './ui/public',
+  },
+  plugins: [react()],
+  server: {
+    proxy: {
+      '/api': {
+        target: {
+          host: '0.0.0.0',
+          protocol: 'http:',
+          port: 9998,
+        },
+      },
+    },
+  },
+});

webpack.common.js

@@ -1,39 +0,0 @@
-const { CleanWebpackPlugin } = require('clean-webpack-plugin');
-const CopyPlugin = require('copy-webpack-plugin');
-const path = require('path');
-
-module.exports = {
-  entry: path.join(__dirname, 'ui', 'src') + '/Index.js',
-  resolve: {
-    extensions: ['.js'],
-  },
-  plugins: [
-    new CleanWebpackPlugin(),
-    new CopyPlugin({
-      patterns: [{ from: path.join(__dirname, 'ui', 'src') + '/index.html', to: path.join(__dirname, 'ui', 'public') }],
-    }),
-  ],
-  output: {
-    path: path.join(__dirname, 'ui', 'public'),
-    publicPath: '/',
-    filename: 'fredy.bundle.js',
-  },
-  performance: { hints: false },
-  module: {
-    rules: [
-      {
-        test: /\.js?$/,
-        exclude: /(node_modules|bower_components)/,
-        use: [{ loader: 'babel-loader' }],
-      },
-      {
-        test: /\.(css|less)$/i,
-        use: ['style-loader', 'css-loader', 'less-loader'],
-      },
-      {
-        test: /\.(png|jpg|gif|svg|eot|ttf|woff|woff2)$/,
-        use: [{ loader: 'url-loader?limit=3000!image-webpack?bypassOnDebug&optimizationLevel=7&interlaced=false' }],
-      },
-    ],
-  },
-};

webpack.dev.js

@@ -1,26 +0,0 @@
-const { merge } = require('webpack-merge');
-const common = require('./webpack.common.js');
-const webpack = require('webpack');
-const path = require('path');
-
-module.exports = merge(common, {
-  devtool: 'inline-source-map',
-  devServer: {
-    static: path.join(__dirname, 'ui', 'public'),
-    port: 9000,
-    proxy: {
-      '/api': {
-        target: {
-          host: '0.0.0.0',
-          protocol: 'http:',
-          port: 9998,
-        },
-      },
-    },
-  },
-  plugins: [
-    new webpack.DefinePlugin({
-      __DEV__: JSON.stringify(true),
-    }),
-  ],
-});

webpack.prod.js

@@ -1,4 +0,0 @@
-const { merge } = require('webpack-merge');
-const common = require('./webpack.common.js');
-
-module.exports = merge(common, {});