using node 18 in github test setup

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 16
+          node-version: 18
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3
-FROM node:16-alpine AS builder
+FROM node:18-alpine AS builder
 COPY --chown=1000:1000 . /fredy
 WORKDIR  /fredy
 USER 1000

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Christian Kellner
+Copyright (c) 2024 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use Node.js 16 or above
+- Make sure to use Node.js 18 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -33,9 +33,6 @@ _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_
          
   <img alt="Job Overview" width="30%" src="https://github.com/orangecoding/fredy/blob/master/doc/screenshot_3.png">
 </p>
-<p align="center">
-
-</p>
 
 ## Understanding the fundamentals
 There are 3 important parts in Fredy, that you need to understand to leverage the full power of _Fredy_.

lib/notification/adapter/mailJet.js

@@ -9,7 +9,7 @@ const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTe
 const emailTemplate = Handlebars.compile(template);
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
-    (adapter) => adapter.id === 'mailjet'
+    (adapter) => adapter.id === config.id,
   ).fields;
   const to = receiver
     .trim()

lib/notification/adapter/mattermost.js

@@ -2,13 +2,13 @@ import { markdown2Html } from '../../services/markdown.js';
 import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === 'mattermost').fields;
+  const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
   message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
   message += newListings.map(
-    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n'
+    (o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
   );
   return fetch(webhook, {
     method: 'POST',

lib/notification/adapter/ntfy.js

@@ -3,7 +3,7 @@ import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === 'ntfy').fields;
+  const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   const promises = newListings.map((newListing) => {

lib/notification/adapter/sendGrid.js

@@ -1,7 +1,7 @@
 import sgMail from '@sendgrid/mail';
 import { markdown2Html } from '../../services/markdown.js';
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === 'sendGrid').fields;
+  const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   sgMail.setApiKey(apiKey);
   const msg = {
     templateId,

lib/notification/adapter/slack.js

@@ -2,7 +2,7 @@ import Slack from 'slack';
 import { markdown2Html } from '../../services/markdown.js';
 const msg = Slack.chat.postMessage;
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === 'slack').fields;
+  const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   return newListings.map((payload) =>
     msg({
       token,
@@ -35,7 +35,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
           ts: new Date().getTime() / 1000,
         },
       ],
-    })
+    }),
   );
 };
 export const config = {

lib/notification/adapter/sqlite.js

@@ -1,25 +0,0 @@
-import { markdown2Html } from '../../services/markdown.js';
-import Database from 'better-sqlite3';
-export const send = ({ serviceName, newListings, jobKey }) => {
-  const db = new Database('db/listings.db');
-  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
-  db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
-  const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
-  newListings.map((listing) => {
-    let insertListing = {};
-    fields.map((field) => {
-      insertListing[field] = listing[field];
-    });
-    insertListing.serviceName = serviceName;
-    insertListing.jobKey = jobKey;
-    insert.run(insertListing);
-  });
-  return Promise.resolve();
-};
-export const config = {
-  id: 'sqlite',
-  name: 'Sqlite',
-  description: 'This adapter stores listings in a local sqlite3 database.',
-  config: {},
-  readme: markdown2Html('lib/notification/adapter/sqlite.md'),
-};

lib/notification/adapter/sqlite.md

@@ -1,3 +0,0 @@
-### Sqlite Adapter
-
-This adapter stores search results in an sqlite database in db/listings.db

lib/notification/adapter/telegram.js

@@ -19,7 +19,7 @@ function shorten(str, len = 30) {
   return str.length > len ? str.substring(0, len) + '...' : str;
 }
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === 'telegram').fields;
+  const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
   //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
@@ -30,7 +30,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
       (o) =>
         `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
         [o.address, o.price, o.size].join(' | ') +
-        '\n\n'
+        '\n\n',
     );
     /**
      * This is to not break the rate limit. It is to only send 1 message per second

lib/provider/kleinanzeigen.js

@@ -19,7 +19,7 @@ const config = {
   sortByDateParam: null,
   crawlFields: {
     id: '.aditem@data-adid | int',
-    price: '.aditem-main--middle--price | removeNewline | trim',
+    price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
     size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
     title: '.aditem-main .text-module-begin a | removeNewline | trim',
     link: '.aditem-main .text-module-begin a@href | removeNewline | trim',

lib/services/scrapingAnt.js

@@ -2,7 +2,7 @@ import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
 import { metaInformation as immoNetInfo } from '../provider/immonet.js';
 import { config } from '../utils.js';
 
-const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${new Buffer(
+const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
   'window.scrollTo(0,document.body.scrollHeight);'
 ).toString('base64')}`;
 

lib/services/storage/LowDashAdapter.js

@@ -1,8 +1,8 @@
 import lodash from 'lodash';
 import { LowSync } from 'lowdb';
 export default class LowdashAdapter extends LowSync {
-  constructor(adapter) {
-    super(adapter);
+  constructor(adapter, defaultData = {}) {
+    super(adapter, defaultData);
     this.chain = lodash.chain(this).get('data');
   }
 }

lib/services/storage/jobStorage.js

@@ -7,11 +7,10 @@ import LowdashAdapter from './LowDashAdapter.js';
 
 const file = path.join(getDirName(), '../', 'db/jobs.json');
 const adapter = new JSONFileSync(file);
-const db = new LowdashAdapter(adapter);
+const db = new LowdashAdapter(adapter, { jobs: [] });
 
 db.read();
 
-db.data ||= { jobs: [] };
 
 export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =

lib/services/storage/listingsStorage.js

@@ -5,12 +5,10 @@ import LowdashAdapter from './LowDashAdapter.js';
 
 const file = path.join(getDirName(), '../', 'db/jobListingData.json');
 const adapter = new JSONFileSync(file);
-const db = new LowdashAdapter(adapter);
+const db = new LowdashAdapter(adapter, {});
 
 db.read();
 
-db.data ||= {};
-
 const buildKey = (jobKey, providerId, endpoint) => {
   let key = `${jobKey}`;
   if (jobKey == null && endpoint == null) {

lib/services/storage/userStorage.js

@@ -6,24 +6,24 @@ import * as jobStorage from './jobStorage.js';
 import path from 'path';
 import LowdashAdapter from './LowDashAdapter.js';
 
+const defaultData = {
+    user: [
+        //you probably want to change the default password ;)
+        {
+            id: nanoid(),
+            lastLogin: Date.now(),
+            username: 'admin',
+            password: hasher.hash('admin'),
+            isAdmin: true,
+        },
+    ],
+};
+
 const file = path.join(getDirName(), '../', 'db/users.json');
 const adapter = new JSONFileSync(file);
-const db = new LowdashAdapter(adapter);
+const db = new LowdashAdapter(adapter, defaultData);
 
 db.read();
-db.data ||= {
-  user: [
-    //you probably want to change the default password ;)
-    {
-      id: nanoid(),
-      lastLogin: Date.now(),
-      username: 'admin',
-      password: hasher.hash('admin'),
-      isAdmin: true,
-      isDemo: false,
-    },
-  ],
-};
 
 export const getUsers = (withPassword) => {
   const jobs = jobStorage.getJobs();

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "7.2.1",
+  "version": "8.0.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -55,54 +55,54 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.31.0",
+    "@douyinfe/semi-ui": "2.49.2",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
-    "@sendgrid/mail": "7.7.0",
-    "@vitejs/plugin-react": "3.1.0",
-    "better-sqlite3": "8.2.0",
+    "@sendgrid/mail": "8.1.0",
+    "@vitejs/plugin-react": "4.2.1",
+    "better-sqlite3": "8.6.0",
     "body-parser": "1.20.2",
     "cookie-session": "2.0.0",
-    "handlebars": "4.7.7",
-    "highcharts": "10.3.3",
-    "highcharts-react-official": "3.2.0",
+    "handlebars": "4.7.8",
+    "highcharts": "11.2.0",
+    "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
-    "lowdb": "5.1.0",
+    "lowdb": "6.0.1",
     "markdown": "^0.5.0",
-    "nanoid": "4.0.1",
-    "node-fetch": "3.3.1",
-    "node-mailjet": "6.0.2",
+    "nanoid": "5.0.4",
+    "node-fetch": "3.3.2",
+    "node-mailjet": "6.0.5",
     "query-string": "8.1.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
-    "react-redux": "8.0.5",
+    "react-redux": "9.0.4",
     "react-router": "5.2.1",
     "react-router-dom": "5.3.0",
-    "redux": "4.2.1",
-    "redux-thunk": "2.4.2",
+    "redux": "5.0.1",
+    "redux-thunk": "3.1.0",
     "restana": "4.9.7",
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "4.2.0",
+    "vite": "5.0.10",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.21.3",
-    "@babel/eslint-parser": "7.21.3",
-    "@babel/preset-env": "7.20.2",
-    "@babel/preset-react": "7.18.6",
-    "chai": "4.3.7",
-    "eslint": "8.36.0",
-    "eslint-config-prettier": "8.7.0",
-    "eslint-plugin-react": "7.32.2",
-    "esmock": "2.1.0",
+    "@babel/core": "7.23.7",
+    "@babel/eslint-parser": "7.23.3",
+    "@babel/preset-env": "7.23.7",
+    "@babel/preset-react": "7.23.3",
+    "chai": "5.0.0",
+    "eslint": "8.56.0",
+    "eslint-config-prettier": "8.8.0",
+    "eslint-plugin-react": "7.33.2",
+    "esmock": "2.6.0",
     "history": "5.3.0",
     "husky": "4.3.8",
-    "less": "4.1.3",
-    "lint-staged": "13.2.0",
+    "less": "4.2.0",
+    "lint-staged": "13.2.2",
     "mocha": "10.2.0",
-    "prettier": "2.8.5",
+    "prettier": "3.1.1",
     "redux-logger": "3.0.6"
   }
 }

test/provider/einsAImmobilien.test.js

@@ -1,11 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { providerConfig, mockFredy } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/einsAImmobilien.js';
 
-const expect = chai.expect;
-
 describe('#einsAImmobilien testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immobilienDe.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { providerConfig, mockFredy } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immobilienDe.js';
-const expect = chai.expect;
+
 describe('#immobilien.de testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immonet.test.js

@@ -1,10 +1,10 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immonet.js';
 import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const expect = chai.expect;
+
 describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immoscout.test.js

@@ -1,10 +1,10 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoscout.js';
 import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
-const expect = chai.expect;
+
 describe('#immoscout testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immoswp.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoswp.js';
-const expect = chai.expect;
+
 describe('#immoswp testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/immowelt.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/immowelt.js';
-const expect = chai.expect;
+
 describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/kleinanzeigen.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/kleinanzeigen.js';
-const expect = chai.expect;
+
 describe('#kleinanzeigen testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/neubauKompass.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
-const expect = chai.expect;
+
 describe('#neubauKompass testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();

test/provider/utils.test.js

@@ -1,7 +1,7 @@
 import utils from '../../lib/utils.js';
 import assert from 'assert';
-import chai from 'chai';
-const expect = chai.expect;
+import { expect } from 'chai';
+
 const fakeWorkingHoursConfig = (from, to) => ({
   workingHours: {
     to,

test/provider/wgGesucht.test.js

@@ -1,9 +1,9 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { get } from '../mocks/mockNotification.js';
 import { mockFredy, providerConfig } from '../utils.js';
-import chai from 'chai';
+import { expect } from 'chai';
 import * as provider from '../../lib/provider/wgGesucht.js';
-const expect = chai.expect;
+
 describe('#wgGesucht testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
@@ -23,7 +23,6 @@ describe('#wgGesucht testsuite()', () => {
           expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.details).to.be.a('string');
-          expect(notify.size).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.link).to.be.a('string');
         });

test/queryStringMutator/queryStringMutator.test.js

@@ -1,16 +1,15 @@
 import fs from 'fs';
-import chai from 'chai';
+import { expect } from 'chai';
 import { readFile } from 'fs/promises';
 import mutator from '../../lib/services/queryStringMutator.js';
 import queryString from 'query-string';
-const expect = chai.expect;
 
 const data = await readFile(new URL('./testData.json', import.meta.url));
 
 const testData = JSON.parse(data);
 
 let _provider = await Promise.all(
-  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`))
+  fs.readdirSync('./lib/provider/').map(async (integPath) => await import(`../../lib/provider/${integPath}`)),
 );
 
 /**

test/similarity/similarity.test.js

@@ -1,6 +1,6 @@
 import SimilarityCacheEntry from '../../lib/services/similarity-check/SimilarityCacheEntry.js';
-import chai from 'chai';
-const expect = chai.expect;
+import { expect } from 'chai';
+
 describe('similarityCheck', () => {
   describe('#similarityCheck()', () => {
     it('should be false', () => {
@@ -29,10 +29,10 @@ describe('similarityCheck', () => {
     it('should be false', () => {
       const check = new SimilarityCacheEntry(0);
       check.setCacheEntry(
-        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.'
+        'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.',
       );
       check.setCacheEntry(
-        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
+        'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
     });
   });