next fredy version

* building multi platform docker images

* upgrading dependencies | using scraping ant for neubaukompass

.github/workflows/docker.yml

@@ -44,3 +44,4 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 18
+          node-version: 20
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18
+FROM node:20
 
 WORKDIR  /fredy
 

README.md

@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use Node.js 18 or above
+- Make sure to use Node.js 20 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -78,8 +78,8 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-### Immoscout / Immonet
+### Immoscout / Immonet / NeubauKompass
-I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+I have added **experimental** support for Immoscout, Immonet and NeubauKompass. They all are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
 To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
 The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)

lib/provider/immoscout.js

@@ -6,7 +6,7 @@ function nullOrEmpty(val) {
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
   return Object.assign(o, { title, address, link });
 }
 function applyBlacklist(o) {

lib/provider/neubauKompass.js

@@ -1,7 +1,11 @@
 import utils from '../utils.js';
 let appliedBlackList = [];
+function nullOrEmpty(val) {
+    return val == null || val.length === 0;
+}
 function normalize(o) {
-  return o;
+    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+  return {...o, link};
 }
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);

lib/services/requestDriver.js

@@ -24,13 +24,16 @@ function makeDriver(headers = {}) {
         },
       });
       const result = await response.text();
+      if (EXPECTED_STATUS_CODES.includes(response.status)) {
+        throw new Error(`${response.status}`);
+      }
       if (cookies.length === 0) {
         cookies = response.headers.raw()['set-cookie'] || [];
       }
       callback(null, result);
     } catch (exception) {
       /* eslint-disable no-console */
-      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status)) {
+      if (!EXPECTED_STATUS_CODES.includes(exception.response?.status) && !EXPECTED_STATUS_CODES.includes(Number(exception.message))) {
         console.error(`Error while trying to scrape data from scraping ant. Received error: ${exception.message}`);
         callback(null, []);
         return;

lib/services/scrapingAnt.js

@@ -1,5 +1,6 @@
 import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
 import { metaInformation as immoNetInfo } from '../provider/immonet.js';
+import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
 import { config } from '../utils.js';
 
 const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
@@ -7,7 +8,7 @@ const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js
 ).toString('base64')}`;
 
 const needScrapingAnt = (id) => {
-  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
 };
 export const transformUrlForScrapingAnt = (url, id) => {
   let urlParams = '';

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "8.0.6",
+  "version": "9.0.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -45,7 +45,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=20.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -55,7 +55,7 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.60.0",
+    "@douyinfe/semi-ui": "2.62.1",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.3",
@@ -64,7 +64,7 @@
     "body-parser": "1.20.2",
     "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "11.4.3",
+    "highcharts": "11.4.6",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
@@ -84,25 +84,25 @@
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.2.13",
+    "vite": "5.3.4",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.24.7",
+    "@babel/core": "7.24.9",
-    "@babel/eslint-parser": "7.24.7",
+    "@babel/eslint-parser": "7.24.8",
-    "@babel/preset-env": "7.24.7",
+    "@babel/preset-env": "7.24.8",
     "@babel/preset-react": "7.24.7",
     "chai": "5.1.1",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.34.2",
+    "eslint-plugin-react": "7.35.0",
-    "esmock": "2.6.5",
+    "esmock": "2.6.7",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.2.0",
     "lint-staged": "13.2.2",
-    "mocha": "10.4.0",
+    "mocha": "10.7.0",
-    "prettier": "3.3.2",
+    "prettier": "3.3.3",
     "redux-logger": "3.0.6"
   }
 }

test/provider/neubauKompass.test.js

@@ -1,36 +1,44 @@
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
+import {get} from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
+import {mockFredy, providerConfig} from '../utils.js';
-import { expect } from 'chai';
+import {expect} from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
 
 describe('#neubauKompass testsuite()', () => {
-  after(() => {
+    after(() => {
-    similarityCache.stopCacheCleanup();
+        similarityCache.stopCacheCleanup();
-  });
+    });
-  provider.init(providerConfig.neubauKompass, [], []);
+    provider.init(providerConfig.neubauKompass, [], []);
-  it('should test neubauKompass provider', async () => {
+    it.only('should test neubauKompass provider', async () => {
-    const Fredy = await mockFredy();
+        const Fredy = await mockFredy();
-    return await new Promise((resolve) => {
+        return await new Promise((resolve) => {
-      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
+            if (!scrapingAnt.isScrapingAntApiKeySet()) {
-      fredy.execute().then((listing) => {
+                /* eslint-disable no-console */
-        expect(listing).to.be.a('array');
+                console.info('Skipping Neubaukompass test as ScrapingAnt Api Key is not set.');
-        const notificationObj = get();
+                /* eslint-enable no-console */
-        expect(notificationObj.serviceName).to.equal('neubauKompass');
+                resolve();
-        notificationObj.payload.forEach((notify) => {
+                return;
-          expect(notify).to.be.a('object');
+            }
-          /** check the actual structure **/
+            const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
-          expect(notify.id).to.be.a('string');
+            fredy.execute().then((listing) => {
-          expect(notify.title).to.be.a('string');
+                expect(listing).to.be.a('array');
-          expect(notify.link).to.be.a('string');
+                const notificationObj = get();
-          expect(notify.address).to.be.a('string');
+                expect(notificationObj.serviceName).to.equal('neubauKompass');
-          /** check the values if possible **/
+                notificationObj.payload.forEach((notify) => {
-          expect(notify.title).to.be.not.empty;
+                    expect(notify).to.be.a('object');
-          expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    /** check the actual structure **/
-          expect(notify.address).to.be.not.empty;
+                    expect(notify.id).to.be.a('string');
-        });
+                    expect(notify.title).to.be.a('string');
-        resolve();
+                    expect(notify.link).to.be.a('string');
-      });
+                    expect(notify.address).to.be.a('string');
+                    /** check the values if possible **/
+                    expect(notify.title).to.be.not.empty;
+                    expect(notify.link).that.does.include('https://www.neubaukompass.de');
+                    expect(notify.address).to.be.not.empty;
+                });
+                resolve();
+            });
+        });
     });
-  });
 });

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              If you chose Immoscout, Immonet or NeubauKompass as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
               (See readme)
             </p>
             <p>