fixing kleinanzeigen test

* building multi platform docker images

* upgrading dependencies | using scraping ant for neubaukompass

.github/workflows/docker.yml

@@ -44,3 +44,4 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2.5.1
         with:
-          node-version: 18
+          node-version: 20
           cache: 'yarn'
       - run: yarn install
       - run: yarn run test

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18
+FROM node:20
 
 WORKDIR  /fredy
 

README.md

@@ -17,7 +17,7 @@ _Fredy_ is supported by JetBrains under Open Source Support Program
 
 ## Usage  
 
-- Make sure to use Node.js 18 or above
+- Make sure to use Node.js 20 or above
 - Run the following commands:
 ```ssh
 yarn (or npm install)
@@ -78,8 +78,8 @@ yarn run test
 # Architecture
 ![Architecture](/doc/architecture.jpg "Architecture")
 
-### Immoscout / Immonet
+### Immoscout / Immonet / NeubauKompass
-I have added **experimental** support for Immoscout and Immonet. They both are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
+I have added **experimental** support for Immoscout, Immonet and NeubauKompass. They all are somewhat special, because they have decided to secure their service from bots using Re-Capture. Finding a way around this is barely possible. For _Fredy_ to be able to bypass this check, I'm using a service called [ScrapingAnt](https://scrapingant.com/). The trick is to use a headless browser, rotating proxies and (once successfully validated) to re-send the cookies each time.
 
 To be able to use Immoscout / Immonet, you need to create an account at ScrapingAnt. Configure the API key in the "General Settings" tab (visible when logged in as administrator).
 The rest will be handled by _Fredy_. Keep in mind, the support is experimental. There might be bugs and you might not always pass the re-capture check, but most of the time it works rather well :)

lib/provider/einsAImmobilien.js

@@ -1,18 +1,22 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
+
 function normalize(o) {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
   if (o.rooms != null) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-  return Object.assign(o, { size, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, size, link });
 }
+
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
   url: null,
   crawlContainer: '.tabelle',

lib/provider/immobilienDe.js

@@ -1,4 +1,4 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
@@ -7,12 +7,12 @@ function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
 function normalize(o) {
-  const id = parseId(shortenLink(o.link));
   const size = o.size || 'N/A m²';
   const price = o.price || 'N/A €';
   const title = o.title || 'No title available';
   const address = o.address || 'No address available';
   const link = shortenLink(o.link);
+  const id = buildHash(parseId(shortenLink(o.link)), o.price);
   return Object.assign(o, { id, price, size, title, address, link });
 }
 function applyBlacklist(o) {

lib/provider/immonet.js

@@ -1,12 +1,12 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function normalize(o) {
-  const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
   const address = o.address.split(' • ')[o.address.split(' • ').length - 1];
   const title = o.title || 'No title available';
   const link = o.id;
+  const id = buildHash(o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length), price);
   return Object.assign(o, { id, address, price, size, title, link });
 }
 function applyBlacklist(o) {

lib/provider/immoscout.js

@@ -1,4 +1,4 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 let appliedBlackList = [];
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
@@ -6,8 +6,9 @@ function nullOrEmpty(val) {
 function normalize(o) {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
-  const link = nullOrEmpty(o.address) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
+  const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
-  return Object.assign(o, { title, address, link });
+  const id = buildHash(o.id, o.price);
+  return Object.assign(o, { id, title, address, link });
 }
 function applyBlacklist(o) {
   return !utils.isOneOf(o.title, appliedBlackList);

lib/provider/immoswp.js

@@ -1,14 +1,15 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-    const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
     const size = o.size || 'N/A m²';
     const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
     const title = o.title || 'No title available';
-    const link = `https://immo.swp.de/immobilien/${id}`;
+    const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
+    const link = `https://immo.swp.de/immobilien/${immoId}`;
     const description = o.description;
+    const id = buildHash(immoId, price);
     return Object.assign(o, {id, price, size, title, link, description});
 }
 

lib/provider/immowelt.js

@@ -1,24 +1,29 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  return o;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id});
 }
+
 function applyBlacklist(o) {
     const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
     const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
     return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
     url: null,
-  crawlContainer: "div[class^='EstateItem-']",
+    crawlContainer: 'div[data-testid="serp-card-testid"]',
     sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
     crawlFields: {
         id: 'a@id',
-    price: "div[class^='KeyFacts-'] [data-test='price'] | removeNewline | trim",
+        price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
-    size: "div[class^='KeyFacts-'] [data-test='area'] | removeNewline | trim",
+        size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: "div[class^='FactsMain-'] h2",
+        title: '.css-1cbj9xw',
         link: 'a@href',
-    address: "div[class^='estateFacts-'] span | removeNewline | trim",
+        address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
     },
     paginate: '#pnlPaging #nlbPlus@href',
     normalize: normalize,

lib/provider/kleinanzeigen.js

@@ -1,10 +1,14 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
+
 function normalize(o) {
     const size = o.size || '--- m²';
-  return Object.assign(o, { size });
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id, size});
 }
+
 function applyBlacklist(o) {
     const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
     const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
@@ -12,6 +16,7 @@ function applyBlacklist(o) {
         appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
     return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
     url: null,
     crawlContainer: '#srchrslt-adtable .ad-listitem ',

lib/provider/neubauKompass.js

@@ -1,11 +1,21 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
-function normalize(o) {
+
-  return o;
+function nullOrEmpty(val) {
+    return val == null || val.length === 0;
 }
+
+function normalize(o) {
+    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id, link});
+}
+
 function applyBlacklist(o) {
     return !utils.isOneOf(o.title, appliedBlackList);
 }
+
 const config = {
     url: null,
     crawlContainer: '.nbk-container >div article',

lib/provider/wgGesucht.js

@@ -1,13 +1,18 @@
-import utils from '../utils.js';
+import utils, {buildHash} from '../utils.js';
+
 let appliedBlackList = [];
+
 function normalize(o) {
-  return o;
+    const id = buildHash(o.id, o.price);
+    return Object.assign(o, {id});
 }
+
 function applyBlacklist(o) {
     const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
     const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
     return o.id != null && titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
     url: null,
     crawlContainer: '#main_column .wgg_card',

lib/services/scrapingAnt.js

@@ -1,5 +1,6 @@
 import { metaInformation as immoScoutInfo } from '../provider/immoscout.js';
 import { metaInformation as immoNetInfo } from '../provider/immonet.js';
+import { metaInformation as neuBauCompassInfo } from '../provider/neubauKompass.js';
 import { config } from '../utils.js';
 
 const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js_snippet=${Buffer.from(
@@ -7,7 +8,7 @@ const additionalImmonetUrlParams = `&wait_for_selector=.content-wrapper-tiles&js
 ).toString('base64')}`;
 
 const needScrapingAnt = (id) => {
-  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id;
+  return id.toLowerCase() === immoScoutInfo.id || id.toLowerCase() === immoNetInfo.id || id.toLowerCase() === neuBauCompassInfo.id.toLowerCase();
 };
 export const transformUrlForScrapingAnt = (url, id) => {
   let urlParams = '';

lib/utils.js

@@ -1,6 +1,7 @@
 import {dirname} from 'node:path';
 import {fileURLToPath} from 'node:url';
 import {readFile} from 'fs/promises';
+import {createHash} from 'crypto';
 
 function isOneOf(word, arr) {
     if (arr == null || arr.length === 0) {
@@ -10,9 +11,11 @@ function isOneOf(word, arr) {
     const blacklist = new RegExp(expression, 'ig');
     return blacklist.test(word);
 }
+
 function nullOrEmpty(val) {
     return val == null || val.length === 0;
 }
+
 function timeStringToMs(timeString, now) {
     const d = new Date(now);
     const parts = timeString.split(':');
@@ -21,6 +24,7 @@ function timeStringToMs(timeString, now) {
     d.setSeconds(0);
     return d.getTime();
 }
+
 function duringWorkingHoursOrNotSet(config, now) {
     const {workingHours} = config;
     if (workingHours == null || nullOrEmpty(workingHours.from) || nullOrEmpty(workingHours.to)) {
@@ -35,6 +39,19 @@ function getDirName() {
     return dirname(fileURLToPath(import.meta.url));
 }
 
+function buildHash(...inputs) {
+    if (inputs == null) {
+        return null;
+    }
+    const cleaned = inputs.filter(i => i != null && i.length > 0);
+    if (cleaned.length === 0) {
+        return null;
+    }
+    return createHash('sha256')
+        .update(cleaned.join(','))
+        .digest('hex');
+}
+
 const config = JSON.parse(await readFile(new URL('../conf/config.json', import.meta.url)));
 
 export {isOneOf};
@@ -42,6 +59,7 @@ export { nullOrEmpty };
 export {duringWorkingHoursOrNotSet};
 export {getDirName};
 export {config};
+export {buildHash};
 export default {
     isOneOf,
     nullOrEmpty,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "8.0.7",
+  "version": "10.0.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",
@@ -45,7 +45,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=20.0.0",
     "npm": ">=7.0.0"
   },
   "browserslist": [
@@ -55,7 +55,7 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.60.0",
+    "@douyinfe/semi-ui": "2.65.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.3",
@@ -64,14 +64,14 @@
     "body-parser": "1.20.2",
     "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "11.4.3",
+    "highcharts": "11.4.8",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
     "markdown": "^0.5.0",
     "nanoid": "5.0.7",
     "node-fetch": "3.3.2",
-    "node-mailjet": "6.0.5",
+    "node-mailjet": "6.0.6",
     "query-string": "8.2.0",
     "react": "18.3.1",
     "react-dom": "18.3.1",
@@ -84,25 +84,25 @@
     "serve-static": "1.15.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.2.13",
+    "vite": "5.4.3",
     "x-ray": "2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "7.24.7",
+    "@babel/core": "7.25.2",
-    "@babel/eslint-parser": "7.24.7",
+    "@babel/eslint-parser": "7.25.1",
-    "@babel/preset-env": "7.24.7",
+    "@babel/preset-env": "7.25.4",
     "@babel/preset-react": "7.24.7",
     "chai": "5.1.1",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.34.2",
+    "eslint-plugin-react": "7.35.2",
-    "esmock": "2.6.5",
+    "esmock": "2.6.7",
     "history": "5.3.0",
     "husky": "4.3.8",
     "less": "4.2.0",
     "lint-staged": "13.2.2",
-    "mocha": "10.4.0",
+    "mocha": "10.7.3",
-    "prettier": "3.3.2",
+    "prettier": "3.3.3",
     "redux-logger": "3.0.6"
   }
 }

test/provider/einsAImmobilien.test.js

@@ -20,7 +20,7 @@ describe('#einsAImmobilien testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('einsAImmobilien');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.price).to.be.a('string');
           expect(notify.size).to.be.a('string');
           expect(notify.title).to.be.a('string');

test/provider/kleinanzeigen.test.js

@@ -20,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => {
         expect(notificationObj.serviceName).to.equal('kleinanzeigen');
         notificationObj.payload.forEach((notify) => {
           /** check the actual structure **/
-          expect(notify.id).to.be.a('number');
+          expect(notify.id).to.be.a('string');
           expect(notify.title).to.be.a('string');
           expect(notify.link).to.be.a('string');
           expect(notify.address).to.be.a('string');

test/provider/neubauKompass.test.js

@@ -3,6 +3,7 @@ import { get } from '../mocks/mockNotification.js';
 import {mockFredy, providerConfig} from '../utils.js';
 import {expect} from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
+import * as scrapingAnt from '../../lib/services/scrapingAnt.js';
 
 describe('#neubauKompass testsuite()', () => {
     after(() => {
@@ -12,6 +13,13 @@ describe('#neubauKompass testsuite()', () => {
     it('should test neubauKompass provider', async () => {
         const Fredy = await mockFredy();
         return await new Promise((resolve) => {
+            if (!scrapingAnt.isScrapingAntApiKeySet()) {
+                /* eslint-disable no-console */
+                console.info('Skipping Neubaukompass test as ScrapingAnt Api Key is not set.');
+                /* eslint-enable no-console */
+                resolve();
+                return;
+            }
             const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
             fredy.execute().then((listing) => {
                 expect(listing).to.be.a('array');

test/utils/utils.test.js

@@ -0,0 +1,16 @@
+import { expect } from 'chai';
+import {buildHash} from '../../lib/utils.js';
+
+describe('utilsCheck', () => {
+  describe('#utilsCheck()', () => {
+      it('should be null when null input', () => {
+          expect(buildHash(null)).to.be.null;
+      });
+      it('should be null when null empty', () => {
+          expect(buildHash('')).to.be.null;
+      });
+      it('should return a value', () => {
+          expect(buildHash('bla', '', null)).to.be.a.string;
+      });
+  });
+});

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              If you chose Immoscout or Immonet as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
+              If you chose Immoscout, Immonet or NeubauKompass as a provider, make sure to also add the scrapingAnt apiKey to the config.json.
               (See readme)
             </p>
             <p>