next release version

.dockerignore

@@ -1,7 +1,7 @@
 node_modules/
 npm-debug.log
 test/
-conf/
 db/
+conf/
 .git/
 .github/

.github/workflows/docker.yml

@@ -18,6 +18,10 @@ jobs:
       contents: read
       packages: write
 
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -51,3 +55,5 @@ jobs:
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

CONTRIBUTING.md

@@ -113,7 +113,7 @@ I'm using Eslint to maintain quote style and quality. Do not skip it...
 
 ##### To do before merging:
 
-- executed tests? (`yarn run test`)
+- executed tests? (`pnpm test`)
 - sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
 
 _Thanks!_ :heart:

Dockerfile

@@ -1,28 +1,35 @@
-FROM node:20
+FROM node:22-slim
 
 WORKDIR /fredy
 
-COPY . /fredy
+# Install Chromium without extra recommended packages and clean apt cache
-
+RUN apt-get update \
-RUN apt-get update && apt-get install -y chromium
+  && apt-get install -y --no-install-recommends chromium \
+  && rm -rf /var/lib/apt/lists/*
 
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
-  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
-# Timeout fix für yarn hinzugefügt
+# Copy lockfiles first to leverage cache for dependencies
-RUN yarn config set network-timeout 600000
+COPY package.json yarn.lock ./
 
-RUN yarn install
+# Set Yarn timeout, install dependencies and PM2 globally
-
+RUN yarn config set network-timeout 600000 \
-RUN yarn global add pm2
+  && yarn install --frozen-lockfile \
+  && yarn global add pm2
 
+# Copy application source and build production assets
+COPY . ./
 RUN yarn run prod
 
-RUN mkdir /db /conf && \
+# Prepare runtime directories and symlinks for data and config
-  chown 1000:1000 /db /conf && \
+RUN mkdir -p /db /conf \
-  chmod 777 -R /db/ && \
+  && chown 1000:1000 /db /conf \
-  ln -s /db /fredy/db && ln -s /conf /fredy/conf
+  && chmod 777 /db /conf \
+  && ln -s /db /fredy/db \
+  && ln -s /conf /fredy/conf
 
 EXPOSE 9998
 
-CMD pm2-runtime index.js
+# Start application using PM2 runtime
+CMD ["pm2-runtime", "index.js"]

README.md

@@ -8,8 +8,6 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
 
 If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
 
-<a href="https://www.producthunt.com/posts/fredy-find-real-estates-damn-easy?embed=true&utm_source=badge-featured&utm_medium=badge&utm_source=badge-fredy&#0045;find&#0045;real&#0045;estates&#0045;damn&#0045;easy" target="_blank"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=965690&theme=light&t=1747292331626" alt="Fredy&#0032;&#0045;&#0032;Find&#0032;Real&#0032;Estates&#0032;Damn&#0032;EasY&#0032; - Your&#0032;personal&#0032;real&#0032;estate&#0032;search&#0032;bot | Product Hunt" style="width: 250px; height: 54px;" width="250" height="54" /></a>
-
 # Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
 If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
 
@@ -48,7 +46,7 @@ A provider contains the URL that points to the search results for the respective
 **It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
 
 #### Adapter
-_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. An adapter dictates how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
 
 #### Jobs
 A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
@@ -84,7 +82,7 @@ yarn run test
 ![Architecture](/doc/architecture.jpg "Architecture")
 
 ### Immoscout
-Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
+Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
 
 # Analytics
 Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 

lib/api/routes/jobRouter.js

@@ -15,7 +15,7 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-  return user.isAdmin || job.userId === job.userId;
+  return user.isAdmin || job.userId === user.id;
 }
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);

lib/notification/adapter/telegram.js

@@ -22,16 +22,18 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
+  // we have to split messages into chunks, because otherwise messages are going to become too big and will fail
   const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
   const promises = chunks.map((chunk) => {
-    let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
+    const messageParagraphs = [];
-    message += chunk.map(
+
+    messageParagraphs.push(`<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:`);
+    messageParagraphs.push(...chunk.map(
       (o) =>
         `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
-        [o.address, o.price, o.size].join(' | ') +
+        [o.address, o.price, o.size].join(' | ')
-        '\n\n',
+    ));
-    );
+
     /**
      * This is to not break the rate limit. It is to only send 1 message per second
      */
@@ -41,7 +43,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
           method: 'post',
           body: JSON.stringify({
             chat_id: chatId,
-            text: message,
+            text: messageParagraphs.join('\n\n'),
             parse_mode: 'HTML',
             disable_web_page_preview: true,
           }),

lib/provider/immowelt.js

@@ -23,7 +23,7 @@ const config = {
     id: 'a@href',
     price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: '.css-jv3zx6',
+    title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },

lib/services/extractor/parser/parser.js

@@ -8,7 +8,7 @@ export function loadParser(text) {
 
 export function parse(crawlContainer, crawlFields, text, url) {
   if (!text) {
-    console.warn('Cannot parse, text was empty for url ', url);
+    console.warn('No content found for ', url);
     return null;
   }
 

lib/services/immoscout/immoscout-web-translater.js

@@ -79,7 +79,7 @@ const PARAM_NAME_MAP = {
   geocoordinates: 'geocoordinates',
   shape: 'shape',
   sorting: 'sorting',
-  newbuilding: 'newbuilding'
+  newbuilding: 'newbuilding',
 };
 
 const EQUIPMENT_MAP = {
@@ -90,7 +90,7 @@ const EQUIPMENT_MAP = {
   garden: 'garden',
   guesttoilet: 'guestToilet',
   balcony: 'balcony',
-  handicappedaccessible: 'handicappedAccessible'
+  handicappedaccessible: 'handicappedAccessible',
 };
 
 const REAL_ESTATE_TYPE = {
@@ -120,7 +120,7 @@ const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
   'wohnung-mit-keller-mieten': { equipment: ['cellar'] },
   // Category "Merkmale"
   'neubauwohnung-mieten': { newbuilding: true },
-  'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] }
+  'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
 };
 
 export function convertWebToMobile(webUrl) {
@@ -165,7 +165,7 @@ export function convertWebToMobile(webUrl) {
     searchType: isRadius ? 'radius' : 'region',
     realestatetype: realType,
     ...(isRadius ? {} : { geocodes }),
-    ...additionalParamsFromWebPath
+    ...additionalParamsFromWebPath,
   };
 
   if (webParams.geocoordinates) {
@@ -176,7 +176,10 @@ export function convertWebToMobile(webUrl) {
     if (key === 'equipment') {
       const items = [].concat(val).flatMap((v) => `${v}`.split(','));
       const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]];
-      mobileParams[PARAM_NAME_MAP[key]] = [...currentEquipmentParams ?? [], ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean)];
+      mobileParams[PARAM_NAME_MAP[key]] = [
+        ...(currentEquipmentParams ?? []),
+        ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
+      ];
     } else {
       mobileParams[PARAM_NAME_MAP[key]] = val;
     }

lib/services/storage/userStorage.js

@@ -1,5 +1,5 @@
 import { JSONFileSync } from 'lowdb/node';
-import {config, getDirName} from '../../utils.js';
+import { config, getDirName } from '../../utils.js';
 import * as hasher from '../security/hash.js';
 import { nanoid } from 'nanoid';
 import * as jobStorage from './jobStorage.js';
@@ -7,23 +7,23 @@ import path from 'path';
 import LowdashAdapter from './LowDashAdapter.js';
 
 const defaultData = {
-    user: [
+  user: [
-        //you probably want to change the default password ;)
+    //you probably want to change the default password ;)
-        {
+    {
-            id: nanoid(),
+      id: nanoid(),
-            lastLogin: Date.now(),
+      lastLogin: Date.now(),
-            username: 'admin',
+      username: 'admin',
-            password: hasher.hash('admin'),
+      password: hasher.hash('admin'),
-            isAdmin: true,
+      isAdmin: true,
-        },
+    },
-        {
+    {
-            id: nanoid(),
+      id: nanoid(),
-            lastLogin: Date.now(),
+      lastLogin: Date.now(),
-            username: 'demo',
+      username: 'demo',
-            password: hasher.hash('demo'),
+      password: hasher.hash('demo'),
-            isAdmin: true,
+      isAdmin: true,
-        },
+    },
-    ],
+  ],
 };
 
 const file = path.join(getDirName(), '../', 'db/users.json');
@@ -86,34 +86,38 @@ export const removeUser = (userId) => {
   db.chain
     .set(
       'user',
-      user.filter((u) => u.id !== userId)
+      user.filter((u) => u.id !== userId),
     )
     .value();
   db.write();
 };
 
 export const handleDemoUser = () => {
-    if(!config.demoMode){
+  if (!config.demoMode) {
-        const user = db.chain.get('user').value();
+    const user = db.chain.get('user').value();
-        db.chain.get('user').value();
+    db.chain
-        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+      .set(
-        db.write();
+        'user',
-    }else {
+        user.filter((u) => u.username !== 'demo'),
-        const demoUser = db.chain
+      )
-            .get('user')
+      .value();
-            .filter((u) => u.username === 'demo')
+    db.write();
-            .value();
+  } else {
-        if (demoUser == null || demoUser.length === 0) {
+    const demoUser = db.chain
-            db.chain.get('user')
+      .get('user')
-                .value()
+      .filter((u) => u.username === 'demo')
-                .push({
+      .value();
-                    id: nanoid(),
+    if (demoUser == null || demoUser.length === 0) {
-                    username: 'demo',
+      db.chain
-                    password: hasher.hash('demo'),
+        .get('user')
-                    isAdmin: true,
+        .value()
-                });
+        .push({
-            db.write();
+          id: nanoid(),
-        }
+          username: 'demo',
+          password: hasher.hash('demo'),
+          isAdmin: true,
+        });
+      db.write();
     }
+  }
 };
-

lib/utils.js

@@ -9,12 +9,9 @@ function inDevMode(){
 }
 
 function isOneOf(word, arr) {
-    if (arr == null || arr.length === 0) {
+    if (!arr || arr.length === 0 || word == null) return false;
-        return false;
+    const lowerWord = word.toLowerCase();
-    }
+    return arr.some(item => lowerWord.indexOf(item.toLowerCase()) !== -1);
-    const expression = String.raw`\b(${arr.join('|')})\b`;
-    const blacklist = new RegExp(expression, 'ig');
-    return blacklist.test(word);
 }
 
 function nullOrEmpty(val) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.2.1",
+  "version": "11.2.6",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node prod.js",
@@ -50,17 +50,17 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.79.0",
+    "@douyinfe/semi-ui": "2.83.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.5",
-    "@vitejs/plugin-react": "4.4.1",
+    "@vitejs/plugin-react": "4.7.0",
     "better-sqlite3": "^11.10.0",
     "body-parser": "2.2.0",
-    "cheerio": "^1.0.0",
+    "cheerio": "^1.1.0",
-    "cookie-session": "2.1.0",
+    "cookie-session": "2.1.1",
     "handlebars": "4.7.8",
-    "highcharts": "12.2.0",
+    "highcharts": "12.3.0",
     "highcharts-react-official": "3.2.2",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
@@ -70,10 +70,10 @@
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.8",
     "package-up": "^5.0.0",
-    "puppeteer": "^24.8.2",
+    "puppeteer": "^24.14.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
-    "query-string": "9.1.2",
+    "query-string": "9.2.2",
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "react-redux": "9.2.0",
@@ -81,28 +81,28 @@
     "react-router-dom": "5.3.0",
     "redux": "5.0.1",
     "redux-thunk": "3.1.0",
-    "restana": "4.9.9",
+    "restana": "5.0.0",
-    "serve-static": "1.16.2",
+    "serve-static": "2.2.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.4.11"
+    "vite": "7.0.5"
   },
   "devDependencies": {
-    "@babel/core": "7.27.1",
+    "@babel/core": "7.27.3",
-    "@babel/eslint-parser": "7.27.1",
+    "@babel/eslint-parser": "7.27.5",
     "@babel/preset-env": "7.27.2",
     "@babel/preset-react": "7.27.1",
-    "chai": "5.2.0",
+    "chai": "5.2.1",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.37.4",
+    "eslint-plugin-react": "7.37.5",
-    "esmock": "2.7.0",
+    "esmock": "2.7.1",
     "history": "5.3.0",
     "husky": "9.1.7",
-    "less": "4.3.0",
+    "less": "4.4.0",
     "lint-staged": "15.5.2",
     "mocha": "10.8.2",
-    "prettier": "3.5.3",
+    "prettier": "3.6.2",
     "redux-logger": "3.0.6"
   }
 }

test/provider/testProvider.json

@@ -37,7 +37,7 @@
     "enabled": true
   },
   "wgGesucht": {
-    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
+    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
     "enabled": true
   }
 }

test/services/immoscout/immoscout-web-translater.test.js

@@ -18,8 +18,7 @@ describe('#immoscout-mobile URL conversion', () => {
 
   // Test URL conversion of web-only SEO path
   it('should convert a SEO web path to the correct query params', () => {
-    const webUrl =
+    const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
-      'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
 
     const converted = convertWebToMobile(webUrl);
     const queryParams = new URL(converted).searchParams;

test/similarity/similarity.test.js

@@ -34,6 +34,7 @@ describe('similarityCheck', () => {
       check.setCacheEntry(
         'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
+      expect(check.hasSimilarEntries('unrelated text')).to.be.false;
     });
   });
 });

ui/src/App.less

@@ -41,3 +41,7 @@ a:active {
   background-color: transparent;
   text-decoration: underline;
 }
+
+.semi-icon {
+  vertical-align: middle;
+}

ui/src/components/table/ProviderTable.jsx

@@ -30,7 +30,7 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
           render: (_, record) => {
             return (
               <div style={{ float: 'right' }}>
-                <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
+                <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
               </div>
             );
           },

ui/src/components/tracking/TrackingModal.jsx

@@ -4,6 +4,7 @@ import Logo from '../logo/Logo.jsx';
 import {xhrPost} from '../../services/xhr.js';
 
 import './TrackingModal.less';
+import inDevelopment from '../../services/developmentMode.js';
 
 const saveResponse = async (analyticsEnabled) => {
     await xhrPost('/api/admin/generalSettings', {
@@ -12,6 +13,9 @@ const saveResponse = async (analyticsEnabled) => {
 };
 
 export default function TrackingModal() {
+    if(inDevelopment()){
+        return null;
+    }
 
     return <Modal
         visible={true}

ui/src/services/developmentMode.js

@@ -0,0 +1,4 @@
+export default function isDevelopmentMode(){
+    const inDevMode= import.meta.env.MODE;
+    return inDevMode != null && inDevMode === 'development';
+}

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import {format} from '../../services/time/timeService';
-import {Banner, Descriptions} from '@douyinfe/semi-ui';
+import {Descriptions} from '@douyinfe/semi-ui';
 
 export default function ProcessingTimes({processingTimes = {}}) {
     if (Object.keys(processingTimes).length === 0) {

ui/src/views/jobs/mutation/JobMutation.jsx

@@ -94,7 +94,7 @@ export default function JobMutator() {
       <form>
         <SegmentPart name="Name">
           <Input
-            autofocus
+            autoFocus
             type="text"
             maxLength={40}
             placeholder="Name"
@@ -124,8 +124,8 @@ export default function JobMutator() {
 
           <ProviderTable
             providerData={providerData}
-            onRemove={(providerId) => {
+            onRemove={(providerUrl) => {
-              setProviderData(providerData.filter((provider) => provider.id !== providerId));
+              setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
             }}
           />
         </SegmentPart>

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              Currently, our Immoscout implementation does not drawing shapes on a map. Use a radius instead.
+              Currently, our Immoscout implementation does not support drawing shapes on a map. Use a radius instead.
             </p>
           </div>
         }