next version

.dockerignore

@@ -1,7 +1,7 @@
 node_modules/
 npm-debug.log
 test/
-conf/
 db/
+conf/
 .git/
 .github/

.github/workflows/docker.yml

@@ -1,4 +1,5 @@
 name: Create and publish Docker image
+
 on:
   push:
     branches:
@@ -17,15 +18,24 @@ jobs:
       contents: read
       packages: write
 
-    steps:
+    concurrency:
-      - name: Set up Docker Buildx
+      group: ${{ github.workflow }}-${{ github.ref }}
-        uses: docker/setup-buildx-action@v1
+      cancel-in-progress: true
 
+    steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -33,15 +43,17 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64, linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

CONTRIBUTING.md

@@ -113,7 +113,7 @@ I'm using Eslint to maintain quote style and quality. Do not skip it...
 
 ##### To do before merging:
 
-- executed tests? (`yarn run test`)
+- executed tests? (`pnpm test`)
 - sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
 
 _Thanks!_ :heart:

Dockerfile

@@ -1,25 +1,35 @@
-FROM node:20
+FROM node:22-slim
 
-WORKDIR  /fredy
+WORKDIR /fredy
 
-COPY . /fredy
+# Install Chromium without extra recommended packages and clean apt cache
-
+RUN apt-get update \
-RUN apt-get update && apt-get install -y chromium
+  && apt-get install -y --no-install-recommends chromium \
+  && rm -rf /var/lib/apt/lists/*
 
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
-  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
-RUN yarn install
+# Copy lockfiles first to leverage cache for dependencies
+COPY package.json yarn.lock ./
 
-RUN yarn global add pm2
+# Set Yarn timeout, install dependencies and PM2 globally
+RUN yarn config set network-timeout 600000 \
+  && yarn install --frozen-lockfile \
+  && yarn global add pm2
 
+# Copy application source and build production assets
+COPY . ./
 RUN yarn run prod
 
-RUN mkdir /db /conf && \
+# Prepare runtime directories and symlinks for data and config
-  chown 1000:1000 /db /conf && \
+RUN mkdir -p /db /conf \
-  chmod 777 -R /db/ && \
+  && chown 1000:1000 /db /conf \
-  ln -s /db /fredy/db && ln -s /conf /fredy/conf
+  && chmod 777 /db /conf \
+  && ln -s /db /fredy/db \
+  && ln -s /conf /fredy/conf
 
 EXPOSE 9998
 
-CMD pm2-runtime index.js
+# Start application using PM2 runtime
+CMD ["pm2-runtime", "index.js"]

README.md

@@ -1,6 +1,6 @@
 <img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">  
 
-![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
+![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)  [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
 
 Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
 
@@ -46,7 +46,7 @@ A provider contains the URL that points to the search results for the respective
 **It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
 
 #### Adapter
-_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. An adapter dictates how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
 
 #### Jobs
 A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
@@ -82,7 +82,7 @@ yarn run test
 ![Architecture](/doc/architecture.jpg "Architecture")
 
 ### Immoscout
-Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
+Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
 
 # Analytics
 Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 
@@ -110,6 +110,10 @@ Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+## Logs  
+
+You can browse the logs with  `docker logs fredy -f`.
+
 ### 👐 Contributing
 Thanks to all the people who already contributed!
 
@@ -119,6 +123,7 @@ Thanks to all the people who already contributed!
 
 See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
-## Logs  
 
-You can browse the logs with  `docker logs fredy -f`.
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=orangecoding/fredy&type=Date)](https://www.star-history.com/#orangecoding/fredy&Date)

lib/api/routes/jobRouter.js

@@ -15,7 +15,7 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-  return user.isAdmin || job.userId === job.userId;
+  return user.isAdmin || job.userId === user.id;
 }
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);

lib/provider/immowelt.js

@@ -23,7 +23,7 @@ const config = {
     id: 'a@href',
     price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: '.css-jv3zx6',
+    title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },

lib/services/demoCleanup.js

@@ -1,29 +1,37 @@
 import { setInterval } from 'node:timers';
-import {removeJobsByUserName} from './storage/jobStorage.js';
+import { removeJobsByUserName } from './storage/jobStorage.js';
-import {config} from '../utils.js';
+import { config } from '../utils.js';
+import { getUsers } from './storage/userStorage.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
  */
 export function cleanupDemoAtMidnight() {
-    const now = new Date();
+  const now = new Date();
-    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+  const millisUntilMidnightUTC =
-        - now.getUTCMinutes() * 60 * 1000
+    (24 - now.getUTCHours()) * 60 * 60 * 1000 -
-        - now.getUTCSeconds() * 1000
+    now.getUTCMinutes() * 60 * 1000 -
-        - now.getUTCMilliseconds();
+    now.getUTCSeconds() * 1000 -
+    now.getUTCMilliseconds();
 
-    setTimeout(() => {
+  cleanup();
+  setTimeout(() => {
+    setInterval(
+      () => {
         cleanup();
-
+      },
-        setInterval(() => {
+      24 * 60 * 60 * 1000,
-            cleanup();
+    );
-        }, 24 * 60 * 60 * 1000);
+  }, millisUntilMidnightUTC);
-
-    }, millisUntilMidnightUTC);
 }
 
-function cleanup(){
+function cleanup() {
-    if(config.demoMode){
+  if (config.demoMode) {
-        removeJobsByUserName('demo');
+    const demoUser = getUsers(false).find((user) => user.username === 'demo');
+    if (demoUser == null) {
+      console.error('Demo user not found, cannot remove Jobs');
+      return;
     }
-}
+    removeJobsByUserName(demoUser.id);
+  }
+}

lib/services/immoscout/immoscout-web-translater.js

@@ -79,6 +79,7 @@ const PARAM_NAME_MAP = {
   geocoordinates: 'geocoordinates',
   shape: 'shape',
   sorting: 'sorting',
+  newbuilding: 'newbuilding',
 };
 
 const EQUIPMENT_MAP = {
@@ -89,6 +90,7 @@ const EQUIPMENT_MAP = {
   garden: 'garden',
   guesttoilet: 'guestToilet',
   balcony: 'balcony',
+  handicappedaccessible: 'handicappedAccessible',
 };
 
 const REAL_ESTATE_TYPE = {
@@ -98,6 +100,29 @@ const REAL_ESTATE_TYPE = {
   'haus-kaufen': 'housebuy',
 };
 
+const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
+  // Category "Balkon/Terrasse"
+  'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
+  'wohnung-mit-garten-mieten': { equipment: ['garden'] },
+  // Category "Wohnungstyp"
+  'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
+  'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
+  'hochparterrewohnung-mieten': { apartmenttypes: ['raisedgroundfloor'] },
+  'etagenwohnung-mieten': { apartmenttypes: ['apartment'] },
+  'loft-mieten': { apartmenttypes: ['loft'] },
+  'maisonette-mieten': { apartmenttypes: ['maisonette'] },
+  'terrassenwohnung-mieten': { apartmenttypes: ['terracedflat'] },
+  'penthouse-mieten': { apartmenttypes: ['penthouse'] },
+  'dachgeschosswohnung-mieten': { apartmenttypes: ['roofstorey'] },
+  // Category "Ausstattung"
+  'wohnung-mit-garage-mieten': { equipment: ['parking'] },
+  'wohnung-mit-einbaukueche-mieten': { equipment: ['builtinkitchen'] },
+  'wohnung-mit-keller-mieten': { equipment: ['cellar'] },
+  // Category "Merkmale"
+  'neubauwohnung-mieten': { newbuilding: true },
+  'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
+};
+
 export function convertWebToMobile(webUrl) {
   let url;
   try {
@@ -112,9 +137,17 @@ export function convertWebToMobile(webUrl) {
   }
 
   const realTypeKey = segments.at(-1);
-  const realType = REAL_ESTATE_TYPE[realTypeKey];
+  let realType = REAL_ESTATE_TYPE[realTypeKey];
+  let additionalParamsFromWebPath;
+
   if (!realType) {
-    throw new Error(`Real estate type not found: ${realTypeKey}`);
+    // Test for seo optimized apartment path (only used on the ImmoScout web app)
+    if (WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey]) {
+      additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey];
+      realType = REAL_ESTATE_TYPE['wohnung-mieten'];
+    } else {
+      throw new Error(`Real estate type not found: ${realTypeKey}`);
+    }
   }
 
   if (segments.includes('shape')) {
@@ -132,6 +165,7 @@ export function convertWebToMobile(webUrl) {
     searchType: isRadius ? 'radius' : 'region',
     realestatetype: realType,
     ...(isRadius ? {} : { geocodes }),
+    ...additionalParamsFromWebPath,
   };
 
   if (webParams.geocoordinates) {
@@ -141,7 +175,11 @@ export function convertWebToMobile(webUrl) {
   for (const [key, val] of Object.entries(webParams)) {
     if (key === 'equipment') {
       const items = [].concat(val).flatMap((v) => `${v}`.split(','));
-      mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean);
+      const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]];
+      mobileParams[PARAM_NAME_MAP[key]] = [
+        ...(currentEquipmentParams ?? []),
+        ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
+      ];
     } else {
       mobileParams[PARAM_NAME_MAP[key]] = val;
     }

lib/services/storage/jobStorage.js

@@ -11,7 +11,6 @@ const db = new LowdashAdapter(adapter, { jobs: [] });
 
 db.read();
 
-
 export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
@@ -77,16 +76,25 @@ export const removeJobsByUserId = (userId) => {
     .value();
   db.write();
 };
-export const removeJobsByUserName = (userName) => {
+export const removeJobsByUserName = (userId) => {
+  let removedDemoJobs = 0;
   db.chain
-      .get('jobs')
+    .get('jobs')
-      .filter((job) => job.username === userName)
+    .filter((job) => job.userId === userId)
-      .forEach((job) => listingStorage.removeListings(job.id));
+    .forEach((job) => {
+      removedDemoJobs++;
+      listingStorage.removeListings(job.id);
+    });
   db.chain
-      .get('jobs')
+    .get('jobs')
-      .remove((job) => job.username === userName)
+    .remove((job) => job.userId === userId)
-      .value();
+    .value();
   db.write();
+  if (removedDemoJobs > 0) {
+    /* eslint-disable no-console */
+    console.log(`Removed ${removedDemoJobs} demo jobs`);
+    /* eslint-enable no-console */
+  }
 };
 export const getJobs = () => {
   return db.chain

lib/services/storage/userStorage.js

@@ -1,5 +1,5 @@
 import { JSONFileSync } from 'lowdb/node';
-import {config, getDirName} from '../../utils.js';
+import { config, getDirName } from '../../utils.js';
 import * as hasher from '../security/hash.js';
 import { nanoid } from 'nanoid';
 import * as jobStorage from './jobStorage.js';
@@ -7,23 +7,23 @@ import path from 'path';
 import LowdashAdapter from './LowDashAdapter.js';
 
 const defaultData = {
-    user: [
+  user: [
-        //you probably want to change the default password ;)
+    //you probably want to change the default password ;)
-        {
+    {
-            id: nanoid(),
+      id: nanoid(),
-            lastLogin: Date.now(),
+      lastLogin: Date.now(),
-            username: 'admin',
+      username: 'admin',
-            password: hasher.hash('admin'),
+      password: hasher.hash('admin'),
-            isAdmin: true,
+      isAdmin: true,
-        },
+    },
-        {
+    {
-            id: nanoid(),
+      id: nanoid(),
-            lastLogin: Date.now(),
+      lastLogin: Date.now(),
-            username: 'demo',
+      username: 'demo',
-            password: hasher.hash('demo'),
+      password: hasher.hash('demo'),
-            isAdmin: true,
+      isAdmin: true,
-        },
+    },
-    ],
+  ],
 };
 
 const file = path.join(getDirName(), '../', 'db/users.json');
@@ -86,34 +86,38 @@ export const removeUser = (userId) => {
   db.chain
     .set(
       'user',
-      user.filter((u) => u.id !== userId)
+      user.filter((u) => u.id !== userId),
     )
     .value();
   db.write();
 };
 
 export const handleDemoUser = () => {
-    if(!config.demoMode){
+  if (!config.demoMode) {
-        const user = db.chain.get('user').value();
+    const user = db.chain.get('user').value();
-        db.chain.get('user').value();
+    db.chain
-        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+      .set(
-        db.write();
+        'user',
-    }else {
+        user.filter((u) => u.username !== 'demo'),
-        const demoUser = db.chain
+      )
-            .get('user')
+      .value();
-            .filter((u) => u.username === 'demo')
+    db.write();
-            .value();
+  } else {
-        if (demoUser == null || demoUser.length === 0) {
+    const demoUser = db.chain
-            db.chain.get('user')
+      .get('user')
-                .value()
+      .filter((u) => u.username === 'demo')
-                .push({
+      .value();
-                    id: nanoid(),
+    if (demoUser == null || demoUser.length === 0) {
-                    username: 'demo',
+      db.chain
-                    password: hasher.hash('demo'),
+        .get('user')
-                    isAdmin: true,
+        .value()
-                });
+        .push({
-            db.write();
+          id: nanoid(),
-        }
+          username: 'demo',
+          password: hasher.hash('demo'),
+          isAdmin: true,
+        });
+      db.write();
     }
+  }
 };
-

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.2.0",
+  "version": "11.2.3",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node prod.js",
@@ -50,11 +50,11 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.79.0",
+    "@douyinfe/semi-ui": "2.80.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.5",
-    "@vitejs/plugin-react": "4.4.1",
+    "@vitejs/plugin-react": "4.5.0",
     "better-sqlite3": "^11.10.0",
     "body-parser": "2.2.0",
     "cheerio": "^1.0.0",
@@ -70,10 +70,10 @@
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.8",
     "package-up": "^5.0.0",
-    "puppeteer": "^24.8.2",
+    "puppeteer": "^24.9.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
-    "query-string": "9.1.2",
+    "query-string": "9.2.0",
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "react-redux": "9.2.0",
@@ -81,21 +81,21 @@
     "react-router-dom": "5.3.0",
     "redux": "5.0.1",
     "redux-thunk": "3.1.0",
-    "restana": "4.9.9",
+    "restana": "5.0.0",
-    "serve-static": "1.16.2",
+    "serve-static": "2.2.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.4.11"
+    "vite": "6.3.5"
   },
   "devDependencies": {
-    "@babel/core": "7.27.1",
+    "@babel/core": "7.27.3",
     "@babel/eslint-parser": "7.27.1",
     "@babel/preset-env": "7.27.2",
     "@babel/preset-react": "7.27.1",
     "chai": "5.2.0",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.37.4",
+    "eslint-plugin-react": "7.37.5",
     "esmock": "2.7.0",
     "history": "5.3.0",
     "husky": "9.1.7",

test/provider/testProvider.json

@@ -37,7 +37,7 @@
     "enabled": true
   },
   "wgGesucht": {
-    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
+    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
     "enabled": true
   }
 }

test/services/immoscout/immoscout-web-translater.test.js

@@ -16,6 +16,15 @@ describe('#immoscout-mobile URL conversion', () => {
     expect(actualMobileUrl).to.equal(expectedMobileUrl);
   });
 
+  // Test URL conversion of web-only SEO path
+  it('should convert a SEO web path to the correct query params', () => {
+    const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
+
+    const converted = convertWebToMobile(webUrl);
+    const queryParams = new URL(converted).searchParams;
+    expect(queryParams.get('equipment').split(',')).to.include.members(['garden', 'balcony']);
+  });
+
   // Test URL conversion with unsupported query parameters
   it('should remove unsupported query parameters', () => {
     const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';

test/similarity/similarity.test.js

@@ -34,6 +34,7 @@ describe('similarityCheck', () => {
       check.setCacheEntry(
         'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
+      expect(check.hasSimilarEntries('unrelated text')).to.be.false;
     });
   });
 });

ui/src/views/jobs/mutation/JobMutation.jsx

@@ -94,7 +94,7 @@ export default function JobMutator() {
       <form>
         <SegmentPart name="Name">
           <Input
-            autofocus
+            autoFocus
             type="text"
             maxLength={40}
             placeholder="Name"