improve docker build

.dockerignore

@@ -1,7 +1,6 @@
 node_modules/
 npm-debug.log
 test/
-conf/
 db/
 .git/
 .github/

.github/workflows/docker.yml

@@ -1,4 +1,5 @@
 name: Create and publish Docker image
+
 on:
   push:
     branches:
@@ -17,15 +18,24 @@ jobs:
       contents: read
       packages: write
 
-    steps:
+    concurrency:
-      - name: Set up Docker Buildx
+      group: ${{ github.workflow }}-${{ github.ref }}
-        uses: docker/setup-buildx-action@v1
+      cancel-in-progress: true
 
+    steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -33,15 +43,17 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64, linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

CONTRIBUTING.md

@@ -113,7 +113,7 @@ I'm using Eslint to maintain quote style and quality. Do not skip it...
 
 ##### To do before merging:
 
-- executed tests? (`yarn run test`)
+- executed tests? (`pnpm test`)
 - sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
 
 _Thanks!_ :heart:

Dockerfile

@@ -1,25 +1,35 @@
-FROM node:20
+FROM node:22-slim
 
-WORKDIR  /fredy
+WORKDIR /fredy
 
-COPY . /fredy
+# Install Chromium without extra recommended packages and clean apt cache
-
+RUN apt-get update \
-RUN apt-get update && apt-get install -y chromium
+  && apt-get install -y --no-install-recommends chromium \
+  && rm -rf /var/lib/apt/lists/*
 
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
-  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
-RUN yarn install
+# Copy lockfiles first to leverage cache for dependencies
+COPY package.json yarn.lock ./
 
-RUN yarn global add pm2
+# Set Yarn timeout, install dependencies and PM2 globally
+RUN yarn config set network-timeout 600000 \
+  && yarn install --frozen-lockfile \
+  && yarn global add pm2
 
+# Copy application source and build production assets
+COPY . ./
 RUN yarn run prod
 
-RUN mkdir /db /conf && \
+# Prepare runtime directories and symlinks for data and config
-  chown 1000:1000 /db /conf && \
+RUN mkdir -p /db /conf \
-  chmod 777 -R /db/ && \
+  && chown 1000:1000 /db /conf \
-  ln -s /db /fredy/db && ln -s /conf /fredy/conf
+  && chmod 777 /db /conf \
+  && ln -s /db /fredy/db \
+  && ln -s /conf /fredy/conf
 
 EXPOSE 9998
 
-CMD pm2-runtime index.js
+# Start application using PM2 runtime
+CMD ["pm2-runtime", "index.js"]

README.md

@@ -1,6 +1,6 @@
 <img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">  
 
-![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
+![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)  [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
 
 Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
 
@@ -8,6 +8,8 @@ _Fredy_ scrapes multiple services (Immonet, Immowelt etc.) and send new listings
 
 If _Fredy_ finds matching results, it will send them to you via Slack, Email, Telegram etc. (More adapters can be configured.) As _Fredy_ stores the listings it has found, new results will not be sent to you twice (and as a side-effect, _Fredy_ can show some statistics). Furthermore, _Fredy_ checks duplicates per scraping so that the same listings are not being sent twice or more when posted on various platforms (which happens more often than one might think).
 
+<a href="https://www.producthunt.com/posts/fredy-find-real-estates-damn-easy?embed=true&utm_source=badge-featured&utm_medium=badge&utm_source=badge-fredy&#0045;find&#0045;real&#0045;estates&#0045;damn&#0045;easy" target="_blank"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=965690&theme=light&t=1747292331626" alt="Fredy&#0032;&#0045;&#0032;Find&#0032;Real&#0032;Estates&#0032;Damn&#0032;EasY&#0032; - Your&#0032;personal&#0032;real&#0032;estate&#0032;search&#0032;bot | Product Hunt" style="width: 250px; height: 54px;" width="250" height="54" /></a>
+
 # Sponsorship [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/orangecoding)    
 If you like my work, consider becoming a sponsor. I'm not expecting anybody to pay for _Fredy_ or any other Open Source Project I'm maintaining, however keep in mind, I'm doing all of this in my spare time :) Thanks.
 
@@ -110,6 +112,10 @@ Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+## Logs  
+
+You can browse the logs with  `docker logs fredy -f`.
+
 ### 👐 Contributing
 Thanks to all the people who already contributed!
 
@@ -119,6 +125,7 @@ Thanks to all the people who already contributed!
 
 See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
-## Logs  
 
-You can browse the logs with  `docker logs fredy -f`.
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=orangecoding/fredy&type=Date)](https://www.star-history.com/#orangecoding/fredy&Date)

lib/provider/immowelt.js

@@ -23,7 +23,7 @@ const config = {
     id: 'a@href',
     price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: '.css-jv3zx6',
+    title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },

lib/services/demoCleanup.js

@@ -1,29 +1,37 @@
 import { setInterval } from 'node:timers';
-import {removeJobsByUserName} from './storage/jobStorage.js';
+import { removeJobsByUserName } from './storage/jobStorage.js';
-import {config} from '../utils.js';
+import { config } from '../utils.js';
+import { getUsers } from './storage/userStorage.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
  */
 export function cleanupDemoAtMidnight() {
-    const now = new Date();
+  const now = new Date();
-    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+  const millisUntilMidnightUTC =
-        - now.getUTCMinutes() * 60 * 1000
+    (24 - now.getUTCHours()) * 60 * 60 * 1000 -
-        - now.getUTCSeconds() * 1000
+    now.getUTCMinutes() * 60 * 1000 -
-        - now.getUTCMilliseconds();
+    now.getUTCSeconds() * 1000 -
+    now.getUTCMilliseconds();
 
-    setTimeout(() => {
+  cleanup();
+  setTimeout(() => {
+    setInterval(
+      () => {
         cleanup();
-
+      },
-        setInterval(() => {
+      24 * 60 * 60 * 1000,
-            cleanup();
+    );
-        }, 24 * 60 * 60 * 1000);
+  }, millisUntilMidnightUTC);
-
-    }, millisUntilMidnightUTC);
 }
 
-function cleanup(){
+function cleanup() {
-    if(config.demoMode){
+  if (config.demoMode) {
-        removeJobsByUserName('demo');
+    const demoUser = getUsers(false).find((user) => user.username === 'demo');
+    if (demoUser == null) {
+      console.error('Demo user not found, cannot remove Jobs');
+      return;
     }
-}
+    removeJobsByUserName(demoUser.id);
+  }
+}

lib/services/immoscout/immoscout-web-translater.js

@@ -79,6 +79,7 @@ const PARAM_NAME_MAP = {
   geocoordinates: 'geocoordinates',
   shape: 'shape',
   sorting: 'sorting',
+  newbuilding: 'newbuilding',
 };
 
 const EQUIPMENT_MAP = {
@@ -89,6 +90,7 @@ const EQUIPMENT_MAP = {
   garden: 'garden',
   guesttoilet: 'guestToilet',
   balcony: 'balcony',
+  handicappedaccessible: 'handicappedAccessible',
 };
 
 const REAL_ESTATE_TYPE = {
@@ -98,6 +100,29 @@ const REAL_ESTATE_TYPE = {
   'haus-kaufen': 'housebuy',
 };
 
+const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
+  // Category "Balkon/Terrasse"
+  'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
+  'wohnung-mit-garten-mieten': { equipment: ['garden'] },
+  // Category "Wohnungstyp"
+  'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
+  'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
+  'hochparterrewohnung-mieten': { apartmenttypes: ['raisedgroundfloor'] },
+  'etagenwohnung-mieten': { apartmenttypes: ['apartment'] },
+  'loft-mieten': { apartmenttypes: ['loft'] },
+  'maisonette-mieten': { apartmenttypes: ['maisonette'] },
+  'terrassenwohnung-mieten': { apartmenttypes: ['terracedflat'] },
+  'penthouse-mieten': { apartmenttypes: ['penthouse'] },
+  'dachgeschosswohnung-mieten': { apartmenttypes: ['roofstorey'] },
+  // Category "Ausstattung"
+  'wohnung-mit-garage-mieten': { equipment: ['parking'] },
+  'wohnung-mit-einbaukueche-mieten': { equipment: ['builtinkitchen'] },
+  'wohnung-mit-keller-mieten': { equipment: ['cellar'] },
+  // Category "Merkmale"
+  'neubauwohnung-mieten': { newbuilding: true },
+  'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
+};
+
 export function convertWebToMobile(webUrl) {
   let url;
   try {
@@ -112,9 +137,17 @@ export function convertWebToMobile(webUrl) {
   }
 
   const realTypeKey = segments.at(-1);
-  const realType = REAL_ESTATE_TYPE[realTypeKey];
+  let realType = REAL_ESTATE_TYPE[realTypeKey];
+  let additionalParamsFromWebPath;
+
   if (!realType) {
-    throw new Error(`Real estate type not found: ${realTypeKey}`);
+    // Test for seo optimized apartment path (only used on the ImmoScout web app)
+    if (WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey]) {
+      additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey];
+      realType = REAL_ESTATE_TYPE['wohnung-mieten'];
+    } else {
+      throw new Error(`Real estate type not found: ${realTypeKey}`);
+    }
   }
 
   if (segments.includes('shape')) {
@@ -132,6 +165,7 @@ export function convertWebToMobile(webUrl) {
     searchType: isRadius ? 'radius' : 'region',
     realestatetype: realType,
     ...(isRadius ? {} : { geocodes }),
+    ...additionalParamsFromWebPath,
   };
 
   if (webParams.geocoordinates) {
@@ -141,7 +175,11 @@ export function convertWebToMobile(webUrl) {
   for (const [key, val] of Object.entries(webParams)) {
     if (key === 'equipment') {
       const items = [].concat(val).flatMap((v) => `${v}`.split(','));
-      mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean);
+      const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]];
+      mobileParams[PARAM_NAME_MAP[key]] = [
+        ...(currentEquipmentParams ?? []),
+        ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
+      ];
     } else {
       mobileParams[PARAM_NAME_MAP[key]] = val;
     }

lib/services/storage/jobStorage.js

@@ -11,7 +11,6 @@ const db = new LowdashAdapter(adapter, { jobs: [] });
 
 db.read();
 
-
 export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
@@ -77,16 +76,25 @@ export const removeJobsByUserId = (userId) => {
     .value();
   db.write();
 };
-export const removeJobsByUserName = (userName) => {
+export const removeJobsByUserName = (userId) => {
+  let removedDemoJobs = 0;
   db.chain
-      .get('jobs')
+    .get('jobs')
-      .filter((job) => job.username === userName)
+    .filter((job) => job.userId === userId)
-      .forEach((job) => listingStorage.removeListings(job.id));
+    .forEach((job) => {
+      removedDemoJobs++;
+      listingStorage.removeListings(job.id);
+    });
   db.chain
-      .get('jobs')
+    .get('jobs')
-      .remove((job) => job.username === userName)
+    .remove((job) => job.userId === userId)
-      .value();
+    .value();
   db.write();
+  if (removedDemoJobs > 0) {
+    /* eslint-disable no-console */
+    console.log(`Removed ${removedDemoJobs} demo jobs`);
+    /* eslint-enable no-console */
+  }
 };
 export const getJobs = () => {
   return db.chain

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.2.0",
+  "version": "11.2.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node prod.js",

test/provider/testProvider.json

@@ -37,7 +37,7 @@
     "enabled": true
   },
   "wgGesucht": {
-    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
+    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
     "enabled": true
   }
 }

test/services/immoscout/immoscout-web-translater.test.js

@@ -16,6 +16,15 @@ describe('#immoscout-mobile URL conversion', () => {
     expect(actualMobileUrl).to.equal(expectedMobileUrl);
   });
 
+  // Test URL conversion of web-only SEO path
+  it('should convert a SEO web path to the correct query params', () => {
+    const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
+
+    const converted = convertWebToMobile(webUrl);
+    const queryParams = new URL(converted).searchParams;
+    expect(queryParams.get('equipment').split(',')).to.include.members(['garden', 'balcony']);
+  });
+
   // Test URL conversion with unsupported query parameters
   it('should remove unsupported query parameters', () => {
     const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';