next version

* feat: rename npm start scripts

.dockerignore

@@ -1,7 +1,7 @@
 node_modules/
 npm-debug.log
 test/
-conf/
 db/
+conf/
 .git/
 .github/

.github/workflows/check_source.yml

@@ -0,0 +1,26 @@
+name: Check the source code
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  check_source_code:
+    name: Check the source code
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Check formatting
+        run: yarn format:check
+
+      - name: Lint
+        run: yarn lint

.github/workflows/docker.yml

@@ -1,4 +1,5 @@
 name: Create and publish Docker image
+
 on:
   push:
     branches:
@@ -17,15 +18,24 @@ jobs:
       contents: read
       packages: write
 
-    steps:
+    concurrency:
-      - name: Set up Docker Buildx
+      group: ${{ github.workflow }}-${{ github.ref }}
-        uses: docker/setup-buildx-action@v1
+      cancel-in-progress: true
 
+    steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -33,15 +43,17 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64, linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/stales.yml

@@ -1,4 +1,4 @@
-name: "Close stale issues and PRs"
+name: Close stale issues and PRs
 
 on:
   schedule:

.github/workflows/test.yml

@@ -13,8 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup Node.js
+      - uses: actions/setup-node@v4
-        uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'yarn'

.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "printWidth": 120
+}

CONTRIBUTING.md

@@ -106,14 +106,14 @@ exports.config = {
 ```
 
 #### Running Tests
-If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome right?
+If you've written a new provider you are an awesome person. If you now write tests for it, you are even more awesome. And who doesn't want to be more awesome, right?
 
 #### Codestyle
-I'm using Eslint to maintain quote style and quality. Do not skip it...
+I'm using ESLint to maintain quote style and quality. Do not skip it...
 
-##### To do before merging:
+##### To-do before merging:
 
-- executed tests? (`yarn run test`)
+- Have you executed the tests? (`yarn test`)
-- sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
+- Are you sure the changes are useful for everybody? Or is it maybe a custom modification just for your case?
 
 _Thanks!_ :heart:

Dockerfile

@@ -1,25 +1,35 @@
-FROM node:20
+FROM node:22-slim
 
 WORKDIR /fredy
 
-COPY . /fredy
+# Install Chromium without extra recommended packages and clean apt cache
-
+RUN apt-get update \
-RUN apt-get update && apt-get install -y chromium
+  && apt-get install -y --no-install-recommends chromium \
+  && rm -rf /var/lib/apt/lists/*
 
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
     PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
-RUN yarn install
+# Copy lockfiles first to leverage cache for dependencies
+COPY package.json yarn.lock .
 
-RUN yarn global add pm2
+# Set Yarn timeout, install dependencies and PM2 globally
+RUN yarn config set network-timeout 600000 \
+  && yarn --frozen-lockfile \
+  && yarn global add pm2
 
-RUN yarn run prod
+# Copy application source and build production assets
+COPY . .
+RUN yarn build:frontend
 
-RUN mkdir /db /conf && \
+# Prepare runtime directories and symlinks for data and config
-  chown 1000:1000 /db /conf && \
+RUN mkdir -p /db /conf \
-  chmod 777 -R /db/ && \
+  && chown 1000:1000 /db /conf \
-  ln -s /db /fredy/db && ln -s /conf /fredy/conf
+  && chmod 777 /db /conf \
+  && ln -s /db /fredy/db \
+  && ln -s /conf /fredy/conf
 
 EXPOSE 9998
 
-CMD pm2-runtime index.js
+# Start application using PM2 runtime
+CMD ["pm2-runtime", "index.js"]

README.md

@@ -1,6 +1,6 @@
 <img src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">  
 
-![Build Status](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
+![Test](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)  [![Create and publish Docker image](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml) ![Check the sourcecode](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
 
 Searching an apartment in Germany can be a frustrating task. Not any longer though, as _Fredy_ will take over and will only notify you once new listings have been found that match your requirements.
 
@@ -23,9 +23,9 @@ If you want to try out _Fredy_, you can access the demo version [here](https://f
 - Make sure to use Node.js 20 or above
 - Run the following commands:
 ```ssh
-yarn (or npm install)
+yarn
-yarn run prod
+yarn run start:backend
-yarn run start
+yarn run start:frontend
 ```
 _Fredy_ will start with the default port, set to `9998`. You can access _Fredy_ by opening your browser at `http://localhost:9998`. The default login is `admin`, both for username and password. You should change the password as soon as possible when you plan to run Fredy on a server.
 
@@ -46,7 +46,7 @@ A provider contains the URL that points to the search results for the respective
 **It is important that you order the search results by date, so that _Fredy_ always picks the latest results first!**
 
 #### Adapter
-_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. A adapter dictactes how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
+_Fredy_ supports multiple adapters, such as Slack, SendGrid, Telegram etc. A search job can have as many adapters as supported by _Fredy_. Each adapter needs different configuration values, which you have to provide when using them. An adapter dictates how the frontend renders by telling the frontend what information it needs in order to send listings to the user.
 
 #### Jobs
 A Job wraps adapters and providers. _Fredy_ runs the configured jobs in a specific interval (can be configured in `/conf/config.json`).
@@ -61,14 +61,13 @@ As an administrator, you can create, edit and remove users from _Fredy_. Be care
 # Development
 
 ### Running Fredy in development mode
-To run _Fredy_ in development mode, you need to run the backend & frontend separately.
 Start the backend with:
 ```shell
-yarn run start
+yarn run start:backend:dev
 ```
 For the frontend, run:
 ```shell
-yarn run dev
+yarn run start:frontend:dev
 ```
 You should now be able to access _Fredy_ from your browser. Check your Terminal to see what port the frontend is running on.
 
@@ -82,7 +81,7 @@ yarn run test
 ![Architecture](/doc/architecture.jpg "Architecture")
 
 ### Immoscout
-Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
+Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
 
 # Analytics
 Fredy is completely free (and will always remain free). However, it would be a huge help if you’d allow me to collect some analytical data. 
@@ -110,6 +109,10 @@ Put your config.json into a path of your choice, such as `/path/to/your/conf/`.
 
 Example: `docker create --name fredy -v /path/to/your/conf/:/conf -p 9998:9998 fredy/fredy`
 
+## Logs  
+
+You can browse the logs with  `docker logs fredy -f`.
+
 ### 👐 Contributing
 Thanks to all the people who already contributed!
 
@@ -119,6 +122,7 @@ Thanks to all the people who already contributed!
 
 See [Contributing](https://github.com/orangecoding/fredy/blob/master/CONTRIBUTING.md)
 
-## Logs  
 
-You can browse the logs with  `docker logs fredy -f`.
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=orangecoding/fredy&type=Date)](https://www.star-history.com/#orangecoding/fredy&Date)

index.js

@@ -25,7 +25,7 @@ if(config.demoMode){
 }
 /* eslint-enable no-console */
 const fetchedProvider = await Promise.all(
-  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
+  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`)),
 );
 
 handleDemoUser();
@@ -58,5 +58,5 @@ setInterval(
     }
     return exec;
   })(),
-  INTERVAL
+  INTERVAL,
 );

lib/api/routes/jobRouter.js

@@ -15,7 +15,7 @@ function doesJobBelongsToUser(job, req) {
   if (user == null) {
     return false;
   }
-  return user.isAdmin || job.userId === job.userId;
+  return user.isAdmin || job.userId === user.id;
 }
 jobRouter.get('/', async (req, res) => {
   const isUserAdmin = isAdmin(req);

lib/notification/adapter/telegram.js

@@ -22,16 +22,18 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-  //we have to split messages into chunk, because otherwise messages are going to become too big and will fail
+  // we have to split messages into chunks, because otherwise messages are going to become too big and will fail
   const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
   const promises = chunks.map((chunk) => {
-    let message = `<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:\n\n`;
+    const messageParagraphs = [];
-    message += chunk.map(
+
+    messageParagraphs.push(`<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:`);
+    messageParagraphs.push(...chunk.map(
       (o) =>
         `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
-        [o.address, o.price, o.size].join(' | ') +
+        [o.address, o.price, o.size].join(' | ')
-        '\n\n',
+    ));
-    );
+
     /**
      * This is to not break the rate limit. It is to only send 1 message per second
      */
@@ -41,7 +43,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
           method: 'post',
           body: JSON.stringify({
             chat_id: chatId,
-            text: message,
+            text: messageParagraphs.join('\n\n'),
             parse_mode: 'HTML',
             disable_web_page_preview: true,
           }),

lib/notification/notify.js

@@ -6,7 +6,7 @@ const adapter = await Promise.all(
   fs
     .readdirSync('./lib/notification/adapter')
     .filter((file) => file.endsWith('.js'))
-    .map(async (integPath) => await import(`${path}/${integPath}`))
+    .map(async (integPath) => await import(`${path}/${integPath}`)),
 );
 
 if (adapter.length === 0) {

lib/provider/immoscout.js

@@ -36,7 +36,7 @@
  */
 
 import utils, { buildHash } from '../utils.js';
-import { convertWebToMobile } from '../services/immoscout/immoscout-web-translater.js';
+import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
 let appliedBlackList = [];
 
 async function getListings(url) {

lib/provider/immowelt.js

@@ -23,7 +23,7 @@ const config = {
     id: 'a@href',
     price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-    title: '.css-jv3zx6',
+    title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
   },

lib/provider/neubauKompass.js

@@ -7,7 +7,9 @@ function nullOrEmpty(val) {
 }
 
 function normalize(o) {
-    const link = nullOrEmpty(o.link) ? 'NO LINK' : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
+  const link = nullOrEmpty(o.link)
+    ? 'NO LINK'
+    : `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
   const id = buildHash(o.link, o.price);
   return Object.assign(o, { id, link });
 }

lib/services/demoCleanup.js

@@ -1,29 +1,37 @@
 import { setInterval } from 'node:timers';
 import { removeJobsByUserName } from './storage/jobStorage.js';
 import { config } from '../utils.js';
+import { getUsers } from './storage/userStorage.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
  */
 export function cleanupDemoAtMidnight() {
   const now = new Date();
-    const millisUntilMidnightUTC = (24 - now.getUTCHours()) * 60 * 60 * 1000
+  const millisUntilMidnightUTC =
-        - now.getUTCMinutes() * 60 * 1000
+    (24 - now.getUTCHours()) * 60 * 60 * 1000 -
-        - now.getUTCSeconds() * 1000
+    now.getUTCMinutes() * 60 * 1000 -
-        - now.getUTCMilliseconds();
+    now.getUTCSeconds() * 1000 -
+    now.getUTCMilliseconds();
 
+  cleanup();
   setTimeout(() => {
+    setInterval(
+      () => {
         cleanup();
-
+      },
-        setInterval(() => {
+      24 * 60 * 60 * 1000,
-            cleanup();
+    );
-        }, 24 * 60 * 60 * 1000);
-
   }, millisUntilMidnightUTC);
 }
 
 function cleanup() {
   if (config.demoMode) {
-        removeJobsByUserName('demo');
+    const demoUser = getUsers(false).find((user) => user.username === 'demo');
+    if (demoUser == null) {
+      console.error('Demo user not found, cannot remove Jobs');
+      return;
+    }
+    removeJobsByUserName(demoUser.id);
   }
 }

lib/services/extractor/parser/parser.js

@@ -8,7 +8,7 @@ export function loadParser(text) {
 
 export function parse(crawlContainer, crawlFields, text, url) {
   if (!text) {
-    console.warn('Cannot parse, text was empty for url ', url);
+    console.warn('No content found for ', url);
     return null;
   }
 

lib/services/immoscout/immoscout-web-translator.js

@@ -79,6 +79,7 @@ const PARAM_NAME_MAP = {
   geocoordinates: 'geocoordinates',
   shape: 'shape',
   sorting: 'sorting',
+  newbuilding: 'newbuilding',
 };
 
 const EQUIPMENT_MAP = {
@@ -89,6 +90,7 @@ const EQUIPMENT_MAP = {
   garden: 'garden',
   guesttoilet: 'guestToilet',
   balcony: 'balcony',
+  handicappedaccessible: 'handicappedAccessible',
 };
 
 const REAL_ESTATE_TYPE = {
@@ -98,6 +100,29 @@ const REAL_ESTATE_TYPE = {
   'haus-kaufen': 'housebuy',
 };
 
+const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
+  // Category "Balkon/Terrasse"
+  'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
+  'wohnung-mit-garten-mieten': { equipment: ['garden'] },
+  // Category "Wohnungstyp"
+  'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
+  'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
+  'hochparterrewohnung-mieten': { apartmenttypes: ['raisedgroundfloor'] },
+  'etagenwohnung-mieten': { apartmenttypes: ['apartment'] },
+  'loft-mieten': { apartmenttypes: ['loft'] },
+  'maisonette-mieten': { apartmenttypes: ['maisonette'] },
+  'terrassenwohnung-mieten': { apartmenttypes: ['terracedflat'] },
+  'penthouse-mieten': { apartmenttypes: ['penthouse'] },
+  'dachgeschosswohnung-mieten': { apartmenttypes: ['roofstorey'] },
+  // Category "Ausstattung"
+  'wohnung-mit-garage-mieten': { equipment: ['parking'] },
+  'wohnung-mit-einbaukueche-mieten': { equipment: ['builtinkitchen'] },
+  'wohnung-mit-keller-mieten': { equipment: ['cellar'] },
+  // Category "Merkmale"
+  'neubauwohnung-mieten': { newbuilding: true },
+  'barrierefreie-wohnung-mieten': { equipment: ['handicappedaccessible'] },
+};
+
 export function convertWebToMobile(webUrl) {
   let url;
   try {
@@ -112,10 +137,18 @@ export function convertWebToMobile(webUrl) {
   }
 
   const realTypeKey = segments.at(-1);
-  const realType = REAL_ESTATE_TYPE[realTypeKey];
+  let realType = REAL_ESTATE_TYPE[realTypeKey];
+  let additionalParamsFromWebPath;
+
   if (!realType) {
+    // Test for seo optimized apartment path (only used on the ImmoScout web app)
+    if (WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey]) {
+      additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey];
+      realType = REAL_ESTATE_TYPE['wohnung-mieten'];
+    } else {
       throw new Error(`Real estate type not found: ${realTypeKey}`);
     }
+  }
 
   if (segments.includes('shape')) {
     throw new Error('Shape is currently not supported using Immoscout');
@@ -132,6 +165,7 @@ export function convertWebToMobile(webUrl) {
     searchType: isRadius ? 'radius' : 'region',
     realestatetype: realType,
     ...(isRadius ? {} : { geocodes }),
+    ...additionalParamsFromWebPath,
   };
 
   if (webParams.geocoordinates) {
@@ -141,7 +175,11 @@ export function convertWebToMobile(webUrl) {
   for (const [key, val] of Object.entries(webParams)) {
     if (key === 'equipment') {
       const items = [].concat(val).flatMap((v) => `${v}`.split(','));
-      mobileParams[PARAM_NAME_MAP[key]] = items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean);
+      const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]];
+      mobileParams[PARAM_NAME_MAP[key]] = [
+        ...(currentEquipmentParams ?? []),
+        ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean),
+      ];
     } else {
       mobileParams[PARAM_NAME_MAP[key]] = val;
     }

lib/services/storage/jobStorage.js

@@ -11,7 +11,6 @@ const db = new LowdashAdapter(adapter, { jobs: [] });
 
 db.read();
 
-
 export const upsertJob = ({ jobId, name, blacklist = [], enabled = true, provider, notificationAdapter, userId }) => {
   const currentJob =
     jobId == null
@@ -77,16 +76,25 @@ export const removeJobsByUserId = (userId) => {
     .value();
   db.write();
 };
-export const removeJobsByUserName = (userName) => {
+export const removeJobsByUserName = (userId) => {
+  let removedDemoJobs = 0;
   db.chain
     .get('jobs')
-      .filter((job) => job.username === userName)
+    .filter((job) => job.userId === userId)
-      .forEach((job) => listingStorage.removeListings(job.id));
+    .forEach((job) => {
+      removedDemoJobs++;
+      listingStorage.removeListings(job.id);
+    });
   db.chain
     .get('jobs')
-      .remove((job) => job.username === userName)
+    .remove((job) => job.userId === userId)
     .value();
   db.write();
+  if (removedDemoJobs > 0) {
+    /* eslint-disable no-console */
+    console.log(`Removed ${removedDemoJobs} demo jobs`);
+    /* eslint-enable no-console */
+  }
 };
 export const getJobs = () => {
   return db.chain

lib/services/storage/userStorage.js

@@ -86,7 +86,7 @@ export const removeUser = (userId) => {
   db.chain
     .set(
       'user',
-      user.filter((u) => u.id !== userId)
+      user.filter((u) => u.id !== userId),
     )
     .value();
   db.write();
@@ -95,8 +95,12 @@ export const removeUser = (userId) => {
 export const handleDemoUser = () => {
   if (!config.demoMode) {
     const user = db.chain.get('user').value();
-        db.chain.get('user').value();
+    db.chain
-        db.chain.set('user', user.filter((u) => u.username !== 'demo')).value();
+      .set(
+        'user',
+        user.filter((u) => u.username !== 'demo'),
+      )
+      .value();
     db.write();
   } else {
     const demoUser = db.chain
@@ -104,7 +108,8 @@ export const handleDemoUser = () => {
       .filter((u) => u.username === 'demo')
       .value();
     if (demoUser == null || demoUser.length === 0) {
-            db.chain.get('user')
+      db.chain
+        .get('user')
         .value()
         .push({
           id: nanoid(),
@@ -116,4 +121,3 @@ export const handleDemoUser = () => {
     }
   }
 };
-

lib/utils.js

@@ -9,12 +9,9 @@ function inDevMode(){
 }
 
 function isOneOf(word, arr) {
-    if (arr == null || arr.length === 0) {
+    if (!arr || arr.length === 0 || word == null) return false;
-        return false;
+    const lowerWord = word.toLowerCase();
-    }
+    return arr.some(item => lowerWord.indexOf(item.toLowerCase()) !== -1);
-    const expression = String.raw`\b(${arr.join('|')})\b`;
-    const blacklist = new RegExp(expression, 'ig');
-    return blacklist.test(word);
 }
 
 function nullOrEmpty(val) {

package.json

@@ -1,21 +1,25 @@
 {
   "name": "fredy",
-  "version": "11.2.0",
+  "version": "11.3.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
-    "start": "node prod.js",
+    "prepare": "husky",
-    "dev": "yarn && rm -rf ./ui/public/* && vite",
+    "start:backend": "x-var NODE_ENV=production node index.js",
-    "ui": "rm -rf ./ui/public/* && vite",
+    "start:backend:dev": "nodemon --watch index.js --watch lib",
-    "prod": "yarn && vite build --emptyOutDir",
+    "start:frontend": "vite -m production",
-    "format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js --single-quote --print-width 120",
+    "start:frontend:dev": "vite",
+    "build:frontend": "vite build",
+    "format": "prettier --write lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js",
+    "format:check": "prettier --check lib/**/*.js ui/src/**/*.jsx test/**/*.js *.js",
     "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
-    "lint": "eslint ./index.js ./lib/**/*.js ./test/**/*.js ./ui/src/**/*.jsx"
+    "lint": "eslint index.js lib/**/*.js test/**/*.js ui/src/**/*.jsx",
+    "lint:fix": "yarn lint --fix"
   },
   "type": "module",
   "lint-staged": {
-    "*.js": [
+    "*.{js,jsx}": [
-      "eslint ./index.js ./lib/**/*.js ./test/**/*.js",
+      "yarn lint",
-      "prettier --single-quote --print-width 120 --write"
+      "yarn format"
     ]
   },
   "main": "index.js",
@@ -50,17 +54,17 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.79.0",
+    "@douyinfe/semi-ui": "2.83.0",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.5",
-    "@vitejs/plugin-react": "4.4.1",
+    "@vitejs/plugin-react": "4.7.0",
     "better-sqlite3": "^11.10.0",
     "body-parser": "2.2.0",
-    "cheerio": "^1.0.0",
+    "cheerio": "^1.1.0",
-    "cookie-session": "2.1.0",
+    "cookie-session": "2.1.1",
     "handlebars": "4.7.8",
-    "highcharts": "12.2.0",
+    "highcharts": "12.3.0",
     "highcharts-react-official": "3.2.2",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
@@ -70,10 +74,10 @@
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.8",
     "package-up": "^5.0.0",
-    "puppeteer": "^24.8.2",
+    "puppeteer": "^24.14.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
-    "query-string": "9.1.2",
+    "query-string": "9.2.2",
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "react-redux": "9.2.0",
@@ -81,28 +85,30 @@
     "react-router-dom": "5.3.0",
     "redux": "5.0.1",
     "redux-thunk": "3.1.0",
-    "restana": "4.9.9",
+    "restana": "5.0.0",
-    "serve-static": "1.16.2",
+    "serve-static": "2.2.0",
     "slack": "11.0.2",
     "string-similarity": "^4.0.4",
-    "vite": "5.4.11"
+    "vite": "7.0.5",
+    "x-var": "^2.1.0"
   },
   "devDependencies": {
-    "@babel/core": "7.27.1",
+    "@babel/core": "7.27.3",
-    "@babel/eslint-parser": "7.27.1",
+    "@babel/eslint-parser": "7.27.5",
     "@babel/preset-env": "7.27.2",
     "@babel/preset-react": "7.27.1",
-    "chai": "5.2.0",
+    "chai": "5.2.1",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.37.4",
+    "eslint-plugin-react": "7.37.5",
-    "esmock": "2.7.0",
+    "esmock": "2.7.1",
     "history": "5.3.0",
     "husky": "9.1.7",
-    "less": "4.3.0",
+    "less": "4.4.0",
     "lint-staged": "15.5.2",
     "mocha": "10.8.2",
-    "prettier": "3.5.3",
+    "nodemon": "^3.1.10",
+    "prettier": "3.6.2",
     "redux-logger": "3.0.6"
   }
 }

prod.js

@@ -1,2 +0,0 @@
-process.env.NODE_ENV = 'production';
-import('./index.js');

reverse-engineered-immoscout.md

@@ -77,4 +77,4 @@ curl -H "User-Agent: ImmoScout24_1410_30_._" \
 
 
 ## Parameters 
-The parameters between web and mobile are very different which is why we have to translate them. Please see `immoscout-web-translator.js`.
+The parameters between web and mobile are very different which is why we have to translate them. Please see [/lib/services/immoscout/immoscout-web-translator.js](https://github.com/orangecoding/fredy/blob/master/lib/services/immoscout/immoscout-web-translator.js).

test/provider/testProvider.json

@@ -37,7 +37,7 @@
     "enabled": true
   },
   "wgGesucht": {
-    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html?offer_filter=1&noDeact=1&city_id=30&category=0&rent_type=0&rMax=5000",
+    "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
     "enabled": true
   }
 }

test/services/immoscout/immoscout-web-translater.test.js

@@ -16,6 +16,15 @@ describe('#immoscout-mobile URL conversion', () => {
     expect(actualMobileUrl).to.equal(expectedMobileUrl);
   });
 
+  // Test URL conversion of web-only SEO path
+  it('should convert a SEO web path to the correct query params', () => {
+    const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mit-balkon-mieten?equipment=garden';
+
+    const converted = convertWebToMobile(webUrl);
+    const queryParams = new URL(converted).searchParams;
+    expect(queryParams.get('equipment').split(',')).to.include.members(['garden', 'balcony']);
+  });
+
   // Test URL conversion with unsupported query parameters
   it('should remove unsupported query parameters', () => {
     const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';

test/similarity/similarity.test.js

@@ -34,6 +34,7 @@ describe('similarityCheck', () => {
       check.setCacheEntry(
         'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.',
       );
+      expect(check.hasSimilarEntries('unrelated text')).to.be.false;
     });
   });
 });

ui/src/App.jsx

@@ -66,15 +66,17 @@ export default function FredyApp() {
 
         {settings.demoMode && (
           <>
-                    <Banner fullMode={true}
+            <Banner
+              fullMode={true}
               type="info"
               bordered
               closeIcon={null}
               description="You're currently viewing the demo version of Fredy. Jobs won't scrape websites, and any changes you make will be reverted at midnight."
             />
             <br />
-                </>)}
+          </>
-                {(settings.analyticsEnabled === null && !settings.demoMode) && <TrackingModal/>}
+        )}
+        {settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
         <Switch>
           <Route name="Insufficient Permission" path={'/403'} component={InsufficientPermission} />
           <Route name="Create new Job" path={'/jobs/new'} component={JobMutation} />

ui/src/App.less

@@ -41,3 +41,7 @@ a:active {
   background-color: transparent;
   text-decoration: underline;
 }
+
+.semi-icon:not(.semi-tabs-bar .semi-tabs-tab .semi-icon) {
+  vertical-align: middle;
+}

ui/src/Index.jsx

@@ -23,5 +23,5 @@ root.render(
         <App />
       </LocaleProvider>
     </HashRouter>
-  </Provider>
+  </Provider>,
 );

ui/src/components/table/ProviderTable.jsx

@@ -30,7 +30,7 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
           render: (_, record) => {
             return (
               <div style={{ float: 'right' }}>
-                <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.id)} />
+                <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
               </div>
             );
           },

ui/src/components/tracking/TrackingModal.jsx

@@ -4,16 +4,21 @@ import Logo from '../logo/Logo.jsx';
 import { xhrPost } from '../../services/xhr.js';
 
 import './TrackingModal.less';
+import inDevelopment from '../../services/developmentMode.js';
 
 const saveResponse = async (analyticsEnabled) => {
   await xhrPost('/api/admin/generalSettings', {
-        analyticsEnabled
+    analyticsEnabled,
   });
 };
 
 export default function TrackingModal() {
+  if (inDevelopment()) {
+    return null;
+  }
 
-    return <Modal
+  return (
+    <Modal
       visible={true}
       onOk={async () => {
         await saveResponse(true);
@@ -32,17 +37,20 @@ export default function TrackingModal() {
       <div className="trackingModal__description">
         <p>Hey 👋</p>
         <p>Fed up with popups? Yeah, me too. But this one’s important, and I promise it will only appear once ;)</p>
-            <p>Fredy is completely free (and will always remain free). If you’d like, you can support me by donating
+        <p>
-                through my GitHub, but there’s absolutely no obligation to do so.</p>
+          Fredy is completely free (and will always remain free). If you’d like, you can support me by donating through
-            <p>However, it would be a huge
+          my GitHub, but there’s absolutely no obligation to do so.
-                help if you’d allow me to collect some analytical data. Wait, before you click "no", let me explain. If
+        </p>
-                you
+        <p>
-                agree, Fredy will send a ping to my Mixpanel project each time it runs.</p>
+          However, it would be a huge help if you’d allow me to collect some analytical data. Wait, before you click
-            <p>The data includes: names of
+          "no", let me explain. If you agree, Fredy will send a ping to my Mixpanel project each time it runs.
-                active adapters/providers, OS, architecture, Node version, and language. The information is entirely
+        </p>
-                anonymous and helps me understand which adapters/providers are most frequently used.</p>
+        <p>
+          The data includes: names of active adapters/providers, OS, architecture, Node version, and language. The
+          information is entirely anonymous and helps me understand which adapters/providers are most frequently used.
+        </p>
         <p>Thanks🤘</p>
       </div>
-    </Modal>;
+    </Modal>
-
+  );
 }

ui/src/services/developmentMode.js

@@ -0,0 +1,4 @@
+export default function isDevelopmentMode(){
+    const inDevMode= import.meta.env.MODE;
+    return inDevMode != null && inDevMode === 'development';
+}

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -8,7 +8,14 @@ import Headline from '../../components/headline/Headline';
 import { xhrPost } from '../../services/xhr';
 import { SegmentPart } from '../../components/segment/SegmentPart';
 import { Banner, Toast } from '@douyinfe/semi-ui';
-import {IconSave, IconCalendar, IconRefresh, IconSignal, IconLineChartStroked, IconSearch} from '@douyinfe/semi-icons';
+import {
+  IconSave,
+  IconCalendar,
+  IconRefresh,
+  IconSignal,
+  IconLineChartStroked,
+  IconSearch,
+} from '@douyinfe/semi-icons';
 import './GeneralSettings.less';
 
 function formatFromTimestamp(ts) {
@@ -97,7 +104,7 @@ const GeneralSettings = function GeneralSettings() {
           to: workingHourTo,
         },
         demoMode,
-                analyticsEnabled
+        analyticsEnabled,
       });
     } catch (exception) {
       console.error(exception);
@@ -175,24 +182,18 @@ const GeneralSettings = function GeneralSettings() {
             </SegmentPart>
             <Divider margin="1rem" />
 
-                        <SegmentPart
+            <SegmentPart name="Analytics" helpText="Insights into the usage of Fredy." Icon={IconLineChartStroked}>
-                            name="Analytics"
-                            helpText="Insights into the usage of Fredy."
-                            Icon={IconLineChartStroked}
-                        >
               <Banner
                 fullMode={false}
                 type="info"
                 closeIcon={null}
-                                title={
+                title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Explanation</div>}
-                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
-                                        Explanation
-                                    </div>
-                                }
                 style={{ marginBottom: '1rem' }}
                 description={
                   <div>
-                                        Analytics are disabled by default. If you choose to enable them, we will begin tracking the following:<br/>
+                    Analytics are disabled by default. If you choose to enable them, we will begin tracking the
+                    following:
+                    <br />
                     <ul>
                       <li>Name of active provider (e.g. Immoscout)</li>
                       <li>Name of active adapter (e.g. Console)</li>
@@ -201,35 +202,26 @@ const GeneralSettings = function GeneralSettings() {
                       <li>node version</li>
                       <li>arch</li>
                     </ul>
-                                        The data is sent anonymously and helps me understand which providers or adapters are being used the most. In the end it helps me to improve fredy.
+                    The data is sent anonymously and helps me understand which providers or adapters are being used the
+                    most. In the end it helps me to improve fredy.
                   </div>
                 }
               />
 
-                            <Checkbox
+              <Checkbox checked={analyticsEnabled} onChange={(e) => setAnalyticsEnabled(e.target.checked)}>
-                                checked={analyticsEnabled}
+                {' '}
-                                onChange={(e) => setAnalyticsEnabled(e.target.checked)}
+                Enabled
-                            > Enabled
               </Checkbox>
-
             </SegmentPart>
 
             <Divider margin="1rem" />
 
-                        <SegmentPart
+            <SegmentPart name="Demo Mode" helpText="If enabled, Fredy runs in demo mode." Icon={IconSearch}>
-                            name="Demo Mode"
-                            helpText="If enabled, Fredy runs in demo mode."
-                            Icon={IconSearch}
-                        >
               <Banner
                 fullMode={false}
                 type="info"
                 closeIcon={null}
-                                title={
+                title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Explanation</div>}
-                                    <div style={{fontWeight: 600, fontSize: '14px', lineHeight: '20px'}}>
-                                        Explanation
-                                    </div>
-                                }
                 style={{ marginBottom: '1rem' }}
                 description={
                   <div>
@@ -239,12 +231,10 @@ const GeneralSettings = function GeneralSettings() {
                 }
               />
 
-                            <Checkbox
+              <Checkbox checked={demoMode} onChange={(e) => setDemoMode(e.target.checked)}>
-                                checked={demoMode}
+                {' '}
-                                onChange={(e) => setDemoMode(e.target.checked)}
+                Enabled
-                            > Enabled
               </Checkbox>
-
             </SegmentPart>
 
             <Divider margin="1rem" />

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
-import {Banner, Descriptions} from '@douyinfe/semi-ui';
+import { Descriptions } from '@douyinfe/semi-ui';
 
 export default function ProcessingTimes({ processingTimes = {} }) {
   if (Object.keys(processingTimes).length === 0) {

ui/src/views/jobs/mutation/JobMutation.jsx

@@ -94,7 +94,7 @@ export default function JobMutator() {
       <form>
         <SegmentPart name="Name">
           <Input
-            autofocus
+            autoFocus
             type="text"
             maxLength={40}
             placeholder="Name"
@@ -124,8 +124,8 @@ export default function JobMutator() {
 
           <ProviderTable
             providerData={providerData}
-            onRemove={(providerId) => {
+            onRemove={(providerUrl) => {
-              setProviderData(providerData.filter((provider) => provider.id !== providerId));
+              setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
             }}
           />
         </SegmentPart>

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx

@@ -83,7 +83,7 @@ export default function NotificationAdapterMutator({
           id: selectedAdapter.id,
           name: selectedAdapter.name,
           fields: selectedAdapter.fields || {},
-        })
+        }),
       );
 
       setSelectedAdapter(null);
@@ -114,7 +114,7 @@ export default function NotificationAdapterMutator({
         setSuccessMessage('It seems like it worked! Please check your service.');
       })
       .catch((error) =>
-        setValidationMessage(`This did not work :-( I've received the following error: ${error.json.message}`)
+        setValidationMessage(`This did not work :-( I've received the following error: ${error.json.message}`),
       );
   };
 
@@ -229,7 +229,7 @@ export default function NotificationAdapterMutator({
           .filter((option) =>
             editNotificationAdapter != null
               ? true
-              : selected.find((selectedOption) => selectedOption.id === option.key) == null
+              : selected.find((selectedOption) => selectedOption.id === option.key) == null,
           )
           .sort(sortAdapter)}
         onChange={(value) => {

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -45,7 +45,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
             url: providerUrl,
             id: selectedProvider.id,
             name: selectedProvider.name,
-          })
+          }),
         );
         setProviderUrl(null);
         setSelectedProvider(null);
@@ -101,7 +101,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         description={
           <div>
             <p>
-              Currently, our Immoscout implementation does not drawing shapes on a map. Use a radius instead.
+              Currently, our Immoscout implementation does not support drawing shapes on a map. Use a radius instead.
             </p>
           </div>
         }

ui/src/views/login/Login.jsx

@@ -87,12 +87,15 @@ export default function Login() {
             Login
           </Button>
           <br />
-                    {demoMode && <Banner fullMode={true}
+          {demoMode && (
+            <Banner
+              fullMode={true}
               type="info"
               bordered
               closeIcon={null}
               description="This is the demo version of Fredy. Use 'demo' as both the username and password to log in."
-                    />}
+            />
+          )}
         </div>
       </form>
     </div>

vite.config.js

@@ -6,6 +6,7 @@ export default defineConfig({
   build: {
     chunkSizeWarningLimit: 9999999,
     outDir: './ui/public',
+    emptyOutDir: true,
   },
   plugins: [react()],
   server: {