Listing management (#223)

* upgrading dependencies, fixing image placeholder

* improving processing times label and hide when screen width is too low

* aligning run now button

* renaming settings -> general settings

* smaller security and memory improvements

* improving footer

* preparing listing management

* improve filtering for listings

* preparing new settings page

* preparing new settings page

* storing settings in db

* next release version

.github/workflows/test.yml

@@ -19,4 +19,4 @@ jobs:
           cache: 'yarn'
 
       - run: yarn install
-      - run: yarn test
+      - run: yarn testGH

conf/config.json

@@ -1 +1 @@
-{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":true,"sqlitepath":"/db"}
+{"sqlitepath":"/db"}

index.js

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import path from 'path';
-import { checkIfConfigIsAccessible, config, getProviders, refreshConfig } from './lib/utils.js';
+import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
 import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
 import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyPipeline from './lib/FredyPipeline.js';
@@ -12,43 +12,52 @@ import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
 import logger from './lib/services/logger.js';
 import { bus } from './lib/services/events/event-bus.js';
 import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
+import { getSettings } from './lib/services/storage/settingsStorage.js';
+import SqliteConnection from './lib/services/storage/SqliteConnection.js';
+
+//in the config, we store the path of the sqlite file, thus we must check if it is available
+const isConfigAccessible = await checkIfConfigIsAccessible();
+await SqliteConnection.init();
 
 // Load configuration before any other startup steps
 await refreshConfig();
 
-const isConfigAccessible = await checkIfConfigIsAccessible();
-
 if (!isConfigAccessible) {
   logger.error('Configuration exists, but is not accessible. Please check the file permission');
   process.exit(1);
 }
 
+// Run DB migrations once at startup and block until finished
+await runMigrations();
+
+const settings = await getSettings();
+
 // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
-const rawDir = config.sqlitepath || '/db';
+const rawDir = settings.sqlitepath || '/db';
 const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
 const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
 if (!fs.existsSync(absDir)) {
   fs.mkdirSync(absDir, { recursive: true });
 }
 
-// Run DB migrations once at startup and block until finished
-await runMigrations();
-
 // Load provider modules once at startup
 const providers = await getProviders();
 
+similarityCache.initSimilarityCache();
+similarityCache.startSimilarityCacheReloader();
+
 //assuming interval is always in minutes
-const INTERVAL = config.interval * 60 * 1000;
+const INTERVAL = settings.interval * 60 * 1000;
 
 // Initialize API only after migrations completed
 await import('./lib/api/api.js');
 
-if (config.demoMode) {
+if (settings.demoMode) {
   logger.info('Running in demo mode');
   cleanupDemoAtMidnight();
 }
 
-logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
 
 ensureAdminUserExists();
 ensureDemoUserExists();
@@ -62,10 +71,10 @@ bus.on('jobs:runAll', () => {
 });
 
 const execute = () => {
-  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(config, Date.now());
+  const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(settings, Date.now());
-  if (!config.demoMode) {
+  if (!settings.demoMode) {
     if (isDuringWorkingHoursOrNotSet) {
-      config.lastRun = Date.now();
+      settings.lastRun = Date.now();
       jobStorage
         .getJobs()
         .filter((job) => job.enabled)

lib/FredyPipeline.js

@@ -183,8 +183,12 @@ class FredyPipeline {
    * @returns {Listing[]} Listings considered unique enough to keep.
    */
   _filterBySimilarListings(listings) {
-    const filteredList = listings.filter((listing) => {
+    return listings.filter((listing) => {
-      const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
+      const similar = this._similarityCache.checkAndAddEntry({
+        title: listing.title,
+        address: listing.address,
+        price: listing.price,
+      });
       if (similar) {
         logger.debug(
           `Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
@@ -192,8 +196,6 @@ class FredyPipeline {
       }
       return !similar;
     });
-    filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
-    return filteredList;
   }
 
   /**

lib/api/api.js

@@ -7,7 +7,6 @@ import { versionRouter } from './routes/versionRouter.js';
 import { loginRouter } from './routes/loginRoute.js';
 import { userRouter } from './routes/userRoute.js';
 import { jobRouter } from './routes/jobRouter.js';
-import { config } from '../utils.js';
 import bodyParser from 'body-parser';
 import restana from 'restana';
 import files from 'serve-static';
@@ -16,9 +15,11 @@ import { getDirName } from '../utils.js';
 import { demoRouter } from './routes/demoRouter.js';
 import logger from '../services/logger.js';
 import { listingsRouter } from './routes/listingsRouter.js';
+import { getSettings } from '../services/storage/settingsStorage.js';
+import { featureRouter } from './routes/featureRouter.js';
 const service = restana();
 const staticService = files(path.join(getDirName(), '../ui/public'));
-const PORT = config.port || 9998;
+const PORT = (await getSettings()).port || 9998;
 
 service.use(bodyParser.json());
 service.use(cookieSession());
@@ -39,6 +40,7 @@ service.use('/api/version', versionRouter);
 service.use('/api/jobs', jobRouter);
 service.use('/api/login', loginRouter);
 service.use('/api/listings', listingsRouter);
+service.use('/api/features', featureRouter);
 //this route is unsecured intentionally as it is being queried from the login page
 service.use('/api/demo', demoRouter);
 

lib/api/routes/demoRouter.js

@@ -1,10 +1,11 @@
 import restana from 'restana';
-import { config } from '../../utils.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const demoRouter = service.newRouter();
 
 demoRouter.get('/', async (req, res) => {
-  res.body = Object.assign({}, { demoMode: config.demoMode });
+  const settings = await getSettings();
+  res.body = Object.assign({}, { demoMode: settings.demoMode });
   res.send();
 });
 

lib/api/routes/featureRouter.js

@@ -0,0 +1,12 @@
+import restana from 'restana';
+import getFeatures from '../../features.js';
+const service = restana();
+const featureRouter = service.newRouter();
+
+featureRouter.get('/', async (req, res) => {
+  const features = getFeatures();
+  res.body = Object.assign({}, { features });
+  res.send();
+});
+
+export { featureRouter };

lib/api/routes/generalSettingsRoute.js

@@ -1,24 +1,30 @@
 import restana from 'restana';
-import { config, getDirName, readConfigFromStorage, refreshConfig } from '../../utils.js';
+import { getDirName } from '../../utils.js';
 import fs from 'fs';
 import { ensureDemoUserExists } from '../../services/storage/userStorage.js';
 import logger from '../../services/logger.js';
+import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
+
 generalSettingsRouter.get('/', async (req, res) => {
-  res.body = Object.assign({}, config);
+  res.body = Object.assign({}, await getSettings());
   res.send();
 });
 generalSettingsRouter.post('/', async (req, res) => {
-  const settings = req.body;
+  const { sqlitepath, ...appSettings } = req.body || {};
+  const localSettings = await getSettings();
+
+  if (localSettings.demoMode) {
+    res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+    return;
+  }
+
   try {
-    if (config.demoMode) {
+    if (typeof sqlitepath !== 'undefined') {
-      res.send(new Error('In demo mode, it is not allowed to change these settings.'));
+      fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
-      return;
     }
-    const currentConfig = await readConfigFromStorage();
+    upsertSettings(appSettings);
-    fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ ...currentConfig, ...settings }));
-    await refreshConfig();
     ensureDemoUserExists();
   } catch (err) {
     logger.error(err);

lib/api/routes/jobRouter.js

@@ -1,10 +1,10 @@
 import restana from 'restana';
 import * as jobStorage from '../../services/storage/jobStorage.js';
 import * as userStorage from '../../services/storage/userStorage.js';
-import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import logger from '../../services/logger.js';
 import { bus } from '../../services/events/event-bus.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 
 const service = restana();
 const jobRouter = service.newRouter();
@@ -44,9 +44,10 @@ jobRouter.get('/', async (req, res) => {
 });
 
 jobRouter.get('/processingTimes', async (req, res) => {
+  const settings = await getSettings();
   res.body = {
-    interval: config.interval,
+    interval: settings.interval,
-    lastRun: config.lastRun || null,
+    lastRun: settings.lastRun || null,
   };
   res.send();
 });

lib/api/routes/loginRoute.js

@@ -1,9 +1,9 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as hasher from '../../services/security/hash.js';
-import { config } from '../../utils.js';
 import { trackDemoAccessed } from '../../services/tracking/Tracker.js';
 import logger from '../../services/logger.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const loginRouter = service.newRouter();
 loginRouter.get('/user', async (req, res) => {
@@ -20,6 +20,7 @@ loginRouter.get('/user', async (req, res) => {
   res.send();
 });
 loginRouter.post('/', async (req, res) => {
+  const settings = await getSettings();
   const { username, password } = req.body;
   const user = userStorage.getUsers(true).find((user) => user.username === username);
   if (user == null) {
@@ -27,7 +28,7 @@ loginRouter.post('/', async (req, res) => {
     return;
   }
   if (user.password === hasher.hash(password)) {
-    if (config.demoMode) {
+    if (settings.demoMode) {
       await trackDemoAccessed();
     }
 

lib/api/routes/userRoute.js

@@ -1,7 +1,7 @@
 import restana from 'restana';
 import * as userStorage from '../../services/storage/userStorage.js';
 import * as jobStorage from '../../services/storage/jobStorage.js';
-import { config } from '../../utils.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
 const service = restana();
 const userRouter = service.newRouter();
 function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
@@ -23,7 +23,8 @@ userRouter.get('/:userId', async (req, res) => {
   res.send();
 });
 userRouter.delete('/', async (req, res) => {
-  if (config.demoMode) {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     res.send(new Error('In demo mode, it is not allowed to remove user.'));
     return;
   }
@@ -44,7 +45,8 @@ userRouter.delete('/', async (req, res) => {
   res.send();
 });
 userRouter.post('/', async (req, res) => {
-  if (config.demoMode) {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     res.send(new Error('In demo mode, it is not allowed to change or add user.'));
     return;
   }

lib/features.js

@@ -0,0 +1,9 @@
+const FEATURES = {
+  WATCHLIST_MANAGEMENT: false,
+};
+
+export default function getFeatures() {
+  return {
+    ...FEATURES,
+  };
+}

lib/notification/adapter/apprise.md

@@ -1,3 +1,8 @@
 ### Apprise Adapter
 
-Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.
+Use [Apprise](https://github.com/caronc/apprise-api#installation) to forward notifications to many different services.
+
+Quick start:
+- Set up an Apprise API instance (see the installation guide linked above).
+- Configure your preferred notification service(s) within Apprise.
+- In Fredy, point the Apprise adapter to your Apprise API endpoint.

lib/notification/adapter/console.md

@@ -1,4 +1,3 @@
 ### Console Adapter
 
-The console adapter prints everything found by Fredy into the console (not sending any notifications to you). This can be useful when you want to check if your search
+The console adapter prints everything found by Fredy to the console (it does not send notifications). This is useful to verify that your search criteria work as expected before enabling a real notification service.
-criteria meet the expectations.

lib/notification/adapter/discord_webhook.md

@@ -1,4 +1,8 @@
-### Discord Adapter
+### Discord Webhook Adapter
 
-To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
+Use a Discord channel webhook to receive notifications.
-Once you have created a webhook, copy and paste the webhook URL.
+
+Quick start:
+- Create a webhook in your target Discord channel. See the "Intro to Webhooks" guide on the Discord support site: https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks
+- Copy the generated webhook URL.
+- In Fredy, configure the Discord adapter with this webhook URL.

lib/notification/adapter/http.js

@@ -0,0 +1,57 @@
+import { markdown2Html } from '../../services/markdown.js';
+
+const mapListing = (listing) => ({
+  address: listing.address,
+  description: listing.description,
+  id: listing.id,
+  imageUrl: listing.image,
+  price: listing.price,
+  size: listing.size,
+  title: listing.title,
+  url: listing.link,
+});
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { authToken, endpointUrl } = notificationConfig.find((a) => a.id === config.id).fields;
+
+  const listings = newListings.map(mapListing);
+  const body = {
+    jobId: jobKey,
+    timestamp: new Date().toISOString(),
+    provider: serviceName,
+    listings,
+  };
+
+  const headers = {
+    'Content-Type': 'application/json',
+  };
+  if (authToken != null) {
+    headers['Authorization'] = `Bearer ${authToken}`;
+  }
+
+  return fetch(endpointUrl, {
+    method: 'POST',
+    headers: headers,
+    body: JSON.stringify(body),
+  });
+};
+
+export const config = {
+  id: 'http',
+  name: 'HTTP',
+  readme: markdown2Html('lib/notification/adapter/http.md'),
+  description: 'Fredy will send a generic HTTP POST request.',
+  fields: {
+    endpointUrl: {
+      description: "Your application's endpoint URL.",
+      label: 'Endpoint URL',
+      type: 'text',
+    },
+    authToken: {
+      description: "Your application's auth token, if required by your endpoint.",
+      label: 'Auth token (optional)',
+      optional: true,
+      type: 'text',
+    },
+  },
+};

lib/notification/adapter/http.md

@@ -0,0 +1,43 @@
+### HTTP Adapter
+
+This is a generic adapter for sending notifications via HTTP requests.
+You can leverage this adapter to integrate with various webhooks or APIs that accept HTTP requests. (e.g. Supabase
+Functions, a Node.js server, etc.)
+
+HTTP adapter supports a `authToken` field, which can be used to include an authorization token in the request headers.
+Your token would be included as a Bearer token in the `Authorization` header, which is a common method for securing API requests.
+
+Request Details:
+<details>
+Request Method: POST  
+
+Headers:
+
+```
+Content Type: `application/json`
+Authorization: Bearer {your-optional-auth-token}
+```
+
+Body:
+
+```json
+{
+  "jobId": "mg1waX4RHmIzL5NDYtYp-",
+  "provider": "immoscout",
+  "timestamp": "2024-06-15T12:34:56Z",
+  "listings": [
+    {
+      "address": "Str. 123, Bielefeld, Germany",
+      "description": "Möbliert: Einziehen & wohlfühlen: Neu möbliert.",
+      "id": "123456789",
+      "imageUrl": "https://<target-url>.com/listings/123456789.jpg",
+      "price": "1.240 €",
+      "size": "38 m²",
+      "title": "Schöne 1-Zimmer-Wohnung in Bielefeld",
+      "url": "https://<target-url>.com/listings/123456789"
+    }
+  ]
+}
+```
+
+</details>

lib/notification/adapter/mailJet.md

@@ -1,8 +1,8 @@
-### MailJet Adapter
+### Mailjet Adapter
 
-To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
+To use [Mailjet](https://mailjet.com), create an account and decide which email address Fredy should send from.
 
-E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.  
+For example, if you use yourGmailAccount@gmail.com, add and verify this address in Mailjet.
-The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
+Provide your public/private API keys in Fredy's configuration. Fredy uses the same email template as for SendGrid.
 
-If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).
+To send to multiple recipients, separate email addresses with commas (e.g., some@email.com, someOther@email.com).

lib/notification/adapter/mattermost.md

@@ -1,5 +1,8 @@
 ### Mattermost Adapter
 
-For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+Receive notifications in Mattermost via an incoming webhook.
 
-As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.
+Quick start:
+- Create an incoming webhook following the Mattermost developer docs: https://docs.mattermost.com/developer/webhooks-incoming.html
+- Copy the webhook URL.
+- In Fredy, configure the Mattermost adapter with this URL and the target channel.

lib/notification/adapter/ntfy.md

@@ -1,5 +1,8 @@
 ### ntfy Adapter
 
-For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+Send push notifications using an ntfy topic.
 
-As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.
+Quick start:
+- Create or choose a topic on your preferred ntfy instance (see docs: https://docs.ntfy.sh/publish/).
+- Copy the publish URL for that topic.
+- In Fredy, configure the ntfy adapter with the topic URL and set a priority.

lib/notification/adapter/pushover.md

@@ -1,5 +1,8 @@
 ### Pushover Adapter
 
-Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+Use Pushover to receive push notifications on your devices.
 
-After setting up the application, please enter both your newly created User key and API token.
+Setup:
+- Follow Pushover's getting-started guide: https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it
+- Create an application and obtain your User Key and API Token.
+- In Fredy, configure the Pushover adapter with both values.

lib/notification/adapter/sendGrid.md

@@ -1,9 +1,12 @@
 ### SendGrid Adapter
 
-SendGrid is a free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
+SendGrid is an email delivery service with a generous free tier, which is more than enough for Fredy.
 
-To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.
+Setup:
+- Create a SendGrid account: https://sendgrid.com/
+- Decide which email address Fredy should send from (e.g., yourGmailAccount@gmail.com), add it to SendGrid, and complete the verification.
+- Create an API key and add it to Fredy's configuration.
+- Create a Dynamic Template in SendGrid. You can copy the template from `/lib/notification/emailTemplate/template.hbs`.
 
-Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new dynamic template. For this new template, I recommend copying and pasting the code from the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
+Sending to multiple recipients:
-
+- Separate email addresses with commas (e.g., some@email.com, someOther@email.com).
-If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).

lib/notification/adapter/slack.md

@@ -1,4 +1,5 @@
-### Slack Adapter
+### Slack Adapter (Legacy)
-IMPORTANT:
+
-Don't use this adapter anymore, it is outdated and only here for backwards compatability reasons. Use the new Slack Adapter with webhooks!
+*IMPORTANT:*
+This legacy adapter is outdated and kept only for backward compatibility. Please use the Slack adapter with webhooks instead.
 

lib/notification/adapter/slack_with_webhooks.md

@@ -1,6 +1,10 @@
-### Slack Adapter
+### Slack Adapter (Webhooks)
 
-IMPORTANT:
+*IMPORTANT:*
-This is the new version of the Slack adapter. I strongly encourage you to use it, the old version is now unmaintained and only kept due to backwards compatability reasons.
+This is the recommended Slack adapter. The old Slack adapter is unmaintained and kept only for backward compatibility.
 
-In order to use [Slack](https://slack.com), you need to create an account. When done, create a new channel and add the Webhook integration to that channel. Copy the webhook url. That's it.
+Setup:
+- Create a Slack account and workspace if you don't have one: https://slack.com
+- Create a channel where you want to receive notifications.
+- Add the Incoming Webhooks integration to that channel and copy the Webhook URL.
+- In Fredy, configure the Slack Webhook adapter with this URL.

lib/notification/adapter/sqlite.md

@@ -1,9 +1,21 @@
 ### SQLite Adapter
 
-This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. This file can be used for further analysis later.
+This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. The file can be used for analysis later.
 
-The database table contains the following columns (all stored as `TEXT` type):
+The table contains the following columns (all stored as `TEXT`):
 
-```
+```json
-['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description', 'image']
+[
+  "serviceName",
+  "jobKey",
+  "id",
+  "size",
+  "rooms",
+  "price",
+  "address",
+  "title",
+  "link",
+  "description",
+  "image"
+]
 ```

lib/notification/adapter/telegram.js

@@ -117,10 +117,24 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
   if (!adapterCfg || !adapterCfg.fields) {
     throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`);
   }
-  const { token, chatId } = adapterCfg.fields;
+  const { token, chatId, messageThreadId } = adapterCfg.fields;
   if (!token || !chatId) {
     throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
   }
+
+  // Optional Telegram topic/thread support (supergroups)
+  let message_thread_id;
+  if (messageThreadId !== undefined && messageThreadId !== null && `${messageThreadId}`.trim() !== '') {
+    const n = Number(messageThreadId);
+    if (Number.isInteger(n) && n > 0) {
+      message_thread_id = n;
+    } else {
+      logger.warn(
+        `Telegram adapter: 'messageThreadId' is invalid ('${messageThreadId}'). It must be a positive integer. Ignoring.`,
+      );
+    }
+  }
+
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
 
@@ -147,6 +161,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
       text: buildText(jobName, serviceName, o),
       parse_mode: 'HTML',
       disable_web_page_preview: true,
+      ...(message_thread_id ? { message_thread_id } : {}),
     };
 
     if (!img) {
@@ -160,6 +175,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
       photo: img,
       caption: buildCaption(jobName, serviceName, o),
       parse_mode: 'HTML',
+      ...(message_thread_id ? { message_thread_id } : {}),
     }).catch(async (e) => {
       logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
       return await throttledCall('sendMessage', textPayload).catch((e) => {
@@ -174,7 +190,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
 
 /**
  * Telegram notification adapter configuration schema.
- * @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string}}}}
+ * @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string},messageThreadId?:{type:string,label:string,description:string}}}}
  */
 export const config = {
   id: 'telegram',
@@ -192,5 +208,12 @@ export const config = {
       label: 'Chat Id',
       description: 'The chat id to send messages to you.',
     },
+    messageThreadId: {
+      type: 'text',
+      optional: true,
+      label: 'Message Thread Id (optional)',
+      description:
+        'Optional: The topic/thread id within a supergroup to post into (Telegram message_thread_id). Provide a positive integer.',
+    },
   },
 };

lib/notification/adapter/telegram.md

@@ -1,12 +1,55 @@
 ### Telegram Adapter
 
-For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
+Use this adapter to send notifications to Telegram via a bot. You will need:
+- A Telegram Bot token (from BotFather)
+- A chat ID (where messages will be sent)
+- Optionally: a thread ID if you want to post into a specific forum topic in a group
 
-A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.  
+#### Create a bot
-After the user has send a message to your bot the first time, you can gather the chatId like this:
+Create a bot with BotFather: open https://telegram.me/BotFather on your phone or in Telegram Desktop and follow the instructions to get your bot token.
 
+#### Getting the chat ID
+A Telegram bot cannot message a user first; you must create a conversation (or add the bot to a group/channel) so Telegram assigns a chat the bot can access.
+
+Steps:
+1. Start a chat with your bot in Telegram (or add the bot to your group/supergroup/channel) and send any message.
+2. Fetch recent updates from the Bot API:
+   ```
+   curl -X GET "https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates"
+   ```
+3. In the JSON response, find the message that you just sent and read `message.chat.id`. That value is your `chatId`.
+   - Private chats: `chat.id` is a positive number
+   - Groups/supergroups: `chat.id` is a negative number
+
+Keep your bot token secret. If `getUpdates` returns an empty list, send a new message and try again, or make sure your bot’s privacy settings allow it to see group messages when used in groups.
+
+#### Getting the thread ID (this is optional to be used for forum topics)
+If you want messages to appear inside a specific forum topic of a supergroup with Topics enabled, you also need a thread ID. In the Telegram Bot API this is called `message_thread_id`.
+
+When you need it:
+- Required only for supergroups with Topics enabled when targeting a topic
+- Not used for private chats, basic groups without Topics, or channels
+
+Steps to obtain it:
+1. In your supergroup, enable Topics (Group settings → Manage group → Topics → Enable). Now add a new topic.
+2. Add your created bot to the topic. (Click on the bot and on "Add to group")
+3. Open the desired topic (or create a new one) and send any message inside that topic.
+4. Call `getUpdates` again:
+   ```
+   curl -X GET "https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates"
+   ```
+4. In the update for the message you sent inside the topic, read `message.message_thread_id`. That number is your `threadId` for this topic.
+
+Example (truncated):
 ```
-curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
+{
+  "message": {
+    "chat": { "id": -1001234567890, "type": "supergroup" },
+    "message_thread_id": 42,
+    "text": "hello from the topic"
+  }
+}
 ```
+Use `chat.id` as `chatId` and `message_thread_id` as `threadId` in your configuration.
 
-A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
+More details about bots and BotFather: https://core.telegram.org/bots#botfather

lib/provider/ohneMakler.js

@@ -0,0 +1,45 @@
+import { isOneOf, buildHash } from '../utils.js';
+import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
+let appliedBlackList = [];
+
+function normalize(o) {
+  const link = metaInformation.baseUrl + o.link;
+  const id = buildHash(o.title, o.link, o.price);
+  return Object.assign(o, { link, id });
+}
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+const config = {
+  url: null,
+  crawlContainer: 'div[data-livecomponent-id*="search/property_list"] .grid > div',
+  sortByDateParam: null,
+  waitForSelector: null,
+  crawlFields: {
+    id: 'a@href',
+    title: 'h4 | removeNewline | trim',
+    price: '.text-xl | trim',
+    size: 'div[title="Wohnfläche"] | trim',
+    address: '.text-slate-800 | removeNewline | trim',
+    image: 'img@src',
+    link: 'a@href',
+  },
+  normalize: normalize,
+  filter: applyBlacklist,
+  activeTester: checkIfListingIsActive,
+};
+
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlackList = blacklist || [];
+};
+
+export const metaInformation = {
+  name: 'OhneMakler',
+  baseUrl: 'https://www.ohne-makler.net/immobilien',
+  id: 'ohneMakler',
+};
+export { config };

lib/provider/regionalimmobilien24.js

@@ -8,7 +8,7 @@ function normalize(o) {
   const title = o.title || 'No title available';
   const link = o.link != null ? decodeURIComponent(o.link) : config.url;
 
-  var urlReg = new RegExp(/url\((.*?)\)/gim);
+  const urlReg = new RegExp(/url\((.*?)\)/gim);
   const image = o.image != null ? urlReg.exec(o.image)[1] : null;
   return Object.assign(o, { id, address, title, link, image });
 }

lib/services/crons/demoCleanup-cron.js

@@ -1,8 +1,8 @@
 import { removeJobsByUserId } from '../storage/jobStorage.js';
-import { config } from '../../utils.js';
 import { getUsers } from '../storage/userStorage.js';
 import logger from '../logger.js';
 import cron from 'node-cron';
+import { getSettings } from '../storage/settingsStorage.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
@@ -11,12 +11,13 @@ export function cleanupDemoAtMidnight() {
   cron.schedule('0 0 * * *', cleanup);
 }
 
-function cleanup() {
+async function cleanup() {
-  if (config.demoMode) {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     const demoUser = getUsers(false).find((user) => user.username === 'demo');
     if (demoUser == null) {
       logger.error('Demo user not found, cannot remove Jobs');
-      return;
+      return Promise.resolve();
     }
     removeJobsByUserId(demoUser.id);
   }

lib/services/crons/tracker-cron.js

@@ -1,10 +1,12 @@
 import cron from 'node-cron';
-import { config, inDevMode } from '../../utils.js';
+import { inDevMode } from '../../utils.js';
 import { trackMainEvent } from '../tracking/Tracker.js';
+import { getSettings } from '../storage/settingsStorage.js';
 
 async function runTask() {
+  const settings = await getSettings();
   //make sure to only send tracking events if the user gave us the green light and we are not in dev mode
-  if (config.analyticsEnabled && !inDevMode()) {
+  if (settings.analyticsEnabled && !inDevMode()) {
     await trackMainEvent();
   }
 }

lib/services/extractor/botPrevention.js

@@ -0,0 +1,274 @@
+import { DEFAULT_HEADER } from './utils.js';
+
+// Helper to safely coerce numbers
+const toInt = (v, d) => {
+  const n = parseInt(v, 10);
+  return Number.isFinite(n) ? n : d;
+};
+
+/**
+ * Compute pre-launch configuration and flags for Puppeteer with bot prevention in mind.
+ * Returns language, user agent, viewport (with optional jitter), and additional launch args.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ */
+export function getPreLaunchConfig(url, options = {}) {
+  const { hostname } = new URL(url);
+
+  const acceptLanguage = options.acceptLanguage || 'de-DE,de;q=0.9,en-US;q=0.7,en;q=0.5';
+  const langForFlag = acceptLanguage.split(',')[0];
+
+  const baseViewport = { width: 1366, height: 768, deviceScaleFactor: 1 };
+  const jitter = options.viewportJitter !== false ? Math.floor(Math.random() * 6) : 0; // 0..5 px
+  const width = toInt(options?.viewport?.width, baseViewport.width) + jitter;
+  const height = toInt(options?.viewport?.height, baseViewport.height) + jitter;
+  const deviceScaleFactor = toInt(options?.viewport?.deviceScaleFactor, baseViewport.deviceScaleFactor);
+  const viewport = { width, height, deviceScaleFactor };
+
+  const userAgent =
+    options.userAgent ||
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+
+  const windowSizeArg = `--window-size=${viewport.width},${viewport.height}`;
+  const langArg = `--lang=${langForFlag}`;
+
+  const extraArgs = [
+    '--disable-blink-features=AutomationControlled',
+    '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--webrtc-ip-handling-policy=default_public_interface_only',
+    '--proxy-bypass-list=<-loopback>',
+  ];
+
+  const headers = {
+    ...DEFAULT_HEADER,
+    'Accept-Language': acceptLanguage,
+    'User-Agent': userAgent,
+    Referer: options?.referer || `https://${hostname}/`,
+    Connection: 'keep-alive',
+    DNT: '1',
+  };
+
+  const timezone = options?.timezone || 'Europe/Berlin';
+
+  return {
+    acceptLanguage,
+    langForFlag,
+    userAgent,
+    viewport,
+    windowSizeArg,
+    langArg,
+    extraArgs,
+    headers,
+    timezone,
+    humanDelay: options?.humanDelay !== false,
+  };
+}
+
+/**
+ * Apply bot-prevention hardening to a Puppeteer page.
+ * Sets UA, viewport, JS enabled, headers, timezone and injects stealth-like patches.
+ *
+ * @param {import('puppeteer').Page} page
+ * @param {ReturnType<typeof getPreLaunchConfig>} cfg
+ */
+export async function applyBotPreventionToPage(page, cfg) {
+  await page.setUserAgent(cfg.userAgent);
+  await page.setViewport(cfg.viewport);
+  await page.setJavaScriptEnabled(true);
+  await page.setExtraHTTPHeaders(cfg.headers);
+  try {
+    if (cfg.timezone) await page.emulateTimezone(cfg.timezone);
+  } catch {
+    // ignore timezone failures
+  }
+
+  // Inject patches as early as possible
+  await page.evaluateOnNewDocument(() => {
+    try {
+      // webdriver
+      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+
+      // chrome runtime
+      // @ts-ignore
+      if (!window.chrome) {
+        // @ts-ignore
+        window.chrome = { runtime: {} };
+      }
+
+      // languages
+      // @ts-ignore
+      Object.defineProperty(navigator, 'languages', {
+        get: () => (window.localStorage.getItem('__LANGS__') || 'de-DE,de').split(','),
+      });
+
+      // plugins
+      // @ts-ignore
+      Object.defineProperty(navigator, 'plugins', {
+        get: () => [{}, {}, {}],
+      });
+
+      // platform and concurrency hints
+      // @ts-ignore
+      Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
+      // @ts-ignore
+      if (typeof navigator.hardwareConcurrency === 'number' && navigator.hardwareConcurrency < 2) {
+        Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 4 });
+      }
+      // @ts-ignore
+      if (typeof navigator.deviceMemory === 'number' && navigator.deviceMemory < 2) {
+        Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
+      }
+
+      // userAgentData (Client Hints)
+      try {
+        // @ts-ignore
+        if ('userAgentData' in navigator) {
+          // @ts-ignore
+          Object.defineProperty(navigator, 'userAgentData', {
+            get: () => ({
+              brands: [
+                { brand: 'Chromium', version: '126' },
+                { brand: 'Google Chrome', version: '126' },
+              ],
+              mobile: false,
+              platform: 'Windows',
+              getHighEntropyValues: async (hints) => {
+                const values = {
+                  platform: 'Windows',
+                  platformVersion: '15.0.0',
+                  architecture: 'x86',
+                  model: '',
+                  uaFullVersion: '126.0.0.0',
+                  bitness: '64',
+                };
+                const out = {};
+                for (const k of hints || []) if (k in values) out[k] = values[k];
+                return out;
+              },
+            }),
+          });
+        }
+      } catch {
+        //noop
+      }
+
+      // Permissions API
+      const origQuery = navigator.permissions && navigator.permissions.query;
+      if (origQuery) {
+        // @ts-ignore
+        navigator.permissions.query = (parameters) =>
+          origQuery.call(navigator.permissions, parameters).then((result) => {
+            if (parameters && parameters.name === 'notifications') {
+              Object.defineProperty(result, 'state', { get: () => Notification.permission });
+            }
+            return result;
+          });
+      }
+
+      // WebGL vendor/renderer
+      const patchWebGL = (proto) => {
+        if (!proto || !proto.getParameter) return;
+        const getParameter = proto.getParameter;
+        // @ts-ignore
+        proto.getParameter = function (param) {
+          const UNMASKED_VENDOR_WEBGL = 0x9245;
+          const UNMASKED_RENDERER_WEBGL = 0x9246;
+          if (param === UNMASKED_VENDOR_WEBGL) return 'Google Inc.';
+          if (param === UNMASKED_RENDERER_WEBGL)
+            return 'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 Ti Direct3D11 vs_5_0 ps_5_0)';
+          return getParameter.call(this, param);
+        };
+      };
+      // @ts-ignore
+      patchWebGL(WebGLRenderingContext?.prototype);
+      // @ts-ignore
+      patchWebGL(WebGL2RenderingContext?.prototype);
+
+      // AudioContext timestamp rounding consistency
+      const patchAudio = (Ctx) => {
+        try {
+          if (!Ctx) return;
+          const proto = Ctx.prototype;
+          const createOsc = proto.createOscillator;
+          proto.createOscillator = function () {
+            const osc = createOsc.call(this);
+            const start = osc.start;
+            osc.start = function (when) {
+              return start.call(this, when || 0);
+            };
+            return osc;
+          };
+        } catch {
+          //noop
+        }
+      };
+      // @ts-ignore
+      patchAudio(window.AudioContext);
+      // @ts-ignore
+      patchAudio(window.OfflineAudioContext);
+
+      // Navigator.connection
+      try {
+        // @ts-ignore
+        Object.defineProperty(navigator, 'connection', { get: () => undefined });
+      } catch {
+        //noop
+      }
+
+      // Consistent outer sizes
+      try {
+        const calcOuter = () => {
+          const w = window.innerWidth + 16;
+          const h = window.innerHeight + 88;
+          return { w, h };
+        };
+        const { w: outerW, h: outerH } = calcOuter();
+        // @ts-ignore
+        Object.defineProperty(window, 'outerWidth', { get: () => outerW });
+        // @ts-ignore
+        Object.defineProperty(window, 'outerHeight', { get: () => outerH });
+      } catch {
+        //noop
+      }
+    } catch {
+      //noop
+    }
+  });
+}
+
+/**
+ * Persist languages value before navigation via localStorage.
+ * @param {import('puppeteer').Page} page
+ * @param {ReturnType<typeof getPreLaunchConfig>} cfg
+ */
+export async function applyLanguagePersistence(page, cfg) {
+  await page.evaluateOnNewDocument((langs) => {
+    try {
+      window.localStorage.setItem('__LANGS__', langs);
+    } catch {
+      // noop
+    }
+  }, cfg.acceptLanguage.split(';')[0]);
+}
+
+/**
+ * Perform subtle human-like interactions post navigation.
+ * @param {import('puppeteer').Page} page
+ * @param {ReturnType<typeof getPreLaunchConfig>} cfg
+ */
+export async function applyPostNavigationHumanSignals(page, cfg) {
+  if (!cfg.humanDelay) return;
+  const delay = 200 + Math.floor(Math.random() * 400);
+  await new Promise((res) => setTimeout(res, delay));
+  try {
+    const vw = cfg.viewport.width;
+    const vh = cfg.viewport.height;
+    const mx = Math.floor(vw * (0.3 + Math.random() * 0.4));
+    const my = Math.floor(vh * (0.3 + Math.random() * 0.4));
+    await page.mouse.move(mx, my, { steps: 10 + Math.floor(Math.random() * 10) });
+    await page.mouse.wheel({ deltaY: 100 + Math.floor(Math.random() * 200) });
+  } catch {
+    // ignore if mouse is unavailable
+  }
+}

lib/services/extractor/puppeteerExtractor.js

@@ -1,6 +1,12 @@
 import puppeteer from 'puppeteer-extra';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
-import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
+import { debug, botDetected } from './utils.js';
+import {
+  getPreLaunchConfig,
+  applyBotPreventionToPage,
+  applyLanguagePersistence,
+  applyPostNavigationHumanSignals,
+} from './botPrevention.js';
 import logger from '../logger.js';
 import fs from 'fs';
 import os from 'os';
@@ -27,23 +33,50 @@ export default async function execute(url, waitForSelector, options) {
       removeUserDataDir = true;
     }
 
+    const launchArgs = [
+      '--no-sandbox',
+      '--disable-gpu',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage',
+      '--disable-crash-reporter',
+      '--no-first-run',
+      '--no-default-browser-check',
+    ];
+    if (options?.proxyUrl) {
+      launchArgs.push(`--proxy-server=${options.proxyUrl}`);
+    }
+    // Prepare bot prevention pre-launch config
+    const preCfg = getPreLaunchConfig(url, options || {});
+    launchArgs.push(preCfg.langArg);
+    launchArgs.push(preCfg.windowSizeArg);
+    launchArgs.push(...preCfg.extraArgs);
+
     browser = await puppeteer.launch({
-      headless: options.puppeteerHeadless ?? true,
+      headless: options?.puppeteerHeadless ?? true,
-      args: [
+      args: launchArgs,
-        '--no-sandbox',
+      timeout: options?.puppeteerTimeout || 30_000,
-        '--disable-gpu',
-        '--disable-setuid-sandbox',
-        '--disable-dev-shm-usage',
-        '--disable-crash-reporter',
-      ],
-      timeout: options.puppeteerTimeout || 30_000,
       userDataDir,
+      executablePath: options?.executablePath, // allow using system Chrome
     });
+
     page = await browser.newPage();
-    await page.setExtraHTTPHeaders(DEFAULT_HEADER);
+    await applyBotPreventionToPage(page, preCfg);
+    // Provide languages value before navigation
+    await applyLanguagePersistence(page, preCfg);
+
+    // Optional cookies
+    if (Array.isArray(options?.cookies) && options.cookies.length > 0) {
+      await page.setCookie(...options.cookies);
+    }
+
+    // Navigation
     const response = await page.goto(url, {
-      waitUntil: 'domcontentloaded',
+      waitUntil: options?.waitUntil || 'domcontentloaded',
     });
+
+    // Optionally wait and add subtle human-like interactions
+    await applyPostNavigationHumanSignals(page, preCfg);
+
     let pageSource;
     // if we're extracting data from a SPA, we must wait for the selector
     if (waitForSelector != null) {
@@ -57,7 +90,7 @@ export default async function execute(url, waitForSelector, options) {
       pageSource = await page.content();
     }
 
-    const statusCode = response.status();
+    const statusCode = response?.status?.() ?? 200;
 
     if (botDetected(pageSource, statusCode)) {
       logger.warn('We have been detected as a bot :-/ Tried url: => ', url);

lib/services/markdown.js

@@ -1,6 +1,4 @@
-import markdown$0 from 'markdown';
 import fs from 'fs';
-const markdown = markdown$0.markdown;
 export function markdown2Html(filePath) {
-  return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
+  return fs.readFileSync(filePath, 'utf8');
 }

lib/services/similarity-check/similarityCache.js

@@ -1,116 +1,94 @@
-import crypto from 'crypto';
-
-const retention = 60 * 60 * 1000;
 /**
- * Internal cache storage.
+ * Similarity cache
- * Maps a SHA-256 hash (string) to its expiry timestamp (number in ms).
- * @type {Map<string, number>}
- */
-const entries = new Map();
-
-/**
- * Reference to the currently scheduled cleanup timer.
- * @type {NodeJS.Timeout | null}
- */
-let timer = null;
-
-/**
- * Generate a SHA-256 hash from a list of input strings.
- * Null or undefined values are ignored.
  *
- * @param {...(string|null|undefined)} strings - Input values to hash
+ * Maintains an in-memory Set of content hashes to detect whether a listing
+ * (identified by a tuple of title, price and address) has been seen before.
+ *
+ * Design notes:
+ * - The cache is refreshed periodically from persistent storage. To avoid
+ *   modification-during-iteration issues, the refresh builds a new Set and
+ *   atomically swaps the reference instead of mutating in place.
+ * - Hashing ignores null/undefined values but preserves falsy-yet-valid values
+ *   like 0. Non-string values are coerced to strings before hashing.
+ *
+ * This module has no persistence of its own; it relies on
+ * getAllEntriesFromListings() for data hydration.
+ * @module similarityCache
+ */
+import crypto from 'crypto';
+import { getAllEntriesFromListings } from '../storage/listingsStorage.js';
+
+/** @type {number} Refresh interval in milliseconds (defaults to one hour). */
+const reloadCycle = 60 * 60 * 1000; // every hour, refresh
+
+/**
+ * Internal cache of content hashes for known listings.
+ *
+ * Each entry is an SHA-256 hex digest produced by toHash(title, price, address).
+ * @type {Set<string>}
+ */
+let cache = new Set();
+
+export const startSimilarityCacheReloader = () => {
+  // Periodically refresh the cache from storage
+  setInterval(() => {
+    initSimilarityCache();
+  }, reloadCycle);
+};
+
+/**
+ * Initialize or refresh the similarity cache from persistent storage.
+ *
+ * Reads all stored listings via getAllEntriesFromListings(), computes a hash for
+ * each, and swaps the in-memory Set atomically to avoid in-place mutations that
+ * could interfere with concurrent iteration.
+ *
+ * This function is idempotent and safe to call at any time.
+ * @returns {void}
+ */
+export const initSimilarityCache = () => {
+  const allEntries = getAllEntriesFromListings();
+  const newCache = new Set();
+  for (const entry of allEntries) {
+    newCache.add(toHash(entry?.title, entry?.price, entry?.address));
+  }
+  // Atomic swap to avoid mutating the cache while it may be iterated elsewhere
+  cache = newCache;
+};
+
+/**
+ * Check if a listing is already known and add it to the cache if not.
+ *
+ * The listing is identified by the combination of its title, price and
+ * address. Null/undefined fields are ignored during hashing. Falsy-but-valid
+ * values (e.g., price 0) are preserved.
+ *
+ * @param {Object} params - Listing fields
+ * @param {string|undefined|null} params.title - The listing title
+ * @param {string|undefined|null} params.address - The listing address
+ * @param {number|string|undefined|null} params.price - The listing price
+ * @returns {boolean} true if the entry already existed in the cache (duplicate), otherwise false
+ */
+export const checkAndAddEntry = ({ title, address, price }) => {
+  const hash = toHash(title, price, address);
+  if (cache.has(hash)) {
+    return true;
+  }
+  cache.add(hash);
+  return false;
+};
+
+/**
+ * Generate an SHA-256 hash from a list of input values.
+ * Null or undefined values are ignored. Falsy but valid values like 0 are preserved.
+ * Non-string values are coerced to strings prior to hashing.
+ *
+ * @param {...(string|number|null|undefined)} strings - Input values to hash
  * @returns {string} Hexadecimal hash
  */
 function toHash(...strings) {
-  return crypto.createHash('sha256').update(strings.filter(Boolean).join('|')).digest('hex');
+  const normalized = strings
-}
+    .filter((v) => v !== null && v !== undefined)
-
+    .map((v) => (typeof v === 'string' ? v : String(v)));
-/**
+  return crypto.createHash('sha256').update(normalized.join('|')).digest('hex');
- * Cleanup expired cache entries and schedule the next cleanup run.
- * This function is invoked automatically by scheduled timers.
- *
- * @private
- */
-function runCleanup() {
-  const now = Date.now();
-  for (const [hash, expiry] of entries) {
-    if (expiry <= now) entries.delete(hash);
-  }
-  scheduleNext();
-}
-
-/**
- * Find the soonest expiry timestamp among all cache entries
- * and schedule a one-shot timer that will trigger at that time.
- * Cancels any existing timer before scheduling a new one.
- *
- * @private
- */
-function scheduleNext() {
-  if (timer) {
-    clearTimeout(timer);
-    timer = null;
-  }
-  let next = Infinity;
-  const now = Date.now();
-  for (const expiry of entries.values()) {
-    if (expiry > now && expiry < next) next = expiry;
-  }
-  if (next !== Infinity) {
-    timer = setTimeout(runCleanup, Math.max(0, next - now));
-  }
-}
-
-/**
- * Add or refresh a cache entry for the given title and address.
- * The entry will automatically expire after the configured retention window.
- *
- * @param {string} title - The title used to build the cache key
- * @param {string} address - The address used to build the cache key
- */
-export function addCacheEntry(title, address) {
-  const hash = toHash(title, address);
-  const expiry = Date.now() + retention;
-  entries.set(hash, expiry);
-  scheduleNext();
-}
-
-/**
- * Check if a cache entry with the same title and address exists
- * and is still valid (not expired).
- *
- * @param {string} title - The title used to build the cache key
- * @param {string} address - The address used to build the cache key
- * @returns {boolean} True if a valid cache entry exists, false otherwise
- */
-export function hasSimilarEntries(title, address) {
-  const hash = toHash(title, address);
-  const expiry = entries.get(hash);
-  if (expiry == null) return false;
-  if (expiry <= Date.now()) {
-    entries.delete(hash);
-    scheduleNext();
-    return false;
-  }
-  return true;
-}
-
-/**
- * Stop any scheduled cleanup timers and prevent further automatic cleanup.
- * Entries that are already in the cache will remain until removed manually
- * or until cleanup is started again by adding new entries.
- */
-export function stopCacheCleanup() {
-  if (timer) clearTimeout(timer);
-  timer = null;
-}
-
-/**
- * this is only for test purposes
- */
-export function invalidateAllForTest() {
-  for (const key of entries.keys()) {
-    entries.set(key, 0);
-  }
-  runCleanup();
 }

lib/services/storage/SqliteConnection.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import Database from 'better-sqlite3';
 import logger from '../../services/logger.js';
-import { config } from '../../utils.js';
+import { readConfigFromStorage } from '../../utils.js';
 
 /**
  * SqliteConnection
@@ -25,6 +25,15 @@ import { config } from '../../utils.js';
 class SqliteConnection {
   static #db = null;
 
+  static #sqlLiteCfg = null;
+
+  static async init() {
+    if (this.#sqlLiteCfg == null) {
+      readConfigFromStorage().then((c) => {
+        this.#sqlLiteCfg = c.sqlitepath;
+      });
+    }
+  }
   /**
    * Returns a singleton instance of better-sqlite3 Database.
    * Respects env var SQLITE_DB_PATH and defaults to db/listings.db.
@@ -32,9 +41,12 @@ class SqliteConnection {
   static getConnection() {
     if (this.#db) return this.#db;
 
+    if (this.#sqlLiteCfg == null) {
+      logger.warn('No sqlitepath configured. Using default db/listings.db');
+    }
+
     // Interpret config.sqlitepath as a directory relative to project root when it starts with '/'
-    const cfg = typeof config === 'object' && config ? config.sqlitepath : undefined;
+    const rawDir = this.#sqlLiteCfg && this.#sqlLiteCfg.length > 0 ? this.#sqlLiteCfg : '/db';
-    const rawDir = cfg && cfg.length > 0 ? cfg : '/db';
     const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
     const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
     const dbPath = path.join(absDir, 'listings.db');

lib/services/storage/listingsStorage.js

@@ -152,8 +152,9 @@ export const storeListings = (jobId, providerId, listings) => {
    */
   function extractNumber(str) {
     if (!str) return null;
-    const match = str.replace(/[.,]/g, '').match(/\d+/);
+    const cleaned = str.replace(/\./g, '').replace(',', '.');
-    return match ? +match[0] : null;
+    const num = parseFloat(cleaned);
+    return isNaN(num) ? null : num;
   }
 
   /**
@@ -310,8 +311,8 @@ export const deleteListingsByJobId = (jobId) => {
   if (!jobId) return;
   return SqliteConnection.execute(
     `DELETE
-                                   FROM listings
+     FROM listings
-                                   WHERE job_id = @jobId`,
+     WHERE job_id = @jobId`,
     { jobId },
   );
 };
@@ -332,3 +333,13 @@ export const deleteListingsById = (ids) => {
     ids,
   );
 };
+
+/**
+ * Return all listings with only the fields: title, address, and price.
+ * This is the single helper requested for simple consumers.
+ *
+ * @returns {{title: string|null, address: string|null, price: number|null}[]}
+ */
+export const getAllEntriesFromListings = () => {
+  return SqliteConnection.query(`SELECT title, address, price FROM listings`);
+};

lib/services/storage/migrations/sql/6.settings.js

@@ -0,0 +1,73 @@
+// Migration: Adding a settings table to store important (config) settings instead of using config file
+import fs from 'fs';
+import path from 'path';
+import { nanoid } from 'nanoid';
+import logger from '../../../logger.js';
+
+export function up(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS settings
+    (
+      id           TEXT PRIMARY KEY,
+      create_date  INTEGER NOT NULL,
+      user_id      TEXT,
+      name         TEXT    NOT NULL,
+      value        jsonb   NOT NULL
+    );
+
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_settings_name ON settings (name);
+  `);
+
+  // Helper to insert one setting row
+  const insertSetting = (name, rawValue) => {
+    try {
+      const id = nanoid();
+      const createDate = Date.now();
+      const value = JSON.stringify(rawValue);
+      db.prepare(
+        `INSERT INTO settings (id, create_date, name, value)
+         VALUES (@id, @create_date, @name, @value)`,
+      ).run({ id, create_date: createDate, name, value });
+    } catch {
+      // Ignore duplicate inserts if any (unique by name)
+    }
+  };
+
+  // Migrate currently existing config.json into settings
+  try {
+    const configPath = path.resolve(process.cwd(), 'conf', 'config.json');
+
+    // Defaults
+    const defaults = {
+      interval: '60',
+      port: 9998,
+      workingHours: { from: '', to: '' },
+      demoMode: false,
+      analyticsEnabled: true,
+    };
+
+    let config = {};
+    if (fs.existsSync(configPath)) {
+      const file = fs.readFileSync(configPath, 'utf8');
+      try {
+        config = JSON.parse(file) || {};
+      } catch (parseErr) {
+        // If parsing fails, still proceed with defaults
+        logger.error(parseErr);
+        config = {};
+      }
+    }
+
+    // Insert each known setting, using the value from config when present, otherwise default
+    insertSetting('interval', config.interval != null ? config.interval : defaults.interval);
+    insertSetting('port', config.port != null ? config.port : defaults.port);
+    insertSetting('workingHours', config.workingHours != null ? config.workingHours : defaults.workingHours);
+    insertSetting('demoMode', config.demoMode != null ? config.demoMode : defaults.demoMode);
+    insertSetting(
+      'analyticsEnabled',
+      config.analyticsEnabled != null ? config.analyticsEnabled : defaults.analyticsEnabled,
+    );
+  } catch (e) {
+    logger.error(e);
+  }
+}

lib/services/storage/settingsStorage.js

@@ -0,0 +1,87 @@
+import { nanoid } from 'nanoid';
+import SqliteConnection from './SqliteConnection.js';
+import { fromJson, readConfigFromStorage, toJson } from '../../utils.js';
+
+// In-memory cache for compiled settings config
+/** @type {Record<string, any>|null} */
+let cachedSettingsConfig = null;
+
+/**
+ * Build a config object from DB rows of settings.
+ * - Unwraps stored shape { value: any } into raw values.
+ * - Add additional config values from file config. E.g. sqlite part cannot be stored in db for obvious reasons ;)
+ * @param {{name:string, value:string|null}[]} rows
+ * @param {{name:value}} configValues
+ * @returns {Record<string, any>}
+ */
+function compileSettings(rows, configValues) {
+  const config = {};
+  for (const r of rows) {
+    const parsed = fromJson(r.value, null);
+    // unwrap { value: any } if present
+    config[r.name] = parsed && typeof parsed === 'object' && 'value' in parsed ? parsed.value : parsed;
+  }
+  return {
+    ...config,
+    ...configValues,
+  };
+}
+
+/**
+ * Force reload the settings config cache from DB and return it.
+ * @returns {Record<string, any>}
+ */
+export async function refreshSettingsCache() {
+  const rows = SqliteConnection.query(`SELECT name, value FROM settings`);
+  const configValues = await readConfigFromStorage();
+  cachedSettingsConfig = compileSettings(rows, configValues);
+  return cachedSettingsConfig;
+}
+
+/**
+ * Get the compiled settings config. Loads it once and caches the result.
+ * @returns {Record<string, any>}
+ */
+export async function getSettings() {
+  if (cachedSettingsConfig == null) {
+    return refreshSettingsCache();
+  }
+  return cachedSettingsConfig;
+}
+
+/**
+ * Upsert settings rows.
+ * - Accepts an object map of name -> value, or an entry {name, value}.
+ * - id: random string (nanoid) when inserting
+ * - create_date: epoch ms when inserting
+ * - name: unique key
+ * - value: JSON string of the raw value (no wrapper)
+ * @param {Record<string, any>|{name:string, value:any}|[string, any][]} settingsMapOrEntry
+ * @returns {void}
+ */
+// Upsert one or more settings by name. Accepts either a single pair or an object map.
+// Preferred usage: upsertSettings({ settingName: any, another: any })
+export function upsertSettings(settingsMapOrEntry, userId = null) {
+  const entries = Array.isArray(settingsMapOrEntry)
+    ? settingsMapOrEntry
+    : typeof settingsMapOrEntry === 'object' &&
+        settingsMapOrEntry != null &&
+        'name' in settingsMapOrEntry &&
+        'value' in settingsMapOrEntry
+      ? [[settingsMapOrEntry.name, settingsMapOrEntry.value]]
+      : Object.entries(settingsMapOrEntry || {});
+
+  for (const [name, rawValue] of entries) {
+    const id = nanoid();
+    const create_date = Date.now();
+    const json = toJson(rawValue);
+    SqliteConnection.execute(
+      `INSERT INTO settings (id, create_date, name, value, user_id)
+       VALUES (@id, @create_date, @name, @value, @userId)
+       ON CONFLICT(name) DO UPDATE SET value = excluded.value`,
+      { id, create_date, name, value: json, userId },
+    );
+  }
+  // keep cache in sync
+  refreshSettingsCache();
+}

lib/services/storage/userStorage.js

@@ -1,7 +1,7 @@
-import { config } from '../../utils.js';
 import * as hasher from '../security/hash.js';
 import { nanoid } from 'nanoid';
 import SqliteConnection from './SqliteConnection.js';
+import { getSettings } from './settingsStorage.js';
 
 /**
  * Get all users.
@@ -129,8 +129,9 @@ export const removeUser = (userId) => {
  * Security: The demo user's password is set to a known value ('demo') and should only be enabled in demoMode.
  * @returns {void}
  */
-export const ensureDemoUserExists = () => {
+export const ensureDemoUserExists = async () => {
-  if (!config.demoMode) {
+  const settings = await getSettings();
+  if (!settings.demoMode) {
     // Remove demo user (and cascade delete their jobs/listings)
     SqliteConnection.execute(`DELETE FROM users WHERE username = 'demo'`);
     return;

lib/services/tracking/Tracker.js

@@ -1,9 +1,10 @@
 import { getJobs } from '../storage/jobStorage.js';
 import { getUniqueId } from './uniqueId.js';
-import { config, getPackageVersion, inDevMode } from '../../utils.js';
+import { getPackageVersion, inDevMode } from '../../utils.js';
 import os from 'os';
 import fetch from 'node-fetch';
 import logger from '../logger.js';
+import { getSettings } from '../storage/settingsStorage.js';
 
 const deviceId = getUniqueId() || 'N/A';
 const version = await getPackageVersion();
@@ -11,7 +12,8 @@ const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
 
 export const trackMainEvent = async () => {
   try {
-    if (config.analyticsEnabled && !inDevMode()) {
+    const settings = await getSettings();
+    if (settings.analyticsEnabled && !inDevMode()) {
       const activeProvider = new Set();
       const activeAdapter = new Set();
 
@@ -44,7 +46,8 @@ export const trackMainEvent = async () => {
  * Note, this will only be used when Fredy runs in demo mode
  */
 export async function trackDemoAccessed() {
-  if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+  const settings = await getSettings();
+  if (settings.analyticsEnabled && !inDevMode() && settings.demoMode) {
     try {
       await fetch(`${FREDY_TRACKING_URL}/demo/accessed`, {
         method: 'POST',
@@ -56,7 +59,8 @@ export async function trackDemoAccessed() {
   }
 }
 
-function enrichTrackingObject(trackingObject) {
+async function enrichTrackingObject(trackingObject) {
+  const settings = await getSettings();
   const operatingSystem = os.platform();
   const osVersion = os.release();
   const arch = process.arch;
@@ -65,7 +69,7 @@ function enrichTrackingObject(trackingObject) {
 
   return {
     ...trackingObject,
-    isDemo: config.demoMode,
+    isDemo: settings.demoMode,
     operatingSystem,
     osVersion,
     arch,

lib/utils.js

@@ -215,10 +215,6 @@ export async function refreshConfig() {
 
   try {
     config = await readConfigFromStorage();
-    //backwards compatibility...
-    config.analyticsEnabled ??= null;
-    config.demoMode ??= false;
-    // default sqlitepath when missing in older configs
     config.sqlitepath ??= '/db';
   } catch (error) {
     config = { ...DEFAULT_CONFIG };
@@ -306,7 +302,6 @@ export {
   getDirName,
   sleep,
   randomBetween,
-  config,
   buildHash,
   getPackageVersion,
   toJson,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "14.2.2",
+  "version": "15.0.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -12,6 +12,7 @@
     "format": "prettier --write \"**/*.js\"",
     "format:check": "prettier --check \"**/*.js\"",
     "test": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 test/**/*.test.js",
+    "testGH": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 --exclude test/provider/immonet.test.js --exclude test/provider/immowelt.test.js test/**/*.test.js",
     "lint": "eslint .",
     "lint:fix": "yarn lint --fix",
     "migratedb": "node lib/services/storage/migrations/migrate.js",
@@ -56,58 +57,57 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-icons": "^2.86.0",
+    "@douyinfe/semi-icons": "^2.89.0",
-    "@douyinfe/semi-ui": "2.86.0",
+    "@douyinfe/semi-ui": "2.89.0",
     "@sendgrid/mail": "8.1.6",
-    "@visactor/react-vchart": "^2.0.5",
+    "@visactor/react-vchart": "^2.0.10",
-    "@visactor/vchart": "^2.0.5",
+    "@visactor/vchart": "^2.0.10",
     "@visactor/vchart-semi-theme": "^1.12.2",
-    "@vitejs/plugin-react": "5.0.4",
+    "@vitejs/plugin-react": "5.1.2",
-    "better-sqlite3": "^12.4.1",
+    "better-sqlite3": "^12.5.0",
-    "body-parser": "2.2.0",
+    "body-parser": "2.2.1",
     "cheerio": "^1.1.2",
     "cookie-session": "2.1.1",
     "handlebars": "4.7.8",
     "lodash": "4.17.21",
-    "markdown": "^0.5.0",
     "nanoid": "5.1.6",
     "node-cron": "^4.2.1",
     "node-fetch": "3.3.2",
-    "node-mailjet": "6.0.9",
+    "node-mailjet": "6.0.11",
-    "p-throttle": "^8.0.0",
+    "p-throttle": "^8.1.0",
     "package-up": "^5.0.0",
-    "puppeteer": "^24.24.0",
+    "puppeteer": "^24.32.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "query-string": "9.3.1",
     "react": "18.3.1",
     "react-dom": "18.3.1",
-    "react-router": "7.9.4",
+    "react-router": "7.10.1",
-    "react-router-dom": "7.9.4",
+    "react-router-dom": "7.10.1",
     "restana": "5.1.0",
     "semver": "^7.7.3",
     "serve-static": "2.2.0",
     "slack": "11.0.2",
-    "vite": "7.1.9",
+    "vite": "7.2.7",
     "x-var": "^3.0.1",
-    "zustand": "^5.0.8"
+    "zustand": "^5.0.9"
   },
   "devDependencies": {
-    "@babel/core": "7.28.4",
+    "@babel/core": "7.28.5",
-    "@babel/eslint-parser": "7.28.4",
+    "@babel/eslint-parser": "7.28.5",
-    "@babel/preset-env": "7.28.3",
+    "@babel/preset-env": "7.28.5",
-    "@babel/preset-react": "7.27.1",
+    "@babel/preset-react": "7.28.5",
-    "chai": "6.2.0",
+    "chai": "6.2.1",
-    "eslint": "9.37.0",
+    "eslint": "9.39.1",
     "eslint-config-prettier": "10.1.8",
     "eslint-plugin-react": "7.37.5",
     "esmock": "2.7.3",
     "history": "5.3.0",
     "husky": "9.1.7",
     "less": "4.4.2",
-    "lint-staged": "16.2.4",
+    "lint-staged": "16.2.7",
-    "mocha": "11.7.4",
+    "mocha": "11.7.5",
-    "nodemon": "^3.1.10",
+    "nodemon": "^3.1.11",
-    "prettier": "3.6.2"
+    "prettier": "3.7.4"
   }
 }

test/FredyPipeline/FredyPipeline.test.js

@@ -1,53 +0,0 @@
-import { expect } from 'chai';
-import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { mockFredy } from '../utils.js';
-
-describe('FredyPipeline', () => {
-  afterEach(() => {
-    similarityCache.invalidateAllForTest();
-  });
-
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
-  describe('_filterBySimilarListings', () => {
-    let fredyRuntime;
-
-    beforeEach(async () => {
-      const FredyRuntime = await mockFredy();
-      fredyRuntime = new FredyRuntime({}, null, 'dummy-provider', 'dummy-job', similarityCache);
-    });
-
-    it('should filter out listings with similar title and address already in cache', () => {
-      similarityCache.addCacheEntry('Penthouse', 'Mustermann Straße 1');
-
-      const listings = [
-        { id: '1', title: 'Penthouse', address: 'Mustermann Straße 1' },
-        { id: '2', title: 'Nice apartment', address: 'Mustermann Straße 15' },
-      ];
-
-      const result = fredyRuntime._filterBySimilarListings(listings);
-
-      expect(result).to.have.length(1);
-      expect(result[0].id).to.equal('2');
-      expect(result[0].title).to.equal('Nice apartment');
-
-      expect(similarityCache.hasSimilarEntries('Nice apartment', 'Mustermann Straße 15')).to.be.true;
-    });
-
-    it('should handle listings with null or undefined address', () => {
-      const listings = [
-        { id: '1', title: 'Penthouse', address: null },
-        { id: '2', title: 'Nice apartment', address: undefined },
-      ];
-
-      const result = fredyRuntime._filterBySimilarListings(listings);
-
-      expect(result).to.have.length(2);
-
-      expect(similarityCache.hasSimilarEntries('Penthouse', null)).to.be.true;
-      expect(similarityCache.hasSimilarEntries('Nice apartment', undefined)).to.be.true;
-    });
-  });
-});

test/provider/einsAImmobilien.test.js

@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/einsAImmobilien.js';
 
 describe('#einsAImmobilien testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
   provider.init(providerConfig.einsAImmobilien, [], []);
   it('should test einsAImmobilien provider', async () => {
     const Fredy = await mockFredy();

test/provider/immobilienDe.test.js

@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immobilienDe.js';
 
 describe('#immobilien.de testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
   provider.init(providerConfig.immobilienDe, [], []);
   it('should test immobilien.de provider', async () => {
     const Fredy = await mockFredy();

test/provider/immonet.test.js

@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immonet.js';
 
 describe('#immonet testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
   it('should test immonet provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.immonet, [], []);

test/provider/immoscout.test.js

@@ -5,10 +5,6 @@ import { get } from '../mocks/mockNotification.js';
 import * as provider from '../../lib/provider/immoscout.js';
 
 describe('#immoscout provider testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
   provider.init(providerConfig.immoscout, [], []);
   it('should test immoscout provider', async () => {
     const Fredy = await mockFredy();

test/provider/immoswp.test.js

@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoswp.js';
 
 describe('#immoswp testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
   provider.init(providerConfig.immoswp, [], []);
   it('should test immoswp provider', async () => {
     const Fredy = await mockFredy();

test/provider/immowelt.test.js

@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immowelt.js';
 
 describe('#immowelt testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
   it('should test immowelt provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);

test/provider/kleinanzeigen.test.js

@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/kleinanzeigen.js';
 
 describe('#kleinanzeigen testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
   it('should test kleinanzeigen provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.kleinanzeigen, [], []);

test/provider/mcMakler.test.js

@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/mcMakler.js';
 
 describe('#mcMakler testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
   it('should test mcMakler provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.mcMakler, []);

test/provider/neubauKompass.test.js

@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
 
 describe('#neubauKompass testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
   provider.init(providerConfig.neubauKompass, [], []);
   it('should test neubauKompass provider', async () => {
     const Fredy = await mockFredy();

test/provider/ohneMakler.test.js

@@ -0,0 +1,33 @@
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { get } from '../mocks/mockNotification.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { expect } from 'chai';
+import * as provider from '../../lib/provider/ohneMakler.js';
+
+describe('#ohneMakler testsuite()', () => {
+  it('should test ohneMakler provider', async () => {
+    const Fredy = await mockFredy();
+    provider.init(providerConfig.ohneMakler, []);
+
+    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'ohneMakler', similarityCache);
+    const listing = await fredy.execute();
+
+    expect(listing).to.be.a('array');
+    const notificationObj = get();
+    expect(notificationObj).to.be.a('object');
+    expect(notificationObj.serviceName).to.equal('ohneMakler');
+    notificationObj.payload.forEach((notify) => {
+      /** check the actual structure **/
+      expect(notify.id).to.be.a('string');
+      expect(notify.price).to.be.a('string');
+      expect(notify.size).to.be.a('string');
+      expect(notify.title).to.be.a('string');
+      expect(notify.link).to.be.a('string');
+      expect(notify.address).to.be.a('string');
+      /** check the values if possible **/
+      expect(notify.size).that.does.include('m²');
+      expect(notify.title).to.be.not.empty;
+      expect(notify.address).to.be.not.empty;
+    });
+  });
+});

test/provider/regionalimmobilien24.test.js

@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/regionalimmobilien24.js';
 
 describe('#regionalimmobilien24 testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
   it('should test regionalimmobilien24 provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.regionalimmobilien24, []);

test/provider/sparkasse.test.js

@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/sparkasse.js';
 
 describe('#sparkasse testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
   it('should test sparkasse provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.sparkasse, []);

test/provider/testProvider.json

@@ -32,6 +32,10 @@
     "url": "https://www.mcmakler.de/immobilien/results?placeId=62649&search=Leipzig%252C+Sachsen&propertyTypes=APARTMENT&page=0",
     "enabled": true
   },
+  "ohneMakler": {
+    "url": "https://www.ohne-makler.net/immobilien/wohnung-kaufen/nordrhein-westfalen/dusseldorf/",
+    "enabled": true
+  },
   "neubauKompass": {
     "url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
     "enabled": true

test/provider/wgGesucht.test.js

@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/wgGesucht.js';
 
 describe('#wgGesucht testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
   provider.init(providerConfig.wgGesucht, [], []);
   it('should test wgGesucht provider', async () => {
     const Fredy = await mockFredy();

test/services/extractor/botPrevention.test.js

@@ -0,0 +1,99 @@
+import { describe, it } from 'mocha';
+import { expect } from 'chai';
+
+import {
+  getPreLaunchConfig,
+  applyBotPreventionToPage,
+  applyLanguagePersistence,
+  applyPostNavigationHumanSignals,
+} from '../../../lib/services/extractor/botPrevention.js';
+
+describe('botPrevention helper', () => {
+  it('getPreLaunchConfig builds deterministic values when jitter disabled', () => {
+    const url = 'https://example.com/some/path';
+    const options = {
+      acceptLanguage: 'de-DE,de;q=0.9',
+      userAgent: 'TestAgent/1.0',
+      viewport: { width: 1200, height: 700, deviceScaleFactor: 2 },
+      viewportJitter: false,
+      referer: 'https://example.com/ref',
+      timezone: 'Europe/Berlin',
+    };
+    const cfg = getPreLaunchConfig(url, options);
+
+    expect(cfg.acceptLanguage).to.equal('de-DE,de;q=0.9');
+    expect(cfg.langArg).to.equal('--lang=de-DE');
+    expect(cfg.windowSizeArg).to.equal('--window-size=1200,700');
+    expect(cfg.viewport).to.deep.equal({ width: 1200, height: 700, deviceScaleFactor: 2 });
+    expect(cfg.userAgent).to.equal('TestAgent/1.0');
+    expect(cfg.headers['Accept-Language']).to.equal('de-DE,de;q=0.9');
+    expect(cfg.headers['User-Agent']).to.equal('TestAgent/1.0');
+    expect(cfg.headers.Referer).to.equal('https://example.com/ref');
+    expect(cfg.extraArgs).to.include('--disable-blink-features=AutomationControlled');
+    expect(cfg.extraArgs).to.include('--proxy-bypass-list=<-loopback>');
+  });
+
+  it('applyBotPreventionToPage sets UA, viewport, headers and injects patches', async () => {
+    const calls = [];
+    const page = {
+      setUserAgent: async (ua) => calls.push(['setUserAgent', ua]),
+      setViewport: async (vp) => calls.push(['setViewport', vp]),
+      setJavaScriptEnabled: async (on) => calls.push(['setJavaScriptEnabled', on]),
+      setExtraHTTPHeaders: async (h) => calls.push(['setExtraHTTPHeaders', h]),
+      emulateTimezone: async (tz) => calls.push(['emulateTimezone', tz]),
+      evaluateOnNewDocument: async (fn) => calls.push(['evaluateOnNewDocument', typeof fn]),
+    };
+    const cfg = getPreLaunchConfig('https://example.org/', {
+      userAgent: 'Foo/Bar',
+      acceptLanguage: 'en-US,en',
+      viewport: { width: 1000, height: 600, deviceScaleFactor: 1 },
+      viewportJitter: false,
+      timezone: 'UTC',
+    });
+
+    await applyBotPreventionToPage(page, cfg);
+
+    expect(calls[0]).to.deep.equal(['setUserAgent', 'Foo/Bar']);
+    expect(calls.some((c) => c[0] === 'setViewport' && c[1].width === 1000 && c[1].height === 600)).to.equal(true);
+    expect(calls.some((c) => c[0] === 'setJavaScriptEnabled' && c[1] === true)).to.equal(true);
+    const headerCall = calls.find((c) => c[0] === 'setExtraHTTPHeaders');
+    expect(headerCall).to.exist;
+    expect(headerCall[1]['Accept-Language']).to.equal('en-US,en');
+    expect(headerCall[1]['User-Agent']).to.equal('Foo/Bar');
+    expect(calls.some((c) => c[0] === 'emulateTimezone' && c[1] === 'UTC')).to.equal(true);
+    expect(calls.some((c) => c[0] === 'evaluateOnNewDocument' && c[1] === 'function')).to.equal(true);
+  });
+
+  it('applyLanguagePersistence stores languages early', async () => {
+    const calls = [];
+    const page = {
+      evaluateOnNewDocument: async (fn, arg) => calls.push(['evaluateOnNewDocument', typeof fn, arg]),
+    };
+    const cfg = getPreLaunchConfig('https://example.org/', {
+      acceptLanguage: 'de-DE,de;q=0.9',
+      viewportJitter: false,
+    });
+    await applyLanguagePersistence(page, cfg);
+    const call = calls[0];
+    expect(call[0]).to.equal('evaluateOnNewDocument');
+    expect(call[1]).to.equal('function');
+    expect(call[2]).to.equal('de-DE,de');
+  });
+
+  it('applyPostNavigationHumanSignals moves mouse and scrolls when enabled', async () => {
+    const mouseCalls = [];
+    const page = {
+      mouse: {
+        move: async (x, y, opts) => mouseCalls.push(['move', x, y, opts && typeof opts.steps === 'number']),
+        wheel: async (opts) => mouseCalls.push(['wheel', typeof opts.deltaY === 'number']),
+      },
+    };
+    const cfg = {
+      humanDelay: true,
+      viewport: { width: 1200, height: 800 },
+    };
+    await applyPostNavigationHumanSignals(page, cfg);
+    expect(mouseCalls.some((c) => c[0] === 'move')).to.equal(true);
+    expect(mouseCalls.some((c) => c[0] === 'wheel')).to.equal(true);
+  });
+});

test/similarity/similarity.test.js

@@ -1,30 +0,0 @@
-import { expect } from 'chai';
-import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-
-describe('similarityCheck', () => {
-  it('should return true on duplicate', () => {
-    similarityCache.addCacheEntry('Hello World', 'Test');
-    expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.true;
-  });
-
-  it('should return true even if one value is null', () => {
-    similarityCache.addCacheEntry('Hello World', null);
-    expect(similarityCache.hasSimilarEntries('Hello World', null)).to.be.true;
-  });
-
-  it('should return true even if one value is an obj', () => {
-    similarityCache.addCacheEntry('Hello World', [{ TR: 'OLOLO' }]);
-    expect(similarityCache.hasSimilarEntries('Hello World', [{ TR: 'OLOLO' }])).to.be.true;
-  });
-
-  it('should return false when no duplicate', () => {
-    similarityCache.addCacheEntry('Hello World__', 'Test');
-    expect(similarityCache.hasSimilarEntries('Hello World___', 'Test')).to.be.false;
-  });
-
-  it('should return false when no duplicate', () => {
-    expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.true;
-    similarityCache.invalidateAllForTest();
-    expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.false;
-  });
-});

test/similarity/similarityCache.test.js

@@ -0,0 +1,62 @@
+import { expect } from 'chai';
+import esmock from 'esmock';
+
+// Helper to create module under test with mocks
+async function loadModuleWith({ entries = [] } = {}) {
+  const mod = await esmock('../../lib/services/similarity-check/similarityCache.js', {
+    // Mock the storage to return our controlled entries
+    '../../lib/services/storage/listingsStorage.js': {
+      getAllEntriesFromListings: () => entries,
+    },
+  });
+  return mod;
+}
+
+describe('similarityCache', () => {
+  it('initSimilarityCache builds cache from storage and enables duplicate detection', async () => {
+    const entries = [
+      { title: 'A', price: 1000, address: 'Main 1' },
+      { title: 'B', price: 0, address: 'Zero St' },
+    ];
+
+    const { initSimilarityCache, checkAndAddEntry } = await loadModuleWith({ entries });
+
+    // Initially, duplicates should not be detected for new data
+    expect(checkAndAddEntry({ title: 'X', price: 200, address: 'Y' })).to.equal(false);
+
+    // Now initialize from storage
+    initSimilarityCache();
+
+    // Exact duplicates should be detected
+    expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).to.equal(true);
+    // Ensure falsy-but-valid price 0 is preserved by hashing and detected as duplicate
+    expect(checkAndAddEntry({ title: 'B', price: 0, address: 'Zero St' })).to.equal(true);
+  });
+
+  it('checkAndAddEntry returns false for new entry then true for duplicate on second call', async () => {
+    const { checkAndAddEntry } = await loadModuleWith();
+
+    const first = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' });
+    const second = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' });
+
+    expect(first).to.equal(false);
+    expect(second).to.equal(true);
+  });
+
+  it('hashing ignores null/undefined but preserves 0 via behavior', async () => {
+    const { checkAndAddEntry } = await loadModuleWith();
+
+    // Add baseline (null address ignored)
+    const add1 = checkAndAddEntry({ title: 'T', price: 1, address: null });
+    expect(add1).to.equal(false);
+    // Duplicate with undefined address should match
+    const dup = checkAndAddEntry({ title: 'T', price: 1, address: undefined });
+    expect(dup).to.equal(true);
+
+    // Now test that price 0 is preserved (not filtered out)
+    const addZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' });
+    expect(addZero).to.equal(false);
+    const dupZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' });
+    expect(dupZero).to.equal(true);
+  });
+});

ui/src/App.jsx

@@ -21,6 +21,7 @@ import Navigation from './components/navigation/Navigation.jsx';
 import { Layout } from '@douyinfe/semi-ui';
 import FredyFooter from './components/footer/FredyFooter.jsx';
 import ProcessingTimes from './views/jobs/ProcessingTimes.jsx';
+import WatchlistManagement from './views/listings/management/WatchlistManagement.jsx';
 
 export default function FredyApp() {
   const actions = useActions();
@@ -34,6 +35,7 @@ export default function FredyApp() {
     async function init() {
       await actions.user.getCurrentUser();
       if (!needsLogin()) {
+        await actions.features.getFeatures();
         await actions.provider.getProvider();
         await actions.jobs.getJobs();
         await actions.jobs.getProcessingTimes();
@@ -91,6 +93,7 @@ export default function FredyApp() {
               <Route path="/jobs/insights/:jobId" element={<JobInsight />} />
               <Route path="/jobs" element={<Jobs />} />
               <Route path="/listings" element={<Listings />} />
+              <Route path="/watchlistManagement" element={<WatchlistManagement />} />
 
               {/* Permission-aware routes */}
               <Route

ui/src/components/footer/FredyFooter.less

@@ -5,6 +5,8 @@
   justify-content: space-between;
   align-items: center;
   height: 1.7rem;
+  border-radius: .3rem;
+  border-top: 1px solid #45464b;
 
   &__version {
     padding-left: .5rem;

ui/src/components/navigation/Navigation.jsx

@@ -1,12 +1,13 @@
 import React from 'react';
 import { Nav } from '@douyinfe/semi-ui';
-import { IconUser, IconStar, IconSetting, IconTerminal } from '@douyinfe/semi-icons';
+import { IconStar, IconSetting, IconTerminal } from '@douyinfe/semi-icons';
 import logoWhite from '../../assets/logo_white.png';
 import Logout from '../logout/Logout.jsx';
 import { useLocation, useNavigate } from 'react-router-dom';
 
 import './Navigate.less';
 import { useScreenWidth } from '../../hooks/screenWidth.js';
+import { useFeature } from '../../hooks/featureHook.js';
 
 export default function Navigation({ isAdmin }) {
   const navigate = useNavigate();
@@ -14,15 +15,28 @@ export default function Navigation({ isAdmin }) {
 
   const width = useScreenWidth();
   const collapsed = width <= 850;
+  const watchlistFeature = useFeature('WATCHLIST_MANAGEMENT') || false;
 
   const items = [
     { itemKey: '/jobs', text: 'Jobs', icon: <IconTerminal /> },
-    { itemKey: '/listings', text: 'Found Listings', icon: <IconStar /> },
+    { itemKey: '/listings', text: 'Listings', icon: <IconStar /> },
   ];
 
   if (isAdmin) {
-    items.push({ itemKey: '/users', text: 'User Management', icon: <IconUser /> });
+    const settingsItems = [
-    items.push({ itemKey: '/generalSettings', text: 'Settings', icon: <IconSetting /> });
+      { itemKey: '/users', text: 'User Management' },
+      { itemKey: '/generalSettings', text: 'General Settings' },
+    ];
+    if (watchlistFeature) {
+      settingsItems.push({ itemKey: '/watchlistManagement', text: 'Watchlist Management' });
+    }
+
+    items.push({
+      itemKey: 'settings',
+      text: 'Settings',
+      icon: <IconSetting />,
+      items: settingsItems,
+    });
   }
 
   function parsePathName(name) {
@@ -32,7 +46,7 @@ export default function Navigation({ isAdmin }) {
 
   return (
     <Nav
-      style={{ height: '100%', width: collapsed ? '' : '13rem' }}
+      style={{ height: '100%', width: collapsed ? '' : '13.2rem' }}
       items={items}
       isCollapsed={collapsed}
       selectedKeys={[parsePathName(location.pathname)]}

ui/src/components/table/ProviderTable.jsx

@@ -1,9 +1,9 @@
 import React from 'react';
 
 import { Empty, Table, Button } from '@douyinfe/semi-ui';
-import { IconDelete } from '@douyinfe/semi-icons';
+import { IconDelete, IconEdit } from '@douyinfe/semi-icons';
 
-export default function ProviderTable({ providerData = [], onRemove } = {}) {
+export default function ProviderTable({ providerData = [], onRemove, onEdit } = {}) {
   return (
     <Table
       pagination={false}
@@ -30,6 +30,8 @@ export default function ProviderTable({ providerData = [], onRemove } = {}) {
           render: (_, record) => {
             return (
               <div style={{ float: 'right' }}>
+                <Button type="secondary" icon={<IconEdit />} onClick={() => onEdit(record)} />
+                <div style={{ display: 'inline-block', width: '16px' }} />
                 <Button type="danger" icon={<IconDelete />} onClick={() => onRemove(record.url)} />
               </div>
             );

ui/src/components/table/listings/ListingsFilter.jsx

@@ -1,53 +0,0 @@
-import { Card, Checkbox, Descriptions, Divider, Select } from '@douyinfe/semi-ui';
-import React from 'react';
-import { useSelector } from '../../../services/state/store.js';
-import { Typography } from '@douyinfe/semi-ui';
-
-import './ListingsFilter.less';
-
-export default function ListingsFilter({ onWatchListFilter, onActivityFilter, onJobNameFilter, onProviderFilter }) {
-  const jobs = useSelector((state) => state.jobs.jobs);
-  const provider = useSelector((state) => state.provider);
-  const { Title } = Typography;
-  return (
-    <Card className="listingsFilter">
-      <Title heading={6}>Filter by:</Title>
-      <Divider />
-      <br />
-      <Descriptions row>
-        <Descriptions.Item itemKey="Watch List">
-          <Checkbox onChange={(e) => onWatchListFilter(e.target.checked)}>Only Watch List</Checkbox>
-        </Descriptions.Item>
-        <Descriptions.Item itemKey="Activity status">
-          <Checkbox onChange={(e) => onActivityFilter(e.target.checked)}>Only Active Listings</Checkbox>
-        </Descriptions.Item>
-        <Descriptions.Item itemKey="Job Name">
-          <Select showClear placeholder="Select Job to Filter" onChange={(val) => onJobNameFilter(val)}>
-            {jobs != null &&
-              jobs.length > 0 &&
-              jobs.map((job) => {
-                return (
-                  <Select.Option value={job.id} key={job.id}>
-                    {job.name}
-                  </Select.Option>
-                );
-              })}
-          </Select>
-        </Descriptions.Item>
-        <Descriptions.Item itemKey="Provider">
-          <Select showClear placeholder="Select Provider to Filter" onChange={(val) => onProviderFilter(val)}>
-            {provider != null &&
-              provider.length > 0 &&
-              provider.map((prov) => {
-                return (
-                  <Select.Option value={prov.id} key={prov.id}>
-                    {prov.name}
-                  </Select.Option>
-                );
-              })}
-          </Select>
-        </Descriptions.Item>
-      </Descriptions>
-    </Card>
-  );
-}

ui/src/components/table/listings/ListingsFilter.less

@@ -1,4 +0,0 @@
-.listingsFilter {
-  margin-bottom: 1rem;
-  background: rgb(53, 54, 60);
-}

ui/src/components/table/listings/ListingsTable.jsx

@@ -1,5 +1,18 @@
 import React, { useState, useEffect, useMemo } from 'react';
-import { Table, Popover, Input, Descriptions, Tag, Image, Empty, Button, Toast, Divider } from '@douyinfe/semi-ui';
+import {
+  Table,
+  Popover,
+  Input,
+  Descriptions,
+  Tag,
+  Image,
+  Empty,
+  Button,
+  Toast,
+  Divider,
+  Space,
+  Select,
+} from '@douyinfe/semi-ui';
 import { useActions, useSelector } from '../../../services/state/store.js';
 import { IconClose, IconDelete, IconSearch, IconStar, IconStarStroked, IconTick } from '@douyinfe/semi-icons';
 import * as timeService from '../../../services/time/timeService.js';
@@ -10,166 +23,224 @@ import './ListingsTable.less';
 import { format } from '../../../services/time/timeService.js';
 import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
 import { xhrDelete, xhrPost } from '../../../services/xhr.js';
-import ListingsFilter from './ListingsFilter.jsx';
+import { useNavigate } from 'react-router-dom';
+import { useFeature } from '../../../hooks/featureHook.js';
 
-const columns = [
+const getColumns = (provider, setProviderFilter, jobs, setJobNameFilter) => {
-  {
+  return [
-    title: '#',
+    {
-    width: 100,
+      title: 'Watchlist',
-    dataIndex: 'isWatched',
+      width: 133,
-    sorter: true,
+      dataIndex: 'isWatched',
-    render: (id, row) => {
+      sorter: true,
-      return (
+      filters: [
-        <div>
+        {
-          <Popover
+          text: 'Show only watched listings',
-            style={{
+          value: 'watchList',
-              padding: '.4rem',
+        },
-              color: 'var(--semi-color-white)',
+      ],
-            }}
+      render: (id, row) => {
-            content={row.isWatched === 1 ? 'Unwatch Listing' : 'Watch Listing'}
+        return (
-          >
+          <div>
-            <Button
+            <Popover
-              icon={
+              style={{
-                row.isWatched === 1 ? (
+                padding: '.4rem',
-                  <IconStar style={{ color: 'rgba(var(--semi-green-5), 1)' }} />
+                color: 'var(--semi-color-white)',
-                ) : (
-                  <IconStarStroked />
-                )
-              }
-              theme="borderless"
-              size="small"
-              onClick={async () => {
-                try {
-                  await xhrPost('/api/listings/watch', { listingId: row.id });
-                  Toast.success(row.isWatched === 1 ? 'Listing removed from Watchlist' : 'Listing added to Watchlist');
-                  row.reloadTable();
-                } catch (e) {
-                  console.error(e);
-                  Toast.error('Failed to operate Watchlist');
-                }
               }}
-            />
+              content={row.isWatched === 1 ? 'Unwatch Listing' : 'Watch Listing'}
-          </Popover>
+            >
-          <Divider layout="vertical" margin="4px" />
+              <Button
-          <Popover
+                icon={
-            style={{
+                  row.isWatched === 1 ? (
-              padding: '.4rem',
+                    <IconStar style={{ color: 'rgba(var(--semi-green-5), 1)' }} />
-              color: 'var(--semi-color-white)',
+                  ) : (
-            }}
+                    <IconStarStroked />
-            content="Delete Listing"
+                  )
-          >
-            <Button
-              icon={<IconDelete />}
-              theme="borderless"
-              size="small"
-              type="danger"
-              onClick={async () => {
-                try {
-                  await xhrDelete('/api/listings/', { ids: [row.id] });
-                  Toast.success('Listing(s) successfully removed');
-                  row.reloadTable();
-                } catch (error) {
-                  Toast.error(error);
                 }
+                theme="borderless"
+                size="small"
+                onClick={async () => {
+                  try {
+                    await xhrPost('/api/listings/watch', { listingId: row.id });
+                    Toast.success(
+                      row.isWatched === 1 ? 'Listing removed from Watchlist' : 'Listing added to Watchlist',
+                    );
+                    row.reloadTable();
+                  } catch (e) {
+                    console.error(e);
+                    Toast.error('Failed to operate Watchlist');
+                  }
+                }}
+              />
+            </Popover>
+            <Divider layout="vertical" margin="4px" />
+            <Popover
+              style={{
+                padding: '.4rem',
+                color: 'var(--semi-color-white)',
               }}
-            />
+              content="Delete Listing"
-          </Popover>
+            >
-        </div>
+              <Button
-      );
+                icon={<IconDelete />}
+                theme="borderless"
+                size="small"
+                type="danger"
+                onClick={async () => {
+                  try {
+                    await xhrDelete('/api/listings/', { ids: [row.id] });
+                    Toast.success('Listing(s) successfully removed');
+                    row.reloadTable();
+                  } catch (error) {
+                    Toast.error(error);
+                  }
+                }}
+              />
+            </Popover>
+          </div>
+        );
+      },
     },
-  },
+    {
-  {
+      title: 'Active',
-    title: 'State',
+      dataIndex: 'is_active',
-    dataIndex: 'is_active',
+      width: 110,
-    width: 84,
+      sorter: true,
-    sorter: true,
+      filters: [
-    render: (value) => {
+        {
-      return value ? (
+          text: 'Show only active listings',
-        <div style={{ color: 'rgba(var(--semi-green-6), 1)' }}>
+          value: 'activityStatus',
-          <Popover
+        },
-            style={{
+      ],
-              padding: '.4rem',
+      render: (value) => {
-              color: 'var(--semi-color-white)',
+        return value ? (
-            }}
+          <div style={{ color: 'rgba(var(--semi-green-6), 1)' }}>
-            content="Listing is still active"
+            <Popover
-          >
+              style={{
-            <IconTick />
+                padding: '.4rem',
-          </Popover>
+                color: 'var(--semi-color-white)',
-        </div>
+              }}
-      ) : (
+              content="Listing is still active"
-        <div style={{ color: 'rgba(var(--semi-red-5), 1)' }}>
+            >
-          <Popover
+              <IconTick />
-            style={{
+            </Popover>
-              padding: '.4rem',
+          </div>
-              color: 'var(--semi-color-white)',
+        ) : (
-            }}
+          <div style={{ color: 'rgba(var(--semi-red-5), 1)' }}>
-            content="Listing is inactive"
+            <Popover
-          >
+              style={{
-            <IconClose />
+                padding: '.4rem',
-          </Popover>
+                color: 'var(--semi-color-white)',
-        </div>
+              }}
-      );
+              content="Listing is inactive"
+            >
+              <IconClose />
+            </Popover>
+          </div>
+        );
+      },
     },
-  },
+    {
-  {
+      title: 'Job-Name',
-    title: 'Job-Name',
+      sorter: true,
-    sorter: true,
+      ellipsis: true,
-    ellipsis: true,
+      dataIndex: 'job_name',
-    dataIndex: 'job_name',
+      width: 150,
-    width: 150,
+      onFilter: () => true,
-  },
+      renderFilterDropdown: () => {
-  {
+        return (
-    title: 'Listing date',
+          <Space vertical style={{ padding: 8 }}>
-    width: 130,
+            <Select showClear placeholder="Select Job to Filter" onChange={(val) => setJobNameFilter(val)}>
-    dataIndex: 'created_at',
+              {jobs != null &&
-    sorter: true,
+                jobs.length > 0 &&
-    render: (text) => timeService.format(text, false),
+                jobs.map((job) => {
-  },
+                  return (
-  {
+                    <Select.Option value={job.id} key={job.id}>
-    title: 'Provider',
+                      {job.name}
-    width: 130,
+                    </Select.Option>
-    dataIndex: 'provider',
+                  );
-    sorter: true,
+                })}
-    render: (text) => text.charAt(0).toUpperCase() + text.slice(1),
+            </Select>
-  },
+          </Space>
-  {
+        );
-    title: 'Price',
+      },
-    width: 110,
-    dataIndex: 'price',
-    sorter: true,
-    render: (text) => text + ' €',
-  },
-  {
-    title: 'Address',
-    width: 150,
-    dataIndex: 'address',
-    sorter: true,
-  },
-  {
-    title: 'Title',
-    dataIndex: 'title',
-    sorter: true,
-    ellipsis: true,
-    render: (text, row) => {
-      return (
-        <a href={row.url} target="_blank" rel="noopener noreferrer">
-          {text}
-        </a>
-      );
     },
-  },
+    {
-];
+      title: 'Listing date',
+      width: 130,
+      dataIndex: 'created_at',
+      sorter: true,
+      render: (text) => timeService.format(text, false),
+    },
+    {
+      title: 'Provider',
+      width: 130,
+      dataIndex: 'provider',
+      sorter: true,
+      render: (text) => text.charAt(0).toUpperCase() + text.slice(1),
+      onFilter: () => true,
+      renderFilterDropdown: () => {
+        return (
+          <Space vertical style={{ padding: 8 }}>
+            <Select showClear placeholder="Select Provider to Filter" onChange={(val) => setProviderFilter(val)}>
+              {provider != null &&
+                provider.length > 0 &&
+                provider.map((prov) => {
+                  return (
+                    <Select.Option value={prov.id} key={prov.id}>
+                      {prov.name}
+                    </Select.Option>
+                  );
+                })}
+            </Select>
+          </Space>
+        );
+      },
+    },
+    {
+      title: 'Price',
+      width: 110,
+      dataIndex: 'price',
+      sorter: true,
+      render: (text) => text + ' €',
+    },
+    {
+      title: 'Address',
+      width: 150,
+      dataIndex: 'address',
+      sorter: true,
+    },
+    {
+      title: 'Title',
+      dataIndex: 'title',
+      sorter: true,
+      ellipsis: true,
+      render: (text, row) => {
+        return (
+          <a href={row.url} target="_blank" rel="noopener noreferrer">
+            {text}
+          </a>
+        );
+      },
+    },
+  ];
+};
 
 const empty = (
   <Empty
     image={<IllustrationNoResult />}
     darkModeImage={<IllustrationNoResultDark />}
-    description="No listings available."
+    description="No listings found."
   />
 );
 
 export default function ListingsTable() {
   const tableData = useSelector((state) => state.listingsTable);
+  const provider = useSelector((state) => state.provider);
+  const jobs = useSelector((state) => state.jobs.jobs);
+  const navigate = useNavigate();
+
+  const watchlistFeature = useFeature('WATCHLIST_MANAGEMENT') || false;
   const actions = useActions();
   const [page, setPage] = useState(1);
   const pageSize = 10;
@@ -179,11 +250,14 @@ export default function ListingsTable() {
   const [jobNameFilter, setJobNameFilter] = useState(null);
   const [activityFilter, setActivityFilter] = useState(null);
   const [providerFilter, setProviderFilter] = useState(null);
+  const [allFilters, setAllFilters] = useState([]);
 
+  const [imageWidth, setImageWidth] = useState('100%');
   const handlePageChange = (_page) => {
     setPage(_page);
   };
 
+  const columns = getColumns(provider, setProviderFilter, jobs, setJobNameFilter);
   const loadTable = () => {
     let sortfield = null;
     let sortdir = null;
@@ -208,14 +282,43 @@ export default function ListingsTable() {
 
   const handleFilterChange = useMemo(() => debounce((value) => setFreeTextFilter(value), 500), []);
 
+  const diffArrays = (primary, secondary) => {
+    const result = {};
+
+    for (const item of secondary) {
+      if (!primary.includes(item)) result[item] = true;
+    }
+
+    for (const item of primary) {
+      if (!secondary.includes(item)) result[item] = false;
+    }
+
+    return [result];
+  };
+
+  useEffect(() => {
+    return () => {
+      // cleanup debounced handler to avoid memory leaks
+      handleFilterChange.cancel && handleFilterChange.cancel();
+    };
+  }, [handleFilterChange]);
+
   const expandRowRender = (record) => {
     return (
       <div className="listingsTable__expanded">
         <div>
           {record.image_url == null ? (
-            <Image height={200} src={no_image} />
+            <Image height={200} width={180} src={no_image} />
           ) : (
-            <Image height={200} src={record.image_url} />
+            <Image
+              height={200}
+              width={imageWidth}
+              src={record.image_url}
+              onError={() => {
+                setImageWidth('180px');
+              }}
+              fallback={<Image height={200} src={no_image} />}
+            />
           )}
         </div>
         <div>
@@ -226,7 +329,7 @@ export default function ListingsTable() {
               </Tag>
             </Descriptions.Item>
             <Descriptions.Item itemKey="Link">
-              <a href={record.link} target="_blank" rel="noreferrer">
+              <a href={record.link} target="_blank" rel="noopener noreferrer">
                 Link to Listing
               </a>
             </Descriptions.Item>
@@ -242,12 +345,6 @@ export default function ListingsTable() {
 
   return (
     <div>
-      <ListingsFilter
-        onActivityFilter={setActivityFilter}
-        onWatchListFilter={setWatchListFilter}
-        onJobNameFilter={setJobNameFilter}
-        onProviderFilter={setProviderFilter}
-      />
       <Input
         prefix={<IconSearch />}
         showClear
@@ -255,6 +352,16 @@ export default function ListingsTable() {
         placeholder="Search"
         onChange={handleFilterChange}
       />
+      {watchlistFeature && (
+        <Button
+          className="listingsTable__setupButton"
+          onClick={() => {
+            navigate('/watchlistManagement');
+          }}
+        >
+          Setup notifications on watchlist changes
+        </Button>
+      )}
       <Table
         rowKey="id"
         empty={empty}
@@ -269,7 +376,23 @@ export default function ListingsTable() {
           };
         })}
         onChange={(changeSet) => {
-          if (changeSet?.extra?.changeType === 'sorter') {
+          if (changeSet?.extra?.changeType === 'filter') {
+            const transformed = changeSet.filters.map((f) => f.dataIndex);
+            const diff = diffArrays(allFilters, transformed);
+            setAllFilters(transformed);
+            diff.forEach((filter) => {
+              switch (Object.keys(filter)[0]) {
+                case 'isWatched':
+                  setWatchListFilter(Object.values(filter)[0]);
+                  break;
+                case 'is_active':
+                  setActivityFilter(Object.values(filter)[0]);
+                  break;
+                default:
+                  console.error('Unknown filter: ', filter.dataIndex);
+              }
+            });
+          } else if (changeSet?.extra?.changeType === 'sorter') {
             setSortData({
               field: changeSet.sorter.dataIndex,
               direction: changeSet.sorter.sortOrder === 'ascend' ? 'asc' : 'desc',

ui/src/components/table/listings/ListingsTable.less

@@ -11,4 +11,8 @@
   &__toolbar {
     margin-bottom: 1rem;
   }
+
+  &__setupButton {
+    margin-bottom: 1rem;
+  }
 }

ui/src/hooks/featureHook.js

@@ -0,0 +1,15 @@
+import { useSelector } from '../services/state/store.js';
+
+export function useFeature(name) {
+  const currentFeatureFlags = useSelector((state) => state.features);
+  if (Object.keys(currentFeatureFlags || {}).length === 0) {
+    return null;
+  }
+
+  if (currentFeatureFlags[name] == null) {
+    console.warn(`Feature flag with name ${name} is unknown.`);
+    return null;
+  }
+
+  return currentFeatureFlags[name];
+}

ui/src/services/state/store.js

@@ -48,6 +48,16 @@ export const useFredyState = create(
             }
           },
         },
+        features: {
+          async getFeatures() {
+            try {
+              const response = await xhrGet('/api/features');
+              set((state) => ({ ...state.features, ...response.json }));
+            } catch (Exception) {
+              console.error('Error while trying to get resource for api/features. Error:', Exception);
+            }
+          },
+        },
         provider: {
           async getProvider() {
             try {
@@ -176,6 +186,7 @@ export const useFredyState = create(
           page: 1,
           result: [],
         },
+        features: {},
         generalSettings: { settings: {} },
         demoMode: { demoMode: false },
         versionUpdate: {},
@@ -192,6 +203,7 @@ export const useFredyState = create(
         versionUpdate: { ...effects.versionUpdate },
         listingsTable: { ...effects.listingsTable },
         provider: { ...effects.provider },
+        features: { ...effects.features },
         jobs: { ...effects.jobs },
         user: { ...effects.user },
       };

ui/src/views/jobs/ProcessingTimes.jsx

@@ -1,16 +1,32 @@
 import React from 'react';
 import { format } from '../../services/time/timeService';
 import { Button, Card, Col, Row, Toast } from '@douyinfe/semi-ui';
-import { IconPlayCircle } from '@douyinfe/semi-icons';
+import {
+  IconClock,
+  IconDoubleChevronLeft,
+  IconDoubleChevronRight,
+  IconPlayCircle,
+  IconSearch,
+} from '@douyinfe/semi-icons';
 import { xhrPost } from '../../services/xhr.js';
 
 import './ProsessingTimes.less';
+import { useScreenWidth } from '../../hooks/screenWidth.js';
 
-function InfoCard({ title, value }) {
+function InfoCard({ title, value, icon }) {
+  const { Meta } = Card;
   return (
-    <Card style={{ maxWidth: '13rem', margin: '1rem', background: 'rgb(53, 54, 60)' }} title={title}>
+    <div
-      {value}
+      style={{
-    </Card>
+        margin: '1rem',
+        background: 'rgb(53, 54, 60)',
+        borderRadius: '.3rem',
+        padding: '1rem',
+        minHeight: '3rem',
+      }}
+    >
+      <Meta title={title} description={value} avatar={icon} />
+    </div>
   );
 }
 
@@ -18,32 +34,57 @@ export default function ProcessingTimes({ processingTimes = {} }) {
   if (Object.keys(processingTimes).length === 0) {
     return null;
   }
+  const width = useScreenWidth();
+  const invisible = width <= 1180;
+
+  if (invisible) {
+    return null;
+  }
+
   return (
     <Row>
       <Col span={6}>
-        <InfoCard title="Processing Interval" value={`${processingTimes.interval} min`} />
+        <InfoCard
+          title="Search Interval"
+          value={`${processingTimes.interval} min`}
+          icon={<IconClock style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
+        />
       </Col>
       {processingTimes.lastRun && (
         <>
           <Col span={6}>
-            <InfoCard title="Last run" value={format(processingTimes.lastRun)} />
+            <InfoCard
+              title="Last search"
+              icon={<IconDoubleChevronLeft style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
+              value={format(processingTimes.lastRun)}
+            />
           </Col>
           <Col span={6}>
-            <InfoCard title="Next run" value={format(processingTimes.lastRun + processingTimes.interval * 60000)} />
+            <InfoCard
+              title="Next search"
+              icon={<IconDoubleChevronRight style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
+              value={format(processingTimes.lastRun + processingTimes.interval * 60000)}
+            />
           </Col>
         </>
       )}
       <Col span={6}>
         <InfoCard
-          title="Find Listings Now"
+          title="Search Now"
+          icon={<IconSearch style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
           value={
             <Button
               size="small"
+              style={{ marginTop: '.2rem' }}
               icon={<IconPlayCircle />}
               aria-label="Start now"
               onClick={async () => {
-                await xhrPost('/api/jobs/startAll', null);
+                try {
-                Toast.success('Successfully triggered Fredy search.');
+                  await xhrPost('/api/jobs/startAll', null);
+                  Toast.success('Successfully triggered Fredy search.');
+                } catch {
+                  Toast.error('Failed to trigger search');
+                }
               }}
             >
               Search now

ui/src/views/jobs/mutation/JobMutation.jsx

@@ -11,7 +11,15 @@ import { useNavigate, useParams } from 'react-router-dom';
 import { Divider, Input, Switch, Button, TagInput, Toast, Select } from '@douyinfe/semi-ui';
 import './JobMutation.less';
 import { SegmentPart } from '../../../components/segment/SegmentPart';
-import { IconBell, IconBriefcase, IconPaperclip, IconPlayCircle, IconPlusCircle, IconUser } from '@douyinfe/semi-icons';
+import {
+  IconBell,
+  IconBriefcase,
+  IconPaperclip,
+  IconPlayCircle,
+  IconPlusCircle,
+  IconUser,
+  IconClear,
+} from '@douyinfe/semi-icons';
 
 export default function JobMutator() {
   const jobs = useSelector((state) => state.jobs.jobs);
@@ -26,6 +34,7 @@ export default function JobMutator() {
   const defaultNotificationAdapter = jobToBeEdit?.notificationAdapter || [];
   const defaultEnabled = jobToBeEdit?.enabled ?? true;
 
+  const [providerToEdit, setProviderToEdit] = useState(null);
   const [providerCreationVisible, setProviderCreationVisibility] = useState(false);
   const [notificationCreationVisible, setNotificationCreationVisibility] = useState(false);
   const [editNotificationAdapter, setEditNotificationAdapter] = useState(null);
@@ -42,6 +51,12 @@ export default function JobMutator() {
     return Boolean(notificationAdapterData.length && providerData.length && name);
   };
 
+  const handleProviderEdit = (data) => {
+    setProviderData(
+      providerData.map((provider) => (provider.url === data.oldProviderToEdit.url ? data.newData : provider)),
+    );
+  };
+
   const mutateJob = async () => {
     try {
       await xhrPost('/api/jobs', {
@@ -70,6 +85,8 @@ export default function JobMutator() {
         onData={(data) => {
           setProviderData([...providerData, data]);
         }}
+        onEditData={handleProviderEdit}
+        providerToEdit={providerToEdit}
       />
 
       {notificationCreationVisible && (
@@ -119,7 +136,10 @@ export default function JobMutator() {
             type="primary"
             icon={<IconPlusCircle />}
             className="jobMutation__newButton"
-            onClick={() => setProviderCreationVisibility(true)}
+            onClick={() => {
+              setProviderToEdit(null);
+              setProviderCreationVisibility(true);
+            }}
           >
             Add new Provider
           </Button>
@@ -129,6 +149,10 @@ export default function JobMutator() {
             onRemove={(providerUrl) => {
               setProviderData(providerData.filter((provider) => provider.url !== providerUrl));
             }}
+            onEdit={(provider) => {
+              setProviderCreationVisibility(true);
+              setProviderToEdit(provider);
+            }}
           />
         </SegmentPart>
         <Divider margin="1rem" />
@@ -160,7 +184,7 @@ export default function JobMutator() {
         </SegmentPart>
         <Divider margin="1rem" />
         <SegmentPart
-          Icon={IconBell}
+          Icon={IconClear}
           name="Blacklist"
           helpText="If a listing contains one of these words, it will be filtered out. Type in a word, then hit enter."
         >

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx

@@ -7,6 +7,7 @@ import { useSelector } from '../../../../../services/state/store';
 import { Banner, Button, Form, Modal, Select, Switch } from '@douyinfe/semi-ui';
 
 import './NotificationAdapterMutator.less';
+import { useScreenWidth } from '../../../../../hooks/screenWidth.js';
 
 const sortAdapter = (a, b) => {
   if (a.name < b.name) {
@@ -21,7 +22,7 @@ const sortAdapter = (a, b) => {
 const validate = (selectedAdapter) => {
   const results = [];
   for (let uiElement of Object.values(selectedAdapter.fields || [])) {
-    if (uiElement.value == null) {
+    if (uiElement.value == null && !uiElement.optional) {
       results.push('All fields are mandatory and must be set.');
       continue;
     }
@@ -36,7 +37,7 @@ const validate = (selectedAdapter) => {
       results.push('A boolean field cannot be of a different type.');
       continue;
     }
-    if (typeof uiElement.value === 'string' && uiElement.value.length === 0) {
+    if (typeof uiElement.value === 'string' && uiElement.value.length === 0 && !uiElement.optional) {
       results.push('All fields are mandatory and must be set.');
     }
   }
@@ -53,6 +54,8 @@ function spreadPrefilledAdapterWithValues(prefilled, fields) {
 }
 
 export default function NotificationAdapterMutator({
+  title,
+  description,
   onVisibilityChanged,
   visible = false,
   selected = [],
@@ -70,6 +73,9 @@ export default function NotificationAdapterMutator({
   const [validationMessage, setValidationMessage] = useState(null);
   const [successMessage, setSuccessMessage] = useState(null);
 
+  const width = useScreenWidth();
+  const isMobile = width <= 850;
+
   const onSubmit = (doStore) => {
     if (doStore) {
       const validationResults = validate(selectedAdapter);
@@ -168,20 +174,21 @@ export default function NotificationAdapterMutator({
 
   return (
     <Modal
-      title="Adding a new Notification Adapter"
+      title={title != null ? title : 'Adding a new Notification Adapter'}
       visible={visible}
-      style={{ width: '95%' }}
+      style={{ width: isMobile ? '95%' : '50rem' }}
+      onCancel={() => onSubmit(false)}
       footer={
         <div>
-          <Button type="secondary" disabled={selectedAdapter == null} style={{ float: 'left' }} onClick={() => onTry()}>
+          <Button type="secondary" disabled={selectedAdapter == null} style={{ float: 'left' }} onClick={onTry}>
             Try
           </Button>
-          <Button type="danger" onClick={() => onSubmit(true)}>
+          <Button theme="light" type="tertiary" onClick={() => onSubmit(false)}>
-            Save
-          </Button>
-          <Button type="primary" onClick={() => onSubmit(false)}>
             Cancel
           </Button>
+          <Button theme="solid" type="primary" onClick={() => onSubmit(true)}>
+            Save
+          </Button>
         </div>
       }
     >
@@ -206,11 +213,15 @@ export default function NotificationAdapterMutator({
         />
       )}
 
-      <p>
+      {description != null ? (
-        When Fredy found new listings, we like to report them to you. To do so, notification adapter can be configured.{' '}
+        <p>{description}</p>
-        <br />
+      ) : (
-        There are multiple ways how Fredy can send new listings to you. Chose your weapon...
+        <p>
-      </p>
+          When Fredy finds new listings, we like to report them to you. To do so, notification adapter can be
+          configured. <br />
+          There are multiple ways how Fredy can send new listings to you. Chose your weapon...
+        </p>
+      )}
 
       <Select
         filter

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationHelpDisplay.jsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Banner } from '@douyinfe/semi-ui';
+import { Banner, MarkdownRender } from '@douyinfe/semi-ui';
 
 export default function Help({ readme }) {
   return (
@@ -8,7 +8,7 @@ export default function Help({ readme }) {
       type="info"
       closeIcon={null}
       title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Information</div>}
-      description={<p dangerouslySetInnerHTML={{ __html: readme }} />}
+      description={<MarkdownRender raw={readme} />}
     />
   );
 }

ui/src/views/jobs/mutation/components/provider/ProviderMutator.jsx

@@ -1,10 +1,11 @@
-import React, { useState } from 'react';
+import React, { useState, useEffect } from 'react';
 
 import { Banner, Modal, Select, Input } from '@douyinfe/semi-ui';
 import { transform } from '../../../../../services/transformer/providerTransformer';
 import { useSelector } from '../../../../../services/state/store';
 import { IconLikeHeart } from '@douyinfe/semi-icons';
 import './ProviderMutator.less';
+import { useScreenWidth } from '../../../../../hooks/screenWidth.js';
 
 const sortProvider = (a, b) => {
   if (a.key < b.key) {
@@ -16,11 +17,35 @@ const sortProvider = (a, b) => {
   return 0;
 };
 
-export default function ProviderMutator({ onVisibilityChanged, visible = false, onData } = {}) {
+const returnOriginalSelectedProvider = (providerToEdit, provider) => {
+  return provider.find((pro) => pro.id === providerToEdit.id);
+};
+
+export default function ProviderMutator({
+  onVisibilityChanged,
+  visible = false,
+  onData,
+  onEditData,
+  providerToEdit,
+} = {}) {
   const provider = useSelector((state) => state.provider);
   const [selectedProvider, setSelectedProvider] = useState(null);
   const [providerUrl, setProviderUrl] = useState(null);
   const [validationMessage, setValidationMessage] = useState(null);
+
+  useEffect(() => {
+    if (providerToEdit) {
+      setSelectedProvider(returnOriginalSelectedProvider(providerToEdit, provider));
+      setProviderUrl(providerToEdit.url);
+    } else {
+      setSelectedProvider(null);
+      setProviderUrl(null);
+    }
+  }, [providerToEdit, visible]);
+
+  const width = useScreenWidth();
+  const isMobile = width <= 850;
+
   const validate = () => {
     if (selectedProvider == null || selectedProvider.length === 0 || providerUrl == null || providerUrl.length === 0) {
       return 'Please select a provider and copy the browser url into the textfield after configuring your search parameter.';
@@ -41,13 +66,24 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
     if (doStore) {
       const validationResult = validate();
       if (validationResult == null) {
-        onData(
+        if (providerToEdit != null) {
-          transform({
+          onEditData({
-            url: providerUrl,
+            newData: transform({
-            id: selectedProvider.id,
+              url: providerUrl,
-            name: selectedProvider.name,
+              id: selectedProvider.id,
-          }),
+              name: selectedProvider.name,
-        );
+            }),
+            oldProviderToEdit: providerToEdit,
+          });
+        } else {
+          onData(
+            transform({
+              url: providerUrl,
+              id: selectedProvider.id,
+              name: selectedProvider.name,
+            }),
+          );
+        }
         setProviderUrl(null);
         setSelectedProvider(null);
         onVisibilityChanged(false);
@@ -63,11 +99,11 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
 
   return (
     <Modal
-      title="Adding a new Provider"
+      title={providerToEdit ? 'Editing an existing Provider' : 'Adding a new Provider'}
       visible={visible}
       onOk={() => onSubmit(true)}
       onCancel={() => onSubmit(false)}
-      style={{ width: '50rem' }}
+      style={{ width: isMobile ? '95%' : '50rem' }}
       okText="Save"
     >
       {validationMessage != null && (
@@ -80,19 +116,26 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
           description={validationMessage}
         />
       )}
-
+      {providerToEdit != null ? (
-      <p>
+        <p>
-        Provider are the <IconLikeHeart style={{ color: '#ff0000' }} /> of Fredy. We're supporting multiple Provider
+          You can now edit the <strong>{providerToEdit.name}</strong> provider's URL in the input field below.
-        such as Immowelt, Kalaydo etc. Select a provider from the list below.
+        </p>
-        <br />
+      ) : (
-        Fredy will then open the provider's url in a new tab.
+        <>
-      </p>
+          <p>
-      <p>
+            Provider are the <IconLikeHeart style={{ color: '#ff0000' }} /> of Fredy. We're supporting multiple Provider
-        You will need to configure your search parameter like you would do when you do a regular search on the
+            such as Immowelt, Kalaydo etc. Select a provider from the list below.
-        provider's website.
+            <br />
-        <br />
+            Fredy will then open the provider's url in a new tab.
-        When the search results are shown on the website, copy the url and paste it into the textfield below.
+          </p>
-      </p>
+          <p>
+            You will need to configure your search parameter like you would do when you do a regular search on the
+            provider's website.
+            <br />
+            When the search results are shown on the website, copy the url and paste it into the textfield below.
+          </p>
+        </>
+      )}
       <Banner
         fullMode={false}
         type="warning"
@@ -112,6 +155,7 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         filter
         placeholder="Select a provider"
         className="providerMutator__fields"
+        disabled={providerToEdit != null}
         optionList={provider
           .map((pro) => {
             return {
@@ -126,7 +170,6 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         onChange={(value) => {
           const selectedProvider = provider.find((pro) => pro.id === value);
           setSelectedProvider(selectedProvider);
-
           window.open(selectedProvider.baseUrl);
         }}
       />
@@ -137,7 +180,8 @@ export default function ProviderMutator({ onVisibilityChanged, visible = false,
         placeholder="Provider Url"
         width={10}
         className="providerMutator__fields"
-        onBlur={(e) => {
+        value={providerUrl}
+        onInput={(e) => {
           setProviderUrl(e.target.value);
         }}
       />

ui/src/views/listings/Listings.jsx

@@ -3,9 +3,5 @@ import React from 'react';
 import ListingsTable from '../../components/table/listings/ListingsTable.jsx';
 
 export default function Listings() {
-  return (
+  return <ListingsTable />;
-    <div>
-      <ListingsTable />
-    </div>
-  );
 }

ui/src/views/listings/management/WatchlistManagement.jsx

@@ -0,0 +1,59 @@
+import React, { useState } from 'react';
+import { IconHorn } from '@douyinfe/semi-icons';
+import { SegmentPart } from '../../../components/segment/SegmentPart.jsx';
+import { Banner, Button, Checkbox, Space } from '@douyinfe/semi-ui';
+import NotificationAdapterMutator from '../../jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx';
+import Headline from '../../../components/headline/Headline.jsx';
+
+export default function WatchlistManagement() {
+  const [notificationChooserVisible, setNotificationChooserVisible] = useState(false);
+  const [notificationAdapterData, setNotificationAdapterData] = useState([]);
+  //TODO: Set default
+  const [activityChanges, setActivityChanges] = useState(false);
+  const [priceChanges, setPriceChanges] = useState(false);
+  return (
+    <div>
+      <SegmentPart
+        name="Notification for Watch List"
+        helpText="You can get notified for changes on listings from your watch list."
+        Icon={IconHorn}
+      >
+        <Banner
+          fullMode={false}
+          type="info"
+          closeIcon={null}
+          title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Note</div>}
+          description="You’ll receive notifications only for listings that are on your watch list. To add listings to it, open the 'Listings' section and tag the ones you want to follow."
+        />
+        <Space />
+        <Headline size={5} text="Notify me when:" style={{ marginTop: '1rem' }} />
+
+        <Checkbox checked={activityChanges} onChange={(e) => setActivityChanges(e.target.checked)}>
+          Listing state changes (e.g. listing becomes inactive)
+        </Checkbox>
+        <Checkbox checked={priceChanges} onChange={(e) => setPriceChanges(e.target.checked)}>
+          Listing price changes
+        </Checkbox>
+
+        <Space />
+        <Headline size={5} text="Notify me with:" style={{ marginTop: '1rem' }} />
+        <Button onClick={() => setNotificationChooserVisible(true)}>Select notification method</Button>
+
+        <NotificationAdapterMutator
+          title="Add notification method"
+          description="When something has changed, Fredy will notify you using the selected notification adapter. Note, some adapter like SqLite are not available here."
+          visible={notificationChooserVisible}
+          onVisibilityChanged={(visible) => {
+            setNotificationChooserVisible(visible);
+          }}
+          selected={notificationAdapterData}
+          editNotificationAdapter={null}
+          onData={(data) => {
+            const oldData = [...notificationAdapterData].filter((o) => o.id !== data.id);
+            setNotificationAdapterData([...oldData, data]);
+          }}
+        />
+      </SegmentPart>
+    </div>
+  );
+}