improve markdown readme's & and adding ability to send telegram messages to a topic in a supergroup

fixing test runner
improving similarity cache
2026-06-16 12:31:07 +00:00 · 2025-10-30 12:42:03 +01:00 · 2025-10-29 10:35:07 +01:00 · 2025-10-29 09:36:05 +01:00 · 2025-10-12 16:47:46 +02:00 · 2025-10-12 16:47:28 +02:00
43 changed files with 583 additions and 482 deletions
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@
 <p align="center">
  <a href="https://fredy.orange-coding.net/" target="_blank">Website</a>&nbsp;&nbsp;|&nbsp;&nbsp;
-  <a href="https://demo-fredy.orange-coding.net/" target="_blank">Demo</a>
+  <a href="https://fredy-demo.orange-coding.net/" target="_blank">Demo</a>
 </p>
 <p align="center">
@@ -202,7 +202,7 @@ flowchart TD
        F2["Adapter 2"]
  end
-    A1 --> B["FredyRuntime"]
+    A1 --> B["FredyPipeline"]
    A2 --> B
    A3 --> B
    B --> C1 & C2 & C3
--- a/index.html
+++ b/index.html
@@ -7,11 +7,14 @@
      content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"
    />
    <meta name="google" content="notranslate" />
      <meta name="apple-mobile-web-app-capable" content="yes" />
      <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
-    <title>Fredy</title>
+      <title>Fredy || Real Estate Finder</title>
  </head>
  <body theme-mode="dark">
    <div id="fredy" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0"></div>
  </body>
  <script type="module" src="/ui/src/Index.jsx"></script>
 </html>
--- a/index.js
+++ b/index.js
@@ -3,7 +3,7 @@ import path from 'path';
 import { checkIfConfigIsAccessible, config, getProviders, refreshConfig } from './lib/utils.js';
 import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
 import * as jobStorage from './lib/services/storage/jobStorage.js';
-import FredyRuntime from './lib/FredyRuntime.js';
+import FredyPipeline from './lib/FredyPipeline.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import { runMigrations } from './lib/services/storage/migrations/migrate.js';
 import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
@@ -37,6 +37,9 @@ await runMigrations();
 // Load provider modules once at startup
 const providers = await getProviders();
 similarityCache.initSimilarityCache();
 similarityCache.startSimilarityCacheReloader();
 //assuming interval is always in minutes
 const INTERVAL = config.interval * 60 * 1000;
@@ -75,7 +78,7 @@ const execute = () => {
            .forEach(async (prov) => {
              const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
              matchedProvider.init(prov, job.blacklist);
-              await new FredyRuntime(
+              await new FredyPipeline(
                matchedProvider.config,
                job.notificationAdapter,
                prov.id,
--- a/lib/FredyPipeline.js
+++ b/lib/FredyPipeline.js
@@ -0,0 +1,216 @@
 import { NoNewListingsWarning } from './errors.js';
 import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
 import * as notify from './notification/notify.js';
 import Extractor from './services/extractor/extractor.js';
 import urlModifier from './services/queryStringMutator.js';
 import logger from './services/logger.js';
 /**
 * @typedef {Object} Listing
 * @property {string} id Stable unique identifier (hash) of the listing.
 * @property {string} title Title or headline of the listing.
 * @property {string} [address] Optional address/location text.
 * @property {string} [price] Optional price text/value.
 * @property {string} [url] Link to the listing detail page.
 * @property {any} [meta] Provider-specific additional metadata.
 */
 /**
 * @typedef {Object} SimilarityCache
 * @property {(title:string, address?:string)=>boolean} hasSimilarEntries Returns true if a similar entry is known.
 * @property {(title:string, address?:string)=>void} addCacheEntry Adds a new entry to the similarity cache.
 */
 /**
 * Runtime orchestrator for fetching, normalizing, filtering, deduplicating, storing,
 * and notifying about new listings from a configured provider.
 *
 * The execution flow is:
 * 1) Prepare provider URL (sorting, etc.)
 * 2) Extract raw listings from the provider
 * 3) Normalize listings to the provider schema
 * 4) Filter out incomplete/blacklisted listings
 * 5) Identify new listings (vs. previously stored hashes)
 * 6) Persist new listings
 * 7) Filter out entries similar to already seen ones
 * 8) Dispatch notifications
 */
 class FredyPipeline {
  /**
   * Create a new runtime instance for a single provider/job execution.
   *
   * @param {Object} providerConfig Provider configuration.
   * @param {string} providerConfig.url Base URL to crawl.
   * @param {string} [providerConfig.sortByDateParam] Query parameter used to enforce sorting by date (provider-specific).
   * @param {string} [providerConfig.waitForSelector] CSS selector to wait for before parsing content.
   * @param {Object.<string, string>} providerConfig.crawlFields Mapping of field names to selectors/paths to extract.
   * @param {string} providerConfig.crawlContainer CSS selector for the container holding listing items.
   * @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape.
   * @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings.
   * @param {(url:string, waitForSelector?:string)=>Promise<void>|Promise<Listing[]>} [providerConfig.getListings] Optional override to fetch listings.
   *
   * @param {Object} notificationConfig Notification configuration passed to notification adapters.
   * @param {string} providerId The ID of the provider currently in use.
   * @param {string} jobKey Key of the job that is currently running (from within the config).
   * @param {SimilarityCache} similarityCache Cache instance for checking similar entries.
   */
  constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
    this._providerConfig = providerConfig;
    this._notificationConfig = notificationConfig;
    this._providerId = providerId;
    this._jobKey = jobKey;
    this._similarityCache = similarityCache;
  }
  /**
   * Execute the end-to-end pipeline for a single provider run.
   *
   * @returns {Promise<Listing[]|void>} Resolves to the list of new (and similarity-filtered) listings
   * after notifications have been sent; resolves to void when there are no new listings.
   */
  execute() {
    return Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
      .then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
      .then(this._normalize.bind(this))
      .then(this._filter.bind(this))
      .then(this._findNew.bind(this))
      .then(this._save.bind(this))
      .then(this._filterBySimilarListings.bind(this))
      .then(this._notify.bind(this))
      .catch(this._handleError.bind(this));
  }
  /**
   * Fetch listings from the provider, using the default Extractor flow unless
   * a provider-specific getListings override is supplied.
   *
   * @param {string} url The provider URL to fetch from.
   * @returns {Promise<Listing[]>} Resolves with an array of listings (empty when none found).
   */
  _getListings(url) {
    const extractor = new Extractor();
    return new Promise((resolve, reject) => {
      extractor
        .execute(url, this._providerConfig.waitForSelector)
        .then(() => {
          const listings = extractor.parseResponseText(
            this._providerConfig.crawlContainer,
            this._providerConfig.crawlFields,
            url,
          );
          resolve(listings == null ? [] : listings);
        })
        .catch((err) => {
          reject(err);
          logger.error(err);
        });
    });
  }
  /**
   * Normalize raw listings into the provider-specific Listing shape.
   *
   * @param {any[]} listings Raw listing entries from the extractor or override.
   * @returns {Listing[]} Normalized listings.
   */
  _normalize(listings) {
    return listings.map(this._providerConfig.normalize);
  }
  /**
   * Filter out listings that are missing required fields and those rejected by the
   * provider's blacklist/filter function.
   *
   * @param {Listing[]} listings Listings to filter.
   * @returns {Listing[]} Filtered listings that pass validation and provider filter.
   */
  _filter(listings) {
    const keys = Object.keys(this._providerConfig.crawlFields);
    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
    return filteredListings.filter(this._providerConfig.filter);
  }
  /**
   * Determine which listings are new by comparing their IDs against stored hashes.
   *
   * @param {Listing[]} listings Listings to evaluate for novelty.
   * @returns {Listing[]} New listings not seen before.
   * @throws {NoNewListingsWarning} When no new listings are found.
   */
  _findNew(listings) {
    logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
    const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
    const newListings = listings.filter((o) => !hashes.includes(o.id));
    if (newListings.length === 0) {
      throw new NoNewListingsWarning();
    }
    return newListings;
  }
  /**
   * Send notifications for new listings using the configured notification adapter(s).
   *
   * @param {Listing[]} newListings New listings to notify about.
   * @returns {Promise<Listing[]>} Resolves to the provided listings after notifications complete.
   * @throws {NoNewListingsWarning} When there are no listings to notify about.
   */
  _notify(newListings) {
    if (newListings.length === 0) {
      throw new NoNewListingsWarning();
    }
    const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
    return Promise.all(sendNotifications).then(() => newListings);
  }
  /**
   * Persist new listings and pass them through.
   *
   * @param {Listing[]} newListings Listings to store.
   * @returns {Listing[]} The same listings, unchanged.
   */
  _save(newListings) {
    logger.debug(`Storing ${newListings.length} new listings (Provider: '${this._providerId}')`);
    storeListings(this._jobKey, this._providerId, newListings);
    return newListings;
  }
  /**
   * Remove listings that are similar to already known entries according to the similarity cache.
   * Adds the remaining listings to the cache.
   *
   * @param {Listing[]} listings Listings to filter by similarity.
   * @returns {Listing[]} Listings considered unique enough to keep.
   */
  _filterBySimilarListings(listings) {
    return listings.filter((listing) => {
      const similar = this._similarityCache.checkAndAddEntry({
        title: listing.title,
        address: listing.address,
        price: listing.price,
      });
      if (similar) {
        logger.debug(
          `Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
        );
      }
      return !similar;
    });
  }
  /**
   * Handle errors occurring in the pipeline, logging levels depending on type.
   *
   * @param {Error} err Error instance thrown by previous steps.
   * @returns {void}
   */
  _handleError(err) {
    if (err.name === 'NoNewListingsWarning') {
      logger.debug(`No new listings found (Provider: '${this._providerId}').`);
    } else {
      logger.error(err);
    }
  }
 }
 export default FredyPipeline;
--- a/lib/FredyRuntime.js
+++ b/lib/FredyRuntime.js
@@ -1,127 +0,0 @@
 import { NoNewListingsWarning } from './errors.js';
 import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
 import * as notify from './notification/notify.js';
 import Extractor from './services/extractor/extractor.js';
 import urlModifier from './services/queryStringMutator.js';
 import logger from './services/logger.js';
 class FredyRuntime {
  /**
   *
   * @param providerConfig the config for the specific provider, we're going to query at the moment
   * @param notificationConfig the config for all notifications
   * @param providerId the id of the provider currently in use
   * @param jobKey key of the job that is currently running (from within the config)
   * @param similarityCache cache instance holding values to check for similarity of entries
   */
  constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
    this._providerConfig = providerConfig;
    this._notificationConfig = notificationConfig;
    this._providerId = providerId;
    this._jobKey = jobKey;
    this._similarityCache = similarityCache;
  }
  execute() {
    return (
      //modify the url to make sure search order is correctly set
      Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
        //scraping the site and try finding new listings
        .then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
        //bring them in a proper form (dictated by the provider)
        .then(this._normalize.bind(this))
        //filter listings with stuff tagged by the blacklist of the provider
        .then(this._filter.bind(this))
        //check if new listings available. if so proceed
        .then(this._findNew.bind(this))
        //store everything in db
        .then(this._save.bind(this))
        //check for similar listings. if found, remove them before notifying
        .then(this._filterBySimilarListings.bind(this))
        //notify the user using the configured notification adapter
        .then(this._notify.bind(this))
        //if an error occurred on the way, handle it here.
        .catch(this._handleError.bind(this))
    );
  }
  _getListings(url) {
    const extractor = new Extractor();
    return new Promise((resolve, reject) => {
      extractor
        .execute(url, this._providerConfig.waitForSelector)
        .then(() => {
          const listings = extractor.parseResponseText(
            this._providerConfig.crawlContainer,
            this._providerConfig.crawlFields,
            url,
          );
          resolve(listings == null ? [] : listings);
        })
        .catch((err) => {
          reject(err);
          logger.error(err);
        });
    });
  }
  _normalize(listings) {
    return listings.map(this._providerConfig.normalize);
  }
  _filter(listings) {
    //only return those where all the fields have been found
    const keys = Object.keys(this._providerConfig.crawlFields);
    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
    return filteredListings.filter(this._providerConfig.filter);
  }
  _findNew(listings) {
    logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
    const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
    const newListings = listings.filter((o) => !hashes.includes(o.id));
    if (newListings.length === 0) {
      throw new NoNewListingsWarning();
    }
    return newListings;
  }
  _notify(newListings) {
    if (newListings.length === 0) {
      throw new NoNewListingsWarning();
    }
    const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
    return Promise.all(sendNotifications).then(() => newListings);
  }
  _save(newListings) {
    logger.debug(`Storing ${newListings.length} new listings (Provider: '${this._providerId}')`);
    storeListings(this._jobKey, this._providerId, newListings);
    return newListings;
  }
  _filterBySimilarListings(listings) {
    const filteredList = listings.filter((listing) => {
      const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
      if (similar) {
        logger.debug(
          `Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
        );
      }
      return !similar;
    });
    filteredList.forEach((filter) => this._similarityCache.addCacheEntry(filter.title, filter.address));
    return filteredList;
  }
  _handleError(err) {
    if (err.name === 'NoNewListingsWarning') {
      logger.debug(`No new listings found (Provider: '${this._providerId}').`);
    } else {
      logger.error(err);
    }
  }
 }
 export default FredyRuntime;
--- a/lib/notification/adapter/apprise.md
+++ b/lib/notification/adapter/apprise.md
@@ -1,3 +1,8 @@
 ### Apprise Adapter
-Refer to the [instructions](https://github.com/caronc/apprise-api#installation) on how to set up an Apprise instance and how to configure your preferred notification service.
+Use [Apprise](https://github.com/caronc/apprise-api#installation) to forward notifications to many different services.
 Quick start:
 - Set up an Apprise API instance (see the installation guide linked above).
 - Configure your preferred notification service(s) within Apprise.
 - In Fredy, point the Apprise adapter to your Apprise API endpoint.
--- a/lib/notification/adapter/console.md
+++ b/lib/notification/adapter/console.md
@@ -1,4 +1,3 @@
 ### Console Adapter
-The console adapter prints everything found by Fredy into the console (not sending any notifications to you). This can be useful when you want to check if your search
+The console adapter prints everything found by Fredy to the console (it does not send notifications). This is useful to verify that your search criteria work as expected before enabling a real notification service.
 criteria meet the expectations.
--- a/lib/notification/adapter/discord_webhook.md
+++ b/lib/notification/adapter/discord_webhook.md
@@ -1,4 +1,8 @@
-### Discord Adapter
+### Discord Webhook Adapter
-To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
+Use a Discord channel webhook to receive notifications.
-Once you have created a webhook, copy and paste the webhook URL.
+
 Quick start:
 - Create a webhook in your target Discord channel. See the "Intro to Webhooks" guide on the Discord support site: https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks
 - Copy the generated webhook URL.
 - In Fredy, configure the Discord adapter with this webhook URL.
--- a/lib/notification/adapter/mailJet.md
+++ b/lib/notification/adapter/mailJet.md
@@ -1,8 +1,8 @@
-### MailJet Adapter
+### Mailjet Adapter
-To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
+To use [Mailjet](https://mailjet.com), create an account and decide which email address Fredy should send from.
-E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.  
+For example, if you use yourGmailAccount@gmail.com, add and verify this address in Mailjet.
-The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
+Provide your public/private API keys in Fredy's configuration. Fredy uses the same email template as for SendGrid.
-If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).
+To send to multiple recipients, separate email addresses with commas (e.g., some@email.com, someOther@email.com).
--- a/lib/notification/adapter/mattermost.md
+++ b/lib/notification/adapter/mattermost.md
@@ -1,5 +1,8 @@
 ### Mattermost Adapter
-For Mattermost, you need to create a incoming webhook. This is pretty easy. Please visit the steps in the [developer docs](https://docs.mattermost.com/developer/webhooks-incoming.html) and follow the instructions.
+Receive notifications in Mattermost via an incoming webhook.
-As a result, you get the webhook URL for configuration in fredy. In addition, the target channel must be defined.
+Quick start:
 - Create an incoming webhook following the Mattermost developer docs: https://docs.mattermost.com/developer/webhooks-incoming.html
 - Copy the webhook URL.
 - In Fredy, configure the Mattermost adapter with this URL and the target channel.
--- a/lib/notification/adapter/ntfy.md
+++ b/lib/notification/adapter/ntfy.md
@@ -1,5 +1,8 @@
 ### ntfy Adapter
-For ntfy, you need to create a topic on your preferred ntfy instance. This is pretty easy. Please visit the steps in the [docs](https://docs.ntfy.sh/publish/) and follow the instructions.
+Send push notifications using an ntfy topic.
-As a result, you get the URL for configuration in fredy. In addition, the priority must be defined.
+Quick start:
 - Create or choose a topic on your preferred ntfy instance (see docs: https://docs.ntfy.sh/publish/).
 - Copy the publish URL for that topic.
 - In Fredy, configure the ntfy adapter with the topic URL and set a priority.
--- a/lib/notification/adapter/pushover.md
+++ b/lib/notification/adapter/pushover.md
@@ -1,5 +1,8 @@
 ### Pushover Adapter
-Refer to the [instructions](https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it) to set up your Pushover application.
+Use Pushover to receive push notifications on your devices.
-After setting up the application, please enter both your newly created User key and API token.
+Setup:
 - Follow Pushover's getting-started guide: https://support.pushover.net/i7-what-is-pushover-and-how-do-i-use-it
 - Create an application and obtain your User Key and API Token.
 - In Fredy, configure the Pushover adapter with both values.
--- a/lib/notification/adapter/sendGrid.md
+++ b/lib/notification/adapter/sendGrid.md
@@ -1,9 +1,12 @@
 ### SendGrid Adapter
-SendGrid is a free email service (free as in "you cannot send more than 100(Sendgrid) and 200(Mailjet) emails a day"), which is more than enough for Fredy.
+SendGrid is an email delivery service with a generous free tier, which is more than enough for Fredy.
-To use [SendGrid](https://sendgrid.com/), you need to create an account. You'll need to decided from which email address you want Fredy to send from. E.g. if you use yourGmailAccount@gmail.com, you have to add this to sendgrid and verify it as well.
+Setup:
 - Create a SendGrid account: https://sendgrid.com/
 - Decide which email address Fredy should send from (e.g., yourGmailAccount@gmail.com), add it to SendGrid, and complete the verification.
 - Create an API key and add it to Fredy's configuration.
 - Create a Dynamic Template in SendGrid. You can copy the template from `/lib/notification/emailTemplate/template.hbs`.
-Lastly you have to create an api-key and feed it into Fredy's config, as well as creating a new dynamic template. For this new template, I recommend copying and pasting the code from the one I have provided under `/lib/notification/emailTemplate/template.hbs`.
+Sending to multiple recipients:
-
+- Separate email addresses with commas (e.g., some@email.com, someOther@email.com).
 If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).
--- a/lib/notification/adapter/slack.md
+++ b/lib/notification/adapter/slack.md
@@ -1,4 +1,5 @@
-### Slack Adapter
+### Slack Adapter (Legacy)
-IMPORTANT:
+
-Don't use this adapter anymore, it is outdated and only here for backwards compatability reasons. Use the new Slack Adapter with webhooks!
+*IMPORTANT:*
 This legacy adapter is outdated and kept only for backward compatibility. Please use the Slack adapter with webhooks instead.
--- a/lib/notification/adapter/slack_with_webhooks.md
+++ b/lib/notification/adapter/slack_with_webhooks.md
@@ -1,6 +1,10 @@
-### Slack Adapter
+### Slack Adapter (Webhooks)
-IMPORTANT:
+*IMPORTANT:*
-This is the new version of the Slack adapter. I strongly encourage you to use it, the old version is now unmaintained and only kept due to backwards compatability reasons.
+This is the recommended Slack adapter. The old Slack adapter is unmaintained and kept only for backward compatibility.
-In order to use [Slack](https://slack.com), you need to create an account. When done, create a new channel and add the Webhook integration to that channel. Copy the webhook url. That's it.
+Setup:
 - Create a Slack account and workspace if you don't have one: https://slack.com
 - Create a channel where you want to receive notifications.
 - Add the Incoming Webhooks integration to that channel and copy the Webhook URL.
 - In Fredy, configure the Slack Webhook adapter with this URL.
--- a/lib/notification/adapter/sqlite.md
+++ b/lib/notification/adapter/sqlite.md
@@ -1,9 +1,21 @@
 ### SQLite Adapter
-This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. This file can be used for further analysis later.
+This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. The file can be used for analysis later.
-The database table contains the following columns (all stored as `TEXT` type):
+The table contains the following columns (all stored as `TEXT`):
-```
+```json
-['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description', 'image']
+[
  "serviceName",
  "jobKey",
  "id",
  "size",
  "rooms",
  "price",
  "address",
  "title",
  "link",
  "description",
  "image"
 ]
 ```
--- a/lib/notification/adapter/telegram.js
+++ b/lib/notification/adapter/telegram.js
@@ -117,10 +117,24 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
  if (!adapterCfg || !adapterCfg.fields) {
    throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`);
  }
-  const { token, chatId } = adapterCfg.fields;
+  const { token, chatId, messageThreadId } = adapterCfg.fields;
  if (!token || !chatId) {
    throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
  }
  // Optional Telegram topic/thread support (supergroups)
  let message_thread_id;
  if (messageThreadId !== undefined && messageThreadId !== null && `${messageThreadId}`.trim() !== '') {
    const n = Number(messageThreadId);
    if (Number.isInteger(n) && n > 0) {
      message_thread_id = n;
    } else {
      logger.warn(
        `Telegram adapter: 'messageThreadId' is invalid ('${messageThreadId}'). It must be a positive integer. Ignoring.`,
      );
    }
  }
  const job = getJob(jobKey);
  const jobName = job == null ? jobKey : job.name;
@@ -147,6 +161,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
      text: buildText(jobName, serviceName, o),
      parse_mode: 'HTML',
      disable_web_page_preview: true,
      ...(message_thread_id ? { message_thread_id } : {}),
    };
    if (!img) {
@@ -160,6 +175,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
      photo: img,
      caption: buildCaption(jobName, serviceName, o),
      parse_mode: 'HTML',
      ...(message_thread_id ? { message_thread_id } : {}),
    }).catch(async (e) => {
      logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
      return await throttledCall('sendMessage', textPayload).catch((e) => {
@@ -174,7 +190,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
 /**
 * Telegram notification adapter configuration schema.
- * @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string}}}}
+ * @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string},messageThreadId?:{type:string,label:string,description:string}}}}
 */
 export const config = {
  id: 'telegram',
@@ -192,5 +208,12 @@ export const config = {
      label: 'Chat Id',
      description: 'The chat id to send messages to you.',
    },
    messageThreadId: {
      type: 'text',
      optional: true,
      label: 'Message Thread Id (optional)',
      description:
        'Optional: The topic/thread id within a supergroup to post into (Telegram message_thread_id). Provide a positive integer.',
    },
  },
 };
--- a/lib/notification/adapter/telegram.md
+++ b/lib/notification/adapter/telegram.md
@@ -1,12 +1,55 @@
 ### Telegram Adapter
-For Telegram, you need to create a Bot. This is pretty easy. Open [this](https://telegram.me/BotFather) url on your smartphone and follow the instructions.
+Use this adapter to send notifications to Telegram via a bot. You will need:
 - A Telegram Bot token (from BotFather)
 - A chat ID (where messages will be sent)
 - Optionally: a thread ID if you want to post into a specific forum topic in a group
-A telegram bot is not allowed to send messages directly to a user, you as a user need to first contact the bot to get a chatId.  
+#### Create a bot
-After the user has send a message to your bot the first time, you can gather the chatId like this:
+Create a bot with BotFather: open https://telegram.me/BotFather on your phone or in Telegram Desktop and follow the instructions to get your bot token.
 #### Getting the chat ID
 A Telegram bot cannot message a user first; you must create a conversation (or add the bot to a group/channel) so Telegram assigns a chat the bot can access.
 Steps:
 1. Start a chat with your bot in Telegram (or add the bot to your group/supergroup/channel) and send any message.
 2. Fetch recent updates from the Bot API:
   ```
   curl -X GET "https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates"
   ```
 3. In the JSON response, find the message that you just sent and read `message.chat.id`. That value is your `chatId`.
   - Private chats: `chat.id` is a positive number
   - Groups/supergroups: `chat.id` is a negative number
 Keep your bot token secret. If `getUpdates` returns an empty list, send a new message and try again, or make sure your bot’s privacy settings allow it to see group messages when used in groups.
 #### Getting the thread ID (this is optional to be used for forum topics)
 If you want messages to appear inside a specific forum topic of a supergroup with Topics enabled, you also need a thread ID. In the Telegram Bot API this is called `message_thread_id`.
 When you need it:
 - Required only for supergroups with Topics enabled when targeting a topic
 - Not used for private chats, basic groups without Topics, or channels
 Steps to obtain it:
 1. In your supergroup, enable Topics (Group settings → Manage group → Topics → Enable). Now add a new topic.
 2. Add your created bot to the topic. (Click on the bot and on "Add to group")
 3. Open the desired topic (or create a new one) and send any message inside that topic.
 4. Call `getUpdates` again:
   ```
   curl -X GET "https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates"
   ```
 4. In the update for the message you sent inside the topic, read `message.message_thread_id`. That number is your `threadId` for this topic.
 Example (truncated):
 ```
-curl -X GET https://api.telegram.org/bot{YOUR_TELEGRAM_TOKEN}/getUpdates
+{
  "message": {
    "chat": { "id": -1001234567890, "type": "supergroup" },
    "message_thread_id": 42,
    "text": "hello from the topic"
  }
 }
 ```
 Use `chat.id` as `chatId` and `message_thread_id` as `threadId` in your configuration.
-A more detailed list of instructions can be found here [https://core.telegram.org/bots#botfather](https://core.telegram.org/bots#botfather)
+More details about bots and BotFather: https://core.telegram.org/bots#botfather
--- a/lib/services/extractor/puppeteerExtractor.js
+++ b/lib/services/extractor/puppeteerExtractor.js
@@ -66,7 +66,7 @@ export default async function execute(url, waitForSelector, options) {
      result = pageSource || (await page.content());
    }
  } catch (error) {
-    logger.error('Error executing with puppeteer executor', error);
+    logger.warn('Error executing with puppeteer executor', error);
    result = null;
  } finally {
    try {
--- a/lib/services/markdown.js
+++ b/lib/services/markdown.js
@@ -1,6 +1,4 @@
 import markdown$0 from 'markdown';
 import fs from 'fs';
 const markdown = markdown$0.markdown;
 export function markdown2Html(filePath) {
-  return markdown.toHTML(fs.readFileSync(filePath, 'utf8'));
+  return fs.readFileSync(filePath, 'utf8');
 }
--- a/lib/services/similarity-check/similarityCache.js
+++ b/lib/services/similarity-check/similarityCache.js
@@ -1,116 +1,94 @@
 import crypto from 'crypto';
 const retention = 60 * 60 * 1000;
 /**
- * Internal cache storage.
+ * Similarity cache
 * Maps a SHA-256 hash (string) to its expiry timestamp (number in ms).
 * @type {Map<string, number>}
 */
 const entries = new Map();
 /**
 * Reference to the currently scheduled cleanup timer.
 * @type {NodeJS.Timeout | null}
 */
 let timer = null;
 /**
 * Generate a SHA-256 hash from a list of input strings.
 * Null or undefined values are ignored.
 *
- * @param {...(string|null|undefined)} strings - Input values to hash
+ * Maintains an in-memory Set of content hashes to detect whether a listing
 * (identified by a tuple of title, price and address) has been seen before.
 *
 * Design notes:
 * - The cache is refreshed periodically from persistent storage. To avoid
 *   modification-during-iteration issues, the refresh builds a new Set and
 *   atomically swaps the reference instead of mutating in place.
 * - Hashing ignores null/undefined values but preserves falsy-yet-valid values
 *   like 0. Non-string values are coerced to strings before hashing.
 *
 * This module has no persistence of its own; it relies on
 * getAllEntriesFromListings() for data hydration.
 * @module similarityCache
 */
 import crypto from 'crypto';
 import { getAllEntriesFromListings } from '../storage/listingsStorage.js';
 /** @type {number} Refresh interval in milliseconds (defaults to one hour). */
 const reloadCycle = 60 * 60 * 1000; // every hour, refresh
 /**
 * Internal cache of content hashes for known listings.
 *
 * Each entry is an SHA-256 hex digest produced by toHash(title, price, address).
 * @type {Set<string>}
 */
 let cache = new Set();
 export const startSimilarityCacheReloader = () => {
  // Periodically refresh the cache from storage
  setInterval(() => {
    initSimilarityCache();
  }, reloadCycle);
 };
 /**
 * Initialize or refresh the similarity cache from persistent storage.
 *
 * Reads all stored listings via getAllEntriesFromListings(), computes a hash for
 * each, and swaps the in-memory Set atomically to avoid in-place mutations that
 * could interfere with concurrent iteration.
 *
 * This function is idempotent and safe to call at any time.
 * @returns {void}
 */
 export const initSimilarityCache = () => {
  const allEntries = getAllEntriesFromListings();
  const newCache = new Set();
  for (const entry of allEntries) {
    newCache.add(toHash(entry?.title, entry?.price, entry?.address));
  }
  // Atomic swap to avoid mutating the cache while it may be iterated elsewhere
  cache = newCache;
 };
 /**
 * Check if a listing is already known and add it to the cache if not.
 *
 * The listing is identified by the combination of its title, price and
 * address. Null/undefined fields are ignored during hashing. Falsy-but-valid
 * values (e.g., price 0) are preserved.
 *
 * @param {Object} params - Listing fields
 * @param {string|undefined|null} params.title - The listing title
 * @param {string|undefined|null} params.address - The listing address
 * @param {number|string|undefined|null} params.price - The listing price
 * @returns {boolean} true if the entry already existed in the cache (duplicate), otherwise false
 */
 export const checkAndAddEntry = ({ title, address, price }) => {
  const hash = toHash(title, price, address);
  if (cache.has(hash)) {
    return true;
  }
  cache.add(hash);
  return false;
 };
 /**
 * Generate an SHA-256 hash from a list of input values.
 * Null or undefined values are ignored. Falsy but valid values like 0 are preserved.
 * Non-string values are coerced to strings prior to hashing.
 *
 * @param {...(string|number|null|undefined)} strings - Input values to hash
 * @returns {string} Hexadecimal hash
 */
 function toHash(...strings) {
-  return crypto.createHash('sha256').update(strings.filter(Boolean).join('|')).digest('hex');
+  const normalized = strings
-}
+    .filter((v) => v !== null && v !== undefined)
-
+    .map((v) => (typeof v === 'string' ? v : String(v)));
-/**
+  return crypto.createHash('sha256').update(normalized.join('|')).digest('hex');
 * Cleanup expired cache entries and schedule the next cleanup run.
 * This function is invoked automatically by scheduled timers.
 *
 * @private
 */
 function runCleanup() {
  const now = Date.now();
  for (const [hash, expiry] of entries) {
    if (expiry <= now) entries.delete(hash);
  }
  scheduleNext();
 }
 /**
 * Find the soonest expiry timestamp among all cache entries
 * and schedule a one-shot timer that will trigger at that time.
 * Cancels any existing timer before scheduling a new one.
 *
 * @private
 */
 function scheduleNext() {
  if (timer) {
    clearTimeout(timer);
    timer = null;
  }
  let next = Infinity;
  const now = Date.now();
  for (const expiry of entries.values()) {
    if (expiry > now && expiry < next) next = expiry;
  }
  if (next !== Infinity) {
    timer = setTimeout(runCleanup, Math.max(0, next - now));
  }
 }
 /**
 * Add or refresh a cache entry for the given title and address.
 * The entry will automatically expire after the configured retention window.
 *
 * @param {string} title - The title used to build the cache key
 * @param {string} address - The address used to build the cache key
 */
 export function addCacheEntry(title, address) {
  const hash = toHash(title, address);
  const expiry = Date.now() + retention;
  entries.set(hash, expiry);
  scheduleNext();
 }
 /**
 * Check if a cache entry with the same title and address exists
 * and is still valid (not expired).
 *
 * @param {string} title - The title used to build the cache key
 * @param {string} address - The address used to build the cache key
 * @returns {boolean} True if a valid cache entry exists, false otherwise
 */
 export function hasSimilarEntries(title, address) {
  const hash = toHash(title, address);
  const expiry = entries.get(hash);
  if (expiry == null) return false;
  if (expiry <= Date.now()) {
    entries.delete(hash);
    scheduleNext();
    return false;
  }
  return true;
 }
 /**
 * Stop any scheduled cleanup timers and prevent further automatic cleanup.
 * Entries that are already in the cache will remain until removed manually
 * or until cleanup is started again by adding new entries.
 */
 export function stopCacheCleanup() {
  if (timer) clearTimeout(timer);
  timer = null;
 }
 /**
 * this is only for test purposes
 */
 export function invalidateAllForTest() {
  for (const key of entries.keys()) {
    entries.set(key, 0);
  }
  runCleanup();
 }
--- a/lib/services/storage/listingsStorage.js
+++ b/lib/services/storage/listingsStorage.js
@@ -310,8 +310,8 @@ export const deleteListingsByJobId = (jobId) => {
  if (!jobId) return;
  return SqliteConnection.execute(
    `DELETE
-                                   FROM listings
+     FROM listings
-                                   WHERE job_id = @jobId`,
+     WHERE job_id = @jobId`,
    { jobId },
  );
 };
@@ -332,3 +332,13 @@ export const deleteListingsById = (ids) => {
    ids,
  );
 };
 /**
 * Return all listings with only the fields: title, address, and price.
 * This is the single helper requested for simple consumers.
 *
 * @returns {{title: string|null, address: string|null, price: number|null}[]}
 */
 export const getAllEntriesFromListings = () => {
  return SqliteConnection.query(`SELECT title, address, price FROM listings`);
 };
--- a/lib/services/storage/migrations/sql/5.job-sharing.js
+++ b/lib/services/storage/migrations/sql/5.job-sharing.js
@@ -1,4 +1,4 @@
-// Migration: Adding a new table to store if somebody "watches" (a.k.a favorite) a listing
+// Migration: Adding a new table to store if somebody shared a job with someone
 export function up(db) {
  db.exec(`
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "fredy",
-  "version": "14.2.1",
+  "version": "14.3.1",
  "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
  "scripts": {
    "prepare": "husky",
@@ -69,21 +69,20 @@
    "cookie-session": "2.1.1",
    "handlebars": "4.7.8",
    "lodash": "4.17.21",
    "markdown": "^0.5.0",
    "nanoid": "5.1.6",
    "node-cron": "^4.2.1",
    "node-fetch": "3.3.2",
    "node-mailjet": "6.0.9",
    "p-throttle": "^8.0.0",
    "package-up": "^5.0.0",
-    "puppeteer": "^24.23.0",
+    "puppeteer": "^24.24.0",
    "puppeteer-extra": "^3.3.6",
    "puppeteer-extra-plugin-stealth": "^2.11.2",
    "query-string": "9.3.1",
    "react": "18.3.1",
    "react-dom": "18.3.1",
-    "react-router": "7.9.3",
+    "react-router": "7.9.4",
-    "react-router-dom": "7.9.3",
+    "react-router-dom": "7.9.4",
    "restana": "5.1.0",
    "semver": "^7.7.3",
    "serve-static": "2.2.0",
@@ -105,7 +104,7 @@
    "history": "5.3.0",
    "husky": "9.1.7",
    "less": "4.4.2",
-    "lint-staged": "16.2.3",
+    "lint-staged": "16.2.4",
    "mocha": "11.7.4",
    "nodemon": "^3.1.10",
    "prettier": "3.6.2"
--- a/test/FredyRuntime/FredyRuntime.test.js
+++ b/test/FredyRuntime/FredyRuntime.test.js
@@ -1,53 +0,0 @@
 import { expect } from 'chai';
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 import { mockFredy } from '../utils.js';
 describe('FredyRuntime', () => {
  afterEach(() => {
    similarityCache.invalidateAllForTest();
  });
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  describe('_filterBySimilarListings', () => {
    let fredyRuntime;
    beforeEach(async () => {
      const FredyRuntime = await mockFredy();
      fredyRuntime = new FredyRuntime({}, null, 'dummy-provider', 'dummy-job', similarityCache);
    });
    it('should filter out listings with similar title and address already in cache', () => {
      similarityCache.addCacheEntry('Penthouse', 'Mustermann Straße 1');
      const listings = [
        { id: '1', title: 'Penthouse', address: 'Mustermann Straße 1' },
        { id: '2', title: 'Nice apartment', address: 'Mustermann Straße 15' },
      ];
      const result = fredyRuntime._filterBySimilarListings(listings);
      expect(result).to.have.length(1);
      expect(result[0].id).to.equal('2');
      expect(result[0].title).to.equal('Nice apartment');
      expect(similarityCache.hasSimilarEntries('Nice apartment', 'Mustermann Straße 15')).to.be.true;
    });
    it('should handle listings with null or undefined address', () => {
      const listings = [
        { id: '1', title: 'Penthouse', address: null },
        { id: '2', title: 'Nice apartment', address: undefined },
      ];
      const result = fredyRuntime._filterBySimilarListings(listings);
      expect(result).to.have.length(2);
      expect(similarityCache.hasSimilarEntries('Penthouse', null)).to.be.true;
      expect(similarityCache.hasSimilarEntries('Nice apartment', undefined)).to.be.true;
    });
  });
 });
--- a/test/provider/einsAImmobilien.test.js
+++ b/test/provider/einsAImmobilien.test.js
@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/einsAImmobilien.js';
 describe('#einsAImmobilien testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  provider.init(providerConfig.einsAImmobilien, [], []);
  it('should test einsAImmobilien provider', async () => {
    const Fredy = await mockFredy();
--- a/test/provider/immobilienDe.test.js
+++ b/test/provider/immobilienDe.test.js
@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immobilienDe.js';
 describe('#immobilien.de testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  provider.init(providerConfig.immobilienDe, [], []);
  it('should test immobilien.de provider', async () => {
    const Fredy = await mockFredy();
--- a/test/provider/immonet.test.js
+++ b/test/provider/immonet.test.js
@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immonet.js';
 describe('#immonet testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  it('should test immonet provider', async () => {
    const Fredy = await mockFredy();
    provider.init(providerConfig.immonet, [], []);
--- a/test/provider/immoscout.test.js
+++ b/test/provider/immoscout.test.js
@@ -5,10 +5,6 @@ import { get } from '../mocks/mockNotification.js';
 import * as provider from '../../lib/provider/immoscout.js';
 describe('#immoscout provider testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  provider.init(providerConfig.immoscout, [], []);
  it('should test immoscout provider', async () => {
    const Fredy = await mockFredy();
--- a/test/provider/immoswp.test.js
+++ b/test/provider/immoswp.test.js
@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immoswp.js';
 describe('#immoswp testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  provider.init(providerConfig.immoswp, [], []);
  it('should test immoswp provider', async () => {
    const Fredy = await mockFredy();
--- a/test/provider/immowelt.test.js
+++ b/test/provider/immowelt.test.js
@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/immowelt.js';
 describe('#immowelt testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  it('should test immowelt provider', async () => {
    const Fredy = await mockFredy();
    provider.init(providerConfig.immowelt, [], []);
--- a/test/provider/kleinanzeigen.test.js
+++ b/test/provider/kleinanzeigen.test.js
@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/kleinanzeigen.js';
 describe('#kleinanzeigen testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  it('should test kleinanzeigen provider', async () => {
    const Fredy = await mockFredy();
    provider.init(providerConfig.kleinanzeigen, [], []);
--- a/test/provider/mcMakler.test.js
+++ b/test/provider/mcMakler.test.js
@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/mcMakler.js';
 describe('#mcMakler testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  it('should test mcMakler provider', async () => {
    const Fredy = await mockFredy();
    provider.init(providerConfig.mcMakler, []);
--- a/test/provider/neubauKompass.test.js
+++ b/test/provider/neubauKompass.test.js
@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/neubauKompass.js';
 describe('#neubauKompass testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  provider.init(providerConfig.neubauKompass, [], []);
  it('should test neubauKompass provider', async () => {
    const Fredy = await mockFredy();
--- a/test/provider/regionalimmobilien24.test.js
+++ b/test/provider/regionalimmobilien24.test.js
@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/regionalimmobilien24.js';
 describe('#regionalimmobilien24 testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  it('should test regionalimmobilien24 provider', async () => {
    const Fredy = await mockFredy();
    provider.init(providerConfig.regionalimmobilien24, []);
--- a/test/provider/sparkasse.test.js
+++ b/test/provider/sparkasse.test.js
@@ -5,10 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/sparkasse.js';
 describe('#sparkasse testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  it('should test sparkasse provider', async () => {
    const Fredy = await mockFredy();
    provider.init(providerConfig.sparkasse, []);
--- a/test/provider/wgGesucht.test.js
+++ b/test/provider/wgGesucht.test.js
@@ -5,9 +5,6 @@ import { expect } from 'chai';
 import * as provider from '../../lib/provider/wgGesucht.js';
 describe('#wgGesucht testsuite()', () => {
  after(() => {
    similarityCache.stopCacheCleanup();
  });
  provider.init(providerConfig.wgGesucht, [], []);
  it('should test wgGesucht provider', async () => {
    const Fredy = await mockFredy();
--- a/test/similarity/similarity.test.js
+++ b/test/similarity/similarity.test.js
@@ -1,30 +0,0 @@
 import { expect } from 'chai';
 import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
 describe('similarityCheck', () => {
  it('should return true on duplicate', () => {
    similarityCache.addCacheEntry('Hello World', 'Test');
    expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.true;
  });
  it('should return true even if one value is null', () => {
    similarityCache.addCacheEntry('Hello World', null);
    expect(similarityCache.hasSimilarEntries('Hello World', null)).to.be.true;
  });
  it('should return true even if one value is an obj', () => {
    similarityCache.addCacheEntry('Hello World', [{ TR: 'OLOLO' }]);
    expect(similarityCache.hasSimilarEntries('Hello World', [{ TR: 'OLOLO' }])).to.be.true;
  });
  it('should return false when no duplicate', () => {
    similarityCache.addCacheEntry('Hello World__', 'Test');
    expect(similarityCache.hasSimilarEntries('Hello World___', 'Test')).to.be.false;
  });
  it('should return false when no duplicate', () => {
    expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.true;
    similarityCache.invalidateAllForTest();
    expect(similarityCache.hasSimilarEntries('Hello World', 'Test')).to.be.false;
  });
 });
--- a/test/similarity/similarityCache.test.js
+++ b/test/similarity/similarityCache.test.js
@@ -0,0 +1,62 @@
 import { expect } from 'chai';
 import esmock from 'esmock';
 // Helper to create module under test with mocks
 async function loadModuleWith({ entries = [] } = {}) {
  const mod = await esmock('../../lib/services/similarity-check/similarityCache.js', {
    // Mock the storage to return our controlled entries
    '../../lib/services/storage/listingsStorage.js': {
      getAllEntriesFromListings: () => entries,
    },
  });
  return mod;
 }
 describe('similarityCache', () => {
  it('initSimilarityCache builds cache from storage and enables duplicate detection', async () => {
    const entries = [
      { title: 'A', price: 1000, address: 'Main 1' },
      { title: 'B', price: 0, address: 'Zero St' },
    ];
    const { initSimilarityCache, checkAndAddEntry } = await loadModuleWith({ entries });
    // Initially, duplicates should not be detected for new data
    expect(checkAndAddEntry({ title: 'X', price: 200, address: 'Y' })).to.equal(false);
    // Now initialize from storage
    initSimilarityCache();
    // Exact duplicates should be detected
    expect(checkAndAddEntry({ title: 'A', price: 1000, address: 'Main 1' })).to.equal(true);
    // Ensure falsy-but-valid price 0 is preserved by hashing and detected as duplicate
    expect(checkAndAddEntry({ title: 'B', price: 0, address: 'Zero St' })).to.equal(true);
  });
  it('checkAndAddEntry returns false for new entry then true for duplicate on second call', async () => {
    const { checkAndAddEntry } = await loadModuleWith();
    const first = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' });
    const second = checkAndAddEntry({ title: 'C', price: 300, address: 'Road 3' });
    expect(first).to.equal(false);
    expect(second).to.equal(true);
  });
  it('hashing ignores null/undefined but preserves 0 via behavior', async () => {
    const { checkAndAddEntry } = await loadModuleWith();
    // Add baseline (null address ignored)
    const add1 = checkAndAddEntry({ title: 'T', price: 1, address: null });
    expect(add1).to.equal(false);
    // Duplicate with undefined address should match
    const dup = checkAndAddEntry({ title: 'T', price: 1, address: undefined });
    expect(dup).to.equal(true);
    // Now test that price 0 is preserved (not filtered out)
    const addZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' });
    expect(addZero).to.equal(false);
    const dupZero = checkAndAddEntry({ title: 'Z', price: 0, address: 'Zero' });
    expect(dupZero).to.equal(true);
  });
 });
--- a/test/utils.js
+++ b/test/utils.js
@@ -6,7 +6,7 @@ import { send } from './mocks/mockNotification.js';
 export const providerConfig = JSON.parse(await readFile(new URL('./provider/testProvider.json', import.meta.url)));
 export const mockFredy = async () => {
-  return await esmock('../lib/FredyRuntime', {
+  return await esmock('../lib/FredyPipeline', {
    '../lib/services/storage/listingsStorage.js': {
      ...mockStore,
    },
--- a/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx
+++ b/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx
@@ -21,7 +21,7 @@ const sortAdapter = (a, b) => {
 const validate = (selectedAdapter) => {
  const results = [];
  for (let uiElement of Object.values(selectedAdapter.fields || [])) {
-    if (uiElement.value == null) {
+    if (uiElement.value == null && !uiElement.optional) {
      results.push('All fields are mandatory and must be set.');
      continue;
    }
@@ -36,7 +36,7 @@ const validate = (selectedAdapter) => {
      results.push('A boolean field cannot be of a different type.');
      continue;
    }
-    if (typeof uiElement.value === 'string' && uiElement.value.length === 0) {
+    if (typeof uiElement.value === 'string' && uiElement.value.length === 0 && !uiElement.optional) {
      results.push('All fields are mandatory and must be set.');
    }
  }
--- a/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationHelpDisplay.jsx
+++ b/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationHelpDisplay.jsx
@@ -1,5 +1,5 @@
 import React from 'react';
-import { Banner } from '@douyinfe/semi-ui';
+import { Banner, MarkdownRender } from '@douyinfe/semi-ui';
 export default function Help({ readme }) {
  return (
@@ -8,7 +8,7 @@ export default function Help({ readme }) {
      type="info"
      closeIcon={null}
      title={<div style={{ fontWeight: 600, fontSize: '14px', lineHeight: '20px' }}>Information</div>}
-      description={<p dangerouslySetInnerHTML={{ __html: readme }} />}
+      description={<MarkdownRender raw={readme} />}
    />
  );
 }
--- a/yarn.lock
+++ b/yarn.lock
@@ -1339,17 +1339,17 @@
  resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
-"@puppeteer/browsers@2.10.10":
+"@puppeteer/browsers@2.10.11":
-  version "2.10.10"
+  version "2.10.11"
-  resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.10.10.tgz#f806f92d966918c931fb9c48052eba2db848beaa"
+  resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.10.11.tgz#e819022871ed63ca8c21a97e3d06963e99ed44a3"
-  integrity sha512-3ZG500+ZeLql8rE0hjfhkycJjDj0pI/btEh3L9IkWUYcOrgP0xCNRq3HbtbqOPbvDhFaAWD88pDFtlLv8ns8gA==
+  integrity sha512-kp3ORGce+oC3qUMJ+g5NH9W4Q7mMG7gV2I+alv0bCbfkZ36B2V/xKCg9uYavSgjmsElhwBneahWjJP7A6fuKLw==
  dependencies:
    debug "^4.4.3"
    extract-zip "^2.0.1"
    progress "^2.0.3"
    proxy-agent "^6.5.0"
    semver "^7.7.2"
-    tar-fs "^3.1.0"
+    tar-fs "^3.1.1"
    yargs "^17.7.2"
 "@resvg/resvg-js-android-arm-eabi@2.4.1":
@@ -1909,11 +1909,6 @@
    "@types/babel__core" "^7.20.5"
    react-refresh "^0.17.0"
 abbrev@1:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
  integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
 abs-svg-path@~0.1.1:
  version "0.1.1"
  resolved "https://registry.yarnpkg.com/abs-svg-path/-/abs-svg-path-0.1.1.tgz#df601c8e8d2ba10d4a76d625e236a9a39c2723bf"
@@ -4566,15 +4561,15 @@ lines-and-columns@^1.1.6:
  resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
  integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
-lint-staged@16.2.3:
+lint-staged@16.2.4:
-  version "16.2.3"
+  version "16.2.4"
-  resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-16.2.3.tgz#790866221d75602510507b5be40b2c7963715960"
+  resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-16.2.4.tgz#1f166370e32d9b7eb10583e86d86e1117f7ab489"
-  integrity sha512-1OnJEESB9zZqsp61XHH2fvpS1es3hRCxMplF/AJUDa8Ho8VrscYDIuxGrj3m8KPXbcWZ8fT9XTMUhEQmOVKpKw==
+  integrity sha512-Pkyr/wd90oAyXk98i/2KwfkIhoYQUMtss769FIT9hFM5ogYZwrk+GRE46yKXSg2ZGhcJ1p38Gf5gmI5Ohjg2yg==
  dependencies:
    commander "^14.0.1"
    listr2 "^9.0.4"
    micromatch "^4.0.8"
-    nano-spawn "^1.0.3"
+    nano-spawn "^2.0.0"
    pidtree "^0.6.0"
    string-argv "^0.3.2"
    yaml "^2.8.1"
@@ -4689,13 +4684,6 @@ markdown-table@^3.0.0:
  resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-3.0.4.tgz#fe44d6d410ff9d6f2ea1797a3f60aa4d2b631c2a"
  integrity sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==
 markdown@^0.5.0:
  version "0.5.0"
  resolved "https://registry.yarnpkg.com/markdown/-/markdown-0.5.0.tgz#28205b565a8ae7592de207463d6637dc182722b2"
  integrity sha512-ctGPIcuqsYoJ493sCtFK7H4UEgMWAUdXeBhPbdsg1W0LsV9yJELAHRsMmWfTgao6nH0/x5gf9FmsbxiXnrgaIQ==
  dependencies:
    nopt "~2.1.1"
 math-intrinsics@^1.1.0:
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9"
@@ -5409,10 +5397,10 @@ ms@^2.1.1, ms@^2.1.3:
  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
  integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
-nano-spawn@^1.0.3:
+nano-spawn@^2.0.0:
-  version "1.0.3"
+  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/nano-spawn/-/nano-spawn-1.0.3.tgz#ef8d89a275eebc8657e67b95fc312a6527a05b8d"
+  resolved "https://registry.yarnpkg.com/nano-spawn/-/nano-spawn-2.0.0.tgz#f1250434c09ae18870d4f729fc54b406cf85a3e1"
-  integrity sha512-jtpsQDetTnvS2Ts1fiRdci5rx0VYws5jGyC+4IYOTnIQ/wwdf6JdomlHBwqC3bJYOvaKu0C2GSZ1A60anrYpaA==
+  integrity sha512-tacvGzUY5o2D8CBh2rrwxyNojUsZNU2zjNTzKQrkgGJQTbGAfArVWXSKMBokBeeg6C7OLRGUEyoFlYbfeWQIqw==
 nanoid@5.1.6:
  version "5.1.6"
@@ -5508,13 +5496,6 @@ nodemon@^3.1.10:
    touch "^3.1.0"
    undefsafe "^2.0.5"
 nopt@~2.1.1:
  version "2.1.2"
  resolved "https://registry.yarnpkg.com/nopt/-/nopt-2.1.2.tgz#6cccd977b80132a07731d6e8ce58c2c8303cf9af"
  integrity sha512-x8vXm7BZ2jE1Txrxh/hO74HTuYZQEbo8edoRcANgdZ4+PCV+pbjd/xdummkmjjC7LU5EjPzlu8zEq/oxWylnKA==
  dependencies:
    abbrev "1"
 normalize-path@^3.0.0, normalize-path@~3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
@@ -5970,12 +5951,12 @@ punycode@^2.1.0:
  resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"
  integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==
-puppeteer-core@24.23.0:
+puppeteer-core@24.24.0:
-  version "24.23.0"
+  version "24.24.0"
-  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.23.0.tgz#1f84abafa480358652ae8df340af984438173a14"
+  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.24.0.tgz#3027c0d59c5246a00e860000e8232745a035e7d6"
-  integrity sha512-yl25C59gb14sOdIiSnJ08XiPP+O2RjuyZmEG+RjYmCXO7au0jcLf7fRiyii96dXGUBW7Zwei/mVKfxMx/POeFw==
+  integrity sha512-RR5AeQ6dIbSepDe9PTtfgK1fgD7TuA9qqyGxPbFCyGfvfkbR7MiqNYdE7AhbTaFIqG3hFBtWwbVKVZF8oEqj7Q==
  dependencies:
-    "@puppeteer/browsers" "2.10.10"
+    "@puppeteer/browsers" "2.10.11"
    chromium-bidi "9.1.0"
    debug "^4.4.3"
    devtools-protocol "0.0.1508733"
@@ -6030,16 +6011,16 @@ puppeteer-extra@^3.3.6:
    debug "^4.1.1"
    deepmerge "^4.2.2"
-puppeteer@^24.23.0:
+puppeteer@^24.24.0:
-  version "24.23.0"
+  version "24.24.0"
-  resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.23.0.tgz#fa3c1bffc1b40c3d7a59b9463d444ff4be69f5c7"
+  resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.24.0.tgz#f58ecdbf99a579b396e6f60636821696fdd1483d"
-  integrity sha512-BVR1Lg8sJGKXY79JARdIssFWK2F6e1j+RyuJP66w4CUmpaXjENicmA3nNpUXA8lcTdDjAndtP+oNdni3T/qQqA==
+  integrity sha512-jRn6T8rSrQZXIplXICpH2zYJ2XrIFY7Ug0+TxRTuwY8ZTL7+MKDvFH0aLG7Xx3ts4twzxIKZmiYo+qg7whNpZw==
  dependencies:
-    "@puppeteer/browsers" "2.10.10"
+    "@puppeteer/browsers" "2.10.11"
    chromium-bidi "9.1.0"
    cosmiconfig "^9.0.0"
    devtools-protocol "0.0.1508733"
-    puppeteer-core "24.23.0"
+    puppeteer-core "24.24.0"
    typed-query-selector "^2.12.0"
 qs@^6.14.0:
@@ -6129,17 +6110,17 @@ react-resizable@^3.0.5:
    prop-types "15.x"
    react-draggable "^4.0.3"
-react-router-dom@7.9.3:
+react-router-dom@7.9.4:
-  version "7.9.3"
+  version "7.9.4"
-  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.9.3.tgz#67ab1655f67b9b6108fe20ed3d4881b53dccf87a"
+  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.9.4.tgz#37d35b4b7f730b37434f2b7e95121ef557a6b538"
-  integrity sha512-1QSbA0TGGFKTAc/aWjpfW/zoEukYfU4dc1dLkT/vvf54JoGMkW+fNA+3oyo2gWVW1GM7BxjJVHz5GnPJv40rvg==
+  integrity sha512-f30P6bIkmYvnHHa5Gcu65deIXoA2+r3Eb6PJIAddvsT9aGlchMatJ51GgpU470aSqRRbFX22T70yQNUGuW3DfA==
  dependencies:
-    react-router "7.9.3"
+    react-router "7.9.4"
-react-router@7.9.3:
+react-router@7.9.4:
-  version "7.9.3"
+  version "7.9.4"
-  resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.9.3.tgz#f2d5ff6181851de3df3acb4e7364fce0dee5fba2"
+  resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.9.4.tgz#2c4249e5d0a6bb8b8f6bf0ede8f5077e4ff8024f"
-  integrity sha512-4o2iWCFIwhI/eYAIL43+cjORXYn/aRQPgtFRRZb3VzoyQ5Uej0Bmqj7437L97N9NJW4wnicSwLOLS+yCXfAPgg==
+  integrity sha512-SD3G8HKviFHg9xj7dNODUKDFgpG4xqD5nhyd0mYoB5iISepuZAvzSr8ywxgxKJ52yRzf/HWtVHc9AWwoTbljvA==
  dependencies:
    cookie "^1.0.1"
    set-cookie-parser "^2.6.0"
@@ -7057,10 +7038,10 @@ tar-fs@^2.0.0:
    pump "^3.0.0"
    tar-stream "^2.1.4"
-tar-fs@^3.1.0:
+tar-fs@^3.1.1:
-  version "3.1.0"
+  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.0.tgz#4675e2254d81410e609d91581a762608de999d25"
+  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.1.tgz#4f164e59fb60f103d472360731e8c6bb4a7fe9ef"
-  integrity sha512-5Mty5y/sOF1YWj1J6GiBodjlDc05CUR8PKXrsnFAiSG0xA+GHeWLovaZPYUDXkH/1iKRf2+M5+OrRgzC7O9b7w==
+  integrity sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==
  dependencies:
    pump "^3.0.0"
    tar-stream "^3.1.5"
Author	SHA1	Message	Date
orangecoding	db3702ed33	improve markdown readme's & and adding ability to send telegram messages to a topic in a supergroup	2025-10-30 12:42:03 +01:00
orangecoding	e3c62d4696	fixing test runner	2025-10-29 10:35:07 +01:00
orangecoding	79a8420dfb	improving similarity cache	2025-10-29 09:36:05 +01:00
orangecoding	d433b13db6	next release version	2025-10-12 16:47:46 +02:00
orangecoding	41d9274dfd	reducing logging	2025-10-12 16:47:28 +02:00
orangecoding	0436c7f7d7	upgrading dependencies / FredyRuntime >> FredyPipeline	2025-10-12 16:43:56 +02:00
Christian Kellner	a1cb57318e	Update README.md	2025-10-11 17:37:51 +02:00
orangecoding	2566db9805	improve index	2025-10-08 15:00:28 +02:00
orangecoding	b48f786fd3	improve docu	2025-10-08 12:16:10 +02:00