santizing ntfy header

README.md

@@ -21,7 +21,7 @@ Finding an apartment or house in Germany can be stressful and
 time-consuming.\
 **Fredy** makes it easier: it automatically scrapes **ImmoScout24,
 Immowelt, Immonet, eBay Kleinanzeigen, and WG-Gesucht** and notifies you
-instantly via **Slack, Telegram, Email, ntfy, discord and more** when new
+instantly via **Slack, Telegram, Email, ntfy, and more** when new
 listings appear.
 
 With a modern architecture, Fredy provides a **clean Web UI**, removes
@@ -35,7 +35,7 @@ same listing twice.
 -   🏠 Scrapes **ImmoScout24, Immowelt, Immonet, eBay Kleinanzeigen,
     WG-Gesucht**
 -   ⚡ Instant notifications: Slack, Telegram, Email (SendGrid,
-    Mailjet), ntfy, discord 
+    Mailjet), ntfy
 -   🔎 Uses the **ImmoScout Mobile API** (reverse engineered)
 -   🌍 Runs anywhere: Docker, Node.js, self-hosted
 -   🖥️ Intuitive **Web UI** to manage searches
@@ -129,7 +129,7 @@ picks up the newest listings first.
 ### Adapter 📡
 
 An **adapter** is the channel through which Fredy notifies you (Slack,
-Telegram, Email, ntfy, discord ...).\
+Telegram, Email, ntfy, ...).\
 Each adapter has its own configuration (e.g. API keys, webhook URLs).\
 You can use multiple adapters at once --- Fredy will send new listings
 through all of them.

lib/notification/adapter/apprise.js

@@ -8,7 +8,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const jobName = job == null ? jobKey : job.name;
   const promises = newListings.map((newListing) => {
     const title = `${jobName} at ${serviceName}: ${newListing.title}`;
-    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
     return fetch(server, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },

lib/notification/adapter/discord_webhook.js

@@ -1,130 +0,0 @@
-import fetch from 'node-fetch';
-import { getJob } from '../../services/storage/jobStorage.js';
-import { markdown2Html } from '../../services/markdown.js';
-import { normalizeImageUrl } from '../../utils.js';
-
-/**
- * Generates an idempotent decimal color code. The input string-based color code is
- * generated using the djb2 hash algorithm.
- *
- * @param {string} str - Input string as color code base
- * @returns {number} Generated decimal color code (0 - 16777215)
- */
-const generateColorFromString = (str) => {
-  let hash = 5381; // initial value
-  const input = String(str);
-
-  for (let i = 0; i < input.length; i++) {
-    // hash * 33 + charCode
-    hash = (hash << 5) + hash + input.charCodeAt(i);
-    // Ensure the hash is 32 bit
-    hash |= 0;
-  }
-
-  let positiveHash = hash >>> 0;
-  const maxColorValue = 16777215;
-  const colorDecimal = positiveHash % maxColorValue;
-
-  return colorDecimal;
-};
-
-/**
- * Creates an embed per listing
- * (-> see https://birdie0.github.io/discord-webhooks-guide/structure/embeds.html).
- *
- * @param {string} jobKey - Key of job (used to set embed color)
- * @param {object} listing - Object holding listing details
- * @returns {object} Discord webhook embed
- */
-const buildEmbed = (jobKey, listing) => {
-  const maxTitleLength = 252; // Max embed title length is 256 characters
-  let title = String(listing.title ?? 'N/A');
-  if (title.length > maxTitleLength) {
-    title = title.substring(0, maxTitleLength) + '...';
-  }
-
-  const fields = [
-    {
-      name: 'Price',
-      value: String(listing.price ?? 'n/a'),
-      inline: true,
-    },
-    {
-      name: 'Size',
-      value: listing?.size?.replace(/2m/g, 'm²') ?? 'n/a',
-      inline: true,
-    },
-    {
-      name: 'Address',
-      value: String(listing.address ?? 'n/a'),
-      inline: true,
-    },
-  ];
-
-  const embed = {
-    title: title,
-    color: generateColorFromString(jobKey),
-    url: listing.link,
-    fields: fields,
-  };
-
-  if (listing.image) {
-    embed.image = {
-      url: normalizeImageUrl(listing.image),
-    };
-  }
-
-  return embed;
-};
-
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const adapter = notificationConfig.find((adapter) => adapter.id === config.id);
-  const webhookUrl = adapter?.fields?.webhookUrl;
-  if (!webhookUrl || newListings.length === 0) return Promise.resolve([]);
-
-  const job = getJob(jobKey);
-  const jobName = job?.name || jobKey;
-
-  const embeds = newListings.map((listing) => buildEmbed(jobKey, listing));
-
-  const maxEmbedsPerMessage = 10; // Discord only allows up to 10 embeds
-  const webhookPromises = [];
-
-  for (let i = 0; i < embeds.length; i += maxEmbedsPerMessage) {
-    // Send multiple Discord messages with up to 10 embeds per message
-    const embedChunk = embeds.slice(i, i + maxEmbedsPerMessage);
-
-    const content = i === 0 ? `*${jobName}:* ${serviceName} found **${newListings.length}** new listings.` : '';
-    const body = JSON.stringify({
-      content: content,
-      embeds: embedChunk,
-    });
-
-    const fetchPromise = fetch(webhookUrl, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body,
-    }).catch((error) => {
-      console.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
-      return Promise.reject(new Error(`Webhook failed: ${error.message}`));
-    });
-
-    webhookPromises.push(fetchPromise);
-  }
-
-  return Promise.allSettled(webhookPromises);
-};
-
-export const config = {
-  id: 'discord_webhook',
-  name: 'Discord Webhook',
-  readme: markdown2Html('lib/notification/adapter/discord_webhook.md'),
-  description: 'Fredy will send new listings to the Discord channel of your choice.',
-  fields: {
-    webhookUrl: {
-      type: 'text',
-      label: 'Webhook URL',
-      description: 'The URL of the Discord webhook to send messages to.',
-    },
-  },
-};

lib/notification/adapter/discord_webhook.md

@@ -1,4 +0,0 @@
-### Discord Adapter
-
-To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
-Once you have created a webhook, copy and paste the webhook URL.

lib/notification/adapter/mattermost.js

@@ -13,10 +13,10 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   return fetch(webhook, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
+    body: {
       channel: channel,
       text: message,
-    }),
+    },
   });
 };
 export const config = {

lib/provider/immonet.js

@@ -29,7 +29,6 @@ const config = {
     address: 'div[data-testid="cardmfe-description-box-address"] | trim',
     image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
     link: 'button@data-base',
-    description: 'div[data-testid="cardmfe-description-text-test-id"] | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immowelt.js

@@ -26,9 +26,8 @@ const config = {
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
     title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
-    description: 'div[data-testid="cardmfe-description-text-test-id"] > div:nth-of-type(2) | removeNewline | trim',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
-    image: 'div[data-testid="cardmfe-picture-box-opacity-layer-test-id"] img@src',
+    image: 'div[data-testid="cardMfe-card-pictureBox-opacity"] img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/regionalimmobilien24.js

@@ -1,49 +0,0 @@
-import { isOneOf, buildHash } from '../utils.js';
-import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
-let appliedBlackList = [];
-
-function normalize(o) {
-  const id = buildHash(o.id, o.price);
-  const address = o.address?.replace(/^adresse /i, '') ?? null;
-  const title = o.title || 'No title available';
-  const link = o.link != null ? decodeURIComponent(o.link) : config.url;
-
-  var urlReg = new RegExp(/url\((.*?)\)/gim);
-  const image = o.image != null ? urlReg.exec(o.image)[1] : null;
-  return Object.assign(o, { id, address, title, link, image });
-}
-function applyBlacklist(o) {
-  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
-}
-const config = {
-  url: null,
-  crawlContainer: '.listentry-content',
-  sortByDateParam: null, // sort by date is standard
-  waitForSelector: 'body',
-  crawlFields: {
-    id: '.listentry-iconbar-share@data-sid | trim',
-    title: 'h2 | trim',
-    price: '.listentry-details-price .listentry-details-v | trim',
-    size: '.listentry-details-size .listentry-details-v | trim',
-    address: '.listentry-adress | trim',
-    image: '.listentry-img@style',
-    link: '.shariff@data-url',
-    description: '.listentry-extras | trim',
-  },
-  normalize: normalize,
-  filter: applyBlacklist,
-  activeTester: checkIfListingIsActive,
-};
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
-};
-export const metaInformation = {
-  name: 'Regionalimmobilien24',
-  baseUrl: 'https://www.regionalimmobilien24.de/',
-  id: 'regionalimmobilien24',
-};
-export { config };

lib/provider/sparkasse.js

@@ -1,46 +0,0 @@
-import { isOneOf, buildHash } from '../utils.js';
-import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
-let appliedBlackList = [];
-
-function normalize(o) {
-  const originalId = o.id.split('/').pop().replace('.html', '');
-  const id = buildHash(originalId, o.price);
-  const size = o.size?.replace(' Wohnfläche', '') ?? null;
-  const title = o.title || 'No title available';
-  const link = o.link != null ? `https://immobilien.sparkasse.de${o.link}` : config.url;
-  return Object.assign(o, { id, size, title, link });
-}
-function applyBlacklist(o) {
-  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
-}
-const config = {
-  url: null,
-  crawlContainer: '.estate-list-item-row',
-  sortByDateParam: 'sortBy=date_desc',
-  waitForSelector: 'body',
-  crawlFields: {
-    id: 'div[data-testid="estate-link"] a@href',
-    title: 'h3 | trim',
-    price: '.estate-list-price | trim',
-    size: '.estate-mainfact:first-child span | trim',
-    address: 'h6 | trim',
-    image: '.estate-list-item-image-container img@src',
-    link: 'div[data-testid="estate-link"] a@href',
-  },
-  normalize: normalize,
-  filter: applyBlacklist,
-  activeTester: checkIfListingIsActive,
-};
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
-};
-export const metaInformation = {
-  name: 'Sparkasse Immobilien',
-  baseUrl: 'https://immobilien.sparkasse.de/',
-  id: 'sparkasse',
-};
-export { config };

lib/services/extractor/utils.js

@@ -3,25 +3,12 @@ import logger from '../logger.js';
 let debuggingOn = false;
 
 export const DEFAULT_HEADER = {
-  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-  'Accept-Language': 'en-US,en;q=0.9',
+  'Accept-Language': 'en-US,en;q=0.5',
-  'Accept-Encoding': 'gzip, deflate, br',
   Connection: 'keep-alive',
   'Upgrade-Insecure-Requests': '1',
   'User-Agent':
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' +
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
-    'AppleWebKit/537.36 (KHTML, like Gecko) ' +
-    'Chrome/140.0.7339.207 Safari/537.36',
-  'Sec-CH-UA': '"Chromium";v="140", "Not.A/Brand";v="8"',
-  'Sec-CH-UA-Mobile': '?0',
-  'Sec-CH-UA-Platform': '"Windows"',
-  'Sec-Fetch-Site': 'none',
-  'Sec-Fetch-Mode': 'navigate',
-  'Sec-Fetch-User': '?1',
-  'Sec-Fetch-Dest': 'document',
-  Referer: 'https://www.google.com/',
-  DNT: '1',
-  TE: 'trailers',
 };
 
 export const setDebug = (options) => {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "12.3.1",
+  "version": "12.2.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",

test/provider/immonet.test.js

@@ -8,30 +8,31 @@ describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
+  provider.init(providerConfig.immonet, [], []);
   it('should test immonet provider', async () => {
     const Fredy = await mockFredy();
-    provider.init(providerConfig.immonet, [], []);
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immonet');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
 
-    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
+          expect(notify.size).that.does.include('m²');
-    const listing = await fredy.execute();
+          expect(notify.title).to.be.not.empty;
-
+          expect(notify.address).to.be.not.empty;
-    expect(listing).to.be.a('array');
+        });
-    const notificationObj = get();
+        resolve();
-    expect(notificationObj).to.be.a('object');
+      });
-    expect(notificationObj.serviceName).to.equal('immonet');
-    notificationObj.payload.forEach((notify) => {
-      /** check the actual structure **/
-      expect(notify.id).to.be.a('string');
-      expect(notify.price).to.be.a('string');
-      expect(notify.size).to.be.a('string');
-      expect(notify.title).to.be.a('string');
-      expect(notify.link).to.be.a('string');
-      expect(notify.address).to.be.a('string');
-      /** check the values if possible **/
-      expect(notify.size).that.does.include('m²');
-      expect(notify.title).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
     });
   });
 });

test/provider/immowelt.test.js

@@ -8,32 +8,33 @@ describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   it('should test immowelt provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);
-
+    return await new Promise((resolve) => {
-    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
-    const listing = await fredy.execute();
+      fredy.execute().then((listing) => {
-
+        expect(listing).to.be.a('array');
-    expect(listing).to.be.a('array');
+        const notificationObj = get();
-    const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
-    expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immowelt');
-    expect(notificationObj.serviceName).to.equal('immowelt');
+        notificationObj.payload.forEach((notify) => {
-    notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
-      /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
-      expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
-      expect(notify.price).to.be.a('string');
+          expect(notify.title).to.be.a('string');
-      expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
-      expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
-      expect(notify.address).to.be.a('string');
+          /** check the values if possible **/
-      /** check the values if possible **/
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
-      if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
+            expect(notify.size).that.does.include('m²');
-        expect(notify.size).that.does.include('m²');
+          }
-      }
+          expect(notify.title).to.be.not.empty;
-      expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immowelt.de');
-      expect(notify.link).that.does.include('https://www.immowelt.de');
+          expect(notify.address).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
     });
   });
 });

test/provider/regionalimmobilien24.test.js

@@ -1,43 +0,0 @@
-import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
-import { expect } from 'chai';
-import * as provider from '../../lib/provider/regionalimmobilien24.js';
-
-describe('#regionalimmobilien24 testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
-  it('should test regionalimmobilien24 provider', async () => {
-    const Fredy = await mockFredy();
-    provider.init(providerConfig.regionalimmobilien24, []);
-
-    const fredy = new Fredy(
-      provider.config,
-      null,
-      provider.metaInformation.id,
-      'regionalimmobilien24',
-      similarityCache,
-    );
-    const listing = await fredy.execute();
-
-    expect(listing).to.be.a('array');
-    const notificationObj = get();
-    expect(notificationObj).to.be.a('object');
-    expect(notificationObj.serviceName).to.equal('regionalimmobilien24');
-    notificationObj.payload.forEach((notify) => {
-      /** check the actual structure **/
-      expect(notify.id).to.be.a('string');
-      expect(notify.price).to.be.a('string');
-      expect(notify.size).to.be.a('string');
-      expect(notify.title).to.be.a('string');
-      expect(notify.link).to.be.a('string');
-      expect(notify.address).to.be.a('string');
-      /** check the values if possible **/
-      expect(notify.size).that.does.include('m²');
-      expect(notify.title).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
-    });
-  });
-});

test/provider/sparkasse.test.js

@@ -1,37 +0,0 @@
-import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
-import { expect } from 'chai';
-import * as provider from '../../lib/provider/sparkasse.js';
-
-describe('#sparkasse testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
-  it('should test sparkasse provider', async () => {
-    const Fredy = await mockFredy();
-    provider.init(providerConfig.sparkasse, []);
-
-    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'sparkasse', similarityCache);
-    const listing = await fredy.execute();
-
-    expect(listing).to.be.a('array');
-    const notificationObj = get();
-    expect(notificationObj).to.be.a('object');
-    expect(notificationObj.serviceName).to.equal('sparkasse');
-    notificationObj.payload.forEach((notify) => {
-      /** check the actual structure **/
-      expect(notify.id).to.be.a('string');
-      expect(notify.price).to.be.a('string');
-      expect(notify.size).to.be.a('string');
-      expect(notify.title).to.be.a('string');
-      expect(notify.link).to.be.a('string');
-      expect(notify.address).to.be.a('string');
-      /** check the values if possible **/
-      expect(notify.size).that.does.include('m²');
-      expect(notify.title).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
-    });
-  });
-});

test/provider/testProvider.json

@@ -32,14 +32,6 @@
     "url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
     "enabled": true
   },
-  "regionalimmobilien24": {
-    "url": "https://www.regionalimmobilien24.de/rostock/rostock/kaufen/haus/-/-/-/?rd=5",
-    "enabled": true
-  },
-  "sparkasse": {
-    "url": "https://immobilien.sparkasse.de/immobilien/treffer?marketingType=buy&objectType=flat&perimeter=10&usageType=residential&zipCityEstateId=62782__Hamburg",
-    "enabled": true
-  },
   "wgGesucht": {
     "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
     "enabled": true

ui/src/components/table/JobTable.jsx

@@ -32,7 +32,7 @@ export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged,
           dataIndex: 'name',
         },
         {
-          title: 'Listings',
+          title: 'Findings',
           dataIndex: 'numberOfFoundListings',
           render: (value) => {
             return value || 0;

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -186,7 +186,7 @@ const GeneralSettings = function GeneralSettings() {
             <Divider margin="1rem" />
             <SegmentPart
               name="Working hours"
-              helpText="During these hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
+              helpText="During this hours, Fredy will search for new apartments. If nothing is configured, Fredy will search around the clock."
               Icon={IconCalendar}
             >
               <div className="generalSettings__timePickerContainer">

ui/src/views/jobs/mutation/JobMutation.jsx

@@ -89,7 +89,7 @@ export default function JobMutator() {
         />
       )}
 
-      <Headline text={jobToBeEdit ? 'Edit Job' : 'Create new Job'} />
+      <Headline text={jobToBeEdit ? 'Edit a Job' : 'Create a new Job'} />
       <form>
         <SegmentPart name="Name">
           <Input