santizing ntfy header

lib/notification/adapter/discord_webhook.js

@@ -1,130 +0,0 @@
-import fetch from 'node-fetch';
-import { getJob } from '../../services/storage/jobStorage.js';
-import { markdown2Html } from '../../services/markdown.js';
-import { normalizeImageUrl } from '../../utils.js';
-
-/**
- * Generates an idempotent decimal color code. The input string-based color code is
- * generated using the djb2 hash algorithm.
- *
- * @param {string} str - Input string as color code base
- * @returns {number} Generated decimal color code (0 - 16777215)
- */
-const generateColorFromString = (str) => {
-  let hash = 5381; // initial value
-  const input = String(str);
-
-  for (let i = 0; i < input.length; i++) {
-    // hash * 33 + charCode
-    hash = (hash << 5) + hash + input.charCodeAt(i);
-    // Ensure the hash is 32 bit
-    hash |= 0;
-  }
-
-  let positiveHash = hash >>> 0;
-  const maxColorValue = 16777215;
-  const colorDecimal = positiveHash % maxColorValue;
-
-  return colorDecimal;
-};
-
-/**
- * Creates an embed per listing
- * (-> see https://birdie0.github.io/discord-webhooks-guide/structure/embeds.html).
- *
- * @param {string} jobKey - Key of job (used to set embed color)
- * @param {object} listing - Object holding listing details
- * @returns {object} Discord webhook embed
- */
-const buildEmbed = (jobKey, listing) => {
-  const maxTitleLength = 252; // Max embed title length is 256 characters
-  let title = String(listing.title ?? 'N/A');
-  if (title.length > maxTitleLength) {
-    title = title.substring(0, maxTitleLength) + '...';
-  }
-
-  const fields = [
-    {
-      name: 'Price',
-      value: String(listing.price ?? 'n/a'),
-      inline: true,
-    },
-    {
-      name: 'Size',
-      value: listing?.size?.replace(/2m/g, 'm²') ?? 'n/a',
-      inline: true,
-    },
-    {
-      name: 'Address',
-      value: String(listing.address ?? 'n/a'),
-      inline: true,
-    },
-  ];
-
-  const embed = {
-    title: title,
-    color: generateColorFromString(jobKey),
-    url: listing.link,
-    fields: fields,
-  };
-
-  if (listing.image) {
-    embed.image = {
-      url: normalizeImageUrl(listing.image),
-    };
-  }
-
-  return embed;
-};
-
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const adapter = notificationConfig.find((adapter) => adapter.id === config.id);
-  const webhookUrl = adapter?.fields?.webhookUrl;
-  if (!webhookUrl || newListings.length === 0) return Promise.resolve([]);
-
-  const job = getJob(jobKey);
-  const jobName = job?.name || jobKey;
-
-  const embeds = newListings.map((listing) => buildEmbed(jobKey, listing));
-
-  const maxEmbedsPerMessage = 10; // Discord only allows up to 10 embeds
-  const webhookPromises = [];
-
-  for (let i = 0; i < embeds.length; i += maxEmbedsPerMessage) {
-    // Send multiple Discord messages with up to 10 embeds per message
-    const embedChunk = embeds.slice(i, i + maxEmbedsPerMessage);
-
-    const content = i === 0 ? `*${jobName}:* ${serviceName} found **${newListings.length}** new listings.` : '';
-    const body = JSON.stringify({
-      content: content,
-      embeds: embedChunk,
-    });
-
-    const fetchPromise = fetch(webhookUrl, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body,
-    }).catch((error) => {
-      console.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
-      return Promise.reject(new Error(`Webhook failed: ${error.message}`));
-    });
-
-    webhookPromises.push(fetchPromise);
-  }
-
-  return Promise.allSettled(webhookPromises);
-};
-
-export const config = {
-  id: 'discord_webhook',
-  name: 'Discord Webhook',
-  readme: markdown2Html('lib/notification/adapter/discord_webhook.md'),
-  description: 'Fredy will send new listings to the Discord channel of your choice.',
-  fields: {
-    webhookUrl: {
-      type: 'text',
-      label: 'Webhook URL',
-      description: 'The URL of the Discord webhook to send messages to.',
-    },
-  },
-};

lib/notification/adapter/discord_webhook.md

@@ -1,4 +0,0 @@
-### Discord Adapter
-
-To use the [Discord](https://discord.com/) Adapter, you need to create a webhook on the Discord channel of your choice. You can follow the instructions of _Making A Webhook_ on [this support website](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks).
-Once you have created a webhook, copy and paste the webhook URL.

lib/provider/immonet.js

@@ -29,7 +29,6 @@ const config = {
     address: 'div[data-testid="cardmfe-description-box-address"] | trim',
     image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
     link: 'button@data-base',
-    description: 'div[data-testid="cardmfe-description-text-test-id"] | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immowelt.js

@@ -26,9 +26,8 @@ const config = {
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
     title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
-    description: 'div[data-testid="cardmfe-description-text-test-id"] > div:nth-of-type(2) | removeNewline | trim',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
-    image: 'div[data-testid="cardmfe-picture-box-opacity-layer-test-id"] img@src',
+    image: 'div[data-testid="cardMfe-card-pictureBox-opacity"] img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/regionalimmobilien24.js

@@ -1,49 +0,0 @@
-import { isOneOf, buildHash } from '../utils.js';
-import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
-let appliedBlackList = [];
-
-function normalize(o) {
-  const id = buildHash(o.id, o.price);
-  const address = o.address?.replace(/^adresse /i, '') ?? null;
-  const title = o.title || 'No title available';
-  const link = o.link != null ? decodeURIComponent(o.link) : config.url;
-
-  var urlReg = new RegExp(/url\((.*?)\)/gim);
-  const image = o.image != null ? urlReg.exec(o.image)[1] : null;
-  return Object.assign(o, { id, address, title, link, image });
-}
-function applyBlacklist(o) {
-  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
-}
-const config = {
-  url: null,
-  crawlContainer: '.listentry-content',
-  sortByDateParam: null, // sort by date is standard
-  waitForSelector: 'body',
-  crawlFields: {
-    id: '.listentry-iconbar-share@data-sid | trim',
-    title: 'h2 | trim',
-    price: '.listentry-details-price .listentry-details-v | trim',
-    size: '.listentry-details-size .listentry-details-v | trim',
-    address: '.listentry-adress | trim',
-    image: '.listentry-img@style',
-    link: '.shariff@data-url',
-    description: '.listentry-extras | trim',
-  },
-  normalize: normalize,
-  filter: applyBlacklist,
-  activeTester: checkIfListingIsActive,
-};
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
-};
-export const metaInformation = {
-  name: 'Regionalimmobilien24',
-  baseUrl: 'https://www.regionalimmobilien24.de/',
-  id: 'regionalimmobilien24',
-};
-export { config };

lib/provider/sparkasse.js

@@ -1,46 +0,0 @@
-import { isOneOf, buildHash } from '../utils.js';
-import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
-let appliedBlackList = [];
-
-function normalize(o) {
-  const originalId = o.id.split('/').pop().replace('.html', '');
-  const id = buildHash(originalId, o.price);
-  const size = o.size?.replace(' Wohnfläche', '') ?? null;
-  const title = o.title || 'No title available';
-  const link = o.link != null ? `https://immobilien.sparkasse.de${o.link}` : config.url;
-  return Object.assign(o, { id, size, title, link });
-}
-function applyBlacklist(o) {
-  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
-  return titleNotBlacklisted && descNotBlacklisted;
-}
-const config = {
-  url: null,
-  crawlContainer: '.estate-list-item-row',
-  sortByDateParam: 'sortBy=date_desc',
-  waitForSelector: 'body',
-  crawlFields: {
-    id: 'div[data-testid="estate-link"] a@href',
-    title: 'h3 | trim',
-    price: '.estate-list-price | trim',
-    size: '.estate-mainfact:first-child span | trim',
-    address: 'h6 | trim',
-    image: '.estate-list-item-image-container img@src',
-    link: 'div[data-testid="estate-link"] a@href',
-  },
-  normalize: normalize,
-  filter: applyBlacklist,
-  activeTester: checkIfListingIsActive,
-};
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist || [];
-};
-export const metaInformation = {
-  name: 'Sparkasse Immobilien',
-  baseUrl: 'https://immobilien.sparkasse.de/',
-  id: 'sparkasse',
-};
-export { config };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "12.3.0",
+  "version": "12.2.2",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",

test/provider/immonet.test.js

@@ -8,30 +8,31 @@ describe('#immonet testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
+  provider.init(providerConfig.immonet, [], []);
   it('should test immonet provider', async () => {
     const Fredy = await mockFredy();
-    provider.init(providerConfig.immonet, [], []);
+    return await new Promise((resolve) => {
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
+      fredy.execute().then((listing) => {
+        expect(listing).to.be.a('array');
+        const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immonet');
+        notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
+          expect(notify.size).to.be.a('string');
+          expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
 
-    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
+          expect(notify.size).that.does.include('m²');
-    const listing = await fredy.execute();
+          expect(notify.title).to.be.not.empty;
-
+          expect(notify.address).to.be.not.empty;
-    expect(listing).to.be.a('array');
+        });
-    const notificationObj = get();
+        resolve();
-    expect(notificationObj).to.be.a('object');
+      });
-    expect(notificationObj.serviceName).to.equal('immonet');
-    notificationObj.payload.forEach((notify) => {
-      /** check the actual structure **/
-      expect(notify.id).to.be.a('string');
-      expect(notify.price).to.be.a('string');
-      expect(notify.size).to.be.a('string');
-      expect(notify.title).to.be.a('string');
-      expect(notify.link).to.be.a('string');
-      expect(notify.address).to.be.a('string');
-      /** check the values if possible **/
-      expect(notify.size).that.does.include('m²');
-      expect(notify.title).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
     });
   });
 });

test/provider/immowelt.test.js

@@ -8,32 +8,33 @@ describe('#immowelt testsuite()', () => {
   after(() => {
     similarityCache.stopCacheCleanup();
   });
-
   it('should test immowelt provider', async () => {
     const Fredy = await mockFredy();
     provider.init(providerConfig.immowelt, [], []);
-
+    return await new Promise((resolve) => {
-    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
+      const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
-    const listing = await fredy.execute();
+      fredy.execute().then((listing) => {
-
+        expect(listing).to.be.a('array');
-    expect(listing).to.be.a('array');
+        const notificationObj = get();
-    const notificationObj = get();
+        expect(notificationObj).to.be.a('object');
-    expect(notificationObj).to.be.a('object');
+        expect(notificationObj.serviceName).to.equal('immowelt');
-    expect(notificationObj.serviceName).to.equal('immowelt');
+        notificationObj.payload.forEach((notify) => {
-    notificationObj.payload.forEach((notify) => {
+          /** check the actual structure **/
-      /** check the actual structure **/
+          expect(notify.id).to.be.a('string');
-      expect(notify.id).to.be.a('string');
+          expect(notify.price).to.be.a('string');
-      expect(notify.price).to.be.a('string');
+          expect(notify.title).to.be.a('string');
-      expect(notify.title).to.be.a('string');
+          expect(notify.link).to.be.a('string');
-      expect(notify.link).to.be.a('string');
+          expect(notify.address).to.be.a('string');
-      expect(notify.address).to.be.a('string');
+          /** check the values if possible **/
-      /** check the values if possible **/
+          if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
-      if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
+            expect(notify.size).that.does.include('m²');
-        expect(notify.size).that.does.include('m²');
+          }
-      }
+          expect(notify.title).to.be.not.empty;
-      expect(notify.title).to.be.not.empty;
+          expect(notify.link).that.does.include('https://www.immowelt.de');
-      expect(notify.link).that.does.include('https://www.immowelt.de');
+          expect(notify.address).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
+        });
+        resolve();
+      });
     });
   });
 });

test/provider/regionalimmobilien24.test.js

@@ -1,43 +0,0 @@
-import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
-import { expect } from 'chai';
-import * as provider from '../../lib/provider/regionalimmobilien24.js';
-
-describe('#regionalimmobilien24 testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
-  it('should test regionalimmobilien24 provider', async () => {
-    const Fredy = await mockFredy();
-    provider.init(providerConfig.regionalimmobilien24, []);
-
-    const fredy = new Fredy(
-      provider.config,
-      null,
-      provider.metaInformation.id,
-      'regionalimmobilien24',
-      similarityCache,
-    );
-    const listing = await fredy.execute();
-
-    expect(listing).to.be.a('array');
-    const notificationObj = get();
-    expect(notificationObj).to.be.a('object');
-    expect(notificationObj.serviceName).to.equal('regionalimmobilien24');
-    notificationObj.payload.forEach((notify) => {
-      /** check the actual structure **/
-      expect(notify.id).to.be.a('string');
-      expect(notify.price).to.be.a('string');
-      expect(notify.size).to.be.a('string');
-      expect(notify.title).to.be.a('string');
-      expect(notify.link).to.be.a('string');
-      expect(notify.address).to.be.a('string');
-      /** check the values if possible **/
-      expect(notify.size).that.does.include('m²');
-      expect(notify.title).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
-    });
-  });
-});

test/provider/sparkasse.test.js

@@ -1,37 +0,0 @@
-import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
-import { get } from '../mocks/mockNotification.js';
-import { mockFredy, providerConfig } from '../utils.js';
-import { expect } from 'chai';
-import * as provider from '../../lib/provider/sparkasse.js';
-
-describe('#sparkasse testsuite()', () => {
-  after(() => {
-    similarityCache.stopCacheCleanup();
-  });
-
-  it('should test sparkasse provider', async () => {
-    const Fredy = await mockFredy();
-    provider.init(providerConfig.sparkasse, []);
-
-    const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'sparkasse', similarityCache);
-    const listing = await fredy.execute();
-
-    expect(listing).to.be.a('array');
-    const notificationObj = get();
-    expect(notificationObj).to.be.a('object');
-    expect(notificationObj.serviceName).to.equal('sparkasse');
-    notificationObj.payload.forEach((notify) => {
-      /** check the actual structure **/
-      expect(notify.id).to.be.a('string');
-      expect(notify.price).to.be.a('string');
-      expect(notify.size).to.be.a('string');
-      expect(notify.title).to.be.a('string');
-      expect(notify.link).to.be.a('string');
-      expect(notify.address).to.be.a('string');
-      /** check the values if possible **/
-      expect(notify.size).that.does.include('m²');
-      expect(notify.title).to.be.not.empty;
-      expect(notify.address).to.be.not.empty;
-    });
-  });
-});

test/provider/testProvider.json

@@ -32,14 +32,6 @@
     "url": "https://www.neubaukompass.de/neubau-immobilien/duesseldorf-region/eigentumswohnung/",
     "enabled": true
   },
-  "regionalimmobilien24": {
-    "url": "https://www.regionalimmobilien24.de/rostock/rostock/kaufen/haus/-/-/-/?rd=5",
-    "enabled": true
-  },
-  "sparkasse": {
-    "url": "https://immobilien.sparkasse.de/immobilien/treffer?marketingType=buy&objectType=flat&perimeter=10&usageType=residential&zipCityEstateId=62782__Hamburg",
-    "enabled": true
-  },
   "wgGesucht": {
     "url": "https://www.wg-gesucht.de/wg-zimmer-in-Duesseldorf.30.0.1.0.html",
     "enabled": true