Adding images to scraping data (#157)

* Fredy now supports pulling the main Image from the listing and send it together with the usual information

lib/notification/adapter/mailJet.js

@@ -2,68 +2,120 @@ import mailjet from 'node-mailjet';
 import path from 'path';
 import fs from 'fs';
 import Handlebars from 'handlebars';
+import fetch from 'node-fetch';
 import { markdown2Html } from '../../services/markdown.js';
-import { getDirName } from '../../utils.js';
+import { getDirName, normalizeImageUrl } from '../../utils.js';
+
 const __dirname = getDirName();
 const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
 const emailTemplate = Handlebars.compile(template);
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+
+const guessMime = (url) => {
+  const lower = url.split('?')[0].toLowerCase();
+  if (lower.endsWith('.png')) return 'image/png';
+  if (lower.endsWith('.gif')) return 'image/gif';
+  return 'image/jpeg';
+};
+
+const toBase64 = async (url) => {
+  try {
+    const res = await fetch(url);
+    if (!res.ok) throw new Error(`Fetch failed with status ${res.status} for URL: ${url}`);
+    const ab = await res.arrayBuffer();
+    return Buffer.from(ab).toString('base64');
+  } catch (error) {
+    console.error(`Error fetching image from ${url}:`, error.message);
+    throw error;
+  }
+};
+
+const mapListingsWithCid = async (serviceName, jobKey, listings) => {
+  const out = [];
+  const attachments = [];
+
+  for (let i = 0; i < listings.length; i++) {
+    const l = listings[i] || {};
+    const imgUrl = normalizeImageUrl(l.image);
+
+    const item = {
+      title: l.title || '',
+      link: l.link || '',
+      address: l.address || '',
+      size: l.size || '',
+      price: l.price || '',
+      serviceName,
+      jobKey,
+      hasImage: false,
+      imageCid: '',
+    };
+
+    if (imgUrl) {
+      try {
+        const base64 = await toBase64(imgUrl);
+        const cid = `listing-${i}`;
+        attachments.push({
+          ContentType: guessMime(imgUrl),
+          Filename: `listing-${i}.${imgUrl.split('.').pop().split('?')[0] || 'jpg'}`,
+          Base64Content: base64,
+          ContentID: cid,
+        });
+        item.hasImage = true;
+        item.imageCid = cid;
+      } catch (error) {
+        console.warn(`Skipping image for listing ${i} due to error: ${error.message}`);
+      }
+    }
+
+    out.push(item);
+  }
+
+  return { listings: out, attachments };
+};
+
+export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
     (adapter) => adapter.id === config.id,
   ).fields;
+
   const to = receiver
     .trim()
     .split(',')
-    .map((r) => ({
-      Email: r.trim(),
-    }));
+    .map((r) => ({ Email: r.trim() }))
+    .filter((r) => r.Email.length > 0);
+
+  const { listings, attachments } = await mapListingsWithCid(serviceName, jobKey, newListings);
+
+  const html = emailTemplate({
+    serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
+    numberOfListings: listings.length,
+    listings,
+  });
+
   return mailjet
     .apiConnect(apiPublicKey, apiPrivateKey)
     .post('send', { version: 'v3.1' })
     .request({
       Messages: [
         {
-          From: {
-            Email: from,
-            Name: 'Fredy',
-          },
+          From: { Email: from, Name: 'Fredy' },
           To: to,
-          Subject: `Fredy found ${newListings.length} new listings for ${serviceName}`,
-          HTMLPart: emailTemplate({
-            serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
-            numberOfListings: newListings.length,
-            listings: newListings,
-          }),
+          Subject: `Fredy found ${listings.length} new listing(s) for ${serviceName}`,
+          HTMLPart: html,
+          InlinedAttachments: attachments,
         },
       ],
     });
 };
+
 export const config = {
   id: 'mailjet',
   name: 'MailJet',
   description: 'MailJet is being used to send new listings via mail.',
   readme: markdown2Html('lib/notification/adapter/mailJet.md'),
   fields: {
-    apiPublicKey: {
-      type: 'text',
-      label: 'Public Api Key',
-      description: 'The public api key needed to access this service.',
-    },
-    apiPrivateKey: {
-      type: 'text',
-      label: 'Private Api Key',
-      description: 'The private api key needed to access this service.',
-    },
-    receiver: {
-      type: 'email',
-      label: 'Receiver Email',
-      description: 'The email address (single one) which Fredy is using to send notifications to.',
-    },
-    from: {
-      type: 'email',
-      label: 'Sender email',
-      description:
-        'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
-    },
+    apiPublicKey: { type: 'text', label: 'Public Api Key' },
+    apiPrivateKey: { type: 'text', label: 'Private Api Key' },
+    receiver: { type: 'email', label: 'Receiver Email' },
+    from: { type: 'email', label: 'Sender email' },
   },
 };

lib/notification/adapter/mailJet.md

@@ -1,8 +1,8 @@
 ### MailJet Adapter
 
-To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from.
+To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
 
 E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.  
 The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
 
-If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).
+If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).

lib/notification/adapter/ntfy.js

@@ -1,32 +1,41 @@
 import { markdown2Html } from '../../services/markdown.js';
 import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
+import { normalizeImageUrl } from '../../utils.js';
 
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
+
   const promises = newListings.map((newListing) => {
     const message = `
-    Address: ${newListing.address} 
-    Size: ${newListing.size.replace(/2m/g, '$m^2$')} 
-    Price: ${newListing.price}
-    Link: ${newListing.link}`;
-    return fetch(server, {
+Address: ${newListing.address}
+Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
+Price: ${newListing.price}
+Link: ${newListing.link}`;
+
+    const headers = {
+      Title: newListing.title,
+      Priority: String(priority),
+      Tags: `${serviceName},${jobName}`,
+      Click: newListing.link,
+    };
+
+    if (newListing.image && typeof newListing.image === 'string') {
+      headers.Attach = normalizeImageUrl(newListing.image);
+    }
+
+    return fetch(`${server}/${topic}`, {
       method: 'POST',
-      body: JSON.stringify({
-        topic: topic,
-        message: message,
-        title: newListing.title,
-        tags: [serviceName, jobName],
-        priority: parseInt(priority),
-        click: newListing.link,
-      }),
+      headers,
+      body: message,
     });
   });
 
   return Promise.all(promises);
 };
+
 export const config = {
   id: 'ntfy',
   name: 'ntfy',

lib/notification/adapter/pushover.js

@@ -2,50 +2,55 @@ import { markdown2Html } from '../../services/markdown.js';
 import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-  const promises = newListings.map((newListing) => {
-    const title = `${jobName} at ${serviceName}: ${newListing.title}`;
-    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
-    return fetch('https://api.pushover.net/1/messages.json', {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({
-        token: token,
-        user: user,
-        message: message,
-        device: device,
-        title: title,
-      }),
-    });
-  });
 
-  return Promise.all(promises)
-    .then((responses) => {
-      // Convert all responses to JSON
-      return Promise.all(responses.map((response) => response.json()));
-    })
-    .then((data) => {
-      // Check for errors in the data
-      const error = data
-        .map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
-        .filter((err) => err !== null);
+  const results = await Promise.all(
+    newListings.map(async (newListing) => {
+      const title = `${jobName} at ${serviceName}: ${newListing.title}`;
+      const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
 
-      if (error.length > 0) {
-        // Reject with the combined error messages
-        return Promise.reject(error.join('; '));
+      const form = new FormData();
+      form.append('token', token);
+      form.append('user', user);
+      form.append('title', title);
+      form.append('message', message);
+      if (device) form.append('device', device);
+
+      // Try to attach image if available
+      if (newListing.image && typeof newListing.image === 'string') {
+        try {
+          const imgRes = await fetch(newListing.image);
+          if (imgRes.ok) {
+            const ab = await imgRes.arrayBuffer();
+            form.append('attachment', new Blob([ab]), 'image.jpg');
+          }
+        } catch {
+          // fail silently, just skip the image
+        }
       }
 
-      return data;
-    })
-    .then(() => {
-      return Promise.resolve();
-    })
-    .catch((error) => {
-      return Promise.reject(error);
-    });
+      const res = await fetch('https://api.pushover.net/1/messages.json', {
+        method: 'POST',
+        body: form,
+      });
+
+      return res.json();
+    }),
+  );
+
+  // Collect errors
+  const errors = results
+    .map((r) => (r.errors && r.errors.length > 0 ? r.errors.join(', ') : null))
+    .filter((e) => e !== null);
+
+  if (errors.length > 0) {
+    return Promise.reject(errors.join('; '));
+  }
+
+  return results;
 };
 
 export const config = {

lib/notification/adapter/sendGrid.js

@@ -1,24 +1,53 @@
 import sgMail from '@sendgrid/mail';
 import { markdown2Html } from '../../services/markdown.js';
+import { normalizeImageUrl } from '../../utils.js';
+
+const mapListings = (serviceName, jobKey, listings) =>
+  listings.map((l) => {
+    const image = normalizeImageUrl(l.image);
+    return {
+      title: l.title || '',
+      link: l.link || '',
+      address: l.address || '',
+      size: l.size || '',
+      price: l.price || '',
+      image,
+      hasImage: Boolean(image),
+      // optional plain text snippet
+      snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
+      serviceName,
+      jobKey,
+    };
+  });
+
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
+
   sgMail.setApiKey(apiKey);
+
+  const to = receiver
+    .trim()
+    .split(',')
+    .map((r) => r.trim())
+    .filter(Boolean);
+
+  const listings = mapListings(serviceName, jobKey, newListings);
+
   const msg = {
     templateId,
-    to: receiver
-      .trim()
-      .split(',')
-      .map((r) => r.trim()),
+    to,
     from,
     subject: `Job ${jobKey} | Service ${serviceName} found ${newListings.length} new listing(s)`,
     dynamic_template_data: {
       serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
       numberOfListings: newListings.length,
-      listings: newListings,
+      listings,
     },
   };
+
   return sgMail.send(msg);
 };
+
 export const config = {
   id: 'sendgrid',
   name: 'SendGrid',

lib/notification/adapter/slack.js

@@ -1,43 +1,61 @@
 import Slack from 'slack';
 import { markdown2Html } from '../../services/markdown.js';
-const msg = Slack.chat.postMessage;
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
-  return newListings.map((payload) =>
-    msg({
-      token,
-      channel,
-      text: `*(${serviceName} - ${jobKey})* - ${payload.title}`,
-      attachments: [
-        {
-          fallback: payload.title,
-          color: '#36a64f',
-          title: 'Link to Exposé',
-          title_link: payload.link,
-          fields: [
-            {
-              title: 'Price',
-              value: payload.price,
-              short: false,
-            },
-            {
-              title: 'Size',
-              value: payload.size,
-              short: false,
-            },
-            {
-              title: 'Address',
-              value: payload.address,
-              short: false,
-            },
-          ],
-          footer: 'Powered by Fredy',
-          ts: new Date().getTime() / 1000,
-        },
+import { normalizeImageUrl } from '../../utils.js';
+
+const buildBlocks = (serviceName, jobKey, p) => {
+  const blocks = [
+    {
+      type: 'header',
+      text: { type: 'plain_text', text: `New Listing from ${serviceName} (${jobKey})`, emoji: false },
+    },
+    {
+      type: 'section',
+      text: { type: 'mrkdwn', text: `*<${p.link}|${p.title}>*` },
+    },
+    {
+      type: 'section',
+      fields: [
+        { type: 'mrkdwn', text: `*Price*\n${p.price ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Size*\n${p.size ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Address*\n${p.address ?? 'n/a'}` },
       ],
-    }),
+    },
+  ];
+
+  const img = normalizeImageUrl(p.image);
+  if (img) {
+    blocks.push({
+      type: 'image',
+      image_url: img,
+      alt_text: p.title || 'listing image',
+    });
+  }
+
+  blocks.push({
+    type: 'context',
+    elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
+  });
+
+  return blocks;
+};
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { token, channel } = notificationConfig.find((a) => a.id === config.id).fields;
+
+  return Promise.allSettled(
+    newListings.map((p) =>
+      Slack.chat.postMessage({
+        token,
+        channel,
+        text: `${serviceName} ${jobKey}: ${p.title}`,
+        blocks: buildBlocks(serviceName, jobKey, p),
+        unfurl_links: false,
+        unfurl_media: false,
+      }),
+    ),
   );
 };
+
 export const config = {
   id: 'slack',
   name: 'Slack',

lib/notification/adapter/slack.md

@@ -1,5 +1,4 @@
 ### Slack Adapter
+IMPORTANT:
+Don't use this adapter anymore, it is outdated and only here for backwards compatability reasons. Use the new Slack Adapter with webhooks!
 
-In order to use [Slack](https://slack.com), you need to create an account. When done, you need to create a new App in your workspace. Give it the permission `chat:write:bot` and `chat:write:user`.
-
-Now you need to create a user token and a channel. Make sure the bot is installed to this channel.

lib/notification/adapter/slack_with_webhooks.js

@@ -0,0 +1,79 @@
+import fetch from 'node-fetch';
+import { markdown2Html } from '../../services/markdown.js';
+import { normalizeImageUrl } from '../../utils.js';
+
+const buildBlocks = (serviceName, jobKey, p) => {
+  const blocks = [
+    {
+      type: 'header',
+      text: { type: 'plain_text', text: `New Listing from ${serviceName} (${jobKey})`, emoji: false },
+    },
+    {
+      type: 'section',
+      text: { type: 'mrkdwn', text: `*<${p.link}|${p.title}>*` },
+    },
+    {
+      type: 'section',
+      fields: [
+        { type: 'mrkdwn', text: `*Price*\n${p.price ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Size*\n${p.size ?? 'n/a'}` },
+        { type: 'mrkdwn', text: `*Address*\n${p.address ?? 'n/a'}` },
+      ],
+    },
+  ];
+
+  const img = normalizeImageUrl(p.image);
+  if (img) {
+    blocks.push({
+      type: 'image',
+      image_url: img,
+      alt_text: p.title || 'listing image',
+    });
+  }
+
+  blocks.push({
+    type: 'context',
+    elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
+  });
+
+  return blocks;
+};
+
+const postJson = (url, body) =>
+  fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body,
+  });
+
+export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const adapter = notificationConfig.find((a) => a.id === config.id);
+  const webhookUrl = adapter?.fields?.webhookUrl;
+  if (!webhookUrl) return Promise.resolve([]);
+
+  const promises = newListings.map((p) => {
+    const body = JSON.stringify({
+      text: `${serviceName} ${jobKey}: ${p.title}`,
+      blocks: buildBlocks(serviceName, jobKey, p),
+      unfurl_links: false,
+      unfurl_media: false,
+    });
+    return postJson(webhookUrl, body);
+  });
+
+  return Promise.allSettled(promises);
+};
+
+export const config = {
+  id: 'slack_with_webhooks',
+  name: 'Slack with Webhooks',
+  readme: markdown2Html('lib/notification/adapter/slack_with_webhooks.md'),
+  description: 'Fredy will send new listings to the slack channel of your choice..',
+  fields: {
+    webhookUrl: {
+      type: 'text',
+      label: 'Webhook-Url',
+      description: 'The Url of the Webhook to send messages to.',
+    },
+  },
+};

lib/notification/adapter/slack_with_webhooks.md

@@ -0,0 +1,6 @@
+### Slack Adapter
+
+IMPORTANT:
+This is the new version of the Slack adapter. I strongly encourage you to use it, the old version is now unmaintained and only kept due to backwards compatability reasons.
+
+In order to use [Slack](https://slack.com), you need to create an account. When done, create a new channel and add the Webhook integration to that channel. Copy the webhook url. That's it.

lib/notification/adapter/sqlite.js

@@ -2,7 +2,19 @@ import { markdown2Html } from '../../services/markdown.js';
 import Database from 'better-sqlite3';
 export const send = ({ serviceName, newListings, jobKey }) => {
   const db = new Database('db/listings.db');
-  const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
+  const fields = [
+    'serviceName',
+    'jobKey',
+    'id',
+    'size',
+    'rooms',
+    'price',
+    'address',
+    'title',
+    'link',
+    'description',
+    'image',
+  ];
   db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
   const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
   newListings.map((listing) => {

lib/notification/adapter/telegram.js

@@ -2,107 +2,97 @@ import { markdown2Html } from '../../services/markdown.js';
 import { getJob } from '../../services/storage/jobStorage.js';
 import fetch from 'node-fetch';
 import pThrottle from 'p-throttle';
+import { normalizeImageUrl } from '../../utils.js';
 
-const MAX_ENTITIES_PER_CHUNK = 8;
 const RATE_LIMIT_INTERVAL = 1000;
 const chatThrottleMap = new Map();
 
 function cleanupOldThrottles() {
   const now = Date.now();
-  const maxAge = RATE_LIMIT_INTERVAL + 1000; // adding extra second
+  const maxAge = RATE_LIMIT_INTERVAL + 1000;
   const toBeDeleted = [];
-
   for (const [chatId, chatThrottle] of chatThrottleMap.entries()) {
-    if (now - chatThrottle.lastUsedAt > maxAge) {
-      toBeDeleted.push(chatId);
-    }
-  }
-
-  for (const chatId of toBeDeleted) {
-    chatThrottleMap.delete(chatId);
+    if (now - chatThrottle.lastUsedAt > maxAge) toBeDeleted.push(chatId);
   }
+  for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId);
 }
 
-/**
- * Returns a throttled async function for sending messages to a specific chat.
- * Telegram enforces a rate limit of 1 message per second per chat (chatId).
- *
- * @param {number} chatId - The chat ID to throttle messages for.
- * @param {Function} fn - The async function to throttle (should send the message).
- * @returns {Function} Throttled async function for sending messages.
- */
 function getThrottled(chatId, call) {
   cleanupOldThrottles();
-
   const now = Date.now();
   const chatThrottle = chatThrottleMap.get(chatId);
-
   if (chatThrottle) {
     chatThrottle.lastUsedAt = now;
     return chatThrottle.throttled;
   }
-
-  // Create new throttled function
-  const newThrottle = {
-    lastUsedAt: now,
-    throttled: pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call),
-  };
-  chatThrottleMap.set(chatId, newThrottle);
-  return newThrottle.throttled;
+  const throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call);
+  chatThrottleMap.set(chatId, { lastUsedAt: now, throttled });
+  return throttled;
 }
 
-/**
- * splitting an array into chunks because Telegram only allows for messages up to
- * 4096 chars, thus we have to split messages into chunks
- * @param inputArray
- * @param perChunk
- */
-const arrayChunks = (inputArray, perChunk) =>
-  inputArray.reduce((all, one, i) => {
-    const ch = Math.floor(i / perChunk);
-    all[ch] = [].concat(all[ch] || [], one);
-    return all;
-  }, []);
-function shorten(str, len = 30) {
-  return str.length > len ? str.substring(0, len) + '...' : str;
+function shorten(str, len = 90) {
+  if (!str) return '';
+  return str.length > len ? str.substring(0, len).trim() + '...' : str;
 }
+
+function escapeHtml(s = '') {
+  return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function buildCaption(jobName, serviceName, o) {
+  const title = shorten((o.title || '').replace(/\*/g, ''), 90);
+  const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
+  return `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n<a href='${escapeHtml(
+    o.link || '',
+  )}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}`.slice(0, 1024);
+}
+
+function buildText(jobName, serviceName, o) {
+  const title = shorten((o.title || '').replace(/\*/g, ''), 90);
+  const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
+  return (
+    `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n` +
+    `<a href='${escapeHtml(o.link || '')}'><b>${escapeHtml(title)}</b></a>\n` +
+    `${escapeHtml(meta)}`
+  );
+}
+
 export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
   const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
   const job = getJob(jobKey);
   const jobName = job == null ? jobKey : job.name;
-  const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
 
-  const getThrottledSend = getThrottled(chatId, async function (body) {
-    await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+  const throttledCall = getThrottled(chatId, async function (endpoint, body) {
+    await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, {
       method: 'post',
       body: JSON.stringify(body),
       headers: { 'Content-Type': 'application/json' },
     });
   });
 
-  const promises = chunks.map((chunk) => {
-    const messageParagraphs = [];
+  const promises = newListings.map(async (o) => {
+    const img = normalizeImageUrl(o.image);
 
-    messageParagraphs.push(`<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:`);
-    messageParagraphs.push(
-      ...chunk.map(
-        (o) =>
-          `<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
-          [o.address, o.price, o.size].join(' | '),
-      ),
-    );
+    if (img) {
+      return throttledCall('sendPhoto', {
+        chat_id: chatId,
+        photo: img,
+        caption: buildCaption(jobName, serviceName, o),
+        parse_mode: 'HTML',
+      });
+    }
 
-    const body = {
+    return throttledCall('sendMessage', {
       chat_id: chatId,
-      text: messageParagraphs.join('\n\n'),
+      text: buildText(jobName, serviceName, o),
       parse_mode: 'HTML',
       disable_web_page_preview: true,
-    };
-
-    return getThrottledSend(body);
+    });
   });
+
   return Promise.all(promises);
 };
+
 export const config = {
   id: 'telegram',
   name: 'Telegram',

lib/notification/emailTemplate/mailjet.hbs

@@ -0,0 +1,123 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <title>Listings</title>
+    <style type="text/css">
+        body { margin:0; padding:0; background:#000000; }
+        table { border-collapse:collapse; }
+        img { border:0; outline:none; text-decoration:none; display:block; }
+        a { text-decoration:none; }
+        .container { width:100%; max-width:640px; margin:0 auto; }
+        .card { background:#111111; border:1px solid #222222; border-radius:8px; overflow:hidden; }
+        .divider { height:2px; line-height:2px; font-size:0; background:#00dc73; }
+        .h1 { font:700 18px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .h2 { font:700 16px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .p { font:400 14px/1.6 Arial, Helvetica, sans-serif; color:#d9d9d9; margin:0; }
+        .meta { font:400 13px/1.5 Arial, Helvetica, sans-serif; color:#bfbfbf; }
+        .btn { background:#00dc73; color:#0b0b0b; font:700 14px/1 Arial, Helvetica, sans-serif; padding:12px 18px; border-radius:6px; display:inline-block; }
+        .sp-8 { height:8px; line-height:8px; font-size:0; }
+        .sp-12 { height:12px; line-height:12px; font-size:0; }
+        .sp-16 { height:16px; line-height:16px; font-size:0; }
+        .sp-20 { height:20px; line-height:20px; font-size:0; }
+        .sp-24 { height:24px; line-height:24px; font-size:0; }
+        @media screen and (max-width:480px){
+            .container { width:100% !important; }
+            .stack { display:block !important; width:100% !important; }
+        }
+    </style>
+</head>
+<body style="background:#000000;">
+<table role="presentation" width="100%" bgcolor="#000000">
+    <tr>
+        <td align="center">
+            <table role="presentation" class="container" width="640">
+                <tr><td class="sp-20"></td></tr>
+
+                <tr>
+                    <td align="center">
+                        <h1 class="h1" style="text-align:center;">
+                            Service {{serviceName}} found {{numberOfListings}} new listings
+                        </h1>
+                    </td>
+                </tr>
+
+                <tr><td class="sp-12"></td></tr>
+                <tr><td class="divider"></td></tr>
+                <tr><td class="sp-16"></td></tr>
+
+                {{#each listings}}
+                    <tr>
+                        <td>
+                            <table role="presentation" class="card" width="100%">
+                                {{#if this.hasImage}}
+                                    <tr>
+                                        <td>
+                                            <a href="{{this.link}}" target="_blank">
+                                                <img src="cid:{{this.imageCid}}" alt="{{this.title}}" width="640"
+                                                     style="width:100%; height:auto; background:#1a1a1a;" />
+                                            </a>
+                                        </td>
+                                    </tr>
+                                {{/if}}
+
+                                <tr>
+                                    <td style="padding:16px 18px 0 18px;">
+                                        <a href="{{this.link}}" target="_blank" style="color:#ffffff;">
+                                            <h2 class="h2">{{this.title}}</h2>
+                                        </a>
+                                    </td>
+                                </tr>
+
+                                <tr><td class="sp-8"></td></tr>
+
+                                <tr>
+                                    <td style="padding:0 18px;">
+                                        <table role="presentation" width="100%">
+                                            <tr>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-right:8px;">
+                                                    <p class="meta"><strong>Price</strong><br/>{{#if this.price}}{{this.price}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-left:8px;">
+                                                    <p class="meta"><strong>Size</strong><br/>{{#if this.size}}{{this.size}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                            <tr><td class="sp-8"></td><td class="sp-8"></td></tr>
+                                            <tr>
+                                                <td colspan="2">
+                                                    <p class="meta"><strong>Address</strong><br/>{{#if this.address}}{{this.address}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                        </table>
+                                    </td>
+                                </tr>
+
+                                <tr><td class="sp-16"></td></tr>
+
+                                <tr>
+                                    <td align="left" style="padding:0 18px 18px 18px;">
+                                        <a href="{{this.link}}" class="btn" target="_blank">View Listing</a>
+                                    </td>
+                                </tr>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr><td class="sp-24"></td></tr>
+                {{/each}}
+
+                <tr><td class="divider"></td></tr>
+                <tr><td class="sp-16"></td></tr>
+                <tr>
+                    <td align="center">
+                        <p class="p" style="color:#9f9f9f;">Powered by Fredy</p>
+                    </td>
+                </tr>
+                <tr><td class="sp-20"></td></tr>
+            </table>
+        </td>
+    </tr>
+</table>
+</body>
+</html>

lib/notification/emailTemplate/template.hbs

@@ -1,237 +1,131 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html data-editor-version="2" class="sg-campaigns" xmlns="http://www.w3.org/1999/xhtml"><head>
-    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1, maximum-scale=1">
-    <!--[if !mso]><!-->
-    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
-    <!--<![endif]-->
-    <!--[if (gte mso 9)|(IE)]>
-    <xml>
-        <o:OfficeDocumentSettings>
-            <o:AllowPNG/>
-            <o:PixelsPerInch>96</o:PixelsPerInch>
-        </o:OfficeDocumentSettings>
-    </xml>
-    <![endif]-->
-    <!--[if (gte mso 9)|(IE)]>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <title>Fredy found some new listings</title>
     <style type="text/css">
-        body {width: 600px;margin: 0 auto;}
-        table {border-collapse: collapse;}
-        table, td {mso-table-lspace: 0pt;mso-table-rspace: 0pt;}
-        img {-ms-interpolation-mode: bicubic;}
-    </style>
-    <![endif]-->
-    <style type="text/css">
-        body, p, div {
-            font-family: arial,helvetica,sans-serif;
-            font-size: 14px;
-        }
-        body {
-            color: #000000;
-        }
-        body a {
-            color: #42ee99;
-            text-decoration: none;
-        }
-        p { margin: 0; padding: 0; }
-        table.wrapper {
-            width:100% !important;
-            table-layout: fixed;
-            -webkit-font-smoothing: antialiased;
-            -webkit-text-size-adjust: 100%;
-            -moz-text-size-adjust: 100%;
-            -ms-text-size-adjust: 100%;
-        }
-        img.max-width {
-            max-width: 100% !important;
-        }
-        .column.of-2 {
-            width: 50%;
-        }
-        .column.of-3 {
-            width: 33.333%;
-        }
-        .column.of-4 {
-            width: 25%;
-        }
-        @media screen and (max-width:480px) {
-            .preheader .rightColumnContent,
-            .footer .rightColumnContent {
-                text-align: left !important;
-            }
-            .preheader .rightColumnContent div,
-            .preheader .rightColumnContent span,
-            .footer .rightColumnContent div,
-            .footer .rightColumnContent span {
-                text-align: left !important;
-            }
-            .preheader .rightColumnContent,
-            .preheader .leftColumnContent {
-                font-size: 80% !important;
-                padding: 5px 0;
-            }
-            table.wrapper-mobile {
-                width: 100% !important;
-                table-layout: fixed;
-            }
-            img.max-width {
-                height: auto !important;
-                max-width: 100% !important;
-            }
-            a.bulletproof-button {
-                display: block !important;
-                width: auto !important;
-                font-size: 80%;
-                padding-left: 0 !important;
-                padding-right: 0 !important;
-            }
-            .columns {
-                width: 100% !important;
-            }
-            .column {
-                display: block !important;
-                width: 100% !important;
-                padding-left: 0 !important;
-                padding-right: 0 !important;
-                margin-left: 0 !important;
-                margin-right: 0 !important;
-            }
+        body { margin:0; padding:0; background:#000000; }
+        table { border-collapse:collapse; }
+        img { border:0; outline:none; text-decoration:none; display:block; }
+        a { text-decoration:none; }
+        .container { width:100%; max-width:640px; margin:0 auto; }
+        .card { background:#111111; border:1px solid #222222; border-radius:8px; overflow:hidden; }
+        .divider { height:2px; line-height:2px; font-size:0; background:#00dc73; }
+        .h1 { font:700 18px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .h2 { font:700 16px/1.4 Arial, Helvetica, sans-serif; color:#ffffff; margin:0; }
+        .p { font:400 14px/1.6 Arial, Helvetica, sans-serif; color:#d9d9d9; margin:0; }
+        .meta { font:400 13px/1.5 Arial, Helvetica, sans-serif; color:#bfbfbf; }
+        .btn { background:#00dc73; color:#0b0b0b; font:700 14px/1 Arial, Helvetica, sans-serif; padding:12px 18px; border-radius:6px; display:inline-block; }
+        .sp-8 { height:8px; line-height:8px; font-size:0; }
+        .sp-12 { height:12px; line-height:12px; font-size:0; }
+        .sp-16 { height:16px; line-height:16px; font-size:0; }
+        .sp-20 { height:20px; line-height:20px; font-size:0; }
+        .sp-24 { height:24px; line-height:24px; font-size:0; }
+        @media screen and (max-width:480px){
+            .container { width:100% !important; }
+            .stack { display:block !important; width:100% !important; }
         }
     </style>
-    <!--user entered Head Start-->
-
-    <!--End Head user entered-->
 </head>
-<body>
-<center class="wrapper" data-link-color="#42ee99" data-body-style="font-size:14px; font-family:arial,helvetica,sans-serif; color:#000000; background-color:#000000;">
-    <div class="webkit">
-        <table cellpadding="0" cellspacing="0" border="0" width="100%" class="wrapper" bgcolor="#000000">
-            <tbody><tr>
-                <td valign="top" bgcolor="#000000" width="100%">
-                    <table width="100%" role="content-container" class="outer" align="center" cellpadding="0" cellspacing="0" border="0">
-                        <tbody><tr>
-                            <td width="100%">
-                                <table width="100%" cellpadding="0" cellspacing="0" border="0">
-                                    <tbody><tr>
+<body style="background:#000000;">
+<table role="presentation" width="100%" bgcolor="#000000">
+    <tr>
+        <td align="center">
+            <table role="presentation" class="container" width="640">
+                <tr><td class="sp-20"></td></tr>
+                <tr>
+                    <td align="center">
+                        <table role="presentation" width="100%">
+                            <tr>
+                                <td align="center">
+                                    <h1 class="h1" style="text-align:center;">
+                                        Service {{serviceName}} found {{numberOfListings}} new listings
+                                    </h1>
+                                </td>
+                            </tr>
+                            <tr><td class="sp-12"></td></tr>
+                            <tr><td class="divider"></td></tr>
+                            <tr><td class="sp-16"></td></tr>
+                        </table>
+                    </td>
+                </tr>
+
+                {{#each listings}}
+                    <tr>
+                        <td>
+                            <table role="presentation" class="card" width="100%">
+                                {{#if this.hasImage}}
+                                    <tr>
                                         <td>
-                                            <!--[if mso]>
-                                            <center>
-                                                <table><tr><td width="600">
-                                            <![endif]-->
-                                            <table width="100%" cellpadding="0" cellspacing="0" border="0" style="width:100%; max-width:600px;" align="center">
-                                                <tbody><tr>
-                                                    <td role="modules-container" style="padding:0px 0px 0px 0px; color:#000000; text-align:left;" bgcolor="#FFFFFF" width="100%" align="left"><table class="module preheader preheader-hide" role="module" data-type="preheader" border="0" cellpadding="0" cellspacing="0" width="100%" style="display: none !important; mso-hide: all; visibility: hidden; opacity: 0; color: transparent; height: 0; width: 0;">
-                                                    </table><table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="vB9TDziyvx65CC2nx3oyRH">
-                                                        <tbody><tr>
-                                                            <td style="padding:0px 0px 20px 0px;" role="module-content" bgcolor="#000000">
-                                                            </td>
-                                                        </tr>
-                                                        </tbody></table><table class="wrapper" role="module" data-type="image" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="uXsDxMnn1bRMmDcX8NB6rW">
-                                                    </table><table class="module" role="module" data-type="text" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="hL6wjQ2qknNd5qDwT1p7Up">
-                                                        <tbody><tr>
-                                                            <td style="background-color:#000000; padding:10px 20px 10px 20px; line-height:40px; text-align:justify;" height="100%" valign="top" bgcolor="#000000"><div><h1 style="text-align: center"><span style="color: #ffffff; font-size: 14px; font-family: verdana,geneva,sans-serif"><strong>Service {{serviceName}} found {{numberOfListings}} new listing(s)!</strong></span></h1><div></div></div></td>
-                                                        </tr>
-                                                        </tbody></table>
-
-                                                        <table border="0" cellpadding="0" cellspacing="0" align="center" width="100%" height="1px" style="line-height:3px; font-size:3px;">
-                                                            <tbody><tr>
-                                                                <td style="padding:0px 0px 1px 0px;" bgcolor="#42ee99"></td>
-                                                            </tr>
-                                                            </tbody></table>
-                                                    </td>
-                                                </tr>
-                                                </tbody>
-                                            </table>
-
-                                            <table class="module" role="module" data-type="text" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="qk51Jjn4bm3rn2Yb31Dxzb">
-                                                <tbody>
-                                                {{#each listings}}
-                                                    <tr>
-                                                        <td style="padding:50px 50px 10px 50px; line-height:22px; text-align:center; color:white" bgcolor="#000000" height="100%" valign="top">
-                                                            <div>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;"><b>{{this.title}}</b></span>
-                                                                <br/>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;">Size: {{#if this.size}}{{this.size}}{{else}}unknown{{/if}}</span>
-                                                                <br/>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;">Price: {{#if this.price}}{{this.price}}{{else}}unknown{{/if}}</span>
-                                                                <br/>
-                                                                <span style="font-size: 12px; font-family: verdana,geneva,sans-serif; color: white;">{{#if this.address}}{{this.address}}{{else}}unknown{{/if}}</span>
-                                                                <br/>
-                                                                <a href="{{this.link}}" target="_blank" style="color:#00dc73; font-size:13px">{{this.link}}</a>
-                                                                <br/>
-                                                                <span style="color: white;">---------------------------</span>
-                                                            </div>
-                                                        </td>
-                                                    </tr>
-                                                {{/each}}
-                                                </tbody>
-                                            </table>
-
-                                            <table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="2ga5f7koD5ApvUfnqUK6aT">
-                                                <tbody><tr>
-                                                    <td style="padding:0px 0px 30px 0px;" role="module-content" bgcolor="#000000">
-                                                    </td>
-                                                </tr>
-                                                </tbody></table>
-                                            <table class="module" role="module" data-type="divider" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="c3nRrjMndqXf1snYDFPSF9">
-                                                <tbody><tr>
-                                                    <td style="padding:0px 0px 0px 0px;" role="module-content" height="100%" valign="top" bgcolor="#000000">
-                                                        <table border="0" cellpadding="0" cellspacing="0" align="center" width="100%" height="2px" style="line-height:1px; font-size:2px;">
-                                                            <tbody><tr>
-                                                                <td style="padding:0px 0px 2px 0px;" bgcolor="#42ee99"></td>
-                                                            </tr>
-                                                            </tbody></table>
-                                                    </td>
-                                                </tr>
-                                                </tbody>
-                                            </table>
-                                            <table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="pa9PeYjCEFyByuP5878Sd2">
-                                                <tbody><tr>
-                                                    <td style="padding:0px 0px 30px 0px;" role="module-content" bgcolor="#000000">
-                                                    </td>
-                                                </tr>
-                                                </tbody></table>
-
-                                            <table class="module" role="module" data-type="social" align="center" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="n7FceQWVnLmounEt32B1gj">
-                                                <tbody>
-                                                <tr>
-                                                    <td valign="top" style="padding:0px 0px 0px 0px; font-size:6px; line-height:10px; background-color:#000000;" align="center">
-                                                        <table align="center">
-                                                            <tbody>
-                                                            <tr><td style="padding: 0px 5px;">
-                                                                <a href="https://github.com/orangecoding/fredy" target="_blank" alt="Fredy" title="Powered by Fredy" style="color:#00dc73; font-size:17px">
-                                                                    Powered by Fredy
-                                                                </a>
-                                                            </td></tr>
-                                                            </tbody>
-                                                        </table>
-                                                    </td>
-                                                </tr>
-                                                </tbody>
-                                            </table></table><table class="module" role="module" data-type="spacer" border="0" cellpadding="0" cellspacing="0" width="100%" style="table-layout: fixed;" data-muid="35xFa9abxGTBYt9yR9BeQ2">
-                                <tbody><tr>
-                                    <td style="padding:0px 0px 30px 0px;" role="module-content" bgcolor="#000000">
+                                            <a href="{{this.link}}" target="_blank">
+                                                <img src="{{this.image}}" alt="{{this.title}}" width="640" style="width:100%;height:auto;background:#1a1a1a;" />
+                                            </a>
+                                        </td>
+                                    </tr>
+                                {{/if}}
+                                <tr>
+                                    <td style="padding:16px 18px 0 18px;">
+                                        <a href="{{this.link}}" target="_blank" style="color:#ffffff;">
+                                            <h2 class="h2">{{this.title}}</h2>
+                                        </a>
                                     </td>
                                 </tr>
-                                </tbody></table></td>
-                        </tr>
-                        </tbody></table>
-                    <!--[if mso]>
-                    </td>
-                    </tr>
-                    </table>
-                    </center>
+                                <tr><td class="sp-8"></td></tr>
+                                <tr>
+                                    <td style="padding:0 18px;">
+                                        <table role="presentation" width="100%">
+                                            <tr>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-right:8px;">
+                                                    <p class="meta"><strong>Price</strong><br/>{{#if this.price}}{{this.price}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                                <td class="stack" style="vertical-align:top; width:50%; padding-left:8px;">
+                                                    <p class="meta"><strong>Size</strong><br/>{{#if this.size}}{{this.size}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                            <tr><td class="sp-8"></td><td class="sp-8"></td></tr>
+                                            <tr>
+                                                <td colspan="2">
+                                                    <p class="meta"><strong>Address</strong><br/>{{#if this.address}}{{this.address}}{{else}}unknown{{/if}}</p>
+                                                </td>
+                                            </tr>
+                                        </table>
+                                    </td>
+                                </tr>
+                                <tr><td class="sp-16"></td></tr>
+                                <tr>
+                                    <td align="left" style="padding:0 18px 18px 18px;">
+                                        <!--[if mso]>
+                    <v:roundrect xmlns:v="urn:schemas-microsoft-com:vml" href="{{this.link}}" arcsize="8%" strokecolor="#00dc73" strokeweight="0" fillcolor="#00dc73" style="height:40px;v-text-anchor:middle;width:180px;">
+                      <w:anchorlock/>
+                      <center style="color:#0b0b0b;font-family:Arial;font-size:14px;font-weight:bold;">
+                        View Listing
+                      </center>
+                    </v:roundrect>
                     <![endif]-->
-                </td>
-            </tr>
-            </tbody></table>
-        </td>
-        </tr>
-        </tbody></table>
-        </td>
-        </tr>
-        </tbody></table>
-    </div>
-</center>
+                                        <!--[if !mso]><!-- -->
+                                        <a href="{{this.link}}" class="btn" target="_blank">View Listing</a>
+                                        <!--<![endif]-->
+                                    </td>
+                                </tr>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr><td class="sp-24"></td></tr>
+                {{/each}}
 
-
-</body></html>
+                <tr><td class="divider"></td></tr>
+                <tr><td class="sp-16"></td></tr>
+                <tr>
+                    <td align="center">
+                        <p class="p" style="color:#9f9f9f;">Powered by Fredy</p>
+                    </td>
+                </tr>
+                <tr><td class="sp-20"></td></tr>
+            </table>
+        </td>
+    </tr>
+</table>
+</body>
+</html>

lib/provider/einsAImmobilien.js

@@ -2,10 +2,12 @@ import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
 
 function normalize(o) {
-  const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
+  const baseUrl = 'https://www.1a-immobilienmarkt.de';
+  const link = `${baseUrl}/expose/${o.id}.html`;
   const price = normalizePrice(o.price);
   const id = buildHash(o.id, price);
-  return Object.assign(o, { id, price, link });
+  const image = baseUrl + o.image;
+  return Object.assign(o, { id, price, link, image });
 }
 
 /**
@@ -41,6 +43,7 @@ const config = {
     price: '.inner_object_data .single_data_price | removeNewline | trim',
     size: '.tabelle .tabelle_inhalt_infos .single_data_box  | removeNewline | trim',
     title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
+    image: '.inner_object_pic img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immobilienDe.js

@@ -1,26 +1,34 @@
 import utils, { buildHash } from '../utils.js';
+
 let appliedBlackList = [];
+
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
 }
+
 function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
+
 function normalize(o) {
+  const baseUrl = 'https://www.immobilien.de';
   const size = o.size || 'N/A m²';
   const price = o.price || 'N/A €';
   const title = o.title || 'No title available';
   const address = o.address || 'No address available';
   const shortLink = shortenLink(o.link);
-  const link = `https://www.immobilien.de/${shortLink}`;
+  const link = `${baseUrl}/${shortLink}`;
+  const image = baseUrl + o.image;
   const id = buildHash(parseId(shortLink), o.price);
-  return Object.assign(o, { id, price, size, title, address, link });
+  return Object.assign(o, { id, price, size, title, address, link, image });
 }
+
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
 }
+
 const config = {
   url: null,
   crawlContainer: '._ref',
@@ -34,6 +42,7 @@ const config = {
     description: '.list_entry .description | trim',
     link: '@href',
     address: '.list_entry .place',
+    image: '.list_entry img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immonet.js

@@ -33,6 +33,7 @@ const config = {
     price: 'div[data-testid="cardmfe-price-testid"] | trim',
     size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
     address: 'div[data-testid="cardmfe-description-box-address"] | trim',
+    image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immoscout.js

@@ -62,6 +62,7 @@ async function getListings(url) {
     .map((expose) => {
       const item = expose.item;
       const [price, size] = item.attributes;
+      const { preview: image } = item.titlePicture;
       return {
         id: item.id,
         price: price?.value,
@@ -69,6 +70,7 @@ async function getListings(url) {
         title: item.title,
         link: `${metaInformation.baseUrl}expose/${item.id}`,
         address: item.address?.line,
+        image,
       };
     });
 }

lib/provider/immoswp.js

@@ -31,6 +31,7 @@ const config = {
     title: '.js-item-title-link@title | trim',
     link: '.ci-search-result__link@href',
     description: '.js-show-more-item-sm | removeNewline | trim',
+    image: 'img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immowelt.js

@@ -26,6 +26,7 @@ const config = {
     title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
     link: 'a@href',
     address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
+    image: 'div[data-testid="cardMfe-card-pictureBox-opacity"] img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/kleinanzeigen.js

@@ -32,6 +32,7 @@ const config = {
     link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
     description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
     address: '.aditem-main--top--left | trim | removeNewline',
+    image: 'img@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/neubauKompass.js

@@ -29,6 +29,7 @@ const config = {
     link: 'a@href',
     address: '.nbk-project-card__description | removeNewline | trim',
     price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
+    image: '.nbk-project-card__image@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/wgGesucht.js

@@ -5,7 +5,8 @@ let appliedBlackList = [];
 function normalize(o) {
   const id = buildHash(o.id, o.price);
   const link = `https://www.wg-gesucht.de${o.link}`;
-  return Object.assign(o, { id, link });
+  const image = o.image != null ? o.image.replace('small', 'large') : null;
+  return Object.assign(o, { id, link, image });
 }
 
 function applyBlacklist(o) {
@@ -26,6 +27,7 @@ const config = {
     size: '.middle .text-right |removeNewline |trim',
     title: '.truncate_title a |removeNewline |trim',
     link: '.truncate_title a@href',
+    image: '.img-responsive@src',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/utils.js

@@ -68,9 +68,29 @@ export async function refreshConfig() {
     console.error('Error reading config file', error);
   }
 }
+
+const RE_GT = />/g;
+const RE_WEBP = /\/format\/webp/gi;
+const RE_EXT = /\.(jpe?g|png|gif)(\?.*)?$/i;
+const HTTPS_PREFIX = 'https://';
+
+const normalizeImageUrl = (url) => {
+  if (typeof url !== 'string' || url.length === 0) return null;
+
+  let u = url.trim().replace(RE_GT, '');
+  if (RE_WEBP.test(u)) u = u.replace(RE_WEBP, '/format/jpg');
+  if (!u.startsWith(HTTPS_PREFIX)) return null;
+  if (!RE_EXT.test(u)) {
+    const jpgIdx = u.toLowerCase().lastIndexOf('.jpg');
+    if (jpgIdx > -1) u = u.slice(0, jpgIdx + 4);
+  }
+  return u;
+};
+
 await refreshConfig();
 
 export { isOneOf };
+export { normalizeImageUrl };
 export { inDevMode };
 export { nullOrEmpty };
 export { duringWorkingHoursOrNotSet };

package.json

@@ -9,7 +9,7 @@
     "start:frontend": "vite -m production",
     "start:frontend:dev": "vite",
     "build:frontend": "vite build",
-    "format": "prettier --write .",
+    "format": "prettier --write \"**/*.js\"",
     "format:check": "prettier --check \"**/*.js\"",
     "test": "mocha --loader=esmock --timeout 3000000 test/**/*.test.js",
     "lint": "eslint .",

test/provider/testProvider.json

@@ -24,10 +24,6 @@
     "url": "https://immo.swp.de/suchergebnisse?l=M%C3%BCnchen&r=0km&_multiselect_r=0km&ut=private&t=apartment%3Arental&a=de.muenchen&pf=&pt=&rf=0&rt=0&sf=50&st=&yf=&yt=&ff=&ft=&s=most_recently_updated_first&pa=&o=&ad=&u=",
     "enabled": true
   },
-  "kalaydo": {
-    "url": "https://www.kalaydo.de/immobilien/eigentumswohnung-kaufen/o/duesseldorf/4/?attr_gt_estate_size_living_area=90.0&attr_gt_no_of_rooms=3.5&maxPrice=420000.00&radius=5&resultsPerPage=50&sorting=-date",
-    "enabled": true
-  },
   "kleinanzeigen": {
     "url": "https://www.kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
     "enabled": true