mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Adding images to scraping data (#157)
* Fredy now supports pulling the main Image from the listing and send it together with the usual information
This commit is contained in:
committed by
GitHub
parent
da743c8279
commit
f0b146fd7f
@@ -2,68 +2,120 @@ import mailjet from 'node-mailjet';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import Handlebars from 'handlebars';
|
||||
import fetch from 'node-fetch';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getDirName } from '../../utils.js';
|
||||
import { getDirName, normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const __dirname = getDirName();
|
||||
const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
|
||||
const emailTemplate = Handlebars.compile(template);
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
|
||||
const guessMime = (url) => {
|
||||
const lower = url.split('?')[0].toLowerCase();
|
||||
if (lower.endsWith('.png')) return 'image/png';
|
||||
if (lower.endsWith('.gif')) return 'image/gif';
|
||||
return 'image/jpeg';
|
||||
};
|
||||
|
||||
const toBase64 = async (url) => {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`Fetch failed with status ${res.status} for URL: ${url}`);
|
||||
const ab = await res.arrayBuffer();
|
||||
return Buffer.from(ab).toString('base64');
|
||||
} catch (error) {
|
||||
console.error(`Error fetching image from ${url}:`, error.message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const mapListingsWithCid = async (serviceName, jobKey, listings) => {
|
||||
const out = [];
|
||||
const attachments = [];
|
||||
|
||||
for (let i = 0; i < listings.length; i++) {
|
||||
const l = listings[i] || {};
|
||||
const imgUrl = normalizeImageUrl(l.image);
|
||||
|
||||
const item = {
|
||||
title: l.title || '',
|
||||
link: l.link || '',
|
||||
address: l.address || '',
|
||||
size: l.size || '',
|
||||
price: l.price || '',
|
||||
serviceName,
|
||||
jobKey,
|
||||
hasImage: false,
|
||||
imageCid: '',
|
||||
};
|
||||
|
||||
if (imgUrl) {
|
||||
try {
|
||||
const base64 = await toBase64(imgUrl);
|
||||
const cid = `listing-${i}`;
|
||||
attachments.push({
|
||||
ContentType: guessMime(imgUrl),
|
||||
Filename: `listing-${i}.${imgUrl.split('.').pop().split('?')[0] || 'jpg'}`,
|
||||
Base64Content: base64,
|
||||
ContentID: cid,
|
||||
});
|
||||
item.hasImage = true;
|
||||
item.imageCid = cid;
|
||||
} catch (error) {
|
||||
console.warn(`Skipping image for listing ${i} due to error: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
out.push(item);
|
||||
}
|
||||
|
||||
return { listings: out, attachments };
|
||||
};
|
||||
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
|
||||
(adapter) => adapter.id === config.id,
|
||||
).fields;
|
||||
|
||||
const to = receiver
|
||||
.trim()
|
||||
.split(',')
|
||||
.map((r) => ({
|
||||
Email: r.trim(),
|
||||
}));
|
||||
.map((r) => ({ Email: r.trim() }))
|
||||
.filter((r) => r.Email.length > 0);
|
||||
|
||||
const { listings, attachments } = await mapListingsWithCid(serviceName, jobKey, newListings);
|
||||
|
||||
const html = emailTemplate({
|
||||
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
|
||||
numberOfListings: listings.length,
|
||||
listings,
|
||||
});
|
||||
|
||||
return mailjet
|
||||
.apiConnect(apiPublicKey, apiPrivateKey)
|
||||
.post('send', { version: 'v3.1' })
|
||||
.request({
|
||||
Messages: [
|
||||
{
|
||||
From: {
|
||||
Email: from,
|
||||
Name: 'Fredy',
|
||||
},
|
||||
From: { Email: from, Name: 'Fredy' },
|
||||
To: to,
|
||||
Subject: `Fredy found ${newListings.length} new listings for ${serviceName}`,
|
||||
HTMLPart: emailTemplate({
|
||||
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
|
||||
numberOfListings: newListings.length,
|
||||
listings: newListings,
|
||||
}),
|
||||
Subject: `Fredy found ${listings.length} new listing(s) for ${serviceName}`,
|
||||
HTMLPart: html,
|
||||
InlinedAttachments: attachments,
|
||||
},
|
||||
],
|
||||
});
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'mailjet',
|
||||
name: 'MailJet',
|
||||
description: 'MailJet is being used to send new listings via mail.',
|
||||
readme: markdown2Html('lib/notification/adapter/mailJet.md'),
|
||||
fields: {
|
||||
apiPublicKey: {
|
||||
type: 'text',
|
||||
label: 'Public Api Key',
|
||||
description: 'The public api key needed to access this service.',
|
||||
},
|
||||
apiPrivateKey: {
|
||||
type: 'text',
|
||||
label: 'Private Api Key',
|
||||
description: 'The private api key needed to access this service.',
|
||||
},
|
||||
receiver: {
|
||||
type: 'email',
|
||||
label: 'Receiver Email',
|
||||
description: 'The email address (single one) which Fredy is using to send notifications to.',
|
||||
},
|
||||
from: {
|
||||
type: 'email',
|
||||
label: 'Sender email',
|
||||
description:
|
||||
'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
|
||||
},
|
||||
apiPublicKey: { type: 'text', label: 'Public Api Key' },
|
||||
apiPrivateKey: { type: 'text', label: 'Private Api Key' },
|
||||
receiver: { type: 'email', label: 'Receiver Email' },
|
||||
from: { type: 'email', label: 'Sender email' },
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
### MailJet Adapter
|
||||
|
||||
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from.
|
||||
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
|
||||
|
||||
E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.
|
||||
The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
|
||||
|
||||
If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).
|
||||
If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).
|
||||
|
||||
@@ -1,32 +1,41 @@
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
|
||||
const promises = newListings.map((newListing) => {
|
||||
const message = `
|
||||
Address: ${newListing.address}
|
||||
Size: ${newListing.size.replace(/2m/g, '$m^2$')}
|
||||
Price: ${newListing.price}
|
||||
Link: ${newListing.link}`;
|
||||
return fetch(server, {
|
||||
Address: ${newListing.address}
|
||||
Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
|
||||
Price: ${newListing.price}
|
||||
Link: ${newListing.link}`;
|
||||
|
||||
const headers = {
|
||||
Title: newListing.title,
|
||||
Priority: String(priority),
|
||||
Tags: `${serviceName},${jobName}`,
|
||||
Click: newListing.link,
|
||||
};
|
||||
|
||||
if (newListing.image && typeof newListing.image === 'string') {
|
||||
headers.Attach = normalizeImageUrl(newListing.image);
|
||||
}
|
||||
|
||||
return fetch(`${server}/${topic}`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
topic: topic,
|
||||
message: message,
|
||||
title: newListing.title,
|
||||
tags: [serviceName, jobName],
|
||||
priority: parseInt(priority),
|
||||
click: newListing.link,
|
||||
}),
|
||||
headers,
|
||||
body: message,
|
||||
});
|
||||
});
|
||||
|
||||
return Promise.all(promises);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'ntfy',
|
||||
name: 'ntfy',
|
||||
|
||||
@@ -2,50 +2,55 @@ import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
const promises = newListings.map((newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
|
||||
return fetch('https://api.pushover.net/1/messages.json', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
token: token,
|
||||
user: user,
|
||||
message: message,
|
||||
device: device,
|
||||
title: title,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
return Promise.all(promises)
|
||||
.then((responses) => {
|
||||
// Convert all responses to JSON
|
||||
return Promise.all(responses.map((response) => response.json()));
|
||||
})
|
||||
.then((data) => {
|
||||
// Check for errors in the data
|
||||
const error = data
|
||||
.map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
|
||||
.filter((err) => err !== null);
|
||||
const results = await Promise.all(
|
||||
newListings.map(async (newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
|
||||
|
||||
if (error.length > 0) {
|
||||
// Reject with the combined error messages
|
||||
return Promise.reject(error.join('; '));
|
||||
const form = new FormData();
|
||||
form.append('token', token);
|
||||
form.append('user', user);
|
||||
form.append('title', title);
|
||||
form.append('message', message);
|
||||
if (device) form.append('device', device);
|
||||
|
||||
// Try to attach image if available
|
||||
if (newListing.image && typeof newListing.image === 'string') {
|
||||
try {
|
||||
const imgRes = await fetch(newListing.image);
|
||||
if (imgRes.ok) {
|
||||
const ab = await imgRes.arrayBuffer();
|
||||
form.append('attachment', new Blob([ab]), 'image.jpg');
|
||||
}
|
||||
} catch {
|
||||
// fail silently, just skip the image
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
})
|
||||
.then(() => {
|
||||
return Promise.resolve();
|
||||
})
|
||||
.catch((error) => {
|
||||
return Promise.reject(error);
|
||||
});
|
||||
const res = await fetch('https://api.pushover.net/1/messages.json', {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
});
|
||||
|
||||
return res.json();
|
||||
}),
|
||||
);
|
||||
|
||||
// Collect errors
|
||||
const errors = results
|
||||
.map((r) => (r.errors && r.errors.length > 0 ? r.errors.join(', ') : null))
|
||||
.filter((e) => e !== null);
|
||||
|
||||
if (errors.length > 0) {
|
||||
return Promise.reject(errors.join('; '));
|
||||
}
|
||||
|
||||
return results;
|
||||
};
|
||||
|
||||
export const config = {
|
||||
|
||||
@@ -1,24 +1,53 @@
|
||||
import sgMail from '@sendgrid/mail';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const mapListings = (serviceName, jobKey, listings) =>
|
||||
listings.map((l) => {
|
||||
const image = normalizeImageUrl(l.image);
|
||||
return {
|
||||
title: l.title || '',
|
||||
link: l.link || '',
|
||||
address: l.address || '',
|
||||
size: l.size || '',
|
||||
price: l.price || '',
|
||||
image,
|
||||
hasImage: Boolean(image),
|
||||
// optional plain text snippet
|
||||
snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
|
||||
serviceName,
|
||||
jobKey,
|
||||
};
|
||||
});
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
|
||||
sgMail.setApiKey(apiKey);
|
||||
|
||||
const to = receiver
|
||||
.trim()
|
||||
.split(',')
|
||||
.map((r) => r.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const listings = mapListings(serviceName, jobKey, newListings);
|
||||
|
||||
const msg = {
|
||||
templateId,
|
||||
to: receiver
|
||||
.trim()
|
||||
.split(',')
|
||||
.map((r) => r.trim()),
|
||||
to,
|
||||
from,
|
||||
subject: `Job ${jobKey} | Service ${serviceName} found ${newListings.length} new listing(s)`,
|
||||
dynamic_template_data: {
|
||||
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
|
||||
numberOfListings: newListings.length,
|
||||
listings: newListings,
|
||||
listings,
|
||||
},
|
||||
};
|
||||
|
||||
return sgMail.send(msg);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'sendgrid',
|
||||
name: 'SendGrid',
|
||||
|
||||
@@ -1,43 +1,61 @@
|
||||
import Slack from 'slack';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
const msg = Slack.chat.postMessage;
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
return newListings.map((payload) =>
|
||||
msg({
|
||||
token,
|
||||
channel,
|
||||
text: `*(${serviceName} - ${jobKey})* - ${payload.title}`,
|
||||
attachments: [
|
||||
{
|
||||
fallback: payload.title,
|
||||
color: '#36a64f',
|
||||
title: 'Link to Exposé',
|
||||
title_link: payload.link,
|
||||
fields: [
|
||||
{
|
||||
title: 'Price',
|
||||
value: payload.price,
|
||||
short: false,
|
||||
},
|
||||
{
|
||||
title: 'Size',
|
||||
value: payload.size,
|
||||
short: false,
|
||||
},
|
||||
{
|
||||
title: 'Address',
|
||||
value: payload.address,
|
||||
short: false,
|
||||
},
|
||||
],
|
||||
footer: 'Powered by Fredy',
|
||||
ts: new Date().getTime() / 1000,
|
||||
},
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const buildBlocks = (serviceName, jobKey, p) => {
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: `New Listing from ${serviceName} (${jobKey})`, emoji: false },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*<${p.link}|${p.title}>*` },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
fields: [
|
||||
{ type: 'mrkdwn', text: `*Price*\n${p.price ?? 'n/a'}` },
|
||||
{ type: 'mrkdwn', text: `*Size*\n${p.size ?? 'n/a'}` },
|
||||
{ type: 'mrkdwn', text: `*Address*\n${p.address ?? 'n/a'}` },
|
||||
],
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
const img = normalizeImageUrl(p.image);
|
||||
if (img) {
|
||||
blocks.push({
|
||||
type: 'image',
|
||||
image_url: img,
|
||||
alt_text: p.title || 'listing image',
|
||||
});
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
type: 'context',
|
||||
elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
|
||||
});
|
||||
|
||||
return blocks;
|
||||
};
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, channel } = notificationConfig.find((a) => a.id === config.id).fields;
|
||||
|
||||
return Promise.allSettled(
|
||||
newListings.map((p) =>
|
||||
Slack.chat.postMessage({
|
||||
token,
|
||||
channel,
|
||||
text: `${serviceName} ${jobKey}: ${p.title}`,
|
||||
blocks: buildBlocks(serviceName, jobKey, p),
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
}),
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'slack',
|
||||
name: 'Slack',
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
### Slack Adapter
|
||||
IMPORTANT:
|
||||
Don't use this adapter anymore, it is outdated and only here for backwards compatability reasons. Use the new Slack Adapter with webhooks!
|
||||
|
||||
In order to use [Slack](https://slack.com), you need to create an account. When done, you need to create a new App in your workspace. Give it the permission `chat:write:bot` and `chat:write:user`.
|
||||
|
||||
Now you need to create a user token and a channel. Make sure the bot is installed to this channel.
|
||||
|
||||
79
lib/notification/adapter/slack_with_webhooks.js
Executable file
79
lib/notification/adapter/slack_with_webhooks.js
Executable file
@@ -0,0 +1,79 @@
|
||||
import fetch from 'node-fetch';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const buildBlocks = (serviceName, jobKey, p) => {
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: `New Listing from ${serviceName} (${jobKey})`, emoji: false },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*<${p.link}|${p.title}>*` },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
fields: [
|
||||
{ type: 'mrkdwn', text: `*Price*\n${p.price ?? 'n/a'}` },
|
||||
{ type: 'mrkdwn', text: `*Size*\n${p.size ?? 'n/a'}` },
|
||||
{ type: 'mrkdwn', text: `*Address*\n${p.address ?? 'n/a'}` },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const img = normalizeImageUrl(p.image);
|
||||
if (img) {
|
||||
blocks.push({
|
||||
type: 'image',
|
||||
image_url: img,
|
||||
alt_text: p.title || 'listing image',
|
||||
});
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
type: 'context',
|
||||
elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
|
||||
});
|
||||
|
||||
return blocks;
|
||||
};
|
||||
|
||||
const postJson = (url, body) =>
|
||||
fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body,
|
||||
});
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const adapter = notificationConfig.find((a) => a.id === config.id);
|
||||
const webhookUrl = adapter?.fields?.webhookUrl;
|
||||
if (!webhookUrl) return Promise.resolve([]);
|
||||
|
||||
const promises = newListings.map((p) => {
|
||||
const body = JSON.stringify({
|
||||
text: `${serviceName} ${jobKey}: ${p.title}`,
|
||||
blocks: buildBlocks(serviceName, jobKey, p),
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
});
|
||||
return postJson(webhookUrl, body);
|
||||
});
|
||||
|
||||
return Promise.allSettled(promises);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'slack_with_webhooks',
|
||||
name: 'Slack with Webhooks',
|
||||
readme: markdown2Html('lib/notification/adapter/slack_with_webhooks.md'),
|
||||
description: 'Fredy will send new listings to the slack channel of your choice..',
|
||||
fields: {
|
||||
webhookUrl: {
|
||||
type: 'text',
|
||||
label: 'Webhook-Url',
|
||||
description: 'The Url of the Webhook to send messages to.',
|
||||
},
|
||||
},
|
||||
};
|
||||
6
lib/notification/adapter/slack_with_webhooks.md
Normal file
6
lib/notification/adapter/slack_with_webhooks.md
Normal file
@@ -0,0 +1,6 @@
|
||||
### Slack Adapter
|
||||
|
||||
IMPORTANT:
|
||||
This is the new version of the Slack adapter. I strongly encourage you to use it, the old version is now unmaintained and only kept due to backwards compatability reasons.
|
||||
|
||||
In order to use [Slack](https://slack.com), you need to create an account. When done, create a new channel and add the Webhook integration to that channel. Copy the webhook url. That's it.
|
||||
@@ -2,7 +2,19 @@ import { markdown2Html } from '../../services/markdown.js';
|
||||
import Database from 'better-sqlite3';
|
||||
export const send = ({ serviceName, newListings, jobKey }) => {
|
||||
const db = new Database('db/listings.db');
|
||||
const fields = ['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description'];
|
||||
const fields = [
|
||||
'serviceName',
|
||||
'jobKey',
|
||||
'id',
|
||||
'size',
|
||||
'rooms',
|
||||
'price',
|
||||
'address',
|
||||
'title',
|
||||
'link',
|
||||
'description',
|
||||
'image',
|
||||
];
|
||||
db.prepare(`CREATE TABLE IF NOT EXISTS listing (${fields.join(' TEXT, ')} TEXT);`).run();
|
||||
const insert = db.prepare(`INSERT INTO listing (${fields.join(', ')}) VALUES (@${fields.join(', @')})`);
|
||||
newListings.map((listing) => {
|
||||
|
||||
@@ -2,107 +2,97 @@ import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
import pThrottle from 'p-throttle';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const MAX_ENTITIES_PER_CHUNK = 8;
|
||||
const RATE_LIMIT_INTERVAL = 1000;
|
||||
const chatThrottleMap = new Map();
|
||||
|
||||
function cleanupOldThrottles() {
|
||||
const now = Date.now();
|
||||
const maxAge = RATE_LIMIT_INTERVAL + 1000; // adding extra second
|
||||
const maxAge = RATE_LIMIT_INTERVAL + 1000;
|
||||
const toBeDeleted = [];
|
||||
|
||||
for (const [chatId, chatThrottle] of chatThrottleMap.entries()) {
|
||||
if (now - chatThrottle.lastUsedAt > maxAge) {
|
||||
toBeDeleted.push(chatId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const chatId of toBeDeleted) {
|
||||
chatThrottleMap.delete(chatId);
|
||||
if (now - chatThrottle.lastUsedAt > maxAge) toBeDeleted.push(chatId);
|
||||
}
|
||||
for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a throttled async function for sending messages to a specific chat.
|
||||
* Telegram enforces a rate limit of 1 message per second per chat (chatId).
|
||||
*
|
||||
* @param {number} chatId - The chat ID to throttle messages for.
|
||||
* @param {Function} fn - The async function to throttle (should send the message).
|
||||
* @returns {Function} Throttled async function for sending messages.
|
||||
*/
|
||||
function getThrottled(chatId, call) {
|
||||
cleanupOldThrottles();
|
||||
|
||||
const now = Date.now();
|
||||
const chatThrottle = chatThrottleMap.get(chatId);
|
||||
|
||||
if (chatThrottle) {
|
||||
chatThrottle.lastUsedAt = now;
|
||||
return chatThrottle.throttled;
|
||||
}
|
||||
|
||||
// Create new throttled function
|
||||
const newThrottle = {
|
||||
lastUsedAt: now,
|
||||
throttled: pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call),
|
||||
};
|
||||
chatThrottleMap.set(chatId, newThrottle);
|
||||
return newThrottle.throttled;
|
||||
const throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call);
|
||||
chatThrottleMap.set(chatId, { lastUsedAt: now, throttled });
|
||||
return throttled;
|
||||
}
|
||||
|
||||
/**
|
||||
* splitting an array into chunks because Telegram only allows for messages up to
|
||||
* 4096 chars, thus we have to split messages into chunks
|
||||
* @param inputArray
|
||||
* @param perChunk
|
||||
*/
|
||||
const arrayChunks = (inputArray, perChunk) =>
|
||||
inputArray.reduce((all, one, i) => {
|
||||
const ch = Math.floor(i / perChunk);
|
||||
all[ch] = [].concat(all[ch] || [], one);
|
||||
return all;
|
||||
}, []);
|
||||
function shorten(str, len = 30) {
|
||||
return str.length > len ? str.substring(0, len) + '...' : str;
|
||||
function shorten(str, len = 90) {
|
||||
if (!str) return '';
|
||||
return str.length > len ? str.substring(0, len).trim() + '...' : str;
|
||||
}
|
||||
|
||||
function escapeHtml(s = '') {
|
||||
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
}
|
||||
|
||||
function buildCaption(jobName, serviceName, o) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
return `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n<a href='${escapeHtml(
|
||||
o.link || '',
|
||||
)}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}`.slice(0, 1024);
|
||||
}
|
||||
|
||||
function buildText(jobName, serviceName, o) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
return (
|
||||
`<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n` +
|
||||
`<a href='${escapeHtml(o.link || '')}'><b>${escapeHtml(title)}</b></a>\n` +
|
||||
`${escapeHtml(meta)}`
|
||||
);
|
||||
}
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK);
|
||||
|
||||
const getThrottledSend = getThrottled(chatId, async function (body) {
|
||||
await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
|
||||
const throttledCall = getThrottled(chatId, async function (endpoint, body) {
|
||||
await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, {
|
||||
method: 'post',
|
||||
body: JSON.stringify(body),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
});
|
||||
|
||||
const promises = chunks.map((chunk) => {
|
||||
const messageParagraphs = [];
|
||||
const promises = newListings.map(async (o) => {
|
||||
const img = normalizeImageUrl(o.image);
|
||||
|
||||
messageParagraphs.push(`<i>${jobName}</i> (${serviceName}) found <b>${newListings.length}</b> new listings:`);
|
||||
messageParagraphs.push(
|
||||
...chunk.map(
|
||||
(o) =>
|
||||
`<a href='${o.link}'><b>${shorten(o.title.replace(/\*/g, ''), 45).trim()}</b></a>\n` +
|
||||
[o.address, o.price, o.size].join(' | '),
|
||||
),
|
||||
);
|
||||
if (img) {
|
||||
return throttledCall('sendPhoto', {
|
||||
chat_id: chatId,
|
||||
photo: img,
|
||||
caption: buildCaption(jobName, serviceName, o),
|
||||
parse_mode: 'HTML',
|
||||
});
|
||||
}
|
||||
|
||||
const body = {
|
||||
return throttledCall('sendMessage', {
|
||||
chat_id: chatId,
|
||||
text: messageParagraphs.join('\n\n'),
|
||||
text: buildText(jobName, serviceName, o),
|
||||
parse_mode: 'HTML',
|
||||
disable_web_page_preview: true,
|
||||
};
|
||||
|
||||
return getThrottledSend(body);
|
||||
});
|
||||
});
|
||||
|
||||
return Promise.all(promises);
|
||||
};
|
||||
|
||||
export const config = {
|
||||
id: 'telegram',
|
||||
name: 'Telegram',
|
||||
|
||||
Reference in New Issue
Block a user