Compare commits

...

4 Commits

Author SHA1 Message Date
orangecoding
b858529f06 next release version 2025-10-05 18:57:52 +02:00
orangecoding
c9bd5dc161 fixing delete listings 2025-10-05 18:57:27 +02:00
orangecoding
daa4a7b8f1 refine telegram adapter 2025-10-05 18:53:17 +02:00
Thomas Brockmöller
035f0e9f83 Check Telegram response (#205) (#211)
* Add error handling and logging to Telegram message sending

* Add debug logging for new listings
2025-10-05 17:06:57 +02:00
4 changed files with 101 additions and 20 deletions

View File

@@ -77,6 +77,7 @@ class FredyRuntime {
} }
_findNew(listings) { _findNew(listings) {
logger.debug(`Checking ${listings.length} listings for new entries (Provider: '${this._providerId}')`);
const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || []; const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
const newListings = listings.filter((o) => !hashes.includes(o.id)); const newListings = listings.filter((o) => !hashes.includes(o.id));
@@ -95,6 +96,7 @@ class FredyRuntime {
} }
_save(newListings) { _save(newListings) {
logger.debug(`Storing ${newListings.length} new listings (Provider: '${this._providerId}')`);
storeListings(this._jobKey, this._providerId, newListings); storeListings(this._jobKey, this._providerId, newListings);
return newListings; return newListings;
} }
@@ -103,7 +105,9 @@ class FredyRuntime {
const filteredList = listings.filter((listing) => { const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address); const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
if (similar) { if (similar) {
logger.debug(`Filtering similar entry for title: ${listing.title} and address ${listing.address}`); logger.debug(
`Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
);
} }
return !similar; return !similar;
}); });
@@ -112,7 +116,11 @@ class FredyRuntime {
} }
_handleError(err) { _handleError(err) {
if (err.name !== 'NoNewListingsWarning') logger.error(err); if (err.name === 'NoNewListingsWarning') {
logger.debug(`No new listings found (Provider: '${this._providerId}').`);
} else {
logger.error(err);
}
} }
} }

View File

@@ -3,10 +3,14 @@ import { getJob } from '../../services/storage/jobStorage.js';
import fetch from 'node-fetch'; import fetch from 'node-fetch';
import pThrottle from 'p-throttle'; import pThrottle from 'p-throttle';
import { normalizeImageUrl } from '../../utils.js'; import { normalizeImageUrl } from '../../utils.js';
import logger from '../../services/logger.js';
const RATE_LIMIT_INTERVAL = 1000; const RATE_LIMIT_INTERVAL = 1000;
const chatThrottleMap = new Map(); const chatThrottleMap = new Map();
/**
* Removes stale throttled call entries to keep memory bounded.
*/
function cleanupOldThrottles() { function cleanupOldThrottles() {
const now = Date.now(); const now = Date.now();
const maxAge = RATE_LIMIT_INTERVAL + 1000; const maxAge = RATE_LIMIT_INTERVAL + 1000;
@@ -17,6 +21,15 @@ function cleanupOldThrottles() {
for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId); for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId);
} }
/**
* Return a throttled wrapper for a chatId to limit Telegram API calls.
* Uses p-throttle with 1 request per RATE_LIMIT_INTERVAL per chat.
*
* @template {Function} T
* @param {string|number} chatId
* @param {T} call - async function (endpoint: string, body: any) => Promise<Response>
* @returns {T}
*/
function getThrottled(chatId, call) { function getThrottled(chatId, call) {
cleanupOldThrottles(); cleanupOldThrottles();
const now = Date.now(); const now = Date.now();
@@ -30,15 +43,38 @@ function getThrottled(chatId, call) {
return throttled; return throttled;
} }
/**
* Shorten a string to a maximum length with an ellipsis suffix.
* @param {string} str
* @param {number} [len=90]
* @returns {string}
*/
function shorten(str, len = 90) { function shorten(str, len = 90) {
if (!str) return ''; if (!str) return '';
return str.length > len ? str.substring(0, len).trim() + '...' : str; return str.length > len ? str.substring(0, len).trim() + '...' : str;
} }
/**
* Escape basic HTML entities for Telegram HTML parse mode.
* @param {string} [s='']
* @returns {string}
*/
function escapeHtml(s = '') { function escapeHtml(s = '') {
return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;'); return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
} }
/**
* Build a Telegram photo caption (max 1024 characters) using HTML parse mode.
* @param {string} jobName
* @param {string} serviceName
* @param {Object} o - Listing object
* @param {string} [o.title]
* @param {string} [o.address]
* @param {string|number} [o.price]
* @param {string|number} [o.size]
* @param {string} [o.link]
* @returns {string}
*/
function buildCaption(jobName, serviceName, o) { function buildCaption(jobName, serviceName, o) {
const title = shorten((o.title || '').replace(/\*/g, ''), 90); const title = shorten((o.title || '').replace(/\*/g, ''), 90);
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | '); const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
@@ -47,6 +83,13 @@ function buildCaption(jobName, serviceName, o) {
)}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}`.slice(0, 1024); )}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}`.slice(0, 1024);
} }
/**
* Build a Telegram message text using HTML parse mode.
* @param {string} jobName
* @param {string} serviceName
* @param {Object} o - Listing object
* @returns {string}
*/
function buildText(jobName, serviceName, o) { function buildText(jobName, serviceName, o) {
const title = shorten((o.title || '').replace(/\*/g, ''), 90); const title = shorten((o.title || '').replace(/\*/g, ''), 90);
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | '); const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
@@ -57,8 +100,27 @@ function buildText(jobName, serviceName, o) {
); );
} }
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { /**
const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields; * Send new listings to Telegram.
* - Respects per-chat Telegram rate limits using a lightweight throttle cache.
* - Falls back to sendMessage when sendPhoto fails or image is missing.
*
* @param {Object} params
* @param {string} params.serviceName - Name of the crawler/service producing the listings.
* @param {Array<Object>} params.newListings - Array of new listing objects.
* @param {Array<Object>} params.notificationConfig - Notification adapters configuration array.
* @param {string} params.jobKey - Storage job key to resolve the human readable job name.
* @returns {Promise<Array<Response>>} Promise resolving when all send operations complete.
*/
export const send = ({ serviceName, newListings = [], notificationConfig, jobKey }) => {
const adapterCfg = notificationConfig.find((adapter) => adapter.id === config.id);
if (!adapterCfg || !adapterCfg.fields) {
throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`);
}
const { token, chatId } = adapterCfg.fields;
if (!token || !chatId) {
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
}
const job = getJob(jobKey); const job = getJob(jobKey);
const jobName = job == null ? jobKey : job.name; const jobName = job == null ? jobKey : job.name;
@@ -68,9 +130,16 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
body: JSON.stringify(body), body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
}); });
if (!res.ok) {
const errorBody = await res.text();
throw new Error(`API error for '${jobName}'. '${endpoint}' returned ${errorBody}`);
}
return res; return res;
}); });
if (!Array.isArray(newListings) || newListings.length === 0) return Promise.resolve([]);
const promises = newListings.map(async (o) => { const promises = newListings.map(async (o) => {
const img = normalizeImageUrl(o.image); const img = normalizeImageUrl(o.image);
const textPayload = { const textPayload = {
@@ -81,28 +150,32 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
}; };
if (!img) { if (!img) {
return throttledCall('sendMessage', textPayload); return await throttledCall('sendMessage', textPayload).catch(async (e) => {
logger.error(`Error sending message to Telegram: ${e.message}`);
});
} }
try { return await throttledCall('sendPhoto', {
return await throttledCall('sendPhoto', { chat_id: chatId,
chat_id: chatId, photo: img,
photo: img, caption: buildCaption(jobName, serviceName, o),
caption: buildCaption(jobName, serviceName, o), parse_mode: 'HTML',
parse_mode: 'HTML', }).catch(async (e) => {
logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
return await throttledCall('sendMessage', textPayload).catch((e) => {
logger.error(`Error sending message to Telegram: ${e.message}`);
throw e;
}); });
} catch (e) { });
// If we see a timeout due to sending an image, try sending it without
if (e && (e.code === 'ETIMEDOUT' || e.errno === 'ETIMEDOUT')) {
return throttledCall('sendMessage', textPayload);
}
throw e;
}
}); });
return Promise.all(promises); return Promise.all(promises);
}; };
/**
* Telegram notification adapter configuration schema.
* @type {{id:string,name:string,readme:string,description:string,fields:{token:{type:string,label:string,description:string},chatId:{type:string,label:string,description:string}}}}
*/
export const config = { export const config = {
id: 'telegram', id: 'telegram',
name: 'Telegram', name: 'Telegram',

View File

@@ -1,6 +1,6 @@
{ {
"name": "fredy", "name": "fredy",
"version": "14.1.0", "version": "14.1.1",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"prepare": "husky", "prepare": "husky",

View File

@@ -65,7 +65,7 @@ const columns = [
type="danger" type="danger"
onClick={async () => { onClick={async () => {
try { try {
await xhrDelete('/api/listings/', { ids: [id] }); await xhrDelete('/api/listings/', { ids: [row.id] });
Toast.success('Listing(s) successfully removed'); Toast.success('Listing(s) successfully removed');
row.reloadTable(); row.reloadTable();
} catch (error) { } catch (error) {