mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
324afee483 | ||
|
|
e95ebb9624 | ||
|
|
c29387c85d | ||
|
|
322ae199b0 |
@@ -99,8 +99,8 @@ class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Optionally, enrich new listings with data from their detail pages.
|
||||
* Only called when the provider config defines a `fetchDetails` function.
|
||||
* Runs all fetches in parallel. Each fetch must handle its own errors
|
||||
* and always resolve (never reject) to avoid aborting other listings.
|
||||
* Fetches are performed sequentially to avoid overloading the provider or
|
||||
* the shared browser instance.
|
||||
*
|
||||
* @param {Listing[]} newListings New listings to enrich.
|
||||
* @returns {Promise<Listing[]>} Resolves with enriched listings.
|
||||
@@ -199,9 +199,9 @@ class FredyPipelineExecutioner {
|
||||
const toDeleteListingByIds = [];
|
||||
const keptListings = newListings.filter((listing) => {
|
||||
const filterOut =
|
||||
(minRooms && listing.rooms && listing.rooms < minRooms) ||
|
||||
(minSize && listing.size && listing.size < minSize) ||
|
||||
(maxPrice && listing.price && listing.price > maxPrice);
|
||||
(minRooms && listing.rooms != null && listing.rooms < minRooms) ||
|
||||
(minSize && listing.size != null && listing.size < minSize) ||
|
||||
(maxPrice && listing.price != null && listing.price > maxPrice);
|
||||
|
||||
if (filterOut) {
|
||||
toDeleteListingByIds.push(listing.id);
|
||||
@@ -223,24 +223,15 @@ class FredyPipelineExecutioner {
|
||||
* @param {string} url The provider URL to fetch from.
|
||||
* @returns {Promise<ParsedListing[]>} Resolves with an array of listings (empty when none found).
|
||||
*/
|
||||
_getListings(url) {
|
||||
async _getListings(url) {
|
||||
const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
|
||||
return new Promise((resolve, reject) => {
|
||||
extractor
|
||||
.execute(url, this._providerConfig.waitForSelector, this._providerId)
|
||||
.then(() => {
|
||||
const listings = extractor.parseResponseText(
|
||||
this._providerConfig.crawlContainer,
|
||||
this._providerConfig.crawlFields,
|
||||
url,
|
||||
);
|
||||
resolve(listings == null ? [] : listings);
|
||||
})
|
||||
.catch((err) => {
|
||||
reject(err);
|
||||
logger.error(err);
|
||||
});
|
||||
});
|
||||
await extractor.execute(url, this._providerConfig.waitForSelector, this._providerId);
|
||||
const listings = extractor.parseResponseText(
|
||||
this._providerConfig.crawlContainer,
|
||||
this._providerConfig.crawlFields,
|
||||
url,
|
||||
);
|
||||
return listings == null ? [] : listings;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -195,6 +195,9 @@ export default async function jobPlugin(fastify) {
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) {
|
||||
return reply.code(404).send({ error: 'Job not found' });
|
||||
}
|
||||
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove the Demo Job ;)' });
|
||||
}
|
||||
@@ -216,6 +219,9 @@ export default async function jobPlugin(fastify) {
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) {
|
||||
return reply.code(404).send({ error: 'Job not found' });
|
||||
}
|
||||
|
||||
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot change the Status of our Demo Job ;)' });
|
||||
|
||||
@@ -8,7 +8,7 @@ import * as watchListStorage from '../../services/storage/watchListStorage.js';
|
||||
import { isAdmin as isAdminFn } from '../security.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { nullOrEmpty } from '../../utils.js';
|
||||
import { getJobs } from '../../services/storage/jobStorage.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { trackPoi } from '../../services/tracking/Tracker.js';
|
||||
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||
@@ -46,9 +46,8 @@ export default async function listingsPlugin(fastify) {
|
||||
|
||||
let jobFilter = null;
|
||||
let jobIdFilter = null;
|
||||
const jobs = getJobs();
|
||||
if (!nullOrEmpty(jobNameFilter)) {
|
||||
const job = jobs.find((j) => j.id === jobNameFilter);
|
||||
const job = getJob(jobNameFilter);
|
||||
jobFilter = job != null ? job.name : null;
|
||||
jobIdFilter = job != null ? job.id : null;
|
||||
}
|
||||
@@ -159,6 +158,16 @@ export default async function listingsPlugin(fastify) {
|
||||
if (settings.demoMode && !isAdminFn(request)) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove listings in demo mode ;)' });
|
||||
}
|
||||
const job = getJob(jobId);
|
||||
if (!job) {
|
||||
return reply.code(404).send({ error: 'Job not found' });
|
||||
}
|
||||
const userId = request.session.currentUser;
|
||||
if (!isAdminFn(request) && job.userId !== userId && !job.shared_with_user.includes(userId)) {
|
||||
return reply
|
||||
.code(403)
|
||||
.send({ error: 'You are trying to remove listings for a job that is not associated to your user' });
|
||||
}
|
||||
listingStorage.deleteListingsByJobId(jobId, hardDelete);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
@@ -169,7 +178,11 @@ export default async function listingsPlugin(fastify) {
|
||||
|
||||
fastify.delete('/', async (request, reply) => {
|
||||
const { ids, hardDelete = false } = request.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
if (settings.demoMode && !isAdminFn(request)) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove listings in demo mode ;)' });
|
||||
}
|
||||
if (Array.isArray(ids) && ids.length > 0) {
|
||||
listingStorage.deleteListingsById(ids, hardDelete);
|
||||
}
|
||||
|
||||
@@ -20,6 +20,9 @@ function getClientIp(request) {
|
||||
|
||||
function isRateLimited(ip) {
|
||||
const now = Date.now();
|
||||
for (const [key, rec] of loginAttempts) {
|
||||
if (now - rec.firstAttempt > LOGIN_WINDOW_MS) loginAttempts.delete(key);
|
||||
}
|
||||
const record = loginAttempts.get(ip);
|
||||
if (!record || now - record.firstAttempt > LOGIN_WINDOW_MS) {
|
||||
loginAttempts.set(ip, { count: 1, firstAttempt: now });
|
||||
|
||||
@@ -3,13 +3,11 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import SqliteConnection from '../../services/storage/SqliteConnection.js';
|
||||
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { getSettings, getUserSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
import { resetGeocoordinatesAndDistanceForUser } from '../../services/storage/listingsStorage.js';
|
||||
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
|
||||
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
|
||||
import { fromJson } from '../../utils.js';
|
||||
import { trackPoi } from '../../services/tracking/Tracker.js';
|
||||
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||
import logger from '../../services/logger.js';
|
||||
@@ -21,12 +19,7 @@ import { runGeoCordTask } from '../../services/crons/geocoding-cron.js';
|
||||
export default async function userSettingsPlugin(fastify) {
|
||||
fastify.get('/', async (request) => {
|
||||
const userId = request.session.currentUser;
|
||||
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
|
||||
const settings = {};
|
||||
for (const r of rows) {
|
||||
settings[r.name] = fromJson(r.value, null);
|
||||
}
|
||||
return settings;
|
||||
return getUserSettings(userId);
|
||||
});
|
||||
|
||||
fastify.get('/autocomplete', async (request, reply) => {
|
||||
|
||||
@@ -7,6 +7,7 @@ import fetch from 'node-fetch';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
import logger from '../../services/logger.js';
|
||||
|
||||
/**
|
||||
* Generates an idempotent decimal color code. The input string-based color code is
|
||||
@@ -67,19 +68,6 @@ const buildEmbed = (jobKey, listing, baseUrl) => {
|
||||
},
|
||||
];
|
||||
|
||||
const embed = {
|
||||
title: title,
|
||||
color: generateColorFromString(jobKey),
|
||||
url: listing.link,
|
||||
fields: fields,
|
||||
};
|
||||
|
||||
if (listing.image) {
|
||||
embed.image = {
|
||||
url: normalizeImageUrl(listing.image),
|
||||
};
|
||||
}
|
||||
|
||||
if (baseUrl && listing.id) {
|
||||
fields.push({
|
||||
name: 'Open in Fredy',
|
||||
@@ -88,6 +76,19 @@ const buildEmbed = (jobKey, listing, baseUrl) => {
|
||||
});
|
||||
}
|
||||
|
||||
const embed = {
|
||||
title: title,
|
||||
color: generateColorFromString(jobKey),
|
||||
url: listing.link,
|
||||
fields,
|
||||
};
|
||||
|
||||
if (listing.image) {
|
||||
embed.image = {
|
||||
url: normalizeImageUrl(listing.image),
|
||||
};
|
||||
}
|
||||
|
||||
return embed;
|
||||
};
|
||||
|
||||
@@ -119,7 +120,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey, bas
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body,
|
||||
}).catch((error) => {
|
||||
console.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
|
||||
logger.error(`Error sending Discord webhook for chunk starting at ${i}:`, error);
|
||||
return Promise.reject(new Error(`Webhook failed: ${error.message}`));
|
||||
});
|
||||
|
||||
|
||||
@@ -12,41 +12,45 @@ import logger from '../../services/logger.js';
|
||||
import { shouldUseMultipart, buildPhotoFormData } from './telegramPhotoUploader.js';
|
||||
|
||||
const RATE_LIMIT_INTERVAL = 1000;
|
||||
const THROTTLE_MAX_IDLE_MS = RATE_LIMIT_INTERVAL + 2000;
|
||||
const chatThrottleMap = new Map();
|
||||
|
||||
/**
|
||||
* Removes stale throttled call entries to keep memory bounded.
|
||||
* An entry is stale when no API call has fired for longer than THROTTLE_MAX_IDLE_MS.
|
||||
*/
|
||||
function cleanupOldThrottles() {
|
||||
const now = Date.now();
|
||||
const maxAge = RATE_LIMIT_INTERVAL + 1000;
|
||||
const toBeDeleted = [];
|
||||
for (const [chatId, chatThrottle] of chatThrottleMap.entries()) {
|
||||
if (now - chatThrottle.lastUsedAt > maxAge) toBeDeleted.push(chatId);
|
||||
if (now - chatThrottle.lastUsedAt > THROTTLE_MAX_IDLE_MS) chatThrottleMap.delete(chatId);
|
||||
}
|
||||
for (const chatId of toBeDeleted) chatThrottleMap.delete(chatId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a throttled wrapper for a chatId to limit Telegram API calls.
|
||||
* Uses p-throttle with 1 request per RATE_LIMIT_INTERVAL per chat.
|
||||
* `lastUsedAt` is refreshed on every actual API call so that the idle window
|
||||
* starts from the last fired call, not from when send() was invoked.
|
||||
*
|
||||
* @template {Function} T
|
||||
* @param {string|number} chatId
|
||||
* @param {T} call - async function (endpoint: string, body: any) => Promise<Response>
|
||||
* @returns {T}
|
||||
* @param {Function} call - async function (endpoint: string, body: any) => Promise<Response>
|
||||
* @returns {Function}
|
||||
*/
|
||||
function getThrottled(chatId, call) {
|
||||
cleanupOldThrottles();
|
||||
const now = Date.now();
|
||||
const chatThrottle = chatThrottleMap.get(chatId);
|
||||
if (chatThrottle) {
|
||||
chatThrottle.lastUsedAt = now;
|
||||
return chatThrottle.throttled;
|
||||
const existing = chatThrottleMap.get(chatId);
|
||||
if (existing) {
|
||||
existing.lastUsedAt = Date.now();
|
||||
return existing.throttled;
|
||||
}
|
||||
const throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(call);
|
||||
chatThrottleMap.set(chatId, { lastUsedAt: now, throttled });
|
||||
return throttled;
|
||||
const entry = { lastUsedAt: Date.now(), throttled: null };
|
||||
chatThrottleMap.set(chatId, entry);
|
||||
entry.throttled = pThrottle({ limit: 1, interval: RATE_LIMIT_INTERVAL })(async (endpoint, body) => {
|
||||
const e = chatThrottleMap.get(chatId);
|
||||
if (e) e.lastUsedAt = Date.now();
|
||||
return call(endpoint, body);
|
||||
});
|
||||
return entry.throttled;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,35 +74,16 @@ function escapeHtml(s = '') {
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a Telegram photo caption (max 1024 characters) using HTML parse mode.
|
||||
* Build a Telegram HTML-formatted message body.
|
||||
* Suitable for both sendMessage (uncapped) and sendPhoto captions (caller must slice to 1024).
|
||||
*
|
||||
* @param {string} jobName
|
||||
* @param {string} serviceName
|
||||
* @param {Object} o - Listing object
|
||||
* @param {string} [o.title]
|
||||
* @param {string} [o.address]
|
||||
* @param {string|number} [o.price]
|
||||
* @param {string|number} [o.size]
|
||||
* @param {string} [o.link]
|
||||
* @param {string} [baseUrl]
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildCaption(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLink =
|
||||
baseUrl && o.id ? `\n<a href='${escapeHtml(`${baseUrl}/#/listings/listing/${o.id}`)}'>Open in Fredy</a>` : '';
|
||||
return `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n<a href='${escapeHtml(
|
||||
o.link || '',
|
||||
)}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}${fredyLink}`.slice(0, 1024);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a Telegram message text using HTML parse mode.
|
||||
* @param {string} jobName
|
||||
* @param {string} serviceName
|
||||
* @param {Object} o - Listing object
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildText(jobName, serviceName, o, baseUrl) {
|
||||
function buildHtmlBody(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLink =
|
||||
@@ -111,14 +96,16 @@ function buildText(jobName, serviceName, o, baseUrl) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a plain text Telegram photo caption (max 4096 characters).
|
||||
* Build a plain-text Telegram photo caption (max 4096 characters).
|
||||
* Meta appears before the link so the most relevant info is visible within the cap.
|
||||
*
|
||||
* @param {string} jobName
|
||||
* @param {string} serviceName
|
||||
* @param {Object} o - Listing object
|
||||
* @param baseUrl
|
||||
* @param {string} [baseUrl]
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildCaptionPlain(jobName, serviceName, o, baseUrl) {
|
||||
function buildPlainCaption(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${o.id}` : '';
|
||||
@@ -126,19 +113,111 @@ function buildCaptionPlain(jobName, serviceName, o, baseUrl) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a plain text Telegram message.
|
||||
* Build a plain-text Telegram message body.
|
||||
* Link appears early so it is tappable without scrolling.
|
||||
*
|
||||
* @param {string} jobName
|
||||
* @param {string} serviceName
|
||||
* @param {Object} o - Listing object
|
||||
* @param {string} [baseUrl]
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildTextPlain(jobName, serviceName, o, baseUrl) {
|
||||
function buildPlainText(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/#/listings/listing/${o.id}` : '';
|
||||
return `${jobName} (${serviceName})\n${title}\n${o.link || ''}\n${meta}${fredyLine}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the raw Telegram API caller for a given bot token.
|
||||
* Handles JSON and multipart (FormData) bodies.
|
||||
*
|
||||
* @param {string} token - Telegram bot token.
|
||||
* @param {string} jobName - Used in error messages.
|
||||
* @returns {(endpoint: string, body: object|FormData) => Promise<Response>}
|
||||
*/
|
||||
function makeTelegramCaller(token, jobName) {
|
||||
return async function (endpoint, body) {
|
||||
const isFormData = body instanceof FormData;
|
||||
const opts = isFormData
|
||||
? { method: 'post', body }
|
||||
: { method: 'post', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' } };
|
||||
const res = await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, opts);
|
||||
if (!res.ok) {
|
||||
const errorBody = await res.text();
|
||||
throw new Error(`API error for '${jobName}'. '${endpoint}' returned ${errorBody}`);
|
||||
}
|
||||
return res;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a single listing to a single Telegram chat, with photo-then-text fallback.
|
||||
*
|
||||
* @param {Function} throttledCall - Throttled Telegram API caller for this chat.
|
||||
* @param {Object} listing - Listing object.
|
||||
* @param {string|number} chatId
|
||||
* @param {Object} opts
|
||||
* @param {string} opts.jobName
|
||||
* @param {string} opts.serviceName
|
||||
* @param {string} opts.baseUrl
|
||||
* @param {boolean} opts.plainText
|
||||
* @param {number|undefined} opts.message_thread_id
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function sendListingToChat(
|
||||
throttledCall,
|
||||
listing,
|
||||
chatId,
|
||||
{ jobName, serviceName, baseUrl, plainText, message_thread_id },
|
||||
) {
|
||||
const img = normalizeImageUrl(listing.image);
|
||||
|
||||
const textPayload = {
|
||||
chat_id: chatId,
|
||||
text: plainText
|
||||
? buildPlainText(jobName, serviceName, listing, baseUrl)
|
||||
: buildHtmlBody(jobName, serviceName, listing, baseUrl),
|
||||
...(plainText ? {} : { parse_mode: 'HTML' }),
|
||||
disable_web_page_preview: true,
|
||||
...(message_thread_id ? { message_thread_id } : {}),
|
||||
};
|
||||
|
||||
if (!img) {
|
||||
return throttledCall('sendMessage', textPayload).catch((e) => {
|
||||
logger.error(`Error sending message to Telegram: ${e.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
const caption = plainText
|
||||
? buildPlainCaption(jobName, serviceName, listing, baseUrl)
|
||||
: buildHtmlBody(jobName, serviceName, listing, baseUrl).slice(0, 1024);
|
||||
const parseMode = plainText ? undefined : 'HTML';
|
||||
|
||||
// .webp URLs (Immowelt/Cloudimage) fail Telegram's URL-based sendPhoto with
|
||||
// "failed to get HTTP URL content". Upload the bytes via multipart instead.
|
||||
const photoCall = shouldUseMultipart(img)
|
||||
? buildPhotoFormData({ chatId, imageUrl: img, caption, parseMode, messageThreadId: message_thread_id }).then((fd) =>
|
||||
throttledCall('sendPhoto', fd),
|
||||
)
|
||||
: throttledCall('sendPhoto', {
|
||||
chat_id: chatId,
|
||||
photo: img,
|
||||
caption,
|
||||
...(parseMode ? { parse_mode: parseMode } : {}),
|
||||
...(message_thread_id ? { message_thread_id } : {}),
|
||||
});
|
||||
|
||||
return photoCall.catch(async (e) => {
|
||||
logger.warn(`Error sending photo to Telegram and use a fallback: ${e.message}`);
|
||||
return throttledCall('sendMessage', textPayload).catch((e) => {
|
||||
logger.error(`Error sending message to Telegram: ${e.message}`);
|
||||
throw e;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send new listings to Telegram.
|
||||
* - Respects per-chat Telegram rate limits using a lightweight throttle cache.
|
||||
@@ -161,6 +240,11 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
|
||||
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
|
||||
}
|
||||
|
||||
const chatIds = String(chatId)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
// Optional Telegram topic/thread support (supergroups)
|
||||
let message_thread_id;
|
||||
if (messageThreadId !== undefined && messageThreadId !== null && `${messageThreadId}`.trim() !== '') {
|
||||
@@ -177,70 +261,16 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
|
||||
const throttledCall = getThrottled(chatId, async function (endpoint, body) {
|
||||
// FormData (multipart) vs JSON. node-fetch sets its own multipart boundary
|
||||
// header, so we must NOT supply Content-Type ourselves in that case.
|
||||
const isFormData = body instanceof FormData;
|
||||
const opts = isFormData
|
||||
? { method: 'post', body }
|
||||
: { method: 'post', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' } };
|
||||
const res = await fetch(`https://api.telegram.org/bot${token}/${endpoint}`, opts);
|
||||
|
||||
if (!res.ok) {
|
||||
const errorBody = await res.text();
|
||||
throw new Error(`API error for '${jobName}'. '${endpoint}' returned ${errorBody}`);
|
||||
}
|
||||
return res;
|
||||
});
|
||||
|
||||
if (!Array.isArray(newListings) || newListings.length === 0) return Promise.resolve([]);
|
||||
|
||||
const promises = newListings.map(async (o) => {
|
||||
const img = normalizeImageUrl(o.image);
|
||||
const textPayload = {
|
||||
chat_id: chatId,
|
||||
text: plainText ? buildTextPlain(jobName, serviceName, o, baseUrl) : buildText(jobName, serviceName, o, baseUrl),
|
||||
...(plainText ? {} : { parse_mode: 'HTML' }),
|
||||
disable_web_page_preview: true,
|
||||
...(message_thread_id ? { message_thread_id } : {}),
|
||||
};
|
||||
|
||||
if (!img) {
|
||||
return await throttledCall('sendMessage', textPayload).catch(async (e) => {
|
||||
logger.error(`Error sending message to Telegram: ${e.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
const caption = plainText
|
||||
? buildCaptionPlain(jobName, serviceName, o, baseUrl)
|
||||
: buildCaption(jobName, serviceName, o, baseUrl);
|
||||
const parseMode = plainText ? undefined : 'HTML';
|
||||
|
||||
// .webp URLs (Immowelt/Cloudimage) fail Telegram's URL-based sendPhoto with
|
||||
// "failed to get HTTP URL content". Upload the bytes via multipart instead;
|
||||
// the rendered chat message is identical.
|
||||
const photoCall = shouldUseMultipart(img)
|
||||
? buildPhotoFormData({ chatId, imageUrl: img, caption, parseMode, messageThreadId: message_thread_id }).then(
|
||||
(fd) => throttledCall('sendPhoto', fd),
|
||||
)
|
||||
: throttledCall('sendPhoto', {
|
||||
chat_id: chatId,
|
||||
photo: img,
|
||||
caption,
|
||||
...(parseMode ? { parse_mode: parseMode } : {}),
|
||||
...(message_thread_id ? { message_thread_id } : {}),
|
||||
});
|
||||
|
||||
return await photoCall.catch(async (e) => {
|
||||
logger.warn(`Error sending photo to Telegram and use a fallback: ${e.message}`);
|
||||
return await throttledCall('sendMessage', textPayload).catch((e) => {
|
||||
logger.error(`Error sending message to Telegram: ${e.message}`);
|
||||
throw e;
|
||||
});
|
||||
});
|
||||
const allPromises = chatIds.flatMap((id) => {
|
||||
const caller = makeTelegramCaller(token, jobName);
|
||||
const throttledCall = getThrottled(id, caller);
|
||||
const opts = { jobName, serviceName, baseUrl, plainText, message_thread_id };
|
||||
return newListings.map((listing) => sendListingToChat(throttledCall, listing, id, opts));
|
||||
});
|
||||
|
||||
return Promise.all(promises);
|
||||
return Promise.all(allPromises);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -261,7 +291,8 @@ export const config = {
|
||||
chatId: {
|
||||
type: 'chatId',
|
||||
label: 'Chat Id',
|
||||
description: 'The chat id to send messages to you.',
|
||||
description:
|
||||
'The chat ID to send messages to. Separate multiple IDs with commas to notify several recipients (e.g. 123456789, 987654321).',
|
||||
},
|
||||
messageThreadId: {
|
||||
type: 'text',
|
||||
|
||||
@@ -21,6 +21,8 @@ Steps:
|
||||
- Private chats: `chat.id` is a positive number
|
||||
- Groups/supergroups: `chat.id` is a negative number
|
||||
|
||||
**Multiple recipients:** To notify several users individually, enter a comma-separated list of chat IDs in the Chat Id field, e.g. `123456789, 987654321`. Each recipient receives the same messages and gets its own independent rate-limit window. This avoids having to create a group and add the bot to it.
|
||||
|
||||
Keep your bot token secret. If `getUpdates` returns an empty list, send a new message and try again, or make sure your bot’s privacy settings allow it to see group messages when used in groups.
|
||||
|
||||
#### Getting the thread ID (this is optional to be used for forum topics)
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import logger from '../services/logger.js';
|
||||
const path = './adapter';
|
||||
|
||||
/** Read every integration existing in ./adapter **/
|
||||
@@ -23,7 +24,13 @@ const findAdapter = (notificationAdapter) => {
|
||||
export const send = (serviceName, newListings, notificationConfig, jobKey, baseUrl) => {
|
||||
//this is not being used in tests, therefore adapter are always set
|
||||
return notificationConfig
|
||||
.filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
|
||||
.map((notificationAdapter) => findAdapter(notificationAdapter))
|
||||
.map((notificationAdapter) => {
|
||||
const found = findAdapter(notificationAdapter);
|
||||
if (!found) {
|
||||
logger.warn(`Notification adapter '${notificationAdapter.id}' not found for job '${jobKey || ''}'`);
|
||||
}
|
||||
return found;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey, baseUrl }));
|
||||
};
|
||||
|
||||
@@ -105,14 +105,11 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
||||
return;
|
||||
}
|
||||
settings.lastRun = now;
|
||||
const jobs = jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.filter((job) => {
|
||||
if (!context) return true; // startup/cron → all
|
||||
if (context.isAdmin) return true; // admin → all
|
||||
return context.userId ? job.userId === context.userId : false; // user → own
|
||||
});
|
||||
const jobs = jobStorage.getJobs().filter((job) => {
|
||||
if (!context) return true; // startup/cron → all
|
||||
if (context.isAdmin) return true; // admin → all
|
||||
return context.userId ? job.userId === context.userId : false; // user → own
|
||||
});
|
||||
|
||||
for (const job of jobs) {
|
||||
await executeJob(job);
|
||||
|
||||
@@ -16,7 +16,7 @@ import logger from '../../services/logger.js';
|
||||
* Concurrency: network-bound checks are executed with a configurable concurrency limit.
|
||||
*
|
||||
* @param {object} [opts]
|
||||
* @param {number} [opts.concurrency=8] Max number of parallel activeTester calls.
|
||||
* @param {number} [opts.concurrency=4] Max number of parallel activeTester calls.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export default async function runActiveChecker(opts = {}) {
|
||||
|
||||
@@ -60,18 +60,14 @@ export const getListingsKpisForJobIds = (jobIds = []) => {
|
||||
|
||||
const placeholders = jobIds.map(() => '?').join(',');
|
||||
const rows = SqliteConnection.query(
|
||||
`SELECT
|
||||
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) OVER() AS active_count,
|
||||
price
|
||||
FROM listings
|
||||
WHERE job_id IN (${placeholders})
|
||||
AND manually_deleted = 0
|
||||
GROUP BY
|
||||
id`,
|
||||
`SELECT is_active, price
|
||||
FROM listings
|
||||
WHERE job_id IN (${placeholders})
|
||||
AND manually_deleted = 0`,
|
||||
jobIds,
|
||||
);
|
||||
|
||||
const activeCount = rows[0]?.active_count ?? 0;
|
||||
const activeCount = rows.filter((r) => r.is_active === 1).length;
|
||||
|
||||
const prices = rows
|
||||
.map((r) => r.price)
|
||||
@@ -308,13 +304,15 @@ export const queryListings = ({
|
||||
}
|
||||
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
|
||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
|
||||
whereParts.push(
|
||||
`(l.title LIKE @filter OR l.address LIKE @filter OR l.provider LIKE @filter OR l.link LIKE @filter)`,
|
||||
);
|
||||
}
|
||||
// activityFilter: when true -> only active listings (is_active = 1), false -> only inactive
|
||||
if (activityFilter === true) {
|
||||
whereParts.push('(is_active = 1)');
|
||||
whereParts.push('(l.is_active = 1)');
|
||||
} else if (activityFilter === false) {
|
||||
whereParts.push('(is_active = 0)');
|
||||
whereParts.push('(l.is_active = 0)');
|
||||
}
|
||||
// Prefer filtering by job id when provided (unambiguous and robust)
|
||||
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
|
||||
@@ -328,7 +326,7 @@ export const queryListings = ({
|
||||
// providerFilter: when provided as string (assumed provider name), filter listings where provider equals that name (exact match)
|
||||
if (providerFilter && String(providerFilter).trim().length > 0) {
|
||||
params.providerName = String(providerFilter).trim();
|
||||
whereParts.push('(provider = @providerName)');
|
||||
whereParts.push('(l.provider = @providerName)');
|
||||
}
|
||||
// watchListFilter: when true -> only watched listings, false -> only unwatched
|
||||
if (watchListFilter === true) {
|
||||
@@ -351,11 +349,11 @@ export const queryListings = ({
|
||||
// Time range filters (unix timestamps in milliseconds)
|
||||
if (Number.isFinite(createdAfter) && createdAfter > 0) {
|
||||
params.createdAfter = createdAfter;
|
||||
whereParts.push('(created_at >= @createdAfter)');
|
||||
whereParts.push('(l.created_at >= @createdAfter)');
|
||||
}
|
||||
if (Number.isFinite(createdBefore) && createdBefore > 0) {
|
||||
params.createdBefore = createdBefore;
|
||||
whereParts.push('(created_at <= @createdBefore)');
|
||||
whereParts.push('(l.created_at <= @createdBefore)');
|
||||
}
|
||||
// Price range filters
|
||||
if (Number.isFinite(minPrice) && minPrice >= 0) {
|
||||
@@ -370,32 +368,22 @@ export const queryListings = ({
|
||||
// Build whereSql (filtering by manually_deleted = 0)
|
||||
whereParts.push('(l.manually_deleted = 0)');
|
||||
|
||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
const whereSqlWithAlias = whereSql
|
||||
.replace(/\btitle\b/g, 'l.title')
|
||||
.replace(/\bdescription\b/g, 'l.description')
|
||||
.replace(/\baddress\b/g, 'l.address')
|
||||
.replace(/\bprovider\b/g, 'l.provider')
|
||||
.replace(/\blink\b/g, 'l.link')
|
||||
.replace(/\bis_active\b/g, 'l.is_active')
|
||||
.replace(/\bj\.user_id\b/g, 'j.user_id')
|
||||
.replace(/\bj\.name\b/g, 'j.name')
|
||||
.replace(/\bwl\.id\b/g, 'wl.id');
|
||||
const whereSqlWithAlias = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
|
||||
// whitelist sortable fields to avoid SQL injection
|
||||
const sortable = new Set(['created_at', 'price', 'size', 'provider', 'title', 'job_name', 'is_active', 'isWatched']);
|
||||
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
|
||||
// whitelist sortable fields to avoid SQL injection; map to fully-qualified expressions
|
||||
const sortableMap = {
|
||||
created_at: 'l.created_at',
|
||||
price: 'l.price',
|
||||
size: 'l.size',
|
||||
provider: 'l.provider',
|
||||
title: 'l.title',
|
||||
job_name: 'j.name',
|
||||
is_active: 'l.is_active',
|
||||
isWatched: 'CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END',
|
||||
};
|
||||
const safeSortExpr = sortField && sortableMap[sortField] ? sortableMap[sortField] : null;
|
||||
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
|
||||
const orderSql = safeSortField ? `ORDER BY ${safeSortField} ${safeSortDir}` : 'ORDER BY created_at DESC';
|
||||
const orderSqlWithAlias = orderSql
|
||||
.replace(/\bcreated_at\b/g, 'l.created_at')
|
||||
.replace(/\bprice\b/g, 'l.price')
|
||||
.replace(/\bsize\b/g, 'l.size')
|
||||
.replace(/\bprovider\b/g, 'l.provider')
|
||||
.replace(/\btitle\b/g, 'l.title')
|
||||
.replace(/\bjob_name\b/g, 'j.name')
|
||||
// Sort by computed watch flag when requested
|
||||
.replace(/\bisWatched\b/g, 'CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END');
|
||||
const orderSqlWithAlias = safeSortExpr ? `ORDER BY ${safeSortExpr} ${safeSortDir}` : 'ORDER BY l.created_at DESC';
|
||||
|
||||
// count total with same WHERE
|
||||
const countRow = SqliteConnection.query(
|
||||
@@ -516,7 +504,7 @@ export const updateListingGeocoordinates = (id, latitude, longitude) => {
|
||||
* @param {string} [params.jobId]
|
||||
* @param {string} [params.userId]
|
||||
* @param {boolean} [params.isAdmin=false]
|
||||
* @returns {{listings: Object[], maxPrice: number}} Object containing listings and maxPrice.
|
||||
* @returns {{listings: Object[]}} Object containing listings.
|
||||
*/
|
||||
export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}) => {
|
||||
const baseWhereParts = [
|
||||
|
||||
@@ -123,8 +123,11 @@ export function upsertSettings(settingsMapOrEntry, userId = null) {
|
||||
);
|
||||
}
|
||||
}
|
||||
// keep cache in sync (only for global settings)
|
||||
// Invalidate cache synchronously so the next getSettings() call rebuilds it.
|
||||
// refreshSettingsCache() is async (reads config.json), so we cannot await it
|
||||
// here without making upsertSettings async everywhere. Nulling is safe because
|
||||
// getSettings() will call refreshSettingsCache() on the next invocation.
|
||||
if (userId == null) {
|
||||
refreshSettingsCache();
|
||||
cachedSettingsConfig = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fredy",
|
||||
"version": "22.3.1",
|
||||
"version": "22.3.2",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"prepare": "husky",
|
||||
|
||||
@@ -335,6 +335,61 @@ describe('telegram send() - mixed batch (regression-safety)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('telegram send() - multiple chat IDs', () => {
|
||||
const listing = {
|
||||
id: '1',
|
||||
title: 'Flat',
|
||||
link: 'https://ex.com',
|
||||
address: 'Berlin',
|
||||
price: '800',
|
||||
size: '50',
|
||||
image: 'https://ex.com/img.jpg',
|
||||
};
|
||||
|
||||
it('sends to every chat ID in a comma-separated list', async () => {
|
||||
mockNodeFetch.mockResolvedValue(jsonOk());
|
||||
|
||||
await send({
|
||||
serviceName: 'immoscout',
|
||||
newListings: [listing],
|
||||
notificationConfig: [{ id: 'telegram', fields: { token: 'TKN', chatId: '111, 222' } }],
|
||||
jobKey: 'Berlin',
|
||||
});
|
||||
|
||||
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||
const bodies = mockNodeFetch.mock.calls.map((c) => JSON.parse(c[1].body));
|
||||
expect(bodies.map((b) => b.chat_id)).toEqual(expect.arrayContaining(['111', '222']));
|
||||
});
|
||||
|
||||
it('trims whitespace around each chat ID', async () => {
|
||||
mockNodeFetch.mockResolvedValue(jsonOk());
|
||||
|
||||
await send({
|
||||
serviceName: 'immoscout',
|
||||
newListings: [listing],
|
||||
notificationConfig: [{ id: 'telegram', fields: { token: 'TKN', chatId: ' 333 , 444 ' } }],
|
||||
jobKey: 'Berlin',
|
||||
});
|
||||
|
||||
expect(mockNodeFetch).toHaveBeenCalledTimes(2);
|
||||
const bodies = mockNodeFetch.mock.calls.map((c) => JSON.parse(c[1].body));
|
||||
expect(bodies.map((b) => b.chat_id)).toEqual(expect.arrayContaining(['333', '444']));
|
||||
});
|
||||
|
||||
it('sends each listing to each chat ID (N listings × M chats)', async () => {
|
||||
mockNodeFetch.mockResolvedValue(jsonOk());
|
||||
|
||||
await send({
|
||||
serviceName: 'immoscout',
|
||||
newListings: [listing, { ...listing, id: '2' }],
|
||||
notificationConfig: [{ id: 'telegram', fields: { token: 'TKN', chatId: '555, 666' } }],
|
||||
jobKey: 'Berlin',
|
||||
});
|
||||
|
||||
expect(mockNodeFetch).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe('telegram send() - config validation', () => {
|
||||
it('throws when telegram adapter config is missing', () => {
|
||||
expect(() =>
|
||||
|
||||
Reference in New Issue
Block a user