From 6c1786dcc2a6767821c8b6e805fc321b3023871d Mon Sep 17 00:00:00 2001 From: Alexander Roidl <34438048+alexanderroidl@users.noreply.github.com> Date: Thu, 31 Jul 2025 21:29:34 +0200 Subject: [PATCH] WIP --- conf/config.json | 2 +- lib/notification/adapter/telegram.js | 130 +++++++++++++----- lib/provider/immoscout.js | 2 +- ...eb-translator.js => convertWebToMobile.js} | 7 +- lib/services/immoscout/index.js | 1 + 5 files changed, 107 insertions(+), 35 deletions(-) rename lib/services/immoscout/{immoscout-web-translator.js => convertWebToMobile.js} (98%) create mode 100644 lib/services/immoscout/index.js diff --git a/conf/config.json b/conf/config.json index e8c9521..3b48654 100755 --- a/conf/config.json +++ b/conf/config.json @@ -1 +1 @@ -{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null} \ No newline at end of file +{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":false} \ No newline at end of file diff --git a/lib/notification/adapter/telegram.js b/lib/notification/adapter/telegram.js index 4a1d516..199d01f 100644 --- a/lib/notification/adapter/telegram.js +++ b/lib/notification/adapter/telegram.js @@ -2,10 +2,13 @@ import { markdown2Html } from '../../services/markdown.js'; import { getJob } from '../../services/storage/jobStorage.js'; import fetch from 'node-fetch'; import pThrottle from 'p-throttle'; +import lodash from 'lodash'; const MAX_ENTITIES_PER_CHUNK = 8; const RATE_LIMIT_INTERVAL = 1000; const chatThrottleMap = new Map(); +const pollingTokens = new Set(); +const updateOffsets = new Map(); function cleanupOldThrottles() { const now = Date.now(); @@ -44,55 +47,120 @@ function getThrottled(chatId, call) { return newThrottle.throttled; } -/** - * splitting an array into chunks because Telegram only allows for messages up to - * 4096 chars, thus we have to split messages into chunks - * @param inputArray - * @param perChunk - */ -const arrayChunks = (inputArray, perChunk) => - inputArray.reduce((all, one, i) => { - const ch = Math.floor(i / perChunk); - all[ch] = [].concat(all[ch] || [], one); - return all; - }, []); -function shorten(str, len = 30) { - return str.length > len ? str.substring(0, len) + '...' : str; +function getCallbackUpdates(token) { + const offset = updateOffsets.get(token) || 0; + return fetch( + `https://api.telegram.org/bot${token}/getUpdates?allowed_updates=["callback_query"]&timeout=30&offset=${offset}`, + { + method: 'get', + headers: { 'Content-Type': 'application/json' }, + }, + ); } -export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { - const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields; - const job = getJob(jobKey); - const jobName = job == null ? jobKey : job.name; - const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK); - const getThrottledSend = getThrottled(chatId, async function (body) { +function getThrottledSend(token, chatId) { + return getThrottled(chatId, async function (body) { await fetch(`https://api.telegram.org/bot${token}/sendMessage`, { method: 'post', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' }, }); }); +} - const promises = chunks.map((chunk) => { +async function pollCallbackUpdates(token) { + setTimeout(() => pollCallbackUpdates(token), 1000); + + let callbackUpdates; + try { + callbackUpdates = await getCallbackUpdates(token); + } catch (error) { + console.error('An error occurred when polling callback updates.', error); + return; + } + + const updatesData = await callbackUpdates.json(); + + if (!updatesData.ok || !updatesData.result || updatesData.result.length === 0) { + return; + } + + // Process each callback query + for (const update of updatesData.result) { + if (update.callback_query) { + const callbackQuery = update.callback_query; + const callbackQueryId = callbackQuery.id; + + try { + // Answer the callback query to remove the loading state + await fetch(`https://api.telegram.org/bot${token}/answerCallbackQuery`, { + method: 'post', + body: JSON.stringify({ + callback_query_id: callbackQueryId, + text: '✅ Opening listing...', + show_alert: false, + }), + headers: { 'Content-Type': 'application/json' }, + }); + } catch (error) { + console.error('Error answering callback query:', error); + } + } + + // Update offset to avoid processing the same update again + updateOffsets.set(token, update.update_id + 1); + } +} + +function startCallbackPolling(token) { + if (!pollingTokens.has(token)) { + pollingTokens.add(token); + pollCallbackUpdates(token); + } +} + +export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { + const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields; + const throttledSend = getThrottledSend(token, chatId); + + // Start callback polling for this token if not already started + startCallbackPolling(token); + + const job = getJob(jobKey); + const jobName = job == null ? jobKey : job.name; + const chunks = lodash.chunk(newListings, MAX_ENTITIES_PER_CHUNK); + + const promises = chunks.map((listingsInChunk, chunkIndex) => { const messageParagraphs = []; + const inlineButtons = []; - messageParagraphs.push(`${jobName} (${serviceName}) found ${newListings.length} new listings:`); messageParagraphs.push( - ...chunk.map( - (o) => - `${shorten(o.title.replace(/\*/g, ''), 45).trim()}\n` + - [o.address, o.price, o.size].join(' | '), - ), + `${jobName} (${serviceName}) found ${newListings.length} new listings (${chunkIndex + 1}/${chunks.length}):`, ); - const body = { + listingsInChunk.forEach((listing, listingIndex) => { + const normalizedTitle = listing.title.replace(/\*/g, '').trim(); + const titleExcerpt = lodash.truncate(normalizedTitle, { length: 45, omission: '…' }); + + messageParagraphs.push(` +${listingIndex + 1}. ${titleExcerpt} +${[listing.address, listing.price, listing.size].join(' | ')}`); + + inlineButtons.push({ + text: `${listingIndex + 1}`, + url: listing.link, + }); + }); + + return throttledSend({ chat_id: chatId, text: messageParagraphs.join('\n\n'), parse_mode: 'HTML', disable_web_page_preview: true, - }; - - return getThrottledSend(body); + reply_markup: { + inline_keyboard: lodash.chunk(inlineButtons, 4), + }, + }); }); return Promise.all(promises); }; diff --git a/lib/provider/immoscout.js b/lib/provider/immoscout.js index 5351370..6a9b752 100644 --- a/lib/provider/immoscout.js +++ b/lib/provider/immoscout.js @@ -36,7 +36,7 @@ */ import utils, { buildHash } from '../utils.js'; -import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js'; +import { convertWebToMobile } from '../services/immoscout/index.js'; let appliedBlackList = []; async function getListings(url) { diff --git a/lib/services/immoscout/immoscout-web-translator.js b/lib/services/immoscout/convertWebToMobile.js similarity index 98% rename from lib/services/immoscout/immoscout-web-translator.js rename to lib/services/immoscout/convertWebToMobile.js index 18a21e0..132fa23 100644 --- a/lib/services/immoscout/immoscout-web-translator.js +++ b/lib/services/immoscout/convertWebToMobile.js @@ -164,6 +164,8 @@ export function convertWebToMobile(webUrl) { const mobileParams = { searchType: isRadius ? 'radius' : 'region', realestatetype: realType, + sorting: '-firstactivation', + pagesize: 25, ...(isRadius ? {} : { geocodes }), ...additionalParamsFromWebPath, }; @@ -180,9 +182,10 @@ export function convertWebToMobile(webUrl) { ...(currentEquipmentParams ?? []), ...items.map((item) => EQUIPMENT_MAP[item.toLowerCase()]).filter(Boolean), ]; - } else { - mobileParams[PARAM_NAME_MAP[key]] = val; + return; } + + mobileParams[PARAM_NAME_MAP[key]] = val; } const mobileQuery = queryString.stringify(mobileParams, { diff --git a/lib/services/immoscout/index.js b/lib/services/immoscout/index.js new file mode 100644 index 0000000..b720bc3 --- /dev/null +++ b/lib/services/immoscout/index.js @@ -0,0 +1 @@ +export * from './convertWebToMobile.js';