Compare commits

...

4 Commits

Author SHA1 Message Date
orangecoding
64d0515c79 next release version 2026-04-20 10:14:26 +02:00
bytedream
cc0164b689 change average price to median price on the dashboard (#300)
* change average price to median price on the dashboard

* Use more efficient median calculation

Co-authored-by: Christian Kellner <weakmap@gmail.com>

* Fix applied suggestion artifacts

* Update sql query and js sort function

* Group sql statement by id

* Revert sort function change

---------

Co-authored-by: Christian Kellner <weakmap@gmail.com>
2026-04-20 10:13:11 +02:00
orangecoding
522bbc2282 upgrade dependencies 2026-04-16 12:07:22 +02:00
Adrian Bartnik
c384781137 Add toggle for plain text message to telegram notification adapter (#299) 2026-04-16 12:05:57 +02:00
7 changed files with 478 additions and 393 deletions

View File

@@ -37,7 +37,7 @@ dashboardRouter.get('/', async (req, res) => {
const totalJobs = jobs.length; const totalJobs = jobs.length;
const totalListings = jobs.reduce((sum, j) => sum + (j.numberOfFoundListings || 0), 0); const totalListings = jobs.reduce((sum, j) => sum + (j.numberOfFoundListings || 0), 0);
const jobIds = jobs.map((j) => j.id); const jobIds = jobs.map((j) => j.id);
const { numberOfActiveListings, avgPriceOfListings } = getListingsKpisForJobIds(jobIds); const { numberOfActiveListings, medianPriceOfListings } = getListingsKpisForJobIds(jobIds);
// Build Pie data in a simple shape the frontend can consume directly // Build Pie data in a simple shape the frontend can consume directly
// Shape: { labels: string[], values: number[] } with values as percentages // Shape: { labels: string[], values: number[] } with values as percentages
const providerPieRaw = getProviderDistributionForJobIds(jobIds); const providerPieRaw = getProviderDistributionForJobIds(jobIds);
@@ -63,7 +63,7 @@ dashboardRouter.get('/', async (req, res) => {
totalJobs, totalJobs,
totalListings, totalListings,
numberOfActiveListings, numberOfActiveListings,
avgPriceOfListings, medianPriceOfListings,
}, },
pie: providerPie, pie: providerPie,
}; };

View File

@@ -105,6 +105,32 @@ function buildText(jobName, serviceName, o) {
); );
} }
/**
* Build a plain text Telegram photo caption (max 4096 characters).
* @param {string} jobName
* @param {string} serviceName
* @param {Object} o - Listing object
* @returns {string}
*/
function buildCaptionPlain(jobName, serviceName, o) {
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
return `${jobName} (${serviceName})\n${title}\n${meta}\n\n${o.link || ''}`.slice(0, 4096);
}
/**
* Build a plain text Telegram message.
* @param {string} jobName
* @param {string} serviceName
* @param {Object} o - Listing object
* @returns {string}
*/
function buildTextPlain(jobName, serviceName, o) {
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
return `${jobName} (${serviceName})\n${title}\n${o.link || ''}\n${meta}`;
}
/** /**
* Send new listings to Telegram. * Send new listings to Telegram.
* - Respects per-chat Telegram rate limits using a lightweight throttle cache. * - Respects per-chat Telegram rate limits using a lightweight throttle cache.
@@ -122,7 +148,7 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
if (!adapterCfg || !adapterCfg.fields) { if (!adapterCfg || !adapterCfg.fields) {
throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`); throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`);
} }
const { token, chatId, messageThreadId } = adapterCfg.fields; const { token, chatId, messageThreadId, plainText } = adapterCfg.fields;
if (!token || !chatId) { if (!token || !chatId) {
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config"); throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
} }
@@ -163,8 +189,8 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
const img = normalizeImageUrl(o.image); const img = normalizeImageUrl(o.image);
const textPayload = { const textPayload = {
chat_id: chatId, chat_id: chatId,
text: buildText(jobName, serviceName, o), text: plainText ? buildTextPlain(jobName, serviceName, o) : buildText(jobName, serviceName, o),
parse_mode: 'HTML', ...(plainText ? {} : { parse_mode: 'HTML' }),
disable_web_page_preview: true, disable_web_page_preview: true,
...(message_thread_id ? { message_thread_id } : {}), ...(message_thread_id ? { message_thread_id } : {}),
}; };
@@ -178,8 +204,8 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
return await throttledCall('sendPhoto', { return await throttledCall('sendPhoto', {
chat_id: chatId, chat_id: chatId,
photo: img, photo: img,
caption: buildCaption(jobName, serviceName, o), caption: plainText ? buildCaptionPlain(jobName, serviceName, o) : buildCaption(jobName, serviceName, o),
parse_mode: 'HTML', ...(plainText ? {} : { parse_mode: 'HTML' }),
...(message_thread_id ? { message_thread_id } : {}), ...(message_thread_id ? { message_thread_id } : {}),
}).catch(async (e) => { }).catch(async (e) => {
logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`); logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
@@ -220,5 +246,11 @@ export const config = {
description: description:
'Optional: The topic/thread id within a supergroup to post into (Telegram message_thread_id). Provide a positive integer.', 'Optional: The topic/thread id within a supergroup to post into (Telegram message_thread_id). Provide a positive integer.',
}, },
plainText: {
type: 'boolean',
optional: true,
label: 'Send as plain text',
description: 'Send messages as plain text instead of HTML formatted.',
},
}, },
}; };

View File

@@ -29,33 +29,47 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => {
* Compute KPI aggregates for a given set of job IDs from the listings table. * Compute KPI aggregates for a given set of job IDs from the listings table.
* *
* - numberOfActiveListings: count of listings where is_active = 1 * - numberOfActiveListings: count of listings where is_active = 1
* - avgPriceOfListings: average of numeric price, rounded to nearest integer * - medianPriceOfListings: median of numeric price, rounded to nearest integer
* *
* When no jobIds are provided, returns zeros. * When no jobIds are provided, returns zeros.
* *
* @param {string[]} jobIds * @param {string[]} jobIds
* @returns {{ numberOfActiveListings: number, avgPriceOfListings: number }} * @returns {{ numberOfActiveListings: number, medianPriceOfListings: number }}
*/ */
export const getListingsKpisForJobIds = (jobIds = []) => { export const getListingsKpisForJobIds = (jobIds = []) => {
if (!Array.isArray(jobIds) || jobIds.length === 0) { if (!Array.isArray(jobIds) || jobIds.length === 0) {
return { numberOfActiveListings: 0, avgPriceOfListings: 0 }; return { numberOfActiveListings: 0, medianPriceOfListings: 0 };
} }
const placeholders = jobIds.map(() => '?').join(','); const placeholders = jobIds.map(() => '?').join(',');
const row = const rows = SqliteConnection.query(
SqliteConnection.query( `SELECT
`SELECT SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) OVER() AS active_count,
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount, price
AVG(price) AS avgPrice FROM listings
FROM listings WHERE job_id IN (${placeholders})
WHERE job_id IN (${placeholders}) AND manually_deleted = 0
AND manually_deleted = 0`, GROUP BY
jobIds, id`,
)[0] || {}; jobIds,
);
const activeCount = rows[0]?.active_count ?? 0;
const prices = rows
.map((r) => r.price)
.filter((p) => p !== null)
.sort((a, b) => a - b);
let medianPrice = 0;
if (prices.length > 0) {
const mid = Math.floor(prices.length / 2);
medianPrice = prices.length % 2 !== 0 ? prices[mid] : (prices[mid - 1] + prices[mid]) / 2;
}
return { return {
numberOfActiveListings: Number(row.activeCount || 0), numberOfActiveListings: activeCount,
avgPriceOfListings: row?.avgPrice == null ? 0 : Math.round(Number(row.avgPrice)), medianPriceOfListings: medianPrice,
}; };
}; };

View File

@@ -1,6 +1,6 @@
{ {
"name": "fredy", "name": "fredy",
"version": "20.3.0", "version": "20.3.2",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"prepare": "husky", "prepare": "husky",
@@ -61,45 +61,45 @@
"Firefox ESR" "Firefox ESR"
], ],
"dependencies": { "dependencies": {
"@douyinfe/semi-icons": "^2.93.0", "@douyinfe/semi-icons": "^2.95.0",
"@douyinfe/semi-ui": "2.93.0", "@douyinfe/semi-ui": "2.95.0",
"@douyinfe/semi-ui-19": "^2.93.0", "@douyinfe/semi-ui-19": "^2.95.0",
"@mapbox/mapbox-gl-draw": "^1.5.1", "@mapbox/mapbox-gl-draw": "^1.5.1",
"@modelcontextprotocol/sdk": "^1.29.0", "@modelcontextprotocol/sdk": "^1.29.0",
"@sendgrid/mail": "8.1.6", "@sendgrid/mail": "8.1.6",
"@turf/boolean-point-in-polygon": "^7.3.4", "@turf/boolean-point-in-polygon": "^7.3.5",
"@vitejs/plugin-react": "6.0.1", "@vitejs/plugin-react": "6.0.1",
"adm-zip": "^0.5.17", "adm-zip": "^0.5.17",
"better-sqlite3": "^12.8.0", "better-sqlite3": "^12.9.0",
"body-parser": "2.2.2", "body-parser": "2.2.2",
"chart.js": "^4.5.1", "chart.js": "^4.5.1",
"cheerio": "^1.2.0", "cheerio": "^1.2.0",
"cookie-session": "2.1.1", "cookie-session": "2.1.1",
"handlebars": "4.7.9", "handlebars": "4.7.9",
"maplibre-gl": "^5.22.0", "maplibre-gl": "^5.23.0",
"nanoid": "5.1.7", "nanoid": "5.1.9",
"node-cron": "^4.2.1", "node-cron": "^4.2.1",
"node-fetch": "3.3.2", "node-fetch": "3.3.2",
"node-mailjet": "6.0.11", "node-mailjet": "6.0.11",
"nodemailer": "^8.0.5", "nodemailer": "^8.0.5",
"p-throttle": "^8.1.0", "p-throttle": "^8.1.0",
"package-up": "^5.0.0", "package-up": "^5.0.0",
"puppeteer": "^24.40.0", "puppeteer": "^24.41.0",
"puppeteer-extra": "^3.3.6", "puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.3.1", "query-string": "9.3.1",
"react": "19.2.4", "react": "19.2.5",
"react-chartjs-2": "^5.3.1", "react-chartjs-2": "^5.3.1",
"react-dom": "19.2.4", "react-dom": "19.2.5",
"react-range-slider-input": "^3.3.5", "react-range-slider-input": "^3.3.5",
"react-router": "7.14.0", "react-router": "7.14.1",
"react-router-dom": "7.14.0", "react-router-dom": "7.14.1",
"resend": "^6.10.0", "resend": "^6.12.0",
"restana": "5.1.0", "restana": "5.2.0",
"semver": "^7.7.4", "semver": "^7.7.4",
"serve-static": "2.2.1", "serve-static": "2.2.1",
"slack": "11.0.2", "slack": "11.0.2",
"vite": "8.0.7", "vite": "8.0.9",
"x-var": "^3.0.1", "x-var": "^3.0.1",
"zustand": "^5.0.12" "zustand": "^5.0.12"
}, },
@@ -110,16 +110,16 @@
"@babel/preset-react": "7.28.5", "@babel/preset-react": "7.28.5",
"@eslint/js": "^10.0.1", "@eslint/js": "^10.0.1",
"chalk": "^5.6.2", "chalk": "^5.6.2",
"eslint": "10.2.0", "eslint": "10.2.1",
"eslint-config-prettier": "10.1.8", "eslint-config-prettier": "10.1.8",
"eslint-plugin-react": "7.37.5", "eslint-plugin-react": "7.37.5",
"globals": "^17.4.0", "globals": "^17.5.0",
"history": "5.3.0", "history": "5.3.0",
"husky": "9.1.7", "husky": "9.1.7",
"less": "4.6.4", "less": "4.6.4",
"lint-staged": "16.4.0", "lint-staged": "16.4.0",
"nodemon": "^3.1.14", "nodemon": "^3.1.14",
"prettier": "3.8.1", "prettier": "3.8.3",
"vitest": "^4.1.3" "vitest": "^4.1.4"
} }
} }

View File

@@ -127,18 +127,18 @@ export default function Dashboard() {
</Col> </Col>
<Col xs={24} sm={12} md={12} lg={6} xl={6}> <Col xs={24} sm={12} md={12} lg={6} xl={6}>
<KpiCard <KpiCard
title="Avg. Price" title="Median Price"
color="purple" color="purple"
value={`${ value={`${
!kpis.avgPriceOfListings !kpis.medianPriceOfListings
? '---' ? '---'
: new Intl.NumberFormat('de-DE', { : new Intl.NumberFormat('de-DE', {
style: 'currency', style: 'currency',
currency: 'EUR', currency: 'EUR',
}).format(kpis.avgPriceOfListings) }).format(kpis.medianPriceOfListings)
}`} }`}
icon={<IconNoteMoney />} icon={<IconNoteMoney />}
description="Avg. Price of listings" description="Median Price of listings"
/> />
</Col> </Col>
</Row> </Row>

View File

@@ -156,14 +156,21 @@ export default function NotificationAdapterMutator({
return ( return (
<Form key={key}> <Form key={key}>
{uiElement.type === 'boolean' ? ( {uiElement.type === 'boolean' ? (
<div style={{ display: 'flex', alignItems: 'center', gap: '16px' }}> <div>
<Switch <div style={{ display: 'flex', alignItems: 'center', gap: '16px' }}>
checked={uiElement.value || false} <Switch
onChange={(checked) => { checked={uiElement.value || false}
setValue(selectedAdapter, uiElement, key, checked); onChange={(checked) => {
}} setValue(selectedAdapter, uiElement, key, checked);
/> }}
{uiElement.label} />
{uiElement.label}
</div>
{uiElement.description && (
<div className="semi-form-field-extra" style={{ marginTop: '4px' }}>
{uiElement.description}
</div>
)}
</div> </div>
) : ( ) : (
<Form.Input <Form.Input

710
yarn.lock

File diff suppressed because it is too large Load Diff