Compare commits

..

9 Commits

Author SHA1 Message Date
orangecoding
7879d0e94a next release version 2026-01-26 12:35:57 +01:00
orangecoding
afd1048c9e hardening the check if a listing is active 2026-01-26 12:34:49 +01:00
orangecoding
acbaab05ed next release version 2026-01-26 12:07:43 +01:00
orangecoding
72fffc526b deleting a listing now sets it to deleted in the db, preventing it from reappearing when scraping happens 2026-01-26 12:07:21 +01:00
orangecoding
9e5989ece3 zoom into map where most markers are 2026-01-26 11:54:47 +01:00
orangecoding
afc200c9e1 improved tooltip in map, improved user-settings handling 2026-01-26 11:50:16 +01:00
orangecoding
59226491f2 improved tooltip in map, improved user-settings handling 2026-01-26 11:20:02 +01:00
orangecoding
28f7760120 adapt link to listing in grid view to behave like a real link 2026-01-26 10:43:38 +01:00
orangecoding
2465514b7a fixing immoscout translator, allowing balcony and garden for purchases 2026-01-26 10:20:21 +01:00
16 changed files with 309 additions and 80 deletions

View File

@@ -12,6 +12,7 @@ import { calculateDistanceForUser } from '../../services/geocoding/distanceServi
import { fromJson } from '../../utils.js';
import { trackFeature } from '../../services/tracking/Tracker.js';
import { FEATURES } from '../../features.js';
import logger from '../../services/logger.js';
const service = restana();
const userSettingsRouter = service.newRouter();
@@ -34,11 +35,12 @@ userSettingsRouter.get('/autocomplete', async (req, res) => {
res.body = results;
res.send();
} catch (error) {
res.status(500).send({ error: error.message });
res.statusCode = 500;
res.send({ error: error.message });
}
});
userSettingsRouter.post('/', async (req, res) => {
userSettingsRouter.post('/home-address', async (req, res) => {
const userId = req.session.currentUser;
const { home_address } = req.body;
@@ -51,15 +53,17 @@ userSettingsRouter.post('/', async (req, res) => {
calculateDistanceForUser(userId);
res.send({ success: true, coords });
} else {
res.status(400).send({ error: 'Could not geocode address' });
res.statusCode = 400;
res.send({ error: 'Could not geocode address' });
}
} else {
// If address is empty, maybe clear it?
upsertSettings({ home_address: null }, userId);
res.send({ success: true });
}
} catch (error) {
res.status(500).send({ error: error.message });
logger.error('Error updating home address settings', error);
res.statusCode = 500;
res.send({ error: error.message });
}
});

View File

@@ -36,7 +36,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
activeTester: (url) => checkIfListingIsActive(url, 'Angebot nicht gefunden'),
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -109,7 +109,9 @@ const REAL_ESTATE_TYPE = {
const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
// Category "Balkon/Terrasse"
'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
'wohnung-kaufen-mit-balkon': { equipment: ['balcony'] },
'wohnung-mit-garten-mieten': { equipment: ['garden'] },
'eigentumswohnung-mit-garten': { equipment: ['garden'] },
// Category "Wohnungstyp"
'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },

View File

@@ -8,38 +8,71 @@ import { randomBetween, sleep } from '../../utils.js';
const maxAttempts = 3;
const userAgents = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
];
/**
* Check if a listing is still active with up to 3 attempts and exponential backoff.
* Backoff waits are capped and the last wait is at most 2000 ms.
* Check if a listing is still active with up to 5 attempts and exponential backoff.
* Backoff waits are randomized and capped.
*
* Rules:
* - HTTP 200 => return 1
* - HTTP 200 => return 1 (if checkForText is provided and found, returns 0)
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
* - HTTP 404 => return 0
* - Other statuses or network errors => retry until attempts are exhausted
*
* @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
* @returns {Promise<Integer>} 1 if active, 0 if not active and -1 if detected as bot
*/
export default async function checkIfListingIsActive(link) {
export default async function checkIfListingIsActive(link, checkForText = null) {
await sleep(randomBetween(50, 100));
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const userAgent = userAgents[Math.floor(Math.random() * userAgents.length)];
const res = await fetch(link, {
redirect: 'manual',
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
'User-Agent': userAgent,
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
Referer: 'https://www.google.com/',
},
});
if (res.status === 200) {
if (checkForText) {
const htmText = await res.text();
if (htmText.includes(checkForText)) {
return 0;
}
}
return 1;
}
if (res.status === 401) return -1;
if (res.status === 403) return -1;
if (res.status === 404) return 0;
if (res.status === 401 || res.status === 403) {
if (attempt < maxAttempts) {
await sleep(backoffDelay(attempt));
continue;
}
return -1;
}
if (res.status === 404 || res.status === 410) return 0;
// For any other status, only retry if attempts remain
if (attempt < maxAttempts) {
@@ -62,13 +95,13 @@ export default async function checkIfListingIsActive(link) {
}
/**
* Exponential backoff delay with cap.
* attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
* Exponential backoff delay with cap and jitter.
* @param {number} attempt 1-based attempt index
* @returns {number} delay in ms
*/
function backoffDelay(attempt) {
const base = 500;
const cap = 2000;
return Math.min(base * 2 ** (attempt - 1), cap);
const delay = Math.min(base * 2 ** (attempt - 1), cap);
return delay + randomBetween(0, 1000);
}

View File

@@ -48,7 +48,8 @@ export const getListingsKpisForJobIds = (jobIds = []) => {
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount,
AVG(price) AS avgPrice
FROM listings
WHERE job_id IN (${placeholders})`,
WHERE job_id IN (${placeholders})
AND manually_deleted = 0`,
jobIds,
)[0] || {};
@@ -80,6 +81,7 @@ export const getProviderDistributionForJobIds = (jobIds = []) => {
`SELECT provider, COUNT(*) AS cnt
FROM listings
WHERE job_id IN (${placeholders})
AND manually_deleted = 0
GROUP BY provider
ORDER BY cnt DESC`,
jobIds,
@@ -118,8 +120,8 @@ export const getActiveOrUnknownListings = () => {
return SqliteConnection.query(
`SELECT *
FROM listings
WHERE is_active is null
OR is_active = 1
WHERE (is_active is null OR is_active = 1)
AND manually_deleted = 0
ORDER BY provider`,
);
};
@@ -306,6 +308,9 @@ export const queryListings = ({
whereParts.push('(wl.id IS NULL)');
}
// Build whereSql (filtering by manually_deleted = 0)
whereParts.push('(l.manually_deleted = 0)');
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
const whereSqlWithAlias = whereSql
.replace(/\btitle\b/g, 'l.title')
@@ -370,8 +375,8 @@ export const queryListings = ({
export const deleteListingsByJobId = (jobId) => {
if (!jobId) return;
return SqliteConnection.execute(
`DELETE
FROM listings
`UPDATE listings
SET manually_deleted = 1
WHERE job_id = @jobId`,
{ jobId },
);
@@ -387,9 +392,9 @@ export const deleteListingsById = (ids) => {
if (!Array.isArray(ids) || ids.length === 0) return;
const placeholders = ids.map(() => '?').join(',');
return SqliteConnection.execute(
`DELETE
FROM listings
WHERE id IN (${placeholders})`,
`UPDATE listings
SET manually_deleted = 1
WHERE id IN (${placeholders})`,
ids,
);
};
@@ -404,6 +409,7 @@ export const getListingsToGeocode = () => {
`SELECT id, address
FROM listings
WHERE is_active = 1
AND manually_deleted = 0
AND address IS NOT NULL
AND (latitude IS NULL OR longitude IS NULL)`,
);
@@ -443,6 +449,7 @@ export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}
'l.latitude != -1',
'l.longitude != -1',
'l.is_active = 1',
'l.manually_deleted = 0',
];
const params = { userId: userId || '__NO_USER__' };
@@ -479,7 +486,7 @@ export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}
* @returns {{title: string|null, address: string|null, price: number|null}[]}
*/
export const getAllEntriesFromListings = () => {
return SqliteConnection.query(`SELECT title, address, price FROM listings`);
return SqliteConnection.query(`SELECT title, address, price FROM listings WHERE manually_deleted = 0`);
};
/**
@@ -493,6 +500,7 @@ export const getGeocoordinatesByAddress = (address) => {
`SELECT latitude, longitude
FROM listings
WHERE address = @address
AND manually_deleted = 0
AND latitude IS NOT NULL
AND longitude IS NOT NULL
AND latitude != -1
@@ -515,6 +523,7 @@ export const getListingsToCalculateDistance = (jobId) => {
FROM listings
WHERE job_id = @jobId
AND is_active = 1
AND manually_deleted = 0
AND latitude IS NOT NULL
AND longitude IS NOT NULL
AND distance_to_destination IS NULL`,
@@ -535,6 +544,7 @@ export const getListingsForUserToCalculateDistance = (userId) => {
JOIN jobs j ON l.job_id = j.id
WHERE j.user_id = @userId
AND l.is_active = 1
AND l.manually_deleted = 0
AND l.latitude IS NOT NULL
AND l.longitude IS NOT NULL`,
{ userId },

View File

@@ -0,0 +1,16 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export function up(db) {
// 1. Add manually_deleted column
db.exec(`ALTER TABLE listings ADD COLUMN manually_deleted INTEGER NOT NULL DEFAULT 0;`);
// 2. Remove change_set column
try {
db.exec(`ALTER TABLE listings DROP COLUMN change_set;`);
} catch {
// if column does not exists for whatever reason
}
}

View File

@@ -90,15 +90,25 @@ export function upsertSettings(settingsMapOrEntry, userId = null) {
: Object.entries(settingsMapOrEntry || {});
for (const [name, rawValue] of entries) {
const id = nanoid();
const create_date = Date.now();
const json = toJson(rawValue);
SqliteConnection.execute(
`INSERT INTO settings (id, create_date, name, value, user_id)
if (rawValue === null) {
SqliteConnection.execute(
`DELETE FROM settings WHERE name = @name AND (user_id = @userId OR (user_id IS NULL AND @userId IS NULL))`,
{
name,
userId,
},
);
} else {
const id = nanoid();
const create_date = Date.now();
const json = toJson(rawValue);
SqliteConnection.execute(
`INSERT INTO settings (id, create_date, name, value, user_id)
VALUES (@id, @create_date, @name, @value, @userId)
ON CONFLICT(name, IFNULL(user_id, 'GLOBAL_SETTING')) DO UPDATE SET value = excluded.value`,
{ id, create_date, name, value: json, userId },
);
{ id, create_date, name, value: json, userId },
);
}
}
// keep cache in sync (only for global settings)
if (userId == null) {

View File

@@ -95,7 +95,7 @@ function isOneOf(word, arr) {
* @returns {boolean}
*/
function nullOrEmpty(val) {
return val == null || val.length === 0 || val === 'null' || val === 'undefined';
return val == null || val.length === 0;
}
/**

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "19.1.0",
"version": "19.2.1",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",

View File

@@ -40,6 +40,8 @@ a:active {
text-decoration: underline;
}
a {outline : none;}
.semi-icon:not(.semi-tabs-bar .semi-tabs-tab .semi-icon) {
vertical-align: middle;
}

View File

@@ -103,6 +103,10 @@ const ListingsGrid = () => {
setPage(_page);
};
const cap = (val) => {
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
};
return (
<div className="listingsGrid">
<div className="listingsGrid__searchbar">
@@ -251,11 +255,9 @@ const ListingsGrid = () => {
bodyStyle={{ padding: '12px' }}
>
<div className="listingsGrid__content">
<a href={item.url} target="_blank" rel="noopener noreferrer" className="listingsGrid__titleLink">
<Text strong ellipsis={{ showTooltip: true }} className="listingsGrid__title">
{item.title}
</Text>
</a>
<Text strong ellipsis={{ showTooltip: true }} className="listingsGrid__title">
{cap(item.title)}
</Text>
<Space vertical align="start" spacing={2} style={{ width: '100%', marginTop: 8 }}>
<Text type="secondary" icon={<IconCart />} size="small">
{item.price}
@@ -287,15 +289,11 @@ const ListingsGrid = () => {
</Space>
<Divider margin=".6rem" />
<div style={{ display: 'flex', justifyContent: 'space-between' }}>
<Button
title="Link to listing"
type="primary"
size="small"
onClick={async () => {
window.open(item.link);
}}
icon={<IconLink />}
/>
<div className="listingsGrid__linkButton">
<a href={item.link} target="_blank" rel="noopener noreferrer">
<IconLink />
</a>
</div>
<Button
title="Remove"

View File

@@ -103,4 +103,17 @@
&__setupButton {
margin-bottom: 1rem;
}
&__linkButton {
background: var(--semi-color-fill-0);
font-size: 14px;
line-height: 20px;
font-weight: 600;
height: 24px;
width: 24px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 3px;
}
}

View File

@@ -4,16 +4,21 @@
*/
import React, { useEffect, useRef, useState } from 'react';
import { renderToString } from 'react-dom/server';
import maplibregl from 'maplibre-gl';
import 'maplibre-gl/dist/maplibre-gl.css';
import { useSelector, useActions } from '../../services/state/store.js';
import { distanceMeters, generateCircleCoords, getBoundsFromCenter } from './mapUtils.js';
import { Select, Space, Typography, Button, Popover, Divider, Switch, Banner } from '@douyinfe/semi-ui-19';
import { IconFilter } from '@douyinfe/semi-icons';
import { distanceMeters, generateCircleCoords, getBoundsFromCenter, getBoundsFromCoords } from './mapUtils.js';
import { Select, Space, Typography, Button, Popover, Divider, Switch, Banner, Toast } from '@douyinfe/semi-ui-19';
import { IconFilter, IconLink } from '@douyinfe/semi-icons';
import { IconDelete } from '@douyinfe/semi-icons';
import no_image from '../../assets/no_image.jpg';
import RangeSlider from 'react-range-slider-input';
import 'react-range-slider-input/dist/style.css';
import './Map.less';
import { xhrDelete } from '../../services/xhr.js';
import { Link } from 'react-router';
const { Text } = Typography;
@@ -97,6 +102,22 @@ export default function MapView() {
return listings.filter((listing) => listing.price && listing.price >= min && listing.price <= max);
};
useEffect(() => {
window.deleteListing = async (id) => {
try {
await xhrDelete('/api/listings/', { ids: [id] });
Toast.success('Listing successfully removed');
fetchListings();
} catch (error) {
Toast.error(error.message || 'Error deleting listing');
}
};
return () => {
delete window.deleteListing;
};
}, []);
useEffect(() => {
if (map.current) return;
@@ -227,26 +248,42 @@ export default function MapView() {
}, [jobId]);
useEffect(() => {
if (!map.current || !homeAddress?.coords) return;
if (!map.current) return;
// We only want to zoom/fly when distanceFilter OR homeAddress actually change,
// not on every render. useEffect dependency array handles this.
if (distanceFilter > 0) {
const bounds = getBoundsFromCenter([homeAddress.coords.lng, homeAddress.coords.lat], distanceFilter);
if (homeAddress?.coords) {
// We only want to zoom/fly when distanceFilter OR homeAddress actually change,
// not on every render. useEffect dependency array handles this.
if (distanceFilter > 0) {
const bounds = getBoundsFromCenter([homeAddress.coords.lng, homeAddress.coords.lat], distanceFilter);
map.current.fitBounds(bounds, {
padding: 20,
maxZoom: 15,
duration: 1000,
});
map.current.fitBounds(bounds, {
padding: 20,
maxZoom: 15,
duration: 1000,
});
} else {
map.current.flyTo({
center: [homeAddress.coords.lng, homeAddress.coords.lat],
zoom: 12,
duration: 1000,
});
}
} else {
map.current.flyTo({
center: [homeAddress.coords.lng, homeAddress.coords.lat],
zoom: 12,
duration: 1000,
});
const filtered = filterListings();
const coords = filtered
.filter((l) => l.latitude != null && l.longitude != null && l.latitude !== -1 && l.longitude !== -1)
.map((l) => [l.longitude, l.latitude]);
if (coords.length > 0) {
const bounds = getBoundsFromCoords(coords);
map.current.fitBounds(bounds, {
padding: 50,
maxZoom: 15,
duration: 1000,
});
}
}
}, [homeAddress?.address, distanceFilter]);
}, [homeAddress?.address, distanceFilter, listings]);
useEffect(() => {
if (!map.current) return;
@@ -325,8 +362,8 @@ export default function MapView() {
? listing.provider.charAt(0).toUpperCase() + listing.provider.slice(1)
: 'N/A';
const popup = new maplibregl.Popup({ offset: 25 }).setHTML(
`<div class="map-popup-content">
const popupContent = `
<div class="map-popup-content">
<img src="${listing.image_url || no_image}" alt="${listing.title}" />
<h4>${listing.title}</h4>
<div class="info">
@@ -334,10 +371,25 @@ export default function MapView() {
<span><strong>Address:</strong> ${listing.address || 'N/A'}</span>
<span><strong>Job:</strong> ${listing.job_name || 'N/A'}</span>
<span><strong>Provider:</strong> ${capitalizedProvider}</span>
<a href="${listing.link}" target="_blank" rel="noopener noreferrer">View Listing</a>
<span><strong>Size:</strong> ${listing.size != null ? `${listing.size}` : 'N/A'}</span>
<div style="display: flex; gap: 8px; margin-top: 8px; justify-content: space-between;">
<div class="map-popup-content__linkButton">
<a href="${listing.link}" target="_blank" rel="noopener noreferrer">
${renderToString(<IconLink />)}
</a>
</div>
<button
class="map-popup-content__deleteButton"
title="Remove"
onclick="deleteListing('${listing.id}')"
>
${renderToString(<IconDelete />)}
</button>
</div>
</div>
</div>`,
);
</div>`;
const popup = new maplibregl.Popup({ offset: 25 }).setHTML(popupContent);
let color = '#3FB1CE'; // Default blue-ish
if (distanceFilter > 0 && homeAddress?.coords) {
@@ -468,7 +520,12 @@ export default function MapView() {
type="warning"
bordered
closeIcon={null}
description="You have not set your home address yet. Please do so in the settings to use the distance filter."
description={
<span>
You have not set your home address yet. Please do so in the <Link to="/userSettings">user settings</Link>{' '}
to use the distance filter.
</span>
}
/>
)}

View File

@@ -43,10 +43,62 @@
}
.info {
font-size: 0.9rem;
font-size: 0.8rem;
display: flex;
flex-direction: column;
gap: 0.2rem;
gap: 0.1rem;
}
&__linkButton {
background: var(--semi-color-primary);
color: white;
font-size: 14px;
line-height: 20px;
font-weight: 600;
height: 32px;
width: 32px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 6px;
a {
color: white;
display: flex;
align-items: center;
justify-content: center;
text-decoration: none;
width: 100%;
height: 100%;
}
&:hover {
background: var(--semi-color-primary-hover);
cursor: pointer;
}
}
&__deleteButton {
background: var(--semi-color-danger);
color: white;
border: none;
font-size: 14px;
height: 32px;
width: 32px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 6px;
cursor: pointer;
padding: 0;
&:hover {
background: var(--semi-color-danger-hover);
}
svg {
fill: currentColor;
}
}
}

View File

@@ -97,3 +97,34 @@ export const getBoundsFromCenter = (center, radiusInKm, padding = 0.15) => {
[lng + offsetLng, lat + offsetLat],
];
};
/**
* Calculates the bounding box for a set of coordinates.
*
* @param {number[][]} coords - Array of [longitude, latitude] coordinates
* @param {number} [padding=0.1] - Padding to add to the bounds
* @returns {number[][]} Bounding box coordinates [[minLon, minLat], [maxLon, maxLat]]
*/
export const getBoundsFromCoords = (coords, padding = 0.1) => {
if (!coords || coords.length === 0) return null;
let minLng = Infinity;
let minLat = Infinity;
let maxLng = -Infinity;
let maxLat = -Infinity;
coords.forEach(([lng, lat]) => {
if (lng < minLng) minLng = lng;
if (lng > maxLng) maxLng = lng;
if (lat < minLat) minLat = lat;
if (lat > maxLat) maxLat = lat;
});
const lngDiff = maxLng - minLng;
const latDiff = maxLat - minLat;
return [
[minLng - lngDiff * padding, minLat - latDiff * padding],
[maxLng + lngDiff * padding, maxLat + latDiff * padding],
];
};

View File

@@ -29,7 +29,7 @@ const UserSettings = () => {
const handleSave = async () => {
setSaving(true);
try {
const response = await xhrPost('/api/user/settings', { home_address: address });
const response = await xhrPost('/api/user/settings/home-address', { home_address: address });
if (response.status === 200) {
setCoords(response.json.coords);
await actions.userSettings.getUserSettings();
@@ -81,6 +81,7 @@ const UserSettings = () => {
<AutoComplete
data={dataSource}
value={address}
showClear
onChange={(v) => setAddress(v)}
onSearch={searchAddress}
placeholder="Enter your home address"