Compare commits

..

54 Commits

Author SHA1 Message Date
orangecoding
4878dc98e3 Merge branch 'master' of github.com:orangecoding/fredy 2026-03-11 15:26:56 +01:00
orangecoding
dc2704997d upgrading dependencies 2026-03-11 15:26:25 +01:00
orangecoding
e107b0fb00 next release version 2026-03-11 15:25:20 +01:00
Promises
6c08675fee Add new properties to real estate translation mappings (#275)
Added few more properties for buying a house
2026-03-11 14:49:21 +01:00
orangecoding
34c4de7267 fixing stdin for mcp 2026-03-10 09:27:04 +01:00
orangecoding
b64a118a18 moving mcp into lib to make it available in docker setup 2026-03-09 16:26:53 +01:00
orangecoding
03cb4d18cb fix formatting 2026-03-09 15:40:29 +01:00
orangecoding
be5c4af3cf adding an MCP Server 🎉 2026-03-09 15:35:29 +01:00
orangecoding
a460b813c1 adding news info 2026-03-08 10:07:51 +01:00
orangecoding
4596442f64 upgrading dependencies | mark listings as 'manually_removed' when filtered 2026-03-08 09:55:46 +01:00
Stephan
0bcfa1d4ad feat(): map area filter (#273)
* feat(): create map component, add area filtering to the job config

* feat(): filter listings by area filter

* chore(): cleanup

* feat(): solve feedback

* feat(): solve most providers

* feat(): solve maybe other providers
2026-03-08 09:44:18 +01:00
orangecoding
0cad05124a fixing immowelt provider 2026-03-08 09:29:40 +01:00
Noah Elijah Till
eb53b68d45 🕵️ More immoscout details (#258)
* 🕵️ More immoscout details

- Added more details to immoscout api - description is now populated with a lot of data from the expose using app API
- You can ignore certificates, if deploying locally and using the http notification adapter
- More details for the test call/example for easier testing + placeholder image + actual values + address (famous Erika Mustermans address see https://de.wikipedia.org/wiki/Mustermann)
- Grater timeout for geocode since the api is sometimes slow in germany
- uiElement, type boolean, now has a label as well

* 👀 Requested changes + some extra

Req:
- using logger
- using node-fetch

Extra:
- boolean input fields will trigger the validate check, because they are set undefined at first - setting them to false if they are undefined now
- added more data to the description (phone number and name of the agent)

*  Fixed import

* ️ Toggle immoscout detail fetching

* ️ Requested change
2026-03-08 09:08:40 +01:00
Tom Dohrmann
ba0732e1f6 add support for fulltext parameter for immoscout (#274) 2026-02-24 09:53:14 +01:00
orangecoding
aa67647bbb adding resend as net notification adapter 2026-02-20 17:08:38 +01:00
orangecoding
7a9d49899b improve reusing of puppeteer by adding a safeguard for broken chrome 2026-02-18 20:16:55 +01:00
orangecoding
9a87c58d3e next release version 2026-02-18 20:06:40 +01:00
orangecoding
fdd7e835e8 improve default puppeteer timeout 2026-02-18 20:06:22 +01:00
Christian Kellner
00d6a12b30 Puppeteer improvements (#270)
* improve puppeteer handling. Now only 1 puppeteer instance is being used which is WAY more efficient

* removing package-lock

* reduce logging

* removing problematic docker command

* Remove Immonet. They now belong to immowelt
2026-02-18 20:05:02 +01:00
orangecoding
05218800d2 fixing app init 2026-02-17 14:28:08 +01:00
orangecoding
19d4721f9f improve welcome screen 2026-02-17 14:03:15 +01:00
orangecoding
a794645393 fixing login route 2026-02-17 12:50:21 +01:00
orangecoding
fd7e228972 adding welcome screen 2026-02-17 12:35:39 +01:00
orangecoding
b86e351007 fixing lint even harder 2026-02-16 13:50:50 +01:00
orangecoding
19c4860da7 fixing eslint harder 2026-02-16 12:59:34 +01:00
orangecoding
d98e06cfdf fixing eslint 2026-02-16 12:40:41 +01:00
orangecoding
6ae0c9749b update dependencies 2026-02-16 12:30:59 +01:00
orangecoding
10e40e038e adding check if fredy is running in docker 2026-02-16 12:29:02 +01:00
orangecoding
4ba6828939 adding release tool 2026-02-05 12:02:18 +01:00
orangecoding
d09770dae2 fancy, almost impossible to see animation on dashboard 2026-02-05 09:54:42 +01:00
orangecoding
248e4d2562 improve tracking 2026-02-04 14:41:55 +01:00
orangecoding
7b8e961b49 adding confirmation dialog if to remove listing entirely from db or just hide it 2026-02-03 14:04:40 +01:00
orangecoding
f66ceccbb4 next release version 2026-01-29 13:01:39 +01:00
orangecoding
a3db725af6 fixing image rendering 2026-01-29 13:01:07 +01:00
orangecoding
0663bd945f smaller demo improvements 2026-01-29 09:46:23 +01:00
orangecoding
bc355fb5fe fixing some bugs the wife found ;) 2026-01-28 21:25:48 +01:00
orangecoding
797421f0d5 hardening demo handling 2026-01-28 16:29:59 +01:00
orangecoding
0b2b42fc75 improve geocoding 2026-01-28 15:55:23 +01:00
Christian Kellner
472169693f Improvements 01 28 (#264)
* improving footer

* improve ui

* upgrading dependencies

* adding glow to all boxes on dashboard

* introducing single listing view

* next release version

* improve screenshots and login page
2026-01-28 14:27:03 +01:00
orangecoding
3117044139 fixing immoscout scraper 2026-01-26 19:52:37 +01:00
orangecoding
7879d0e94a next release version 2026-01-26 12:35:57 +01:00
orangecoding
afd1048c9e hardening the check if a listing is active 2026-01-26 12:34:49 +01:00
orangecoding
acbaab05ed next release version 2026-01-26 12:07:43 +01:00
orangecoding
72fffc526b deleting a listing now sets it to deleted in the db, preventing it from reappearing when scraping happens 2026-01-26 12:07:21 +01:00
orangecoding
9e5989ece3 zoom into map where most markers are 2026-01-26 11:54:47 +01:00
orangecoding
afc200c9e1 improved tooltip in map, improved user-settings handling 2026-01-26 11:50:16 +01:00
orangecoding
59226491f2 improved tooltip in map, improved user-settings handling 2026-01-26 11:20:02 +01:00
orangecoding
28f7760120 adapt link to listing in grid view to behave like a real link 2026-01-26 10:43:38 +01:00
orangecoding
2465514b7a fixing immoscout translator, allowing balcony and garden for purchases 2026-01-26 10:20:21 +01:00
Christian Kellner
9dde377fe6 possibility to display distance (#262) 2026-01-25 13:52:56 +01:00
Katrin Leinweber
28a3a7f372 Use EUR-symbol to match Map.jsx (see d43c5b3) (#261)
Co-authored-by: Katrin Leinweber <katrinleinweber@noreply.github.com>
2026-01-25 12:32:11 +01:00
orangecoding
e859250545 next release version 2026-01-22 15:10:31 +00:00
Christian Kellner
4dd0370ec1 Calculating the distance (#255)
* migra for distance

* adding distance calculator

* adding ability to store home address

* improve distance calculation

* calculating distance

* show distance in grid view

* upgrading dependencies

* moving to react 19

* ability to clone a job

* fixing tests

* polishing
2026-01-22 16:09:36 +01:00
orangecoding
51b4e51f3f fixing setting kleinanzeigen listings to inactive if not available anymore 2026-01-16 11:36:51 +01:00
138 changed files with 6556 additions and 15318 deletions

1
.gitignore vendored
View File

@@ -6,3 +6,4 @@ npm-debug.log
.DS_Store
.idea
.vscode
tools/release/config.json

View File

@@ -1,94 +0,0 @@
Newer release changelog see https://github.com/orangecoding/fredy/releases
---
###### [V5.5.0]
- Upgrading dependencies
- fixing provider
- allow multiple instances of 1 provider
- **BREAKING**: Minimum node version is now 16
###### [V5.4.6]
- Adding Instana node.js monitoring
-
###### [V5.4.5]
- Adding Instana node.js monitoring
###### [V5.4.4]
- Add support for Immo Südwest Presse (immo.swp.de)
- Telegram: Use job name instead of ID and link in title
- Fix race condition if user ID is in session but not in user store
- Allow visiting the original provider URL
###### [V5.4.3]
- re-writing readme
- improving docker build
- using github's actions to build docker and test automatically
###### [V5.4.2]
- Fixing prod build
###### [V5.4.1]
- Upgrading dependencies
- Provider urls are now automagically been changed to include the correct sort order for search results
```
Note: It has been an point of confusion since the very beginning of Fredy, that people simply copied the url, but
did not take care of sorting the search results by date. If this is not done, Fredy will most likely not see the latest
results, thus cannot report them. This release fixes it by adding the necessary params (or replaces them).
```
###### [V5.3.0]
- Upgrading dependencies
- It's now possible to send mails to multiple receiver using comma separation for MailJet & Sendgrid
- Fixing Immowelt scraping
###### [V5.2.0]
- Upgrading dependencies
- Adding new similarity check layer (Duplicates are being removed now)
- Adding paging for search results
###### [V5.1.0]
- Upgrading dependencies
- NodeJS 12.13 is now the minimum supported version
- Adding general settings as new configuration page to ui
- Adding new feature working hours
###### [V5.0.0]
- Upgrading dependencies
- NodeJS 12 is now the minimum supported version
###### [V4.0.0]
Bringing back Immoscout :tada:
###### [V3.0.0]
This is basically a re-write, your old config file will not be compatible anymore. Please re-created your search jobs
on the new ui and use the values from your previous config file if needed.
```
- We're getting rid of manual config changes, Fredy, now ships with a UI so that it's easy for you to create and edit jobs
```
###### [V2.0.0]
```
- Fredy can now run multiple search job on one instance
- Changed lot's of the structure of Fredy to make this happen
[BREAKING CHANGES]
- The config has been changed, the config of V1.x will not work any longer
- Sources have been renamed to provider
```

View File

@@ -35,6 +35,7 @@ WORKDIR /fredy
RUN apk add --no-cache chromium curl
ENV NODE_ENV=production \
IS_DOCKER=true \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser

View File

@@ -154,6 +154,13 @@ to Slack + Telegram."\
Jobs run automatically at the interval you configure (see
`/conf/config.json`).
### MCP Server 🤖
Starting with **V20**, Fredy ships with a built-in **MCP Server **. This allows you to connect Fredy to LLMs (like Claude, ChatGPT, or local models via LM Studio) and query your real estate data using natural language.
The local LLM can even enrich existing listings by checking the listing online.
For more information on how to set it up and use it, please refer to the [MCP Readme](lib/mcp/README.md).
------------------------------------------------------------------------
## Immoscout

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.7 MiB

After

Width:  |  Height:  |  Size: 4.8 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 402 KiB

After

Width:  |  Height:  |  Size: 531 KiB

View File

@@ -8,20 +8,20 @@ import js from '@eslint/js';
import prettier from 'eslint-config-prettier';
import globals from 'globals';
import react from 'eslint-plugin-react';
import babelParser from '@babel/eslint-parser';
export default [
{
files: ['**/*.{js,jsx,ts,tsx}'],
ignores: ['**/node_modules/**', '**/dist/**', '**/build/**', '**/public/**', 'db/**', 'conf/**'],
},
js.configs.recommended,
prettier,
{
files: ['**/*.{js,jsx}'],
languageOptions: {
parser: babelParser,
ecmaVersion: 'latest',
sourceType: 'module',
ecmaVersion: 2021,
parserOptions: {
ecmaFeatures: { jsx: true },
},
globals: {
...globals.browser,
...globals.node,
@@ -32,70 +32,14 @@ export default [
after: 'readonly',
it: 'readonly',
},
parserOptions: { requireConfigFile: false },
},
plugins: { react },
rules: {
eqeqeq: [2, 'allow-null'],
strict: 0,
'no-redeclare': [2, { builtinGlobals: false }],
'class-methods-use-this': 'off',
indent: ['off', 2],
'linebreak-style': ['error', 'unix'],
quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }],
semi: ['error', 'always'],
'no-console': ['error', { allow: ['warn', 'error'] }],
'jsx-quotes': ['error', 'prefer-double'],
'react/display-name': 'off',
'react/forbid-prop-types': 'off',
'react/jsx-closing-bracket-location': 'off',
'react/jsx-curly-spacing': 'off',
'react/jsx-handler-names': ['off', { eventHandlerPrefix: 'handle', eventHandlerPropPrefix: 'on' }],
'react/jsx-indent-props': 'off',
'react/jsx-key': 'off',
'react/jsx-max-props-per-line': 'off',
'react/jsx-no-bind': ['error', { ignoreRefs: true, allowArrowFunctions: true, allowBind: false }],
'react/jsx-no-duplicate-props': ['error', { ignoreCase: true }],
'react/jsx-no-literals': 'off',
'react/jsx-no-undef': 'error',
'react/jsx-pascal-case': ['error', { allowAllCaps: true, ignore: [] }],
'react/sort-prop-types': ['off', { ignoreCase: true, callbacksLast: false, requiredFirst: false }],
'react/jsx-sort-prop-types': 'off',
'react/jsx-sort-props': 'off',
'react/jsx-uses-react': 'error',
'react/jsx-uses-vars': 'error',
'react/no-danger': 'warn',
'react/no-deprecated': 'error',
'react/no-did-mount-set-state': 'error',
'react/no-did-update-set-state': 'warn',
'react/no-direct-mutation-state': 'off',
'react/no-is-mounted': 'error',
'react/no-set-state': 'off',
'react/no-string-refs': 'warn',
'react/no-unknown-property': 'error',
'react/prop-types': ['error', { ignore: [], customValidators: [], skipUndeclared: true }],
'react/react-in-jsx-scope': 'error',
'react/require-extension': 'off',
'react/require-render-return': 'error',
'react/self-closing-comp': 'warn',
'react/sort-comp': 'off',
'react/jsx-wrap-multilines': ['warn', { declaration: true, assignment: true, return: true }],
'react/wrap-multilines': 'off',
'react/jsx-first-prop-new-line': 'off',
'react/jsx-equals-spacing': ['warn', 'never'],
'react/jsx-no-target-blank': 'error',
'react/jsx-filename-extension': ['error', { extensions: ['.jsx'] }],
'react/jsx-no-comment-textnodes': 'error',
'react/no-comment-textnodes': 'off',
'react/no-render-return-value': 'error',
'react/require-optimization': ['off', { allowDecorators: [] }],
'react/no-find-dom-node': 'warn',
'react/forbid-component-props': ['off', { forbid: [] }],
'react/no-danger-with-children': 'error',
'react/no-unused-prop-types': ['warn', { customValidators: [], skipShapeProps: true }],
'react/style-prop-object': 'error',
'react/no-children-prop': 'warn',
},
settings: { react: { version: 'detect' } },
rules: {
...js.configs.recommended.rules,
'no-console': ['error', { allow: ['warn', 'error'] }],
},
},
prettier,
];

View File

@@ -7,7 +7,7 @@
content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"
/>
<meta name="google" content="notranslate" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<title>Fredy || Real Estate Finder</title>

View File

@@ -8,7 +8,6 @@ import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/ut
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
import logger from './lib/services/logger.js';
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
@@ -54,7 +53,6 @@ await import('./lib/api/api.js');
if (settings.demoMode) {
logger.info('Running in demo mode');
cleanupDemoAtMidnight();
}
ensureAdminUserExists();

View File

@@ -4,12 +4,21 @@
*/
import { NoNewListingsWarning } from './errors.js';
import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
import {
storeListings,
getKnownListingHashesForJobAndProvider,
deleteListingsById,
} from './services/storage/listingsStorage.js';
import { getJob } from './services/storage/jobStorage.js';
import * as notify from './notification/notify.js';
import Extractor from './services/extractor/extractor.js';
import urlModifier from './services/queryStringMutator.js';
import logger from './services/logger.js';
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
import { distanceMeters } from './services/listings/distanceCalculator.js';
import { getUserSettings } from './services/storage/settingsStorage.js';
import { updateListingDistance } from './services/storage/listingsStorage.js';
import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
/**
* @typedef {Object} Listing
@@ -54,18 +63,21 @@ class FredyPipelineExecutioner {
* @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape.
* @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings.
* @param {(url:string, waitForSelector?:string)=>Promise<void>|Promise<Listing[]>} [providerConfig.getListings] Optional override to fetch listings.
*
* @param {Object} notificationConfig Notification configuration passed to notification adapters.
* @param {Object} spatialFilter Optional spatial filter configuration.
* @param {string} providerId The ID of the provider currently in use.
* @param {string} jobKey Key of the job that is currently running (from within the config).
* @param {SimilarityCache} similarityCache Cache instance for checking similar entries.
* @param browser
*/
constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
constructor(providerConfig, notificationConfig, spatialFilter, providerId, jobKey, similarityCache, browser) {
this._providerConfig = providerConfig;
this._notificationConfig = notificationConfig;
this._spatialFilter = spatialFilter;
this._providerId = providerId;
this._jobKey = jobKey;
this._similarityCache = similarityCache;
this._browser = browser;
}
/**
@@ -82,7 +94,9 @@ class FredyPipelineExecutioner {
.then(this._findNew.bind(this))
.then(this._geocode.bind(this))
.then(this._save.bind(this))
.then(this._calculateDistance.bind(this))
.then(this._filterBySimilarListings.bind(this))
.then(this._filterByArea.bind(this))
.then(this._notify.bind(this))
.catch(this._handleError.bind(this));
}
@@ -106,6 +120,47 @@ class FredyPipelineExecutioner {
return newListings;
}
/**
* Filter listings by area using the provider's area filter if available.
* Only filters if areaFilter is set on the provider AND the listing has coordinates.
*
* @param {Listing[]} newListings New listings to filter by area.
* @returns {Promise<Listing[]>} Resolves with listings that are within the area (or not filtered if no area is set).
*/
_filterByArea(newListings) {
const polygonFeatures = this._spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon');
// If no area filter is set, return all listings
if (!polygonFeatures?.length) {
return newListings;
}
const filteredIds = [];
// Filter listings by area - keep only those within the polygon
const keptListings = newListings.filter((listing) => {
// If listing doesn't have coordinates, keep it (don't filter out)
if (listing.latitude == null || listing.longitude == null) {
return true;
}
// Check if the point is inside the polygons
const point = [listing.longitude, listing.latitude]; // GeoJSON format: [lon, lat]
const isInPolygon = polygonFeatures.some((feature) => booleanPointInPolygon(point, feature));
if (!isInPolygon) {
filteredIds.push(listing.id);
}
return isInPolygon;
});
if (filteredIds.length > 0) {
deleteListingsById(filteredIds);
}
return keptListings;
}
/**
* Fetch listings from the provider, using the default Extractor flow unless
* a provider-specific getListings override is supplied.
@@ -114,7 +169,7 @@ class FredyPipelineExecutioner {
* @returns {Promise<Listing[]>} Resolves with an array of listings (empty when none found).
*/
_getListings(url) {
const extractor = new Extractor();
const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
return new Promise((resolve, reject) => {
extractor
.execute(url, this._providerConfig.waitForSelector)
@@ -201,6 +256,42 @@ class FredyPipelineExecutioner {
return newListings;
}
/**
* Calculate distance for new listings.
*
* @param {Listing[]} listings
* @returns {Listing[]}
* @private
*/
_calculateDistance(listings) {
if (listings.length === 0) return [];
const job = getJob(this._jobKey);
const userId = job?.userId;
if (userId == null || typeof userId !== 'string') {
logger.debug('Skipping distance calculation: userId is missing or invalid');
return listings;
}
const userSettings = getUserSettings(userId);
const homeAddress = userSettings?.home_address;
if (!homeAddress || !homeAddress.coords) {
return listings;
}
const { lat, lng } = homeAddress.coords;
for (const listing of listings) {
if (listing.latitude != null && listing.longitude != null) {
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
updateListingDistance(listing.id, dist);
listing.distance_to_destination = dist;
}
}
return listings;
}
/**
* Remove listings that are similar to already known entries according to the similarity cache.
* Adds the remaining listings to the cache.
@@ -209,7 +300,8 @@ class FredyPipelineExecutioner {
* @returns {Listing[]} Listings considered unique enough to keep.
*/
_filterBySimilarListings(listings) {
return listings.filter((listing) => {
const filteredIds = [];
const keptListings = listings.filter((listing) => {
const similar = this._similarityCache.checkAndAddEntry({
title: listing.title,
address: listing.address,
@@ -219,9 +311,16 @@ class FredyPipelineExecutioner {
logger.debug(
`Filtering similar entry for title '${listing.title}' and address '${listing.address}' (Provider: '${this._providerId}')`,
);
filteredIds.push(listing.id);
}
return !similar;
});
if (filteredIds.length > 0) {
deleteListingsById(filteredIds);
}
return keptListings;
}
/**

10
lib/TRACKING_POIS.js Normal file
View File

@@ -0,0 +1,10 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export const TRACKING_POIS = {
DISTANCE_ADDRESS_ENTERED: 'DISTANCE_ADDRESS_ENTERED',
WELCOME_FINISHED: 'WELCOME_FINISHED',
WELCOME_SKIPPED: 'WELCOME_SKIPPED',
};

View File

@@ -10,6 +10,7 @@ import { providerRouter } from './routes/providerRouter.js';
import { versionRouter } from './routes/versionRouter.js';
import { loginRouter } from './routes/loginRoute.js';
import { userRouter } from './routes/userRoute.js';
import { userSettingsRouter } from './routes/userSettingsRoute.js';
import { jobRouter } from './routes/jobRouter.js';
import bodyParser from 'body-parser';
import restana from 'restana';
@@ -20,9 +21,10 @@ import { demoRouter } from './routes/demoRouter.js';
import logger from '../services/logger.js';
import { listingsRouter } from './routes/listingsRouter.js';
import { getSettings } from '../services/storage/settingsStorage.js';
import { featureRouter } from './routes/featureRouter.js';
import { dashboardRouter } from './routes/dashboardRouter.js';
import { backupRouter } from './routes/backupRouter.js';
import { trackingRouter } from './routes/trackingRoute.js';
import { registerMcpRoutes } from '../mcp/mcpHttpRoute.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../ui/public'));
const PORT = (await getSettings()).port || 9998;
@@ -35,7 +37,8 @@ service.use('/api/jobs', authInterceptor());
service.use('/api/version', authInterceptor());
service.use('/api/listings', authInterceptor());
service.use('/api/dashboard', authInterceptor());
service.use('/api/features', authInterceptor());
service.use('/api/user/settings', authInterceptor());
service.use('/api/tracking', authInterceptor());
// /admin can only be accessed when user is having admin permissions
service.use('/api/admin', adminInterceptor());
@@ -44,15 +47,19 @@ service.use('/api/admin/generalSettings', generalSettingsRouter);
service.use('/api/admin/backup', backupRouter);
service.use('/api/jobs/provider', providerRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/user/settings', userSettingsRouter);
service.use('/api/version', versionRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
service.use('/api/listings', listingsRouter);
service.use('/api/features', featureRouter);
service.use('/api/dashboard', dashboardRouter);
service.use('/api/tracking', trackingRouter);
//this route is unsecured intentionally as it is being queried from the login page
service.use('/api/demo', demoRouter);
// MCP Streamable HTTP endpoint (secured via Bearer token, not cookie-session)
registerMcpRoutes(service);
service.start(PORT).then(() => {
logger.debug(`Started API service on port ${PORT}`);
});

View File

@@ -1,17 +0,0 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import getFeatures from '../../features.js';
const service = restana();
const featureRouter = service.newRouter();
featureRouter.get('/', async (req, res) => {
const features = getFeatures();
res.body = Object.assign({}, { features });
res.send();
});
export { featureRouter };

View File

@@ -11,10 +11,13 @@ import logger from '../../services/logger.js';
import { bus } from '../../services/events/event-bus.js';
import { isRunning as isJobRunning } from '../../services/jobs/run-state.js';
import { addClient as addSseClient, removeClient } from '../../services/sse/sse-broker.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const jobRouter = service.newRouter();
const DEMO_JOB_NAME = 'Demo-Job';
function doesJobBelongsToUser(job, req) {
const userId = req.session.currentUser;
if (userId == null) {
@@ -160,7 +163,17 @@ jobRouter.post('/:jobId/run', async (req, res) => {
});
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled, shareWithUsers = [] } = req.body;
const {
provider,
notificationAdapter,
name,
blacklist = [],
jobId,
enabled,
shareWithUsers = [],
spatialFilter = null,
} = req.body;
const settings = await getSettings();
try {
let jobFromDb = jobStorage.getJob(jobId);
@@ -169,6 +182,11 @@ jobRouter.post('/', async (req, res) => {
return;
}
if (settings.demoMode && jobFromDb && jobFromDb.name === DEMO_JOB_NAME) {
res.send(new Error('Sorry, but you cannot change the Status of our Demo Job ;)'));
return;
}
jobStorage.upsertJob({
userId: req.session.currentUser,
jobId,
@@ -178,6 +196,7 @@ jobRouter.post('/', async (req, res) => {
provider,
notificationAdapter,
shareWithUsers,
spatialFilter,
});
} catch (error) {
res.send(new Error(error));
@@ -188,8 +207,14 @@ jobRouter.post('/', async (req, res) => {
jobRouter.delete('', async (req, res) => {
const { jobId } = req.body;
const settings = await getSettings();
try {
const job = jobStorage.getJob(jobId);
if (settings.demoMode && job.name === DEMO_JOB_NAME) {
res.send(new Error('Sorry, but you cannot remove the Demo Job ;)'));
return;
}
if (!doesJobBelongsToUser(job, req)) {
res.send(new Error('You are trying to remove a job that is not associated to your user'));
} else {
@@ -204,8 +229,15 @@ jobRouter.delete('', async (req, res) => {
jobRouter.put('/:jobId/status', async (req, res) => {
const { status } = req.body;
const { jobId } = req.params;
const settings = await getSettings();
try {
const job = jobStorage.getJob(jobId);
if (settings.demoMode && job.name === DEMO_JOB_NAME) {
res.send(new Error('Sorry, but you cannot change the Status of our Demo Job ;)'));
return;
}
if (!doesJobBelongsToUser(job, req)) {
res.send(new Error('You are trying change a job that is not associated to your user'));
} else {

View File

@@ -10,6 +10,7 @@ import { isAdmin as isAdminFn } from '../security.js';
import logger from '../../services/logger.js';
import { nullOrEmpty } from '../../utils.js';
import { getJobs } from '../../services/storage/jobStorage.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
@@ -74,6 +75,18 @@ listingsRouter.get('/map', async (req, res) => {
res.send();
});
listingsRouter.get('/:listingId', async (req, res) => {
const { listingId } = req.params;
const listing = listingStorage.getListingById(listingId, req.session.currentUser, isAdminFn(req));
if (!listing) {
res.statusCode = 404;
res.body = { message: 'Listing not found' };
return res.send();
}
res.body = listing;
res.send();
});
// Toggle watch state for the current user on a listing
listingsRouter.post('/watch', async (req, res) => {
try {
@@ -94,9 +107,15 @@ listingsRouter.post('/watch', async (req, res) => {
});
listingsRouter.delete('/job', async (req, res) => {
const { jobId } = req.body;
const { jobId, hardDelete = false } = req.body;
const settings = await getSettings();
try {
listingStorage.deleteListingsByJobId(jobId);
if (settings.demoMode) {
res.send(new Error('Sorry, but you cannot remove listings in demo mode ;)'));
return;
}
listingStorage.deleteListingsByJobId(jobId, hardDelete);
} catch (error) {
res.send(new Error(error));
logger.error(error);
@@ -105,10 +124,10 @@ listingsRouter.delete('/job', async (req, res) => {
});
listingsRouter.delete('/', async (req, res) => {
const { ids } = req.body;
const { ids, hardDelete = false } = req.body;
try {
if (Array.isArray(ids) && ids.length > 0) {
listingStorage.deleteListingsById(ids);
listingStorage.deleteListingsById(ids, hardDelete);
}
} catch (error) {
res.send(new Error(error));

View File

@@ -5,6 +5,8 @@
import fs from 'fs';
import restana from 'restana';
import logger from '../../services/logger.js';
const service = restana();
const notificationAdapterRouter = service.newRouter();
const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
@@ -34,11 +36,14 @@ notificationAdapterRouter.post('/try', async (req, res) => {
serviceName: 'TestCall',
newListings: [
{
price: '42 €',
title: 'This is a test listing',
address: 'some address',
size: '666 2m',
link: 'https://www.orange-coding.net',
address: 'Heidestrasse 17, 51147 Köln',
description: exampleDescription,
id: '1',
imageUrl: 'https://placehold.co/600x400/png',
price: '1.000 €',
size: '76 m²',
title: 'Stilvolle gepflegte 3-Raum-Wohnung mit gehobener Innenausstattung',
url: 'https://www.orange-coding.net',
},
],
notificationConfig,
@@ -46,6 +51,7 @@ notificationAdapterRouter.post('/try', async (req, res) => {
});
res.send();
} catch (Exception) {
logger.error('Error during notification adapter test:', Exception);
res.send(new Error(Exception));
}
});
@@ -54,3 +60,51 @@ notificationAdapterRouter.get('/', async (req, res) => {
res.send();
});
export { notificationAdapterRouter };
const exampleDescription = `
Wohnungstyp: Etagenwohnung
Nutzfläche: 76 m²
Etage: 2 von 3
Schlafzimmer: 1
Badezimmer: 1
Bezugsfrei ab: 1.4.2026
Haustiere: Nein
Garage/Stellplatz: Tiefgarage
Anzahl Garage/Stellplatz: 1
Kaltmiete (zzgl. Nebenkosten): 1.000 €
Preis/m²: 13,16 €/m²
Nebenkosten: 230 €
Heizkosten in Nebenkosten enthalten: Ja
Gesamtmiete: 1.230 €
Kaution: 3.000,00
Preis pro Parkfläche: 60 €
Baujahr: 2000
Objektzustand: Modernisiert
Qualität der Ausstattung: Gehoben
Heizungsart: Fernwärme
Energieausweistyp: Verbrauchsausweis
Energieausweis: liegt vor
Endenergieverbrauch: 72 kWh/(m²∙a)
Baujahr laut Energieausweis: 2000
Diese moderne 3-Zimmer-Wohnung liegt direkt neben einem Park und nur wenige Minuten von der S-Bahn-Haltestelle entfernt. Das Stadtzentrum sowie Freizeiteinrichtungen sind 1,5 km entfernt.
Die Wohnung ist ideal für Paare oder kleine Familien geeignet.
Ausstattung:
- neuer hochwertiger Bodenbelag (Holzoptik) in allen Räumen außer Bad/Küche
- sonniger Balkon (Süd)
- Tiefgaragenstellplatz
- Kellerabteil
- gepflegtes Mehrfamilienhaus
Die Küche ist vom Mieter nach eigenen Wünschen einzurichten.
Vermietung direkt vom Eigentümer - provisionsfrei!
Lage:
• Park: 1 Minute zu Fuß
• S-Bahn Station: 2 Minuten zu Fuß
• Supermärkte, Restaurants, täglicher Bedarf in der Nähe
• Gute Anbindung Richtung Großstadt und Flughafen
`;

View File

@@ -0,0 +1,37 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import { trackPoi } from '../../services/tracking/Tracker.js';
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
import logger from '../../services/logger.js';
const service = restana();
const trackingRouter = service.newRouter();
trackingRouter.get('/trackingPois', async (req, res) => {
res.body = TRACKING_POIS;
res.send();
});
trackingRouter.post('/poi', async (req, res) => {
const { poi } = req.body;
if (!poi) {
res.statusCode = 400;
res.send({ error: 'Feature name is required' });
return;
}
try {
await trackPoi(poi);
res.send({ success: true });
} catch (error) {
logger.error('Error tracking feature', error);
res.statusCode = 500;
res.send({ error: error.message });
}
});
export { trackingRouter };

View File

@@ -0,0 +1,121 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import SqliteConnection from '../../services/storage/SqliteConnection.js';
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
import { resetGeocoordinatesAndDistanceForUser } from '../../services/storage/listingsStorage.js';
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
import { fromJson } from '../../utils.js';
import { trackPoi } from '../../services/tracking/Tracker.js';
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
import logger from '../../services/logger.js';
import { runGeoCordTask } from '../../services/crons/geocoding-cron.js';
const service = restana();
const userSettingsRouter = service.newRouter();
userSettingsRouter.get('/', async (req, res) => {
const userId = req.session.currentUser;
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
const settings = {};
for (const r of rows) {
settings[r.name] = fromJson(r.value, null);
}
res.body = settings;
res.send();
});
userSettingsRouter.get('/autocomplete', async (req, res) => {
const { q } = req.query;
try {
const results = await autocompleteAddress(q);
res.body = results;
res.send();
} catch (error) {
res.statusCode = 500;
res.send({ error: error.message });
}
});
userSettingsRouter.post('/home-address', async (req, res) => {
const userId = req.session.currentUser;
const { home_address } = req.body;
const settings = await getSettings();
if (settings.demoMode) {
res.send(new Error('In demo mode, it is not allowed to change the home address.'));
return;
}
try {
if (home_address) {
await trackPoi(TRACKING_POIS.DISTANCE_ADDRESS_ENTERED);
const coords = await geocodeAddress(home_address);
if (coords && coords.lat !== -1) {
upsertSettings({ home_address: { address: home_address, coords } }, userId);
resetGeocoordinatesAndDistanceForUser(userId);
//we do NOT wait for this to finish, as we don't want to block the response
runGeoCordTask();
res.send({ success: true, coords });
} else {
res.statusCode = 400;
res.send({ error: 'Could not geocode address' });
}
} else {
upsertSettings({ home_address: null }, userId);
res.send({ success: true });
}
} catch (error) {
logger.error('Error updating home address settings', error);
res.statusCode = 500;
res.send({ error: error.message });
}
});
userSettingsRouter.post('/news-hash', async (req, res) => {
const userId = req.session.currentUser;
const { news_hash } = req.body;
const globalSettings = await getSettings();
if (globalSettings.demoMode) {
res.statusCode = 403;
res.send({ error: 'In demo mode, it is not allowed to change settings.' });
return;
}
try {
upsertSettings({ news_hash }, userId);
res.send({ success: true });
} catch (error) {
logger.error('Error updating news hash', error);
res.statusCode = 500;
res.send({ error: error.message });
}
});
userSettingsRouter.post('/immoscout-details', async (req, res) => {
const userId = req.session.currentUser;
const { immoscout_details } = req.body;
const globalSettings = await getSettings();
if (globalSettings.demoMode) {
res.statusCode = 403;
res.send({ error: 'In demo mode, it is not allowed to change settings.' });
return;
}
try {
upsertSettings({ immoscout_details: !!immoscout_details }, userId);
res.send({ success: true });
} catch (error) {
logger.error('Error updating immoscout details setting', error);
res.statusCode = 500;
res.send({ error: error.message });
}
});
export { userSettingsRouter };

275
lib/mcp/README.md Normal file
View File

@@ -0,0 +1,275 @@
# Fredy MCP Server
The Fredy MCP Server exposes your real estate jobs and listings data to LLM clients. It supports two transports:
- **Stdio**: for local LLM clients (Claude Desktop, LM Studio, llm-cli, mcp-cli, etc.)
- **Streamable HTTP**: for remote LLM clients (ChatGPT, cloud-hosted agents, etc.)
## Authentication
All MCP access is **token-based** based. Every Fredy user is automatically assigned a **permanent, non-expiring MCP token** when their account is created. This token is a secret and should be treated like a password.
### Where to find your token
MCP tokens are displayed in the **User Management** list (Admin → Users). Each user's token is shown in the **"MCP Token"** column.
> **Important:** MCP tokens never expire. They are permanent secrets tied to each user account. If a token is compromised, you must change the token! If you chose to use a token from an admin account, the LLM can query information from ALL jobs/listings.
## Available Tools
| Tool | Description |
|------|--------------------------------------------------------------------------------|
| `list_jobs` | List real estate search jobs with pagination and text filtering |
| `get_job` | Get detailed information about a specific job |
| `list_listings` | Search and list real estate listings with pagination, text search, and filters |
| `get_listing` | Get full details of a single listing |
| `get_current_date_time` | Gets the current date/time for the llm to be used |
### Tool Details
#### list_jobs
- `page` (number, optional) Page number (default: 1)
- `pageSize` (number, optional) Results per page (default: 50, max: 1000). Use pagination to fetch more.
- `filter` (string, optional) Free-text filter on job name
Response: markdown table with columns ID, Name, Enabled, Active Listings. Includes summary and pagination info.
#### get_job
- `jobId` (string, required) The job ID to retrieve
#### list_listings
- `page` (number, optional) Page number (default: 1)
- `pageSize` (number, optional) Results per page (default: 50, max: 1000). Use pagination to fetch more.
- `filter` (string, optional) Free-text search across title, address, provider, link
- `jobId` (string, optional) Filter listings by job ID
- `activeOnly` (boolean, optional) When true, only show active listings
- `provider` (string, optional) Filter by provider name
- `createdAfter` (number, optional) Only include listings created at or after this unix timestamp in milliseconds (e.g. `1772008362564`). Useful for queries like "give me all listings from today".
- `createdBefore` (number, optional) Only include listings created at or before this unix timestamp in milliseconds (e.g. `1772008362564`).
- `minPrice` (number, optional) Only include listings with price >= this value (e.g. `500`). Numeric, no currency symbol.
- `maxPrice` (number, optional) Only include listings with price <= this value (e.g. `1500`). Numeric, no currency symbol.
- `sortField` (string, optional) Sort by: created_at, price, size, provider, title, is_active
- `sortDir` (string, optional) Sort direction: asc or desc
Response: markdown table with columns ID, Title, Address, Price, Size, Provider, Active, Created, Job. Includes summary and pagination info. Use `get_listing` for full details.
> **Note:** All timestamps are **unix timestamps in milliseconds** (e.g. `1772008362564`), not seconds.
#### get_listing
- `listingId` (string, required) The listing ID to retrieve
## Usage with Local LLM (stdio transport)
The stdio transport communicates over stdin/stdout and is ideal for local LLM tools.
### Quick Start
```bash
MCP_TOKEN=fredy_<your-token> node mcp/stdio.js
# or
MCP_TOKEN=fredy_<your-token> yarn mcp:stdio
```
### Testing with MCP Inspector
The [MCP Inspector](https://github.com/modelcontextprotocol/inspector) lets you interactively test your MCP server in a browser UI.
```bash
npx @modelcontextprotocol/inspector -e MCP_TOKEN=fredy_<your-token> -- node mcp/stdio.js
```
Once the inspector is running, open the URL shown in your terminal (usually `http://localhost:6274`). You can then:
1. Click **Connect** to establish the stdio connection
2. Go to the **Tools** tab to see all available tools
3. Select a tool, fill in parameters, and click **Run** to test it
### LM Studio Configuration
[LM Studio](https://lmstudio.ai/) supports MCP servers natively, allowing your local LLM to access Fredy's jobs and listings data.
#### Setup
1. Open **LM Studio** and load a model that supports tool use (e.g., Qwen 2.5, Llama 3.1, Mistral, etc.)
2. In the right side under **Integrations** click on "# install" and "edit mcp.json"
3. Edit the LM Studio MCP config file directly (`~/.lmstudio/config/mcp.json` or via the UI export):
```json
{
"mcpServers": {
"fredy": {
"command": "node",
"args": ["/absolute/path/to/fredy/mcp/stdio.js"],
"env": {
"MCP_TOKEN": "fredy_<your-token>"
}
}
}
}
```
4. Toggle the server **on**: LM Studio will spawn the stdio process and connect
5. You should see the Fredy tools appear as available tools
#### Suggestion on LLM
After testing numerous LLM's, I got the best results with Qwen 3.5 or Qwen 2.5.. E.g. `Qwen2.5-14B-Instruct-1M-8bit`.
#### Usage
Once connected, simply ask your LLM about your real estate data in natural language:
- *"Show me all my active search jobs"*
- *"List the latest listings from my Berlin apartment search"*
- *"Get details for listing XYZ"*
- *"What are the cheapest listings across all my jobs?"*
The LLM will automatically call the appropriate Fredy MCP tools and present the results.
> **Tip:** Make sure Fredy is running and the database is accessible before starting the MCP server in LM Studio. The stdio transport initializes its own database connection, so Fredy's main process does not need to be running, but the database file must exist and be up-to-date (migrations applied).
## Usage with Remote LLM (Streamable HTTP transport)
The HTTP transport is automatically available when Fredy is running. It uses the MCP Streamable HTTP protocol at:
```
POST /api/mcp JSON-RPC messages (initialize, tool calls)
GET /api/mcp SSE stream for server-initiated notifications
DELETE /api/mcp Terminate session
```
### Authentication
All requests must include the token as a Bearer token:
```
Authorization: Bearer fredy_<your-token>
```
### Example: Initialize a session
```bash
curl -X POST http://localhost:9998/api/mcp \
-H "Content-Type: application/json" \
-H "Authorization: Bearer fredy_<your-token>" \
-d '{
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-03-26",
"capabilities": {},
"clientInfo": { "name": "test-client", "version": "1.0.0" }
}
}'
```
### Example: Call a tool
```bash
curl -X POST http://localhost:9998/api/mcp \
-H "Content-Type: application/json" \
-H "Authorization: Bearer fredy_<your-token>" \
-H "Mcp-Session-Id: <session-id-from-init-response>" \
-d '{
"jsonrpc": "2.0",
"id": 2,
"method": "tools/call",
"params": {
"name": "list_jobs",
"arguments": { "page": 1, "pageSize": 10 }
}
}'
```
## Security
- Every user is automatically assigned a permanent MCP token at account creation **tokens never expire**
- Tokens are cryptographically random (256-bit) and prefixed with `fredy_`
- Each token is scoped to a single user the LLM can only access that user's data
- Non-admin users only see their own jobs and jobs shared with them
- Tokens are stored in the `mcp_token` column of the `users` table
- Tokens are deleted automatically when the owning user is removed
- The `/api/mcp` endpoint uses Bearer token auth (independent of cookie-session)
- Treat MCP tokens like passwords do not share them publicly
## Response Format
All tool responses use **markdown** instead of JSON for maximum LLM readability and token efficiency:
- **List responses** (list_jobs, list_listings) use markdown tables with a summary line and pagination footer
- **Detail responses** (get_job, get_listing) use markdown key-value lists
- **Error responses** include the tool name and error message
Example list response:
```
**Tool:** list_listings | **Status:** OK
Found **85** listing(s). Showing page 1 of 2 (50 on this page). More pages available — use page=2 to continue.
| ID | Title | Address | Price | Size | Provider | Active | Created | Job |
|----|-------|---------|-------|------|----------|--------|---------|-----|
| abc123 | Nice flat | Berlin | 1200 | 70 | immoscout | yes | 2026-02-25 10:30:00 | My Search |
| ... | ... | ... | ... | ... | ... | ... | ... | ... |
Use **get_listing** with an ID for full details (description, link, image).
**Page:** 1/2 | **Has more:** yes
```
Example detail response:
```
**Tool:** get_listing | **Status:** OK
### Listing: Nice flat
- **ID:** abc123
- **Title:** Nice flat
- **Address:** Berlin
- **Price:** 1200
- **Size:** 70
- **Provider:** immoscout
- **Link:** https://...
- **Active:** yes
- **Created:** 2026-02-25 10:30:00
```
Markdown is used because it is significantly more token-efficient than JSON (~40-60% fewer tokens for tabular data) and natively understood by all LLMs.
## Architecture
```
┌─────────────────┐ stdio ┌──────────────┐
│ Local LLM │◄──────────────►│ mcp/stdio.js│
│ (LM Studio, │ │ (transport) │
│ Claude, etc.) │ │ │
└─────────────────┘ └──────┬───────┘
┌─────────────────┐ Streamable HTTP ┌────┴────────┐
│ Remote LLM │◄───────────────►│ /api/mcp │
│ │ (Bearer token) │ (transport) │
└─────────────────┘ └──────┬───────┘
┌──────────┴──────────┐
│ mcpAuthentication │
│ (token validation, │
│ access control) │
└──────────┬──────────┘
┌────────┴────────┐
│ mcpAdapter.js │
│ (tool routing │
│ + data fetch) │
└────────┬────────┘
┌────────┴────────┐
│ mcpNormalizer.js│
│ (markdown │
│ formatting) │
└────────┬────────┘
┌──────┴───────┐
│ Fredy DB │
│ (SQLite) │
└──────────────┘
```

231
lib/mcp/mcpAdapter.js Normal file
View File

@@ -0,0 +1,231 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';
import { queryJobs, getJob } from '../services/storage/jobStorage.js';
import { queryListings, getListingById } from '../services/storage/listingsStorage.js';
import { authenticateToolCall, checkJobAccess } from './mcpAuthentication.js';
import {
normalizeListJobs,
normalizeGetJob,
normalizeListListings,
normalizeGetListing,
normalizeError,
} from './mcpNormalizer.js';
/**
* Create a configured MCP server instance with all Fredy tools registered.
*
* The adapter fetches raw data from storage and delegates response formatting
* to the normalizer layer (mcpNormalizer.js) which produces a consistent
* { ok, summary, data, meta } envelope for every tool response.
*
* Each tool call requires a userId (resolved from the MCP token before invocation).
* Tools respect user scoping: non-admin users only see their own jobs/listings.
*
* @returns {McpServer}
*/
export function createMcpServer() {
const server = new McpServer(
{
name: 'fredy-mcp',
version: '1.0.0',
},
{
capabilities: {
tools: {},
},
instructions:
'Fredy MCP Server query real estate jobs and listings. ' +
'All timestamps are unix timestamps in milliseconds (e.g. 1772008362564). ' +
'Use list_jobs to browse jobs, get_job for details, ' +
'list_listings to search listings (supports time filters like createdAfter/createdBefore), ' +
'and get_listing for full details of a single listing. ' +
'Responses are formatted as markdown with a summary, data (tables for lists, key-value for details), and pagination info. ' +
'Always present results to the user as soon as you have them — do NOT call the tool again unless you need additional pages or different data.',
},
);
// ── list_jobs ───────────────────────────────────────────────────────
server.tool(
'list_jobs',
'List real estate search jobs for the authenticated user. ' +
'Returns up to 50 jobs per page by default. Use pagination (page parameter) to fetch more. ' +
'Check meta.hasMore to know if there are additional pages.',
{
page: z.number().optional().describe('Page number (default: 1)'),
pageSize: z
.number()
.optional()
.describe('Results per page (default: 50, max: 1000). Start with the default and paginate if needed.'),
filter: z.string().optional().describe('Free-text filter on job name'),
},
async ({ page, pageSize, filter }, extra) => {
const { user, error } = authenticateToolCall(extra, 'list_jobs');
if (error) return normalizeError(error, 'list_jobs');
const safePage = page ?? 1;
const safePageSize = pageSize ?? 50;
const result = queryJobs({
page: safePage,
pageSize: safePageSize,
freeTextFilter: filter,
userId: user.id,
isAdmin: user.isAdmin,
});
return normalizeListJobs(result, { page: safePage, pageSize: safePageSize });
},
);
// ── get_job ─────────────────────────────────────────────────────────
server.tool(
'get_job',
'Get detailed information about a specific job by its ID.',
{
jobId: z.string().describe('The job ID to retrieve'),
},
async ({ jobId }, extra) => {
const { user, error } = authenticateToolCall(extra, 'get_job');
if (error) return normalizeError(error, 'get_job');
const job = getJob(jobId);
if (!job) {
return normalizeError('Job not found.', 'get_job');
}
if (!checkJobAccess(user, job)) {
return normalizeError('Access denied.', 'get_job');
}
return normalizeGetJob(job);
},
);
// ── list_listings ───────────────────────────────────────────────────
server.tool(
'list_listings',
'Search and list real estate listings. Returns up to 50 listings per page by default. ' +
'Use pagination (page parameter) to fetch more. Check meta.hasMore in the response. ' +
'Supports text search, time filtering, and various filters. ' +
'All timestamps are unix timestamps in milliseconds (e.g. 1772008362564). ' +
'Use createdAfter/createdBefore to filter by time, e.g. "give me all listings from today". ' +
'Use get_listing to get full details (description, link, image) for a specific listing.',
{
page: z.number().optional().describe('Page number (default: 1)'),
pageSize: z
.number()
.optional()
.describe('Results per page (default: 50, max: 1000). Start with the default and paginate if needed.'),
filter: z.string().optional().describe('Free-text search across title, address, provider, link'),
jobId: z.string().optional().describe('Filter listings by job ID'),
activeOnly: z.boolean().optional().describe('When true, only show active listings'),
provider: z.string().optional().describe('Filter by provider name'),
createdAfter: z
.number()
.optional()
.describe(
'Only include listings created at or after this unix timestamp in milliseconds (e.g. 1772008362564). Useful for queries like "listings from today".',
),
createdBefore: z
.number()
.optional()
.describe(
'Only include listings created at or before this unix timestamp in milliseconds (e.g. 1772008362564).',
),
minPrice: z
.number()
.optional()
.describe(
'Only include listings with price >= this value (e.g. 500). Price is a numeric value (no currency symbol).',
),
maxPrice: z
.number()
.optional()
.describe(
'Only include listings with price <= this value (e.g. 1500). Price is a numeric value (no currency symbol).',
),
sortField: z.string().optional().describe('Sort by: created_at, price, size, provider, title, is_active'),
sortDir: z.string().optional().describe('Sort direction: asc or desc'),
},
async (
{
page,
pageSize,
filter,
jobId,
activeOnly,
provider,
createdAfter,
createdBefore,
minPrice,
maxPrice,
sortField,
sortDir,
},
extra,
) => {
const { user, error } = authenticateToolCall(extra, 'list_listings');
if (error) return normalizeError(error, 'list_listings');
const safePage = page ?? 1;
const safePageSize = pageSize ?? 50;
const result = queryListings({
page: safePage,
pageSize: safePageSize,
freeTextFilter: filter,
jobIdFilter: jobId,
activityFilter: activeOnly === true ? true : activeOnly === false ? false : undefined,
providerFilter: provider,
createdAfter: createdAfter ?? null,
createdBefore: createdBefore ?? null,
minPrice: minPrice ?? null,
maxPrice: maxPrice ?? null,
sortField: sortField ?? null,
sortDir: sortDir ?? 'desc',
userId: user.id,
isAdmin: user.isAdmin,
});
return normalizeListListings(result, { page: safePage, pageSize: safePageSize });
},
);
// ── get_listing ─────────────────────────────────────────────────────
server.tool(
'get_listing',
'Get full details of a single listing by its ID.',
{
listingId: z.string().describe('The listing ID to retrieve'),
},
async ({ listingId }, extra) => {
const { user, error } = authenticateToolCall(extra, 'get_listing');
if (error) return normalizeError(error, 'get_listing');
const listing = getListingById(listingId, user.id, user.isAdmin);
if (!listing) {
return normalizeError('Listing not found or access denied.', 'get_listing');
}
return normalizeGetListing(listing);
},
);
// ── get_current_date_ime ─────────────────────────────────────────────────────
server.tool('get_current_date_time', 'Returns the current date and time.', {}, () => {
return {
content: [{ type: 'text', text: `Timestring: ${new Date().toLocaleString()}, MS since 1970: ${Date.now()}` }],
};
});
return server;
}

View File

@@ -0,0 +1,66 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/**
* MCP Authentication Layer
*
* Centralizes all authentication and authorization logic for MCP tool calls
* and HTTP requests. Ensures consistent access control across all transports.
*/
import { getUser, validateMcpToken } from '../services/storage/userStorage.js';
/**
* Authenticate an MCP tool call by extracting and validating the user from authInfo.
*
* @param {{ authInfo?: { userId?: string } }} extra - The extra context passed by the MCP SDK.
* @returns {{ user: object|null, error: string|null }}
* - On success: { user: <userObject>, error: null }
* - On failure: { user: null, error: <errorMessage> }
*/
export function authenticateToolCall(extra) {
const userId = extra?.authInfo?.userId;
if (!userId) {
return { user: null, error: 'Authentication required. Please provide a valid MCP API token.' };
}
const user = getUser(userId);
if (!user) {
return { user: null, error: 'Authentication required. Please provide a valid MCP API token.' };
}
return { user, error: null };
}
/**
* Check whether a user has access to a specific job.
* Admins have access to all jobs. Non-admins can only access their own jobs
* or jobs explicitly shared with them.
*
* @param {object} user - The authenticated user object.
* @param {object} job - The job object from storage.
* @returns {boolean} True if the user is allowed to access this job.
*/
export function checkJobAccess(user, job) {
if (user.isAdmin) return true;
if (job.userId === user.id) return true;
if (Array.isArray(job.shared_with_user) && job.shared_with_user.includes(user.id)) return true;
return false;
}
/**
* Authenticate an HTTP request by extracting and validating the Bearer token
* from the Authorization header.
*
* @param {import('http').IncomingMessage} req
* @returns {{ userId: string } | null} The authenticated user info, or null if invalid.
*/
export function authenticateRequest(req) {
const authHeader = req.headers['authorization'] || '';
if (!authHeader.startsWith('Bearer ')) return null;
const token = authHeader.slice(7).trim();
if (!token) return null;
return validateMcpToken(token);
}

131
lib/mcp/mcpHttpRoute.js Normal file
View File

@@ -0,0 +1,131 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
import { createMcpServer } from './mcpAdapter.js';
import { authenticateRequest } from './mcpAuthentication.js';
import logger from '../services/logger.js';
import crypto from 'crypto';
/**
* Active transports keyed by session id.
* Each session gets its own McpServer + StreamableHTTPServerTransport pair.
* @type {Map<string, { server: McpServer, transport: StreamableHTTPServerTransport }>}
*/
const sessions = new Map();
/**
* Get or create a session for the given session id with authentication.
* @param {string|undefined} sessionId
* @param {{ userId: string }} auth
* @returns {{ server: McpServer, transport: StreamableHTTPServerTransport }}
*/
function getOrCreateSession(sessionId, auth) {
if (sessionId && sessions.has(sessionId)) {
return sessions.get(sessionId);
}
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => crypto.randomUUID(),
onsessioninitialized: (sid) => {
sessions.set(sid, entry);
logger.debug(`MCP session created: ${sid}`);
},
});
const server = createMcpServer();
const entry = { server, transport, userId: auth.userId };
transport.onclose = () => {
const sid = transport.sessionId;
if (sid) {
sessions.delete(sid);
logger.debug(`MCP session closed: ${sid}`);
}
};
return entry;
}
/**
* Register MCP Streamable HTTP routes on a restana service.
*
* Mounts handlers at /api/mcp to handle the MCP Streamable HTTP protocol:
* - POST /api/mcp JSON-RPC messages (initialize, tool calls, etc.)
* - GET /api/mcp SSE stream for server-initiated notifications
* - DELETE /api/mcp session termination
*
* All endpoints require a valid Bearer token in the Authorization header.
*
* @param {import('restana').Service} service - The restana service instance.
*/
export function registerMcpRoutes(service) {
// POST main JSON-RPC endpoint
service.post('/api/mcp', async (req, res) => {
const auth = authenticateRequest(req);
if (!auth) {
res.statusCode = 401;
return res.send({ error: 'Unauthorized. Provide a valid Bearer token.' });
}
const sessionId = req.headers['mcp-session-id'];
const { server, transport } = getOrCreateSession(sessionId, auth);
// Connect server to transport if not already connected
if (!transport.onmessage) {
await server.connect(transport);
}
// Inject authInfo so tools can access the authenticated user
req.auth = { userId: auth.userId };
await transport.handleRequest(req, res, req.body);
});
// GET SSE stream for server-initiated messages
service.get('/api/mcp', async (req, res) => {
const auth = authenticateRequest(req);
if (!auth) {
res.statusCode = 401;
return res.send({ error: 'Unauthorized. Provide a valid Bearer token.' });
}
const sessionId = req.headers['mcp-session-id'];
if (!sessionId || !sessions.has(sessionId)) {
res.statusCode = 400;
return res.send({ error: 'Invalid or missing session. Send an initialize request first.' });
}
const { transport } = sessions.get(sessionId);
await transport.handleRequest(req, res);
});
// DELETE terminate session
service.delete('/api/mcp', async (req, res) => {
const auth = authenticateRequest(req);
if (!auth) {
res.statusCode = 401;
return res.send({ error: 'Unauthorized. Provide a valid Bearer token.' });
}
const sessionId = req.headers['mcp-session-id'];
if (!sessionId || !sessions.has(sessionId)) {
res.statusCode = 404;
return res.send({ error: 'Session not found.' });
}
const { transport } = sessions.get(sessionId);
await transport.close();
sessions.delete(sessionId);
res.statusCode = 200;
res.send({ ok: true });
});
logger.debug('MCP Streamable HTTP endpoint registered at /api/mcp');
}

180
lib/mcp/mcpNormalizer.js Normal file
View File

@@ -0,0 +1,180 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/**
* MCP Response Normalizer
*
* Transforms raw adapter data into LLM-friendly markdown responses.
* Markdown is significantly better than JSON for LLM consumption because:
* - LLMs are trained extensively on markdown text
* - Markdown tables are ~40-60% more token-efficient than JSON arrays
* - Less syntactic noise (no quotes, brackets, commas around every value)
* - Natively readable and structured
*
* Each response follows a consistent structure:
* 1. Status line (OK/ERROR + tool name)
* 2. Summary (human-readable description)
* 3. Data (markdown table for lists, key-value for single items)
* 4. Pagination info (for list responses)
*/
/**
* Wrap a markdown string as an MCP text content result.
* @param {string} markdown
* @param {boolean} [isError=false]
* @returns {{ content: Array, isError?: boolean }}
*/
function toMcpResponse(markdown, isError = false) {
const result = {
content: [{ type: 'text', text: markdown }],
};
if (isError) result.isError = true;
return result;
}
/**
* Format a unix timestamp (ms) as a human-readable date string.
* @param {number|null|undefined} ts
* @returns {string}
*/
function formatDate(ts) {
if (ts == null) return '';
return new Date(ts)
.toISOString()
.replace('T', ' ')
.replace(/\.\d{3}Z$/, '');
}
/**
* Escape pipe characters in table cell values.
* @param {*} val
* @returns {string}
*/
function cell(val) {
if (val == null) return '';
return String(val).replace(/\|/g, '\\|').replace(/\n/g, ' ');
}
/**
* Normalize a list_jobs response.
* @param {{ totalNumber: number, page: number, result: object[] }} queryResult
* @param {{ page: number, pageSize: number }} params
* @returns {{ content: Array }}
*/
export function normalizeListJobs(queryResult, { page, pageSize }) {
const maxPage = Math.max(1, Math.ceil(queryResult.totalNumber / pageSize));
const hasMore = page < maxPage;
const jobs = queryResult.result;
let md = `**Tool:** list_jobs | **Status:** OK\n\n`;
md += `Found **${queryResult.totalNumber}** job(s). Showing page ${page} of ${maxPage} (${jobs.length} on this page).`;
if (hasMore) md += ` More pages available — use page=${page + 1} to continue.`;
md += '\n\n';
if (jobs.length > 0) {
md += `| ID | Name | Enabled | Active Listings |\n`;
md += `|----|------|---------|----------------|\n`;
for (const j of jobs) {
md += `| ${cell(j.id)} | ${cell(j.name)} | ${j.enabled ? 'yes' : 'no'} | ${j.numberOfFoundListings ?? 0} |\n`;
}
} else {
md += `No jobs found.\n`;
}
md += `\n**Page:** ${page}/${maxPage} | **Has more:** ${hasMore ? 'yes' : 'no'}`;
return toMcpResponse(md);
}
/**
* Normalize a get_job response.
* @param {object} job - The job object from storage.
* @returns {{ content: Array }}
*/
export function normalizeGetJob(job) {
const providers = (job.provider ?? []).map((p) => p.id || p);
let md = `**Tool:** get_job | **Status:** OK\n\n`;
md += `### Job: ${job.name || job.id}\n\n`;
md += `- **ID:** ${job.id}\n`;
md += `- **Name:** ${job.name || ''}\n`;
md += `- **Enabled:** ${job.enabled ? 'yes' : 'no'}\n`;
md += `- **Active Listings:** ${job.numberOfFoundListings ?? 0}\n`;
md += `- **Providers:** ${providers.length > 0 ? providers.join(', ') : ''}\n`;
md += `- **Blacklist:** ${(job.blacklist ?? []).length > 0 ? job.blacklist.join(', ') : ''}\n`;
return toMcpResponse(md);
}
/**
* Normalize a list_listings response.
* @param {{ totalNumber: number, page: number, result: object[] }} queryResult
* @param {{ page: number, pageSize: number }} params
* @returns {{ content: Array }}
*/
export function normalizeListListings(queryResult, { page, pageSize }) {
const maxPage = Math.max(1, Math.ceil(queryResult.totalNumber / pageSize));
const hasMore = page < maxPage;
const listings = queryResult.result;
let md = `**Tool:** list_listings | **Status:** OK\n\n`;
md += `Found **${queryResult.totalNumber}** listing(s). Showing page ${page} of ${maxPage} (${listings.length} on this page).`;
if (hasMore) md += ` More pages available — use page=${page + 1} to continue.`;
md += '\n\n';
if (listings.length > 0) {
md += `| ID | Title | Address | Price | Size | Provider | Active | Created | Job |\n`;
md += `|----|-------|---------|-------|------|----------|--------|---------|-----|\n`;
for (const l of listings) {
md += `| ${cell(l.id)} | ${cell(l.title)} | ${cell(l.address)} | ${cell(l.price)} | ${cell(l.size)} | ${cell(l.provider)} | ${l.is_active ? 'yes' : 'no'} | ${formatDate(l.created_at)} | ${cell(l.job_name)} |\n`;
}
md += `\nUse **get_listing** with an ID for full details (description, link, image).\n`;
} else {
md += `No listings found.\n`;
}
md += `\n**Page:** ${page}/${maxPage} | **Has more:** ${hasMore ? 'yes' : 'no'}`;
return toMcpResponse(md);
}
/**
* Normalize a get_listing response.
* @param {object} listing - The listing object from storage.
* @returns {{ content: Array }}
*/
export function normalizeGetListing(listing) {
let md = `**Tool:** get_listing | **Status:** OK\n\n`;
md += `### Listing: ${listing.title || listing.id}\n\n`;
md += `- **ID:** ${listing.id}\n`;
md += `- **Title:** ${listing.title || ''}\n`;
md += `- **Description:** ${listing.description || ''}\n`;
md += `- **Address:** ${listing.address || ''}\n`;
md += `- **Price:** ${listing.price ?? ''}\n`;
md += `- **Size:** ${listing.size ?? ''}\n`;
md += `- **Provider:** ${listing.provider || ''}\n`;
md += `- **Link:** ${listing.link || ''}\n`;
md += `- **Image:** ${listing.image_url || ''}\n`;
md += `- **Active:** ${listing.is_active ? 'yes' : 'no'}\n`;
md += `- **Created:** ${formatDate(listing.created_at)}\n`;
md += `- **Job:** ${listing.job_name || ''}\n`;
if (listing.latitude != null && listing.longitude != null) {
md += `- **Location:** ${listing.latitude}, ${listing.longitude}\n`;
}
if (listing.distance_to_destination != null) {
md += `- **Distance to destination:** ${listing.distance_to_destination}\n`;
}
return toMcpResponse(md);
}
/**
* Normalize an error response.
* @param {string} message - The error message.
* @param {string} [tool] - Optional tool name for context.
* @returns {{ content: Array, isError: boolean }}
*/
export function normalizeError(message, tool) {
const md = `**Tool:** ${tool ?? 'unknown'} | **Status:** ERROR\n\n${message}`;
return toMcpResponse(md, true);
}

76
lib/mcp/stdio.js Normal file
View File

@@ -0,0 +1,76 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/**
* Fredy MCP Server stdio transport
*
* Launches the MCP server over stdin/stdout so that local LLM clients
* (e.g. Claude Desktop, llm-cli, mcp-cli) can connect directly.
*
* Usage:
* MCP_TOKEN=fredy_<your-token> node mcp/stdio.js
*
* The MCP_TOKEN environment variable must contain a valid Fredy MCP token.
* Each user has a permanent, non-expiring token shown in the user management list.
*/
import { fileURLToPath } from 'url';
import path from 'path';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import SqliteConnection from '../services/storage/SqliteConnection.js';
import { runMigrations } from '../services/storage/migrations/migrate.js';
import { createMcpServer } from './mcpAdapter.js';
import { validateMcpToken } from '../services/storage/userStorage.js';
// Ensure cwd is the project root so that relative DB/config paths resolve correctly
// (LM Studio and other MCP hosts may spawn this process from an arbitrary directory)
const __dirname = path.dirname(fileURLToPath(import.meta.url));
process.chdir(path.resolve(__dirname, '..', '..'));
// Initialize the database (required for standalone usage)
await SqliteConnection.init();
await runMigrations();
const token = process.env.MCP_TOKEN;
if (!token) {
process.stderr.write('Error: MCP_TOKEN environment variable is required.\n');
process.stderr.write('Each user has a permanent MCP token shown in the user management list.\n');
process.exit(1);
}
const auth = validateMcpToken(token);
if (!auth) {
process.stderr.write('Error: Invalid MCP_TOKEN. Token not found or user no longer exists.\n');
process.exit(1);
}
const mcpServer = createMcpServer();
// Wrap the stdio transport to inject authInfo into every message
const transport = new StdioServerTransport();
// Patch: the MCP SDK passes authInfo through the transport's onmessage extra param.
// For stdio we inject the resolved user from the token.
const patchedTransport = new Proxy(transport, {
set(target, prop, value) {
if (prop === 'onmessage') {
target.onmessage = (message, extra) => {
value(message, { ...extra, authInfo: { userId: auth.userId } });
};
return true;
}
target[prop] = value;
return true;
},
get(target, prop) {
return target[prop];
},
});
await mcpServer.connect(patchedTransport);
process.stderr.write('Fredy MCP Server running on stdio\n');

View File

@@ -16,8 +16,8 @@ const mapListing = (listing) => ({
url: listing.link,
});
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { authToken, endpointUrl } = notificationConfig.find((a) => a.id === config.id).fields;
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { authToken, endpointUrl, selfSignedCerts } = notificationConfig.find((a) => a.id === config.id).fields;
const listings = newListings.map(mapListing);
const body = {
@@ -34,11 +34,20 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
headers['Authorization'] = `Bearer ${authToken}`;
}
return fetch(endpointUrl, {
let fetchOptions = {
method: 'POST',
headers: headers,
headers,
timeout: 10000,
body: JSON.stringify(body),
});
};
if (selfSignedCerts === true) {
fetchOptions.dispatcher = new (await import('undici')).Agent({
connect: { rejectUnauthorized: false },
});
}
return fetch(endpointUrl, fetchOptions);
};
export const config = {
@@ -52,6 +61,10 @@ export const config = {
label: 'Endpoint URL',
type: 'text',
},
selfSignedCerts: {
label: 'Self-signed certificates',
type: 'boolean',
},
authToken: {
description: "Your application's auth token, if required by your endpoint.",
label: 'Auth token (optional)',

View File

@@ -0,0 +1,88 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { Resend } from 'resend';
import path from 'path';
import fs from 'fs';
import Handlebars from 'handlebars';
import { markdown2Html } from '../../services/markdown.js';
import { getDirName, normalizeImageUrl } from '../../utils.js';
const __dirname = getDirName();
const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
const emailTemplate = Handlebars.compile(template);
const mapListings = (serviceName, jobKey, listings) =>
listings.map((l) => {
const image = normalizeImageUrl(l.image);
return {
title: l.title || '',
link: l.link || '',
address: l.address || '',
size: l.size || '',
price: l.price || '',
image,
hasImage: Boolean(image),
serviceName,
jobKey,
};
});
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
const { apiKey, receiver, from } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
const to = receiver
.trim()
.split(',')
.map((r) => r.trim())
.filter(Boolean);
const resend = new Resend(apiKey);
const listings = mapListings(serviceName, jobKey, newListings);
const html = emailTemplate({
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
numberOfListings: listings.length,
listings,
});
const { error } = await resend.emails.send({
from,
to,
subject: `Fredy found ${listings.length} new listing(s) for ${serviceName}`,
html,
});
if (!error) {
return Promise.resolve();
} else {
return Promise.reject(error.message);
}
};
export const config = {
id: 'resend',
name: 'Resend',
description: 'Resend is being used to send new listings via mail.',
readme: markdown2Html('lib/notification/adapter/resend.md'),
fields: {
apiKey: {
type: 'text',
label: 'Api Key',
description: 'The Resend API key used to send emails.',
},
receiver: {
type: 'email',
label: 'Receiver Email',
description: 'Comma-separated email addresses Fredy will send notifications to.',
},
from: {
type: 'email',
label: 'Sender Email',
description: 'The verified email address or domain you send from in Resend.',
},
},
};

View File

@@ -0,0 +1,17 @@
### Resend Adapter
Resend is a modern email delivery service that Fredy can use to send notifications.
Setup:
- Create a Resend account: https://resend.com/
- Create an API key and add it to Fredy's configuration.
- Choose the sender address (e.g., you@yourdomain.com). Verify the domain (https://resend.com/domains/) in Resend before using it.
- Optional for local testing: you can use `onboarding@resend.dev`, but Resend may restrict who you can send to when using test domains.
Multiple recipients:
- Separate email addresses with commas (e.g., some@email.com, someOther@email.com).
Notes & Troubleshooting:
- Ensure the `from` address is verified or belongs to a verified domain in Resend.
- If emails don't arrive, check your spam folder and Resend dashboard logs.
- The template displays listing images via their public URLs; make sure images are reachable.

View File

@@ -9,7 +9,9 @@ import checkIfListingIsActive from '../services/listings/listingActiveTester.js'
let appliedBlackList = [];
function shortenLink(link) {
return link.substring(0, link.indexOf('?'));
if (!link) return '';
const index = link.indexOf('?');
return index === -1 ? link : link.substring(0, index);
}
function parseId(shortenedLink) {
@@ -23,7 +25,7 @@ function normalize(o) {
const title = o.title || 'No title available';
const address = o.address || null;
const shortLink = shortenLink(o.link);
const link = `${baseUrl}/${shortLink}`;
const link = baseUrl + shortLink;
const image = baseUrl + o.image;
const id = buildHash(parseId(shortLink), o.price);
return Object.assign(o, { id, price, size, title, address, link, image });
@@ -37,18 +39,18 @@ function applyBlacklist(o) {
const config = {
url: null,
crawlContainer: '._ref',
crawlContainer: 'a:has(div.list_entry)',
sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
waitForSelector: 'body',
crawlFields: {
id: '@href', //will be transformed later
price: '.list_entry .immo_preis .label_info',
size: '.list_entry .flaeche .label_info | removeNewline | trim',
title: '.list_entry .part_text h3 span',
description: '.list_entry .description | trim',
price: '.immo_preis .label_info',
size: '.flaeche .label_info | removeNewline | trim',
title: 'h3 span',
description: '.description | trim',
link: '@href',
address: '.list_entry .place',
image: '.list_entry img@src',
address: '.place',
image: 'img@src',
},
normalize: normalize,
filter: applyBlacklist,

View File

@@ -1,53 +0,0 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { isOneOf, buildHash } from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
const price = o.price.replace('Kaufpreis ', '');
const address = o.address?.split(' • ')?.pop() ?? null;
const title = o.title || 'No title available';
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
const id = buildHash(title, price);
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
sortByDateParam: 'sortby=19',
waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
crawlFields: {
id: 'button@title |trim',
title: 'button@title |trim',
price: 'div[data-testid="cardmfe-price-testid"] | trim',
size: 'div[data-testid="cardmfe-keyfacts-testid"] | trim',
address: 'div[data-testid="cardmfe-description-box-address"] | trim',
image: 'div[data-testid="cardmfe-picture-box-test-id"] img@src',
link: 'button@data-base',
description: 'div[data-testid="cardmfe-description-text-test-id"] | trim',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
export const metaInformation = {
name: 'Immonet',
baseUrl: 'https://www.immonet.de/',
id: 'immonet',
};
export { config };

View File

@@ -8,7 +8,7 @@
*
* The mobile API provides the following endpoints:
* - GET /search/total?{search parameters}: Returns the total number of listings for the given query
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
* Example: `curl -H "User-Agent: ImmoScout_27.12_26.2_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
*
* - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
* data specifying additional results (advertisements) to return. The format is as follows:
@@ -20,12 +20,12 @@
* ```
* It is not necessary to provide data for the specified keys.
*
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.12_26.2_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
* - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
* listing response.
*
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
* Example: `curl -H "User-Agent: ImmoScout_27.12_26.2_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
*
*
* It is necessary to set the correct User Agent (see `getListings`) in the request header.
@@ -46,13 +46,15 @@ import {
convertWebToMobile,
} from '../services/immoscout/immoscout-web-translator.js';
import logger from '../services/logger.js';
import { getUserSettings } from '../services/storage/settingsStorage.js';
let appliedBlackList = [];
let currentUserId = null;
async function getListings(url) {
const response = await fetch(url, {
method: 'POST',
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
'User-Agent': 'ImmoScout_27.12_26.2_._',
'Content-Type': 'application/json',
},
body: JSON.stringify({
@@ -66,29 +68,92 @@ async function getListings(url) {
}
const responseBody = await response.json();
return responseBody.resultListItems
.filter((item) => item.type === 'EXPOSE_RESULT')
.map((expose) => {
const item = expose.item;
const [price, size] = item.attributes;
const image = item?.titlePicture?.preview ?? null;
return {
id: item.id,
price: price?.value,
size: size?.value,
title: item.title,
description: item.description,
link: `${metaInformation.baseUrl}expose/${item.id}`,
address: item.address?.line,
image,
};
});
return Promise.all(
responseBody.resultListItems
.filter((item) => item.type === 'EXPOSE_RESULT')
.map(async (expose) => {
const item = expose.item;
const [price, size] = item.attributes;
const image = item?.titlePicture?.full ?? item?.titlePicture?.preview ?? null;
let listing = {
id: item.id,
price: price?.value,
size: size?.value,
title: item.title,
link: `${metaInformation.baseUrl}expose/${item.id}`,
address: item.address?.line,
image,
};
if (currentUserId) {
const userSettings = getUserSettings(currentUserId);
if (userSettings.immoscout_details) {
return await pushDetails(listing);
}
}
return listing;
}),
);
}
async function pushDetails(listing) {
const detailed = await fetch(`https://api.mobile.immobilienscout24.de/expose/${listing.id}`, {
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
'Content-Type': 'application/json',
},
});
if (!detailed.ok) {
logger.error('Error fetching listing details from ImmoScout Mobile API:', detailed.statusText);
return listing;
}
const detailBody = await detailed.json();
listing.description = buildDescription(detailBody);
return listing;
}
function buildDescription(detailBody) {
const sections = detailBody.sections || [];
const contact = detailBody.contact || {};
const cData = contact?.contactData || {};
const agentName = cData?.agent?.name || '';
const agentCompany = cData?.agent?.company || '';
const stars = cData?.agent?.rating?.numberOfStars || '';
const phoneNumbers = contact?.phoneNumbers || [];
const phoneNumbersMapped = phoneNumbers
.map((p) => `${p.label}: ${p.text}`)
.join('\n')
.trim();
const attributes = sections
.filter((s) => s.type === 'ATTRIBUTE_LIST')
.flatMap((s) => s.attributes)
.filter((attr) => attr.label && attr.text)
.map((attr) => `${attr.label} ${attr.text}`)
.join('\n');
const freeText = sections
.filter((s) => s.type === 'TEXT_AREA')
.map((s) => {
return `${s.title}\n${s.text}`;
})
.join('\n\n');
return (
`Agent: ${agentName ? agentName : 'Unbekannt'} ${agentCompany ? `(${agentCompany}) ` : ''}${stars ? `- ${stars} stars` : ''}\n` +
(phoneNumbersMapped ? `Phone Numbers:\n${phoneNumbersMapped}` : '') +
'\n\n' +
attributes.trim() +
'\n\n' +
freeText.trim()
);
}
async function isListingActive(link) {
const result = await fetch(convertImmoscoutListingToMobileListing(link), {
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
'User-Agent': 'ImmoScout_27.12_26.2_._',
},
});
@@ -137,6 +202,7 @@ export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = convertWebToMobile(sourceConfig.url);
appliedBlackList = blacklist || [];
currentUserId = sourceConfig.userId || null;
};
export const metaInformation = {
name: 'Immoscout',

View File

@@ -36,7 +36,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
activeTester: (url) => checkIfListingIsActive(url, 'Angebot nicht gefunden'),
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -4,6 +4,7 @@
*/
import * as utils from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
@@ -40,6 +41,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklistTerms) => {

View File

@@ -1,29 +0,0 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { removeJobsByUserId } from '../storage/jobStorage.js';
import { getUsers } from '../storage/userStorage.js';
import logger from '../logger.js';
import cron from 'node-cron';
import { getSettings } from '../storage/settingsStorage.js';
/**
* if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
*/
export function cleanupDemoAtMidnight() {
cron.schedule('0 0 * * *', cleanup);
}
async function cleanup() {
const settings = await getSettings();
if (settings.demoMode) {
const demoUser = getUsers(false).find((user) => user.username === 'demo');
if (demoUser == null) {
logger.error('Demo user not found, cannot remove Jobs');
return Promise.resolve();
}
removeJobsByUserId(demoUser.id);
}
}

View File

@@ -6,28 +6,41 @@
import cron from 'node-cron';
import { getListingsToGeocode, updateListingGeocoordinates } from '../storage/listingsStorage.js';
import { geocodeAddress, isGeocodingPaused } from '../geocoding/geoCodingService.js';
import { getJobs } from '../storage/jobStorage.js';
import { calculateDistanceForJob } from '../geocoding/distanceService.js';
import { getSettings } from '../storage/settingsStorage.js';
import logger from '../logger.js';
async function runTask() {
export async function runGeoCordTask() {
const listings = getListingsToGeocode();
if (listings.length === 0) {
return;
if (listings.length > 0) {
for (const listing of listings) {
if (isGeocodingPaused()) {
break;
}
const coords = await geocodeAddress(listing.address);
if (coords) {
updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
}
}
}
for (const listing of listings) {
if (isGeocodingPaused()) {
break;
}
const coords = await geocodeAddress(listing.address);
if (coords) {
updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
}
//additional run
const jobs = getJobs();
for (const job of jobs) {
calculateDistanceForJob(job.id, job.userId);
}
}
export async function initGeocodingCron() {
const settings = await getSettings();
if (settings.demoMode) {
logger.info('Do not start geo service as we are in demo mode');
return;
}
// run directly on start
await runTask();
await runGeoCordTask();
// then every 6 hours
cron.schedule('0 */6 * * *', runTask);
cron.schedule('0 */6 * * *', runGeoCordTask);
}

View File

@@ -5,12 +5,19 @@
import cron from 'node-cron';
import runActiveChecker from '../listings/listingActiveService.js';
import logger from '../logger.js';
import { getSettings } from '../storage/settingsStorage.js';
async function runTask() {
await runActiveChecker();
}
export async function initActiveCheckerCron() {
const settings = await getSettings();
if (settings.demoMode) {
logger.info('Do not start listing active checker as we are in demo mode');
return;
}
//run directly on start
await runTask();
// then every day at 1 am

View File

@@ -19,52 +19,80 @@ import path from 'path';
puppeteer.use(StealthPlugin());
export default async function execute(url, waitForSelector, options) {
let browser;
let page;
let result = null;
export async function launchBrowser(url, options) {
const preCfg = getPreLaunchConfig(url, options || {});
const launchArgs = [
'--no-sandbox',
'--disable-gpu',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-crash-reporter',
'--no-first-run',
'--no-default-browser-check',
preCfg.langArg,
preCfg.windowSizeArg,
...preCfg.extraArgs,
];
if (options?.proxyUrl) {
launchArgs.push(`--proxy-server=${options.proxyUrl}`);
}
let userDataDir;
let removeUserDataDir = false;
if (options && options.userDataDir) {
userDataDir = options.userDataDir;
} else {
const prefix = path.join(os.tmpdir(), 'puppeteer-fredy-');
userDataDir = fs.mkdtempSync(prefix);
removeUserDataDir = true;
}
const browser = await puppeteer.launch({
headless: options?.puppeteerHeadless ?? true,
args: launchArgs,
timeout: options?.puppeteerTimeout || 45_000,
userDataDir,
executablePath: options?.executablePath,
});
browser.__fredy_userDataDir = userDataDir;
browser.__fredy_removeUserDataDir = removeUserDataDir;
return browser;
}
export async function closeBrowser(browser) {
if (!browser) return;
const userDataDir = browser.__fredy_userDataDir;
const removeUserDataDir = browser.__fredy_removeUserDataDir;
try {
await browser.close();
} catch {
// ignore
}
if (removeUserDataDir && userDataDir) {
try {
await fs.promises.rm(userDataDir, { recursive: true, force: true });
} catch {
// ignore
}
}
}
export default async function execute(url, waitForSelector, options) {
let browser = options?.browser;
let isExternalBrowser = !!browser;
let page;
let result;
try {
debug(`Sending request to ${url} using Puppeteer.`);
// Prepare a dedicated temporary userDataDir to avoid leaking /tmp/.org.chromium.* dirs
if (options && options.userDataDir) {
userDataDir = options.userDataDir;
removeUserDataDir = !!options.cleanupUserDataDir;
} else {
const prefix = path.join(os.tmpdir(), 'puppeteer-fredy-');
userDataDir = fs.mkdtempSync(prefix);
removeUserDataDir = true;
if (!isExternalBrowser) {
browser = await launchBrowser(url, options);
}
const launchArgs = [
'--no-sandbox',
'--disable-gpu',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-crash-reporter',
'--no-first-run',
'--no-default-browser-check',
];
if (options?.proxyUrl) {
launchArgs.push(`--proxy-server=${options.proxyUrl}`);
}
// Prepare bot prevention pre-launch config
const preCfg = getPreLaunchConfig(url, options || {});
launchArgs.push(preCfg.langArg);
launchArgs.push(preCfg.windowSizeArg);
launchArgs.push(...preCfg.extraArgs);
browser = await puppeteer.launch({
headless: options?.puppeteerHeadless ?? true,
args: launchArgs,
timeout: options?.puppeteerTimeout || 30_000,
userDataDir,
executablePath: options?.executablePath, // allow using system Chrome
});
page = await browser.newPage();
const preCfg = getPreLaunchConfig(url, options || {});
await applyBotPreventionToPage(page, preCfg);
// Provide languages value before navigation
await applyLanguagePersistence(page, preCfg);
@@ -104,7 +132,7 @@ export default async function execute(url, waitForSelector, options) {
result = pageSource || (await page.content());
}
} catch (error) {
if (error?.message?.includes('Timeout')) {
if (error?.name?.includes('Timeout')) {
logger.debug('Error executing with puppeteer executor', error);
} else {
logger.warn('Error executing with puppeteer executor', error);
@@ -118,19 +146,8 @@ export default async function execute(url, waitForSelector, options) {
} catch {
// ignore
}
try {
if (browser != null) {
await browser.close();
}
} catch {
// ignore
}
try {
if (removeUserDataDir && userDataDir) {
await fs.promises.rm(userDataDir, { recursive: true, force: true });
}
} catch {
// ignore
if (browser != null && !isExternalBrowser) {
await closeBrowser(browser);
}
}
return result;

View File

@@ -0,0 +1,26 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { autocomplete as nominatimAutocomplete } from './client/nominatimClient.js';
import logger from '../logger.js';
/**
* Autocompletes an address using Nominatim.
*
* @param {string} query - The search query.
* @returns {Promise<string[]>} List of matching addresses.
*/
export async function autocompleteAddress(query) {
if (!query) {
return [];
}
try {
return await nominatimAutocomplete(query);
} catch (error) {
logger.error('Error during address autocomplete:', error);
return [];
}
}

View File

@@ -67,6 +67,7 @@ async function doGeocode(address) {
try {
const response = await fetch(url, {
agent,
timeout: 60000,
headers: {
'User-Agent': userAgent,
},
@@ -100,6 +101,53 @@ async function doGeocode(address) {
}
}
/**
* Autocompletes an address using Nominatim.
*
* @param {string} query - The search query.
* @returns {Promise<string[]>} List of matching addresses.
*/
async function doAutocomplete(query) {
if (Date.now() - last429 < PAUSE_DURATION) {
return [];
}
const url = `${API_URL}?q=${encodeURIComponent(query)}&format=json&addressdetails=1&limit=5&countrycodes=de`;
try {
const response = await fetch(url, {
agent,
headers: {
'User-Agent': userAgent,
},
});
if (response.status === 429) {
logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
last429 = Date.now();
return [];
}
if (!response.ok) {
logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
return [];
}
const data = await response.json();
if (Array.isArray(data)) {
return data.map((item) => item.display_name);
}
return [];
} catch (error) {
logger.error('Error during Nominatim autocomplete:', error);
return [];
}
}
export const geocode = throttle(doGeocode);
export const autocomplete = throttle(doAutocomplete);
export const isPaused = () => Date.now() - last429 < PAUSE_DURATION;

View File

@@ -0,0 +1,61 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { distanceMeters } from '../listings/distanceCalculator.js';
import {
getListingsToCalculateDistance,
getListingsForUserToCalculateDistance,
updateListingDistance,
} from '../storage/listingsStorage.js';
import { getUserSettings } from '../storage/settingsStorage.js';
/**
* Calculates and updates distances for listings of a specific job.
* Only processes listings where distance_to_destination is null.
*
* @param {string} jobId
* @param {string} userId
* @returns {void}
*/
export function calculateDistanceForJob(jobId, userId) {
const userSettings = getUserSettings(userId);
const homeAddress = userSettings.home_address;
if (!homeAddress || !homeAddress.coords) {
return;
}
const listings = getListingsToCalculateDistance(jobId);
const { lat, lng } = homeAddress.coords;
for (const listing of listings) {
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
updateListingDistance(listing.id, dist);
}
}
/**
* Calculates and updates distances for all active listings of a user.
* Usually called when the user updates their home address.
*
* @param {string} userId
* @returns {void}
*/
export function calculateDistanceForUser(userId) {
const userSettings = getUserSettings(userId);
const homeAddress = userSettings.home_address;
if (!homeAddress || !homeAddress.coords) {
return;
}
const listings = getListingsForUserToCalculateDistance(userId);
const { lat, lng } = homeAddress.coords;
for (const listing of listings) {
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
updateListingDistance(listing.id, dist);
}
}

View File

@@ -79,6 +79,8 @@ const PARAM_NAME_MAP = {
price: 'price',
constructionyear: 'constructionyear',
apartmenttypes: 'apartmenttypes',
buildingtypes: 'buildingtypes',
ground: 'ground',
pricetype: 'pricetype',
floor: 'floor',
geocodes: 'geocodes',
@@ -86,6 +88,7 @@ const PARAM_NAME_MAP = {
shape: 'shape',
sorting: 'sorting',
newbuilding: 'newbuilding',
fulltext: 'fulltext',
};
const EQUIPMENT_MAP = {
@@ -97,19 +100,28 @@ const EQUIPMENT_MAP = {
guesttoilet: 'guestToilet',
balcony: 'balcony',
handicappedaccessible: 'handicappedAccessible',
lodgerflat: 'lodgerflat',
};
const REAL_ESTATE_TYPE = {
'haus-mieten': 'houserent',
'wohnung-mieten': 'apartmentrent',
'wohnung-kaufen': 'apartmentbuy',
'wohnung-kaufen-mit-balkon': 'apartmentbuy',
'eigentumswohnung-mit-garten': 'apartmentbuy',
'haus-kaufen': 'housebuy',
'haus-mit-keller-kaufen': 'housebuy',
'luxushaus-kaufen': 'housebuy',
'villa-kaufen': 'housebuy',
'neubauhaus-kaufen': 'housebuy',
};
const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
// Category "Balkon/Terrasse"
'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
'wohnung-kaufen-mit-balkon': { equipment: ['balcony'] },
'wohnung-mit-garten-mieten': { equipment: ['garden'] },
'eigentumswohnung-mit-garten': { equipment: ['garden'] },
// Category "Wohnungstyp"
'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
@@ -144,7 +156,7 @@ export function convertWebToMobile(webUrl) {
const realTypeKey = segments.at(-1);
let realType = REAL_ESTATE_TYPE[realTypeKey];
let additionalParamsFromWebPath;
let additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey] || null;
if (!realType) {
// Test for seo optimized apartment path (only used on the ImmoScout web app)
@@ -165,7 +177,7 @@ export function convertWebToMobile(webUrl) {
Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
);
const geocodes = `/${segments.slice(2, 5).join('/')}`;
const geocodes = `/${segments.slice(2, segments.length - 1).join('/')}`;
const isRadius = segments.includes('radius');
const mobileParams = {
searchType: isRadius ? 'radius' : 'region',

View File

@@ -13,6 +13,7 @@ import FredyPipelineExecutioner from '../../FredyPipelineExecutioner.js';
import * as similarityCache from '../similarity-check/similarityCache.js';
import { isRunning, markFinished, markRunning } from './run-state.js';
import { sendToUsers } from '../sse/sse-broker.js';
import * as puppeteerExtractor from '../extractor/puppeteerExtractor.js';
/**
* Initializes the job execution service.
@@ -94,7 +95,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
* @param {{userId?: string, isAdmin?: boolean}} [context] - Who requested the run; determines job filtering.
* @returns {void}
*/
function runAll(respectWorkingHours = true, context = undefined) {
async function runAll(respectWorkingHours = true, context = undefined) {
if (settings.demoMode) return;
const now = Date.now();
const withinHours = duringWorkingHoursOrNotSet(settings, now);
@@ -103,15 +104,18 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
return;
}
settings.lastRun = now;
jobStorage
const jobs = jobStorage
.getJobs()
.filter((job) => job.enabled)
.filter((job) => {
if (!context) return true; // startup/cron → all
if (context.isAdmin) return true; // admin → all
return context.userId ? job.userId === context.userId : false; // user → own
})
.forEach((job) => executeJob(job));
});
for (const job of jobs) {
await executeJob(job);
}
}
/**
@@ -154,28 +158,43 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
} catch (err) {
logger.warn('Failed to emit start status for job', job.id, err);
}
let browser;
try {
const jobProviders = job.provider.filter(
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
);
const executions = jobProviders.map(async (prov) => {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyPipelineExecutioner(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
});
const results = await Promise.allSettled(executions);
for (const r of results) {
if (r.status === 'rejected') {
logger.error(r.reason);
for (const prov of jobProviders) {
try {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init({ ...prov, userId: job.userId }, job.blacklist);
if (browser && !browser.isConnected()) {
logger.debug('Browser is disconnected, nullifying to launch a new one.');
await puppeteerExtractor.closeBrowser(browser);
browser = null;
}
if (!browser && matchedProvider.config.getListings == null) {
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
}
await new FredyPipelineExecutioner(
matchedProvider.config,
job.notificationAdapter,
job.spatialFilter,
prov.id,
job.id,
similarityCache,
browser,
).execute();
} catch (err) {
logger.error(err);
}
}
} finally {
if (browser) {
await puppeteerExtractor.closeBrowser(browser);
}
markFinished(job.id);
try {
bus.emit('jobs:status', { jobId: job.id, running: false });

View File

@@ -0,0 +1,35 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
const R = 6371000; // Earth radius in meters
/**
* Calculate the great-circle distance between two points on Earth using the Haversine formula.
* This is to calculate the distance between the listing address & the address provided by the user. I know, it is only
* a rough estimation as this calculates the distance as a straight line, but it's more convenient than using an external
* service and still gives a good approximation for sorting purposes.
* Returns distance in meters.
*
* @param {number} lat1
* @param {number} lon1
* @param {number} lat2
* @param {number} lon2
* @returns {number}
*/
export function distanceMeters(lat1, lon1, lat2, lon2) {
const toRad = (deg) => (deg * Math.PI) / 180;
const phi1 = toRad(lat1);
const phi2 = toRad(lat2);
const dPhi = toRad(lat2 - lat1);
const dLambda = toRad(lon2 - lon1);
const a =
Math.sin(dPhi / 2) * Math.sin(dPhi / 2) +
Math.cos(phi1) * Math.cos(phi2) * Math.sin(dLambda / 2) * Math.sin(dLambda / 2);
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
return Math.round(R * c * 10) / 10;
}

View File

@@ -8,37 +8,71 @@ import { randomBetween, sleep } from '../../utils.js';
const maxAttempts = 3;
const userAgents = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
];
/**
* Check if a listing is still active with up to 3 attempts and exponential backoff.
* Backoff waits are capped and the last wait is at most 2000 ms.
* Check if a listing is still active with up to 5 attempts and exponential backoff.
* Backoff waits are randomized and capped.
*
* Rules:
* - HTTP 200 => return 1
* - HTTP 200 => return 1 (if checkForText is provided and found, returns 0)
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
* - HTTP 404 => return 0
* - Other statuses or network errors => retry until attempts are exhausted
*
* @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
* @returns {Promise<Integer>} 1 if active, 0 if not active and -1 if detected as bot
*/
export default async function checkIfListingIsActive(link) {
export default async function checkIfListingIsActive(link, checkForText = null) {
await sleep(randomBetween(50, 100));
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const userAgent = userAgents[Math.floor(Math.random() * userAgents.length)];
const res = await fetch(link, {
redirect: 'manual',
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
'User-Agent': userAgent,
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
Referer: 'https://www.google.com/',
},
});
if (res.status === 200) {
if (checkForText) {
const htmText = await res.text();
if (htmText.includes(checkForText)) {
return 0;
}
}
return 1;
}
if (res.status === 401) return -1;
if (res.status === 403) return -1;
if (res.status === 404) return 0;
if (res.status === 401 || res.status === 403) {
if (attempt < maxAttempts) {
await sleep(backoffDelay(attempt));
continue;
}
return -1;
}
if (res.status === 404 || res.status === 410) return 0;
// For any other status, only retry if attempts remain
if (attempt < maxAttempts) {
@@ -61,13 +95,13 @@ export default async function checkIfListingIsActive(link) {
}
/**
* Exponential backoff delay with cap.
* attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
* Exponential backoff delay with cap and jitter.
* @param {number} attempt 1-based attempt index
* @returns {number} delay in ms
*/
function backoffDelay(attempt) {
const base = 500;
const cap = 2000;
return Math.min(base * 2 ** (attempt - 1), cap);
const delay = Math.min(base * 2 ** (attempt - 1), cap);
return delay + randomBetween(0, 1000);
}

View File

@@ -34,9 +34,8 @@ class SqliteConnection {
static async init() {
if (this.#sqlLiteCfg == null) {
readConfigFromStorage().then((c) => {
this.#sqlLiteCfg = c.sqlitepath;
});
const c = await readConfigFromStorage();
this.#sqlLiteCfg = c.sqlitepath;
}
}
/**

View File

@@ -30,6 +30,7 @@ export const upsertJob = ({
notificationAdapter,
userId,
shareWithUsers = [],
spatialFilter = null,
}) => {
const id = jobId || nanoid();
const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0];
@@ -37,12 +38,13 @@ export const upsertJob = ({
if (existing) {
SqliteConnection.execute(
`UPDATE jobs
SET enabled = @enabled,
name = @name,
blacklist = @blacklist,
provider = @provider,
notification_adapter = @notification_adapter,
shared_with_user = @shareWithUsers
SET enabled = @enabled,
name = @name,
blacklist = @blacklist,
provider = @provider,
notification_adapter = @notification_adapter,
shared_with_user = @shareWithUsers,
spatial_filter = @spatialFilter
WHERE id = @id`,
{
id,
@@ -52,12 +54,13 @@ export const upsertJob = ({
shareWithUsers: toJson(shareWithUsers ?? []),
provider: toJson(provider ?? []),
notification_adapter: toJson(notificationAdapter ?? []),
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
},
);
} else {
SqliteConnection.execute(
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user)
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers)`,
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter)
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`,
{
id,
user_id: ownerId,
@@ -67,6 +70,7 @@ export const upsertJob = ({
provider: toJson(provider ?? []),
shareWithUsers: toJson(shareWithUsers ?? []),
notification_adapter: toJson(notificationAdapter ?? []),
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
},
);
}
@@ -87,10 +91,11 @@ export const getJob = (jobId) => {
j.provider,
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
FROM jobs j
WHERE j.id = @id
LIMIT 1`,
j.spatial_filter AS spatialFilter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
FROM jobs j
WHERE j.id = @id
LIMIT 1`,
{ id: jobId },
)[0];
if (!row) return null;
@@ -101,6 +106,7 @@ export const getJob = (jobId) => {
provider: fromJson(row.provider, []),
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
spatialFilter: fromJson(row.spatialFilter, null),
};
};
@@ -150,9 +156,11 @@ export const getJobs = () => {
j.provider,
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
FROM jobs j
ORDER BY j.name IS NULL, j.name`,
j.spatial_filter AS spatialFilter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
FROM jobs j
WHERE j.enabled = 1
ORDER BY j.name IS NULL, j.name`,
);
return rows.map((row) => ({
...row,
@@ -161,6 +169,7 @@ export const getJobs = () => {
provider: fromJson(row.provider, []),
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
spatialFilter: fromJson(row.spatialFilter, null),
}));
};
@@ -189,7 +198,7 @@ export const queryJobs = ({
isAdmin = false,
} = {}) => {
// sanitize inputs
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(1000, Math.floor(pageSize)) : 50;
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
const offset = (safePage - 1) * safePageSize;
@@ -250,11 +259,12 @@ export const queryJobs = ({
j.provider,
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
j.spatial_filter AS spatialFilter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
FROM jobs j
${whereSql}
${orderSql}
LIMIT @limit OFFSET @offset`,
${orderSql}
LIMIT @limit OFFSET @offset`,
params,
);
@@ -265,6 +275,7 @@ export const queryJobs = ({
provider: fromJson(row.provider, []),
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
spatialFilter: fromJson(row.spatialFilter, null),
}));
return { totalNumber, page: safePage, result };

View File

@@ -48,7 +48,8 @@ export const getListingsKpisForJobIds = (jobIds = []) => {
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount,
AVG(price) AS avgPrice
FROM listings
WHERE job_id IN (${placeholders})`,
WHERE job_id IN (${placeholders})
AND manually_deleted = 0`,
jobIds,
)[0] || {};
@@ -80,6 +81,7 @@ export const getProviderDistributionForJobIds = (jobIds = []) => {
`SELECT provider, COUNT(*) AS cnt
FROM listings
WHERE job_id IN (${placeholders})
AND manually_deleted = 0
GROUP BY provider
ORDER BY cnt DESC`,
jobIds,
@@ -118,8 +120,8 @@ export const getActiveOrUnknownListings = () => {
return SqliteConnection.query(
`SELECT *
FROM listings
WHERE is_active is null
OR is_active = 1
WHERE (is_active is null OR is_active = 1)
AND manually_deleted = 0
ORDER BY provider`,
);
};
@@ -240,6 +242,8 @@ export const storeListings = (jobId, providerId, listings) => {
* @param {object} [params.watchListFilter]
* @param {string|null} [params.sortField=null] - One of: 'created_at','price','size','provider','title'.
* @param {('asc'|'desc')} [params.sortDir='asc']
* @param {number} [params.createdAfter] - Only include listings created at or after this unix timestamp (ms).
* @param {number} [params.createdBefore] - Only include listings created at or before this unix timestamp (ms).
* @param {string} [params.userId] - Current user id used to scope listings (ignored for admins).
* @param {boolean} [params.isAdmin=false] - When true, returns all listings.
* @returns {{ totalNumber:number, page:number, result:Object[] }}
@@ -255,11 +259,15 @@ export const queryListings = ({
freeTextFilter,
sortField = null,
sortDir = 'asc',
createdAfter = null,
createdBefore = null,
minPrice = null,
maxPrice = null,
userId = null,
isAdmin = false,
} = {}) => {
// sanitize inputs
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(1000, Math.floor(pageSize)) : 50;
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
const offset = (safePage - 1) * safePageSize;
@@ -305,6 +313,27 @@ export const queryListings = ({
} else if (watchListFilter === false) {
whereParts.push('(wl.id IS NULL)');
}
// Time range filters (unix timestamps in milliseconds)
if (Number.isFinite(createdAfter) && createdAfter > 0) {
params.createdAfter = createdAfter;
whereParts.push('(created_at >= @createdAfter)');
}
if (Number.isFinite(createdBefore) && createdBefore > 0) {
params.createdBefore = createdBefore;
whereParts.push('(created_at <= @createdBefore)');
}
// Price range filters
if (Number.isFinite(minPrice) && minPrice >= 0) {
params.minPrice = minPrice;
whereParts.push('(l.price >= @minPrice)');
}
if (Number.isFinite(maxPrice) && maxPrice >= 0) {
params.maxPrice = maxPrice;
whereParts.push('(l.price <= @maxPrice)');
}
// Build whereSql (filtering by manually_deleted = 0)
whereParts.push('(l.manually_deleted = 0)');
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
const whereSqlWithAlias = whereSql
@@ -365,13 +394,21 @@ export const queryListings = ({
* Delete all listings for a given job id.
*
* @param {string} jobId - The job identifier whose listings should be removed.
* @returns {any} The result from SqliteConnection.execute (may contain changes count).
* @param {boolean} [hardDelete=false] - Whether to hard delete from DB or just mark as deleted.
* @returns {any} The result from SqliteConnection.execute.
*/
export const deleteListingsByJobId = (jobId) => {
export const deleteListingsByJobId = (jobId, hardDelete = false) => {
if (!jobId) return;
if (hardDelete) {
return SqliteConnection.execute(
`DELETE FROM listings
WHERE job_id = @jobId`,
{ jobId },
);
}
return SqliteConnection.execute(
`DELETE
FROM listings
`UPDATE listings
SET manually_deleted = 1
WHERE job_id = @jobId`,
{ jobId },
);
@@ -381,15 +418,23 @@ export const deleteListingsByJobId = (jobId) => {
* Delete listings by a list of listing IDs.
*
* @param {string[]} ids - Array of listing IDs to delete.
* @param {boolean} [hardDelete=false] - Whether to hard delete from DB or just mark as deleted.
* @returns {any} The result from SqliteConnection.execute.
*/
export const deleteListingsById = (ids) => {
export const deleteListingsById = (ids, hardDelete = false) => {
if (!Array.isArray(ids) || ids.length === 0) return;
const placeholders = ids.map(() => '?').join(',');
if (hardDelete) {
return SqliteConnection.execute(
`DELETE FROM listings
WHERE id IN (${placeholders})`,
ids,
);
}
return SqliteConnection.execute(
`DELETE
FROM listings
WHERE id IN (${placeholders})`,
`UPDATE listings
SET manually_deleted = 1
WHERE id IN (${placeholders})`,
ids,
);
};
@@ -404,6 +449,7 @@ export const getListingsToGeocode = () => {
`SELECT id, address
FROM listings
WHERE is_active = 1
AND manually_deleted = 0
AND address IS NOT NULL
AND (latitude IS NULL OR longitude IS NULL)`,
);
@@ -443,6 +489,7 @@ export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}
'l.latitude != -1',
'l.longitude != -1',
'l.is_active = 1',
'l.manually_deleted = 0',
];
const params = { userId: userId || '__NO_USER__' };
@@ -479,7 +526,7 @@ export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}
* @returns {{title: string|null, address: string|null, price: number|null}[]}
*/
export const getAllEntriesFromListings = () => {
return SqliteConnection.query(`SELECT title, address, price FROM listings`);
return SqliteConnection.query(`SELECT title, address, price FROM listings WHERE manually_deleted = 0`);
};
/**
@@ -493,6 +540,7 @@ export const getGeocoordinatesByAddress = (address) => {
`SELECT latitude, longitude
FROM listings
WHERE address = @address
AND manually_deleted = 0
AND latitude IS NOT NULL
AND longitude IS NOT NULL
AND latitude != -1
@@ -502,3 +550,105 @@ export const getGeocoordinatesByAddress = (address) => {
)[0];
return row ? { lat: row.latitude, lng: row.longitude } : null;
};
/**
* Return all active listings for a given job that have geocoordinates but no distance set.
*
* @param {string} jobId
* @returns {Object[]}
*/
export const getListingsToCalculateDistance = (jobId) => {
return SqliteConnection.query(
`SELECT id, latitude, longitude
FROM listings
WHERE job_id = @jobId
AND is_active = 1
AND manually_deleted = 0
AND latitude IS NOT NULL
AND longitude IS NOT NULL
AND distance_to_destination IS NULL`,
{ jobId },
);
};
/**
* Return all active listings for a given user (across all jobs) that have geocoordinates.
*
* @param {string} userId
* @returns {Object[]}
*/
export const getListingsForUserToCalculateDistance = (userId) => {
return SqliteConnection.query(
`SELECT l.id, l.latitude, l.longitude
FROM listings l
JOIN jobs j ON l.job_id = j.id
WHERE j.user_id = @userId
AND l.is_active = 1
AND l.manually_deleted = 0
AND l.latitude IS NOT NULL
AND l.longitude IS NOT NULL`,
{ userId },
);
};
/**
* Update the distance to destination for a listing.
*
* @param {string} id
* @param {number} distance
* @returns {void}
*/
export const updateListingDistance = (id, distance) => {
SqliteConnection.execute(
`UPDATE listings
SET distance_to_destination = @distance
WHERE id = @id`,
{ id, distance },
);
};
/**
* Return a single listing by id.
*
* @param {string} id
* @param {string} userId
* @param {boolean} isAdmin
* @returns {Object|null}
*/
export const getListingById = (id, userId = null, isAdmin = false) => {
const params = { id, userId: userId || '__NO_USER__' };
let whereScoping = '';
if (!isAdmin) {
whereScoping = `AND (j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`;
}
return (
SqliteConnection.query(
`SELECT l.*, j.name AS job_name, CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END AS isWatched
FROM listings l
LEFT JOIN jobs j ON j.id = l.job_id
LEFT JOIN watch_list wl ON wl.listing_id = l.id AND wl.user_id = @userId
WHERE l.id = @id AND l.manually_deleted = 0 ${whereScoping}`,
params,
)[0] || null
);
};
/**
* Resets geocoordinates and distance for all listings related to a user.
*
* @param {string} userId
* @returns {void}
*/
export const resetGeocoordinatesAndDistanceForUser = (userId) => {
SqliteConnection.execute(
`UPDATE listings
SET latitude = NULL,
longitude = NULL,
distance_to_destination = NULL
WHERE job_id IN (
SELECT id FROM jobs j
WHERE j.user_id = @userId
)`,
{ userId },
);
};

View File

@@ -88,7 +88,7 @@ export function up(db) {
}
} catch (e) {
// If parsing fails, let it throw to rollback the migration
throw new Error(`Failed to import users from ${usersJsonPath}: ${e.message}`);
throw new Error(`Failed to import users from ${usersJsonPath}: ${e.message}`, { cause: e });
}
}
@@ -116,7 +116,7 @@ export function up(db) {
}
}
} catch (e) {
throw new Error(`Failed to import jobs from ${jobsJsonPath}: ${e.message}`);
throw new Error(`Failed to import jobs from ${jobsJsonPath}: ${e.message}`, { cause: e });
}
}
}

View File

@@ -0,0 +1,16 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export function up(db) {
// 1. Add manually_deleted column
db.exec(`ALTER TABLE listings ADD COLUMN manually_deleted INTEGER NOT NULL DEFAULT 0;`);
// 2. Remove change_set column
try {
db.exec(`ALTER TABLE listings DROP COLUMN change_set;`);
} catch {
// if column does not exists for whatever reason
}
}

View File

@@ -0,0 +1,11 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
// Migration: Add spatial_filter column to jobs table for storing GeoJSON-based spatial filters
export function up(db) {
db.exec(`
ALTER TABLE jobs ADD COLUMN spatial_filter JSONB DEFAULT NULL;
`);
}

View File

@@ -0,0 +1,25 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import crypto from 'crypto';
// Migration: Add mcp_token column to users table.
// Each user gets a permanent, non-expiring secret token used for MCP API authentication.
// Tokens are auto-generated for all existing users during this migration.
export function up(db) {
db.exec(`ALTER TABLE users ADD COLUMN mcp_token TEXT`);
// Backfill all existing users that don't have a token yet
const users = db.prepare(`SELECT id FROM users WHERE mcp_token IS NULL`).all();
const update = db.prepare(`UPDATE users SET mcp_token = @token WHERE id = @id`);
for (const user of users) {
const token = `fredy_${crypto.randomBytes(32).toString('hex')}`;
update.run({ id: user.id, token });
}
}

View File

@@ -0,0 +1,12 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
// Migration: Removing city field and adding distance field
export function up(db) {
db.exec(`
ALTER TABLE listings ADD COLUMN distance_to_destination INTEGER;
`);
}

View File

@@ -0,0 +1,20 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export function up(db) {
// 1. Remove old unique index
db.exec(`DROP INDEX IF EXISTS idx_settings_name;`);
// 2. Add new unique index for name and user_id.
// Since user_id can be NULL, we need a special index or use coalesce for the index.
// In SQLite, multiple NULLs are allowed in a UNIQUE index, which is fine for our global settings (user_id IS NULL).
// But we want only one global setting for a given name.
// Actually, in SQLite, UNIQUE allows multiple NULL values.
// To have only one NULL user_id for a name, we can use a partial index or COALESCE.
db.exec(`
CREATE UNIQUE INDEX idx_settings_name_user_id ON settings (name, IFNULL(user_id, 'GLOBAL_SETTING'));
`);
}

View File

@@ -37,12 +37,25 @@ function compileSettings(rows, configValues) {
* @returns {Record<string, any>}
*/
export async function refreshSettingsCache() {
const rows = SqliteConnection.query(`SELECT name, value FROM settings`);
const rows = SqliteConnection.query(`SELECT name, value FROM settings WHERE user_id IS NULL`);
const configValues = await readConfigFromStorage();
cachedSettingsConfig = compileSettings(rows, configValues);
return cachedSettingsConfig;
}
/**
* Retrieves user-specific settings from the database.
* @param {string} userId
* @returns {Record<string, any>}
*/
export function getUserSettings(userId) {
if (!userId || typeof userId !== 'string') {
return {};
}
const userRows = SqliteConnection.query(`SELECT name, value FROM settings WHERE user_id = @userId`, { userId });
return compileSettings(userRows, {});
}
/**
* Get the compiled settings config. Loads it once and caches the result.
* @returns {Record<string, any>}
@@ -77,16 +90,28 @@ export function upsertSettings(settingsMapOrEntry, userId = null) {
: Object.entries(settingsMapOrEntry || {});
for (const [name, rawValue] of entries) {
const id = nanoid();
const create_date = Date.now();
const json = toJson(rawValue);
SqliteConnection.execute(
`INSERT INTO settings (id, create_date, name, value, user_id)
if (rawValue === null) {
SqliteConnection.execute(
`DELETE FROM settings WHERE name = @name AND (user_id = @userId OR (user_id IS NULL AND @userId IS NULL))`,
{
name,
userId,
},
);
} else {
const id = nanoid();
const create_date = Date.now();
const json = toJson(rawValue);
SqliteConnection.execute(
`INSERT INTO settings (id, create_date, name, value, user_id)
VALUES (@id, @create_date, @name, @value, @userId)
ON CONFLICT(name) DO UPDATE SET value = excluded.value`,
{ id, create_date, name, value: json, userId },
);
ON CONFLICT(name, IFNULL(user_id, 'GLOBAL_SETTING')) DO UPDATE SET value = excluded.value`,
{ id, create_date, name, value: json, userId },
);
}
}
// keep cache in sync (only for global settings)
if (userId == null) {
refreshSettingsCache();
}
// keep cache in sync
refreshSettingsCache();
}

View File

@@ -5,8 +5,17 @@
import * as hasher from '../security/hash.js';
import { nanoid } from 'nanoid';
import crypto from 'crypto';
import SqliteConnection from './SqliteConnection.js';
import { getSettings } from './settingsStorage.js';
import { inDevMode } from '../../utils.js';
/**
* Generate a permanent, non-expiring MCP API token.
* These tokens are secrets that never expire and are used for MCP authentication.
* @returns {string}
*/
const generateMcpToken = () => `fredy_${crypto.randomBytes(32).toString('hex')}`;
/**
* Get all users.
@@ -20,7 +29,7 @@ import { getSettings } from './settingsStorage.js';
*/
export const getUsers = (withPassword) => {
const rows = SqliteConnection.query(
`SELECT u.id, u.username, u.password, u.last_login AS lastLogin, u.is_admin AS isAdmin,
`SELECT u.id, u.username, u.password, u.last_login AS lastLogin, u.is_admin AS isAdmin, u.mcp_token AS mcpToken,
(SELECT COUNT(1) FROM jobs j WHERE j.user_id = u.id) AS numberOfJobs
FROM users u
ORDER BY u.username`,
@@ -40,7 +49,7 @@ export const getUsers = (withPassword) => {
*/
export const getUser = (id) => {
const rows = SqliteConnection.query(
`SELECT u.id, u.username, u.password, u.last_login AS lastLogin, u.is_admin AS isAdmin,
`SELECT u.id, u.username, u.password, u.last_login AS lastLogin, u.is_admin AS isAdmin, u.mcp_token AS mcpToken,
(SELECT COUNT(1) FROM jobs j WHERE j.user_id = u.id) AS numberOfJobs
FROM users u
WHERE u.id = @id
@@ -87,14 +96,15 @@ export const upsertUser = ({ username, password, userId, isAdmin }) => {
}
} else {
SqliteConnection.execute(
`INSERT INTO users (id, username, password, last_login, is_admin)
VALUES (@id, @username, @password, @last_login, @is_admin)`,
`INSERT INTO users (id, username, password, last_login, is_admin, mcp_token)
VALUES (@id, @username, @password, @last_login, @is_admin, @mcp_token)`,
{
id,
username,
password: hasher.hash(password || ''),
last_login: null,
is_admin: isAdmin ? 1 : 0,
mcp_token: generateMcpToken(),
},
);
}
@@ -137,17 +147,21 @@ export const removeUser = (userId) => {
export const ensureDemoUserExists = async () => {
const settings = await getSettings();
if (!settings.demoMode) {
// Remove demo user (and cascade delete their jobs/listings)
SqliteConnection.execute(`DELETE FROM users WHERE username = 'demo'`);
if (!inDevMode()) {
// Remove demo user (and cascade delete their jobs/listings)
SqliteConnection.execute(`DELETE
FROM users
WHERE username = 'demo'`);
}
return;
}
// Ensure demo user exists when demo mode is on
const existing = SqliteConnection.query(`SELECT id FROM users WHERE username = 'demo' LIMIT 1`);
if (existing.length === 0) {
SqliteConnection.execute(
`INSERT INTO users (id, username, password, last_login, is_admin)
VALUES (@id, 'demo', @password, NULL, 1)`,
{ id: nanoid(), password: hasher.hash('demo') },
`INSERT INTO users (id, username, password, last_login, is_admin, mcp_token)
VALUES (@id, 'demo', @password, NULL, 1, @mcp_token)`,
{ id: nanoid(), password: hasher.hash('demo'), mcp_token: generateMcpToken() },
);
}
};
@@ -162,13 +176,25 @@ export const ensureDemoUserExists = async () => {
* Security: On a fresh instance, a default admin/admin is created; change this password immediately.
* @returns {void}
*/
/**
* Validate an MCP API token and return the associated user id.
* MCP tokens are permanent secrets stored in the users table that never expire.
* @param {string} token - The raw token string (e.g. fredy_...).
* @returns {{ userId: string } | null} The user id or null if invalid.
*/
export const validateMcpToken = (token) => {
if (!token) return null;
const row = SqliteConnection.query(`SELECT id FROM users WHERE mcp_token = @token LIMIT 1`, { token })[0];
return row ? { userId: row.id } : null;
};
export const ensureAdminUserExists = () => {
const anyUser = SqliteConnection.query(`SELECT id FROM users LIMIT 1`).length > 0;
if (!anyUser) {
SqliteConnection.execute(
`INSERT INTO users (id, username, password, last_login, is_admin)
VALUES (@id, 'admin', @password, @last_login, 1)`,
{ id: nanoid(), password: hasher.hash('admin'), last_login: Date.now() },
`INSERT INTO users (id, username, password, last_login, is_admin, mcp_token)
VALUES (@id, 'admin', @password, @last_login, 1, @mcp_token)`,
{ id: nanoid(), password: hasher.hash('admin'), last_login: Date.now(), mcp_token: generateMcpToken() },
);
return;
}

View File

@@ -14,73 +14,89 @@ import { getSettings } from '../storage/settingsStorage.js';
const deviceId = getUniqueId() || 'N/A';
const version = await getPackageVersion();
const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
const isDocker = process.env.IS_DOCKER != null;
export const trackMainEvent = async () => {
const staticTrackingData = {
operatingSystem: os.platform(),
osVersion: os.release(),
isDocker,
arch: process.arch,
language: process.env.LANG || 'en',
nodeVersion: process.version || 'N/A',
deviceId,
version,
};
const shouldTrack = async () => {
const settings = await getSettings();
return settings.analyticsEnabled && !inDevMode();
};
const sendTrackingData = async (endpoint, payload) => {
try {
const settings = await getSettings();
if (settings.analyticsEnabled && !inDevMode()) {
const activeProvider = new Set();
const activeAdapter = new Set();
const jobs = getJobs();
if (jobs != null && jobs.length > 0) {
jobs.forEach((job) => {
job.provider.forEach((provider) => activeProvider.add(provider.id));
job.notificationAdapter.forEach((adapter) => activeAdapter.add(adapter.id));
});
const trackingObj = enrichTrackingObject({
adapter: Array.from(activeAdapter),
provider: Array.from(activeProvider),
});
await fetch(`${FREDY_TRACKING_URL}/main`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(trackingObj),
});
}
const response = await fetch(`${FREDY_TRACKING_URL}${endpoint}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: payload ? JSON.stringify(payload) : undefined,
});
if (!response.ok) {
logger.warn(`Error sending tracking data to ${endpoint}. Status: ${response.status}`);
}
} catch (error) {
logger.warn('Error sending tracking data', error);
logger.warn(`Error sending tracking data to ${endpoint}`, error);
}
};
export const trackMainEvent = async () => {
if (!(await shouldTrack())) return;
const activeProvider = new Set();
const activeAdapter = new Set();
const jobs = getJobs();
if (jobs != null && jobs.length > 0) {
jobs.forEach((job) => {
job.provider.forEach((provider) => activeProvider.add(provider.id));
job.notificationAdapter.forEach((adapter) => activeAdapter.add(adapter.id));
});
const trackingObj = await enrichTrackingObject({
adapter: Array.from(activeAdapter),
provider: Array.from(activeProvider),
});
await sendTrackingData('/main', trackingObj);
}
};
export const trackPoi = async (poi) => {
if (!(await shouldTrack())) return;
const trackingObj = await enrichTrackingObject({
feature: poi,
});
await sendTrackingData('/feature', trackingObj);
};
/**
* Note, this will only be used when Fredy runs in demo mode
*/
export async function trackDemoAccessed() {
const settings = await getSettings();
if (settings.analyticsEnabled && !inDevMode() && settings.demoMode) {
try {
await fetch(`${FREDY_TRACKING_URL}/demo/accessed`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
});
} catch (error) {
logger.warn('Error sending tracking data', error);
}
const trackingObj = await enrichTrackingObject({});
await sendTrackingData('/demo/accessed', trackingObj);
}
}
async function enrichTrackingObject(trackingObject) {
const settings = await getSettings();
const operatingSystem = os.platform();
const osVersion = os.release();
const arch = process.arch;
const language = process.env.LANG || 'en';
const nodeVersion = process.version || 'N/A';
return {
...trackingObject,
...staticTrackingData,
isDemo: settings.demoMode,
operatingSystem,
osVersion,
arch,
nodeVersion,
language,
deviceId,
version,
};
}

View File

@@ -95,7 +95,7 @@ function isOneOf(word, arr) {
* @returns {boolean}
*/
function nullOrEmpty(val) {
return val == null || val.length === 0 || val === 'null' || val === 'undefined';
return val == null || val.length === 0;
}
/**

13467
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "18.0.1",
"version": "20.0.3",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",
@@ -14,10 +14,12 @@
"test": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 test/**/*.test.js",
"testGH": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 --exclude test/provider/immonet.test.js --exclude test/provider/immobilienDe.test.js --exclude test/provider/immowelt.test.js test/**/*.test.js",
"lint": "eslint .",
"mcp:stdio": "node lib/mcp/stdio.js",
"lint:fix": "yarn lint --fix",
"migratedb": "node lib/services/storage/migrations/migrate.js",
"migratedb:overwrite": "x-var MIGRATION_ALLOW_CHECKSUM_UPDATE=true node lib/services/storage/migrations/migrate.js",
"copyright": "node ./copyright.js"
"copyright": "node ./copyright.js",
"release": "node ./tools/release/release.js"
},
"type": "module",
"lint-staged": {
@@ -59,59 +61,67 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-icons": "^2.90.13",
"@douyinfe/semi-ui": "2.90.13",
"@douyinfe/semi-icons": "^2.92.2",
"@douyinfe/semi-ui": "2.92.2",
"@douyinfe/semi-ui-19": "^2.92.2",
"@mapbox/mapbox-gl-draw": "^1.5.1",
"@sendgrid/mail": "8.1.6",
"@vitejs/plugin-react": "5.1.2",
"@vitejs/plugin-react": "5.1.4",
"@modelcontextprotocol/sdk": "^1.27.1",
"adm-zip": "^0.5.16",
"better-sqlite3": "^12.6.0",
"better-sqlite3": "^12.6.2",
"body-parser": "2.2.2",
"chart.js": "^4.5.1",
"cheerio": "^1.1.2",
"cheerio": "^1.2.0",
"@turf/boolean-point-in-polygon": "^7.3.4",
"cookie-session": "2.1.1",
"handlebars": "4.7.8",
"lodash": "4.17.21",
"maplibre-gl": "^5.16.0",
"lodash": "4.17.23",
"maplibre-gl": "^5.20.0",
"nanoid": "5.1.6",
"node-cron": "^4.2.1",
"node-fetch": "3.3.2",
"node-mailjet": "6.0.11",
"p-throttle": "^8.1.0",
"package-up": "^5.0.0",
"puppeteer": "^24.35.0",
"puppeteer": "^24.39.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.3.1",
"react": "18.3.1",
"react": "19.2.4",
"react-chartjs-2": "^5.3.1",
"react-dom": "18.3.1",
"react-dom": "19.2.4",
"react-range-slider-input": "^3.3.2",
"react-router": "7.12.0",
"react-router-dom": "7.12.0",
"react-router": "7.13.1",
"react-router-dom": "7.13.1",
"resend": "^6.9.3",
"restana": "5.1.0",
"semver": "^7.7.3",
"semver": "^7.7.4",
"serve-static": "2.2.1",
"slack": "11.0.2",
"vite": "7.3.1",
"x-var": "^3.0.1",
"zustand": "^5.0.10"
"zustand": "^5.0.11"
},
"devDependencies": {
"@babel/core": "7.28.6",
"@babel/core": "7.29.0",
"@babel/eslint-parser": "7.28.6",
"@babel/preset-env": "7.28.6",
"@babel/preset-env": "7.29.0",
"@babel/preset-react": "7.28.5",
"@eslint/js": "^10.0.1",
"chai": "6.2.2",
"eslint": "9.39.2",
"chalk": "^5.6.2",
"eslint": "10.0.3",
"eslint-config-prettier": "10.1.8",
"eslint-plugin-react": "7.37.5",
"esmock": "2.7.3",
"globals": "^17.4.0",
"history": "5.3.0",
"husky": "9.1.7",
"less": "4.5.1",
"lint-staged": "16.2.7",
"less": "4.6.2",
"lint-staged": "16.3.3",
"mocha": "11.7.5",
"nodemon": "^3.1.11",
"prettier": "3.8.0"
"nodemon": "^3.1.14",
"prettier": "3.8.1"
}
}

View File

@@ -41,7 +41,7 @@ Challenges:
_Returns the total number of listings for the given query._
```
curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
curl -H "User-Agent: ImmoScout_27.12_26.2_._" \
-H "Accept: application/json" \
"https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin"
```
@@ -63,7 +63,7 @@ _The body is json encoded and contains data specifying additional results (adver
```
curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' \
-H "Connection: keep-alive" \
-H "User-Agent: ImmoScout_27.3_26.0_._" \
-H "User-Agent: ImmoScout_27.12_26.2_._" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-d '{"supportedResultListType":[],"userData":{}}'
@@ -78,7 +78,7 @@ curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calc
The response contains additional details not included in the listing response.
```
curl -H "User-Agent: ImmoScout_27.3_26.0_._" \
curl -H "User-Agent: ImmoScout_27.12_26.2_._" \
-H "Accept: application/json" \
"https://api.mobile.immobilienscout24.de/expose/158382494"
```

View File

@@ -3,6 +3,7 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/* eslint-disable no-unused-vars */
const db = {};
export const storeListings = (jobKey, providerId, listings) => {
if (!Array.isArray(listings)) throw Error('Not a valid array');
@@ -11,3 +12,20 @@ export const storeListings = (jobKey, providerId, listings) => {
export const getKnownListingHashesForJobAndProvider = (jobKey, providerId) => {
return db[providerId] || [];
};
export const getGeocoordinatesByAddress = (any) => {
return null;
};
export function getUserSettings(userId) {
return null;
}
export const updateListingDistance = (id, distance) => {
// noop
};
export const deletedIds = [];
export const deleteListingsById = (ids) => {
deletedIds.push(...ids);
};
/* eslint-enable no-unused-vars */

View File

@@ -0,0 +1,89 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { expect } from 'chai';
import { mockFredy } from './utils.js';
import * as mockStore from './mocks/mockStore.js';
describe('Issue reproduction: listings filtered by similarity or area should be marked as manually deleted', () => {
it('should call deleteListingsById when listings are filtered by similarity', async () => {
const Fredy = await mockFredy();
const mockSimilarityCache = {
checkAndAddEntry: () => true, // always similar
};
const providerConfig = {
url: 'http://example.com',
getListings: () => Promise.resolve([{ id: '1', title: 'test', address: 'addr', price: '100' }]),
normalize: (l) => l,
filter: () => true,
crawlFields: { id: 'id', title: 'title', address: 'address', price: 'price' },
};
const fredy = new Fredy(providerConfig, null, null, 'test-provider', 'test-job', mockSimilarityCache);
// Clear deletedIds before test
mockStore.deletedIds.length = 0;
try {
await fredy.execute();
} catch {
// Might throw NoNewListingsWarning if all are filtered out
}
expect(mockStore.deletedIds).to.include('1');
});
it('should call deleteListingsById when listings are filtered by area', async () => {
const Fredy = await mockFredy();
const mockSimilarityCache = {
checkAndAddEntry: () => false, // never similar
};
const spatialFilter = {
features: [
{
type: 'Feature',
geometry: {
type: 'Polygon',
coordinates: [
[
[0, 0],
[0, 1],
[1, 1],
[1, 0],
[0, 0],
],
],
},
},
],
};
const providerConfig = {
url: 'http://example.com',
getListings: () =>
Promise.resolve([{ id: '2', title: 'test', address: 'addr', price: '100', latitude: 2, longitude: 2 }]), // outside polygon
normalize: (l) => l,
filter: () => true,
crawlFields: { id: 'id', title: 'title', address: 'address', price: 'price' },
};
const fredy = new Fredy(providerConfig, null, spatialFilter, 'test-provider', 'test-job', mockSimilarityCache);
// Clear deletedIds before test
mockStore.deletedIds.length = 0;
try {
await fredy.execute();
} catch {
// Might throw NoNewListingsWarning if all are filtered out
}
expect(mockStore.deletedIds).to.include('2');
});
});

View File

@@ -14,7 +14,14 @@ describe('#einsAImmobilien testsuite()', () => {
it('should test einsAImmobilien provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'einsAImmobilien', similarityCache);
const fredy = new Fredy(
provider.config,
null,
null,
provider.metaInformation.id,
'einsAImmobilien',
similarityCache,
);
fredy.execute().then((listings) => {
expect(listings).to.be.a('array');
const notificationObj = get();

View File

@@ -14,7 +14,7 @@ describe('#immobilien.de testsuite()', () => {
it('should test immobilien.de provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();

View File

@@ -1,38 +0,0 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
import { get } from '../mocks/mockNotification.js';
import { mockFredy, providerConfig } from '../utils.js';
import { expect } from 'chai';
import * as provider from '../../lib/provider/immonet.js';
describe('#immonet testsuite()', () => {
it('should test immonet provider', async () => {
const Fredy = await mockFredy();
provider.init(providerConfig.immonet, [], []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immonet', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');
const notificationObj = get();
expect(notificationObj).to.be.a('object');
expect(notificationObj.serviceName).to.equal('immonet');
notificationObj.payload.forEach((notify) => {
/** check the actual structure **/
expect(notify.id).to.be.a('string');
expect(notify.price).to.be.a('string');
expect(notify.size).to.be.a('string');
expect(notify.title).to.be.a('string');
expect(notify.link).to.be.a('string');
expect(notify.address).to.be.a('string');
/** check the values if possible **/
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty;
expect(notify.address).to.be.not.empty;
});
});
});

View File

@@ -14,7 +14,7 @@ describe('#immoscout provider testsuite()', () => {
it('should test immoscout provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, '', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, '', similarityCache);
fredy.execute().then((listings) => {
expect(listings).to.be.a('array');
const notificationObj = get();

View File

@@ -14,7 +14,7 @@ describe('#immoswp testsuite()', () => {
it('should test immoswp provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immoswp', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immoswp', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();

View File

@@ -14,7 +14,7 @@ describe('#immowelt testsuite()', () => {
const Fredy = await mockFredy();
provider.init(providerConfig.immowelt, [], []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'immowelt', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immowelt', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');

View File

@@ -14,7 +14,14 @@ describe('#kleinanzeigen testsuite()', () => {
const Fredy = await mockFredy();
provider.init(providerConfig.kleinanzeigen, [], []);
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'kleinanzeigen', similarityCache);
const fredy = new Fredy(
provider.config,
null,
null,
provider.metaInformation.id,
'kleinanzeigen',
similarityCache,
);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();

View File

@@ -14,7 +14,7 @@ describe('#mcMakler testsuite()', () => {
const Fredy = await mockFredy();
provider.init(providerConfig.mcMakler, []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'mcMakler', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'mcMakler', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');

View File

@@ -14,7 +14,14 @@ describe('#neubauKompass testsuite()', () => {
it('should test neubauKompass provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'neubauKompass', similarityCache);
const fredy = new Fredy(
provider.config,
null,
null,
provider.metaInformation.id,
'neubauKompass',
similarityCache,
);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();

View File

@@ -14,7 +14,7 @@ describe('#ohneMakler testsuite()', () => {
const Fredy = await mockFredy();
provider.init(providerConfig.ohneMakler, []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'ohneMakler', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'ohneMakler', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');

View File

@@ -17,6 +17,7 @@ describe('#regionalimmobilien24 testsuite()', () => {
const fredy = new Fredy(
provider.config,
null,
null,
provider.metaInformation.id,
'regionalimmobilien24',
similarityCache,

View File

@@ -14,7 +14,7 @@ describe('#sparkasse testsuite()', () => {
const Fredy = await mockFredy();
provider.init(providerConfig.sparkasse, []);
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'sparkasse', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'sparkasse', similarityCache);
const listing = await fredy.execute();
expect(listing).to.be.a('array');

View File

@@ -8,10 +8,6 @@
"url": "https://www.immobilien.de/Wohnen/Suchergebnisse-51797.html?search._digest=true&search._filter=wohnen&search.flaeche_von=50&search.objektart=wohnung&search.preis_bis=1200&search.typ=mieten&search.umkreis=15&search.wo=district%3A2434%2C2695%2C2621%2C2700%2C2967%2C2734%2C2909%2C2955%2C2392%2C2746%2C2767%2C2982%2C2904%2C2612%2C2892%2C2587%2C2871%2C2975%2C2591%2C2887%2C2569%2C2640%2C2735&sort_col=*created_ts&sort_dir=desc",
"enabled": true
},
"immonet": {
"url": "https://www.immonet.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2112&order=Default&m=homepage_new_search_classified_search_result",
"enabled": true
},
"immowelt": {
"url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
"enabled": true

View File

@@ -14,7 +14,7 @@ describe('#wgGesucht testsuite()', () => {
it('should test wgGesucht provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
fredy.execute().then((listing) => {
expect(listing).to.be.a('array');
const notificationObj = get();

View File

@@ -14,7 +14,14 @@ describe('#wohnungsboerse testsuite()', () => {
it('should test wohnungsboerse provider', async () => {
const Fredy = await mockFredy();
return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'wohnungsboerse', similarityCache);
const fredy = new Fredy(
provider.config,
null,
null,
provider.metaInformation.id,
'wohnungsboerse',
similarityCache,
);
fredy.execute().then((listings) => {
expect(listings).to.be.a('array');
const notificationObj = get();

View File

@@ -14,12 +14,6 @@
"shouldBecome": "https://www.wg-gesucht.de/1-zimmer-wohnungen-in-Dusseldorf.30.1.1.0.html?sort_column=0&sort_order=0",
"id": "wgGesucht"
},
{
"url": "https://www.immonet.de/immobiliensuche/sel.do?sortby=0&suchart=1&objecttype=1&marketingtype=2&parentcat=1&locationname=d%C3%BCsseldorf",
"shouldBecome": "https://www.immonet.de/immobiliensuche/sel.do?sortby=19&suchart=1&objecttype=1&marketingtype=2&parentcat=1&locationname=d%C3%BCsseldorf",
"id": "immonet"
},
{
"url": "https://www.neubaukompass.de/neubau-immobilien/berlin-region/",
"shouldBecome": "https://www.neubaukompass.de/neubau-immobilien/berlin-region/?Sortierung=Id&Richtung=DESC",

View File

@@ -58,7 +58,7 @@ describe('#immoscout-mobile URL conversion', () => {
const response = await fetch(url, {
method: 'POST',
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
'User-Agent': 'ImmoScout_27.12_26.2_._',
'Content-Type': 'application/json',
},
body: JSON.stringify({
@@ -75,7 +75,9 @@ describe('#immoscout-mobile URL conversion', () => {
expect(responseBody.totalResults).to.be.greaterThan(0);
expect(responseBody.totalResults).to.be.greaterThan(0);
expect(responseBody.resultListItems.length).to.greaterThan(0);
expect(responseBody.resultListItems[0].item.realEstateType).to.equal(type);
expect(responseBody.resultListItems.filter((r) => r.type === 'EXPOSE_RESULT')[0].item.realEstateType).to.equal(
type,
);
}
});
});

View File

@@ -15,6 +15,15 @@ export const mockFredy = async () => {
'../lib/services/storage/listingsStorage.js': {
...mockStore,
},
'../lib/services/storage/settingsStorage.js': {
...mockStore,
},
'../lib/services/geocoding/geoCodingService.js': {
geocodeAddress: mockStore.getGeocoordinatesByAddress,
},
'../lib/services/storage/jobStorage.js': {
getJob: (jobKey) => ({ id: jobKey, userId: 'user1' }),
},
'../lib/notification/notify.js': {
send,
},

196
tools/release/release.js Normal file
View File

@@ -0,0 +1,196 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import fs from 'fs';
import path from 'path';
import { execSync, spawn } from 'child_process';
import fetch from 'node-fetch';
import chalk from 'chalk';
import { fileURLToPath } from 'url';
/**
* Release Tool for Fredy
*
* This tool automates the process of creating a GitHub release.
* It fetches the latest release, compares it with the current master branch,
* allows manual editing of commit messages, and creates a new release on GitHub.
*/
// Define __dirname for ESM
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// Configuration and Paths
const CONFIG_PATH = path.join(__dirname, 'config.json');
const PACKAGE_JSON_PATH = path.join(__dirname, '../../package.json');
const REPO = 'orangecoding/fredy';
const config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
const GITHUB_TOKEN = config.github_token;
/**
* Main function to execute the release process
*/
async function createRelease() {
/* eslint-disable no-console */
try {
console.log(chalk.cyan('🚀 Starting release process...'));
// 1. Load Configuration
if (!fs.existsSync(CONFIG_PATH)) {
console.error(chalk.red('❌ Error: config.json not found in tools/release/'));
process.exit(1);
}
if (!GITHUB_TOKEN) {
console.error(chalk.red('❌ Error: GitHub token not configured.'));
process.exit(1);
}
// 2. Get current version from package.json
const packageJson = JSON.parse(fs.readFileSync(PACKAGE_JSON_PATH, 'utf8'));
const version = packageJson.version;
const tag = version; // Using version as tag
console.log(chalk.blue(`📦 Target version: ${version}`));
// 3. Check if release already exists
console.log(chalk.yellow('🔍 Checking if release already exists...'));
const existingReleaseResponse = await fetch(`https://api.github.com/repos/${REPO}/releases/tags/${tag}`, {
headers: {
Authorization: `token ${GITHUB_TOKEN}`,
Accept: 'application/vnd.github.v3+json',
},
});
if (existingReleaseResponse.status === 200) {
console.error(chalk.red(`❌ Error: A release with tag ${tag} already exists.`));
process.exit(1);
}
// 4. Fetch latest release to find the starting point for the diff
console.log(chalk.yellow('📡 Fetching latest release from GitHub...'));
const latestReleaseResponse = await fetch(`https://api.github.com/repos/${REPO}/releases/latest`, {
headers: {
Authorization: `token ${GITHUB_TOKEN}`,
Accept: 'application/vnd.github.v3+json',
},
});
if (!latestReleaseResponse.ok) {
console.error(chalk.red('❌ Error fetching latest release.'));
const errorData = await latestReleaseResponse.json();
console.error(chalk.red(JSON.stringify(errorData)));
process.exit(1);
}
const latestRelease = await latestReleaseResponse.json();
const latestTag = latestRelease.tag_name;
console.log(chalk.green(`✅ Latest release found: ${latestTag}`));
// 5. Ensure the latest tag is available locally
console.log(chalk.yellow(`📡 Fetching tag ${latestTag} from remote...`));
try {
execSync(`git fetch origin tag ${latestTag} --no-tags`);
} catch (error) {
console.error(chalk.red(`❌ Error fetching tag ${latestTag} from origin.`));
console.error(error.message);
// We don't exit here, maybe it's already there but fetch failed for some reason
}
// 6. Get commit messages between latest tag and current HEAD
console.log(chalk.yellow(`Git diff: ${latestTag} .. HEAD`));
let commitMessages;
try {
commitMessages = execSync(`git log ${latestTag}..HEAD --pretty=format:"- %s"`).toString().trim();
} catch (error) {
console.error(chalk.red('❌ Error running git log. Make sure the latest tag is available locally.'), error);
process.exit(1);
}
if (!commitMessages) {
console.log(chalk.magenta('⚠️ No new commits found since last release.'));
commitMessages = '- No changes recorded';
}
// 7. Open commit messages in editor for manual adjustment
const tempFilePath = path.join(__dirname, 'CHANGELOG_EDIT.tmp');
const initialContent = `# Release Notes for ${version}\n# Edit the messages below. Lines starting with # will be ignored.\n\n${commitMessages}`;
fs.writeFileSync(tempFilePath, initialContent);
console.log(chalk.blue('📝 Opening editor for release notes (using nano or $EDITOR)...'));
await openInEditor(tempFilePath);
// 8. Read edited content
let editedContent = fs
.readFileSync(tempFilePath, 'utf8')
.split('\n')
.filter((line) => !line.startsWith('#'))
.join('\n')
.trim();
fs.unlinkSync(tempFilePath); // Clean up temp file
if (!editedContent) {
console.error(chalk.red('❌ Release notes are empty. Aborting release.'));
process.exit(1);
}
// 9. Create the new release
console.log(chalk.cyan(`🚀 Creating release ${version} on GitHub...`));
const createResponse = await fetch(`https://api.github.com/repos/${REPO}/releases`, {
method: 'POST',
headers: {
Authorization: `token ${GITHUB_TOKEN}`,
Accept: 'application/vnd.github.v3+json',
'Content-Type': 'application/json',
},
body: JSON.stringify({
tag_name: tag,
name: version,
body: editedContent,
draft: false,
prerelease: false,
}),
});
if (createResponse.status === 201) {
const data = await createResponse.json();
console.log(chalk.green('🎉 Release successfully created!'));
console.log(chalk.green(`🔗 URL: ${data.html_url}`));
} else {
const errorData = await createResponse.json();
console.error(chalk.red('❌ Failed to create release.'));
console.error(chalk.red(JSON.stringify(errorData, null, 2)));
process.exit(1);
}
} catch (error) {
console.error(chalk.red('💥 An unexpected error occurred:'));
console.error(error);
process.exit(1);
}
}
/**
* Helper to open a file in a terminal editor
* @param {string} filePath
*/
function openInEditor(filePath) {
return new Promise((resolve, reject) => {
const editor = process.env.EDITOR || 'nano';
const child = spawn(editor, [filePath], {
stdio: 'inherit',
});
child.on('exit', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`Editor exited with code ${code}`));
}
});
});
}
await createRelease();
/* eslint-enable no-console */

View File

@@ -8,6 +8,7 @@ import React, { useEffect } from 'react';
import InsufficientPermission from './components/permission/InsufficientPermission';
import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
import GeneralSettings from './views/generalSettings/GeneralSettings';
import UserSettings from './views/userSettings/UserSettings';
import JobMutation from './views/jobs/mutation/JobMutation';
import UserMutator from './views/user/mutation/UserMutator';
import { useActions, useSelector } from './services/state/store';
@@ -18,15 +19,17 @@ import Jobs from './views/jobs/Jobs';
import './App.less';
import TrackingModal from './components/tracking/TrackingModal.jsx';
import { Banner, Divider } from '@douyinfe/semi-ui';
import { Banner } from '@douyinfe/semi-ui-19';
import VersionBanner from './components/version/VersionBanner.jsx';
import Listings from './views/listings/Listings.jsx';
import MapView from './views/listings/Map.jsx';
import Navigation from './components/navigation/Navigation.jsx';
import { Layout } from '@douyinfe/semi-ui';
import { Layout } from '@douyinfe/semi-ui-19';
import FredyFooter from './components/footer/FredyFooter.jsx';
import WatchlistManagement from './views/listings/management/WatchlistManagement.jsx';
import Dashboard from './views/dashboard/Dashboard.jsx';
import ListingDetail from './views/listings/ListingDetail.jsx';
import NewsModal from './components/news/NewsModal.jsx';
export default function FredyApp() {
const actions = useActions();
@@ -39,13 +42,14 @@ export default function FredyApp() {
async function init() {
await actions.user.getCurrentUser();
if (!needsLogin()) {
await actions.features.getFeatures();
await actions.provider.getProvider();
await actions.jobsData.getJobs();
await actions.jobsData.getSharableUserList();
await actions.notificationAdapter.getAdapter();
await actions.generalSettings.getGeneralSettings();
await actions.userSettings.getUserSettings();
await actions.versionUpdate.getVersionUpdate();
await actions.tracking.getTrackingPois();
}
setLoading(false);
}
@@ -58,7 +62,7 @@ export default function FredyApp() {
};
const isAdmin = () => currentUser != null && currentUser.isAdmin;
const { Footer, Sider, Content } = Layout;
const { Sider, Content } = Layout;
return loading ? null : needsLogin() ? (
<Routes>
@@ -67,11 +71,11 @@ export default function FredyApp() {
</Routes>
) : (
<Layout className="app">
<Layout className="app">
<Sider>
<Navigation isAdmin={isAdmin()} />
</Sider>
<Content>
<Sider>
<Navigation isAdmin={isAdmin()} />
</Sider>
<Layout className="app__main">
<Content className="app__content">
{versionUpdate?.newVersion && <VersionBanner />}
{settings.demoMode && (
<>
@@ -86,60 +90,58 @@ export default function FredyApp() {
</>
)}
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
<Divider />
<div className="app__content">
<Routes>
<Route path="/403" element={<InsufficientPermission />} />
<Route path="/jobs/new" element={<JobMutation />} />
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
<Route path="/dashboard" element={<Dashboard />} />
<Route path="/jobs" element={<Jobs />} />
<Route path="/listings" element={<Listings />} />
<Route path="/map" element={<MapView />} />
<Route path="/watchlistManagement" element={<WatchlistManagement />} />
{!settings.demoMode && <NewsModal />}
<Routes>
<Route path="/403" element={<InsufficientPermission />} />
<Route path="/jobs/new" element={<JobMutation />} />
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
<Route path="/dashboard" element={<Dashboard />} />
<Route path="/jobs" element={<Jobs />} />
<Route path="/listings" element={<Listings />} />
<Route path="/listings/listing/:listingId" element={<ListingDetail />} />
<Route path="/map" element={<MapView />} />
<Route path="/watchlistManagement" element={<WatchlistManagement />} />
{/* Permission-aware routes */}
<Route
path="/users/new"
element={
<PermissionAwareRoute currentUser={currentUser}>
<UserMutator />
</PermissionAwareRoute>
}
/>
<Route
path="/users/edit/:userId"
element={
<PermissionAwareRoute currentUser={currentUser}>
<UserMutator />
</PermissionAwareRoute>
}
/>
<Route
path="/users"
element={
<PermissionAwareRoute currentUser={currentUser}>
<Users />
</PermissionAwareRoute>
}
/>
<Route
path="/generalSettings"
element={
<PermissionAwareRoute currentUser={currentUser}>
<GeneralSettings />
</PermissionAwareRoute>
}
/>
{/* Permission-aware routes */}
<Route
path="/users/new"
element={
<PermissionAwareRoute currentUser={currentUser}>
<UserMutator />
</PermissionAwareRoute>
}
/>
<Route
path="/users/edit/:userId"
element={
<PermissionAwareRoute currentUser={currentUser}>
<UserMutator />
</PermissionAwareRoute>
}
/>
<Route
path="/users"
element={
<PermissionAwareRoute currentUser={currentUser}>
<Users />
</PermissionAwareRoute>
}
/>
<Route path="/userSettings" element={<UserSettings />} />
<Route
path="/generalSettings"
element={
<PermissionAwareRoute currentUser={currentUser}>
<GeneralSettings />
</PermissionAwareRoute>
}
/>
<Route path="/" element={<Navigate to="/dashboard" replace />} />
</Routes>
</div>
<Route path="/" element={<Navigate to="/dashboard" replace />} />
</Routes>
</Content>
</Layout>
<Footer>
<FredyFooter />
</Footer>
</Layout>
</Layout>
);
}

View File

@@ -1,45 +1,31 @@
.app {
height: 100%;
width: 100%;
height: 100vh;
width: 100vw;
&__main {
height: 100vh;
display: flex;
flex-direction: column;
overflow: hidden;
}
&__content {
margin: 1rem;
flex: 1;
overflow-y: auto;
overflow-x: hidden;
position: relative;
padding: 24px;
background-color: var(--semi-color-bg-0);
box-sizing: border-box;
display: flex;
flex-direction: column;
@media (max-width: 768px) {
padding: 12px;
}
}
}
.ui.inverted.segment {
background: #31303078 !important;
}
.ui.black.label,
.ui.black.labels .label {
background-color: #31303078 !important;
}
a:link {
color: #54a9ff;
background-color: transparent;
text-decoration: none;
}
a:visited {
color: #54a9ff;
background-color: transparent;
text-decoration: none;
}
a:hover {
color: #54a9ff;
background-color: transparent;
text-decoration: underline;
}
a:active {
color: #54a9ff;
background-color: transparent;
text-decoration: underline;
}
.semi-icon:not(.semi-tabs-bar .semi-tabs-tab .semi-icon) {
vertical-align: middle;
}

View File

@@ -3,12 +3,10 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { HashRouter } from 'react-router-dom';
import { createRoot } from 'react-dom/client';
import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
import { LocaleProvider } from '@douyinfe/semi-ui';
import en_US from '@douyinfe/semi-ui-19/lib/es/locale/source/en_US';
import { LocaleProvider } from '@douyinfe/semi-ui-19';
import App from './App';
import './Index.less';

View File

@@ -0,0 +1,11 @@
{
"key": "00e6b81777a275f5a140fc9101cb943810db6a69f6eb3927319c5aee0c876511",
"content":
[
{
"title": "Fredy goes AI",
"text": "With Fredy v20.0.0, we are introducing Fredys own MCP server. This brings a powerful new capability: you can connect your local LLM directly to Fredy and explore the data it collects in a much more flexible way.<br/><br/>The MCP server exposes Fredys tools and findings through a structured interface, allowing your LLM to query listings, inspect collected details, and analyze results programmatically. Instead of manually searching through the data, you can simply ask your model questions and let it dig into what Fredy has discovered for you.<br/><br/>In practice, this means your local LLM can interact with Fredy almost like an assistant: investigating properties, summarizing listings, filtering results, or helping you identify interesting opportunities based on the data Fredy gathered.",
"media": "news.mp4"
}
]
}

BIN
ui/src/assets/news/news.mp4 Normal file

Binary file not shown.

View File

@@ -0,0 +1,70 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { useState } from 'react';
import { Modal, Radio, RadioGroup, Typography } from '@douyinfe/semi-ui-19';
const { Text } = Typography;
const ListingDeletionModal = ({
visible,
onConfirm,
onCancel,
title = 'Delete Listings',
showOptions = true,
message = 'How would you like to delete the selected listing(s)?',
}) => {
const [deleteType, setDeleteType] = useState('soft');
const handleOk = () => {
onConfirm(!showOptions || deleteType === 'hard');
};
return (
<Modal
title={title}
visible={visible}
onOk={handleOk}
onCancel={onCancel}
okText="Confirm"
cancelText="Cancel"
style={{ maxWidth: '500px' }}
>
<div style={{ marginBottom: 16 }}>
<Text>{message}</Text>
</div>
{showOptions && (
<RadioGroup value={deleteType} onChange={(e) => setDeleteType(e.target.value)} style={{ width: '100%' }}>
<Radio value="soft" style={{ alignItems: 'flex-start', width: '100%' }}>
<div style={{ marginLeft: 8 }}>
<Text strong>Mark as deleted (Soft Delete)</Text>
<br />
<Text type="secondary">
Listings are kept in the database but marked as hidden. They will <b>not</b> re-appear during the next
scraping session.
</Text>
</div>
</Radio>
<Radio value="hard" style={{ marginTop: 16, alignItems: 'flex-start', width: '100%' }}>
<div style={{ marginLeft: 8 }}>
<Text strong>Remove from database (Hard Delete)</Text>
<br />
<Text type="secondary">
Listings are completely removed from the database.
<br />
<Text type="warning">
Consequence: They might re-appear when scraping the next time because Fredy won't know they were
previously found.
</Text>
</Text>
</div>
</Radio>
</RadioGroup>
)}
</Modal>
);
};
export default ListingDeletionModal;

View File

@@ -1,92 +1,80 @@
@import "DashboardCardColors.less";
.color-variant(@bg, @border, @text) {
background-color: @bg;
border: 1px solid @border;
color: @text;
}
.dashboard-card {
box-sizing: border-box;
padding: .8rem;
border-radius: .5rem;
border-width: 1px;
font-weight: 600;
box-shadow: 0 6px 20px rgba(0,0,0,0.08);
/* Make all KPI boxes the same size regardless of content/font */
width: 100%;
max-width: none;
height: 10rem;
display: flex;
flex-direction: column;
height: 140px;
margin-bottom: 16px;
transition: transform 0.2s;
background-color: rgba(36, 36, 36, 0.9);
backdrop-filter: blur(8px);
border: 1px solid var(--semi-color-border);
--pulse-color: rgba(255, 255, 255, 0.1);
position: relative;
z-index: 1;
overflow: visible;
&.blue {
.color-variant(@color-blue-bg, @color-blue-border, @color-blue-text);
}
&.orange {
.color-variant(@color-orange-bg, @color-orange-border, @color-orange-text);
}
&.green {
.color-variant(@color-green-bg, @color-green-border, @color-green-text);
}
&.purple {
.color-variant(@color-purple-bg, @color-purple-border, @color-purple-text);
}
&.gray {
.color-variant(@color-gray-bg, @color-gray-border, @color-gray-text);
}
&__header {
display: flex;
align-items: center;
gap: .6rem;
/* Keep header from growing content height */
min-height: 2rem;
overflow: hidden;
&::after {
content: '';
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
border-radius: inherit;
box-shadow: 0 4px 25px -2px var(--pulse-color);
opacity: 0;
animation: pulse 5s infinite ease-in-out;
pointer-events: none;
z-index: -1;
will-change: opacity;
}
&__icon {
border-radius: .6rem;
display: grid;
place-items: center;
}
&__title {
font-weight: 600;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-size: 20px;
display: flex;
align-items: center;
justify-content: center;
}
&__content {
margin-top: .4rem;
font-size: .7rem;
flex: 1 1 auto;
display: flex;
flex-direction: column;
justify-content: center;
overflow: hidden;
width: 100%;
}
&__value {
margin: 0;
font-size: 1.5rem;
line-height: 1.1;
color: #fff;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-weight: 700;
margin-bottom: 4px;
color: var(--semi-color-text-0);
}
&__desc {
opacity: .8;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
&.blue {
--pulse-color: var(--semi-color-primary);
box-shadow: 0 4px 20px -5px var(--pulse-color);
}
&.orange {
--pulse-color: var(--semi-color-warning);
box-shadow: 0 4px 20px -5px var(--pulse-color);
}
&.green {
--pulse-color: var(--semi-color-success);
box-shadow: 0 4px 20px -5px var(--pulse-color);
}
&.purple {
--pulse-color: var(--semi-color-info);
box-shadow: 0 4px 20px -5px var(--pulse-color);
}
&.gray {
--pulse-color: rgba(255, 255, 255, 0.2);
box-shadow: 0 4px 20px -5px var(--pulse-color);
}
}
@keyframes pulse {
0%, 100% {
opacity: 0.1;
}
50% {
opacity: 0.5;
}
}

View File

@@ -3,12 +3,7 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { Card, Typography, Space } from '@douyinfe/semi-ui-19';
import './DashboardCard.less';
export default function KpiCard({
@@ -20,21 +15,28 @@ export default function KpiCard({
color = 'gray',
children,
}) {
const { Text } = Typography;
return (
<div className={`dashboard-card ${color}`}>
<div className="dashboard-card__header">
<div className="dashboard-card__icon">{icon}</div>
<div className="dashboard-card__title">
<span>{title}</span>
<Card className={`dashboard-card ${color}`} bodyStyle={{ padding: '16px' }}>
<Space vertical align="start" spacing="tight" style={{ width: '100%' }}>
<Space>
<div className="dashboard-card__icon">{icon}</div>
<Text strong className="dashboard-card__title">
{title}
</Text>
</Space>
<div className="dashboard-card__content">
<div className="dashboard-card__value" style={{ fontSize: valueFontSize }}>
{value}
{children}
</div>
{description && (
<Text size="small" type="tertiary" className="dashboard-card__desc">
{description}
</Text>
)}
</div>
</div>
<div className="dashboard-card__content">
<p className="dashboard-card__value" style={{ fontSize: valueFontSize }}>
{value}
{children}
</p>
{description && <span className="dashboard-card__desc">{description}</span>}
</div>
</div>
</Space>
</Card>
);
}

View File

@@ -3,22 +3,25 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import './FredyFooter.less';
import { useSelector } from '../../services/state/store.js';
import { Typography } from '@douyinfe/semi-ui';
import { Typography, Layout, Space, Divider } from '@douyinfe/semi-ui-19';
export default function FredyFooter() {
const { Text } = Typography;
const { Footer } = Layout;
const version = useSelector((state) => state.versionUpdate.versionUpdate);
return (
<div className="fredyFooter">
<div className="fredyFooter__version">
<Text type="tertiary">Fredy V{version?.localFredyVersion || 'N/A'}</Text>
</div>
<div className="fredyFooter__copyRight">
<Text link={{ href: 'https://github.com/orangecoding', target: '_blank' }}>Made with </Text>
</div>
</div>
<Footer className="fredyFooter">
<Space split={<Divider layout="vertical" />}>
<Text type="tertiary" size="small">
Fredy V{version?.localFredyVersion || 'N/A'}
</Text>
<Text size="small" link={{ href: 'https://github.com/orangecoding', target: '_blank' }}>
Made with
</Text>
</Space>
</Footer>
);
}

View File

@@ -1,20 +1,12 @@
.fredyFooter {
background:rgb(53, 54, 60);
color: white;
background-color: var(--semi-color-bg-1);
display: flex;
justify-content: space-between;
justify-content: flex-end;
align-items: center;
height: 1.7rem;
border-radius: .3rem;
border-top: 1px solid #45464b;
&__version {
padding-left: .5rem;
font-size: small;
}
&__copyRight {
padding-right: 1rem;
}
padding: 0 1rem;
height: 32px;
border-top: 1px solid var(--semi-color-border);
z-index: 1000;
position: relative;
flex-shrink: 0;
}

View File

@@ -3,7 +3,7 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React, { useState, useEffect, useMemo, useRef } from 'react';
import { useState, useEffect, useMemo, useRef } from 'react';
import {
Card,
Col,
@@ -20,12 +20,13 @@ import {
Pagination,
Toast,
Empty,
} from '@douyinfe/semi-ui';
} from '@douyinfe/semi-ui-19';
import {
IconAlertTriangle,
IconDelete,
IconDescend2,
IconEdit,
IconCopy,
IconPlayCircle,
IconBriefcase,
IconBell,
@@ -34,6 +35,7 @@ import {
IconPlusCircle,
} from '@douyinfe/semi-icons';
import { useNavigate } from 'react-router-dom';
import ListingDeletionModal from '../../ListingDeletionModal.jsx';
import { useActions, useSelector } from '../../../services/state/store.js';
import { xhrDelete, xhrPut, xhrPost } from '../../../services/xhr.js';
import debounce from 'lodash/debounce';
@@ -59,6 +61,9 @@ const JobGrid = () => {
const [activityFilter, setActivityFilter] = useState(null);
const [showFilterBar, setShowFilterBar] = useState(false);
const [deleteModalVisible, setDeleteModalVisible] = useState(false);
const [pendingDeletion, setPendingDeletion] = useState(null); // { type: 'job'|'listings', jobId }
const pendingJobIdRef = useRef(null);
const evtSourceRef = useRef(null);
@@ -124,24 +129,35 @@ const JobGrid = () => {
};
}, [handleFilterChange]);
const onJobRemoval = async (jobId) => {
try {
await xhrDelete('/api/jobs', { jobId });
Toast.success('Job successfully removed');
loadData();
actions.jobsData.getJobs(); // refresh select list too
} catch (error) {
Toast.error(error);
}
const onJobRemoval = (jobId) => {
setPendingDeletion({ type: 'job', jobId });
setDeleteModalVisible(true);
};
const onListingRemoval = async (jobId) => {
const onListingRemoval = (jobId) => {
setPendingDeletion({ type: 'listings', jobId });
setDeleteModalVisible(true);
};
const confirmDeletion = async (hardDelete) => {
const { type, jobId } = pendingDeletion;
try {
await xhrDelete('/api/listings/job', { jobId });
Toast.success('Listings successfully removed');
if (type === 'job') {
await xhrDelete('/api/jobs', { jobId });
Toast.success('Job and listings successfully removed');
} else if (type === 'listings') {
await xhrDelete('/api/listings/job', { jobId, hardDelete });
Toast.success('Listings successfully removed');
}
loadData();
if (type === 'job') {
actions.jobsData.getJobs(); // refresh select list too
}
} catch (error) {
Toast.error(error);
Toast.error(error.message || 'Error performing deletion');
} finally {
setDeleteModalVisible(false);
setPendingDeletion(null);
}
};
@@ -184,29 +200,21 @@ const JobGrid = () => {
return (
<div className="jobGrid">
<div style={{ display: 'flex', flexDirection: 'column' }}>
<Button
style={{ width: '7rem', margin: 0 }}
type="primary"
icon={<IconPlusCircle />}
className="jobs__newButton"
onClick={() => navigate('/jobs/new')}
>
<Space vertical align="start" style={{ width: '100%', marginBottom: '16px' }} spacing="medium">
<Button type="primary" icon={<IconPlusCircle />} onClick={() => navigate('/jobs/new')}>
New Job
</Button>
<div className="jobGrid__searchbar">
<div className="jobGrid__searchbar" style={{ width: '100%' }}>
<Input prefix={<IconSearch />} showClear placeholder="Search" onChange={handleFilterChange} />
<Popover content="Filter / Sort Results" style={{ color: 'white', padding: '.5rem' }}>
<Button
icon={<IconFilter />}
onClick={() => {
setShowFilterBar(!showFilterBar);
}}
/>
</Popover>
<Button
icon={<IconFilter />}
style={{ marginLeft: '8px' }}
onClick={() => {
setShowFilterBar(!showFilterBar);
}}
/>
</div>
</div>
</Space>
{showFilterBar && (
<div className="jobGrid__toolbar">
@@ -274,7 +282,6 @@ const JobGrid = () => {
<Card
className="jobGrid__card"
bodyStyle={{ padding: '16px' }}
headerLine={true}
title={
<div className="jobGrid__header">
<Title heading={5} ellipsis={{ showTooltip: true }} className="jobGrid__title">
@@ -287,7 +294,9 @@ const JobGrid = () => {
'This job has been shared with you by another user, therefor it is read-only.',
)}
>
<IconAlertTriangle style={{ color: 'rgba(var(--semi-yellow-7), 1)', marginLeft: '8px' }} />
<div>
<IconAlertTriangle style={{ color: 'rgba(var(--semi-yellow-7), 1)', marginLeft: '8px' }} />
</div>
</Popover>
)}
</div>
@@ -343,40 +352,61 @@ const JobGrid = () => {
<div className="jobGrid__actions">
<Popover content={getPopoverContent('Run Job')}>
<Button
type="primary"
theme="solid"
icon={<IconPlayCircle />}
disabled={job.isOnlyShared || job.running}
onClick={() => onJobRun(job.id)}
/>
<div>
<Button
type="primary"
style={{ background: '#21aa21b5' }}
size="small"
theme="solid"
icon={<IconPlayCircle />}
disabled={job.isOnlyShared || job.running}
onClick={() => onJobRun(job.id)}
/>
</div>
</Popover>
<Popover content={getPopoverContent('Edit a Job')}>
<Button
type="secondary"
theme="solid"
icon={<IconEdit />}
disabled={job.isOnlyShared}
onClick={() => navigate(`/jobs/edit/${job.id}`)}
/>
<div>
<Button
type="secondary"
size="small"
icon={<IconEdit />}
disabled={job.isOnlyShared}
onClick={() => navigate(`/jobs/edit/${job.id}`)}
/>
</div>
</Popover>
<Popover content={getPopoverContent('Clone Job')}>
<div>
<Button
type="tertiary"
size="small"
icon={<IconCopy />}
disabled={job.isOnlyShared}
onClick={() => navigate('/jobs/new', { state: { cloneFrom: job.id } })}
/>
</div>
</Popover>
<Popover content={getPopoverContent('Delete all found Listings of this Job')}>
<Button
type="danger"
theme="solid"
icon={<IconDescend2 />}
disabled={job.isOnlyShared}
onClick={() => onListingRemoval(job.id)}
/>
<div>
<Button
type="danger"
size="small"
icon={<IconDescend2 />}
disabled={job.isOnlyShared}
onClick={() => onListingRemoval(job.id)}
/>
</div>
</Popover>
<Popover content={getPopoverContent('Delete Job')}>
<Button
type="danger"
theme="solid"
icon={<IconDelete />}
disabled={job.isOnlyShared}
onClick={() => onJobRemoval(job.id)}
/>
<div>
<Button
type="danger"
size="small"
icon={<IconDelete />}
disabled={job.isOnlyShared}
onClick={() => onJobRemoval(job.id)}
/>
</div>
</Popover>
</div>
</div>
@@ -395,6 +425,21 @@ const JobGrid = () => {
/>
</div>
)}
<ListingDeletionModal
visible={deleteModalVisible}
title={pendingDeletion?.type === 'job' ? 'Delete Job' : 'Delete Listings'}
showOptions={pendingDeletion?.type !== 'job'}
message={
pendingDeletion?.type === 'job'
? 'Are you sure you want to delete this job? All associated listings will be removed from the database.'
: 'How would you like to delete the selected listing(s)?'
}
onConfirm={confirmDeletion}
onCancel={() => {
setDeleteModalVisible(false);
setPendingDeletion(null);
}}
/>
</div>
);
};

View File

@@ -1,11 +1,16 @@
.jobGrid {
&__card {
height: 100%;
transition: transform 0.2s;
transition: transform 0.2s, box-shadow 0.2s;
background-color: rgba(36, 36, 36, 0.9);
backdrop-filter: blur(8px);
border: 1px solid var(--semi-color-border);
box-shadow: 0 0 15px -3px rgb(78 78 78 / 50%);
&:hover {
transform: translateY(-4px);
box-shadow: var(--semi-shadow-elevated);
box-shadow: 0 0 15px -3px rgb(78 78 78 / 70%);
background-color: rgba(36, 36, 36, 1);
}
}
@@ -19,12 +24,14 @@
&__toolbar {
&__card {
border-radius: 5px;
border-radius: var(--semi-border-radius-medium);
display: flex;
flex-direction: column;
gap: .3rem;
background: #232429;
background: rgba(36, 36, 36, 0.9);
backdrop-filter: blur(8px);
padding: 0.5rem;
border: 1px solid var(--semi-color-border);
}
}

View File

@@ -3,7 +3,7 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React, { useState, useEffect, useMemo } from 'react';
import { useState, useEffect, useMemo } from 'react';
import {
Card,
Col,
@@ -19,7 +19,7 @@ import {
Select,
Popover,
Empty,
} from '@douyinfe/semi-ui';
} from '@douyinfe/semi-ui-19';
import {
IconBriefcase,
IconCart,
@@ -31,7 +31,11 @@ import {
IconStarStroked,
IconSearch,
IconFilter,
IconActivity,
IconEyeOpened,
} from '@douyinfe/semi-icons';
import { useNavigate } from 'react-router-dom';
import ListingDeletionModal from '../../ListingDeletionModal.jsx';
import no_image from '../../../assets/no_image.jpg';
import * as timeService from '../../../services/time/timeService.js';
import { xhrDelete, xhrPost } from '../../../services/xhr.js';
@@ -48,6 +52,7 @@ const ListingsGrid = () => {
const providers = useSelector((state) => state.provider);
const jobs = useSelector((state) => state.jobsData.jobs);
const actions = useActions();
const navigate = useNavigate();
const [page, setPage] = useState(1);
const pageSize = 40;
@@ -61,6 +66,9 @@ const ListingsGrid = () => {
const [providerFilter, setProviderFilter] = useState(null);
const [showFilterBar, setShowFilterBar] = useState(false);
const [deleteModalVisible, setDeleteModalVisible] = useState(false);
const [listingToDelete, setListingToDelete] = useState(null);
const loadData = () => {
actions.listingsData.getListingsData({
page,
@@ -102,17 +110,36 @@ const ListingsGrid = () => {
setPage(_page);
};
const confirmDeletion = async (hardDelete) => {
try {
await xhrDelete('/api/listings/', { ids: [listingToDelete], hardDelete });
Toast.success('Listing successfully removed');
loadData();
} catch (error) {
Toast.error(error.message || 'Error deleting listing');
} finally {
setDeleteModalVisible(false);
setListingToDelete(null);
}
};
const cap = (val) => {
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
};
return (
<div className="listingsGrid">
<div className="listingsGrid__searchbar">
<Input prefix={<IconSearch />} showClear placeholder="Search" onChange={handleFilterChange} />
<Popover content="Filter / Sort Results" style={{ color: 'white', padding: '.5rem' }}>
<Button
icon={<IconFilter />}
onClick={() => {
setShowFilterBar(!showFilterBar);
}}
/>
<div>
<Button
icon={<IconFilter />}
onClick={() => {
setShowFilterBar(!showFilterBar);
}}
/>
</div>
</Popover>
</div>
{showFilterBar && (
@@ -216,6 +243,8 @@ const ListingsGrid = () => {
<Col key={item.id} xs={24} sm={12} md={8} lg={6} xl={4} xxl={6}>
<Card
className={`listingsGrid__card ${!item.is_active ? 'listingsGrid__card--inactive' : ''}`}
style={{ cursor: 'pointer' }}
onClick={() => navigate(`/listings/listing/${item.id}`)}
cover={
<div style={{ position: 'relative' }}>
<div className="listingsGrid__imageContainer">
@@ -248,11 +277,9 @@ const ListingsGrid = () => {
bodyStyle={{ padding: '12px' }}
>
<div className="listingsGrid__content">
<a href={item.url} target="_blank" rel="noopener noreferrer" className="listingsGrid__titleLink">
<Text strong ellipsis={{ showTooltip: true }} className="listingsGrid__title">
{item.title}
</Text>
</a>
<Text strong ellipsis={{ showTooltip: true }} className="listingsGrid__title">
{cap(item.title)}
</Text>
<Space vertical align="start" spacing={2} style={{ width: '100%', marginTop: 8 }}>
<Text type="secondary" icon={<IconCart />} size="small">
{item.price}
@@ -272,31 +299,40 @@ const ListingsGrid = () => {
<Text type="tertiary" size="small" icon={<IconBriefcase />}>
{item.provider.charAt(0).toUpperCase() + item.provider.slice(1)}
</Text>
{item.distance_to_destination ? (
<Text type="tertiary" size="small" icon={<IconActivity />}>
{item.distance_to_destination} m to chosen address
</Text>
) : (
<Text type="tertiary" size="small" icon={<IconActivity />}>
Distance cannot be calculated, provide an address
</Text>
)}
</Space>
<Divider margin=".6rem" />
<div style={{ display: 'flex', justifyContent: 'space-between' }}>
<div className="listingsGrid__linkButton" onClick={(e) => e.stopPropagation()}>
<a href={item.link} target="_blank" rel="noopener noreferrer">
<IconLink />
</a>
</div>
<Button
title="Link to listing"
type="primary"
type="secondary"
size="small"
onClick={async () => {
window.open(item.link);
}}
icon={<IconLink />}
title="View Details"
onClick={() => navigate(`/listings/listing/${item.id}`)}
icon={<IconEyeOpened />}
/>
<Button
title="Remove"
type="danger"
size="small"
onClick={async () => {
try {
await xhrDelete('/api/listings/', { ids: [item.id] });
Toast.success('Listing(s) successfully removed');
loadData();
} catch (error) {
Toast.error(error);
}
onClick={(e) => {
e.stopPropagation();
setListingToDelete(item.id);
setDeleteModalVisible(true);
}}
icon={<IconDelete />}
/>
@@ -317,6 +353,14 @@ const ListingsGrid = () => {
/>
</div>
)}
<ListingDeletionModal
visible={deleteModalVisible}
onConfirm={confirmDeletion}
onCancel={() => {
setDeleteModalVisible(false);
setListingToDelete(null);
}}
/>
</div>
);
};

View File

@@ -33,11 +33,15 @@
&__card {
height: 100%;
transition: transform 0.2s;
transition: transform 0.2s, box-shadow 0.2s;
background-color: rgba(36, 36, 36, 0.9);
backdrop-filter: blur(8px);
border: 1px solid var(--semi-color-border);
&:hover {
transform: translateY(-4px);
box-shadow: var(--semi-shadow-elevated);
background-color: rgba(36, 36, 36, 1);
}
&--inactive {
@@ -90,17 +94,45 @@
}
&__toolbar {
&__card {
border-radius: 5px;
border-radius: var(--semi-border-radius-medium);
display: flex;
flex-direction: column;
gap: .3rem;
background: #232429;
background: rgba(36, 36, 36, 0.9);
backdrop-filter: blur(8px);
padding: 0.5rem;
border: 1px solid var(--semi-color-border);
}
}
&__setupButton {
margin-bottom: 1rem;
}
&__linkButton {
background: var(--semi-color-primary);
font-size: 14px;
line-height: 20px;
font-weight: 600;
height: 24px;
width: 24px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 3px;
a {
color: white;
display: flex;
align-items: center;
justify-content: center;
width: 100%;
height: 100%;
}
&:hover {
background: var(--semi-color-primary-hover);
}
}
}

View File

@@ -3,8 +3,7 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { Typography } from '@douyinfe/semi-ui';
import { Typography } from '@douyinfe/semi-ui-19';
export default function Headline({ text, size = 3 } = {}) {
const { Title } = Typography;

View File

@@ -3,7 +3,6 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import logo from '../../assets/logo.png';
import logoWhite from '../../assets/logo_white.png';

View File

@@ -3,8 +3,7 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { Button } from '@douyinfe/semi-ui';
import { Button } from '@douyinfe/semi-ui-19';
import { xhrPost } from '../../services/xhr';
import { IconUser } from '@douyinfe/semi-icons';

View File

@@ -0,0 +1,213 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { useEffect, useRef } from 'react';
import maplibregl from 'maplibre-gl';
import 'maplibre-gl/dist/maplibre-gl.css';
import '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw.css';
import { fixMapboxDrawCompatibility, addDrawingControl, setupAreaFilterEventListeners } from './MapDrawingExtension.js';
import './Map.less';
export const GERMANY_BOUNDS = [
[5.866, 47.27], // Southwest coordinates
[15.042, 55.059], // Northeast coordinates
];
export const STYLES = {
STANDARD: 'https://tiles.openfreemap.org/styles/bright',
SATELLITE: {
version: 8,
sources: {
'satellite-tiles': {
type: 'raster',
tiles: ['https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}'],
tileSize: 256,
attribution:
'Tiles &copy; Esri &mdash; Source: Esri, i-cubed, USDA, USGS, AEX, GeoEye, Getmapping, Aerogrid, IGN, IGP, UPR-EGP, and the GIS User Community',
},
'satellite-labels': {
type: 'raster',
tiles: [
'https://server.arcgisonline.com/ArcGIS/rest/services/Reference/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}',
],
tileSize: 256,
attribution: '© Esri',
},
},
layers: [
{
id: 'satellite-tiles',
type: 'raster',
source: 'satellite-tiles',
minzoom: 0,
maxzoom: 19,
},
{
id: 'satellite-labels',
type: 'raster',
source: 'satellite-labels',
minzoom: 0,
maxzoom: 19,
},
],
},
};
export default function Map({
style = 'STANDARD',
show3dBuildings = false,
onMapReady = null,
enableDrawing = false,
initialSpatialFilter = null,
onDrawingChange = null,
}) {
const mapContainerRef = useRef(null);
const mapRef = useRef(null);
const drawRef = useRef(null);
// Initialize map - ONLY when container changes, never reinitialize
useEffect(() => {
if (mapRef.current) return; // Map already exists, don't reinitialize
mapRef.current = new maplibregl.Map({
container: mapContainerRef.current,
style: STYLES[style],
center: [10.4515, 51.1657], // Center of Germany
zoom: 4,
maxBounds: GERMANY_BOUNDS,
antialias: true,
});
mapRef.current.addControl(
new maplibregl.NavigationControl({
showCompass: true,
visualizePitch: true,
visualizeRoll: true,
}),
'top-right',
);
mapRef.current.addControl(
new maplibregl.GeolocateControl({
positionOptions: {
enableHighAccuracy: true,
},
trackUserLocation: true,
}),
);
// Initialize drawing extension only if enabled
if (enableDrawing) {
fixMapboxDrawCompatibility();
drawRef.current = addDrawingControl(mapRef.current);
}
// Call onMapReady callback if provided
if (onMapReady) {
onMapReady(mapRef.current);
}
return () => {
if (mapRef.current) {
mapRef.current.remove();
mapRef.current = null;
}
};
}, [mapContainerRef]); // ONLY depend on mapContainerRef - nothing else!
// Load spatial filter and setup area filter event listeners
useEffect(() => {
if (!mapRef.current || !drawRef.current || !enableDrawing) return;
// Load initial spatial filter if provided
if (initialSpatialFilter) {
try {
drawRef.current.set(initialSpatialFilter);
} catch (error) {
console.error('Error loading spatial filter:', error);
}
}
// Setup drawing event listeners
const cleanup = setupAreaFilterEventListeners(mapRef.current, drawRef.current, onDrawingChange);
return cleanup;
}, [initialSpatialFilter, onDrawingChange, enableDrawing]);
// Handle style changes
useEffect(() => {
if (mapRef.current) {
mapRef.current.setStyle(STYLES[style]);
}
}, [style]);
// Handle 3D buildings layer
useEffect(() => {
if (!mapRef.current) return;
const add3dLayer = () => {
if (!mapRef.current || !mapRef.current.isStyleLoaded()) return;
if (show3dBuildings) {
if (!mapRef.current.getSource('openfreemap')) {
mapRef.current.addSource('openfreemap', {
type: 'vector',
url: 'https://tiles.openfreemap.org/planet',
});
}
if (!mapRef.current.getLayer('3d-buildings')) {
const layers = mapRef.current.getStyle().layers;
let labelLayerId;
for (let i = 0; i < layers.length; i++) {
if (layers[i].type === 'symbol' && layers[i].layout?.['text-field']) {
labelLayerId = layers[i].id;
break;
}
}
mapRef.current.addLayer(
{
id: '3d-buildings',
source: 'openfreemap',
'source-layer': 'building',
type: 'fill-extrusion',
minzoom: 15,
filter: ['!=', ['get', 'hide_3d'], true],
paint: {
'fill-extrusion-color': [
'interpolate',
['linear'],
['get', 'render_height'],
0,
'lightgray',
200,
'royalblue',
400,
'lightblue',
],
'fill-extrusion-height': ['interpolate', ['linear'], ['zoom'], 15, 0, 16, ['get', 'render_height']],
'fill-extrusion-base': ['case', ['>=', ['get', 'zoom'], 16], ['get', 'render_min_height'], 0],
'fill-extrusion-opacity': 0.6,
},
},
labelLayerId,
);
}
} else {
if (mapRef.current.getLayer('3d-buildings')) {
mapRef.current.removeLayer('3d-buildings');
}
}
};
add3dLayer();
}, [show3dBuildings, style]);
// Handle pitch for 3D
useEffect(() => {
if (!mapRef.current) return;
mapRef.current.setPitch(show3dBuildings ? 45 : 0);
}, [show3dBuildings]);
return <div ref={mapContainerRef} className="map-container" />;
}

View File

@@ -0,0 +1,45 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
.map-container {
height: 100%;
}
/* Fix Mapbox Draw cursors for MapLibre GL compatibility */
.maplibregl-map.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive {
cursor: pointer;
}
.maplibregl-map.mouse-move .maplibregl-canvas-container.maplibregl-interactive {
cursor: move;
}
.maplibregl-map.mouse-add .maplibregl-canvas-container.maplibregl-interactive {
cursor: crosshair;
}
.maplibregl-map.mouse-move.mode-direct_select .maplibregl-canvas-container.maplibregl-interactive {
cursor: grab;
cursor: -moz-grab;
cursor: -webkit-grab;
}
.maplibregl-map.mode-direct_select.feature-vertex.mouse-move .maplibregl-canvas-container.maplibregl-interactive {
cursor: move;
}
.maplibregl-map.mode-direct_select.feature-midpoint.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive {
cursor: cell;
}
.maplibregl-map.mode-direct_select.feature-feature.mouse-move .maplibregl-canvas-container.maplibregl-interactive {
cursor: move;
}
.maplibregl-map.mode-static.mouse-pointer .maplibregl-canvas-container.maplibregl-interactive {
cursor: grab;
cursor: -moz-grab;
cursor: -webkit-grab;
}

View File

@@ -0,0 +1,177 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import MapboxDraw from '@mapbox/mapbox-gl-draw';
const drawStyles = [
{
id: 'gl-draw-polygon-fill-inactive',
type: 'fill',
filter: ['all', ['==', 'active', 'false'], ['==', '$type', 'Polygon'], ['!=', 'mode', 'static']],
paint: { 'fill-color': '#3bb2d0', 'fill-outline-color': '#3bb2d0', 'fill-opacity': 0.1 },
},
{
id: 'gl-draw-polygon-fill-active',
type: 'fill',
filter: ['all', ['==', 'active', 'true'], ['==', '$type', 'Polygon']],
paint: { 'fill-color': '#fbb03b', 'fill-outline-color': '#fbb03b', 'fill-opacity': 0.1 },
},
{
id: 'gl-draw-polygon-midpoint',
type: 'circle',
filter: ['all', ['==', '$type', 'Point'], ['==', 'meta', 'midpoint']],
paint: { 'circle-radius': 3, 'circle-color': '#fbb03b' },
},
{
id: 'gl-draw-polygon-stroke-inactive',
type: 'line',
filter: ['all', ['==', 'active', 'false'], ['==', '$type', 'Polygon'], ['!=', 'mode', 'static']],
layout: { 'line-cap': 'round', 'line-join': 'round' },
paint: { 'line-color': '#3bb2d0', 'line-width': 2 },
},
{
id: 'gl-draw-polygon-stroke-active',
type: 'line',
filter: ['all', ['==', 'active', 'true'], ['==', '$type', 'Polygon']],
layout: { 'line-cap': 'round', 'line-join': 'round' },
paint: { 'line-color': '#fbb03b', 'line-dasharray': [0.2, 2], 'line-width': 2 },
},
{
id: 'gl-draw-line-inactive',
type: 'line',
filter: ['all', ['==', 'active', 'false'], ['==', '$type', 'LineString'], ['!=', 'mode', 'static']],
layout: { 'line-cap': 'round', 'line-join': 'round' },
paint: { 'line-color': '#3bb2d0', 'line-width': 2 },
},
{
id: 'gl-draw-line-active',
type: 'line',
filter: ['all', ['==', '$type', 'LineString'], ['==', 'active', 'true']],
layout: { 'line-cap': 'round', 'line-join': 'round' },
paint: { 'line-color': '#fbb03b', 'line-dasharray': [0.2, 2], 'line-width': 2 },
},
{
id: 'gl-draw-polygon-and-line-vertex-stroke-inactive',
type: 'circle',
filter: ['all', ['==', 'meta', 'vertex'], ['==', '$type', 'Point'], ['!=', 'mode', 'static']],
paint: { 'circle-radius': 5, 'circle-color': '#fff' },
},
{
id: 'gl-draw-polygon-and-line-vertex-inactive',
type: 'circle',
filter: ['all', ['==', 'meta', 'vertex'], ['==', '$type', 'Point'], ['!=', 'mode', 'static']],
paint: { 'circle-radius': 3, 'circle-color': '#fbb03b' },
},
{
id: 'gl-draw-point-point-stroke-inactive',
type: 'circle',
filter: [
'all',
['==', 'active', 'false'],
['==', '$type', 'Point'],
['==', 'meta', 'feature'],
['!=', 'mode', 'static'],
],
paint: { 'circle-radius': 5, 'circle-opacity': 1, 'circle-color': '#fff' },
},
{
id: 'gl-draw-point-inactive',
type: 'circle',
filter: [
'all',
['==', 'active', 'false'],
['==', '$type', 'Point'],
['==', 'meta', 'feature'],
['!=', 'mode', 'static'],
],
paint: { 'circle-radius': 3, 'circle-color': '#3bb2d0' },
},
{
id: 'gl-draw-point-stroke-active',
type: 'circle',
filter: ['all', ['==', '$type', 'Point'], ['==', 'active', 'true'], ['!=', 'meta', 'midpoint']],
paint: { 'circle-radius': 7, 'circle-color': '#fff' },
},
{
id: 'gl-draw-point-active',
type: 'circle',
filter: ['all', ['==', '$type', 'Point'], ['!=', 'meta', 'midpoint'], ['==', 'active', 'true']],
paint: { 'circle-radius': 5, 'circle-color': '#fbb03b' },
},
{
id: 'gl-draw-polygon-fill-static',
type: 'fill',
filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'Polygon']],
paint: { 'fill-color': '#404040', 'fill-outline-color': '#404040', 'fill-opacity': 0.1 },
},
{
id: 'gl-draw-polygon-stroke-static',
type: 'line',
filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'Polygon']],
layout: { 'line-cap': 'round', 'line-join': 'round' },
paint: { 'line-color': '#404040', 'line-width': 2 },
},
{
id: 'gl-draw-line-static',
type: 'line',
filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'LineString']],
layout: { 'line-cap': 'round', 'line-join': 'round' },
paint: { 'line-color': '#404040', 'line-width': 2 },
},
{
id: 'gl-draw-point-static',
type: 'circle',
filter: ['all', ['==', 'mode', 'static'], ['==', '$type', 'Point']],
paint: { 'circle-radius': 5, 'circle-color': '#404040' },
},
];
export function fixMapboxDrawCompatibility() {
MapboxDraw.constants.classes.CANVAS = 'maplibregl-canvas';
MapboxDraw.constants.classes.CONTROL_BASE = 'maplibregl-ctrl';
MapboxDraw.constants.classes.CONTROL_PREFIX = 'maplibregl-ctrl-';
MapboxDraw.constants.classes.CONTROL_GROUP = 'maplibregl-ctrl-group';
MapboxDraw.constants.classes.ATTRIBUTION = 'maplibregl-ctrl-attrib';
}
export function addDrawingControl(map) {
const draw = new MapboxDraw({
displayControlsDefault: false,
controls: {
polygon: true,
trash: true,
},
styles: drawStyles,
});
map.addControl(draw, 'top-left');
return draw;
}
export function setupAreaFilterEventListeners(map, draw, onDrawingChange) {
if (!map || !draw) return () => {};
const handleDrawChange = () => {
if (draw) {
const data = draw.getAll();
if (onDrawingChange) {
onDrawingChange(data);
}
}
};
map.on('draw.create', handleDrawChange);
map.on('draw.update', handleDrawChange);
map.on('draw.delete', handleDrawChange);
// Return cleanup function
return () => {
if (map) {
map.off('draw.create', handleDrawChange);
map.off('draw.update', handleDrawChange);
map.off('draw.delete', handleDrawChange);
}
};
}

Some files were not shown because too many files have changed in this diff Show More