Compare commits

..

32 Commits

Author SHA1 Message Date
orangecoding
bc355fb5fe fixing some bugs the wife found ;) 2026-01-28 21:25:48 +01:00
orangecoding
797421f0d5 hardening demo handling 2026-01-28 16:29:59 +01:00
orangecoding
0b2b42fc75 improve geocoding 2026-01-28 15:55:23 +01:00
Christian Kellner
472169693f Improvements 01 28 (#264)
* improving footer

* improve ui

* upgrading dependencies

* adding glow to all boxes on dashboard

* introducing single listing view

* next release version

* improve screenshots and login page
2026-01-28 14:27:03 +01:00
orangecoding
3117044139 fixing immoscout scraper 2026-01-26 19:52:37 +01:00
orangecoding
7879d0e94a next release version 2026-01-26 12:35:57 +01:00
orangecoding
afd1048c9e hardening the check if a listing is active 2026-01-26 12:34:49 +01:00
orangecoding
acbaab05ed next release version 2026-01-26 12:07:43 +01:00
orangecoding
72fffc526b deleting a listing now sets it to deleted in the db, preventing it from reappearing when scraping happens 2026-01-26 12:07:21 +01:00
orangecoding
9e5989ece3 zoom into map where most markers are 2026-01-26 11:54:47 +01:00
orangecoding
afc200c9e1 improved tooltip in map, improved user-settings handling 2026-01-26 11:50:16 +01:00
orangecoding
59226491f2 improved tooltip in map, improved user-settings handling 2026-01-26 11:20:02 +01:00
orangecoding
28f7760120 adapt link to listing in grid view to behave like a real link 2026-01-26 10:43:38 +01:00
orangecoding
2465514b7a fixing immoscout translator, allowing balcony and garden for purchases 2026-01-26 10:20:21 +01:00
Christian Kellner
9dde377fe6 possibility to display distance (#262) 2026-01-25 13:52:56 +01:00
Katrin Leinweber
28a3a7f372 Use EUR-symbol to match Map.jsx (see d43c5b3) (#261)
Co-authored-by: Katrin Leinweber <katrinleinweber@noreply.github.com>
2026-01-25 12:32:11 +01:00
orangecoding
e859250545 next release version 2026-01-22 15:10:31 +00:00
Christian Kellner
4dd0370ec1 Calculating the distance (#255)
* migra for distance

* adding distance calculator

* adding ability to store home address

* improve distance calculation

* calculating distance

* show distance in grid view

* upgrading dependencies

* moving to react 19

* ability to clone a job

* fixing tests

* polishing
2026-01-22 16:09:36 +01:00
orangecoding
51b4e51f3f fixing setting kleinanzeigen listings to inactive if not available anymore 2026-01-16 11:36:51 +01:00
orangecoding
fa1899765c fixing some rendering issues in map 2026-01-16 10:46:50 +01:00
Christian Kellner
d43c5b3f97 Map View in Fredy :D (#253)
* init map view

* switching off 3d buildings when sattelite view is on

* rename menu items

* upgrading dependencies, adding provider to popups

* adding screenshot for map view

* fixing readme

* next release version
2026-01-12 15:00:36 +01:00
orangecoding
7fd8be07a2 adding wohnungsboerse provider 2026-01-09 11:37:03 +01:00
orangecoding
2926ee7e08 upgrading dependencies 2026-01-06 09:51:04 +01:00
Christian Kellner
9506d1a9db next release version 2026-01-06 08:13:39 +01:00
Christian Kellner
feaa06c132 Update LICENSE to 2026 2026-01-04 06:46:32 +01:00
Timur
ad46500d4e Fix: correct baseUrl for ohne-makler provider - Fixes #251 (#252) 2026-01-02 08:36:39 +01:00
Christian Kellner
3c209a8f97 Redesigning listing table (#248)
* redesigning listing table

* getting rid of old listing table view

* improving listing grid
2025-12-23 08:47:51 +01:00
orangecoding
398259ff20 next release version 2025-12-18 19:25:33 +01:00
orangecoding
cf030bfa39 next release version / fixing valuers not being shown when editing a notification adapter 2025-12-18 19:24:48 +01:00
orangecoding
5dc976c7e3 ability to start jobs individually 2025-12-18 19:16:28 +01:00
orangecoding
05f1bc61c9 fixing tests 2025-12-17 16:35:24 +01:00
orangecoding
6e8a35a836 adding backup/restore ability 2025-12-17 15:48:56 +01:00
201 changed files with 21042 additions and 2990 deletions

View File

@@ -34,7 +34,8 @@ WORKDIR /fredy
# Using Alpine's chromium package which is much smaller
RUN apk add --no-cache chromium curl
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
ENV NODE_ENV=production \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Install build dependencies for native modules, then remove them after yarn install

View File

@@ -210,5 +210,5 @@ different name or branding without the explicit written permission of the
original copyright holder.
Copyright (c) 2025 Christian Kellner
Copyright (c) 2026 Christian Kellner
Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause

View File

@@ -119,7 +119,7 @@ Should you use [Unraid](https://unraid.net/), you can now install Fredy from the
## 📸 Screenshots
| Fredy Main Overview | Job Configuration | Found Listings |
| Fredy Maps View | Dashboard | Found Listings |
|--------------------------------------------------|-----------------------------------------------------------------------|-----------------------------------------------------------------------------|
| ![Screenshot showing Fredy](doc/screenshot1.png) | ![Screenshot showing job configuration in Fredy](doc/screenshot3.png) | ![Screenshot showing found listings in Fredy](doc/screenshot2.png) |
@@ -206,7 +206,7 @@ flowchart TD
F2["Adapter 2"]
end
A1 --> B["FredyPipeline"]
A1 --> B["FredyPipelineExecutioner"]
A2 --> B
A3 --> B
B --> C1 & C2 & C3

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -30,12 +30,16 @@ async function getAllFiles(dir = '.') {
/* eslint-disable no-console */
async function addCopyright(files) {
const oldCopyrightRegex =
/^(\/\*\n \* Copyright \(c\) \d{4} by Christian Kellner\.\n \* Licensed under Apache-2.0 with Commons Clause and Attribution\/Naming Clause\n \*\/\n\n)+/;
for (let file of files) {
try {
let content = await fs.readFile(file, 'utf8');
if (!content.startsWith(COPYRIGHT)) {
await fs.writeFile(file, COPYRIGHT + content);
console.log(`Added copyright to ${file}`);
const strippedContent = content.replace(oldCopyrightRegex, '');
const newContent = COPYRIGHT + strippedContent;
if (content !== newContent) {
await fs.writeFile(file, newContent);
console.log(`Added/Updated copyright in ${file}`);
}
} catch (err) {
console.error(`Error processing ${file}: ${err}`);

Binary file not shown.

Before

Width:  |  Height:  |  Size: 197 KiB

After

Width:  |  Height:  |  Size: 3.7 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 512 KiB

After

Width:  |  Height:  |  Size: 4.8 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 402 KiB

After

Width:  |  Height:  |  Size: 531 KiB

View File

@@ -5,6 +5,8 @@ services:
context: .
dockerfile: Dockerfile
image: ghcr.io/orangecoding/fredy
environment:
- NODE_ENV=production
volumes:
- ./conf:/conf
- ./db:/db

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -7,7 +7,7 @@
content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"
/>
<meta name="google" content="notranslate" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<title>Fredy || Real Estate Finder</title>

View File

@@ -1,24 +1,20 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import fs from 'fs';
import path from 'path';
import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import * as jobStorage from './lib/services/storage/jobStorage.js';
import FredyPipeline from './lib/FredyPipeline.js';
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
import logger from './lib/services/logger.js';
import { bus } from './lib/services/events/event-bus.js';
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
import { initGeocodingCron } from './lib/services/crons/geocoding-cron.js';
import { getSettings } from './lib/services/storage/settingsStorage.js';
import SqliteConnection from './lib/services/storage/SqliteConnection.js';
import SqliteConnection, { computeDbPath } from './lib/services/storage/SqliteConnection.js';
import { initJobExecutionService } from './lib/services/jobs/jobExecutionService.js';
//in the config, we store the path of the sqlite file, thus we must check if it is available
const isConfigAccessible = await checkIfConfigIsAccessible();
@@ -37,12 +33,10 @@ await runMigrations();
const settings = await getSettings();
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
const rawDir = settings.sqlitepath || '/db';
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
if (!fs.existsSync(absDir)) {
fs.mkdirSync(absDir, { recursive: true });
// Ensure the sqlite directory exists before loading anything else (based on config.sqlitepath)
const { dir: sqliteDir } = await computeDbPath();
if (!fs.existsSync(sqliteDir)) {
fs.mkdirSync(sqliteDir, { recursive: true });
}
// Load provider modules once at startup
@@ -59,55 +53,16 @@ await import('./lib/api/api.js');
if (settings.demoMode) {
logger.info('Running in demo mode');
cleanupDemoAtMidnight();
}
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
ensureAdminUserExists();
ensureDemoUserExists();
await initTrackerCron();
//do not wait for this to finish, let it run in the background
initActiveCheckerCron();
initGeocodingCron();
bus.on('jobs:runAll', () => {
logger.debug('Running Fredy Job manually');
execute();
});
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
const execute = () => {
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(settings, Date.now());
if (!settings.demoMode) {
if (isDuringWorkingHoursOrNotSet) {
settings.lastRun = Date.now();
jobStorage
.getJobs()
.filter((job) => job.enabled)
.forEach((job) => {
job.provider
.filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
.forEach(async (prov) => {
try {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyPipeline(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
} catch (error) {
logger.error(error);
}
});
});
} else {
logger.debug('Working hours set. Skipping as outside of working hours.');
}
}
};
setInterval(execute, INTERVAL);
//start once at startup
execute();
// Initialize the lean Job Execution Service (schedules and bus listeners)
initJobExecutionService({ providers, settings, intervalMs: INTERVAL });

View File

@@ -1,14 +1,19 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { NoNewListingsWarning } from './errors.js';
import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
import { getJob } from './services/storage/jobStorage.js';
import * as notify from './notification/notify.js';
import Extractor from './services/extractor/extractor.js';
import urlModifier from './services/queryStringMutator.js';
import logger from './services/logger.js';
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
import { distanceMeters } from './services/listings/distanceCalculator.js';
import { getUserSettings } from './services/storage/settingsStorage.js';
import { updateListingDistance } from './services/storage/listingsStorage.js';
/**
* @typedef {Object} Listing
@@ -40,7 +45,7 @@ import logger from './services/logger.js';
* 7) Filter out entries similar to already seen ones
* 8) Dispatch notifications
*/
class FredyPipeline {
class FredyPipelineExecutioner {
/**
* Create a new runtime instance for a single provider/job execution.
*
@@ -79,12 +84,33 @@ class FredyPipeline {
.then(this._normalize.bind(this))
.then(this._filter.bind(this))
.then(this._findNew.bind(this))
.then(this._geocode.bind(this))
.then(this._save.bind(this))
.then(this._calculateDistance.bind(this))
.then(this._filterBySimilarListings.bind(this))
.then(this._notify.bind(this))
.catch(this._handleError.bind(this));
}
/**
* Geocode new listings.
*
* @param {Listing[]} newListings New listings to geocode.
* @returns {Promise<Listing[]>} Resolves with the listings (potentially with added coordinates).
*/
async _geocode(newListings) {
for (const listing of newListings) {
if (listing.address) {
const coords = await geocodeAddress(listing.address);
if (coords) {
listing.latitude = coords.lat;
listing.longitude = coords.lng;
}
}
}
return newListings;
}
/**
* Fetch listings from the provider, using the default Extractor flow unless
* a provider-specific getListings override is supplied.
@@ -180,6 +206,42 @@ class FredyPipeline {
return newListings;
}
/**
* Calculate distance for new listings.
*
* @param {Listing[]} listings
* @returns {Listing[]}
* @private
*/
_calculateDistance(listings) {
if (listings.length === 0) return [];
const job = getJob(this._jobKey);
const userId = job?.userId;
if (userId == null || typeof userId !== 'string') {
logger.debug('Skipping distance calculation: userId is missing or invalid');
return listings;
}
const userSettings = getUserSettings(userId);
const homeAddress = userSettings?.home_address;
if (!homeAddress || !homeAddress.coords) {
return listings;
}
const { lat, lng } = homeAddress.coords;
for (const listing of listings) {
if (listing.latitude != null && listing.longitude != null) {
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
updateListingDistance(listing.id, dist);
listing.distance_to_destination = dist;
}
}
return listings;
}
/**
* Remove listings that are similar to already known entries according to the similarity cache.
* Adds the remaining listings to the cache.
@@ -218,4 +280,4 @@ class FredyPipeline {
}
}
export default FredyPipeline;
export default FredyPipelineExecutioner;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -10,6 +10,7 @@ import { providerRouter } from './routes/providerRouter.js';
import { versionRouter } from './routes/versionRouter.js';
import { loginRouter } from './routes/loginRoute.js';
import { userRouter } from './routes/userRoute.js';
import { userSettingsRouter } from './routes/userSettingsRoute.js';
import { jobRouter } from './routes/jobRouter.js';
import bodyParser from 'body-parser';
import restana from 'restana';
@@ -20,8 +21,8 @@ import { demoRouter } from './routes/demoRouter.js';
import logger from '../services/logger.js';
import { listingsRouter } from './routes/listingsRouter.js';
import { getSettings } from '../services/storage/settingsStorage.js';
import { featureRouter } from './routes/featureRouter.js';
import { dashboardRouter } from './routes/dashboardRouter.js';
import { backupRouter } from './routes/backupRouter.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../ui/public'));
const PORT = (await getSettings()).port || 9998;
@@ -34,19 +35,20 @@ service.use('/api/jobs', authInterceptor());
service.use('/api/version', authInterceptor());
service.use('/api/listings', authInterceptor());
service.use('/api/dashboard', authInterceptor());
service.use('/api/features', authInterceptor());
service.use('/api/user/settings', authInterceptor());
// /admin can only be accessed when user is having admin permissions
service.use('/api/admin', adminInterceptor());
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
service.use('/api/admin/generalSettings', generalSettingsRouter);
service.use('/api/admin/backup', backupRouter);
service.use('/api/jobs/provider', providerRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/user/settings', userSettingsRouter);
service.use('/api/version', versionRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
service.use('/api/listings', listingsRouter);
service.use('/api/features', featureRouter);
service.use('/api/dashboard', dashboardRouter);
//this route is unsecured intentionally as it is being queried from the login page
service.use('/api/demo', demoRouter);

View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import {
buildBackupFileName,
createBackupZip,
precheckRestore,
restoreFromZip,
} from '../../services/storage/backupRestoreService.js';
/**
* Backup & Restore Admin Router
*
* Endpoints:
* - GET /api/admin/backup
* Returns the current database as a zip download. Content-Type: application/zip
* - POST /api/admin/backup/restore?dryRun=true
* Accepts a zip file (raw body). Returns a compatibility report, does not restore.
* - POST /api/admin/backup/restore?force=true|false
* Accepts a zip file (raw body). Restores the database; when incompatible and force=false, returns 400.
*/
const service = restana();
const backupRouter = service.newRouter();
backupRouter.get('/', async (req, res) => {
const zipBuffer = await createBackupZip();
const fileName = await buildBackupFileName();
res.setHeader('Content-Type', 'application/zip');
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
res.send(zipBuffer);
});
/**
* Read the full request body as a Buffer. Used for raw zip uploads.
* @param {import('http').IncomingMessage} req
* @returns {Promise<Buffer>}
*/
function readBody(req) {
return new Promise((resolve, reject) => {
const chunks = [];
req.on('data', (c) => chunks.push(c));
req.on('end', () => resolve(Buffer.concat(chunks)));
req.on('error', (e) => reject(e));
});
}
// Upload endpoint. Accepts raw zip (Content-Type: application/zip or application/octet-stream)
// Query parameters:
// - dryRun=true => only validate and return compatibility info
// - force=true => proceed even if incompatible
backupRouter.post('/restore', async (req, res) => {
const { dryRun = 'false', force = 'false' } = req.query || {};
const doDryRun = String(dryRun) === 'true';
const doForce = String(force) === 'true';
const body = await readBody(req);
if (doDryRun) {
res.body = await precheckRestore(body);
return res.send();
}
try {
res.body = await restoreFromZip(body, { force: doForce });
return res.send();
} catch (e) {
res.statusCode = 400;
res.body = { message: e?.message || 'Restore failed', details: e?.payload || null };
return res.send();
}
});
export { backupRouter };

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,17 +0,0 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import getFeatures from '../../features.js';
const service = restana();
const featureRouter = service.newRouter();
featureRouter.get('/', async (req, res) => {
const features = getFeatures();
res.body = Object.assign({}, { features });
res.send();
});
export { featureRouter };

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -9,10 +9,15 @@ import * as userStorage from '../../services/storage/userStorage.js';
import { isAdmin } from '../security.js';
import logger from '../../services/logger.js';
import { bus } from '../../services/events/event-bus.js';
import { isRunning as isJobRunning } from '../../services/jobs/run-state.js';
import { addClient as addSseClient, removeClient } from '../../services/sse/sse-broker.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const jobRouter = service.newRouter();
const DEMO_JOB_NAME = 'Demo-Job';
function doesJobBelongsToUser(job, req) {
const userId = req.session.currentUser;
if (userId == null) {
@@ -37,6 +42,7 @@ jobRouter.get('/', async (req, res) => {
.map((job) => {
return {
...job,
running: isJobRunning(job.id),
isOnlyShared:
!isUserAdmin &&
job.userId !== req.session.currentUser &&
@@ -47,13 +53,118 @@ jobRouter.get('/', async (req, res) => {
res.send();
});
jobRouter.post('/startAll', async (req, res) => {
bus.emit('jobs:runAll');
jobRouter.get('/data', async (req, res) => {
const { page, pageSize = 50, activityFilter, sortfield = null, sortdir = 'asc', freeTextFilter } = req.query || {};
// normalize booleans
const toBool = (v) => {
if (v === true || v === 'true' || v === 1 || v === '1') return true;
if (v === false || v === 'false' || v === 0 || v === '0') return false;
return null;
};
const normalizedActivity = toBool(activityFilter);
const queryResult = jobStorage.queryJobs({
page: page ? parseInt(page, 10) : 1,
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
freeTextFilter: freeTextFilter || null,
activityFilter: normalizedActivity,
sortField: sortfield || null,
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
userId: req.session.currentUser,
isAdmin: isAdmin(req),
});
const isUserAdmin = isAdmin(req);
// Map result to include runtime status
queryResult.result = queryResult.result.map((job) => {
return {
...job,
running: isJobRunning(job.id),
isOnlyShared:
!isUserAdmin &&
job.userId !== req.session.currentUser &&
job.shared_with_user.includes(req.session.currentUser),
};
});
res.body = queryResult;
res.send();
});
// Server-Sent Events for job status updates
jobRouter.get('/events', async (req, res) => {
const userId = req.session.currentUser;
if (userId == null) {
res.send({ message: 'Unauthorized' }, 401);
return;
}
// SSE headers
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
try {
// Initial comment to establish stream
res.write(': connected\n\n');
addSseClient(userId, res);
// Cleanup on close/aborted
const onClose = () => removeClient(userId, res);
// restana exposes original req/res; use both close and finish
req.on('close', onClose);
req.on('aborted', onClose);
res.on('close', onClose);
} catch (e) {
logger.error('Error establishing SSE connection', e);
try {
res.end();
} catch {
//noop
}
}
});
jobRouter.post('/startAll', async (req, res) => {
try {
const userId = req.session.currentUser;
// Emit only the userId; handler will decide based on admin/ownership
bus.emit('jobs:runAll', { userId });
res.send({ message: 'Run all accepted' }, 202);
} catch (err) {
logger.error('Failed to trigger startAll', err);
res.send({ message: 'Unexpected error' }, 500);
}
});
// Trigger a single job run
jobRouter.post('/:jobId/run', async (req, res) => {
const { jobId } = req.params;
try {
const job = jobStorage.getJob(jobId);
if (!job) {
res.send({ message: 'Job not found' }, 404);
return;
}
if (!doesJobBelongsToUser(job, req)) {
res.send({ message: 'You are trying to run a job that is not associated to your user' }, 403);
return;
}
if (isJobRunning(jobId)) {
res.send({ message: 'Job is already running' }, 409);
return;
}
// fire and forget; actual execution handled by index.js listener
bus.emit('jobs:runOne', { jobId });
res.send({ message: 'Job run accepted' }, 202);
} catch (error) {
logger.error(error);
res.send({ message: 'Unexpected error triggering job' }, 500);
}
});
jobRouter.post('/', async (req, res) => {
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled, shareWithUsers = [] } = req.body;
const settings = await getSettings();
try {
let jobFromDb = jobStorage.getJob(jobId);
@@ -62,6 +173,11 @@ jobRouter.post('/', async (req, res) => {
return;
}
if (settings.demoMode && jobFromDb && jobFromDb.name === DEMO_JOB_NAME) {
res.send(new Error('Sorry, but you cannot change the Status of our Demo Job ;)'));
return;
}
jobStorage.upsertJob({
userId: req.session.currentUser,
jobId,
@@ -81,8 +197,14 @@ jobRouter.post('/', async (req, res) => {
jobRouter.delete('', async (req, res) => {
const { jobId } = req.body;
const settings = await getSettings();
try {
const job = jobStorage.getJob(jobId);
if (settings.demoMode && job.name === DEMO_JOB_NAME) {
res.send(new Error('Sorry, but you cannot remove the Demo Job ;)'));
return;
}
if (!doesJobBelongsToUser(job, req)) {
res.send(new Error('You are trying to remove a job that is not associated to your user'));
} else {
@@ -97,8 +219,15 @@ jobRouter.delete('', async (req, res) => {
jobRouter.put('/:jobId/status', async (req, res) => {
const { status } = req.body;
const { jobId } = req.params;
const settings = await getSettings();
try {
const job = jobStorage.getJob(jobId);
if (settings.demoMode && job.name === DEMO_JOB_NAME) {
res.send(new Error('Sorry, but you cannot change the Status of our Demo Job ;)'));
return;
}
if (!doesJobBelongsToUser(job, req)) {
res.send(new Error('You are trying change a job that is not associated to your user'));
} else {

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -10,6 +10,7 @@ import { isAdmin as isAdminFn } from '../security.js';
import logger from '../../services/logger.js';
import { nullOrEmpty } from '../../utils.js';
import { getJobs } from '../../services/storage/jobStorage.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
@@ -28,10 +29,14 @@ listingsRouter.get('/table', async (req, res) => {
freeTextFilter,
} = req.query || {};
// normalize booleans (accept true, 'true', 1, '1')
const toBool = (v) => v === true || v === 'true' || v === 1 || v === '1';
const normalizedActivity = toBool(activityFilter) ? true : null;
const normalizedWatch = toBool(watchListFilter) ? true : null;
// normalize booleans (accept true, 'true', 1, '1' for true; false, 'false', 0, '0' for false)
const toBool = (v) => {
if (v === true || v === 'true' || v === 1 || v === '1') return true;
if (v === false || v === 'false' || v === 0 || v === '0') return false;
return null;
};
const normalizedActivity = toBool(activityFilter);
const normalizedWatch = toBool(watchListFilter);
let jobFilter = null;
let jobIdFilter = null;
@@ -59,6 +64,29 @@ listingsRouter.get('/table', async (req, res) => {
res.send();
});
listingsRouter.get('/map', async (req, res) => {
const { jobId } = req.query || {};
res.body = listingStorage.getListingsForMap({
jobId: nullOrEmpty(jobId) ? null : jobId,
userId: req.session.currentUser,
isAdmin: isAdminFn(req),
});
res.send();
});
listingsRouter.get('/:listingId', async (req, res) => {
const { listingId } = req.params;
const listing = listingStorage.getListingById(listingId, req.session.currentUser, isAdminFn(req));
if (!listing) {
res.statusCode = 404;
res.body = { message: 'Listing not found' };
return res.send();
}
res.body = listing;
res.send();
});
// Toggle watch state for the current user on a listing
listingsRouter.post('/watch', async (req, res) => {
try {
@@ -80,7 +108,13 @@ listingsRouter.post('/watch', async (req, res) => {
listingsRouter.delete('/job', async (req, res) => {
const { jobId } = req.body;
const settings = await getSettings();
try {
if (settings.demoMode) {
res.send(new Error('Sorry, but you cannot remove listings in demo mode ;)'));
return;
}
listingStorage.deleteListingsByJobId(jobId);
} catch (error) {
res.send(new Error(error));

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -0,0 +1,79 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import SqliteConnection from '../../services/storage/SqliteConnection.js';
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
import { resetGeocoordinatesAndDistanceForUser } from '../../services/storage/listingsStorage.js';
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
import { fromJson } from '../../utils.js';
import { trackFeature } from '../../services/tracking/Tracker.js';
import { FEATURES } from '../../features.js';
import logger from '../../services/logger.js';
import { runGeoCordTask } from '../../services/crons/geocoding-cron.js';
const service = restana();
const userSettingsRouter = service.newRouter();
userSettingsRouter.get('/', async (req, res) => {
const userId = req.session.currentUser;
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
const settings = {};
for (const r of rows) {
settings[r.name] = fromJson(r.value, null);
}
res.body = settings;
res.send();
});
userSettingsRouter.get('/autocomplete', async (req, res) => {
const { q } = req.query;
try {
const results = await autocompleteAddress(q);
res.body = results;
res.send();
} catch (error) {
res.statusCode = 500;
res.send({ error: error.message });
}
});
userSettingsRouter.post('/home-address', async (req, res) => {
const userId = req.session.currentUser;
const { home_address } = req.body;
const settings = await getSettings();
if (settings.demoMode) {
res.send(new Error('In demo mode, it is not allowed to change the home address.'));
return;
}
try {
if (home_address) {
await trackFeature(FEATURES.DISTANCE_ADDRESS_ENTERED);
const coords = await geocodeAddress(home_address);
if (coords && coords.lat !== -1) {
upsertSettings({ home_address: { address: home_address, coords } }, userId);
resetGeocoordinatesAndDistanceForUser(userId);
//we do NOT wait for this to finish, as we don't want to block the response
runGeoCordTask();
res.send({ success: true, coords });
} else {
res.statusCode = 400;
res.send({ error: 'Could not geocode address' });
}
} else {
upsertSettings({ home_address: null }, userId);
res.send({ success: true });
}
} catch (error) {
logger.error('Error updating home address settings', error);
res.statusCode = 500;
res.send({ error: error.message });
}
});
export { userSettingsRouter };

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,14 +1,8 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
const FEATURES = {
WATCHLIST_MANAGEMENT: false,
export const FEATURES = {
DISTANCE_ADDRESS_ENTERED: 'DISTANCE_ADDRESS_ENTERED',
};
export default function getFeatures() {
return {
...FEATURES,
};
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -8,7 +8,7 @@
*
* The mobile API provides the following endpoints:
* - GET /search/total?{search parameters}: Returns the total number of listings for the given query
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
* Example: `curl -H "User-Agent: ImmoScout_27.12_26.2_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
*
* - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
* data specifying additional results (advertisements) to return. The format is as follows:
@@ -20,12 +20,12 @@
* ```
* It is not necessary to provide data for the specified keys.
*
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.3_26.0_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout_27.12_26.2_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
* - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
* listing response.
*
* Example: `curl -H "User-Agent: ImmoScout_27.3_26.0_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
* Example: `curl -H "User-Agent: ImmoScout_27.12_26.2_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
*
*
* It is necessary to set the correct User Agent (see `getListings`) in the request header.
@@ -52,7 +52,7 @@ async function getListings(url) {
const response = await fetch(url, {
method: 'POST',
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
'User-Agent': 'ImmoScout_27.12_26.2_._',
'Content-Type': 'application/json',
},
body: JSON.stringify({
@@ -88,7 +88,7 @@ async function getListings(url) {
async function isListingActive(link) {
const result = await fetch(convertImmoscoutListingToMobileListing(link), {
headers: {
'User-Agent': 'ImmoScout_27.3_26.0_._',
'User-Agent': 'ImmoScout_27.12_26.2_._',
},
});

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -44,7 +44,7 @@ export const init = (sourceConfig, blacklist) => {
export const metaInformation = {
name: 'OhneMakler',
baseUrl: 'https://www.ohne-makler.net/immobilien',
baseUrl: 'https://www.ohne-makler.net',
id: 'ohneMakler',
};
export { config };

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -36,7 +36,7 @@ const config = {
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
activeTester: (url) => checkIfListingIsActive(url, 'Angebot nicht gefunden'),
};
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -0,0 +1,58 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import * as utils from '../utils.js';
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
let appliedBlackList = [];
function normalize(o) {
const id = o.link.split('/').pop();
const price = o.price;
const size = o.size;
const rooms = o.rooms;
const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim());
const address = `${part}, ${city}`;
return Object.assign(o, { id, price, size, rooms, address });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return o.id != null && o.title != null && titleNotBlacklisted && descNotBlacklisted && o.link.startsWith(o.link);
}
const config = {
url: null,
sortByDateParam: null,
waitForSelector: 'body',
crawlContainer: '.search_result_container > a',
crawlFields: {
id: '*',
title: 'h3 | trim',
price: 'dl:nth-of-type(1) dd | removeNewline | trim',
rooms: 'dl:nth-of-type(2) dd | removeNewline | trim',
size: 'dl:nth-of-type(3) dd | removeNewline | trim',
description: 'div.before\\:icon-location_marker | trim',
link: '@href',
imageUrl: 'img@src',
},
normalize: normalize,
filter: applyBlacklist,
activeTester: checkIfListingIsActive,
};
export const init = (sourceConfig, blacklistTerms) => {
config.url = sourceConfig.url;
appliedBlackList = blacklistTerms || [];
};
export const metaInformation = {
name: 'Wohnungsboerse',
baseUrl: 'https://www.wohnungsboerse.net',
id: 'wohnungsboerse',
};
export { config };

View File

@@ -1,29 +0,0 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { removeJobsByUserId } from '../storage/jobStorage.js';
import { getUsers } from '../storage/userStorage.js';
import logger from '../logger.js';
import cron from 'node-cron';
import { getSettings } from '../storage/settingsStorage.js';
/**
* if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
*/
export function cleanupDemoAtMidnight() {
cron.schedule('0 0 * * *', cleanup);
}
async function cleanup() {
const settings = await getSettings();
if (settings.demoMode) {
const demoUser = getUsers(false).find((user) => user.username === 'demo');
if (demoUser == null) {
logger.error('Demo user not found, cannot remove Jobs');
return Promise.resolve();
}
removeJobsByUserId(demoUser.id);
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import cron from 'node-cron';
import { getListingsToGeocode, updateListingGeocoordinates } from '../storage/listingsStorage.js';
import { geocodeAddress, isGeocodingPaused } from '../geocoding/geoCodingService.js';
import { getJobs } from '../storage/jobStorage.js';
import { calculateDistanceForJob } from '../geocoding/distanceService.js';
export async function runGeoCordTask() {
const listings = getListingsToGeocode();
if (listings.length > 0) {
for (const listing of listings) {
if (isGeocodingPaused()) {
break;
}
const coords = await geocodeAddress(listing.address);
if (coords) {
updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
}
}
}
//additional run
const jobs = getJobs();
for (const job of jobs) {
calculateDistanceForJob(job.id, job.userId);
}
}
export async function initGeocodingCron() {
// run directly on start
await runGeoCordTask();
// then every 6 hours
cron.schedule('0 */6 * * *', runGeoCordTask);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -104,7 +104,11 @@ export default async function execute(url, waitForSelector, options) {
result = pageSource || (await page.content());
}
} catch (error) {
logger.warn('Error executing with puppeteer executor', error);
if (error?.message?.includes('Timeout')) {
logger.debug('Error executing with puppeteer executor', error);
} else {
logger.warn('Error executing with puppeteer executor', error);
}
result = null;
} finally {
try {

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -0,0 +1,26 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { autocomplete as nominatimAutocomplete } from './client/nominatimClient.js';
import logger from '../logger.js';
/**
* Autocompletes an address using Nominatim.
*
* @param {string} query - The search query.
* @returns {Promise<string[]>} List of matching addresses.
*/
export async function autocompleteAddress(query) {
if (!query) {
return [];
}
try {
return await nominatimAutocomplete(query);
} catch (error) {
logger.error('Error during address autocomplete:', error);
return [];
}
}

View File

@@ -0,0 +1,152 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import os from 'os';
import crypto from 'crypto';
import https from 'https';
import fetch from 'node-fetch';
import pThrottle from 'p-throttle';
import logger from '../../logger.js';
const API_URL = 'https://nominatim.openstreetmap.org/search';
const agent = new https.Agent({
keepAlive: true,
keepAliveMsecs: 1000,
});
const throttle = pThrottle({
limit: 1,
interval: 1000,
});
function computeMachineId() {
const hostname = os.hostname() || 'unknown-host';
const nets = os.networkInterfaces?.() || {};
const macs = [];
for (const ifname of Object.keys(nets)) {
for (const addr of nets[ifname] || []) {
if (!addr) continue;
if (addr.internal) continue;
if (addr.mac && addr.mac !== '00:00:00:00:00:00') macs.push(addr.mac);
}
}
macs.sort();
const raw = [hostname, os.platform(), os.arch(), ...macs].join('|');
return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 20);
}
/**
* Nominatim requires a specific User-Agent.
* Since Fredy is self-hosted, we use a unique machine ID to make it specific.
*/
const userAgent = `Fredy-Self-Hosted (${computeMachineId()}; https://github.com/orangecoding/fredy)`;
let last429 = 0;
const PAUSE_DURATION = 3600000; // 1 hour
/**
* Geocodes an address using Nominatim.
*
* @param {string} address - The address to geocode.
* @returns {Promise<{lat: number, lng: number}|null>} The geocoordinates or null if error. {lat: -1, lng: -1} if not found.
*/
async function doGeocode(address) {
if (Date.now() - last429 < PAUSE_DURATION) {
return null;
}
const url = `${API_URL}?q=${encodeURIComponent(address)}&format=json&countrycodes=de`;
try {
const response = await fetch(url, {
agent,
headers: {
'User-Agent': userAgent,
},
});
if (response.status === 429) {
logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
last429 = Date.now();
return null;
}
if (!response.ok) {
logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
return null;
}
const data = await response.json();
if (Array.isArray(data) && data.length > 0) {
const result = data[0];
return {
lat: parseFloat(result.lat),
lng: parseFloat(result.lon),
};
}
return { lat: -1, lng: -1 };
} catch (error) {
logger.error('Error during Nominatim geocoding:', error);
return null;
}
}
/**
* Autocompletes an address using Nominatim.
*
* @param {string} query - The search query.
* @returns {Promise<string[]>} List of matching addresses.
*/
async function doAutocomplete(query) {
if (Date.now() - last429 < PAUSE_DURATION) {
return [];
}
const url = `${API_URL}?q=${encodeURIComponent(query)}&format=json&addressdetails=1&limit=5&countrycodes=de`;
try {
const response = await fetch(url, {
agent,
headers: {
'User-Agent': userAgent,
},
});
if (response.status === 429) {
logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
last429 = Date.now();
return [];
}
if (!response.ok) {
logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
return [];
}
const data = await response.json();
if (Array.isArray(data)) {
return data.map((item) => item.display_name);
}
return [];
} catch (error) {
logger.error('Error during Nominatim autocomplete:', error);
return [];
}
}
export const geocode = throttle(doGeocode);
export const autocomplete = throttle(doAutocomplete);
export const isPaused = () => Date.now() - last429 < PAUSE_DURATION;

View File

@@ -0,0 +1,61 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { distanceMeters } from '../listings/distanceCalculator.js';
import {
getListingsToCalculateDistance,
getListingsForUserToCalculateDistance,
updateListingDistance,
} from '../storage/listingsStorage.js';
import { getUserSettings } from '../storage/settingsStorage.js';
/**
* Calculates and updates distances for listings of a specific job.
* Only processes listings where distance_to_destination is null.
*
* @param {string} jobId
* @param {string} userId
* @returns {void}
*/
export function calculateDistanceForJob(jobId, userId) {
const userSettings = getUserSettings(userId);
const homeAddress = userSettings.home_address;
if (!homeAddress || !homeAddress.coords) {
return;
}
const listings = getListingsToCalculateDistance(jobId);
const { lat, lng } = homeAddress.coords;
for (const listing of listings) {
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
updateListingDistance(listing.id, dist);
}
}
/**
* Calculates and updates distances for all active listings of a user.
* Usually called when the user updates their home address.
*
* @param {string} userId
* @returns {void}
*/
export function calculateDistanceForUser(userId) {
const userSettings = getUserSettings(userId);
const homeAddress = userSettings.home_address;
if (!homeAddress || !homeAddress.coords) {
return;
}
const listings = getListingsForUserToCalculateDistance(userId);
const { lat, lng } = homeAddress.coords;
for (const listing of listings) {
const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
updateListingDistance(listing.id, dist);
}
}

View File

@@ -0,0 +1,43 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { getGeocoordinatesByAddress } from '../storage/listingsStorage.js';
import { geocode as nominatimGeocode, isPaused as isNominatimPaused } from './client/nominatimClient.js';
import logger from '../logger.js';
/**
* Geocodes an address using Nominatim or cached results from the database.
*
* @param {string} address - The address to geocode.
* @returns {Promise<{lat: number, lng: number}|null>} The geocoordinates or null if error. {lat: -1, lng: -1} if not found.
*/
export async function geocodeAddress(address) {
if (!address) {
return null;
}
try {
// 1. Check if we already have this address geocoded in our database
const cachedCoordinates = getGeocoordinatesByAddress(address);
if (cachedCoordinates) {
logger.debug(`Found cached geocoordinates for address: ${address}`);
return cachedCoordinates;
}
// 2. If not, use Nominatim
return await nominatimGeocode(address);
} catch (error) {
logger.error('Error during geocoding:', error);
return null;
}
}
/**
* Checks if we are currently in a rate limit pause.
* @returns {boolean}
*/
export function isGeocodingPaused() {
return isNominatimPaused();
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -103,13 +103,17 @@ const REAL_ESTATE_TYPE = {
'haus-mieten': 'houserent',
'wohnung-mieten': 'apartmentrent',
'wohnung-kaufen': 'apartmentbuy',
'wohnung-kaufen-mit-balkon': 'apartmentbuy',
'eigentumswohnung-mit-garten': 'apartmentbuy',
'haus-kaufen': 'housebuy',
};
const WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP = {
// Category "Balkon/Terrasse"
'wohnung-mit-balkon-mieten': { equipment: ['balcony'] },
'wohnung-kaufen-mit-balkon': { equipment: ['balcony'] },
'wohnung-mit-garten-mieten': { equipment: ['garden'] },
'eigentumswohnung-mit-garten': { equipment: ['garden'] },
// Category "Wohnungstyp"
'souterrainwohnung-mieten': { apartmenttypes: ['halfbasement'] },
'erdgeschosswohnung-mieten': { apartmenttypes: ['groundfloor'] },
@@ -144,7 +148,7 @@ export function convertWebToMobile(webUrl) {
const realTypeKey = segments.at(-1);
let realType = REAL_ESTATE_TYPE[realTypeKey];
let additionalParamsFromWebPath;
let additionalParamsFromWebPath = WEB_PATH_TO_APARTMENT_EQUIPMENT_MAP[realTypeKey] || null;
if (!realType) {
// Test for seo optimized apartment path (only used on the ImmoScout web app)
@@ -165,7 +169,7 @@ export function convertWebToMobile(webUrl) {
Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
);
const geocodes = `/${segments.slice(2, 5).join('/')}`;
const geocodes = `/${segments.slice(2, segments.length - 1).join('/')}`;
const isRadius = segments.includes('radius');
const mobileParams = {
searchType: isRadius ? 'radius' : 'region',

View File

@@ -0,0 +1,187 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import logger from '../logger.js';
import { bus } from '../events/event-bus.js';
import * as jobStorage from '../storage/jobStorage.js';
import * as userStorage from '../storage/userStorage.js';
import { getUser } from '../storage/userStorage.js';
import { duringWorkingHoursOrNotSet } from '../../utils.js';
import FredyPipelineExecutioner from '../../FredyPipelineExecutioner.js';
import * as similarityCache from '../similarity-check/similarityCache.js';
import { isRunning, markFinished, markRunning } from './run-state.js';
import { sendToUsers } from '../sse/sse-broker.js';
/**
* Initializes the job execution service.
* - Registers event-bus listeners for `jobs:runAll`, `jobs:runOne`, and `jobs:status`.
* - Starts the periodic scheduler (if `intervalMs` > 0) and performs an initial run respecting working hours.
* - Forwards job status updates to affected users via Server-Sent Events (SSE).
*
* This function is intentionally side-effectful and exposes no external API.
*
* @param {Object} deps - Dependencies required to initialize the service.
* @param {Array<Object>} deps.providers - Loaded provider modules. Each module must expose `metaInformation.id`, `config`, and `init(config, blacklist)`.
* @param {Object} deps.settings - Global settings object (read/write). Must include `demoMode`, `interval`, and working-hours attributes used by `duringWorkingHoursOrNotSet`.
* @param {number} deps.intervalMs - Scheduler interval in milliseconds. If not finite or <= 0, the scheduler is not started.
* @returns {void}
*/
export function initJobExecutionService({ providers, settings, intervalMs }) {
// Forward job status via SSE to relevant recipients
bus.on('jobs:status', ({ jobId, running }) => {
try {
const recipients = resolveRecipients(jobId);
if (recipients.length > 0) {
sendToUsers(recipients, 'jobStatus', { jobId, running });
}
} catch (err) {
logger.warn('Failed to forward job status', jobId, err);
}
});
// Listen for "run all" requests (admin = all, user = own)
bus.on('jobs:runAll', (payload) => {
const userId = payload?.userId ?? null;
const user = userId ? getUser(userId) : null;
const isAdmin = !!user?.isAdmin;
if (isAdmin) {
logger.debug('Running all jobs manually (admin request)');
} else if (userId) {
logger.debug(`Running all jobs manually for user ${userId}`);
} else {
logger.debug('Running all jobs manually (no user provided)');
}
runAll(false, { userId, isAdmin });
});
// Listen for single job run requests
bus.on('jobs:runOne', ({ jobId }) => {
logger.debug(`Running single job manually: ${jobId}`);
// fire and forget, do not block the bus
runSingle(jobId);
});
// Start scheduler and initial run
if (Number.isFinite(intervalMs) && intervalMs > 0) {
setInterval(() => runAll(true), intervalMs);
}
// start once at startup, respecting working hours
runAll(true);
/**
* Resolve all recipients who should receive SSE updates for a job.
* Includes job owner, users with whom the job is shared, and all admins.
*
* @param {string} jobId
* @returns {string[]} unique userIds
*/
function resolveRecipients(jobId) {
const job = jobStorage.getJob(jobId);
if (!job) return [];
const admins = (userStorage.getUsers && userStorage.getUsers(false)) || [];
const adminIds = admins.filter((u) => u.isAdmin).map((u) => u.id);
const shared = Array.isArray(job.shared_with_user) ? job.shared_with_user : [];
const recipients = [job.userId, ...shared, ...adminIds].filter(Boolean);
return Array.from(new Set(recipients));
}
/**
* Execute all enabled jobs, optionally filtering by context (admin/owner) and respecting working hours.
*
* @param {boolean} [respectWorkingHours=true] - If true, skip execution when outside configured working hours.
* @param {{userId?: string, isAdmin?: boolean}} [context] - Who requested the run; determines job filtering.
* @returns {void}
*/
function runAll(respectWorkingHours = true, context = undefined) {
if (settings.demoMode) return;
const now = Date.now();
const withinHours = duringWorkingHoursOrNotSet(settings, now);
if (respectWorkingHours && !withinHours) {
logger.debug('Working hours set. Skipping as outside of working hours.');
return;
}
settings.lastRun = now;
jobStorage
.getJobs()
.filter((job) => job.enabled)
.filter((job) => {
if (!context) return true; // startup/cron → all
if (context.isAdmin) return true; // admin → all
return context.userId ? job.userId === context.userId : false; // user → own
})
.forEach((job) => executeJob(job));
}
/**
* Execute a single job by id.
* Manual runs are allowed even if the job is disabled, but never duplicated when already running.
*
* @param {string} jobId
* @returns {Promise<void>}
*/
async function runSingle(jobId) {
if (settings.demoMode) return;
const job = jobStorage.getJob(jobId);
if (!job) return;
// allow manual run even if disabled; keep guard to avoid duplicates
await executeJob(job);
}
/**
* Executes one job across all of its configured providers.
* Emits SSE start/finish events via the bus and ensures the run-state guard is always cleared.
* Provider errors are surfaced via logging but do not abort other providers.
*
* @param {Object} job
* @param {string} job.id
* @param {Array<{id:string}>} job.provider
* @param {Array<string>} [job.blacklist]
* @param {*} job.notificationAdapter
* @returns {Promise<void>}
*/
async function executeJob(job) {
if (isRunning(job.id)) {
logger.debug(`Job ${job.id} is already running. Skipping.`);
return;
}
const acquired = markRunning(job.id);
if (!acquired) return;
// notify listeners (SSE) that the job started
try {
bus.emit('jobs:status', { jobId: job.id, running: true });
} catch (err) {
logger.warn('Failed to emit start status for job', job.id, err);
}
try {
const jobProviders = job.provider.filter(
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
);
const executions = jobProviders.map(async (prov) => {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyPipelineExecutioner(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
});
const results = await Promise.allSettled(executions);
for (const r of results) {
if (r.status === 'rejected') {
logger.error(r.reason);
}
}
} finally {
markFinished(job.id);
try {
bus.emit('jobs:status', { jobId: job.id, running: false });
} catch (err) {
logger.warn('Failed to emit finish status for job', job.id, err);
}
}
}
}

View File

@@ -0,0 +1,42 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/**
* Simple in-memory running state registry for jobs.
* Prevents concurrent execution of the same job within a single process.
* This registry is reset on process restart.
* @type {Set<string>}
*/
const running = new Set();
/**
* Check if a job is currently marked as running.
* @param {string} jobId
* @returns {boolean}
*/
export function isRunning(jobId) {
return running.has(jobId);
}
/**
* Try to mark a job as running.
* If it was already running, returns false and does not modify the set.
* @param {string} jobId
* @returns {boolean} true if the job was successfully marked as running
*/
export function markRunning(jobId) {
if (running.has(jobId)) return false;
running.add(jobId);
return true;
}
/**
* Mark a job as finished (remove from the running registry).
* @param {string} jobId
* @returns {void}
*/
export function markFinished(jobId) {
running.delete(jobId);
}

View File

@@ -0,0 +1,35 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
const R = 6371000; // Earth radius in meters
/**
* Calculate the great-circle distance between two points on Earth using the Haversine formula.
* This is to calculate the distance between the listing address & the address provided by the user. I know, it is only
* a rough estimation as this calculates the distance as a straight line, but it's more convenient than using an external
* service and still gives a good approximation for sorting purposes.
* Returns distance in meters.
*
* @param {number} lat1
* @param {number} lon1
* @param {number} lat2
* @param {number} lon2
* @returns {number}
*/
export function distanceMeters(lat1, lon1, lat2, lon2) {
const toRad = (deg) => (deg * Math.PI) / 180;
const phi1 = toRad(lat1);
const phi2 = toRad(lat2);
const dPhi = toRad(lat2 - lat1);
const dLambda = toRad(lon2 - lon1);
const a =
Math.sin(dPhi / 2) * Math.sin(dPhi / 2) +
Math.cos(phi1) * Math.cos(phi2) * Math.sin(dLambda / 2) * Math.sin(dLambda / 2);
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
return Math.round(R * c * 10) / 10;
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -8,37 +8,71 @@ import { randomBetween, sleep } from '../../utils.js';
const maxAttempts = 3;
const userAgents = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
];
/**
* Check if a listing is still active with up to 3 attempts and exponential backoff.
* Backoff waits are capped and the last wait is at most 2000 ms.
* Check if a listing is still active with up to 5 attempts and exponential backoff.
* Backoff waits are randomized and capped.
*
* Rules:
* - HTTP 200 => return 1
* - HTTP 200 => return 1 (if checkForText is provided and found, returns 0)
* - HTTP 401/403 => return -1 (most certainly detected as a bot)
* - HTTP 404 => return 0
* - Other statuses or network errors => retry until attempts are exhausted
*
* @returns {Promise<Integer>} 1 if active, o if not active and -1 if detected as bot
* @returns {Promise<Integer>} 1 if active, 0 if not active and -1 if detected as bot
*/
export default async function checkIfListingIsActive(link) {
export default async function checkIfListingIsActive(link, checkForText = null) {
await sleep(randomBetween(50, 100));
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const userAgent = userAgents[Math.floor(Math.random() * userAgents.length)];
const res = await fetch(link, {
redirect: 'manual',
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
'User-Agent': userAgent,
Accept:
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'max-age=0',
'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
Referer: 'https://www.google.com/',
},
});
if (res.status === 200) {
if (checkForText) {
const htmText = await res.text();
if (htmText.includes(checkForText)) {
return 0;
}
}
return 1;
}
if (res.status === 401) return -1;
if (res.status === 403) return -1;
if (res.status === 404) return 0;
if (res.status === 401 || res.status === 403) {
if (attempt < maxAttempts) {
await sleep(backoffDelay(attempt));
continue;
}
return -1;
}
if (res.status === 404 || res.status === 410) return 0;
// For any other status, only retry if attempts remain
if (attempt < maxAttempts) {
@@ -61,13 +95,13 @@ export default async function checkIfListingIsActive(link) {
}
/**
* Exponential backoff delay with cap.
* attempt: 1 -> 500ms, 2 -> 1000ms, 3 -> 2000ms (cap)
* Exponential backoff delay with cap and jitter.
* @param {number} attempt 1-based attempt index
* @returns {number} delay in ms
*/
function backoffDelay(attempt) {
const base = 500;
const cap = 2000;
return Math.min(base * 2 ** (attempt - 1), cap);
const delay = Math.min(base * 2 ** (attempt - 1), cap);
return delay + randomBetween(0, 1000);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -0,0 +1,108 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/**
* In-memory SSE client registry.
* Maps a userId to a Set of Node.js ServerResponse objects representing open streams.
* @type {Map<string, Set<import('http').ServerResponse>>}
*/
const clients = new Map(); // Map<userId, Set<ServerResponse>>
/**
* Write a single SSE event frame to a response.
*
* @param {import('http').ServerResponse} res - The open SSE HTTP response.
* @param {string} [event] - Optional event name (sent as `event:`). If omitted, a generic message is sent.
* @param {any} [data] - Optional payload. Objects are JSON.stringified.
* @returns {void}
*/
function writeEvent(res, event, data) {
try {
if (event) {
res.write(`event: ${event}\n`);
}
if (data !== undefined) {
const payload = typeof data === 'string' ? data : JSON.stringify(data);
res.write(`data: ${payload}\n`);
}
res.write('\n');
} catch {
// ignore write errors here; cleanup happens on close
}
}
/**
* Register a new SSE client for the given user.
*
* @param {string} userId
* @param {import('http').ServerResponse} res
* @returns {void}
*/
export function addClient(userId, res) {
let set = clients.get(userId);
if (!set) {
set = new Set();
clients.set(userId, set);
}
set.add(res);
// send a hello event
writeEvent(res, 'hello', { ok: true });
}
/**
* Unregister a specific SSE client for a user. Removes the user entry when empty.
*
* @param {string} userId
* @param {import('http').ServerResponse} res
* @returns {void}
*/
export function removeClient(userId, res) {
const set = clients.get(userId);
if (!set) return;
set.delete(res);
if (set.size === 0) clients.delete(userId);
}
/**
* Send an SSE event to all open connections of a user.
*
* @param {string} userId
* @param {string} event
* @param {any} data
* @returns {void}
*/
export function sendToUser(userId, event, data) {
const set = clients.get(userId);
if (!set) return;
for (const res of set) {
writeEvent(res, event, data);
}
}
/**
* Broadcast an SSE event to multiple users (unique by id).
*
* @param {string[]} userIds
* @param {string} event
* @param {any} data
* @returns {void}
*/
export function sendToUsers(userIds, event, data) {
const unique = Array.from(new Set(userIds));
unique.forEach((id) => sendToUser(id, event, data));
}
// Heartbeat to keep connections alive on proxies (every 25s)
setInterval(() => {
for (const set of clients.values()) {
for (const res of set) {
try {
res.write(`: ping ${Date.now()}\n\n`);
} catch {
// ignore
}
}
}
}, 25000);

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -155,3 +155,21 @@ class SqliteConnection {
}
export default SqliteConnection;
// Centralized DB path computation to avoid duplication across modules
// Returns: { dir, dbPath }
/**
* Compute the absolute SQLite database directory and file path based on configuration.
* Ensures the directory exists on disk.
* @returns {Promise<{dir:string, dbPath:string}>} Absolute directory and database file path.
*/
export async function computeDbPath() {
const cfg = await readConfigFromStorage();
const rawDir = cfg?.sqlitepath && cfg.sqlitepath.length > 0 ? cfg.sqlitepath : '/db';
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
const dbPath = path.join(absDir, 'listings.db');
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
return { dir: absDir, dbPath };
}

View File

@@ -0,0 +1,320 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import fs from 'fs';
import os from 'os';
import path from 'path';
import SqliteConnection, { computeDbPath } from './SqliteConnection.js';
import logger from '../../services/logger.js';
import { getPackageVersion } from '../../utils.js';
import { runMigrations, listMigrationFiles } from './migrations/migrate.js';
/**
* Lazily resolve and cache the AdmZip constructor via dynamic import.
* This keeps startup costs low and avoids ESM/CJS interop pitfalls.
* @returns {Promise<any>} AdmZip constructor (class)
*/
let _AdmZipSingleton = null;
async function getAdmZip() {
if (_AdmZipSingleton) return _AdmZipSingleton;
// Allow tests to provide a mock constructor without ESM loader intricacies
if (globalThis && globalThis.__TEST_ADM_ZIP__) {
_AdmZipSingleton = globalThis.__TEST_ADM_ZIP__;
return _AdmZipSingleton;
}
const mod = await import('adm-zip');
_AdmZipSingleton = (mod && mod.default) || mod;
return _AdmZipSingleton;
}
/**
* Extract numeric migration id from a migration file name like "12.add-users.js".
* @param {string} name
* @returns {number} Parsed id or 0 when not parsable
*/
function parseMigrationIdFromName(name) {
if (typeof name !== 'string') return 0;
const m = name.match(/^(\d+)\./);
return m ? parseInt(m[1], 10) : 0;
}
/**
* Read the highest migration id from available migration files.
* @returns {number} Highest migration id from files, or 0 when none.
*/
function getLatestMigrationIdFromFiles() {
try {
const files = listMigrationFiles();
const ids = files.map((f) => f.id);
return ids.length > 0 ? Math.max(...ids) : 0;
} catch (e) {
logger.warn('Failed to scan migrations directory:', e.message);
return 0;
}
}
/**
* Inspect the current database and return the highest applied migration id.
* @returns {number} Max id from schema_migrations, or 0 when table/rows are missing.
*/
function getCurrentDbMigration() {
try {
const exists = SqliteConnection.tableExists('schema_migrations');
if (!exists) return 0;
const rows = SqliteConnection.query('SELECT name FROM schema_migrations');
if (!rows || rows.length === 0) return 0;
return rows.reduce((max, r) => Math.max(max, parseMigrationIdFromName(r.name)), 0);
} catch (e) {
logger.warn('Failed to read current DB migration:', e.message);
return 0;
}
}
/**
* Create a consistent SQLite snapshot using the native backup API into a temp folder.
* @returns {Promise<{tempDir:string, backupPath:string}>}
*/
async function createTempBackupFile() {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-db-'));
const backupPath = path.join(tempDir, 'listings.db');
// Ensure connection is open and create a consistent snapshot
const db = SqliteConnection.getConnection();
await db.backup(backupPath);
return { tempDir, backupPath };
}
/**
* Build a zip buffer that contains the DB snapshot and metadata marker.
* Files:
* - listings.db
* - fredy-backup.json { formatVersion, createdAt, dbMigration, fredyVersion }
* @returns {Promise<Buffer>}
*/
async function buildBackupZipBuffer() {
const { backupPath, tempDir } = await createTempBackupFile();
try {
const AdmZip = await getAdmZip();
const zip = new AdmZip();
const meta = {
formatVersion: 1,
createdAt: new Date().toISOString(),
dbMigration: getCurrentDbMigration(),
fredyVersion: await getPackageVersion(),
};
// add files
zip.addLocalFile(backupPath, '', 'listings.db');
zip.addFile('fredy-backup.json', Buffer.from(JSON.stringify(meta, null, 2), 'utf-8'));
return zip.toBuffer();
} finally {
// cleanup temp
try {
fs.rmSync(tempDir, { recursive: true, force: true });
} catch (e) {
logger.debug('Failed to cleanup temp backup dir:', e.message);
}
}
}
/**
* Read and parse the metadata file from a backup zip buffer.
* @param {Buffer} zipBuffer
* @returns {Promise<any|null>} Parsed JSON or null when missing/invalid.
*/
async function readMetadataFromZip(zipBuffer) {
const AdmZip = await getAdmZip();
const zip = new AdmZip(zipBuffer);
const entry = zip.getEntry('fredy-backup.json');
if (!entry) return null;
try {
const txt = entry.getData().toString('utf-8');
return JSON.parse(txt);
} catch {
return null;
}
}
/**
* Check if a backup zip contains a listings.db entry.
* @param {Buffer} zipBuffer
* @returns {Promise<boolean>}
*/
async function hasListingsDbInZip(zipBuffer) {
const AdmZip = await getAdmZip();
const zip = new AdmZip(zipBuffer);
return zip.getEntry('listings.db') != null || zip.getEntries().some((e) => /listings\.db$/i.test(e.entryName));
}
/**
* Extract the listings.db from a backup zip buffer to a temp directory.
* @param {Buffer} zipBuffer
* @returns {Promise<{tempDir:string, dbPath:string}>}
*/
async function extractListingsDbToTemp(zipBuffer) {
const AdmZip = await getAdmZip();
const zip = new AdmZip(zipBuffer);
const entry = zip.getEntry('listings.db') || zip.getEntries().find((e) => /listings\.db$/i.test(e.entryName));
if (!entry) throw new Error('Backup zip does not contain listings.db');
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-restore-'));
const outPath = path.join(tempDir, 'listings.db');
fs.writeFileSync(outPath, entry.getData());
return { tempDir, dbPath: outPath };
}
/**
* Public: Create a backup zip buffer ready for download.
* @returns {Promise<Buffer>}
*/
export async function createBackupZip() {
return buildBackupZipBuffer();
}
/**
* Analyze a backup zip for compatibility with the current codebase.
* - Missing DB yields danger.
* - Newer backup migration than required yields danger.
* - Older backup yields warning but is considered compatible (auto-migrate).
* - Equal version yields info.
* @param {Buffer} zipBuffer
* @returns {Promise<{compatible:boolean,severity:'danger'|'warning'|'info',message:string,backupMigration:number|null,requiredMigration:number,fredyVersion?:string|null}>>}
*/
export async function precheckRestore(zipBuffer) {
if (!zipBuffer || zipBuffer.length === 0) {
return {
compatible: false,
severity: 'danger',
message: 'Empty upload',
backupMigration: null,
requiredMigration: getLatestMigrationIdFromFiles(),
};
}
if (!(await hasListingsDbInZip(zipBuffer))) {
return {
compatible: false,
severity: 'danger',
message: 'Zip file is missing the database file (listings.db).',
backupMigration: null,
requiredMigration: getLatestMigrationIdFromFiles(),
};
}
const meta = await readMetadataFromZip(zipBuffer);
const requiredMigration = getLatestMigrationIdFromFiles();
const backupMigration = meta?.dbMigration ?? null;
const fredyVersion = meta?.fredyVersion ?? null;
if (backupMigration == null) {
return {
compatible: false,
severity: 'danger',
message:
'Backup metadata is missing the migration marker. Cannot validate compatibility. It is NOT advised to continue!',
backupMigration,
requiredMigration,
fredyVersion,
};
}
if (backupMigration > requiredMigration) {
return {
compatible: false,
severity: 'danger',
message:
'Backup schema is newer than this Fredy version. Please upgrade Fredy to a version that supports this backup or proceed at your own risk.',
backupMigration,
requiredMigration,
fredyVersion,
};
}
if (backupMigration < requiredMigration) {
return {
compatible: true,
severity: 'warning',
message:
'Backup contains an older database schema than this Fredy version requires. We will apply automatic migrations right after the restore to upgrade the database.',
backupMigration,
requiredMigration,
fredyVersion,
};
}
return {
compatible: true,
severity: 'info',
message: 'Backup is compatible with the current Fredy version.',
backupMigration,
requiredMigration,
fredyVersion,
};
}
/**
* Perform a restore from a validated backup zip.
* - Optionally forces restore when incompatible.
* - Replaces the on-disk DB and runs migrations when needed.
* @param {Buffer} zipBuffer
* @param {{force?:boolean}} [opts]
* @returns {Promise<{restored:true,warning:string|null,details:any}>}
* @throws Error with code 'INCOMPATIBLE' when not forced and incompatible
*/
export async function restoreFromZip(zipBuffer, { force = false } = {}) {
const check = await precheckRestore(zipBuffer);
if (!check.compatible && !force) {
const err = new Error(check.message || 'Backup is incompatible');
err.code = 'INCOMPATIBLE';
err.payload = check;
throw err;
}
const { dbPath } = await computeDbPath();
const { tempDir, dbPath: tempDbPath } = await extractListingsDbToTemp(zipBuffer);
try {
// Close existing connection to allow file replacement
SqliteConnection.close();
// Backup existing DB file
try {
if (fs.existsSync(dbPath)) {
const backupName = `${dbPath}.bak-${Date.now()}`;
fs.copyFileSync(dbPath, backupName);
}
} catch (e) {
logger.warn('Failed to create on-disk backup copy of current DB:', e.message);
}
// Replace DB with the one from the zip
fs.copyFileSync(tempDbPath, dbPath);
// Re-run migrations when needed
if (check.backupMigration < check.requiredMigration) {
await runMigrations();
} else {
// Ensure we can re-open the DB
SqliteConnection.getConnection();
}
} finally {
try {
fs.rmSync(tempDir, { recursive: true, force: true });
} catch (e) {
logger.debug('Failed to cleanup temp restore dir:', e.message);
}
}
return { restored: true, warning: check.severity !== 'info' ? check.message : null, details: check };
}
/**
* Build the backup file name with current date and Fredy version.
* Pattern: YYYY-MM-DD-FredyBackup-{version}.zip
* @returns {Promise<string>}
*/
export async function buildBackupFileName() {
const dt = new Date();
const yyyy = dt.getFullYear();
const mm = String(dt.getMonth() + 1).padStart(2, '0');
const dd = String(dt.getDate()).padStart(2, '0');
const version = await getPackageVersion();
return `${yyyy}-${mm}-${dd}-FredyBackup-${version}.zip`.replaceAll(' ', '');
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -85,6 +85,7 @@ export const getJob = (jobId) => {
j.name,
j.blacklist,
j.provider,
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
FROM jobs j
@@ -98,6 +99,7 @@ export const getJob = (jobId) => {
enabled: !!row.enabled,
blacklist: fromJson(row.blacklist, []),
provider: fromJson(row.provider, []),
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
};
};
@@ -161,3 +163,109 @@ export const getJobs = () => {
notificationAdapter: fromJson(row.notificationAdapter, []),
}));
};
/**
* Query jobs with pagination, filtering and sorting.
*
* @param {Object} params
* @param {number} [params.pageSize=50]
* @param {number} [params.page=1]
* @param {string} [params.freeTextFilter]
* @param {object} [params.activityFilter]
* @param {string|null} [params.sortField=null]
* @param {('asc'|'desc')} [params.sortDir='asc']
* @param {string} [params.userId] - Current user id used to scope jobs (ignored for admins).
* @param {boolean} [params.isAdmin=false] - When true, returns all jobs.
* @returns {{ totalNumber:number, page:number, result:Object[] }}
*/
export const queryJobs = ({
pageSize = 50,
page = 1,
activityFilter,
freeTextFilter,
sortField = null,
sortDir = 'asc',
userId = null,
isAdmin = false,
} = {}) => {
// sanitize inputs
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
const offset = (safePage - 1) * safePageSize;
// build WHERE filter
const whereParts = [];
const params = { limit: safePageSize, offset };
params.userId = userId || '__NO_USER__';
if (!isAdmin) {
whereParts.push(
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
);
}
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
params.filter = `%${String(freeTextFilter).trim()}%`;
whereParts.push(`(j.name LIKE @filter)`);
}
if (activityFilter === true) {
whereParts.push('(j.enabled = 1)');
} else if (activityFilter === false) {
whereParts.push('(j.enabled = 0)');
}
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
// whitelist sortable fields
const sortable = new Set(['name', 'numberOfFoundListings', 'enabled']);
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
let orderSql = 'ORDER BY j.name IS NULL, j.name ASC';
if (safeSortField) {
if (safeSortField === 'numberOfFoundListings') {
orderSql = `ORDER BY numberOfFoundListings ${safeSortDir}`;
} else {
orderSql = `ORDER BY j.${safeSortField} ${safeSortDir}`;
}
}
// count total
const countRow = SqliteConnection.query(
`SELECT COUNT(1) as cnt
FROM jobs j
${whereSql}`,
params,
);
const totalNumber = countRow?.[0]?.cnt ?? 0;
// fetch page
const rows = SqliteConnection.query(
`SELECT j.id,
j.user_id AS userId,
j.enabled,
j.name,
j.blacklist,
j.provider,
j.shared_with_user,
j.notification_adapter AS notificationAdapter,
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
FROM jobs j
${whereSql}
${orderSql}
LIMIT @limit OFFSET @offset`,
params,
);
const result = rows.map((row) => ({
...row,
enabled: !!row.enabled,
blacklist: fromJson(row.blacklist, []),
provider: fromJson(row.provider, []),
shared_with_user: fromJson(row.shared_with_user, []),
notificationAdapter: fromJson(row.notificationAdapter, []),
}));
return { totalNumber, page: safePage, result };
};

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -48,7 +48,8 @@ export const getListingsKpisForJobIds = (jobIds = []) => {
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount,
AVG(price) AS avgPrice
FROM listings
WHERE job_id IN (${placeholders})`,
WHERE job_id IN (${placeholders})
AND manually_deleted = 0`,
jobIds,
)[0] || {};
@@ -80,6 +81,7 @@ export const getProviderDistributionForJobIds = (jobIds = []) => {
`SELECT provider, COUNT(*) AS cnt
FROM listings
WHERE job_id IN (${placeholders})
AND manually_deleted = 0
GROUP BY provider
ORDER BY cnt DESC`,
jobIds,
@@ -118,8 +120,8 @@ export const getActiveOrUnknownListings = () => {
return SqliteConnection.query(
`SELECT *
FROM listings
WHERE is_active is null
OR is_active = 1
WHERE (is_active is null OR is_active = 1)
AND manually_deleted = 0
ORDER BY provider`,
);
};
@@ -173,9 +175,9 @@ export const storeListings = (jobId, providerId, listings) => {
SqliteConnection.withTransaction((db) => {
const stmt = db.prepare(
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
link, created_at, is_active)
link, created_at, is_active, latitude, longitude)
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
@created_at, 1)
@created_at, 1, @latitude, @longitude)
ON CONFLICT(job_id, hash) DO NOTHING`,
);
@@ -193,6 +195,8 @@ export const storeListings = (jobId, providerId, listings) => {
address: removeParentheses(item.address),
link: item.link,
created_at: Date.now(),
latitude: item.latitude || null,
longitude: item.longitude || null,
};
stmt.run(params);
}
@@ -277,9 +281,11 @@ export const queryListings = ({
params.filter = `%${String(freeTextFilter).trim()}%`;
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
}
// activityFilter: when true -> only active listings (is_active = 1)
// activityFilter: when true -> only active listings (is_active = 1), false -> only inactive
if (activityFilter === true) {
whereParts.push('(is_active = 1)');
} else if (activityFilter === false) {
whereParts.push('(is_active = 0)');
}
// Prefer filtering by job id when provided (unambiguous and robust)
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
@@ -295,11 +301,16 @@ export const queryListings = ({
params.providerName = String(providerFilter).trim();
whereParts.push('(provider = @providerName)');
}
// watchListFilter: when true -> only watched listings
// watchListFilter: when true -> only watched listings, false -> only unwatched
if (watchListFilter === true) {
whereParts.push('(wl.id IS NOT NULL)');
} else if (watchListFilter === false) {
whereParts.push('(wl.id IS NULL)');
}
// Build whereSql (filtering by manually_deleted = 0)
whereParts.push('(l.manually_deleted = 0)');
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
const whereSqlWithAlias = whereSql
.replace(/\btitle\b/g, 'l.title')
@@ -364,8 +375,8 @@ export const queryListings = ({
export const deleteListingsByJobId = (jobId) => {
if (!jobId) return;
return SqliteConnection.execute(
`DELETE
FROM listings
`UPDATE listings
SET manually_deleted = 1
WHERE job_id = @jobId`,
{ jobId },
);
@@ -381,13 +392,93 @@ export const deleteListingsById = (ids) => {
if (!Array.isArray(ids) || ids.length === 0) return;
const placeholders = ids.map(() => '?').join(',');
return SqliteConnection.execute(
`DELETE
FROM listings
WHERE id IN (${placeholders})`,
`UPDATE listings
SET manually_deleted = 1
WHERE id IN (${placeholders})`,
ids,
);
};
/**
* Return all listings that are active, have an address, and do not yet have geocoordinates.
*
* @returns {Object[]} Array of listing objects {id, address}.
*/
export const getListingsToGeocode = () => {
return SqliteConnection.query(
`SELECT id, address
FROM listings
WHERE is_active = 1
AND manually_deleted = 0
AND address IS NOT NULL
AND (latitude IS NULL OR longitude IS NULL)`,
);
};
/**
* Update the geocoordinates for a listing.
*
* @param {string} id - The listing ID.
* @param {number} latitude
* @param {number} longitude
* @returns {void}
*/
export const updateListingGeocoordinates = (id, latitude, longitude) => {
SqliteConnection.execute(
`UPDATE listings
SET latitude = @latitude,
longitude = @longitude
WHERE id = @id`,
{ id, latitude, longitude },
);
};
/**
* Return listings with geocoordinates for the map view, with optional filtering.
*
* @param {Object} params
* @param {string} [params.jobId]
* @param {string} [params.userId]
* @param {boolean} [params.isAdmin=false]
* @returns {{listings: Object[], maxPrice: number}} Object containing listings and maxPrice.
*/
export const getListingsForMap = ({ jobId, userId = null, isAdmin = false } = {}) => {
const baseWhereParts = [
'l.latitude IS NOT NULL',
'l.longitude IS NOT NULL',
'l.latitude != -1',
'l.longitude != -1',
'l.is_active = 1',
'l.manually_deleted = 0',
];
const params = { userId: userId || '__NO_USER__' };
if (!isAdmin) {
baseWhereParts.push(
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
);
}
if (jobId) {
params.jobId = jobId;
baseWhereParts.push('l.job_id = @jobId');
}
const wherePartsForListings = [...baseWhereParts];
const listings = SqliteConnection.query(
`SELECT l.*, j.name AS job_name
FROM listings l
LEFT JOIN jobs j ON j.id = l.job_id
WHERE ${wherePartsForListings.join(' AND ')}`,
params,
);
return {
listings,
};
};
/**
* Return all listings with only the fields: title, address, and price.
* This is the single helper requested for simple consumers.
@@ -395,5 +486,129 @@ export const deleteListingsById = (ids) => {
* @returns {{title: string|null, address: string|null, price: number|null}[]}
*/
export const getAllEntriesFromListings = () => {
return SqliteConnection.query(`SELECT title, address, price FROM listings`);
return SqliteConnection.query(`SELECT title, address, price FROM listings WHERE manually_deleted = 0`);
};
/**
* Return geocoordinates for a given address if it has been geocoded before.
*
* @param {string} address
* @returns {{lat: number, lng: number}|null}
*/
export const getGeocoordinatesByAddress = (address) => {
const row = SqliteConnection.query(
`SELECT latitude, longitude
FROM listings
WHERE address = @address
AND manually_deleted = 0
AND latitude IS NOT NULL
AND longitude IS NOT NULL
AND latitude != -1
AND longitude != -1
LIMIT 1`,
{ address },
)[0];
return row ? { lat: row.latitude, lng: row.longitude } : null;
};
/**
* Return all active listings for a given job that have geocoordinates but no distance set.
*
* @param {string} jobId
* @returns {Object[]}
*/
export const getListingsToCalculateDistance = (jobId) => {
return SqliteConnection.query(
`SELECT id, latitude, longitude
FROM listings
WHERE job_id = @jobId
AND is_active = 1
AND manually_deleted = 0
AND latitude IS NOT NULL
AND longitude IS NOT NULL
AND distance_to_destination IS NULL`,
{ jobId },
);
};
/**
* Return all active listings for a given user (across all jobs) that have geocoordinates.
*
* @param {string} userId
* @returns {Object[]}
*/
export const getListingsForUserToCalculateDistance = (userId) => {
return SqliteConnection.query(
`SELECT l.id, l.latitude, l.longitude
FROM listings l
JOIN jobs j ON l.job_id = j.id
WHERE j.user_id = @userId
AND l.is_active = 1
AND l.manually_deleted = 0
AND l.latitude IS NOT NULL
AND l.longitude IS NOT NULL`,
{ userId },
);
};
/**
* Update the distance to destination for a listing.
*
* @param {string} id
* @param {number} distance
* @returns {void}
*/
export const updateListingDistance = (id, distance) => {
SqliteConnection.execute(
`UPDATE listings
SET distance_to_destination = @distance
WHERE id = @id`,
{ id, distance },
);
};
/**
* Return a single listing by id.
*
* @param {string} id
* @param {string} userId
* @param {boolean} isAdmin
* @returns {Object|null}
*/
export const getListingById = (id, userId = null, isAdmin = false) => {
const params = { id, userId: userId || '__NO_USER__' };
let whereScoping = '';
if (!isAdmin) {
whereScoping = `AND (j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`;
}
return (
SqliteConnection.query(
`SELECT l.*, j.name AS job_name, CASE WHEN wl.id IS NOT NULL THEN 1 ELSE 0 END AS isWatched
FROM listings l
LEFT JOIN jobs j ON j.id = l.job_id
LEFT JOIN watch_list wl ON wl.listing_id = l.id AND wl.user_id = @userId
WHERE l.id = @id AND l.manually_deleted = 0 ${whereScoping}`,
params,
)[0] || null
);
};
/**
* Resets geocoordinates and distance for all listings related to a user.
*
* @param {string} userId
* @returns {void}
*/
export const resetGeocoordinatesAndDistanceForUser = (userId) => {
SqliteConnection.execute(
`UPDATE listings
SET latitude = NULL,
longitude = NULL,
distance_to_destination = NULL
WHERE job_id IN (
SELECT id FROM jobs j
WHERE j.user_id = @userId
)`,
{ userId },
);
};

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
@@ -35,7 +35,11 @@ import SqliteConnection from '../SqliteConnection.js';
import logger from '../../logger.js';
const ROOT = path.resolve('.');
const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
/**
* Absolute path to the migrations directory (lib/services/storage/migrations/sql).
* @type {string}
*/
export const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
/**
* Ensures that the given directory exists, creating it recursively if needed.
@@ -50,7 +54,7 @@ function ensureDir(p) {
* Migration files must follow the format: <number>.<label>.js
* @returns {Array<{id:number, name:string, label:string, path:string}>}
*/
function listMigrationFiles() {
export function listMigrationFiles() {
ensureDir(MIGRATIONS_DIR);
return fs
.readdirSync(MIGRATIONS_DIR)

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -0,0 +1,16 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
export function up(db) {
// 1. Add manually_deleted column
db.exec(`ALTER TABLE listings ADD COLUMN manually_deleted INTEGER NOT NULL DEFAULT 0;`);
// 2. Remove change_set column
try {
db.exec(`ALTER TABLE listings DROP COLUMN change_set;`);
} catch {
// if column does not exists for whatever reason
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/

View File

@@ -0,0 +1,13 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
// Migration: Add geocoordinates to listings for map display
export function up(db) {
db.exec(`
ALTER TABLE listings ADD COLUMN latitude REAL;
ALTER TABLE listings ADD COLUMN longitude REAL;
`);
}

View File

@@ -0,0 +1,12 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
// Migration: Removing city field and adding distance field
export function up(db) {
db.exec(`
ALTER TABLE listings ADD COLUMN distance_to_destination INTEGER;
`);
}

Some files were not shown because too many files have changed in this diff Show More