Calculating the distance (#255)

* migra for distance

* adding distance calculator

* adding ability to store home address

* improve distance calculation

* calculating distance

* show distance in grid view

* upgrading dependencies

* moving to react 19

* ability to clone a job

* fixing tests

* polishing

lib/FredyPipelineExecutioner.js

@@ -5,11 +5,15 @@
 
 import { NoNewListingsWarning } from './errors.js';
 import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
+import { getJob } from './services/storage/jobStorage.js';
 import * as notify from './notification/notify.js';
 import Extractor from './services/extractor/extractor.js';
 import urlModifier from './services/queryStringMutator.js';
 import logger from './services/logger.js';
 import { geocodeAddress } from './services/geocoding/geoCodingService.js';
+import { distanceMeters } from './services/listings/distanceCalculator.js';
+import { getUserSettings } from './services/storage/settingsStorage.js';
+import { updateListingDistance } from './services/storage/listingsStorage.js';
 
 /**
  * @typedef {Object} Listing
@@ -82,6 +86,7 @@ class FredyPipelineExecutioner {
       .then(this._findNew.bind(this))
       .then(this._geocode.bind(this))
       .then(this._save.bind(this))
+      .then(this._calculateDistance.bind(this))
       .then(this._filterBySimilarListings.bind(this))
       .then(this._notify.bind(this))
       .catch(this._handleError.bind(this));
@@ -201,6 +206,42 @@ class FredyPipelineExecutioner {
     return newListings;
   }
 
+  /**
+   * Calculate distance for new listings.
+   *
+   * @param {Listing[]} listings
+   * @returns {Listing[]}
+   * @private
+   */
+  _calculateDistance(listings) {
+    if (listings.length === 0) return [];
+
+    const job = getJob(this._jobKey);
+    const userId = job?.userId;
+
+    if (userId == null || typeof userId !== 'string') {
+      logger.debug('Skipping distance calculation: userId is missing or invalid');
+      return listings;
+    }
+
+    const userSettings = getUserSettings(userId);
+    const homeAddress = userSettings?.home_address;
+
+    if (!homeAddress || !homeAddress.coords) {
+      return listings;
+    }
+
+    const { lat, lng } = homeAddress.coords;
+    for (const listing of listings) {
+      if (listing.latitude != null && listing.longitude != null) {
+        const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
+        updateListingDistance(listing.id, dist);
+        listing.distance_to_destination = dist;
+      }
+    }
+    return listings;
+  }
+
   /**
    * Remove listings that are similar to already known entries according to the similarity cache.
    * Adds the remaining listings to the cache.

lib/api/api.js

@@ -10,6 +10,7 @@ import { providerRouter } from './routes/providerRouter.js';
 import { versionRouter } from './routes/versionRouter.js';
 import { loginRouter } from './routes/loginRoute.js';
 import { userRouter } from './routes/userRoute.js';
+import { userSettingsRouter } from './routes/userSettingsRoute.js';
 import { jobRouter } from './routes/jobRouter.js';
 import bodyParser from 'body-parser';
 import restana from 'restana';
@@ -20,7 +21,6 @@ import { demoRouter } from './routes/demoRouter.js';
 import logger from '../services/logger.js';
 import { listingsRouter } from './routes/listingsRouter.js';
 import { getSettings } from '../services/storage/settingsStorage.js';
-import { featureRouter } from './routes/featureRouter.js';
 import { dashboardRouter } from './routes/dashboardRouter.js';
 import { backupRouter } from './routes/backupRouter.js';
 const service = restana();
@@ -36,6 +36,7 @@ service.use('/api/version', authInterceptor());
 service.use('/api/listings', authInterceptor());
 service.use('/api/dashboard', authInterceptor());
 service.use('/api/features', authInterceptor());
+service.use('/api/user/settings', authInterceptor());
 
 // /admin can only be accessed when user is having admin permissions
 service.use('/api/admin', adminInterceptor());
@@ -44,11 +45,11 @@ service.use('/api/admin/generalSettings', generalSettingsRouter);
 service.use('/api/admin/backup', backupRouter);
 service.use('/api/jobs/provider', providerRouter);
 service.use('/api/admin/users', userRouter);
+service.use('/api/user/settings', userSettingsRouter);
 service.use('/api/version', versionRouter);
 service.use('/api/jobs', jobRouter);
 service.use('/api/login', loginRouter);
 service.use('/api/listings', listingsRouter);
-service.use('/api/features', featureRouter);
 service.use('/api/dashboard', dashboardRouter);
 //this route is unsecured intentionally as it is being queried from the login page
 service.use('/api/demo', demoRouter);

lib/api/routes/featureRouter.js

@@ -1,17 +0,0 @@
-/*
- * Copyright (c) 2026 by Christian Kellner.
- * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
- */
-
-import restana from 'restana';
-import getFeatures from '../../features.js';
-const service = restana();
-const featureRouter = service.newRouter();
-
-featureRouter.get('/', async (req, res) => {
-  const features = getFeatures();
-  res.body = Object.assign({}, { features });
-  res.send();
-});
-
-export { featureRouter };

lib/api/routes/userSettingsRoute.js

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+import restana from 'restana';
+import SqliteConnection from '../../services/storage/SqliteConnection.js';
+import { upsertSettings } from '../../services/storage/settingsStorage.js';
+import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
+import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
+import { calculateDistanceForUser } from '../../services/geocoding/distanceService.js';
+import { fromJson } from '../../utils.js';
+import { trackFeature } from '../../services/tracking/Tracker.js';
+import { FEATURES } from '../../features.js';
+
+const service = restana();
+const userSettingsRouter = service.newRouter();
+
+userSettingsRouter.get('/', async (req, res) => {
+  const userId = req.session.currentUser;
+  const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
+  const settings = {};
+  for (const r of rows) {
+    settings[r.name] = fromJson(r.value, null);
+  }
+  res.body = settings;
+  res.send();
+});
+
+userSettingsRouter.get('/autocomplete', async (req, res) => {
+  const { q } = req.query;
+  try {
+    const results = await autocompleteAddress(q);
+    res.body = results;
+    res.send();
+  } catch (error) {
+    res.status(500).send({ error: error.message });
+  }
+});
+
+userSettingsRouter.post('/', async (req, res) => {
+  const userId = req.session.currentUser;
+  const { home_address } = req.body;
+
+  try {
+    if (home_address) {
+      await trackFeature(FEATURES.DISTANCE_ADDRESS_ENTERED);
+      const coords = await geocodeAddress(home_address);
+      if (coords && coords.lat !== -1) {
+        upsertSettings({ home_address: { address: home_address, coords } }, userId);
+        calculateDistanceForUser(userId);
+        res.send({ success: true, coords });
+      } else {
+        res.status(400).send({ error: 'Could not geocode address' });
+      }
+    } else {
+      // If address is empty, maybe clear it?
+      upsertSettings({ home_address: null }, userId);
+      res.send({ success: true });
+    }
+  } catch (error) {
+    res.status(500).send({ error: error.message });
+  }
+});
+
+export { userSettingsRouter };

lib/features.js

@@ -3,12 +3,6 @@
  * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
  */
 
-const FEATURES = {
-  WATCHLIST_MANAGEMENT: false,
+export const FEATURES = {
+  DISTANCE_ADDRESS_ENTERED: 'DISTANCE_ADDRESS_ENTERED',
 };
-
-export default function getFeatures() {
-  return {
-    ...FEATURES,
-  };
-}

lib/services/crons/geocoding-cron.js

@@ -6,22 +6,28 @@
 import cron from 'node-cron';
 import { getListingsToGeocode, updateListingGeocoordinates } from '../storage/listingsStorage.js';
 import { geocodeAddress, isGeocodingPaused } from '../geocoding/geoCodingService.js';
+import { getJobs } from '../storage/jobStorage.js';
+import { calculateDistanceForJob } from '../geocoding/distanceService.js';
 
 async function runTask() {
   const listings = getListingsToGeocode();
-  if (listings.length === 0) {
-    return;
+  if (listings.length > 0) {
+    for (const listing of listings) {
+      if (isGeocodingPaused()) {
+        break;
+      }
+
+      const coords = await geocodeAddress(listing.address);
+      if (coords) {
+        updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
+      }
+    }
   }
 
-  for (const listing of listings) {
-    if (isGeocodingPaused()) {
-      break;
-    }
-
-    const coords = await geocodeAddress(listing.address);
-    if (coords) {
-      updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
-    }
+  //additional run
+  const jobs = getJobs();
+  for (const job of jobs) {
+    calculateDistanceForJob(job.id, job.userId);
   }
 }
 

lib/services/geocoding/autocompleteService.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+import { autocomplete as nominatimAutocomplete } from './client/nominatimClient.js';
+import logger from '../logger.js';
+
+/**
+ * Autocompletes an address using Nominatim.
+ *
+ * @param {string} query - The search query.
+ * @returns {Promise<string[]>} List of matching addresses.
+ */
+export async function autocompleteAddress(query) {
+  if (!query) {
+    return [];
+  }
+
+  try {
+    return await nominatimAutocomplete(query);
+  } catch (error) {
+    logger.error('Error during address autocomplete:', error);
+    return [];
+  }
+}

lib/services/geocoding/client/nominatimClient.js

@@ -100,6 +100,53 @@ async function doGeocode(address) {
   }
 }
 
+/**
+ * Autocompletes an address using Nominatim.
+ *
+ * @param {string} query - The search query.
+ * @returns {Promise<string[]>} List of matching addresses.
+ */
+async function doAutocomplete(query) {
+  if (Date.now() - last429 < PAUSE_DURATION) {
+    return [];
+  }
+
+  const url = `${API_URL}?q=${encodeURIComponent(query)}&format=json&addressdetails=1&limit=5&countrycodes=de`;
+
+  try {
+    const response = await fetch(url, {
+      agent,
+      headers: {
+        'User-Agent': userAgent,
+      },
+    });
+
+    if (response.status === 429) {
+      logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
+      last429 = Date.now();
+      return [];
+    }
+
+    if (!response.ok) {
+      logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
+      return [];
+    }
+
+    const data = await response.json();
+
+    if (Array.isArray(data)) {
+      return data.map((item) => item.display_name);
+    }
+
+    return [];
+  } catch (error) {
+    logger.error('Error during Nominatim autocomplete:', error);
+    return [];
+  }
+}
+
 export const geocode = throttle(doGeocode);
 
+export const autocomplete = throttle(doAutocomplete);
+
 export const isPaused = () => Date.now() - last429 < PAUSE_DURATION;

lib/services/geocoding/distanceService.js

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+import { distanceMeters } from '../listings/distanceCalculator.js';
+import {
+  getListingsToCalculateDistance,
+  getListingsForUserToCalculateDistance,
+  updateListingDistance,
+} from '../storage/listingsStorage.js';
+import { getUserSettings } from '../storage/settingsStorage.js';
+
+/**
+ * Calculates and updates distances for listings of a specific job.
+ * Only processes listings where distance_to_destination is null.
+ *
+ * @param {string} jobId
+ * @param {string} userId
+ * @returns {void}
+ */
+export function calculateDistanceForJob(jobId, userId) {
+  const userSettings = getUserSettings(userId);
+  const homeAddress = userSettings.home_address;
+
+  if (!homeAddress || !homeAddress.coords) {
+    return;
+  }
+
+  const listings = getListingsToCalculateDistance(jobId);
+  const { lat, lng } = homeAddress.coords;
+
+  for (const listing of listings) {
+    const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
+    updateListingDistance(listing.id, dist);
+  }
+}
+
+/**
+ * Calculates and updates distances for all active listings of a user.
+ * Usually called when the user updates their home address.
+ *
+ * @param {string} userId
+ * @returns {void}
+ */
+export function calculateDistanceForUser(userId) {
+  const userSettings = getUserSettings(userId);
+  const homeAddress = userSettings.home_address;
+
+  if (!homeAddress || !homeAddress.coords) {
+    return;
+  }
+
+  const listings = getListingsForUserToCalculateDistance(userId);
+  const { lat, lng } = homeAddress.coords;
+
+  for (const listing of listings) {
+    const dist = distanceMeters(lat, lng, listing.latitude, listing.longitude);
+    updateListingDistance(listing.id, dist);
+  }
+}

lib/services/listings/distanceCalculator.js

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+const R = 6371000; // Earth radius in meters
+/**
+ * Calculate the great-circle distance between two points on Earth using the Haversine formula.
+ * This is to calculate the distance between the listing address & the address provided by the user. I know, it is only
+ * a rough estimation as this calculates the distance as a straight line, but it's more convenient than using an external
+ * service and still gives a good approximation for sorting purposes.
+ * Returns distance in meters.
+ *
+ * @param {number} lat1
+ * @param {number} lon1
+ * @param {number} lat2
+ * @param {number} lon2
+ * @returns {number}
+ */
+export function distanceMeters(lat1, lon1, lat2, lon2) {
+  const toRad = (deg) => (deg * Math.PI) / 180;
+
+  const phi1 = toRad(lat1);
+  const phi2 = toRad(lat2);
+  const dPhi = toRad(lat2 - lat1);
+  const dLambda = toRad(lon2 - lon1);
+
+  const a =
+    Math.sin(dPhi / 2) * Math.sin(dPhi / 2) +
+    Math.cos(phi1) * Math.cos(phi2) * Math.sin(dLambda / 2) * Math.sin(dLambda / 2);
+
+  const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+
+  return Math.round(R * c * 10) / 10;
+}

lib/services/storage/listingsStorage.js

@@ -502,3 +502,57 @@ export const getGeocoordinatesByAddress = (address) => {
   )[0];
   return row ? { lat: row.latitude, lng: row.longitude } : null;
 };
+
+/**
+ * Return all active listings for a given job that have geocoordinates but no distance set.
+ *
+ * @param {string} jobId
+ * @returns {Object[]}
+ */
+export const getListingsToCalculateDistance = (jobId) => {
+  return SqliteConnection.query(
+    `SELECT id, latitude, longitude
+     FROM listings
+     WHERE job_id = @jobId
+       AND is_active = 1
+       AND latitude IS NOT NULL
+       AND longitude IS NOT NULL
+       AND distance_to_destination IS NULL`,
+    { jobId },
+  );
+};
+
+/**
+ * Return all active listings for a given user (across all jobs) that have geocoordinates.
+ *
+ * @param {string} userId
+ * @returns {Object[]}
+ */
+export const getListingsForUserToCalculateDistance = (userId) => {
+  return SqliteConnection.query(
+    `SELECT l.id, l.latitude, l.longitude
+     FROM listings l
+     JOIN jobs j ON l.job_id = j.id
+     WHERE j.user_id = @userId
+       AND l.is_active = 1
+       AND l.latitude IS NOT NULL
+       AND l.longitude IS NOT NULL`,
+    { userId },
+  );
+};
+
+/**
+ * Update the distance to destination for a listing.
+ *
+ * @param {string} id
+ * @param {number} distance
+ * @returns {void}
+ */
+export const updateListingDistance = (id, distance) => {
+  SqliteConnection.execute(
+    `UPDATE listings
+     SET distance_to_destination = @distance
+     WHERE id = @id`,
+    { id, distance },
+  );
+};

lib/services/storage/migrations/sql/8.distances.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+// Migration: Removing city field and adding distance field
+
+export function up(db) {
+  db.exec(`
+    ALTER TABLE listings ADD COLUMN distance_to_destination INTEGER;
+  `);
+}

lib/services/storage/migrations/sql/9.user-settings-unique-constraint.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+export function up(db) {
+  // 1. Remove old unique index
+  db.exec(`DROP INDEX IF EXISTS idx_settings_name;`);
+
+  // 2. Add new unique index for name and user_id.
+  // Since user_id can be NULL, we need a special index or use coalesce for the index.
+  // In SQLite, multiple NULLs are allowed in a UNIQUE index, which is fine for our global settings (user_id IS NULL).
+  // But we want only one global setting for a given name.
+  // Actually, in SQLite, UNIQUE allows multiple NULL values.
+  // To have only one NULL user_id for a name, we can use a partial index or COALESCE.
+
+  db.exec(`
+    CREATE UNIQUE INDEX idx_settings_name_user_id ON settings (name, IFNULL(user_id, 'GLOBAL_SETTING'));
+  `);
+}

lib/services/storage/settingsStorage.js

@@ -37,12 +37,25 @@ function compileSettings(rows, configValues) {
  * @returns {Record<string, any>}
  */
 export async function refreshSettingsCache() {
-  const rows = SqliteConnection.query(`SELECT name, value FROM settings`);
+  const rows = SqliteConnection.query(`SELECT name, value FROM settings WHERE user_id IS NULL`);
   const configValues = await readConfigFromStorage();
   cachedSettingsConfig = compileSettings(rows, configValues);
   return cachedSettingsConfig;
 }
 
+/**
+ * Retrieves user-specific settings from the database.
+ * @param {string} userId
+ * @returns {Record<string, any>}
+ */
+export function getUserSettings(userId) {
+  if (!userId || typeof userId !== 'string') {
+    return {};
+  }
+  const userRows = SqliteConnection.query(`SELECT name, value FROM settings WHERE user_id = @userId`, { userId });
+  return compileSettings(userRows, {});
+}
+
 /**
  * Get the compiled settings config. Loads it once and caches the result.
  * @returns {Record<string, any>}
@@ -83,10 +96,12 @@ export function upsertSettings(settingsMapOrEntry, userId = null) {
     SqliteConnection.execute(
       `INSERT INTO settings (id, create_date, name, value, user_id)
        VALUES (@id, @create_date, @name, @value, @userId)
-       ON CONFLICT(name) DO UPDATE SET value = excluded.value`,
+       ON CONFLICT(name, IFNULL(user_id, 'GLOBAL_SETTING')) DO UPDATE SET value = excluded.value`,
       { id, create_date, name, value: json, userId },
     );
   }
-  // keep cache in sync
-  refreshSettingsCache();
+  // keep cache in sync (only for global settings)
+  if (userId == null) {
+    refreshSettingsCache();
+  }
 }

lib/services/tracking/Tracker.js

@@ -47,6 +47,25 @@ export const trackMainEvent = async () => {
   }
 };
 
+export const trackFeature = async (feature) => {
+  try {
+    const settings = await getSettings();
+    if (settings.analyticsEnabled && !inDevMode()) {
+      const trackingObj = await enrichTrackingObject({
+        feature,
+      });
+
+      await fetch(`${FREDY_TRACKING_URL}/feature`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(trackingObj),
+      });
+    }
+  } catch (error) {
+    logger.warn('Error tracking feature', error);
+  }
+};
+
 /**
  * Note, this will only be used when Fredy runs in demo mode
  */