Listing management (#223)

* upgrading dependencies, fixing image placeholder

* improving processing times label and hide when screen width is too low

* aligning run now button

* renaming settings -> general settings

* smaller security and memory improvements

* improving footer

* preparing listing management

* improve filtering for listings

* preparing new settings page

* preparing new settings page

* storing settings in db

* next release version

lib/services/crons/demoCleanup-cron.js

@@ -1,8 +1,8 @@
 import { removeJobsByUserId } from '../storage/jobStorage.js';
-import { config } from '../../utils.js';
 import { getUsers } from '../storage/userStorage.js';
 import logger from '../logger.js';
 import cron from 'node-cron';
+import { getSettings } from '../storage/settingsStorage.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
@@ -11,12 +11,13 @@ export function cleanupDemoAtMidnight() {
   cron.schedule('0 0 * * *', cleanup);
 }
 
-function cleanup() {
-  if (config.demoMode) {
+async function cleanup() {
+  const settings = await getSettings();
+  if (settings.demoMode) {
     const demoUser = getUsers(false).find((user) => user.username === 'demo');
     if (demoUser == null) {
       logger.error('Demo user not found, cannot remove Jobs');
-      return;
+      return Promise.resolve();
     }
     removeJobsByUserId(demoUser.id);
   }

lib/services/crons/tracker-cron.js

@@ -1,10 +1,12 @@
 import cron from 'node-cron';
-import { config, inDevMode } from '../../utils.js';
+import { inDevMode } from '../../utils.js';
 import { trackMainEvent } from '../tracking/Tracker.js';
+import { getSettings } from '../storage/settingsStorage.js';
 
 async function runTask() {
+  const settings = await getSettings();
   //make sure to only send tracking events if the user gave us the green light and we are not in dev mode
-  if (config.analyticsEnabled && !inDevMode()) {
+  if (settings.analyticsEnabled && !inDevMode()) {
     await trackMainEvent();
   }
 }

lib/services/storage/SqliteConnection.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import Database from 'better-sqlite3';
 import logger from '../../services/logger.js';
-import { config } from '../../utils.js';
+import { readConfigFromStorage } from '../../utils.js';
 
 /**
  * SqliteConnection
@@ -25,6 +25,15 @@ import { config } from '../../utils.js';
 class SqliteConnection {
   static #db = null;
 
+  static #sqlLiteCfg = null;
+
+  static async init() {
+    if (this.#sqlLiteCfg == null) {
+      readConfigFromStorage().then((c) => {
+        this.#sqlLiteCfg = c.sqlitepath;
+      });
+    }
+  }
   /**
    * Returns a singleton instance of better-sqlite3 Database.
    * Respects env var SQLITE_DB_PATH and defaults to db/listings.db.
@@ -32,9 +41,12 @@ class SqliteConnection {
   static getConnection() {
     if (this.#db) return this.#db;
 
+    if (this.#sqlLiteCfg == null) {
+      logger.warn('No sqlitepath configured. Using default db/listings.db');
+    }
+
     // Interpret config.sqlitepath as a directory relative to project root when it starts with '/'
-    const cfg = typeof config === 'object' && config ? config.sqlitepath : undefined;
-    const rawDir = cfg && cfg.length > 0 ? cfg : '/db';
+    const rawDir = this.#sqlLiteCfg && this.#sqlLiteCfg.length > 0 ? this.#sqlLiteCfg : '/db';
     const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
     const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
     const dbPath = path.join(absDir, 'listings.db');

lib/services/storage/migrations/sql/6.settings.js

@@ -0,0 +1,73 @@
+// Migration: Adding a settings table to store important (config) settings instead of using config file
+import fs from 'fs';
+import path from 'path';
+import { nanoid } from 'nanoid';
+import logger from '../../../logger.js';
+
+export function up(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS settings
+    (
+      id           TEXT PRIMARY KEY,
+      create_date  INTEGER NOT NULL,
+      user_id      TEXT,
+      name         TEXT    NOT NULL,
+      value        jsonb   NOT NULL
+    );
+
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_settings_name ON settings (name);
+  `);
+
+  // Helper to insert one setting row
+  const insertSetting = (name, rawValue) => {
+    try {
+      const id = nanoid();
+      const createDate = Date.now();
+      const value = JSON.stringify(rawValue);
+      db.prepare(
+        `INSERT INTO settings (id, create_date, name, value)
+         VALUES (@id, @create_date, @name, @value)`,
+      ).run({ id, create_date: createDate, name, value });
+    } catch {
+      // Ignore duplicate inserts if any (unique by name)
+    }
+  };
+
+  // Migrate currently existing config.json into settings
+  try {
+    const configPath = path.resolve(process.cwd(), 'conf', 'config.json');
+
+    // Defaults
+    const defaults = {
+      interval: '60',
+      port: 9998,
+      workingHours: { from: '', to: '' },
+      demoMode: false,
+      analyticsEnabled: true,
+    };
+
+    let config = {};
+    if (fs.existsSync(configPath)) {
+      const file = fs.readFileSync(configPath, 'utf8');
+      try {
+        config = JSON.parse(file) || {};
+      } catch (parseErr) {
+        // If parsing fails, still proceed with defaults
+        logger.error(parseErr);
+        config = {};
+      }
+    }
+
+    // Insert each known setting, using the value from config when present, otherwise default
+    insertSetting('interval', config.interval != null ? config.interval : defaults.interval);
+    insertSetting('port', config.port != null ? config.port : defaults.port);
+    insertSetting('workingHours', config.workingHours != null ? config.workingHours : defaults.workingHours);
+    insertSetting('demoMode', config.demoMode != null ? config.demoMode : defaults.demoMode);
+    insertSetting(
+      'analyticsEnabled',
+      config.analyticsEnabled != null ? config.analyticsEnabled : defaults.analyticsEnabled,
+    );
+  } catch (e) {
+    logger.error(e);
+  }
+}

lib/services/storage/settingsStorage.js

@@ -0,0 +1,87 @@
+import { nanoid } from 'nanoid';
+import SqliteConnection from './SqliteConnection.js';
+import { fromJson, readConfigFromStorage, toJson } from '../../utils.js';
+
+// In-memory cache for compiled settings config
+/** @type {Record<string, any>|null} */
+let cachedSettingsConfig = null;
+
+/**
+ * Build a config object from DB rows of settings.
+ * - Unwraps stored shape { value: any } into raw values.
+ * - Add additional config values from file config. E.g. sqlite part cannot be stored in db for obvious reasons ;)
+ * @param {{name:string, value:string|null}[]} rows
+ * @param {{name:value}} configValues
+ * @returns {Record<string, any>}
+ */
+function compileSettings(rows, configValues) {
+  const config = {};
+  for (const r of rows) {
+    const parsed = fromJson(r.value, null);
+    // unwrap { value: any } if present
+    config[r.name] = parsed && typeof parsed === 'object' && 'value' in parsed ? parsed.value : parsed;
+  }
+  return {
+    ...config,
+    ...configValues,
+  };
+}
+
+/**
+ * Force reload the settings config cache from DB and return it.
+ * @returns {Record<string, any>}
+ */
+export async function refreshSettingsCache() {
+  const rows = SqliteConnection.query(`SELECT name, value FROM settings`);
+  const configValues = await readConfigFromStorage();
+  cachedSettingsConfig = compileSettings(rows, configValues);
+  return cachedSettingsConfig;
+}
+
+/**
+ * Get the compiled settings config. Loads it once and caches the result.
+ * @returns {Record<string, any>}
+ */
+export async function getSettings() {
+  if (cachedSettingsConfig == null) {
+    return refreshSettingsCache();
+  }
+  return cachedSettingsConfig;
+}
+
+/**
+ * Upsert settings rows.
+ * - Accepts an object map of name -> value, or an entry {name, value}.
+ * - id: random string (nanoid) when inserting
+ * - create_date: epoch ms when inserting
+ * - name: unique key
+ * - value: JSON string of the raw value (no wrapper)
+ * @param {Record<string, any>|{name:string, value:any}|[string, any][]} settingsMapOrEntry
+ * @returns {void}
+ */
+// Upsert one or more settings by name. Accepts either a single pair or an object map.
+// Preferred usage: upsertSettings({ settingName: any, another: any })
+export function upsertSettings(settingsMapOrEntry, userId = null) {
+  const entries = Array.isArray(settingsMapOrEntry)
+    ? settingsMapOrEntry
+    : typeof settingsMapOrEntry === 'object' &&
+        settingsMapOrEntry != null &&
+        'name' in settingsMapOrEntry &&
+        'value' in settingsMapOrEntry
+      ? [[settingsMapOrEntry.name, settingsMapOrEntry.value]]
+      : Object.entries(settingsMapOrEntry || {});
+
+  for (const [name, rawValue] of entries) {
+    const id = nanoid();
+    const create_date = Date.now();
+    const json = toJson(rawValue);
+    SqliteConnection.execute(
+      `INSERT INTO settings (id, create_date, name, value, user_id)
+       VALUES (@id, @create_date, @name, @value, @userId)
+       ON CONFLICT(name) DO UPDATE SET value = excluded.value`,
+      { id, create_date, name, value: json, userId },
+    );
+  }
+  // keep cache in sync
+  refreshSettingsCache();
+}

lib/services/storage/userStorage.js

@@ -1,7 +1,7 @@
-import { config } from '../../utils.js';
 import * as hasher from '../security/hash.js';
 import { nanoid } from 'nanoid';
 import SqliteConnection from './SqliteConnection.js';
+import { getSettings } from './settingsStorage.js';
 
 /**
  * Get all users.
@@ -129,8 +129,9 @@ export const removeUser = (userId) => {
  * Security: The demo user's password is set to a known value ('demo') and should only be enabled in demoMode.
  * @returns {void}
  */
-export const ensureDemoUserExists = () => {
-  if (!config.demoMode) {
+export const ensureDemoUserExists = async () => {
+  const settings = await getSettings();
+  if (!settings.demoMode) {
     // Remove demo user (and cascade delete their jobs/listings)
     SqliteConnection.execute(`DELETE FROM users WHERE username = 'demo'`);
     return;

lib/services/tracking/Tracker.js

@@ -1,9 +1,10 @@
 import { getJobs } from '../storage/jobStorage.js';
 import { getUniqueId } from './uniqueId.js';
-import { config, getPackageVersion, inDevMode } from '../../utils.js';
+import { getPackageVersion, inDevMode } from '../../utils.js';
 import os from 'os';
 import fetch from 'node-fetch';
 import logger from '../logger.js';
+import { getSettings } from '../storage/settingsStorage.js';
 
 const deviceId = getUniqueId() || 'N/A';
 const version = await getPackageVersion();
@@ -11,7 +12,8 @@ const FREDY_TRACKING_URL = 'https://fredy.orange-coding.net/tracking';
 
 export const trackMainEvent = async () => {
   try {
-    if (config.analyticsEnabled && !inDevMode()) {
+    const settings = await getSettings();
+    if (settings.analyticsEnabled && !inDevMode()) {
       const activeProvider = new Set();
       const activeAdapter = new Set();
 
@@ -44,7 +46,8 @@ export const trackMainEvent = async () => {
  * Note, this will only be used when Fredy runs in demo mode
  */
 export async function trackDemoAccessed() {
-  if (config.analyticsEnabled && !inDevMode() && config.demoMode) {
+  const settings = await getSettings();
+  if (settings.analyticsEnabled && !inDevMode() && settings.demoMode) {
     try {
       await fetch(`${FREDY_TRACKING_URL}/demo/accessed`, {
         method: 'POST',
@@ -56,7 +59,8 @@ export async function trackDemoAccessed() {
   }
 }
 
-function enrichTrackingObject(trackingObject) {
+async function enrichTrackingObject(trackingObject) {
+  const settings = await getSettings();
   const operatingSystem = os.platform();
   const osVersion = os.release();
   const arch = process.arch;
@@ -65,7 +69,7 @@ function enrichTrackingObject(trackingObject) {
 
   return {
     ...trackingObject,
-    isDemo: config.demoMode,
+    isDemo: settings.demoMode,
     operatingSystem,
     osVersion,
     arch,