next release version

README.md

@@ -1,11 +1,15 @@
 <p align="center">
-  <a href="https://fredy.orange-coding.net/">
-    <img alt="Expo logo" width="400" src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png">
-  </a>
+
+<a href="https://fredy.orange-coding.net/">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/orangecoding/fredy/blob/master/doc/logo_white.png" width="400">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/orangecoding/fredy/blob/master/doc/logo.png" width="400">
+  <img alt="Jetbrains Open Source" src="https://github.com/orangecoding/fredy/blob/master/doc/logo.png">
+</picture>
+</a>
 </p>
 
-
- ![Tests](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
+![Tests](https://github.com/orangecoding/fredy/actions/workflows/test.yml/badge.svg)
 [![Docker](https://github.com/orangecoding/fredy/actions/workflows/docker.yml/badge.svg)](https://github.com/orangecoding/fredy/actions/workflows/docker.yml)
 ![Source](https://github.com/orangecoding/fredy/actions/workflows/check_source.yml/badge.svg)
 ![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fghcr-badge.elias.eu.org%2Fapi%2Forangecoding%2Ffredy%2Ffredy&query=%24.downloadCount&label=Docker%20Pulls)
@@ -49,7 +53,12 @@ I maintain Fredy and other open-source projects in my free time.\
 If you find it useful, consider supporting the project 💙
 
 Fredy is proudly backed by the **JetBrains Open Source Support Program**.   
-[<img src="https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg" alt="JetBrains" width="120"/>](https://jb.gg/OpenSourceSupport)
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://www.jetbrains.com/company/brand/img/logo_jb_dos_3.svg">
+  <source media="(prefers-color-scheme: light)" srcset="https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg">
+  <img alt="Jetbrains Open Source" src="https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg">
+</picture>
 
 ------------------------------------------------------------------------
 

index.js

@@ -9,6 +9,7 @@ import './lib/api/api.js';
 import { handleDemoUser } from './lib/services/storage/userStorage.js';
 import { cleanupDemoAtMidnight } from './lib/services/demoCleanup.js';
 import { initTrackerCron } from './lib/services/tracking/Tracker-Cron.js';
+import logger from './lib/services/logger.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
@@ -17,13 +18,11 @@ const path = './lib/provider';
 const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
 //assuming interval is always in minutes
 const INTERVAL = config.interval * 60 * 1000;
-/* eslint-disable no-console */
-console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
+logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
 if (config.demoMode) {
-  console.info('Running in demo mode');
+  logger.info('Running in demo mode');
   cleanupDemoAtMidnight();
 }
-/* eslint-enable no-console */
 const fetchedProvider = await Promise.all(
   provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`)),
 );
@@ -51,9 +50,7 @@ setInterval(
               });
           });
       } else {
-        /* eslint-disable no-console */
-        console.debug('Working hours set. Skipping as outside of working hours.');
-        /* eslint-enable no-console */
+        logger.debug('Working hours set. Skipping as outside of working hours.');
       }
     }
     return exec;

lib/FredyRuntime.js

@@ -3,6 +3,7 @@ import { setKnownListings, getKnownListings } from './services/storage/listingsS
 import * as notify from './notification/notify.js';
 import Extractor from './services/extractor/extractor.js';
 import urlModifier from './services/queryStringMutator.js';
+import logger from './services/logger.js';
 
 class FredyRuntime {
   /**
@@ -59,7 +60,7 @@ class FredyRuntime {
         })
         .catch((err) => {
           reject(err);
-          console.error(err);
+          logger.error(err);
         });
     });
   }
@@ -104,9 +105,7 @@ class FredyRuntime {
     const filteredList = listings.filter((listing) => {
       const similar = this._similarityCache.hasSimilarEntries(listing.title, listing.address);
       if (similar) {
-        /* eslint-disable no-console */
-        console.debug(`Filtering similar entry for title: ${listing.title} and address ${listing.address}`);
-        /* eslint-enable no-console */
+        logger.debug(`Filtering similar entry for title: ${listing.title} and address ${listing.address}`);
       }
       return !similar;
     });
@@ -115,7 +114,7 @@ class FredyRuntime {
   }
 
   _handleError(err) {
-    if (err.name !== 'NoNewListingsWarning') console.error(err);
+    if (err.name !== 'NoNewListingsWarning') logger.error(err);
   }
 }
 

lib/api/api.js

@@ -13,6 +13,7 @@ import files from 'serve-static';
 import path from 'path';
 import { getDirName } from '../utils.js';
 import { demoRouter } from './routes/demoRouter.js';
+import logger from '../services/logger.js';
 const service = restana();
 const staticService = files(path.join(getDirName(), '../ui/public'));
 const PORT = config.port || 9998;
@@ -34,7 +35,6 @@ service.use('/api/login', loginRouter);
 //this route is unsecured intentionally as it is being queried from the login page
 service.use('/api/demo', demoRouter);
 
-/* eslint-disable no-console */
 service.start(PORT).then(() => {
-  console.info(`Started API service on port ${PORT}`);
+  logger.debug(`Started API service on port ${PORT}`);
 });

lib/api/routes/generalSettingsRoute.js

@@ -2,6 +2,7 @@ import restana from 'restana';
 import { config, getDirName, readConfigFromStorage, refreshConfig } from '../../utils.js';
 import fs from 'fs';
 import { handleDemoUser } from '../../services/storage/userStorage.js';
+import logger from '../../services/logger.js';
 const service = restana();
 const generalSettingsRouter = service.newRouter();
 generalSettingsRouter.get('/', async (req, res) => {
@@ -20,7 +21,7 @@ generalSettingsRouter.post('/', async (req, res) => {
     await refreshConfig();
     handleDemoUser();
   } catch (err) {
-    console.error(err);
+    logger.error(err);
     res.send(new Error('Error while trying to write settings.'));
     return;
   }

lib/api/routes/jobRouter.js

@@ -3,6 +3,7 @@ import * as jobStorage from '../../services/storage/jobStorage.js';
 import * as userStorage from '../../services/storage/userStorage.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
+import logger from '../../services/logger.js';
 const service = restana();
 const jobRouter = service.newRouter();
 function doesJobBelongsToUser(job, req) {
@@ -43,7 +44,7 @@ jobRouter.post('/', async (req, res) => {
     });
   } catch (error) {
     res.send(new Error(error));
-    console.error(error);
+    logger.error(error);
   }
   res.send();
 });
@@ -58,7 +59,7 @@ jobRouter.delete('', async (req, res) => {
     }
   } catch (error) {
     res.send(new Error(error));
-    console.error(error);
+    logger.error(error);
   }
   res.send();
 });
@@ -77,7 +78,7 @@ jobRouter.put('/:jobId/status', async (req, res) => {
     }
   } catch (error) {
     res.send(new Error(error));
-    console.error(error);
+    logger.error(error);
   }
   res.send();
 });

lib/api/routes/loginRoute.js

@@ -3,6 +3,7 @@ import * as userStorage from '../../services/storage/userStorage.js';
 import * as hasher from '../../services/security/hash.js';
 import { config } from '../../utils.js';
 import { trackDemoAccessed } from '../../services/tracking/Tracker.js';
+import logger from '../../services/logger.js';
 const service = restana();
 const loginRouter = service.newRouter();
 loginRouter.get('/user', async (req, res) => {
@@ -35,7 +36,7 @@ loginRouter.post('/', async (req, res) => {
     res.send(200);
     return;
   } else {
-    console.error(`User ${username} tried to login, but password was wrong.`);
+    logger.error(`User ${username} tried to login, but password was wrong.`);
   }
   res.send(401);
 });

lib/notification/adapter/mailJet.js

@@ -5,6 +5,7 @@ import Handlebars from 'handlebars';
 import fetch from 'node-fetch';
 import { markdown2Html } from '../../services/markdown.js';
 import { getDirName, normalizeImageUrl } from '../../utils.js';
+import logger from '../../services/logger.js';
 
 const __dirname = getDirName();
 const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
@@ -24,7 +25,7 @@ const toBase64 = async (url) => {
     const ab = await res.arrayBuffer();
     return Buffer.from(ab).toString('base64');
   } catch (error) {
-    console.error(`Error fetching image from ${url}:`, error.message);
+    logger.error(`Error fetching image from ${url}:`, error.message);
     throw error;
   }
 };
@@ -62,7 +63,7 @@ const mapListingsWithCid = async (serviceName, jobKey, listings) => {
         item.hasImage = true;
         item.imageCid = cid;
       } catch (error) {
-        console.warn(`Skipping image for listing ${i} due to error: ${error.message}`);
+        logger.warn(`Skipping image for listing ${i} due to error: ${error.message}`);
       }
     }
 

lib/notification/adapter/sqlite.js

@@ -1,7 +1,18 @@
 import { markdown2Html } from '../../services/markdown.js';
 import Database from 'better-sqlite3';
-export const send = ({ serviceName, newListings, jobKey }) => {
-  const db = new Database('db/listings.db');
+import path from 'path';
+import fs from 'fs';
+
+export const send = ({ serviceName, newListings, jobKey, notificationConfig }) => {
+  const sqliteConfig = notificationConfig.find((adapter) => adapter.id === config.id);
+  const dbPath = sqliteConfig?.fields?.dbPath || 'db/listings.db';
+
+  const dbDir = path.dirname(dbPath);
+  if (!fs.existsSync(dbDir)) {
+    fs.mkdirSync(dbDir, { recursive: true });
+  }
+
+  const db = new Database(dbPath);
   const fields = [
     'serviceName',
     'jobKey',
@@ -30,8 +41,16 @@ export const send = ({ serviceName, newListings, jobKey }) => {
 };
 export const config = {
   id: 'sqlite',
-  name: 'Sqlite',
-  description: 'This adapter stores listings in a local sqlite3 database.',
-  config: {},
+  name: 'SQLite',
+  description: 'This adapter stores listings in a local SQLite 3 database.',
+  fields: {
+    dbPath: {
+      type: 'text',
+      label: 'Database Path',
+      description:
+        'Path to the SQLite database file (e.g., db/listings.db). If not specified, defaults to db/listings.db',
+      placeholder: 'db/listings.db',
+    },
+  },
   readme: markdown2Html('lib/notification/adapter/sqlite.md'),
 };

lib/notification/adapter/sqlite.md

@@ -1,9 +1,9 @@
-### Sqlite Adapter
+### SQLite Adapter
 
-This adapter stores search results in a sqlite database located in db/listings.db. This file can be used for further analysis later on.
+This adapter stores search results in an SQLite database. By default, the database is located at `db/listings.db`, but you can configure a custom location. This file can be used for further analysis later.
 
-Fields are:
+The database table contains the following columns (all stored as `TEXT` type):
 
 ```
-['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description']
+['serviceName', 'jobKey', 'id', 'size', 'rooms', 'price', 'address', 'title', 'link', 'description', 'image']
 ```

lib/provider/immoscout.js

@@ -37,6 +37,7 @@
 
 import utils, { buildHash } from '../utils.js';
 import { convertWebToMobile } from '../services/immoscout/immoscout-web-translator.js';
+import logger from '../services/logger.js';
 let appliedBlackList = [];
 
 async function getListings(url) {
@@ -52,7 +53,7 @@ async function getListings(url) {
     }),
   });
   if (!response.ok) {
-    console.error('Error fetching data from ImmoScout Mobile API:', response.statusText);
+    logger.error('Error fetching data from ImmoScout Mobile API:', response.statusText);
     return [];
   }
 

lib/services/demoCleanup.js

@@ -2,6 +2,7 @@ import { setInterval } from 'node:timers';
 import { removeJobsByUserName } from './storage/jobStorage.js';
 import { config } from '../utils.js';
 import { getUsers } from './storage/userStorage.js';
+import logger from './logger.js';
 
 /**
  * if we are running in demo environment, we have to cleanup the db files (specifically the jobs table)
@@ -29,7 +30,7 @@ function cleanup() {
   if (config.demoMode) {
     const demoUser = getUsers(false).find((user) => user.username === 'demo');
     if (demoUser == null) {
-      console.error('Demo user not found, cannot remove Jobs');
+      logger.error('Demo user not found, cannot remove Jobs');
       return;
     }
     removeJobsByUserName(demoUser.id);

lib/services/extractor/extractor.js

@@ -1,6 +1,7 @@
 import { setDebug } from './utils.js';
 import puppeteerExtractor from './puppeteerExtractor.js';
 import { loadParser, parse } from './parser/parser.js';
+import logger from '../logger.js';
 
 const DEFAULT_OPTIONS = {
   debug: false,
@@ -32,7 +33,7 @@ export default class Extractor {
         loadParser(this.responseText);
       }
     } catch (error) {
-      console.error('Error trying to load page.', error);
+      logger.error('Error trying to load page.', error);
     }
     return this;
   };

lib/services/extractor/parser/parser.js

@@ -1,4 +1,5 @@
 import * as cheerio from 'cheerio';
+import logger from '../../logger.js';
 
 let $ = null;
 
@@ -8,19 +9,19 @@ export function loadParser(text) {
 
 export function parse(crawlContainer, crawlFields, text, url) {
   if (!text) {
-    console.warn('No content found for ', url);
+    logger.warn('No content found for ', url);
     return null;
   }
 
   if (!crawlContainer || !crawlFields) {
-    console.warn('Cannot parse, selector was empty for url ', url);
+    logger.warn('Cannot parse, selector was empty for url ', url);
     return null;
   }
 
   const result = [];
 
   if ($(crawlContainer).length === 0) {
-    console.warn('No elements in crawl container found for url ', url);
+    logger.warn('No elements in crawl container found for url ', url);
     return null;
   }
 
@@ -58,7 +59,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
 
         parsedObject[key] = value || null;
       } catch (error) {
-        console.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
+        logger.error(`Error parsing field '${key}' with selector '${fieldSelector}':`, error);
         parsedObject[key] = null;
       }
     }
@@ -66,9 +67,7 @@ export function parse(crawlContainer, crawlFields, text, url) {
     if (parsedObject.id != null) {
       result.push(parsedObject);
     } else {
-      /* eslint-disable no-console */
-      console.debug('ID not found. Not relaying object.');
-      /* eslint-enable no-console */
+      logger.debug('ID not found. Not relaying object.');
     }
   });
 
@@ -91,7 +90,7 @@ function applyModifiers(value, modifiers) {
         value = value.replace(/\n/g, ' ');
         break;
       default:
-        console.warn(`Unknown modifier: ${modifier}`);
+        logger.warn(`Unknown modifier: ${modifier}`);
     }
   });
 

lib/services/extractor/puppeteerExtractor.js

@@ -1,6 +1,7 @@
 import puppeteer from 'puppeteer-extra';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 import { debug, DEFAULT_HEADER, botDetected } from './utils.js';
+import logger from '../logger.js';
 
 puppeteer.use(StealthPlugin());
 
@@ -33,13 +34,13 @@ export default async function execute(url, waitForSelector, options) {
     const statusCode = response.status();
 
     if (botDetected(pageSource, statusCode)) {
-      console.warn('We have been detected as a bot :-/ Tried url: => ', url);
+      logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
       return null;
     }
 
     return await page.content();
   } catch (error) {
-    console.error('Error executing with puppeteer executor', error);
+    logger.error('Error executing with puppeteer executor', error);
     return null;
   } finally {
     if (browser != null) {

lib/services/extractor/utils.js

@@ -1,3 +1,5 @@
+import logger from '../logger.js';
+
 let debuggingOn = false;
 
 export const DEFAULT_HEADER = {
@@ -15,9 +17,7 @@ export const setDebug = (options) => {
 
 export const debug = (message) => {
   if (debuggingOn) {
-    /* eslint-disable no-console */
-    console.debug(message);
-    /* eslint-enable no-console */
+    logger.debug(message);
   }
 };
 

lib/services/logger.js

@@ -0,0 +1,59 @@
+const COLORS = {
+  debug: '\x1b[36m',
+  info: '\x1b[32m',
+  warn: '\x1b[33m',
+  error: '\x1b[31m',
+  reset: '\x1b[0m',
+};
+
+const env = process.env.NODE_ENV || 'development';
+const useColor = process.stdout.isTTY || process.stderr.isTTY;
+
+function ts() {
+  const d = new Date();
+  const yyyy = d.getFullYear();
+  const mm = String(d.getMonth() + 1).padStart(2, '0');
+  const dd = String(d.getDate()).padStart(2, '0');
+  const hh = String(d.getHours()).padStart(2, '0');
+  const mi = String(d.getMinutes()).padStart(2, '0');
+  const ss = String(d.getSeconds()).padStart(2, '0');
+  return `${yyyy}-${mm}-${dd} ${hh}:${mi}:${ss}`;
+}
+
+function lvl(level) {
+  const upper = level.toUpperCase();
+  if (!useColor) return upper;
+  return `${COLORS[level] || ''}${upper}${COLORS.reset}`;
+}
+
+/* eslint-disable no-console */
+function log(level, ...args) {
+  if (level === 'debug' && env !== 'development') {
+    return; // Skip debug logs in non-development environments
+  }
+
+  const prefix = `[${ts()}] ${lvl(level)}:`;
+  switch (level) {
+    case 'debug':
+      console.debug(prefix, ...args);
+      break;
+    case 'info':
+      console.info(prefix, ...args);
+      break;
+    case 'warn':
+      console.warn(prefix, ...args);
+      break;
+    case 'error':
+      console.error(prefix, ...args);
+      break;
+    default:
+      console.log(prefix, ...args);
+  }
+}
+
+export default {
+  debug: (...a) => log('debug', ...a),
+  info: (...a) => log('info', ...a),
+  warn: (...a) => log('warn', ...a),
+  error: (...a) => log('error', ...a),
+};

lib/services/storage/jobStorage.js

@@ -4,6 +4,7 @@ import * as listingStorage from './listingsStorage.js';
 import { getDirName } from '../../utils.js';
 import path from 'path';
 import LowdashAdapter from './LowDashAdapter.js';
+import logger from '../logger.js';
 
 const file = path.join(getDirName(), '../', 'db/jobs.json');
 const adapter = new JSONFileSync(file);
@@ -91,9 +92,7 @@ export const removeJobsByUserName = (userId) => {
     .value();
   db.write();
   if (removedDemoJobs > 0) {
-    /* eslint-disable no-console */
-    console.log(`Removed ${removedDemoJobs} demo jobs`);
-    /* eslint-enable no-console */
+    logger.info(`Removed ${removedDemoJobs} demo jobs`);
   }
 };
 export const getJobs = () => {

lib/services/tracking/Tracker.js

@@ -5,6 +5,7 @@ import os from 'os';
 import { readFileSync } from 'fs';
 import { packageUp } from 'package-up';
 import fetch from 'node-fetch';
+import logger from '../logger.js';
 
 const deviceId = getUniqueId() || 'N/A';
 const version = await getPackageVersion();
@@ -37,7 +38,7 @@ export const trackMainEvent = async () => {
       }
     }
   } catch (error) {
-    console.warn('Error sending tracking data', error);
+    logger.warn('Error sending tracking data', error);
   }
 };
 
@@ -52,7 +53,7 @@ export async function trackDemoAccessed() {
         headers: { 'Content-Type': 'application/json' },
       });
     } catch (error) {
-      console.warn('Error sending tracking data', error);
+      logger.warn('Error sending tracking data', error);
     }
   }
 }
@@ -84,7 +85,7 @@ async function getPackageVersion() {
     const json = JSON.parse(packageJson);
     return json.version;
   } catch (error) {
-    console.error('Error reading version from package.json', error);
+    logger.error('Error reading version from package.json', error);
   }
   return 'N/A';
 }

lib/utils.js

@@ -4,6 +4,7 @@ import { readFile } from 'fs/promises';
 import { createHash } from 'crypto';
 import { DEFAULT_CONFIG } from './defaultConfig.js';
 import fs from 'fs';
+import logger from './services/logger.js';
 
 const RE_GT = />/g;
 const RE_WEBP = /\/format\/webp/gi;
@@ -74,8 +75,7 @@ export async function refreshConfig() {
     config.demoMode ??= false;
   } catch (error) {
     config = { ...DEFAULT_CONFIG };
-    /* eslint-disable no-console */
-    console.info('Error reading config file.', error);
+    logger.info('Error reading config file.', error);
   }
 }
 
@@ -84,7 +84,7 @@ export async function refreshConfig() {
  */
 const checkIfConfigExistsAndWriteIfNot = () => {
   if (!fs.existsSync(`${getDirName()}/../conf/config.json`)) {
-    console.info('Could not find config file. Will create one with default values now');
+    logger.info('Could not find config file. Will create one with default values now');
     fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ ...DEFAULT_CONFIG }));
   }
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.6.3",
+  "version": "11.6.6",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",