upgrading dependencies

lib/FredyPipelineExecutioner.js

@@ -227,7 +227,7 @@ class FredyPipelineExecutioner {
     const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
     return new Promise((resolve, reject) => {
       extractor
-        .execute(url, this._providerConfig.waitForSelector)
+        .execute(url, this._providerConfig.waitForSelector, this._providerId)
         .then(() => {
           const listings = extractor.parseResponseText(
             this._providerConfig.crawlContainer,

lib/TRACKING_POIS.js

@@ -7,4 +7,8 @@ export const TRACKING_POIS = {
   DISTANCE_ADDRESS_ENTERED: 'DISTANCE_ADDRESS_ENTERED',
   WELCOME_FINISHED: 'WELCOME_FINISHED',
   WELCOME_SKIPPED: 'WELCOME_SKIPPED',
+  JOBS_TABLE_VIEW: 'JOBS_TABLE_VIEW',
+  LISTING_TABLE_VIEW: 'LISTING_TABLE_VIEW',
+  BASE_URL_SETTING: 'BASE_URL_SETTING',
+  DETECTED_AS_BOT: 'DETECTED_AS_BOT',
 };

lib/api/api.js

@@ -76,13 +76,13 @@ fastify.register(async (app) => {
   app.register(dashboardPlugin, { prefix: '/api/dashboard' });
   app.register(userSettingsPlugin, { prefix: '/api/user/settings' });
   app.register(trackingPlugin, { prefix: '/api/tracking' });
+  app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
 });
 
 // Admin-only routes
 fastify.register(async (app) => {
   app.addHook('preHandler', authHook);
   app.addHook('preHandler', adminHook);
-  app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
   app.register(backupPlugin, { prefix: '/api/admin/backup' });
   app.register(userPlugin, { prefix: '/api/admin/users' });
 });

lib/api/routes/backupRouter.js

@@ -9,6 +9,10 @@ import {
   precheckRestore,
   restoreFromZip,
 } from '../../services/storage/backupRestoreService.js';
+import { getSettings } from '../../services/storage/settingsStorage.js';
+import { isAdmin } from '../security.js';
+
+const DEMO_MODE_ERROR = 'Backup and restore are not available in demo mode.';
 
 /**
  * @param {import('fastify').FastifyInstance} fastify
@@ -21,7 +25,11 @@ export default async function backupPlugin(fastify) {
     (req, body, done) => done(null, body),
   );
 
-  fastify.get('/', async (_request, reply) => {
+  fastify.get('/', async (request, reply) => {
+    const settings = await getSettings();
+    if (settings.demoMode && !isAdmin(request)) {
+      return reply.code(403).send({ error: DEMO_MODE_ERROR });
+    }
     const zipBuffer = await createBackupZip();
     const fileName = await buildBackupFileName();
     reply.header('Content-Type', 'application/zip');
@@ -30,6 +38,10 @@ export default async function backupPlugin(fastify) {
   });
 
   fastify.post('/restore', async (request, reply) => {
+    const settings = await getSettings();
+    if (settings.demoMode && !isAdmin(request)) {
+      return reply.code(403).send({ error: DEMO_MODE_ERROR });
+    }
     const { dryRun = 'false', force = 'false' } = request.query || {};
     const doDryRun = String(dryRun) === 'true';
     const doForce = String(force) === 'true';

lib/api/routes/generalSettingsRoute.js

@@ -9,6 +9,8 @@ import { ensureDemoUserExists } from '../../services/storage/userStorage.js';
 import logger from '../../services/logger.js';
 import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
 import { isAdmin } from '../security.js';
+import { trackPoi } from '../../services/tracking/Tracker.js';
+import { TRACKING_POIS } from '../../TRACKING_POIS.js';
 
 /**
  * @param {import('fastify').FastifyInstance} fastify
@@ -25,16 +27,23 @@ export default async function generalSettingsPlugin(fastify) {
     }
     const localSettings = await getSettings();
 
-    if (localSettings.demoMode && !isAdmin(request)) {
-      return reply.code(403).send({ error: 'In demo mode, it is not allowed to change these settings.' });
+    if (!isAdmin(request)) {
+      const reason = localSettings.demoMode
+        ? 'In demo mode, it is not allowed to change these settings.'
+        : 'Only admins can change these settings.';
+      return reply.code(403).send({ error: reason });
     }
 
     try {
       if (typeof sqlitepath !== 'undefined') {
         fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
       }
+
       upsertSettings(appSettings);
       ensureDemoUserExists();
+      if (appSettings.baseUrl != null) {
+        await trackPoi(TRACKING_POIS.BASE_URL_SETTING);
+      }
     } catch (err) {
       logger.error(err);
       return reply.code(500).send({ error: 'Error while trying to write settings.' });

lib/api/routes/userSettingsRoute.js

@@ -118,6 +118,10 @@ export default async function userSettingsPlugin(fastify) {
       return reply.code(400).send({ error: 'listings_view_mode must be "grid" or "table".' });
     }
 
+    if (listings_view_mode === 'table') {
+      await trackPoi(TRACKING_POIS.LISTING_TABLE_VIEW);
+    }
+
     try {
       upsertSettings({ listings_view_mode }, userId);
       return { success: true };
@@ -135,6 +139,10 @@ export default async function userSettingsPlugin(fastify) {
       return reply.code(400).send({ error: 'jobs_view_mode must be "grid" or "table".' });
     }
 
+    if (jobs_view_mode === 'table') {
+      await trackPoi(TRACKING_POIS.JOBS_TABLE_VIEW);
+    }
+
     try {
       upsertSettings({ jobs_view_mode }, userId);
       return { success: true };

lib/provider/immobilienDe.js

@@ -26,7 +26,7 @@ function parseId(shortenedLink) {
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, null, { browser });
+    const html = await puppeteerExtractor(listing.link, null, { browser, name: 'immobilienDe_details' });
     if (!html) return listing;
 
     const $ = cheerio.load(html);

lib/provider/immowelt.js

@@ -16,7 +16,7 @@ let appliedBlackList = [];
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, null, { browser });
+    const html = await puppeteerExtractor(listing.link, null, { browser, name: 'immowelt_details' });
     if (!html) return listing;
 
     const $ = cheerio.load(html);

lib/provider/kleinanzeigen.js

@@ -128,7 +128,7 @@ async function enrichListingFromDetails(listing, browser) {
   if (!absoluteLink) return listing;
 
   try {
-    const html = await puppeteerExtractor(absoluteLink, null, { browser });
+    const html = await puppeteerExtractor(absoluteLink, null, { browser, name: 'kleinanzeigen_details' });
     if (!html) return { ...listing, link: absoluteLink };
 
     const { detailAddress, detailDescription } = extractDetailFromHtml(html);
@@ -196,8 +196,8 @@ const config = {
     id: '.aditem@data-adid',
     price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
     tags: '.aditem-main--middle--tags | removeNewline | trim',
-    title: '.aditem-main .text-module-begin a | removeNewline | trim',
-    link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
+    title: '.aditem-main .text-module-begin | removeNewline | trim',
+    link: '.aditem@data-href',
     description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
     address: '.aditem-main--top--left | trim | removeNewline',
     image: 'img@src',

lib/provider/sparkasse.js

@@ -16,7 +16,7 @@ let appliedBlackList = [];
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, 'body', { browser });
+    const html = await puppeteerExtractor(listing.link, 'body', { browser, name: 'sparkasse_details' });
 
     const $ = cheerio.load(html);
     const nextDataRaw = $('#__NEXT_DATA__').text;

lib/provider/wgGesucht.js

@@ -16,7 +16,7 @@ let appliedBlackList = [];
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, null, { browser });
+    const html = await puppeteerExtractor(listing.link, null, { browser, name: 'wgGesucht_details' });
     if (!html) return listing;
 
     const $ = cheerio.load(html);

lib/services/extractor/extractor.js

@@ -29,11 +29,12 @@ export default class Extractor {
    * your response will never contain what you are really looking for
    * @param url
    * @param waitForSelector
+   * @param jobKey
    */
-  execute = async (url, waitForSelector = null) => {
+  execute = async (url, waitForSelector = null, jobKey = null) => {
     this.responseText = null;
     try {
-      this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+      this.responseText = await puppeteerExtractor(url, waitForSelector, { ...this.options, name: jobKey });
       if (this.responseText != null) {
         loadParser(this.responseText);
       }

lib/services/extractor/puppeteerExtractor.js

@@ -4,9 +4,11 @@
  */
 
 import { launch } from 'cloakbrowser/puppeteer';
-import { debug, botDetected } from './utils.js';
+import { botDetected, debug } from './utils.js';
 import { getPreLaunchConfig } from './botPrevention.js';
 import logger from '../logger.js';
+import { trackPoi } from '../tracking/Tracker.js';
+import { TRACKING_POIS } from '../../TRACKING_POIS.js';
 
 /**
  * Launch a CloakBrowser/Puppeteer browser instance with stealth and humanizer enabled.
@@ -48,7 +50,7 @@ export async function launchBrowser(url, options) {
     preCfg.windowSizeArg,
   ];
 
-  const browser = await launch({
+  return await launch({
     headless: options?.puppeteerHeadless ?? true,
     humanize: true,
     args,
@@ -57,8 +59,6 @@ export async function launchBrowser(url, options) {
     ...(options?.proxyUrl ? { proxy: options.proxyUrl } : {}),
     ...(preCfg.timezone ? { timezone: preCfg.timezone } : {}),
   });
-
-  return browser;
 }
 
 /**
@@ -145,6 +145,13 @@ export default async function execute(url, waitForSelector, options) {
 
     if (botDetected(pageSource, statusCode)) {
       logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
+
+      if (options != null && options.name != null) {
+        await trackPoi(TRACKING_POIS.DETECTED_AS_BOT + '_' + options.name);
+      } else {
+        await trackPoi(TRACKING_POIS.DETECTED_AS_BOT);
+      }
+
       result = null;
     } else {
       result = pageSource || (await page.content());

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "22.0.0",
+  "version": "22.0.9",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -62,9 +62,9 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-icons": "^2.96.1",
-    "@douyinfe/semi-ui": "2.96.1",
-    "@douyinfe/semi-ui-19": "^2.96.1",
+    "@douyinfe/semi-icons": "^2.99.0",
+    "@douyinfe/semi-ui": "2.99.0",
+    "@douyinfe/semi-ui-19": "^2.99.0",
     "@fastify/cookie": "^11.0.2",
     "@fastify/helmet": "^13.0.2",
     "@fastify/session": "^11.1.1",
@@ -73,12 +73,12 @@
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@sendgrid/mail": "8.1.6",
     "@turf/boolean-point-in-polygon": "^7.3.5",
-    "@vitejs/plugin-react": "6.0.1",
+    "@vitejs/plugin-react": "6.0.2",
     "adm-zip": "^0.5.17",
-    "better-sqlite3": "^12.9.0",
+    "better-sqlite3": "^12.10.0",
     "chart.js": "^4.5.1",
     "cheerio": "^1.2.0",
-    "cloakbrowser": "^0.3.27",
+    "cloakbrowser": "^0.3.30",
     "fastify": "^5.8.5",
     "handlebars": "4.7.9",
     "maplibre-gl": "^5.24.0",
@@ -89,18 +89,18 @@
     "nodemailer": "^8.0.7",
     "p-throttle": "^8.1.0",
     "package-up": "^5.0.0",
-    "puppeteer-core": "^24.43.0",
+    "puppeteer-core": "^25.0.4",
     "query-string": "9.3.1",
     "react": "19.2.6",
     "react-chartjs-2": "^5.3.1",
     "react-dom": "19.2.6",
     "react-range-slider-input": "^3.3.5",
-    "react-router": "7.15.0",
-    "react-router-dom": "7.15.0",
+    "react-router": "7.15.1",
+    "react-router-dom": "7.15.1",
     "resend": "^6.12.3",
-    "semver": "^7.7.4",
+    "semver": "^7.8.1",
     "slack": "11.0.2",
-    "vite": "8.0.11",
+    "vite": "8.0.14",
     "x-var": "^3.0.1",
     "zustand": "^5.0.13"
   },
@@ -111,16 +111,16 @@
     "@babel/preset-react": "7.28.5",
     "@eslint/js": "^10.0.1",
     "chalk": "^5.6.2",
-    "eslint": "10.3.0",
+    "eslint": "10.4.0",
     "eslint-config-prettier": "10.1.8",
     "eslint-plugin-react": "7.37.5",
     "globals": "^17.6.0",
     "history": "5.3.0",
     "husky": "9.1.7",
     "less": "4.6.4",
-    "lint-staged": "16.4.0",
+    "lint-staged": "17.0.5",
     "nodemon": "^3.1.14",
     "prettier": "3.8.3",
-    "vitest": "^4.1.5"
+    "vitest": "^4.1.7"
   }
 }

test/services/jobs/jobExecutionService.test.js

@@ -38,6 +38,7 @@ describe('services/jobs/jobExecutionService', () => {
     }));
     vi.doMock(utilsPath, () => ({
       duringWorkingHoursOrNotSet: () => false,
+      getPackageVersion: async () => '0.0.0-test',
     }));
     vi.doMock(loggerPath, () => {
       const m = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} };

tools/testFixtures/downloadFixtures.js

@@ -95,7 +95,10 @@ async function downloadHtmlProvider(name, providerConfig, launchBrowser, closeBr
 
   const browser = await launchBrowser(providerConfig.url, {});
   try {
-    const html = await puppeteerExtractor(providerConfig.url, providerConfig.waitForSelector, { browser });
+    const html = await puppeteerExtractor(providerConfig.url, providerConfig.waitForSelector, {
+      browser,
+      name: 'dowload_fixtures',
+    });
 
     if (!html) {
       console.warn(`  Failed to download ${name}`);

ui/src/components/grid/jobs/JobGrid.jsx

@@ -174,7 +174,7 @@ const JobGrid = () => {
       Toast.success('Job status successfully changed');
       loadData();
     } catch (error) {
-      Toast.error(error);
+      Toast.error(error.error);
     }
   };
 

ui/src/views/generalSettings/GeneralSettings.jsx

@@ -54,6 +54,7 @@ const GeneralSettings = function GeneralSettings() {
   const [loading, setLoading] = React.useState(true);
 
   const settings = useSelector((state) => state.generalSettings.settings);
+  const currentUser = useSelector((state) => state.user.currentUser);
 
   const [interval, setInterval] = React.useState('');
   const [port, setPort] = React.useState('');
@@ -467,12 +468,26 @@ const GeneralSettings = function GeneralSettings() {
               itemKey="backup"
             >
               <div className="generalSettings__tab-content">
+                {demoMode && !currentUser?.isAdmin && (
+                  <Banner
+                    fullMode={false}
+                    type="warning"
+                    closeIcon={null}
+                    style={{ marginBottom: '12px' }}
+                    description="Backup and restore are not available in demo mode."
+                  />
+                )}
                 <SegmentPart
                   name="Backup & Restore"
                   helpText="Download a zipped backup of your database or restore from a backup zip."
                 >
                   <div style={{ display: 'flex', gap: '0.5rem', alignItems: 'center', flexWrap: 'wrap' }}>
-                    <Button theme="solid" icon={<IconSave />} onClick={handleDownloadBackup}>
+                    <Button
+                      theme="solid"
+                      icon={<IconSave />}
+                      onClick={handleDownloadBackup}
+                      disabled={demoMode && !currentUser?.isAdmin}
+                    >
                       Download Backup
                     </Button>
                     <input
@@ -482,7 +497,12 @@ const GeneralSettings = function GeneralSettings() {
                       style={{ display: 'none' }}
                       onChange={handleSelectRestoreFile}
                     />
-                    <Button onClick={handleOpenFilePicker} theme="light" icon={<IconFolder />}>
+                    <Button
+                      onClick={handleOpenFilePicker}
+                      theme="light"
+                      icon={<IconFolder />}
+                      disabled={demoMode && !currentUser?.isAdmin}
+                    >
                       Restore from Zip
                     </Button>
                   </div>

ui/src/views/user/Users.jsx

@@ -37,7 +37,7 @@ const Users = function Users() {
       await actions.jobsData.getJobs();
       await actions.user.getUsers();
     } catch (error) {
-      Toast.error(error);
+      Toast.error(error.error);
       setUserIdToBeRemoved(null);
     }
   };