upgrading pois

lib/FredyPipelineExecutioner.js

@@ -227,7 +227,7 @@ class FredyPipelineExecutioner {
     const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
     return new Promise((resolve, reject) => {
       extractor
-        .execute(url, this._providerConfig.waitForSelector)
+        .execute(url, this._providerConfig.waitForSelector, this._providerId)
         .then(() => {
           const listings = extractor.parseResponseText(
             this._providerConfig.crawlContainer,

lib/api/api.js

@@ -76,13 +76,13 @@ fastify.register(async (app) => {
   app.register(dashboardPlugin, { prefix: '/api/dashboard' });
   app.register(userSettingsPlugin, { prefix: '/api/user/settings' });
   app.register(trackingPlugin, { prefix: '/api/tracking' });
+  app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
 });
 
 // Admin-only routes
 fastify.register(async (app) => {
   app.addHook('preHandler', authHook);
   app.addHook('preHandler', adminHook);
-  app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
   app.register(backupPlugin, { prefix: '/api/admin/backup' });
   app.register(userPlugin, { prefix: '/api/admin/users' });
 });

lib/api/routes/generalSettingsRoute.js

@@ -27,8 +27,11 @@ export default async function generalSettingsPlugin(fastify) {
     }
     const localSettings = await getSettings();
 
-    if (localSettings.demoMode && !isAdmin(request)) {
+    if (!isAdmin(request)) {
-      return reply.code(403).send({ error: 'In demo mode, it is not allowed to change these settings.' });
+      const reason = localSettings.demoMode
+        ? 'In demo mode, it is not allowed to change these settings.'
+        : 'Only admins can change these settings.';
+      return reply.code(403).send({ error: reason });
     }
 
     try {

lib/provider/immobilienDe.js

@@ -26,7 +26,7 @@ function parseId(shortenedLink) {
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, null, { browser });
+    const html = await puppeteerExtractor(listing.link, null, { browser, name: 'immobilienDe_details' });
     if (!html) return listing;
 
     const $ = cheerio.load(html);

lib/provider/immowelt.js

@@ -16,7 +16,7 @@ let appliedBlackList = [];
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, null, { browser });
+    const html = await puppeteerExtractor(listing.link, null, { browser, name: 'immowelt_details' });
     if (!html) return listing;
 
     const $ = cheerio.load(html);

lib/provider/kleinanzeigen.js

@@ -128,7 +128,7 @@ async function enrichListingFromDetails(listing, browser) {
   if (!absoluteLink) return listing;
 
   try {
-    const html = await puppeteerExtractor(absoluteLink, null, { browser });
+    const html = await puppeteerExtractor(absoluteLink, null, { browser, name: 'kleinanzeigen_details' });
     if (!html) return { ...listing, link: absoluteLink };
 
     const { detailAddress, detailDescription } = extractDetailFromHtml(html);

lib/provider/sparkasse.js

@@ -16,7 +16,7 @@ let appliedBlackList = [];
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, 'body', { browser });
+    const html = await puppeteerExtractor(listing.link, 'body', { browser, name: 'sparkasse_details' });
 
     const $ = cheerio.load(html);
     const nextDataRaw = $('#__NEXT_DATA__').text;

lib/provider/wgGesucht.js

@@ -16,7 +16,7 @@ let appliedBlackList = [];
 
 async function fetchDetails(listing, browser) {
   try {
-    const html = await puppeteerExtractor(listing.link, null, { browser });
+    const html = await puppeteerExtractor(listing.link, null, { browser, name: 'wgGesucht_details' });
     if (!html) return listing;
 
     const $ = cheerio.load(html);

lib/services/extractor/extractor.js

@@ -29,11 +29,12 @@ export default class Extractor {
    * your response will never contain what you are really looking for
    * @param url
    * @param waitForSelector
+   * @param jobKey
    */
-  execute = async (url, waitForSelector = null) => {
+  execute = async (url, waitForSelector = null, jobKey = null) => {
     this.responseText = null;
     try {
-      this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
+      this.responseText = await puppeteerExtractor(url, waitForSelector, { ...this.options, name: jobKey });
       if (this.responseText != null) {
         loadParser(this.responseText);
       }

lib/services/extractor/puppeteerExtractor.js

@@ -148,7 +148,11 @@ export default async function execute(url, waitForSelector, options) {
     if (botDetected(pageSource, statusCode)) {
       logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
 
-      await trackPoi(TRACKING_POIS.DETECTED_AS_BOT);
+      if (options != null && options.name != null) {
+        await trackPoi(TRACKING_POIS.DETECTED_AS_BOT + '_' + options.name);
+      } else {
+        await trackPoi(TRACKING_POIS.DETECTED_AS_BOT);
+      }
 
       result = null;
     } else {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "22.0.4",
+  "version": "22.0.7",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "prepare": "husky",
@@ -75,7 +75,7 @@
     "@turf/boolean-point-in-polygon": "^7.3.5",
     "@vitejs/plugin-react": "6.0.1",
     "adm-zip": "^0.5.17",
-    "better-sqlite3": "^12.9.0",
+    "better-sqlite3": "^12.10.0",
     "chart.js": "^4.5.1",
     "cheerio": "^1.2.0",
     "cloakbrowser": "^0.3.28",

tools/testFixtures/downloadFixtures.js

@@ -95,7 +95,10 @@ async function downloadHtmlProvider(name, providerConfig, launchBrowser, closeBr
 
   const browser = await launchBrowser(providerConfig.url, {});
   try {
-    const html = await puppeteerExtractor(providerConfig.url, providerConfig.waitForSelector, { browser });
+    const html = await puppeteerExtractor(providerConfig.url, providerConfig.waitForSelector, {
+      browser,
+      name: 'dowload_fixtures',
+    });
 
     if (!html) {
       console.warn(`  Failed to download ${name}`);

ui/src/components/grid/jobs/JobGrid.jsx

@@ -174,7 +174,7 @@ const JobGrid = () => {
       Toast.success('Job status successfully changed');
       loadData();
     } catch (error) {
-      Toast.error(error);
+      Toast.error(error.error);
     }
   };
 

ui/src/views/user/Users.jsx

@@ -37,7 +37,7 @@ const Users = function Users() {
       await actions.jobsData.getJobs();
       await actions.user.getUsers();
     } catch (error) {
-      Toast.error(error);
+      Toast.error(error.error);
       setUserIdToBeRemoved(null);
     }
   };

yarn.lock

@@ -2675,10 +2675,10 @@ basic-ftp@^5.0.2:
   resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.3.1.tgz#3148ee9af43c0522514a4f973fecb1d3cbb6d71e"
   integrity sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==
 
-better-sqlite3@^12.9.0:
+better-sqlite3@^12.10.0:
-  version "12.9.0"
+  version "12.10.0"
-  resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.9.0.tgz#32498c99ba3fb36f604fbb5c70667c5f68c00414"
+  resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-12.10.0.tgz#bde622d14a18008583a53bc53501ae98f1a12221"
-  integrity sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ==
+  integrity sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==
   dependencies:
     bindings "^1.5.0"
     prebuild-install "^7.1.1"