better logging

Dockerfile

@@ -4,6 +4,11 @@ WORKDIR  /fredy
 
 COPY . /fredy
 
+RUN apt-get update && apt-get install -y chromium
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+  PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
+
 RUN yarn install
 
 RUN yarn global add pm2

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Christian Kellner
+Copyright (c) 2025 Christian Kellner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

lib/FredyRuntime.js

@@ -47,11 +47,17 @@ class FredyRuntime {
   _getListings(url) {
     const extractor = new Extractor();
     return new Promise((resolve, reject) => {
-            extractor.execute(url,this._providerConfig.waitForSelector)
+      extractor
+        .execute(url, this._providerConfig.waitForSelector)
         .then(() => {
-                    const listings = extractor.parseResponseText(this._providerConfig.crawlContainer, this._providerConfig.crawlFields);
+          const listings = extractor.parseResponseText(
+            this._providerConfig.crawlContainer,
+            this._providerConfig.crawlFields,
+            url,
+          );
           resolve(listings == null ? [] : listings);
-                }).catch(err => {
+        })
+        .catch((err) => {
           reject(err);
           /* eslint-disable no-console */
           console.error(err);

lib/api/routes/jobRouter.js

@@ -1,8 +1,6 @@
 import restana from 'restana';
-import fetch from 'node-fetch';
 import * as jobStorage from '../../services/storage/jobStorage.js';
 import * as userStorage from '../../services/storage/userStorage.js';
-import * as immoscoutProvider from '../../provider/immoscout.js';
 import { config } from '../../utils.js';
 import { isAdmin } from '../security.js';
 import { trackDemoJobCreated } from '../../services/tracking/Tracker.js';
@@ -28,7 +26,7 @@ jobRouter.get('/', async (req, res) => {
 jobRouter.get('/processingTimes', async (req, res) => {
   res.body = {
     interval: config.interval,
-    lastRun: config.lastRun || null
+    lastRun: config.lastRun || null,
   };
   res.send();
 });
@@ -51,7 +49,7 @@ jobRouter.post('/', async (req, res) => {
   trackDemoJobCreated({
     name,
     provider,
-    adapter: notificationAdapter
+    adapter: notificationAdapter,
   });
   res.send();
 });

lib/provider/immonet.js

@@ -26,7 +26,7 @@ const config = {
   url: null,
   crawlContainer: 'div[data-testid="serp-core-classified-card-testid"]',
   sortByDateParam: 'sortby=19',
-  waitForSelector: 'div[data-testid="serp-core-classified-card-testid"]',
+  waitForSelector: 'div[data-testid="serp-resultscount-testid"]',
   crawlFields: {
     id: 'button@title |trim', // immonet is a piece of sh*t. See comment above
     title: 'button@title |trim',

lib/provider/immowelt.js

@@ -18,7 +18,7 @@ const config = {
   crawlContainer:
     'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
   sortByDateParam: 'order=DateDesc',
-  waitForSelector: 'div[data-testid="cardmfe-price-testid"]',
+  waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
   crawlFields: {
     id: 'a@href',
     price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',

lib/provider/kleinanzeigen.js

@@ -15,7 +15,7 @@ function applyBlacklist(o) {
     const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
     const isBlacklistedDistrict =
         appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
-    return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
+    return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {

lib/services/extractor/extractor.js

@@ -4,16 +4,15 @@ import {loadParser, parse} from './parser/parser.js';
 
 const DEFAULT_OPTIONS = {
   debug: false,
-    puppeteerTimeout: 20_000,
+  puppeteerTimeout: 60_000,
-    puppeteerHeadless: true
+  puppeteerHeadless: true,
-
 };
 
 export default class Extractor {
   constructor(options) {
     this.options = {
       ...DEFAULT_OPTIONS,
-            ...options
+      ...options,
     };
     this.responseText = null;
     setDebug(this.options);
@@ -38,8 +37,7 @@ export default class Extractor {
     return this;
   };
 
-
+  parseResponseText = (crawlContainer, crawlFields, url) => {
-    parseResponseText = (crawlContainer, crawlFields) => {
+    return parse(crawlContainer, crawlFields, this.responseText, url);
-        return parse(crawlContainer, crawlFields, this.responseText);
   };
 }

lib/services/extractor/parser/parser.js

@@ -6,21 +6,22 @@ export function loadParser(text) {
   $ = cheerio.load(text);
 }
 
-export function parse(crawlContainer, crawlFields, text) {
+export function parse(crawlContainer, crawlFields, text, url) {
   if (!text) {
-        console.warn('Cannot parse, text was empty.');
+    console.warn('Cannot parse, text was empty for url ', url);
     return null;
   }
 
   if (!crawlContainer || !crawlFields) {
-        console.warn('Cannot parse, selector was empty.');
+    console.warn('Cannot parse, selector was empty for url ', url);
     return null;
   }
 
   const result = [];
 
   if ($(crawlContainer).length === 0) {
-        console.error('No elements in crawl container found!');
+    console.warn('No elements in crawl container found for url ', url);
+    return null;
   }
 
   $(crawlContainer).each((_, element) => {
@@ -32,8 +33,9 @@ export function parse(crawlContainer, crawlFields, text) {
       let value;
 
       try {
-
+        const selector = fieldSelector.includes('|')
-                const selector = fieldSelector.includes('|') ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim() : fieldSelector;
+          ? fieldSelector.substring(0, fieldSelector.indexOf('|')).trim()
+          : fieldSelector;
 
         if (selector.includes('@')) {
           const [sel, attr] = selector.split('@');
@@ -48,7 +50,9 @@ export function parse(crawlContainer, crawlFields, text) {
 
         // Apply modifiers if specified
         if (fieldSelector.includes('|')) {
-                    const [_, ...modifiers] = fieldSelector.split('|').map(s => s.trim());
+          /* eslint-disable no-unused-vars */
+          const [_, ...modifiers] = fieldSelector.split('|').map((s) => s.trim());
+          /* eslint-disable no-unused-vars */
           value = applyModifiers(value, modifiers);
         }
 
@@ -73,7 +77,7 @@ export function parse(crawlContainer, crawlFields, text) {
 function applyModifiers(value, modifiers) {
   if (!value) return value;
 
-    modifiers.forEach(modifier => {
+  modifiers.forEach((modifier) => {
     switch (modifier) {
       case 'int':
         value = parseInt(value, 10);
@@ -91,4 +95,3 @@ function applyModifiers(value, modifiers) {
 
   return value;
 }
-

lib/services/extractor/puppeteerExtractor.js

@@ -11,18 +11,19 @@ export default async function execute(url, waitForSelector, options) {
 
     browser = await puppeteer.launch({
       headless: options.puppeteerHeadless ?? true,
-            args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox']
+      args: ['--no-sandbox', '--disable-gpu', '--disable-setuid-sandbox'],
+      timeout: options.puppeteerTimeout || 30_000,
     });
     let page = await browser.newPage();
     await page.setExtraHTTPHeaders(DEFAULT_HEADER);
     const response = await page.goto(url, {
-            waitUntil: 'domcontentloaded'
+      waitUntil: 'domcontentloaded',
     });
     let pageSource;
     //if we're extracting data from a spa, we must wait for the selector
     if (waitForSelector != null) {
       await page.waitForSelector(waitForSelector);
-            pageSource = await page.evaluate(selector => {
+      pageSource = await page.evaluate((selector) => {
         return document.querySelector(selector).innerHTML;
       }, waitForSelector);
     } else {

lib/services/extractor/utils.js

@@ -1,34 +1,31 @@
 let debuggingOn = false;
 
 export const DEFAULT_HEADER = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
   'Accept-Language': 'en-US,en;q=0.5',
-    'Connection': 'keep-alive',
+  Connection: 'keep-alive',
   'Upgrade-Insecure-Requests': '1',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
 };
 
-export const setDebug = options => {
+export const setDebug = (options) => {
   debuggingOn = !!options?.debug;
 };
 
 export const debug = (message) => {
   if (debuggingOn) {
+    /* eslint-disable no-console */
     console.debug(message);
+    /* eslint-enable no-console */
   }
 };
 
 export const botDetected = (pageSource, statusCode) => {
-    const suspiciousStatusCodes = [
+  const suspiciousStatusCodes = [403, 429];
-        403, 429
+  const botDetectionPatterns = [/verify you are human/i, /access denied/i, /x-amz-cf-id/i];
-    ];
-    const botDetectionPatterns = [
-        /verify you are human/i,
-        /access denied/i,
-        /x-amz-cf-id/i,
-    ];
 
-    const detectedInSource = botDetectionPatterns.some(pattern => pattern.test(pageSource));
+  const detectedInSource = botDetectionPatterns.some((pattern) => pattern.test(pageSource));
   const detectedByStatus = suspiciousStatusCodes.includes(statusCode);
 
   return detectedInSource || detectedByStatus;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "11.0.0",
+  "version": "11.0.1",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node prod.js",
@@ -50,17 +50,17 @@
     "Firefox ESR"
   ],
   "dependencies": {
-    "@douyinfe/semi-ui": "2.71.3",
+    "@douyinfe/semi-ui": "2.72.3",
     "@rematch/core": "2.2.0",
     "@rematch/loading": "2.1.2",
     "@sendgrid/mail": "8.1.4",
     "@vitejs/plugin-react": "4.3.4",
-    "better-sqlite3": "^11.7.0",
+    "better-sqlite3": "^11.7.2",
     "body-parser": "1.20.3",
     "cheerio": "^1.0.0",
     "cookie-session": "2.1.0",
     "handlebars": "4.7.8",
-    "highcharts": "12.1.0",
+    "highcharts": "12.1.2",
     "highcharts-react-official": "3.2.1",
     "lodash": "4.17.21",
     "lowdb": "6.0.1",
@@ -70,7 +70,7 @@
     "node-fetch": "3.3.2",
     "node-mailjet": "6.0.6",
     "package-up": "^5.0.0",
-    "puppeteer": "^23.10.4",
+    "puppeteer": "^23.11.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "query-string": "9.1.1",
@@ -95,12 +95,12 @@
     "chai": "5.1.2",
     "eslint": "8.56.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-react": "7.37.2",
+    "eslint-plugin-react": "7.37.3",
     "esmock": "2.6.9",
     "history": "5.3.0",
     "husky": "9.1.7",
     "less": "4.2.1",
-    "lint-staged": "15.2.11",
+    "lint-staged": "15.3.0",
     "mocha": "10.8.2",
     "prettier": "3.4.2",
     "redux-logger": "3.0.6"