🕵️ More immoscout details (#258)

* 🕵️ More immoscout details

- Added more details to immoscout api - description is now populated with a lot of data from the expose using app API
- You can ignore certificates, if deploying locally and using the http notification adapter
- More details for the test call/example for easier testing + placeholder image + actual values + address (famous Erika Mustermans address see https://de.wikipedia.org/wiki/Mustermann)
- Grater timeout for geocode since the api is sometimes slow in germany
- uiElement, type boolean, now has a label as well

* 👀 Requested changes + some extra

Req:
- using logger
- using node-fetch

Extra:
- boolean input fields will trigger the validate check, because they are set undefined at first - setting them to false if they are undefined now
- added more data to the description (phone number and name of the agent)

* ✅ Fixed import

* ✅️ Toggle immoscout detail fetching

* ✅️ Requested change

lib/api/routes/notificationAdapterRouter.js

@@ -5,6 +5,8 @@
 
 import fs from 'fs';
 import restana from 'restana';
+import logger from '../../services/logger.js';
+
 const service = restana();
 const notificationAdapterRouter = service.newRouter();
 const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
@@ -34,11 +36,14 @@ notificationAdapterRouter.post('/try', async (req, res) => {
       serviceName: 'TestCall',
       newListings: [
         {
-          price: '42 €',
-          title: 'This is a test listing',
-          address: 'some address',
-          size: '666 2m',
-          link: 'https://www.orange-coding.net',
+          address: 'Heidestrasse 17, 51147 Köln',
+          description: exampleDescription,
+          id: '1',
+          imageUrl: 'https://placehold.co/600x400/png',
+          price: '1.000 €',
+          size: '76 m²',
+          title: 'Stilvolle gepflegte 3-Raum-Wohnung mit gehobener Innenausstattung',
+          url: 'https://www.orange-coding.net',
         },
       ],
       notificationConfig,
@@ -46,6 +51,7 @@ notificationAdapterRouter.post('/try', async (req, res) => {
     });
     res.send();
   } catch (Exception) {
+    logger.error('Error during notification adapter test:', Exception);
     res.send(new Error(Exception));
   }
 });
@@ -54,3 +60,51 @@ notificationAdapterRouter.get('/', async (req, res) => {
   res.send();
 });
 export { notificationAdapterRouter };
+
+const exampleDescription = `
+Wohnungstyp: Etagenwohnung
+Nutzfläche: 76 m²
+Etage: 2 von 3
+Schlafzimmer: 1
+Badezimmer: 1
+Bezugsfrei ab: 1.4.2026
+Haustiere: Nein
+Garage/Stellplatz: Tiefgarage
+Anzahl Garage/Stellplatz: 1
+Kaltmiete (zzgl. Nebenkosten): 1.000 €
+Preis/m²: 13,16 €/m²
+Nebenkosten: 230 €
+Heizkosten in Nebenkosten enthalten: Ja
+Gesamtmiete: 1.230 €
+Kaution: 3.000,00
+Preis pro Parkfläche: 60 €
+Baujahr: 2000
+Objektzustand: Modernisiert
+Qualität der Ausstattung: Gehoben
+Heizungsart: Fernwärme
+Energieausweistyp: Verbrauchsausweis
+Energieausweis: liegt vor
+Endenergieverbrauch: 72 kWh/(m²∙a)
+Baujahr laut Energieausweis: 2000
+
+Diese moderne 3-Zimmer-Wohnung liegt direkt neben einem Park und nur wenige Minuten von der S-Bahn-Haltestelle entfernt. Das Stadtzentrum sowie Freizeiteinrichtungen sind 1,5 km entfernt.
+
+Die Wohnung ist ideal für Paare oder kleine Familien geeignet.
+
+Ausstattung:
+- neuer hochwertiger Bodenbelag (Holzoptik) in allen Räumen außer Bad/Küche
+- sonniger Balkon (Süd)
+- Tiefgaragenstellplatz 
+- Kellerabteil
+- gepflegtes Mehrfamilienhaus
+
+Die Küche ist vom Mieter nach eigenen Wünschen einzurichten.
+
+Vermietung direkt vom Eigentümer - provisionsfrei!
+
+Lage:
+• Park: 1 Minute zu Fuß
+• S-Bahn Station: 2 Minuten zu Fuß  
+• Supermärkte, Restaurants, täglicher Bedarf in der Nähe
+• Gute Anbindung Richtung Großstadt und Flughafen
+`;

lib/api/routes/userSettingsRoute.js

@@ -97,4 +97,25 @@ userSettingsRouter.post('/news-hash', async (req, res) => {
   }
 });
 
+userSettingsRouter.post('/immoscout-details', async (req, res) => {
+  const userId = req.session.currentUser;
+  const { immoscout_details } = req.body;
+
+  const globalSettings = await getSettings();
+  if (globalSettings.demoMode) {
+    res.statusCode = 403;
+    res.send({ error: 'In demo mode, it is not allowed to change settings.' });
+    return;
+  }
+
+  try {
+    upsertSettings({ immoscout_details: !!immoscout_details }, userId);
+    res.send({ success: true });
+  } catch (error) {
+    logger.error('Error updating immoscout details setting', error);
+    res.statusCode = 500;
+    res.send({ error: error.message });
+  }
+});
+
 export { userSettingsRouter };

lib/notification/adapter/http.js

@@ -16,8 +16,8 @@ const mapListing = (listing) => ({
   url: listing.link,
 });
 
-export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
-  const { authToken, endpointUrl } = notificationConfig.find((a) => a.id === config.id).fields;
+export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
+  const { authToken, endpointUrl, selfSignedCerts } = notificationConfig.find((a) => a.id === config.id).fields;
 
   const listings = newListings.map(mapListing);
   const body = {
@@ -34,11 +34,20 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
     headers['Authorization'] = `Bearer ${authToken}`;
   }
 
-  return fetch(endpointUrl, {
+  let fetchOptions = {
     method: 'POST',
-    headers: headers,
+    headers,
+    timeout: 10000,
     body: JSON.stringify(body),
-  });
+  };
+
+  if (selfSignedCerts === true) {
+    fetchOptions.dispatcher = new (await import('undici')).Agent({
+      connect: { rejectUnauthorized: false },
+    });
+  }
+
+  return fetch(endpointUrl, fetchOptions);
 };
 
 export const config = {
@@ -52,6 +61,10 @@ export const config = {
       label: 'Endpoint URL',
       type: 'text',
     },
+    selfSignedCerts: {
+      label: 'Self-signed certificates',
+      type: 'boolean',
+    },
     authToken: {
       description: "Your application's auth token, if required by your endpoint.",
       label: 'Auth token (optional)',

lib/provider/immoscout.js

@@ -46,7 +46,9 @@ import {
   convertWebToMobile,
 } from '../services/immoscout/immoscout-web-translator.js';
 import logger from '../services/logger.js';
+import { getUserSettings } from '../services/storage/settingsStorage.js';
 let appliedBlackList = [];
+let currentUserId = null;
 
 async function getListings(url) {
   const response = await fetch(url, {
@@ -66,23 +68,86 @@ async function getListings(url) {
   }
 
   const responseBody = await response.json();
-  return responseBody.resultListItems
-    .filter((item) => item.type === 'EXPOSE_RESULT')
-    .map((expose) => {
-      const item = expose.item;
-      const [price, size] = item.attributes;
-      const image = item?.titlePicture?.preview ?? null;
-      return {
-        id: item.id,
-        price: price?.value,
-        size: size?.value,
-        title: item.title,
-        description: item.description,
-        link: `${metaInformation.baseUrl}expose/${item.id}`,
-        address: item.address?.line,
-        image,
-      };
-    });
+  return Promise.all(
+    responseBody.resultListItems
+      .filter((item) => item.type === 'EXPOSE_RESULT')
+      .map(async (expose) => {
+        const item = expose.item;
+        const [price, size] = item.attributes;
+        const image = item?.titlePicture?.full ?? item?.titlePicture?.preview ?? null;
+        let listing = {
+          id: item.id,
+          price: price?.value,
+          size: size?.value,
+          title: item.title,
+          link: `${metaInformation.baseUrl}expose/${item.id}`,
+          address: item.address?.line,
+          image,
+        };
+        if (currentUserId) {
+          const userSettings = getUserSettings(currentUserId);
+          if (userSettings.immoscout_details) {
+            return await pushDetails(listing);
+          }
+        }
+        return listing;
+      }),
+  );
+}
+
+async function pushDetails(listing) {
+  const detailed = await fetch(`https://api.mobile.immobilienscout24.de/expose/${listing.id}`, {
+    headers: {
+      'User-Agent': 'ImmoScout_27.3_26.0_._',
+      'Content-Type': 'application/json',
+    },
+  });
+  if (!detailed.ok) {
+    logger.error('Error fetching listing details from ImmoScout Mobile API:', detailed.statusText);
+    return listing;
+  }
+  const detailBody = await detailed.json();
+
+  listing.description = buildDescription(detailBody);
+
+  return listing;
+}
+
+function buildDescription(detailBody) {
+  const sections = detailBody.sections || [];
+  const contact = detailBody.contact || {};
+  const cData = contact?.contactData || {};
+  const agentName = cData?.agent?.name || '';
+  const agentCompany = cData?.agent?.company || '';
+  const stars = cData?.agent?.rating?.numberOfStars || '';
+  const phoneNumbers = contact?.phoneNumbers || [];
+  const phoneNumbersMapped = phoneNumbers
+    .map((p) => `${p.label}: ${p.text}`)
+    .join('\n')
+    .trim();
+
+  const attributes = sections
+    .filter((s) => s.type === 'ATTRIBUTE_LIST')
+    .flatMap((s) => s.attributes)
+    .filter((attr) => attr.label && attr.text)
+    .map((attr) => `${attr.label} ${attr.text}`)
+    .join('\n');
+
+  const freeText = sections
+    .filter((s) => s.type === 'TEXT_AREA')
+    .map((s) => {
+      return `${s.title}\n${s.text}`;
+    })
+    .join('\n\n');
+
+  return (
+    `Agent: ${agentName ? agentName : 'Unbekannt'} ${agentCompany ? `(${agentCompany}) ` : ''}${stars ? `- ${stars} stars` : ''}\n` +
+    (phoneNumbersMapped ? `Phone Numbers:\n${phoneNumbersMapped}` : '') +
+    '\n\n' +
+    attributes.trim() +
+    '\n\n' +
+    freeText.trim()
+  );
 }
 
 async function isListingActive(link) {
@@ -137,6 +202,7 @@ export const init = (sourceConfig, blacklist) => {
   config.enabled = sourceConfig.enabled;
   config.url = convertWebToMobile(sourceConfig.url);
   appliedBlackList = blacklist || [];
+  currentUserId = sourceConfig.userId || null;
 };
 export const metaInformation = {
   name: 'Immoscout',

lib/services/geocoding/client/nominatimClient.js

@@ -67,6 +67,7 @@ async function doGeocode(address) {
   try {
     const response = await fetch(url, {
       agent,
+      timeout: 60000,
       headers: {
         'User-Agent': userAgent,
       },

lib/services/jobs/jobExecutionService.js

@@ -166,7 +166,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
       for (const prov of jobProviders) {
         try {
           const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
-          matchedProvider.init(prov, job.blacklist);
+          matchedProvider.init({ ...prov, userId: job.userId }, job.blacklist);
 
           if (browser && !browser.isConnected()) {
             logger.debug('Browser is disconnected, nullifying to launch a new one.');

ui/src/services/state/store.js

@@ -304,6 +304,20 @@ export const useFredyState = create(
               throw Exception;
             }
           },
+          async setImmoscoutDetails(enabled) {
+            try {
+              await xhrPost('/api/user/settings/immoscout-details', { immoscout_details: enabled });
+              set((state) => ({
+                userSettings: {
+                  ...state.userSettings,
+                  settings: { ...state.userSettings.settings, immoscout_details: enabled },
+                },
+              }));
+            } catch (Exception) {
+              console.error('Error while trying to update immoscout details setting. Error:', Exception);
+              throw Exception;
+            }
+          },
         },
       };
 

ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx

@@ -27,10 +27,13 @@ const sortAdapter = (a, b) => {
 const validate = (selectedAdapter) => {
   const results = [];
   for (let uiElement of Object.values(selectedAdapter.fields || [])) {
-    if (uiElement.value == null && !uiElement.optional) {
+    if (uiElement.value == null && !uiElement.optional && uiElement.type !== 'boolean') {
       results.push('All fields are mandatory and must be set.');
       continue;
     }
+    if (uiElement.type === 'boolean' && typeof uiElement.value !== 'boolean') {
+      uiElement.value = false;
+    }
     if (uiElement.type === 'number') {
       const numberValue = parseFloat(uiElement.value);
       if (isNaN(numberValue) || numberValue < 0) {
@@ -153,12 +156,15 @@ export default function NotificationAdapterMutator({
       return (
         <Form key={key}>
           {uiElement.type === 'boolean' ? (
-            <Switch
+            <div style={{ display: 'flex', alignItems: 'center', gap: '16px' }}>
+              <Switch
               checked={uiElement.value || false}
               onChange={(checked) => {
                 setValue(selectedAdapter, uiElement, key, checked);
               }}
             />
+              {uiElement.label}
+            </div>
           ) : (
             <Form.Input
               style={{ width: '100%' }}

ui/src/views/userSettings/UserSettings.jsx

@@ -4,8 +4,8 @@
  */
 
 import { useEffect, useState, useMemo } from 'react';
-import { Divider, Button, AutoComplete, Toast, Banner } from '@douyinfe/semi-ui-19';
-import { IconSave, IconHome } from '@douyinfe/semi-icons';
+import { Divider, Button, AutoComplete, Toast, Banner, Switch } from '@douyinfe/semi-ui-19';
+import { IconSave, IconHome, IconSearch } from '@douyinfe/semi-icons';
 import { useSelector, useActions, useIsLoading } from '../../services/state/store';
 import { xhrGet } from '../../services/xhr';
 import { SegmentPart } from '../../components/segment/SegmentPart';
@@ -14,6 +14,7 @@ import debounce from 'lodash/debounce';
 const UserSettings = () => {
   const actions = useActions();
   const homeAddress = useSelector((state) => state.userSettings.settings.home_address);
+  const immoscoutDetails = useSelector((state) => state.userSettings.settings.immoscout_details);
   const [address, setAddress] = useState(homeAddress?.address || '');
   const [coords, setCoords] = useState(homeAddress?.coords || null);
   const saving = useIsLoading(actions.userSettings.setHomeAddress);
@@ -84,6 +85,33 @@ const UserSettings = () => {
         </div>
       </SegmentPart>
       <Divider />
+      <SegmentPart
+        name="ImmoScout Details"
+        Icon={IconSearch}
+        helpText="When enabled, Fredy will fetch additional details (description, attributes, agent info) for each listing from ImmoScout. This provides richer notifications but makes an extra API call per listing."
+      >
+        <Banner
+          type="warning"
+          description="Enabling this feature significantly increases the number of API requests to ImmoScout. This raises the likelihood of being detected and rate-limited or blocked. Use at your own risk."
+          closeIcon={null}
+          style={{ marginBottom: '12px', maxWidth: '600px' }}
+        />
+        <div style={{ display: 'flex', alignItems: 'center', gap: '12px' }}>
+          <Switch
+            checked={!!immoscoutDetails}
+            onChange={async (checked) => {
+              try {
+                await actions.userSettings.setImmoscoutDetails(checked);
+                Toast.success('ImmoScout details setting updated.');
+              } catch {
+                Toast.error('Failed to update setting.');
+              }
+            }}
+          />
+          <span>Fetch detailed ImmoScout listings</span>
+        </div>
+      </SegmentPart>
+      <Divider />
       <div style={{ marginTop: '20px' }}>
         <Button icon={<IconSave />} theme="solid" type="primary" onClick={handleSave} loading={saving}>
           Save Settings