regex for einsAImmobilien price normalization | filter listings that does not have all required keys

lib/FredyRuntime.js

@@ -87,7 +87,10 @@ class FredyRuntime {
     return listings.map(this._providerConfig.normalize);
   }
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);

lib/notification/adapter/apprise.js

@@ -8,7 +8,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const jobName = job == null ? jobKey : job.name;
   const promises = newListings.map((newListing) => {
     const title = `${jobName} at ${serviceName}: ${newListing.title}`;
-    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\Link: ${newListing.link}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
     return fetch(server, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },

lib/provider/einsAImmobilien.js

@@ -1,4 +1,4 @@
-import utils, {buildHash} from '../utils.js';
+import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
 
 function normalize(o) {
@@ -7,10 +7,28 @@ function normalize(o) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-  const id = buildHash(o.id, o.price);
+  const price = normalizePrice(o.price);
-  return Object.assign(o, { id, size, link });
+  const id = buildHash(o.id, price);
+  return Object.assign(o, { id, price, size, link });
 }
 
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
+}
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
@@ -27,7 +45,6 @@ const config = {
     size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
     rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
     title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "10.0.0",
+  "version": "10.1.0",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",