fixing tests

lib/FredyRuntime.js

@@ -87,7 +87,10 @@ class FredyRuntime {
     return listings.map(this._providerConfig.normalize);
   }
   _filter(listings) {
-    return listings.filter(this._providerConfig.filter);
+    //only return those where all the fields have been found
+    const keys = Object.keys(this._providerConfig.crawlFields);
+    const filteredListings = listings.filter((item) => keys.every((key) => key in item));
+    return filteredListings.filter(this._providerConfig.filter);
   }
   _findNew(listings) {
     const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);

lib/notification/adapter/apprise.js

@@ -8,7 +8,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
   const jobName = job == null ? jobKey : job.name;
   const promises = newListings.map((newListing) => {
     const title = `${jobName} at ${serviceName}: ${newListing.title}`;
-    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\Link: ${newListing.link}`;
+    const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nink: ${newListing.link}`;
     return fetch(server, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },

lib/provider/einsAImmobilien.js

@@ -1,4 +1,4 @@
-import utils, {buildHash} from '../utils.js';
+import utils, { buildHash } from '../utils.js';
 let appliedBlackList = [];
 
 function normalize(o) {
@@ -7,10 +7,28 @@ function normalize(o) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
-  const id = buildHash(o.id, o.price);
+  const price = normalizePrice(o.price);
-  return Object.assign(o, { id, size, link });
+  const id = buildHash(o.id, price);
+  return Object.assign(o, { id, price, size, link });
 }
 
+/**
+ * einsAImmobilien sometimes use a weird pricing label such as `775.700,00 EUR Kaufpreis ab 2.475 € mtl`.
+ * Make sure to extract only the actual price out of the string.
+ * @param price
+ * @returns {*}
+ */
+function normalizePrice(price) {
+  if (price == null) {
+    return null;
+  }
+  const regex = /(\d{1,3}(?:\.\d{3})*,\d{2})\s?(EUR|€)/g;
+  const result = price.match(regex);
+  if (result == null || result.length === 0) {
+    return price;
+  }
+  return result[0];
+}
 function applyBlacklist(o) {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
@@ -27,7 +45,6 @@ const config = {
     size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
     rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
     title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
   },
   normalize: normalize,
   filter: applyBlacklist,

lib/provider/immowelt.js

@@ -1,42 +1,43 @@
-import utils, {buildHash} from '../utils.js';
+import utils, { buildHash } from '../utils.js';
 
 let appliedBlackList = [];
 
 function normalize(o) {
-    const id = buildHash(o.id, o.price);
+  const id = buildHash(o.id, o.price);
-    return Object.assign(o, {id});
+  return Object.assign(o, { id });
 }
 
 function applyBlacklist(o) {
-    const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
+  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-    const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
+  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-    return titleNotBlacklisted && descNotBlacklisted;
+  return titleNotBlacklisted && descNotBlacklisted;
 }
 
 const config = {
-    url: null,
+  url: null,
-    crawlContainer: 'div[data-testid="serp-card-testid"]',
+  crawlContainer:
-    sortByDateParam: 'sd=DESC&sf=TIMESTAMP',
+    'div[data-testid="serp-card-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"])',
-    crawlFields: {
+  sortByDateParam: 'order=DateDesc',
-        id: 'a@id',
+  crawlFields: {
-        price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
+    id: 'a@id',
-        size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
+    price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
-        title: '.css-1cbj9xw',
+    size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
-        link: 'a@href',
+    title: '.css-1cbj9xw',
-        address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
+    link: 'a@href',
-    },
+    address: 'div[data-testid="cardmfe-description-box-address"] | removeNewline | trim',
-    paginate: '#pnlPaging #nlbPlus@href',
+  },
-    normalize: normalize,
+  paginate: '#pnlPaging #nlbPlus@href',
-    filter: applyBlacklist,
+  normalize: normalize,
+  filter: applyBlacklist,
 };
 export const init = (sourceConfig, blacklist) => {
-    config.enabled = sourceConfig.enabled;
+  config.enabled = sourceConfig.enabled;
-    config.url = sourceConfig.url;
+  config.url = sourceConfig.url;
-    appliedBlackList = blacklist || [];
+  appliedBlackList = blacklist || [];
 };
 export const metaInformation = {
-    name: 'Immowelt',
+  name: 'Immowelt',
-    baseUrl: 'https://www.immowelt.de/',
+  baseUrl: 'https://www.immowelt.de/',
-    id: 'immowelt',
+  id: 'immowelt',
 };
-export {config};
+export { config };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fredy",
-  "version": "10.0.0",
+  "version": "10.1.1",
   "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
   "scripts": {
     "start": "node index.js",

test/provider/testProvider.json

@@ -13,7 +13,7 @@
     "enabled": true
   },
   "immowelt": {
-    "url": "https://www.immowelt.de/liste/duesseldorf/wohnungen/kaufen?d=true&rmi=3&sd=DESC&sf=TIMESTAMP&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
     "enabled": true
   },
   "immoscout": {

test/queryStringMutator/testData.json

@@ -1,7 +1,7 @@
 [
   {
-    "url": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=PRIMARY_PRICE_AMOUNT&sp=1",
+    "url": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350",
-    "shouldBecome": "https://www.immowelt.de/liste/40589/wohnungen/mieten?d=true&sd=DESC&sf=TIMESTAMP&sp=1",
+    "shouldBecome": "https://www.immowelt.de/classified-search?distributionTypes=Buy,Buy_Auction,Compulsory_Auction&estateTypes=House,Apartment&locations=AD08DE2350&order=DateDesc",
     "id": "immowelt"
   },
   {