UI (#15)

Adding new Admin UI. Updating Fredy to V3.0.0 as it has been a large rewrite. Thanks for all contributions and help on the way!
2026-06-16 12:31:07 +00:00 · 2021-01-21 16:09:23 +01:00
parent 8185bfe818
commit b2847f6834
124 changed files with 9768 additions and 1495 deletions
--- a/lib/provider/einsAImmobilien.js
+++ b/lib/provider/einsAImmobilien.js
@@ -20,7 +20,6 @@ function applyBlacklist(o) {
 }

 const config = {
-  enabled: null,
  url: null,
  crawlContainer: '.tabelle',
  crawlFields: {
@@ -29,20 +28,23 @@ const config = {
    size: '.tabelle .inner_object_data .data_boxes div:nth-child(1)',
    rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)',
    title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
-    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim'
+    description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
  },
  paginate: '.pagination_blocks div:last a@href',
  normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };

 exports.init = (sourceConfig, blacklist) => {
  config.enabled = sourceConfig.enabled;
  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };

-//must match the id of the source given in the config!
-exports.id = () => 'einsAImmobilien';
+exports.metaInformation = {
+  name: '1a Immobilien',
+  baseUrl: 'https://www.1a-immobilienmarkt.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};

 exports.config = config;
--- a/lib/provider/immonet.js
+++ b/lib/provider/immonet.js
@@ -20,7 +20,6 @@ function applyBlacklist(o) {
 }

 const config = {
-  enabled: null,
  url: null,
  crawlContainer: '#result-list-stage .item',
  crawlFields: {
@@ -29,20 +28,23 @@ const config = {
    size: 'div[id*="selArea_"] | trim',
    title: '.item a img@title',
    link: 'a[id*="lnkImgToDetails_"]@href',
-    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim'
+    address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim',
  },
  paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
  normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };

 exports.init = (sourceConfig, blacklist) => {
  config.enabled = sourceConfig.enabled;
  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };

-//must match the id of the source given in the config!
-exports.id = () => 'immonet';
+exports.metaInformation = {
+  name: 'Immonet',
+  baseUrl: 'https://www.immonet.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};

 exports.config = config;
--- a/lib/provider/immoscout.js
+++ b/lib/provider/immoscout.js
@@ -1,42 +0,0 @@
-const utils = require('../utils');
-
-let appliedBlackList = [];
-
-function normalize(o) {
-  const title = o.title.replace('NEU', '');
-  const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
-
-  return Object.assign(o, { title, address });
-}
-
-function applyBlacklist(o) {
-  return !utils.isOneOf(o.title, appliedBlackList);
-}
-
-const config = {
-  enabled: null,
-  url: null,
-  crawlContainer: '#resultListItems li.result-list__listing',
-  crawlFields: {
-    id: '.result-list-entry@data-obid | int',
-    price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
-    size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
-    title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
-    link: '.result-list-entry .result-list-entry__brand-title-container@href',
-    address: '.result-list-entry .result-list-entry__map-link'
-  },
-  paginate: '#pager .align-right a@href',
-  normalize: normalize,
-  filter: applyBlacklist
-};
-
-exports.init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
-};
-
-//must match the id of the source given in the config!
-exports.id = () => 'immoscout';
-
-exports.config = config;
--- a/lib/provider/immowelt.js
+++ b/lib/provider/immowelt.js
@@ -17,7 +17,6 @@ function applyBlacklist(o) {
 }

 const config = {
-  enabled: null,
  url: null,
  crawlContainer: '.immoliste .js-object.listitem_wrap ',
  crawlFields: {
@@ -26,20 +25,22 @@ const config = {
    size: '.js-object.listitem_wrap .hardfacts_3 div:nth-child(2)| removeNewline | trim',
    title: '.listcontent.clear h2',
    link: 'a@href',
-    address: '.listcontent .details .listlocation| removeNewline | trim'
+    address: '.listcontent .details .listlocation| removeNewline | trim',
  },
  paginate: '#pnlPaging #nlbPlus@href',
  normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };

 exports.init = (sourceConfig, blacklist) => {
  config.enabled = sourceConfig.enabled;
  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
+};
+exports.metaInformation = {
+  name: 'Immowelt',
+  baseUrl: 'https://www.immowelt.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
 };

-//must match the id of the source given in the config!
-exports.id = () => 'immowelt';
-
 exports.config = config;
--- a/lib/provider/kalaydo.js
+++ b/lib/provider/kalaydo.js
@@ -1,56 +0,0 @@
-const utils = require('../utils');
-
-let appliedBlackList = [];
-let appliedBlacklistedDistricts = [];
-
-function normalize(o) {
-  const id = o.id
-    .split('/')
-    .filter(Boolean)
-    .reverse()[0];
-  const price = o.price == null ? 'unknown' : o.price.trim().replace('Preis', '');
-  let size = o.size == null ? 'unknown' : o.size.replace('Wohnfläche: ', '').replace('ca. ', '');
-  size += ' / ' + o.rooms;
-  const address = '---';
-
-  return Object.assign(o, { id, price, size, address });
-}
-
-function applyBlacklist(o) {
-  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
-  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
-
-  const isBlacklistedDistrict =
-    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
-
-  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
-}
-
-const config = {
-  enabled: null,
-  url: null,
-  crawlContainer: '#resultList .resultitem-content-container',
-  crawlFields: {
-    id: '.resultitem-content-container a@href',
-    price: '.description .rent | removeNewline | trim',
-    title: '.resultitem-content-container a@title',
-    link: '.resultitem-content-container a@href',
-    rooms: '.resultitem-content-container .no-of-rooms | removeNewline  | trim',
-    size: '.resultitem-content-container .living-area | removeNewline | trim'
-  },
-  paginate: '.markt_pagination_pageLinkNext .markt_pagination_link@href',
-  normalize: normalize,
-  filter: applyBlacklist
-};
-
-exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
-  config.enabled = sourceConfig.enabled;
-  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
-  appliedBlacklistedDistricts = blacklistedDistricts;
-};
-
-//must match the id of the source given in the config!
-exports.id = () => 'kalaydo';
-
-exports.config = config;
--- a/lib/provider/kleinanzeigen.js
+++ b/lib/provider/kleinanzeigen.js
@@ -13,13 +13,12 @@ function applyBlacklist(o) {
  const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
  const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
  const isBlacklistedDistrict =
-      appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
+    appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);

  return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
 }

 const config = {
-  enabled: null,
  url: null,
  crawlContainer: '#srchrslt-adtable .ad-listitem',
  crawlFields: {
@@ -29,22 +28,24 @@ const config = {
    title: '.aditem-main .text-module-begin a | removeNewline | trim',
    link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
    description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
-    address: '.aditem-details | trim | removeNewline'
+    address: '.aditem-details | trim | removeNewline',
  },
  paginate: '#srchrslt-pagination .pagination-next@href',
  normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
+};
+
+exports.metaInformation = {
+  name: 'Ebay Kleinanzeigen',
+  baseUrl: 'https://www.ebay-kleinanzeigen.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
 };

 exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
-    config.enabled = sourceConfig.enabled;
-    config.url = sourceConfig.url;
-    appliedBlacklistedDistricts = blacklistedDistricts;
-    appliedBlackList = blacklist;
+  config.enabled = sourceConfig.enabled;
+  config.url = sourceConfig.url;
+  appliedBlacklistedDistricts = blacklistedDistricts || [];
+  appliedBlackList = blacklist || [];
 };

-//must match the id of the source given in the config!
-exports.id = () => 'kleinanzeigen';
-
 exports.config = config;
-
--- a/lib/provider/neubauKompass.js
+++ b/lib/provider/neubauKompass.js
@@ -11,7 +11,6 @@ function applyBlacklist(o) {
 }

 const config = {
-  enabled: null,
  url: null,
  crawlContainer: '.nbk-container >div article',
  crawlFields: {
@@ -28,10 +27,13 @@ const config = {
 exports.init = (sourceConfig, blacklist) => {
  config.enabled = sourceConfig.enabled;
  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };

-//must match the id of the source given in the config!
-exports.id = () => 'neubauKompass';
+exports.metaInformation = {
+  name: 'Neubau Kompass',
+  baseUrl: 'https://www.neubaukompass.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};

 exports.config = config;
--- a/lib/provider/wgGesucht.js
+++ b/lib/provider/wgGesucht.js
@@ -14,7 +14,6 @@ function applyBlacklist(o) {
 }

 const config = {
-  enabled: null,
  url: null,
  crawlContainer: '#main_column .wgg_card',
  crawlFields: {
@@ -23,20 +22,23 @@ const config = {
    price: '.middle .col-xs-3 |removeNewline |trim',
    size: '.middle .text-right |removeNewline |trim',
    title: '.truncate_title a |removeNewline |trim',
-    link: '.truncate_title a@href'
+    link: '.truncate_title a@href',
  },
  paginate: '.pagination-sm:first a:last@href',
  normalize: normalize,
-  filter: applyBlacklist
+  filter: applyBlacklist,
 };

 exports.init = (sourceConfig, blacklist) => {
  config.enabled = sourceConfig.enabled;
  config.url = sourceConfig.url;
-  appliedBlackList = blacklist;
+  appliedBlackList = blacklist || [];
 };

-//must match the id of the source given in the config!
-exports.id = () => 'wgGesucht';
+exports.metaInformation = {
+  name: 'Wg gesucht',
+  baseUrl: 'https://www.wg-gesucht.de/',
+  id: __filename.slice(__dirname.length + 1, -3),
+};

 exports.config = config;