Making Fredy an ESM project (#70)

Making Fredy an ESM project
This commit is contained in:
Christian Kellner
2023-03-13 13:42:43 +01:00
committed by GitHub
parent 7d0ec72a0c
commit 2c5eceb0c1
69 changed files with 862 additions and 1567 deletions

View File

@@ -1,24 +1,18 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
if (o.rooms != null) {
size += ` / / ${o.rooms.trim()}`;
}
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
return Object.assign(o, { size, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.tabelle',
@@ -34,17 +28,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: '1a Immobilien',
baseUrl: 'https://www.1a-immobilienmarkt.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'einsAImmobilien',
};
exports.config = config;
export { config };

View File

@@ -1,15 +1,11 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function shortenLink(link) {
return link.substring(0, link.indexOf('?'));
}
function parseId(shortenedLink) {
return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
}
function normalize(o) {
const id = parseId(shortenLink(o.link));
const size = o.size || 'N/A m²';
@@ -19,14 +15,11 @@ function normalize(o) {
const link = shortenLink(o.link);
return Object.assign(o, { id, price, size, title, address, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.estates_list .list_immo a._ref',
@@ -43,17 +36,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immobilien.de',
baseUrl: 'https://www.immobilien.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immobilienDe',
};
exports.config = config;
export { config };

View File

@@ -1,7 +1,5 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
@@ -13,14 +11,11 @@ function normalize(o) {
const link = `https://www.immonet.de/angebot/${id}`;
return Object.assign(o, { id, address, price, size, title, link });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '#result-list-stage .item',
@@ -36,17 +31,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immonet',
baseUrl: 'https://www.immonet.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immonet',
};
exports.config = config;
export { config };

View File

@@ -1,22 +1,17 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function nullOrEmpty(val) {
return val == null || val.length === 0;
}
function normalize(o) {
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
return Object.assign(o, { title, address, link });
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
crawlContainer: '#resultListItems li.result-list__listing',
@@ -33,17 +28,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immoscout',
baseUrl: 'https://www.immobilienscout24.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immoscout',
};
exports.config = config;
export { config };

View File

@@ -1,7 +1,5 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
const size = o.size || 'N/A m²';
@@ -12,14 +10,11 @@ function normalize(o) {
const description = o.description;
return Object.assign(o, { id, address, price, size, title, link, description });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '.js-serp-item',
@@ -36,17 +31,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immo Südwest Presse',
baseUrl: 'https://immo.swp.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immoswp',
};
exports.config = config;
export { config };

View File

@@ -1,18 +1,13 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: "div[class^='EstateItem-']",
@@ -29,16 +24,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Immowelt',
baseUrl: 'https://www.immowelt.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'immowelt',
};
exports.config = config;
export { config };

View File

@@ -1,23 +1,17 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
let appliedBlacklistedDistricts = [];
function normalize(o) {
const size = o.size || '--- m²';
return Object.assign(o, { size });
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const isBlacklistedDistrict =
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '#srchrslt-adtable .ad-listitem ',
@@ -36,18 +30,15 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.metaInformation = {
export const metaInformation = {
name: 'Ebay Kleinanzeigen',
baseUrl: 'https://www.ebay-kleinanzeigen.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'kleinanzeigen',
};
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlacklistedDistricts = blacklistedDistricts || [];
appliedBlackList = blacklist || [];
};
exports.config = config;
export { config };

View File

@@ -1,15 +1,11 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, appliedBlackList);
}
const config = {
url: null,
crawlContainer: '.nbk-container >div article',
@@ -25,17 +21,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Neubau Kompass',
baseUrl: 'https://www.neubaukompass.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'neubauKompass',
};
exports.config = config;
export { config };

View File

@@ -1,18 +1,13 @@
const utils = require('../utils');
import utils from '../utils.js';
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
}
const config = {
url: null,
crawlContainer: '#main_column .wgg_card',
@@ -28,17 +23,14 @@ const config = {
normalize: normalize,
filter: applyBlacklist,
};
exports.init = (sourceConfig, blacklist) => {
export const init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist || [];
};
exports.metaInformation = {
export const metaInformation = {
name: 'Wg gesucht',
baseUrl: 'https://www.wg-gesucht.de/',
id: __filename.slice(__dirname.length + 1, -3),
id: 'wgGesucht',
};
exports.config = config;
export { config };