External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Concurrent-Jobs (#7)

Browse Source 
Fredy 2.0.0 introduces the concept of search jobs. You can now configure multiple of these jobs running in the same instance of Fredy. Also a new API has been created 🎉
This commit is contained in:
Christian Kellner
2020-02-26 09:05:20 +01:00
committed by GitHub
parent 52a32f7453
commit 770d30af95
50 changed files with 3297 additions and 1564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
lib/provider/einsAImmobilien.js
View File

@@ -1,11 +1,11 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
let appliedBlackList = [];
function normalize(o) {
let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
if(o.rooms != null){
size+=` / / ${o.rooms.trim()}`;
if (o.rooms != null) {
size += ` / / ${o.rooms.trim()}`;
}
const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
@@ -13,16 +13,15 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const einsAImmobilien = {
name: 'einsAImmobilien',
enabled: config.sources.einsAImmobilien.enabled,
url: config.sources.einsAImmobilien.url,
const config = {
enabled: null,
url: null,
crawlContainer: '.tabelle',
crawlFields: {
id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
@@ -37,4 +36,13 @@ const einsAImmobilien = {
filter: applyBlacklist
};
module.exports = new Fredy(einsAImmobilien);
exports.init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'einsAImmobilien';
exports.config = config;

26
lib/provider/immonet.js
View File

@@ -1,7 +1,7 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
let appliedBlackList = [];
function normalize(o) {
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
@@ -13,16 +13,15 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const immonet = {
name: 'immonet',
enabled: config.sources.immonet.enabled,
url: config.sources.immonet.url,
const config = {
enabled: null,
url: null,
crawlContainer: '#result-list-stage .item',
crawlFields: {
id: '@id',
@@ -37,4 +36,13 @@ const immonet = {
filter: applyBlacklist
};
module.exports = new Fredy(immonet);
exports.init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'immonet';
exports.config = config;

32
lib/provider/immoscout.js
View File

@@ -1,7 +1,7 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
let appliedBlackList = [];
function normalize(o) {
const title = o.title.replace('NEU', '');
const address = (o.address || '').replace(/\(.*\),.*$/, '').trim();
@@ -10,25 +10,33 @@ function normalize(o) {
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, config.blacklist);
return !utils.isOneOf(o.title, appliedBlackList);
}
const immoscout = {
name: 'immoscout',
enabled: config.sources.immoscout.enabled,
url: config.sources.immoscout.url,
const config = {
enabled: null,
url: null,
crawlContainer: '#resultListItems li.result-list__listing',
crawlFields: {
crawlFields: {
id: '.result-list-entry@data-obid | int',
price: '.result-list-entry .result-list-entry__criteria .grid-item:first-child dd | removeNewline | trim',
size: '.result-list-entry .result-list-entry__criteria .grid-item:nth-child(2) dd | removeNewline | trim',
title: '.result-list-entry .result-list-entry__brand-title-container h5 | removeNewline | trim',
link: '.result-list-entry .result-list-entry__brand-title-container@href',
address: '.result-list-entry .result-list-entry__map-link'
},
},
paginate: '#pager .align-right a@href',
normalize: normalize,
filter: applyBlacklist
normalize: normalize,
filter: applyBlacklist
};
module.exports = new Fredy(immoscout);
exports.init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'immoscout';
exports.config = config;

26
lib/provider/immowelt.js
View File

@@ -1,7 +1,7 @@
const Fredy = require('../fredy');
const config = require('../../conf/config.json');
const utils = require('../utils');
let appliedBlackList = [];
function normalize(o) {
const size = o.size == null ? '--- m²' : o.size.split('Wohnfläche')[1].replace(' (ca.) ', '');
const address = o.address;
@@ -10,16 +10,15 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return titleNotBlacklisted && descNotBlacklisted;
}
const immowelt = {
name: 'immowelt',
enabled: config.sources.immowelt.enabled,
url: config.sources.immowelt.url,
const config = {
enabled: null,
url: null,
crawlContainer: '.immoliste .js-object.listitem_wrap ',
crawlFields: {
id: '@data-estateid | int',
@@ -34,4 +33,13 @@ const immowelt = {
filter: applyBlacklist
};
module.exports = new Fredy(immowelt);
exports.init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'immowelt';
exports.config = config;

31
lib/provider/kalaydo.js
View File

@@ -1,7 +1,8 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
let appliedBlackList = [];
let appliedBlacklistedDistricts = [];
function normalize(o) {
const id = o.id
.split('/')
@@ -16,19 +17,18 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const isBlacklistedDistrict =
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.title, config.blacklistedDistrics);
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.title, appliedBlacklistedDistricts);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
const kalaydo = {
name: 'kalaydo',
enabled: config.sources.kalaydo.enabled,
url: config.sources.kalaydo.url,
const config = {
enabled: null,
url: null,
crawlContainer: '#resultList .resultitem-content-container',
crawlFields: {
id: '.resultitem-content-container a@href',
@@ -43,4 +43,15 @@ const kalaydo = {
filter: applyBlacklist
};
module.exports = new Fredy(kalaydo);
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
appliedBlacklistedDistricts = blacklistedDistricts;
};
//must match the id of the source given in the config!
exports.id = () => 'kalaydo';
exports.config = config;

31
lib/provider/kleinanzeigen.js
View File

@@ -1,7 +1,8 @@
const Fredy = require('../fredy');
const config = require('../../conf/config.json');
const utils = require('../utils');
let appliedBlackList = [];
let appliedBlacklistedDistricts = [];
function normalize(o) {
const size = o.size || '--- m²';
@@ -9,18 +10,17 @@ function normalize(o) {
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
const isBlacklistedDistrict =
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
appliedBlacklistedDistricts.length === 0 ? false : utils.isOneOf(o.description, appliedBlacklistedDistricts);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
}
const kleinanzeigen = {
name: 'kleinanzeigen',
enabled: config.sources.kleinanzeigen.enabled,
url: config.sources.kleinanzeigen.url,
const config = {
enabled: null,
url: null,
crawlContainer: '#srchrslt-adtable .ad-listitem',
crawlFields: {
id: '.aditem@data-adid | int',
@@ -36,4 +36,15 @@ const kleinanzeigen = {
filter: applyBlacklist
};
module.exports = new Fredy(kleinanzeigen);
exports.init = (sourceConfig, blacklist, blacklistedDistricts) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlacklistedDistricts = blacklistedDistricts;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'kleinanzeigen';
exports.config = config;

24
lib/provider/neubauKompass.js
View File

@@ -1,19 +1,18 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
return !utils.isOneOf(o.title, config.blacklist);
return !utils.isOneOf(o.title, appliedBlackList);
}
const neubauKompass = {
name: 'neubauKompass',
enabled: config.sources.neubauKompass.enabled,
url: config.sources.neubauKompass.url,
const config = {
enabled: null,
url: null,
crawlContainer: '.row article',
crawlFields: {
id: '@id',
@@ -26,4 +25,13 @@ const neubauKompass = {
filter: applyBlacklist
};
module.exports = new Fredy(neubauKompass);
exports.init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'neubauKompass';
exports.config = config;

38
lib/provider/wgGesucht.js
View File

@@ -1,34 +1,42 @@
const config = require('../../conf/config.json');
const Fredy = require('../fredy');
const utils = require('../utils');
let appliedBlackList = [];
function normalize(o) {
return o;
}
function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
}
const wgGesucht = {
name: 'wgGesucht',
enabled: config.sources.wgGesucht.enabled,
url: config.sources.wgGesucht.url,
crawlContainer: '#main_column .panel:not(.display-none):not(.noprint)',
const config = {
enabled: null,
url: null,
crawlContainer: '#main_column .wgg_card',
crawlFields: {
id: '@data-id',
details: ' .list-details-costs-col |removeNewline |trim',
title: '.headline .detailansicht |removeNewline |trim',
description: '.list-details-category-location |removeNewline |trim',
link: '.headline .detailansicht@href'
details: '.row .noprint .col-xs-11 |removeNewline |trim',
price: '.middle .col-xs-3 |removeNewline |trim',
size: '.middle .text-right |removeNewline |trim',
title: '.truncate_title a |removeNewline |trim',
link: '.truncate_title a@href'
},
paginate: '.pagination-sm:first a:last@href',
normalize: normalize,
filter: applyBlacklist
};
module.exports = new Fredy(wgGesucht);
exports.init = (sourceConfig, blacklist) => {
config.enabled = sourceConfig.enabled;
config.url = sourceConfig.url;
appliedBlackList = blacklist;
};
//must match the id of the source given in the config!
exports.id = () => 'wgGesucht';
exports.config = config;