External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixing neubaukompass && updating dependencies

Browse Source
This commit is contained in:
Christian Kellner
2019-10-24 21:25:19 +02:00
parent 8e225f761c
commit 9106144d9c
8 changed files with 341 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
lib/provider/immonet.js
View File

@@ -3,38 +3,38 @@ const Fredy = require('../fredy');
const utils = require('../utils'); const utils = require('../utils');
function normalize(o) { function normalize(o) {
const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length)); const id = parseInt(o.id.substring(o.id.indexOf('_') + 1, o.id.length));
const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²'; const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
const price = o.price.replace('Kaufpreis ', ''); const price = o.price.replace('Kaufpreis ', '');
const address = o.address.split(' • ')[1]; const address = o.address.split(' • ')[1];
const title = o.title || 'No title available'; const title = o.title || 'No title available';
return Object.assign(o, {id, address, price, size, title}); return Object.assign(o, { id, address, price, size, title });
} }
function applyBlacklist(o) { function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist); const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist); const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
return titleNotBlacklisted && descNotBlacklisted; return titleNotBlacklisted && descNotBlacklisted;
} }
const immonet = { const immonet = {
name: 'immonet', name: 'immonet',
enabled: config.sources.immonet.enabled, enabled: config.sources.immonet.enabled,
url: config.sources.immonet.url, url: config.sources.immonet.url,
crawlContainer: '#result-list-stage .item', crawlContainer: '#result-list-stage .item',
crawlFields: { crawlFields: {
id: '@id', id: '@id',
price: 'div[id*="selPrice_"] | trim', price: 'div[id*="selPrice_"] | trim',
size: 'div[id*="selArea_"] | trim', size: 'div[id*="selArea_"] | trim',
title: '.item a img@title', title: '.item a img@title',
link: 'a[id*="lnkImgToDetails_"]@href', link: 'a[id*="lnkImgToDetails_"]@href',
address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim' address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim'
}, },
paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href', paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist filter: applyBlacklist
}; };
module.exports = new Fredy(immonet); module.exports = new Fredy(immonet);

46
lib/provider/kleinanzeigen.js
View File

@@ -3,37 +3,37 @@ const config = require('../../conf/config.json');
const utils = require('../utils'); const utils = require('../utils');
function normalize(o) { function normalize(o) {
const size = o.size || '--- m²'; const size = o.size || '--- m²';
return Object.assign(o, {size}); return Object.assign(o, { size });
} }
function applyBlacklist(o) { function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist); const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist); const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
const isBlacklistedDistrict = const isBlacklistedDistrict =
config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics); config.blacklistedDistrics.length === 0 ? false : utils.isOneOf(o.description, config.blacklistedDistrics);
return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted; return !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
} }
const kleinanzeigen = { const kleinanzeigen = {
name: 'kleinanzeigen', name: 'kleinanzeigen',
enabled: config.sources.kleinanzeigen.enabled, enabled: config.sources.kleinanzeigen.enabled,
url: config.sources.kleinanzeigen.url, url: config.sources.kleinanzeigen.url,
crawlContainer: '#srchrslt-adtable .ad-listitem', crawlContainer: '#srchrslt-adtable .ad-listitem',
crawlFields: { crawlFields: {
id: '.aditem@data-adid | int', id: '.aditem@data-adid | int',
price: '.aditem-details strong | removeNewline | trim', price: '.aditem-details strong | removeNewline | trim',
size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim', size: '.aditem-main .text-module-end span:nth-child(2) | removeNewline | trim',
title: '.aditem-main .text-module-begin a | removeNewline | trim', title: '.aditem-main .text-module-begin a | removeNewline | trim',
link: '.aditem-main .text-module-begin a@href | removeNewline | trim', link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
description: '.aditem-main p:not(.text-module-end) | removeNewline | trim', description: '.aditem-main p:not(.text-module-end) | removeNewline | trim',
address: '.aditem-details | trim | removeNewline' address: '.aditem-details | trim | removeNewline'
}, },
paginate: '#srchrslt-pagination .pagination-next@href', paginate: '#srchrslt-pagination .pagination-next@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist filter: applyBlacklist
}; };
module.exports = new Fredy(kleinanzeigen); module.exports = new Fredy(kleinanzeigen);

30
lib/provider/neubauKompass.js
View File

@@ -3,27 +3,27 @@ const Fredy = require('../fredy');
const utils = require('../utils'); const utils = require('../utils');
function normalize(o) { function normalize(o) {
return o; return o;
} }
function applyBlacklist(o) { function applyBlacklist(o) {
return !utils.isOneOf(o.title, config.blacklist); return !utils.isOneOf(o.title, config.blacklist);
} }
const neubauKompass = { const neubauKompass = {
name: 'neubauKompass', name: 'neubauKompass',
enabled: config.sources.neubauKompass.enabled, enabled: config.sources.neubauKompass.enabled,
url: config.sources.neubauKompass.url, url: config.sources.neubauKompass.url,
crawlContainer: '.row article', crawlContainer: '.row article',
crawlFields: { crawlFields: {
id: '@id', id: '@id',
title: 'a@title | removeNewline | trim', title: 'div.p-2 > a@title | removeNewline | trim',
link: 'a@href', link: 'div.p-2 > a@href',
address: 'div.p-2 > p:nth-child(3) | removeNewline | trim' address: 'div.p-2 > p | removeNewline | trim'
}, },
paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href', paginate: '.numbered-pager__bottom .numbered-pager--info li:nth-child(2) a@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist filter: applyBlacklist
}; };
module.exports = new Fredy(neubauKompass); module.exports = new Fredy(neubauKompass);

36
lib/provider/wgGesucht.js
View File

@@ -3,31 +3,31 @@ const Fredy = require('../fredy');
const utils = require('../utils'); const utils = require('../utils');
function normalize(o) { function normalize(o) {
return o; return o;
} }
function applyBlacklist(o) { function applyBlacklist(o) {
const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist); const titleNotBlacklisted = !utils.isOneOf(o.title, config.blacklist);
const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist); const descNotBlacklisted = !utils.isOneOf(o.description, config.blacklist);
return titleNotBlacklisted && descNotBlacklisted; return titleNotBlacklisted && descNotBlacklisted;
} }
const wgGesucht = { const wgGesucht = {
name: 'wgGesucht', name: 'wgGesucht',
enabled: config.sources.wgGesucht.enabled, enabled: config.sources.wgGesucht.enabled,
url: config.sources.wgGesucht.url, url: config.sources.wgGesucht.url,
crawlContainer: '#main_column .panel:not(.display-none):not(.noprint)', crawlContainer: '#main_column .panel:not(.display-none):not(.noprint)',
crawlFields: { crawlFields: {
id: '@data-id', id: '@data-id',
details: '.detail-size-price-wrapper .detailansicht |removeNewline |trim', details: '.detail-size-price-wrapper .detailansicht |removeNewline |trim',
title: '.headline .detailansicht |removeNewline |trim', title: '.headline .detailansicht |removeNewline |trim',
description: '.list-details-panel-inner p |removeNewline |trim', description: '.list-details-panel-inner p |removeNewline |trim',
link: '.headline .detailansicht@href' link: '.headline .detailansicht@href'
}, },
paginate: '.pagination-sm:first a:last@href', paginate: '.pagination-sm:first a:last@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist filter: applyBlacklist
}; };
module.exports = new Fredy(wgGesucht); module.exports = new Fredy(wgGesucht);

43
lib/services/stats.js
View File

@@ -1,36 +1,35 @@
const config = require('../../conf/config.json'); const config = require('../../conf/config.json');
let stats = { let stats = {
lastScrape: {}, lastScrape: {},
foundScrapes: {} foundScrapes: {}
}; };
if (config.enableStats) { if (config.enableStats) {
const http = require('http'); const http = require('http');
http http
.createServer((req, res) => { .createServer((req, res) => {
res.writeHead(200, {'Content-Type': 'application/json'}); res.writeHead(200, { 'Content-Type': 'application/json' });
res.end( res.end(
JSON.stringify({ JSON.stringify({
config, config,
stats stats
})
);
}) })
.listen(config.statsPort, '127.0.0.1'); );
})
.listen(config.statsPort, '127.0.0.1');
} }
const datetime = date => { const datetime = date => {
return `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()} ${date.getHours()}:${date.getMinutes()}`; return `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()} ${date.getHours()}:${date.getMinutes()}`;
}; };
exports.setLastScrape = (serviceName, numberOfNewListsings) => { exports.setLastScrape = (serviceName, numberOfNewListsings) => {
const d = new Date(); const d = new Date();
const dt = datetime(d); const dt = datetime(d);
stats.lastScrape[serviceName] = d.toString(); stats.lastScrape[serviceName] = d.toString();
if (numberOfNewListsings > 0) { if (numberOfNewListsings > 0) {
stats.foundScrapes[dt] = stats.foundScrapes[dt] || {}; stats.foundScrapes[dt] = stats.foundScrapes[dt] || {};
stats.foundScrapes[dt][serviceName] = numberOfNewListsings; stats.foundScrapes[dt][serviceName] = numberOfNewListsings;
} }
}; };

10
package.json
View File

@@ -1,11 +1,11 @@
{ {
"name": "Fredy", "name": "Fredy",
"version": "1.2.0", "version": "1.2.1",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"start": "node index.js", "start": "node index.js",
"format": "prettier --write lib/**/*.js test/**/*.js *.js --single-quote --print-width 120", "format": "prettier --write lib/**/*.js test/**/*.js *.js --single-quote --print-width 120",
"test": "mocha --timeout 8000" "test": "mocha --timeout 9000"
}, },
"main": "index.js", "main": "index.js",
"author": "Christian Kellner", "author": "Christian Kellner",
@@ -31,12 +31,12 @@
"chai": "4.2.0", "chai": "4.2.0",
"lowdb": "1.0.0", "lowdb": "1.0.0",
"request-x-ray": "0.1.4", "request-x-ray": "0.1.4",
"slack": "10.1.1", "slack": "11.0.2",
"tg-yarl": "1.3.0", "tg-yarl": "1.3.0",
"x-ray": "2.3.3" "x-ray": "2.3.4"
}, },
"devDependencies": { "devDependencies": {
"mocha": "6.1.4", "mocha": "6.2.2",
"prettier": "1.18.2", "prettier": "1.18.2",
"proxyquire": "1.8.0" "proxyquire": "1.8.0"
} }

1
test/kleinanzeigen.test.js
View File

@@ -40,7 +40,6 @@ describe('#kleinanzeigen testsuite()', () => {
expect(notify.id).to.equal( expect(notify.id).to.equal(
kleinanzeigenDbContent.kleinanzeigen[idx] kleinanzeigenDbContent.kleinanzeigen[idx]
); );
expect(notify.size).that.does.include('m²');
expect(notify.title).to.be.not.empty; expect(notify.title).to.be.not.empty;
expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de'); expect(notify.link).that.does.include('https://www.ebay-kleinanzeigen.de');
expect(notify.address).to.be.not.empty; expect(notify.address).to.be.not.empty;

656
yarn.lock
View File

File diff suppressed because it is too large Load Diff