Compare commits

...

2 Commits

Author SHA1 Message Date
Christian Kellner
59e6d287fc adding similarity check (#29)
* adding similarity check

* adding paging

* fixing tests

* docu

* better error handling

* fixing tests

* adjusting page limit

* fixing login screen

* cleanup

* upgrade browser list

* prevent spamming the log

* fixing tests

* removing job listings when removing a job or the user
2021-06-28 08:52:09 +02:00
Christian Kellner
88c046dbd4 Update config.json
Set the default interval to 1 hour. 30 mins was a little too much
2021-05-31 08:58:41 +02:00
27 changed files with 1115 additions and 879 deletions

View File

@@ -4,6 +4,7 @@ module.exports = {
es6: true, es6: true,
node: true, node: true,
browser: true, browser: true,
mocha: true,
}, },
parser: 'babel-eslint', parser: 'babel-eslint',
extends: ['eslint:recommended', 'prettier'], extends: ['eslint:recommended', 'prettier'],
@@ -11,6 +12,7 @@ module.exports = {
globals: { globals: {
Promise: false, Promise: false,
describe: true, describe: true,
after: true,
it: true, it: true,
fetch: true, fetch: true,
}, },

View File

@@ -1,8 +1,13 @@
###### [V5.2.0]
- Upgrading dependencies
- Adding new similarity check layer (Duplicates are being removed now)
- Adding paging for search results
###### [V5.1.0] ###### [V5.1.0]
- Upgrading dependencies - Upgrading dependencies
- NodeJS 12.13 is now the minimum supported version - NodeJS 12.13 is now the minimum supported version
- Adding general settings as new configuration page to ui - Adding general settings as new configuration page to ui
- Adding new feature working hours - Adding new feature working hours
###### [V5.0.0] ###### [V5.0.0]
- Upgrading dependencies - Upgrading dependencies

View File

@@ -1 +1 @@
{"interval":"30","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}} {"interval":"60","port":9998,"scrapingAnt":{"apiKey":""},"workingHours":{"from":"","to":""}}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 134 KiB

After

Width:  |  Height:  |  Size: 189 KiB

View File

@@ -9,8 +9,9 @@ const path = './lib/provider';
const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js')); const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
const config = require('./conf/config.json'); const config = require('./conf/config.json');
const jobStorage = require('./lib/services/storage/jobStorage'); const similarityCache = require('./lib/services/similarity-check/similarityCache');
const { setLastJobExecution } = require('./lib/services/storage/listingsStorage'); const { setLastJobExecution } = require('./lib/services/storage/listingsStorage');
const jobStorage = require('./lib/services/storage/jobStorage');
const FredyRuntime = require('./lib/FredyRuntime'); const FredyRuntime = require('./lib/FredyRuntime');
const { duringWorkingHoursOrNotSet } = require('./lib/utils'); const { duringWorkingHoursOrNotSet } = require('./lib/utils');
@@ -50,7 +51,13 @@ setInterval(
throw new Error(`Provider Config for provider with id ${providerId} not found.`); throw new Error(`Provider Config for provider with id ${providerId} not found.`);
} }
pro.init(providerConfig, job.blacklist); pro.init(providerConfig, job.blacklist);
await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute(); await new FredyRuntime(
pro.config,
job.notificationAdapter,
providerId,
job.id,
similarityCache
).execute();
setLastJobExecution(job.id); setLastJobExecution(job.id);
}); });
}); });

View File

@@ -1,4 +1,4 @@
const { NoNewListingsError } = require('./errors'); const { NoNewListingsWarning } = require('./errors');
const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage'); const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage');
const notify = require('./notification/notify'); const notify = require('./notification/notify');
@@ -12,12 +12,14 @@ class FredyRuntime {
* @param notificationConfig the config for all notifications * @param notificationConfig the config for all notifications
* @param providerId the id of the provider currently in use * @param providerId the id of the provider currently in use
* @param jobKey key of the job that is currently running (from within the config) * @param jobKey key of the job that is currently running (from within the config)
* @param similarityCache cache instance holding values to check for similarity of entries
*/ */
constructor(providerConfig, notificationConfig, providerId, jobKey) { constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) {
this._providerConfig = providerConfig; this._providerConfig = providerConfig;
this._notificationConfig = notificationConfig; this._notificationConfig = notificationConfig;
this._providerId = providerId; this._providerId = providerId;
this._jobKey = jobKey; this._jobKey = jobKey;
this._similarityCache = similarityCache;
} }
execute() { execute() {
@@ -33,6 +35,8 @@ class FredyRuntime {
.then(this._findNew.bind(this)) .then(this._findNew.bind(this))
//store everything in db //store everything in db
.then(this._save.bind(this)) .then(this._save.bind(this))
//check for similar listings. if found, remove them before notifying
.then(this._filterBySimilarListings.bind(this))
//notify the user using the configured notification adapter //notify the user using the configured notification adapter
.then(this._notify.bind(this)) .then(this._notify.bind(this))
//if an error occurred on the way, handle it here. //if an error occurred on the way, handle it here.
@@ -53,14 +57,29 @@ class FredyRuntime {
} }
const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url; const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
try { try {
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]) if (this._providerConfig.paginate != null) {
.then((listings) => { xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
resolve(listings == null ? [] : listings); //the first 2 pages should be enough here
}) //TODO: Think about automagically sort by date
.catch((err) => { .limit(2)
reject(err); .paginate(this._providerConfig.paginate)
console.error(err); .then((listings) => {
}); resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
} else {
xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields])
.then((listings) => {
resolve(listings == null ? [] : listings);
})
.catch((err) => {
reject(err);
console.error(err);
});
}
} catch (error) { } catch (error) {
reject(error); reject(error);
console.error(error); console.error(error);
@@ -80,7 +99,7 @@ class FredyRuntime {
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null); const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
if (newListings.length === 0) { if (newListings.length === 0) {
throw new NoNewListingsError(); throw new NoNewListingsWarning();
} }
return newListings; return newListings;
@@ -100,8 +119,22 @@ class FredyRuntime {
return newListings; return newListings;
} }
_filterBySimilarListings(listings) {
const filteredList = listings.filter((listing) => {
const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title);
if (similar) {
/* eslint-disable no-console */
console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title);
/* eslint-enable no-console */
}
return !similar;
});
filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title));
return filteredList;
}
_handleError(err) { _handleError(err) {
if (err.name !== 'NoNewListingsError') console.error(err); if (err.name !== 'NoNewListingsWarning') console.error(err);
} }
} }

View File

@@ -10,6 +10,6 @@ class ExtendableError extends Error {
} }
} }
class NoNewListingsError extends ExtendableError {} class NoNewListingsWarning extends ExtendableError {}
module.exports = { NoNewListingsError }; module.exports = { NoNewListingsWarning };

View File

@@ -30,7 +30,6 @@ const config = {
title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim', title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim', description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim',
}, },
paginate: '.pagination_blocks div:last a@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist, filter: applyBlacklist,
}; };

View File

@@ -20,7 +20,7 @@ function applyBlacklist(o) {
const config = { const config = {
url: null, url: null,
crawlContainer: '#srchrslt-adtable .ad-listitem', crawlContainer: '#srchrslt-adtable .ad-listitem ',
crawlFields: { crawlFields: {
id: '.aditem@data-adid | int', id: '.aditem@data-adid | int',
price: '.aditem-main--middle--price | removeNewline | trim', price: '.aditem-main--middle--price | removeNewline | trim',

View File

@@ -24,7 +24,6 @@ const config = {
title: '.truncate_title a |removeNewline |trim', title: '.truncate_title a |removeNewline |trim',
link: '.truncate_title a@href', link: '.truncate_title a@href',
}, },
paginate: '.pagination-sm:first a:last@href',
normalize: normalize, normalize: normalize,
filter: applyBlacklist, filter: applyBlacklist,
}; };

View File

@@ -0,0 +1,36 @@
const stringSimilarity = require('string-similarity');
//if the score is higher than this, it will be considered a match
const MAX_DICE_INDEX = 0.7;
/**
* The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
*
* @type {module.SimilarityCacheEntry}
*/
module.exports = class SimilarityCacheEntry {
constructor(time) {
this.time = time;
this.values = [];
}
setCacheEntry = (entry) => {
this.values.push(entry);
};
getTime = () => {
return this.time;
};
hasSimilarEntries = (value) => {
if (this.values.length > 0) {
for (let i = 0; i < this.values.length; i++) {
const index = stringSimilarity.compareTwoStrings(value, this.values[i]);
if (index >= MAX_DICE_INDEX) {
return true;
}
}
}
return false;
};
};

View File

@@ -0,0 +1,63 @@
/**
* each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
* a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
* the user won't see any duplicates
*
* The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
*
* @type {module.SimilarityCacheEntry|{}}
*/
const SimilarityCacheEntry = require('./SimilarityCacheEntry');
const config = require('../../../conf/config.json');
//5 minutes
let retention = 5 * 60 * 1000;
const intervalInMs = config.interval * 60 * 1000;
//an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
if (intervalInMs <= retention) {
retention = Math.floor(intervalInMs / 2);
}
//jobid -> SimilarityCacheEntry
const cache = {};
let intervalId;
exports.addCacheEntry = (jobId, value) => {
cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
cache[jobId].setCacheEntry(value);
};
exports.hasSimilarEntries = (jobId, value) => {
if (cache[jobId] == null) {
return false;
}
return cache[jobId].hasSimilarEntries(value);
};
/**
* cleanup
*/
intervalId = setInterval(() => {
const keysToBeRemoved = [];
const now = Date.now();
Object.keys(cache).forEach((key) => {
if (cache[key].getTime() + retention < now) {
keysToBeRemoved.push(key);
}
});
if (keysToBeRemoved.length > 0) {
keysToBeRemoved.forEach((key) => delete cache[key]);
}
}, 10000);
/**
* mostly used for tests
*/
exports.stopCacheCleanup = () => {
clearInterval(intervalId);
};

View File

@@ -61,12 +61,18 @@ exports.setJobStatus = ({ jobId, status }) => {
}; };
exports.removeJob = (jobId) => { exports.removeJob = (jobId) => {
listingStorage.removeListings(jobId);
db.get('jobs') db.get('jobs')
.remove((job) => job.id === jobId) .remove((job) => job.id === jobId)
.write(); .write();
}; };
exports.removeJobsByUserId = (userId) => { exports.removeJobsByUserId = (userId) => {
db.get('jobs')
.value()
.filter((job) => job.userId === userId)
.forEach((job) => listingStorage.removeListings(job.id));
db.get('jobs') db.get('jobs')
.remove((job) => job.userId === userId) .remove((job) => job.userId === userId)
.write(); .write();

View File

@@ -47,3 +47,7 @@ exports.setLastJobExecution = (jobId) => {
const key = buildKey(jobId, null, 'lastExecution'); const key = buildKey(jobId, null, 'lastExecution');
return db.set(key, Date.now()).write(); return db.set(key, Date.now()).write();
}; };
exports.removeListings = (jobId) => {
db.unset(jobId).write();
};

View File

@@ -1,6 +1,6 @@
{ {
"name": "fredy", "name": "fredy",
"version": "5.1.0", "version": "5.2.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].", "description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": { "scripts": {
"start": "node index.js", "start": "node index.js",
@@ -53,12 +53,12 @@
"dependencies": { "dependencies": {
"@rematch/core": "2.0.1", "@rematch/core": "2.0.1",
"@rematch/loading": "2.0.1", "@rematch/loading": "2.0.1",
"@sendgrid/mail": "7.4.4", "@sendgrid/mail": "7.4.5",
"axios": "0.21.1", "axios": "0.21.1",
"body-parser": "1.19.0", "body-parser": "1.19.0",
"cookie-session": "1.4.0", "cookie-session": "1.4.0",
"handlebars": "4.7.7", "handlebars": "4.7.7",
"highcharts": "9.1.0", "highcharts": "9.1.2",
"highcharts-react-official": "3.0.0", "highcharts-react-official": "3.0.0",
"lowdb": "1.0.0", "lowdb": "1.0.0",
"markdown": "^0.5.0", "markdown": "^0.5.0",
@@ -76,36 +76,37 @@
"semantic-ui-react": "2.0.3", "semantic-ui-react": "2.0.3",
"serve-static": "^1.14.1", "serve-static": "^1.14.1",
"slack": "11.0.2", "slack": "11.0.2",
"string-similarity": "^4.0.4",
"x-ray": "2.3.4" "x-ray": "2.3.4"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "7.14.3", "@babel/core": "7.14.6",
"@babel/preset-env": "7.14.2", "@babel/preset-env": "7.14.7",
"@babel/preset-react": "7.13.13", "@babel/preset-react": "7.14.5",
"babel-eslint": "10.1.0", "babel-eslint": "10.1.0",
"babel-loader": "8.2.2", "babel-loader": "8.2.2",
"chai": "4.3.4", "chai": "4.3.4",
"clean-webpack-plugin": "3.0.0", "clean-webpack-plugin": "3.0.0",
"copy-webpack-plugin": "9.0.0", "copy-webpack-plugin": "9.0.1",
"css-loader": "5.2.6", "css-loader": "5.2.6",
"eslint": "7.27.0", "eslint": "7.29.0",
"eslint-config-prettier": "8.3.0", "eslint-config-prettier": "8.3.0",
"eslint-plugin-react": "7.23.2", "eslint-plugin-react": "7.24.0",
"file-loader": "6.2.0", "file-loader": "6.2.0",
"history": "5.0.0", "history": "5.0.0",
"husky": "4.3.8", "husky": "4.3.8",
"less": "4.1.1", "less": "4.1.1",
"less-loader": "9.0.0", "less-loader": "10.0.0",
"lint-staged": "11.0.0", "lint-staged": "11.0.0",
"mocha": "8.4.0", "mocha": "9.0.1",
"prettier": "2.3.0", "prettier": "2.3.2",
"proxyquire": "2.1.3", "proxyquire": "2.1.3",
"redux-logger": "3.0.6", "redux-logger": "3.0.6",
"style-loader": "2.0.0", "style-loader": "3.0.0",
"url-loader": "4.1.1", "url-loader": "4.1.1",
"webpack": "5.37.1", "webpack": "5.40.0",
"webpack-cli": "3.3.12", "webpack-cli": "3.3.12",
"webpack-dev-server": "3.11.2", "webpack-dev-server": "3.11.2",
"webpack-merge": "5.7.3" "webpack-merge": "5.8.0"
} }
} }

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -6,6 +7,10 @@ const expect = require('chai').expect;
const provider = require('../../lib/provider/einsAImmobilien'); const provider = require('../../lib/provider/einsAImmobilien');
describe('#einsAImmobilien testsuite()', () => { describe('#einsAImmobilien testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.einsAImmobilien, [], []); provider.init(providerConfig.einsAImmobilien, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
@@ -17,7 +22,7 @@ describe('#einsAImmobilien testsuite()', () => {
it('should test einsAImmobilien provider', async () => { it('should test einsAImmobilien provider', async () => {
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listings) => { fredy.execute().then((listings) => {
expect(listings).to.be.a('array'); expect(listings).to.be.a('array');

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -6,6 +7,10 @@ const expect = require('chai').expect;
const provider = require('../../lib/provider/immonet'); const provider = require('../../lib/provider/immonet');
describe('#immonet testsuite()', () => { describe('#immonet testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immonet, [], []); provider.init(providerConfig.immonet, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': { './services/storage/listingsStorage': {
@@ -16,7 +21,7 @@ describe('#immonet testsuite()', () => {
it('should test immonet provider', async () => { it('should test immonet provider', async () => {
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -7,6 +8,9 @@ const provider = require('../../lib/provider/immoscout');
const scrapingAnt = require('../../lib/services/scrapingAnt'); const scrapingAnt = require('../../lib/services/scrapingAnt');
describe('#immoscout testsuite()', () => { describe('#immoscout testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.immoscout, [], []); provider.init(providerConfig.immoscout, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': { './services/storage/listingsStorage': {
@@ -25,7 +29,7 @@ describe('#immoscout testsuite()', () => {
return; return;
} }
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -6,6 +7,9 @@ const expect = require('chai').expect;
const provider = require('../../lib/provider/immowelt'); const provider = require('../../lib/provider/immowelt');
describe('#immowelt testsuite()', () => { describe('#immowelt testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test immowelt provider', async () => { it('should test immowelt provider', async () => {
provider.init(providerConfig.immowelt, [], []); provider.init(providerConfig.immowelt, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
@@ -16,7 +20,7 @@ describe('#immowelt testsuite()', () => {
}); });
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -6,6 +7,9 @@ const expect = require('chai').expect;
const provider = require('../../lib/provider/kleinanzeigen'); const provider = require('../../lib/provider/kleinanzeigen');
describe('#kleinanzeigen testsuite()', () => { describe('#kleinanzeigen testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
it('should test kleinanzeigen provider', async () => { it('should test kleinanzeigen provider', async () => {
provider.init(providerConfig.kleinanzeigen, [], []); provider.init(providerConfig.kleinanzeigen, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
@@ -16,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => {
}); });
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -6,6 +7,9 @@ const expect = require('chai').expect;
const provider = require('../../lib/provider/neubauKompass'); const provider = require('../../lib/provider/neubauKompass');
describe('#neubauKompass testsuite()', () => { describe('#neubauKompass testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.neubauKompass, [], []); provider.init(providerConfig.neubauKompass, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': { './services/storage/listingsStorage': {
@@ -16,7 +20,7 @@ describe('#neubauKompass testsuite()', () => {
it('should test neubauKompass provider', async () => { it('should test neubauKompass provider', async () => {
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');

View File

@@ -21,7 +21,7 @@
"enabled": true "enabled": true
}, },
"kleinanzeigen": { "kleinanzeigen": {
"url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,", "url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5",
"enabled": true "enabled": true
}, },
"neubauKompass": { "neubauKompass": {

View File

@@ -1,3 +1,4 @@
const similarityCache = require('../../lib/services/similarity-check/similarityCache');
const mockNotification = require('../mocks/mockNotification'); const mockNotification = require('../mocks/mockNotification');
const providerConfig = require('./testProvider.json'); const providerConfig = require('./testProvider.json');
const mockStore = require('../mocks/mockStore'); const mockStore = require('../mocks/mockStore');
@@ -6,6 +7,9 @@ const expect = require('chai').expect;
const provider = require('../../lib/provider/wgGesucht'); const provider = require('../../lib/provider/wgGesucht');
describe('#wgGesucht testsuite()', () => { describe('#wgGesucht testsuite()', () => {
after(() => {
similarityCache.stopCacheCleanup();
});
provider.init(providerConfig.wgGesucht, [], []); provider.init(providerConfig.wgGesucht, [], []);
const Fredy = proxyquire('../../lib/FredyRuntime', { const Fredy = proxyquire('../../lib/FredyRuntime', {
'./services/storage/listingsStorage': { './services/storage/listingsStorage': {
@@ -16,7 +20,7 @@ describe('#wgGesucht testsuite()', () => {
it('should test wgGesucht provider', async () => { it('should test wgGesucht provider', async () => {
return await new Promise((resolve) => { return await new Promise((resolve) => {
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache);
fredy.execute().then((listing) => { fredy.execute().then((listing) => {
expect(listing).to.be.a('array'); expect(listing).to.be.a('array');
const notificationObj = mockNotification.get(); const notificationObj = mockNotification.get();

View File

@@ -0,0 +1,39 @@
const SimilarityCacheEntry = require('../../lib/services/similarity-check/SimilarityCacheEntry');
const expect = require('chai').expect;
describe('similarityCheck', () => {
describe('#similarityCheck()', () => {
it('should be false', () => {
const check = new SimilarityCacheEntry(0);
check.setCacheEntry('Hallo');
expect(check.hasSimilarEntries('Welt')).to.be.false;
});
it('should be true', () => {
const check = new SimilarityCacheEntry(0);
check.setCacheEntry('Hallo');
expect(check.hasSimilarEntries('hallo')).to.be.true;
});
it('should be true', () => {
const check = new SimilarityCacheEntry(0);
check.setCacheEntry('Selling an incredible house in san francisco');
expect(check.hasSimilarEntries('incredible house in san francisco for sale')).to.be.true;
});
it('should be true', () => {
const check = new SimilarityCacheEntry(0);
check.setCacheEntry('a');
check.setCacheEntry('b');
check.setCacheEntry('c');
check.setCacheEntry('d');
expect(check.hasSimilarEntries('b')).to.be.true;
});
it('should be false', () => {
const check = new SimilarityCacheEntry(0);
check.setCacheEntry(
'The index is known by several other names, especially SørensenDice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the sen ending.'
);
check.setCacheEntry(
'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.'
);
});
});
});

View File

@@ -40,9 +40,8 @@ export default function Login() {
return ( return (
<div className="login"> <div className="login">
<div className="login__bgImage" style={{ background: `url("${cityBackground}")` }} />
<Logo /> <Logo />
<div className="login__bgImage" style={{ background: `url(${cityBackground})` }} />
<form> <form>
<div className="login__loginWrapper"> <div className="login__loginWrapper">
{error && <Message negative icon="error" content={error} />} {error && <Message negative icon="error" content={error} />}

View File

@@ -2,18 +2,17 @@
display: flex; display: flex;
justify-content: center; justify-content: center;
align-items: center; align-items: center;
width:100%; width: 100%;
height: 100%; height: 100%;
&__bgImage { &__bgImage {
background-size: cover; background-size: cover;
filter: blur(8px); filter: blur(8px);
-webkit-filter: blur(8px); -webkit-filter: blur(8px);
background-size: cover;
position: absolute; position: absolute;
top: 0; top: 0;
left: 0; left: 0;
z-index: -1; z-index: 0;
right: 0; right: 0;
bottom: 0; bottom: 0;
} }
@@ -23,9 +22,14 @@
border-radius: 30px; border-radius: 30px;
height: 25rem; height: 25rem;
width: 30rem; width: 30rem;
z-index: 1;
background-color: #151313ab; background-color: #151313ab;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
padding: 2rem; padding: 2rem;
} }
form {
z-index: 1;
}
} }

1669
yarn.lock

File diff suppressed because it is too large Load Diff