diff --git a/.eslintrc.js b/.eslintrc.js index a5d4807..a43cf16 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -4,6 +4,7 @@ module.exports = { es6: true, node: true, browser: true, + mocha: true, }, parser: 'babel-eslint', extends: ['eslint:recommended', 'prettier'], @@ -11,6 +12,7 @@ module.exports = { globals: { Promise: false, describe: true, + after: true, it: true, fetch: true, }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bd5ee7..cfb3941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ +###### [V5.2.0] +- Upgrading dependencies +- Adding new similarity check layer (Duplicates are being removed now) +- Adding paging for search results + ###### [V5.1.0] - Upgrading dependencies - NodeJS 12.13 is now the minimum supported version - Adding general settings as new configuration page to ui -- Adding new feature working hours +- Adding new feature working hours ###### [V5.0.0] - Upgrading dependencies diff --git a/doc/architecture.jpg b/doc/architecture.jpg index 2201db1..eb3d548 100644 Binary files a/doc/architecture.jpg and b/doc/architecture.jpg differ diff --git a/index.js b/index.js index 77e4e76..259373b 100755 --- a/index.js +++ b/index.js @@ -9,8 +9,9 @@ const path = './lib/provider'; const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js')); const config = require('./conf/config.json'); -const jobStorage = require('./lib/services/storage/jobStorage'); +const similarityCache = require('./lib/services/similarity-check/similarityCache'); const { setLastJobExecution } = require('./lib/services/storage/listingsStorage'); +const jobStorage = require('./lib/services/storage/jobStorage'); const FredyRuntime = require('./lib/FredyRuntime'); const { duringWorkingHoursOrNotSet } = require('./lib/utils'); @@ -50,7 +51,13 @@ setInterval( throw new Error(`Provider Config for provider with id ${providerId} not found.`); } pro.init(providerConfig, job.blacklist); - await new FredyRuntime(pro.config, job.notificationAdapter, providerId, job.id).execute(); + await new FredyRuntime( + pro.config, + job.notificationAdapter, + providerId, + job.id, + similarityCache + ).execute(); setLastJobExecution(job.id); }); }); diff --git a/lib/FredyRuntime.js b/lib/FredyRuntime.js index 342e3f2..0513d36 100755 --- a/lib/FredyRuntime.js +++ b/lib/FredyRuntime.js @@ -1,4 +1,4 @@ -const { NoNewListingsError } = require('./errors'); +const { NoNewListingsWarning } = require('./errors'); const { setKnownListings, getKnownListings } = require('./services/storage/listingsStorage'); const notify = require('./notification/notify'); @@ -12,12 +12,14 @@ class FredyRuntime { * @param notificationConfig the config for all notifications * @param providerId the id of the provider currently in use * @param jobKey key of the job that is currently running (from within the config) + * @param similarityCache cache instance holding values to check for similarity of entries */ - constructor(providerConfig, notificationConfig, providerId, jobKey) { + constructor(providerConfig, notificationConfig, providerId, jobKey, similarityCache) { this._providerConfig = providerConfig; this._notificationConfig = notificationConfig; this._providerId = providerId; this._jobKey = jobKey; + this._similarityCache = similarityCache; } execute() { @@ -33,6 +35,8 @@ class FredyRuntime { .then(this._findNew.bind(this)) //store everything in db .then(this._save.bind(this)) + //check for similar listings. if found, remove them before notifying + .then(this._filterBySimilarListings.bind(this)) //notify the user using the configured notification adapter .then(this._notify.bind(this)) //if an error occurred on the way, handle it here. @@ -53,14 +57,29 @@ class FredyRuntime { } const u = scrapingAnt.isImmoscout(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url; try { - xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]) - .then((listings) => { - resolve(listings == null ? [] : listings); - }) - .catch((err) => { - reject(err); - console.error(err); - }); + if (this._providerConfig.paginate != null) { + xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]) + //the first 2 pages should be enough here + //TODO: Think about automagically sort by date + .limit(2) + .paginate(this._providerConfig.paginate) + .then((listings) => { + resolve(listings == null ? [] : listings); + }) + .catch((err) => { + reject(err); + console.error(err); + }); + } else { + xray(u, this._providerConfig.crawlContainer, [this._providerConfig.crawlFields]) + .then((listings) => { + resolve(listings == null ? [] : listings); + }) + .catch((err) => { + reject(err); + console.error(err); + }); + } } catch (error) { reject(error); console.error(error); @@ -80,7 +99,7 @@ class FredyRuntime { const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null); if (newListings.length === 0) { - throw new NoNewListingsError(); + throw new NoNewListingsWarning(); } return newListings; @@ -100,8 +119,22 @@ class FredyRuntime { return newListings; } + _filterBySimilarListings(listings) { + const filteredList = listings.filter((listing) => { + const similar = this._similarityCache.hasSimilarEntries(this._jobKey, listing.title); + if (similar) { + /* eslint-disable no-console */ + console.debug(`Filtering similar entry for job with id ${this._jobKey} with title: `, listing.title); + /* eslint-enable no-console */ + } + return !similar; + }); + filteredList.forEach((filter) => this._similarityCache.addCacheEntry(this._jobKey, filter.title)); + return filteredList; + } + _handleError(err) { - if (err.name !== 'NoNewListingsError') console.error(err); + if (err.name !== 'NoNewListingsWarning') console.error(err); } } diff --git a/lib/errors.js b/lib/errors.js index 543419e..4aa13a6 100755 --- a/lib/errors.js +++ b/lib/errors.js @@ -10,6 +10,6 @@ class ExtendableError extends Error { } } -class NoNewListingsError extends ExtendableError {} +class NoNewListingsWarning extends ExtendableError {} -module.exports = { NoNewListingsError }; +module.exports = { NoNewListingsWarning }; diff --git a/lib/provider/einsAImmobilien.js b/lib/provider/einsAImmobilien.js index 7e84a6e..1b4e752 100755 --- a/lib/provider/einsAImmobilien.js +++ b/lib/provider/einsAImmobilien.js @@ -30,7 +30,6 @@ const config = { title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim', description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim', }, - paginate: '.pagination_blocks div:last a@href', normalize: normalize, filter: applyBlacklist, }; diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index 5e22dea..61755ba 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -20,7 +20,7 @@ function applyBlacklist(o) { const config = { url: null, - crawlContainer: '#srchrslt-adtable .ad-listitem', + crawlContainer: '#srchrslt-adtable .ad-listitem ', crawlFields: { id: '.aditem@data-adid | int', price: '.aditem-main--middle--price | removeNewline | trim', diff --git a/lib/provider/wgGesucht.js b/lib/provider/wgGesucht.js index b74dd71..73e76ba 100755 --- a/lib/provider/wgGesucht.js +++ b/lib/provider/wgGesucht.js @@ -24,7 +24,6 @@ const config = { title: '.truncate_title a |removeNewline |trim', link: '.truncate_title a@href', }, - paginate: '.pagination-sm:first a:last@href', normalize: normalize, filter: applyBlacklist, }; diff --git a/lib/services/similarity-check/SimilarityCacheEntry.js b/lib/services/similarity-check/SimilarityCacheEntry.js new file mode 100644 index 0000000..e9fcf42 --- /dev/null +++ b/lib/services/similarity-check/SimilarityCacheEntry.js @@ -0,0 +1,36 @@ +const stringSimilarity = require('string-similarity'); + +//if the score is higher than this, it will be considered a match +const MAX_DICE_INDEX = 0.7; + +/** + * The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient + * + * @type {module.SimilarityCacheEntry} + */ +module.exports = class SimilarityCacheEntry { + constructor(time) { + this.time = time; + this.values = []; + } + + setCacheEntry = (entry) => { + this.values.push(entry); + }; + + getTime = () => { + return this.time; + }; + + hasSimilarEntries = (value) => { + if (this.values.length > 0) { + for (let i = 0; i < this.values.length; i++) { + const index = stringSimilarity.compareTwoStrings(value, this.values[i]); + if (index >= MAX_DICE_INDEX) { + return true; + } + } + } + return false; + }; +}; diff --git a/lib/services/similarity-check/similarityCache.js b/lib/services/similarity-check/similarityCache.js new file mode 100644 index 0000000..e6af41a --- /dev/null +++ b/lib/services/similarity-check/similarityCache.js @@ -0,0 +1,63 @@ +/** + * each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides + * a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus + * the user won't see any duplicates + * + * The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins. + * + * @type {module.SimilarityCacheEntry|{}} + */ +const SimilarityCacheEntry = require('./SimilarityCacheEntry'); +const config = require('../../../conf/config.json'); + +//5 minutes +let retention = 5 * 60 * 1000; + +const intervalInMs = config.interval * 60 * 1000; +//an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit. +if (intervalInMs <= retention) { + retention = Math.floor(intervalInMs / 2); +} + +//jobid -> SimilarityCacheEntry +const cache = {}; + +let intervalId; + +exports.addCacheEntry = (jobId, value) => { + cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now()); + cache[jobId].setCacheEntry(value); +}; + +exports.hasSimilarEntries = (jobId, value) => { + if (cache[jobId] == null) { + return false; + } + + return cache[jobId].hasSimilarEntries(value); +}; + +/** + * cleanup + */ +intervalId = setInterval(() => { + const keysToBeRemoved = []; + const now = Date.now(); + + Object.keys(cache).forEach((key) => { + if (cache[key].getTime() + retention < now) { + keysToBeRemoved.push(key); + } + }); + + if (keysToBeRemoved.length > 0) { + keysToBeRemoved.forEach((key) => delete cache[key]); + } +}, 10000); + +/** + * mostly used for tests + */ +exports.stopCacheCleanup = () => { + clearInterval(intervalId); +}; diff --git a/lib/services/storage/jobStorage.js b/lib/services/storage/jobStorage.js index 964d1e5..adb5b12 100644 --- a/lib/services/storage/jobStorage.js +++ b/lib/services/storage/jobStorage.js @@ -61,12 +61,18 @@ exports.setJobStatus = ({ jobId, status }) => { }; exports.removeJob = (jobId) => { + listingStorage.removeListings(jobId); db.get('jobs') .remove((job) => job.id === jobId) .write(); }; exports.removeJobsByUserId = (userId) => { + db.get('jobs') + .value() + .filter((job) => job.userId === userId) + .forEach((job) => listingStorage.removeListings(job.id)); + db.get('jobs') .remove((job) => job.userId === userId) .write(); diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index cf57bf9..c25632e 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -47,3 +47,7 @@ exports.setLastJobExecution = (jobId) => { const key = buildKey(jobId, null, 'lastExecution'); return db.set(key, Date.now()).write(); }; + +exports.removeListings = (jobId) => { + db.unset(jobId).write(); +}; diff --git a/package.json b/package.json index 1dc867a..8561a07 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fredy", - "version": "5.1.0", + "version": "5.2.0", "description": "[F]ind [R]eal [E]states [d]amn eas[y].", "scripts": { "start": "node index.js", @@ -53,12 +53,12 @@ "dependencies": { "@rematch/core": "2.0.1", "@rematch/loading": "2.0.1", - "@sendgrid/mail": "7.4.4", + "@sendgrid/mail": "7.4.5", "axios": "0.21.1", "body-parser": "1.19.0", "cookie-session": "1.4.0", "handlebars": "4.7.7", - "highcharts": "9.1.0", + "highcharts": "9.1.2", "highcharts-react-official": "3.0.0", "lowdb": "1.0.0", "markdown": "^0.5.0", @@ -76,36 +76,37 @@ "semantic-ui-react": "2.0.3", "serve-static": "^1.14.1", "slack": "11.0.2", + "string-similarity": "^4.0.4", "x-ray": "2.3.4" }, "devDependencies": { - "@babel/core": "7.14.3", - "@babel/preset-env": "7.14.2", - "@babel/preset-react": "7.13.13", + "@babel/core": "7.14.6", + "@babel/preset-env": "7.14.7", + "@babel/preset-react": "7.14.5", "babel-eslint": "10.1.0", "babel-loader": "8.2.2", "chai": "4.3.4", "clean-webpack-plugin": "3.0.0", - "copy-webpack-plugin": "9.0.0", + "copy-webpack-plugin": "9.0.1", "css-loader": "5.2.6", - "eslint": "7.27.0", + "eslint": "7.29.0", "eslint-config-prettier": "8.3.0", - "eslint-plugin-react": "7.23.2", + "eslint-plugin-react": "7.24.0", "file-loader": "6.2.0", "history": "5.0.0", "husky": "4.3.8", "less": "4.1.1", - "less-loader": "9.0.0", + "less-loader": "10.0.0", "lint-staged": "11.0.0", - "mocha": "8.4.0", - "prettier": "2.3.0", + "mocha": "9.0.1", + "prettier": "2.3.2", "proxyquire": "2.1.3", "redux-logger": "3.0.6", - "style-loader": "2.0.0", + "style-loader": "3.0.0", "url-loader": "4.1.1", - "webpack": "5.37.1", + "webpack": "5.40.0", "webpack-cli": "3.3.12", "webpack-dev-server": "3.11.2", - "webpack-merge": "5.7.3" + "webpack-merge": "5.8.0" } } diff --git a/test/provider/einsAImmobilien.test.js b/test/provider/einsAImmobilien.test.js index 38e0169..1fa2c64 100644 --- a/test/provider/einsAImmobilien.test.js +++ b/test/provider/einsAImmobilien.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -6,6 +7,10 @@ const expect = require('chai').expect; const provider = require('../../lib/provider/einsAImmobilien'); describe('#einsAImmobilien testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); + provider.init(providerConfig.einsAImmobilien, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { @@ -17,7 +22,7 @@ describe('#einsAImmobilien testsuite()', () => { it('should test einsAImmobilien provider', async () => { return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listings) => { expect(listings).to.be.a('array'); diff --git a/test/provider/immonet.test.js b/test/provider/immonet.test.js index 8badba9..b2fb8ae 100644 --- a/test/provider/immonet.test.js +++ b/test/provider/immonet.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -6,6 +7,10 @@ const expect = require('chai').expect; const provider = require('../../lib/provider/immonet'); describe('#immonet testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); + provider.init(providerConfig.immonet, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { './services/storage/listingsStorage': { @@ -16,7 +21,7 @@ describe('#immonet testsuite()', () => { it('should test immonet provider', async () => { return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); diff --git a/test/provider/immoscout.test.js b/test/provider/immoscout.test.js index 657f793..1b7cae4 100644 --- a/test/provider/immoscout.test.js +++ b/test/provider/immoscout.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -7,6 +8,9 @@ const provider = require('../../lib/provider/immoscout'); const scrapingAnt = require('../../lib/services/scrapingAnt'); describe('#immoscout testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); provider.init(providerConfig.immoscout, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { './services/storage/listingsStorage': { @@ -25,7 +29,7 @@ describe('#immoscout testsuite()', () => { return; } - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); diff --git a/test/provider/immowelt.test.js b/test/provider/immowelt.test.js index f8326f5..841ce3a 100644 --- a/test/provider/immowelt.test.js +++ b/test/provider/immowelt.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -6,6 +7,9 @@ const expect = require('chai').expect; const provider = require('../../lib/provider/immowelt'); describe('#immowelt testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); it('should test immowelt provider', async () => { provider.init(providerConfig.immowelt, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { @@ -16,7 +20,7 @@ describe('#immowelt testsuite()', () => { }); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); diff --git a/test/provider/kleinanzeigen.test.js b/test/provider/kleinanzeigen.test.js index 2532ae2..d9a5e2c 100644 --- a/test/provider/kleinanzeigen.test.js +++ b/test/provider/kleinanzeigen.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -6,6 +7,9 @@ const expect = require('chai').expect; const provider = require('../../lib/provider/kleinanzeigen'); describe('#kleinanzeigen testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); it('should test kleinanzeigen provider', async () => { provider.init(providerConfig.kleinanzeigen, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { @@ -16,7 +20,7 @@ describe('#kleinanzeigen testsuite()', () => { }); return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); diff --git a/test/provider/neubauKompass.test.js b/test/provider/neubauKompass.test.js index f1a8ccc..a0e419c 100644 --- a/test/provider/neubauKompass.test.js +++ b/test/provider/neubauKompass.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -6,6 +7,9 @@ const expect = require('chai').expect; const provider = require('../../lib/provider/neubauKompass'); describe('#neubauKompass testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); provider.init(providerConfig.neubauKompass, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { './services/storage/listingsStorage': { @@ -16,7 +20,7 @@ describe('#neubauKompass testsuite()', () => { it('should test neubauKompass provider', async () => { return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); diff --git a/test/provider/testProvider.json b/test/provider/testProvider.json index 4d4dbbe..6988cf5 100644 --- a/test/provider/testProvider.json +++ b/test/provider/testProvider.json @@ -21,7 +21,7 @@ "enabled": true }, "kleinanzeigen": { - "url": "https://www.ebay-kleinanzeigen.de/s-wohnung-kaufen/duesseldorf/anzeige:angebote/preis::420000/wohnung/k0c196l2068r5+wohnung_kaufen.qm_d:90,+wohnung_kaufen.zimmer_d:3.5,", + "url": "https://www.ebay-kleinanzeigen.de/s-immobilien/duesseldorf/anzeige:angebote/wohnung/k0c195l2068r5", "enabled": true }, "neubauKompass": { diff --git a/test/provider/wgGesucht.test.js b/test/provider/wgGesucht.test.js index 5487e44..dc85245 100644 --- a/test/provider/wgGesucht.test.js +++ b/test/provider/wgGesucht.test.js @@ -1,3 +1,4 @@ +const similarityCache = require('../../lib/services/similarity-check/similarityCache'); const mockNotification = require('../mocks/mockNotification'); const providerConfig = require('./testProvider.json'); const mockStore = require('../mocks/mockStore'); @@ -6,6 +7,9 @@ const expect = require('chai').expect; const provider = require('../../lib/provider/wgGesucht'); describe('#wgGesucht testsuite()', () => { + after(() => { + similarityCache.stopCacheCleanup(); + }); provider.init(providerConfig.wgGesucht, [], []); const Fredy = proxyquire('../../lib/FredyRuntime', { './services/storage/listingsStorage': { @@ -16,7 +20,7 @@ describe('#wgGesucht testsuite()', () => { it('should test wgGesucht provider', async () => { return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1'); + const fredy = new Fredy(provider.config, null, provider.metaInformation.id, 'test1', similarityCache); fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = mockNotification.get(); diff --git a/test/similarity/similarity.test.js b/test/similarity/similarity.test.js new file mode 100644 index 0000000..673b33d --- /dev/null +++ b/test/similarity/similarity.test.js @@ -0,0 +1,39 @@ +const SimilarityCacheEntry = require('../../lib/services/similarity-check/SimilarityCacheEntry'); +const expect = require('chai').expect; + +describe('similarityCheck', () => { + describe('#similarityCheck()', () => { + it('should be false', () => { + const check = new SimilarityCacheEntry(0); + check.setCacheEntry('Hallo'); + expect(check.hasSimilarEntries('Welt')).to.be.false; + }); + it('should be true', () => { + const check = new SimilarityCacheEntry(0); + check.setCacheEntry('Hallo'); + expect(check.hasSimilarEntries('hallo')).to.be.true; + }); + it('should be true', () => { + const check = new SimilarityCacheEntry(0); + check.setCacheEntry('Selling an incredible house in san francisco'); + expect(check.hasSimilarEntries('incredible house in san francisco for sale')).to.be.true; + }); + it('should be true', () => { + const check = new SimilarityCacheEntry(0); + check.setCacheEntry('a'); + check.setCacheEntry('b'); + check.setCacheEntry('c'); + check.setCacheEntry('d'); + expect(check.hasSimilarEntries('b')).to.be.true; + }); + it('should be false', () => { + const check = new SimilarityCacheEntry(0); + check.setCacheEntry( + 'The index is known by several other names, especially Sørensen–Dice index,[3] Sørensen index and Dice\'s coefficient. Other variations include the "similarity coefficient" or "index", such as Dice similarity coefficient (DSC). Common alternate spellings for Sørensen are Sorenson, Soerenson and Sörenson, and all three can also be seen with the –sen ending.' + ); + check.setCacheEntry( + 'where |X| and |Y| are the cardinalities of the two sets (i.e. the number of elements in each set). The Sørensen index equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.' + ); + }); + }); +}); diff --git a/ui/src/views/login/Login.js b/ui/src/views/login/Login.js index a474da7..7840547 100644 --- a/ui/src/views/login/Login.js +++ b/ui/src/views/login/Login.js @@ -40,9 +40,8 @@ export default function Login() { return (