External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Making Fredy an ESM project (#70)

Browse Source 
Making Fredy an ESM project
This commit is contained in:
Christian Kellner
2023-03-13 13:42:43 +01:00
committed by GitHub
parent 7d0ec72a0c
commit 2c5eceb0c1
69 changed files with 862 additions and 1567 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
lib/services/similarity-check/SimilarityCacheEntry.js
View File

@@ -1,27 +1,17 @@
const stringSimilarity = require('string-similarity');
import stringSimilarity from 'string-similarity';
//if the score is higher than this, it will be considered a match
const MAX_DICE_INDEX = 0.7;
/**
* The similarity check is based on the dice coefficient. => https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
*
* @type {module.SimilarityCacheEntry}
*/
module.exports = class SimilarityCacheEntry {
export default (class SimilarityCacheEntry {
constructor(time) {
this.time = time;
this.values = [];
}
setCacheEntry = (entry) => {
this.values.push(entry);
};
getTime = () => {
return this.time;
};
hasSimilarEntries = (value) => {
if (this.values.length > 0) {
for (let i = 0; i < this.values.length; i++) {
@@ -33,4 +23,4 @@ module.exports = class SimilarityCacheEntry {
}
return false;
};
};
});

49
lib/services/similarity-check/similarityCache.js
View File

@@ -1,63 +1,40 @@
/**
* each job that runs scrapes all provider. This cache holds the titles of the found listing(s) and provides
* a similarity check. if this check returns true, it will not be forwarded to the notification adapter, thus
* the user won't see any duplicates
*
* The retention of this cache is per default 5 minutes, but can be smaller if the interval is > 5 mins.
*
* @type {module.SimilarityCacheEntry|{}}
*/
const SimilarityCacheEntry = require('./SimilarityCacheEntry');
const config = require('../../../conf/config.json');
import SimilarityCacheEntry from './SimilarityCacheEntry.js';
import { config } from '../../utils.js';
//5 minutes
let retention = 5 * 60 * 1000;
const intervalInMs = config.interval * 60 * 1000;
//an interval below 5 mins sounds crazy, but there are ppl out there doing crazy shit.
if (intervalInMs <= retention) {
retention = Math.floor(intervalInMs / 2);
}
//jobid -> SimilarityCacheEntry
const cache = {};
let intervalId;
exports.addCacheEntry = (jobId, value) => {
cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
cache[jobId].setCacheEntry(value);
};
exports.hasSimilarEntries = (jobId, value) => {
if (cache[jobId] == null) {
return false;
}
return cache[jobId].hasSimilarEntries(value);
};
/**
* cleanup
*/
intervalId = setInterval(() => {
const keysToBeRemoved = [];
const now = Date.now();
Object.keys(cache).forEach((key) => {
if (cache[key].getTime() + retention < now) {
keysToBeRemoved.push(key);
}
});
if (keysToBeRemoved.length > 0) {
keysToBeRemoved.forEach((key) => delete cache[key]);
}
}, 10000);
/**
* mostly used for tests
*/
exports.stopCacheCleanup = () => {
export const addCacheEntry = (jobId, value) => {
cache[jobId] = cache[jobId] || new SimilarityCacheEntry(Date.now());
cache[jobId].setCacheEntry(value);
};
export const hasSimilarEntries = (jobId, value) => {
if (cache[jobId] == null) {
return false;
}
return cache[jobId].hasSimilarEntries(value);
};
export const stopCacheCleanup = () => {
clearInterval(intervalId);
};