External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate to SQLite (#174)

Browse Source 
* Migrating Fredy from LowDb to SqLite 🎉

* adding new sql migration system for future sql migrations

* adding setting to change  sqlite path for db files

* create migration plan for graceful migration lowdb -> sqlite

* Improving Documentation

* adding test for sqliteconnection

* upgrading dependencies

* making nodejs 22 as min version

* improve scraper

* adding overwrite ability for db migra
This commit is contained in:
Christian Kellner
2025-09-18 15:38:23 +02:00
committed by GitHub
parent 18fdbd761a
commit 8d95f052c6
31 changed files with 1636 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
lib/FredyRuntime.js
View File

@@ -1,5 +1,5 @@
import { NoNewListingsWarning } from './errors.js';
import { setKnownListings, getKnownListings } from './services/storage/listingsStorage.js';
import { storeListings, getKnownListingHashesForJobAndProvider } from './services/storage/listingsStorage.js';
import * as notify from './notification/notify.js';
import Extractor from './services/extractor/extractor.js';
import urlModifier from './services/queryStringMutator.js';
@@ -77,7 +77,9 @@ class FredyRuntime {
}
_findNew(listings) {
const newListings = listings.filter((o) => getKnownListings(this._jobKey, this._providerId)[o.id] == null);
const hashes = getKnownListingHashesForJobAndProvider(this._jobKey, this._providerId) || [];
const newListings = listings.filter((o) => !hashes.includes(o.id));
if (newListings.length === 0) {
throw new NoNewListingsWarning();
}
@@ -93,11 +95,7 @@ class FredyRuntime {
}
_save(newListings) {
const currentListings = getKnownListings(this._jobKey, this._providerId) || {};
newListings.forEach((listing) => {
currentListings[listing.id] = Date.now();
});
setKnownListings(this._jobKey, this._providerId, currentListings);
storeListings(this._jobKey, this._providerId, newListings);
return newListings;
}