From 8d95f052c6b78d4b09bebd76295ac1b359b8c4b0 Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Thu, 18 Sep 2025 15:38:23 +0200 Subject: [PATCH] Migrate to SQLite (#174) * Migrating Fredy from LowDb to SqLite :tada: * adding new sql migration system for future sql migrations * adding setting to change sqlite path for db files * create migration plan for graceful migration lowdb -> sqlite * Improving Documentation * adding test for sqliteconnection * upgrading dependencies * making nodejs 22 as min version * improve scraper * adding overwrite ability for db migra --- .github/workflows/check_source.yml | 2 +- .github/workflows/test.yml | 2 +- .gitignore | 3 +- README.md | 2 +- conf/config.json | 2 +- db/.gitkeep | 0 db/migrations/migrate.js | 199 +++++++++++ db/migrations/sql/0.init.js | 16 + .../sql/1.create-fredy-base-structure.js | 117 +++++++ index.js | 38 +- lib/FredyRuntime.js | 12 +- lib/api/routes/generalSettingsRoute.js | 4 +- lib/defaultConfig.js | 2 + lib/provider/einsAImmobilien.js | 4 +- lib/provider/immobilienDe.js | 6 +- lib/provider/immonet.js | 12 +- lib/provider/immoscout.js | 1 + lib/services/demoCleanup.js | 4 +- lib/services/storage/LowDashAdapter.js | 8 - lib/services/storage/SqliteConnection.js | 140 ++++++++ lib/services/storage/jobStorage.js | 216 +++++++----- lib/services/storage/listingsStorage.js | 180 +++++++--- lib/services/storage/userStorage.js | 279 +++++++++------ lib/utils.js | 100 +++++- package.json | 23 +- test/db/migrations/migrate.test.js | 329 ++++++++++++++++++ test/mocks/mockStore.js | 4 +- test/provider/einsAImmobilien.test.js | 1 + test/storage/SqliteConnection.test.js | 142 ++++++++ .../views/generalSettings/GeneralSettings.jsx | 40 ++- yarn.lock | 160 ++++----- 31 files changed, 1636 insertions(+), 412 deletions(-) create mode 100644 db/.gitkeep create mode 100644 db/migrations/migrate.js create mode 100644 db/migrations/sql/0.init.js create mode 100644 db/migrations/sql/1.create-fredy-base-structure.js delete mode 100644 lib/services/storage/LowDashAdapter.js create mode 100644 lib/services/storage/SqliteConnection.js create mode 100644 test/db/migrations/migrate.test.js create mode 100644 test/storage/SqliteConnection.test.js diff --git a/.github/workflows/check_source.yml b/.github/workflows/check_source.yml index a3d1b92..f0c34fa 100644 --- a/.github/workflows/check_source.yml +++ b/.github/workflows/check_source.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: 'yarn' - name: Install dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d5ce6ee..236ddf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: 'yarn' - run: yarn install diff --git a/.gitignore b/.gitignore index 45b5ac8..f59f123 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ ui/public/ -db/ +db/*.json +db/*.db* npm-debug.log .DS_Store .idea diff --git a/README.md b/README.md index 0f3aac6..9e47046 100755 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ docker logs fredy -f ### Manual (Node.js) -- Requirement: **Node.js 20 or higher** +- Requirement: **Node.js 22 or higher** - Install dependencies and start: ``` bash diff --git a/conf/config.json b/conf/config.json index e8c9521..eb00028 100644 --- a/conf/config.json +++ b/conf/config.json @@ -1 +1 @@ -{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null} \ No newline at end of file +{"interval":"60","port":9998,"workingHours":{"from":"","to":""},"demoMode":false,"analyticsEnabled":null,"sqlitepath":"/db"} \ No newline at end of file diff --git a/db/.gitkeep b/db/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/db/migrations/migrate.js b/db/migrations/migrate.js new file mode 100644 index 0000000..6fa2029 --- /dev/null +++ b/db/migrations/migrate.js @@ -0,0 +1,199 @@ +/** + * Migration Runner for better-sqlite3 + * I know there are external libs out there, but + * a) most of them are pretty bloated + * b) I wanted to have something that fit's this limited use-case + * c) I was searching for justifications anyway to build a migration system on my own. Don't judge me ;) + * + * Executes all migration files in db/migrations/sql in natural order. + * Each migration runs in its own transaction. If a migration fails, only that + * migration is rolled back and the process stops with a non-zero exit code. + * Already applied migrations are skipped using the schema_migrations table. + * + * Usage: + * CLI: yarn run migratedb + * Programmatic: + * import { runMigrations } from './db/migrations/migrate.js'; + * await runMigrations(); + * + * Migration file format (example: db/migrations/sql/1.add-users.js): + * export function up(db) { + * db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)"); + * } + * + */ +import fs from 'fs'; +import path from 'path'; +import {pathToFileURL} from 'url'; +import crypto from 'crypto'; +import SqliteConnection from '../../lib/services/storage/SqliteConnection.js'; +import logger from '../../lib/services/logger.js'; + +const ROOT = path.resolve('.'); +const MIGRATIONS_DIR = path.join(ROOT, 'db', 'migrations', 'sql'); + +/** + * Ensures that the given directory exists, creating it recursively if needed. + * @param {string} p - Path to the directory. + */ +function ensureDir(p) { + if (!fs.existsSync(p)) fs.mkdirSync(p, {recursive: true}); +} + +/** + * Lists all migration files in the migrations directory. + * Migration files must follow the format: .