diff --git a/index.js b/index.js index 2c059b1..75a763b 100755 --- a/index.js +++ b/index.js @@ -4,7 +4,6 @@ */ import fs from 'fs'; -import path from 'path'; import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js'; import * as similarityCache from './lib/services/similarity-check/similarityCache.js'; import * as jobStorage from './lib/services/storage/jobStorage.js'; @@ -18,7 +17,7 @@ import logger from './lib/services/logger.js'; import { bus } from './lib/services/events/event-bus.js'; import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js'; import { getSettings } from './lib/services/storage/settingsStorage.js'; -import SqliteConnection from './lib/services/storage/SqliteConnection.js'; +import SqliteConnection, { computeDbPath } from './lib/services/storage/SqliteConnection.js'; //in the config, we store the path of the sqlite file, thus we must check if it is available const isConfigAccessible = await checkIfConfigIsAccessible(); @@ -38,11 +37,9 @@ await runMigrations(); const settings = await getSettings(); // Ensure sqlite directory exists before loading anything else (based on config.sqlitepath) -const rawDir = settings.sqlitepath || '/db'; -const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir; -const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir); -if (!fs.existsSync(absDir)) { - fs.mkdirSync(absDir, { recursive: true }); +const { dir: sqliteDir } = await computeDbPath(); +if (!fs.existsSync(sqliteDir)) { + fs.mkdirSync(sqliteDir, { recursive: true }); } // Load provider modules once at startup diff --git a/lib/api/api.js b/lib/api/api.js index bec5158..abce65a 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -22,6 +22,7 @@ import { listingsRouter } from './routes/listingsRouter.js'; import { getSettings } from '../services/storage/settingsStorage.js'; import { featureRouter } from './routes/featureRouter.js'; import { dashboardRouter } from './routes/dashboardRouter.js'; +import { backupRouter } from './routes/backupRouter.js'; const service = restana(); const staticService = files(path.join(getDirName(), '../ui/public')); const PORT = (await getSettings()).port || 9998; @@ -40,6 +41,7 @@ service.use('/api/features', authInterceptor()); service.use('/api/admin', adminInterceptor()); service.use('/api/jobs/notificationAdapter', notificationAdapterRouter); service.use('/api/admin/generalSettings', generalSettingsRouter); +service.use('/api/admin/backup', backupRouter); service.use('/api/jobs/provider', providerRouter); service.use('/api/admin/users', userRouter); service.use('/api/version', versionRouter); diff --git a/lib/api/routes/backupRouter.js b/lib/api/routes/backupRouter.js new file mode 100644 index 0000000..2e35757 --- /dev/null +++ b/lib/api/routes/backupRouter.js @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import restana from 'restana'; +import { + buildBackupFileName, + createBackupZip, + precheckRestore, + restoreFromZip, +} from '../../services/storage/backupRestoreService.js'; + +/** + * Backup & Restore Admin Router + * + * Endpoints: + * - GET /api/admin/backup + * Returns the current database as a zip download. Content-Type: application/zip + * - POST /api/admin/backup/restore?dryRun=true + * Accepts a zip file (raw body). Returns a compatibility report, does not restore. + * - POST /api/admin/backup/restore?force=true|false + * Accepts a zip file (raw body). Restores the database; when incompatible and force=false, returns 400. + */ +const service = restana(); +const backupRouter = service.newRouter(); + +backupRouter.get('/', async (req, res) => { + const zipBuffer = await createBackupZip(); + const fileName = await buildBackupFileName(); + res.setHeader('Content-Type', 'application/zip'); + res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`); + res.send(zipBuffer); +}); + +/** + * Read the full request body as a Buffer. Used for raw zip uploads. + * @param {import('http').IncomingMessage} req + * @returns {Promise} + */ +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on('data', (c) => chunks.push(c)); + req.on('end', () => resolve(Buffer.concat(chunks))); + req.on('error', (e) => reject(e)); + }); +} + +// Upload endpoint. Accepts raw zip (Content-Type: application/zip or application/octet-stream) +// Query parameters: +// - dryRun=true => only validate and return compatibility info +// - force=true => proceed even if incompatible +backupRouter.post('/restore', async (req, res) => { + const { dryRun = 'false', force = 'false' } = req.query || {}; + const doDryRun = String(dryRun) === 'true'; + const doForce = String(force) === 'true'; + const body = await readBody(req); + + if (doDryRun) { + res.body = await precheckRestore(body); + return res.send(); + } + + try { + res.body = await restoreFromZip(body, { force: doForce }); + return res.send(); + } catch (e) { + res.statusCode = 400; + res.body = { message: e?.message || 'Restore failed', details: e?.payload || null }; + return res.send(); + } +}); + +export { backupRouter }; diff --git a/lib/services/storage/SqliteConnection.js b/lib/services/storage/SqliteConnection.js index 0eb1f90..06e9d93 100644 --- a/lib/services/storage/SqliteConnection.js +++ b/lib/services/storage/SqliteConnection.js @@ -155,3 +155,21 @@ class SqliteConnection { } export default SqliteConnection; + +// Centralized DB path computation to avoid duplication across modules +// Returns: { dir, dbPath } +/** + * Compute the absolute SQLite database directory and file path based on configuration. + * Ensures the directory exists on disk. + * @returns {Promise<{dir:string, dbPath:string}>} Absolute directory and database file path. + */ +export async function computeDbPath() { + const cfg = await readConfigFromStorage(); + const rawDir = cfg?.sqlitepath && cfg.sqlitepath.length > 0 ? cfg.sqlitepath : '/db'; + const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir; + const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir); + const dbPath = path.join(absDir, 'listings.db'); + const dir = path.dirname(dbPath); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + return { dir: absDir, dbPath }; +} diff --git a/lib/services/storage/backupRestoreService.js b/lib/services/storage/backupRestoreService.js new file mode 100644 index 0000000..4d23f41 --- /dev/null +++ b/lib/services/storage/backupRestoreService.js @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2025 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import SqliteConnection, { computeDbPath } from './SqliteConnection.js'; +import logger from '../../services/logger.js'; +import { getPackageVersion } from '../../utils.js'; +import { runMigrations, listMigrationFiles } from './migrations/migrate.js'; + +/** + * Lazily resolve and cache the AdmZip constructor via dynamic import. + * This keeps startup costs low and avoids ESM/CJS interop pitfalls. + * @returns {Promise} AdmZip constructor (class) + */ +let _AdmZipSingleton = null; +async function getAdmZip() { + if (_AdmZipSingleton) return _AdmZipSingleton; + const mod = await import('adm-zip'); + _AdmZipSingleton = mod.default || mod; + return _AdmZipSingleton; +} + +/** + * Extract numeric migration id from a migration file name like "12.add-users.js". + * @param {string} name + * @returns {number} Parsed id or 0 when not parsable + */ +function parseMigrationIdFromName(name) { + if (typeof name !== 'string') return 0; + const m = name.match(/^(\d+)\./); + return m ? parseInt(m[1], 10) : 0; +} + +/** + * Read the highest migration id from available migration files. + * @returns {number} Highest migration id from files, or 0 when none. + */ +function getLatestMigrationIdFromFiles() { + try { + const files = listMigrationFiles(); + const ids = files.map((f) => f.id); + return ids.length > 0 ? Math.max(...ids) : 0; + } catch (e) { + logger.warn('Failed to scan migrations directory:', e.message); + return 0; + } +} + +/** + * Inspect the current database and return the highest applied migration id. + * @returns {number} Max id from schema_migrations, or 0 when table/rows are missing. + */ +function getCurrentDbMigration() { + try { + const exists = SqliteConnection.tableExists('schema_migrations'); + if (!exists) return 0; + const rows = SqliteConnection.query('SELECT name FROM schema_migrations'); + if (!rows || rows.length === 0) return 0; + return rows.reduce((max, r) => Math.max(max, parseMigrationIdFromName(r.name)), 0); + } catch (e) { + logger.warn('Failed to read current DB migration:', e.message); + return 0; + } +} + +/** + * Create a consistent SQLite snapshot using the native backup API into a temp folder. + * @returns {Promise<{tempDir:string, backupPath:string}>} + */ +async function createTempBackupFile() { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-db-')); + const backupPath = path.join(tempDir, 'listings.db'); + // Ensure connection is open and create a consistent snapshot + const db = SqliteConnection.getConnection(); + await db.backup(backupPath); + return { tempDir, backupPath }; +} + +/** + * Build a zip buffer that contains the DB snapshot and metadata marker. + * Files: + * - listings.db + * - fredy-backup.json { formatVersion, createdAt, dbMigration, fredyVersion } + * @returns {Promise} + */ +async function buildBackupZipBuffer() { + const { backupPath, tempDir } = await createTempBackupFile(); + try { + const AdmZip = await getAdmZip(); + const zip = new AdmZip(); + const meta = { + formatVersion: 1, + createdAt: new Date().toISOString(), + dbMigration: getCurrentDbMigration(), + fredyVersion: await getPackageVersion(), + }; + // add files + zip.addLocalFile(backupPath, '', 'listings.db'); + zip.addFile('fredy-backup.json', Buffer.from(JSON.stringify(meta, null, 2), 'utf-8')); + return zip.toBuffer(); + } finally { + // cleanup temp + try { + fs.rmSync(tempDir, { recursive: true, force: true }); + } catch (e) { + logger.debug('Failed to cleanup temp backup dir:', e.message); + } + } +} + +/** + * Read and parse the metadata file from a backup zip buffer. + * @param {Buffer} zipBuffer + * @returns {Promise} Parsed JSON or null when missing/invalid. + */ +async function readMetadataFromZip(zipBuffer) { + const AdmZip = await getAdmZip(); + const zip = new AdmZip(zipBuffer); + const entry = zip.getEntry('fredy-backup.json'); + if (!entry) return null; + try { + const txt = entry.getData().toString('utf-8'); + return JSON.parse(txt); + } catch { + return null; + } +} + +/** + * Check if a backup zip contains a listings.db entry. + * @param {Buffer} zipBuffer + * @returns {Promise} + */ +async function hasListingsDbInZip(zipBuffer) { + const AdmZip = await getAdmZip(); + const zip = new AdmZip(zipBuffer); + return zip.getEntry('listings.db') != null || zip.getEntries().some((e) => /listings\.db$/i.test(e.entryName)); +} + +/** + * Extract the listings.db from a backup zip buffer to a temp directory. + * @param {Buffer} zipBuffer + * @returns {Promise<{tempDir:string, dbPath:string}>} + */ +async function extractListingsDbToTemp(zipBuffer) { + const AdmZip = await getAdmZip(); + const zip = new AdmZip(zipBuffer); + const entry = zip.getEntry('listings.db') || zip.getEntries().find((e) => /listings\.db$/i.test(e.entryName)); + if (!entry) throw new Error('Backup zip does not contain listings.db'); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-restore-')); + const outPath = path.join(tempDir, 'listings.db'); + fs.writeFileSync(outPath, entry.getData()); + return { tempDir, dbPath: outPath }; +} + +/** + * Public: Create a backup zip buffer ready for download. + * @returns {Promise} + */ +export async function createBackupZip() { + return buildBackupZipBuffer(); +} + +/** + * Analyze a backup zip for compatibility with the current codebase. + * - Missing DB yields danger. + * - Newer backup migration than required yields danger. + * - Older backup yields warning but is considered compatible (auto-migrate). + * - Equal version yields info. + * @param {Buffer} zipBuffer + * @returns {Promise<{compatible:boolean,severity:'danger'|'warning'|'info',message:string,backupMigration:number|null,requiredMigration:number,fredyVersion?:string|null}>>} + */ +export async function precheckRestore(zipBuffer) { + if (!zipBuffer || zipBuffer.length === 0) { + return { + compatible: false, + severity: 'danger', + message: 'Empty upload', + backupMigration: null, + requiredMigration: getLatestMigrationIdFromFiles(), + }; + } + if (!(await hasListingsDbInZip(zipBuffer))) { + return { + compatible: false, + severity: 'danger', + message: 'Zip file is missing the database file (listings.db).', + backupMigration: null, + requiredMigration: getLatestMigrationIdFromFiles(), + }; + } + const meta = await readMetadataFromZip(zipBuffer); + const requiredMigration = getLatestMigrationIdFromFiles(); + const backupMigration = meta?.dbMigration ?? null; + const fredyVersion = meta?.fredyVersion ?? null; + + if (backupMigration == null) { + return { + compatible: false, + severity: 'danger', + message: + 'Backup metadata is missing the migration marker. Cannot validate compatibility. It is NOT advised to continue!', + backupMigration, + requiredMigration, + fredyVersion, + }; + } + + if (backupMigration > requiredMigration) { + return { + compatible: false, + severity: 'danger', + message: + 'Backup schema is newer than this Fredy version. Please upgrade Fredy to a version that supports this backup or proceed at your own risk.', + backupMigration, + requiredMigration, + fredyVersion, + }; + } + + if (backupMigration < requiredMigration) { + return { + compatible: true, + severity: 'warning', + message: + 'Backup contains an older database schema than this Fredy version requires. We will apply automatic migrations right after the restore to upgrade the database.', + backupMigration, + requiredMigration, + fredyVersion, + }; + } + + return { + compatible: true, + severity: 'info', + message: 'Backup is compatible with the current Fredy version.', + backupMigration, + requiredMigration, + fredyVersion, + }; +} + +/** + * Perform a restore from a validated backup zip. + * - Optionally forces restore when incompatible. + * - Replaces the on-disk DB and runs migrations when needed. + * @param {Buffer} zipBuffer + * @param {{force?:boolean}} [opts] + * @returns {Promise<{restored:true,warning:string|null,details:any}>} + * @throws Error with code 'INCOMPATIBLE' when not forced and incompatible + */ +export async function restoreFromZip(zipBuffer, { force = false } = {}) { + const check = await precheckRestore(zipBuffer); + if (!check.compatible && !force) { + const err = new Error(check.message || 'Backup is incompatible'); + err.code = 'INCOMPATIBLE'; + err.payload = check; + throw err; + } + + const { dbPath } = await computeDbPath(); + const { tempDir, dbPath: tempDbPath } = await extractListingsDbToTemp(zipBuffer); + + try { + // Close existing connection to allow file replacement + SqliteConnection.close(); + + // Backup existing DB file + try { + if (fs.existsSync(dbPath)) { + const backupName = `${dbPath}.bak-${Date.now()}`; + fs.copyFileSync(dbPath, backupName); + } + } catch (e) { + logger.warn('Failed to create on-disk backup copy of current DB:', e.message); + } + + // Replace DB with the one from the zip + fs.copyFileSync(tempDbPath, dbPath); + + // Re-run migrations when needed + if (check.backupMigration < check.requiredMigration) { + await runMigrations(); + } else { + // Ensure we can re-open the DB + SqliteConnection.getConnection(); + } + } finally { + try { + fs.rmSync(tempDir, { recursive: true, force: true }); + } catch (e) { + logger.debug('Failed to cleanup temp restore dir:', e.message); + } + } + + return { restored: true, warning: check.severity !== 'info' ? check.message : null, details: check }; +} + +/** + * Build the backup file name with current date and Fredy version. + * Pattern: YYYY-MM-DD-FredyBackup-{version}.zip + * @returns {Promise} + */ +export async function buildBackupFileName() { + const dt = new Date(); + const yyyy = dt.getFullYear(); + const mm = String(dt.getMonth() + 1).padStart(2, '0'); + const dd = String(dt.getDate()).padStart(2, '0'); + const version = await getPackageVersion(); + return `${yyyy}-${mm}-${dd}-FredyBackup-${version}.zip`.replaceAll(' ', ''); +} diff --git a/lib/services/storage/migrations/migrate.js b/lib/services/storage/migrations/migrate.js index 0b5281d..136fe9f 100644 --- a/lib/services/storage/migrations/migrate.js +++ b/lib/services/storage/migrations/migrate.js @@ -35,7 +35,11 @@ import SqliteConnection from '../SqliteConnection.js'; import logger from '../../logger.js'; const ROOT = path.resolve('.'); -const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql'); +/** + * Absolute path to the migrations directory (lib/services/storage/migrations/sql). + * @type {string} + */ +export const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql'); /** * Ensures that the given directory exists, creating it recursively if needed. @@ -50,7 +54,7 @@ function ensureDir(p) { * Migration files must follow the format: .