mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
fixing docker migration path
This commit is contained in:
orangecoding
197
lib/services/storage/migrations/migrate.js
Normal file
197
lib/services/storage/migrations/migrate.js
Normal file
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* Migration Runner for better-sqlite3
|
||||
* I know there are external libs out there, but
|
||||
* a) most of them are pretty bloated
|
||||
* b) I wanted to have something that fit's this limited use-case
|
||||
* c) I was searching for justifications anyway to build a migration system on my own. Don't judge me ;)
|
||||
*
|
||||
* Executes all migration files in lib/services/storage/migrations/sql in natural order.
|
||||
* Each migration runs in its own transaction. If a migration fails, only that
|
||||
* migration is rolled back and the process stops with a non-zero exit code.
|
||||
* Already applied migrations are skipped using the schema_migrations table.
|
||||
*
|
||||
* Usage:
|
||||
* CLI: yarn run migratedb
|
||||
* Programmatic:
|
||||
* import { runMigrations } from './lib/services/storage/migrations/migrate.js';
|
||||
* await runMigrations();
|
||||
*
|
||||
* Migration file format (example: lib/services/storage/migrations/sql/1.add-users.js):
|
||||
* export function up(db) {
|
||||
* db.exec("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)");
|
||||
* }
|
||||
*
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import crypto from 'crypto';
|
||||
import SqliteConnection from '../SqliteConnection.js';
|
||||
import logger from '../../logger.js';
|
||||
|
||||
const ROOT = path.resolve('.');
|
||||
const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
|
||||
|
||||
/**
|
||||
* Ensures that the given directory exists, creating it recursively if needed.
|
||||
* @param {string} p - Path to the directory.
|
||||
*/
|
||||
function ensureDir(p) {
|
||||
if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all migration files in the migrations directory.
|
||||
* Migration files must follow the format: <number>.<label>.js
|
||||
* @returns {Array<{id:number, name:string, label:string, path:string}>}
|
||||
*/
|
||||
function listMigrationFiles() {
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
return fs
|
||||
.readdirSync(MIGRATIONS_DIR)
|
||||
.filter((f) => /^\d+\..+\.js$/.test(f))
|
||||
.map((file) => {
|
||||
const [idStr, ...rest] = file.split('.');
|
||||
const id = Number.parseInt(idStr, 10);
|
||||
const label = rest.slice(0, -1).join('.');
|
||||
const fullPath = path.join(MIGRATIONS_DIR, file);
|
||||
return { id, name: file, label, path: fullPath };
|
||||
})
|
||||
.sort((a, b) => (a.id === b.id ? a.name.localeCompare(b.name) : a.id - b.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the SHA-256 checksum of a file.
|
||||
* @param {string} filePath - Path to the file.
|
||||
* @returns {string} Hex-encoded checksum.
|
||||
*/
|
||||
function sha256File(filePath) {
|
||||
const buf = fs.readFileSync(filePath);
|
||||
return crypto.createHash('sha256').update(buf).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamically imports a migration module and returns its `up` function.
|
||||
* @param {string} filePath - Path to the migration file.
|
||||
* @returns {Promise<Function>} Migration function.
|
||||
* @throws {Error} If the migration file does not export a valid function.
|
||||
*/
|
||||
async function loadMigrationModule(filePath) {
|
||||
const testImporter = globalThis.__TEST_MIGRATE_IMPORT__;
|
||||
const url = pathToFileURL(filePath);
|
||||
const mod = testImporter ? await testImporter(filePath, url) : await import(url.href);
|
||||
const fn = mod.up || mod.default;
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`Migration ${filePath} must export function up(db) or default function(db)`);
|
||||
}
|
||||
return fn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all previously executed migrations from the database.
|
||||
* @returns {Map<string,string>} Map of migration name to checksum.
|
||||
*/
|
||||
function loadExecutedMigrations() {
|
||||
const executed = new Map();
|
||||
const hasTable = SqliteConnection.tableExists('schema_migrations');
|
||||
if (!hasTable) return executed;
|
||||
const rows = SqliteConnection.query('SELECT name, checksum FROM schema_migrations ORDER BY applied_at ASC');
|
||||
for (const r of rows) executed.set(r.name, r.checksum);
|
||||
return executed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes all pending migrations.
|
||||
* Ensures that each migration runs inside its own transaction.
|
||||
* Already applied migrations are skipped, unless checksum updates are allowed.
|
||||
* On success, updates the schema_migrations table and runs PRAGMA optimize.
|
||||
*/
|
||||
export async function runMigrations() {
|
||||
ensureDir(path.join(ROOT, 'db'));
|
||||
ensureDir(MIGRATIONS_DIR);
|
||||
|
||||
const files = listMigrationFiles();
|
||||
if (files.length === 0) {
|
||||
logger.info('No migration files found under', MIGRATIONS_DIR);
|
||||
return;
|
||||
}
|
||||
|
||||
SqliteConnection.getConnection();
|
||||
|
||||
const executed = loadExecutedMigrations();
|
||||
|
||||
let appliedMigrations = 0;
|
||||
for (const m of files) {
|
||||
const checksum = sha256File(m.path);
|
||||
|
||||
if (executed.has(m.name)) {
|
||||
const prev = executed.get(m.name);
|
||||
if (prev !== checksum) {
|
||||
const allow = (process.env.MIGRATION_ALLOW_CHECKSUM_UPDATE || '').toLowerCase();
|
||||
const allowUpdate = allow === '1' || allow === 'true' || allow === 'yes';
|
||||
if (allowUpdate) {
|
||||
logger.warn(
|
||||
`Checksum mismatch for already executed migration ${m.name}, but MIGRATION_ALLOW_CHECKSUM_UPDATE is enabled. ` +
|
||||
`Updating recorded checksum and continuing without re-running the migration.`,
|
||||
);
|
||||
SqliteConnection.execute('UPDATE schema_migrations SET checksum = @checksum WHERE name = @name', {
|
||||
checksum,
|
||||
name: m.name,
|
||||
});
|
||||
executed.set(m.name, checksum);
|
||||
} else {
|
||||
throw new Error(
|
||||
`Checksum mismatch for already executed migration ${m.name}. ` +
|
||||
`Do not modify applied migrations. Create a new migration instead.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
appliedMigrations++;
|
||||
logger.info(`Applying migration: ${m.name}`);
|
||||
const fn = await loadMigrationModule(m.path);
|
||||
|
||||
try {
|
||||
let duration = 0;
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const t0 = Date.now();
|
||||
fn(db);
|
||||
duration = Date.now() - t0;
|
||||
db.prepare(
|
||||
"INSERT INTO schema_migrations (name, checksum, applied_at, duration_ms) VALUES (?, ?, datetime('now'), ?)",
|
||||
).run(m.name, checksum, duration);
|
||||
});
|
||||
logger.info(`Migration applied: ${m.name} (${duration} ms)`);
|
||||
} catch (e) {
|
||||
logger.error(`Migration failed and was rolled back: ${m.name}`, e);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
SqliteConnection.optimize();
|
||||
if (appliedMigrations > 0) {
|
||||
logger.info('All migrations completed successfully.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects whether the current file is being executed directly via Node.js.
|
||||
* This allows `node lib/services/storage/migrations/migrate.js` to run migrations directly.
|
||||
* @returns {boolean} True if the file was run directly.
|
||||
*/
|
||||
const isDirectRun = (() => {
|
||||
try {
|
||||
const thisFile = import.meta.url;
|
||||
const invoked = pathToFileURL(process.argv[1] || '').href;
|
||||
return thisFile === invoked;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
if (isDirectRun) {
|
||||
await runMigrations();
|
||||
}
|
||||
16
lib/services/storage/migrations/sql/0.init.js
Normal file
16
lib/services/storage/migrations/sql/0.init.js
Normal file
@@ -0,0 +1,16 @@
|
||||
// Initial migration: creates schema_migrations table used by the migration runner.
|
||||
//
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
checksum TEXT NOT NULL,
|
||||
applied_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_schema_migrations_applied_at
|
||||
ON schema_migrations(applied_at);
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// Migration: Create fredy's base structure (users, jobs and listings) import initial
|
||||
// data from JSON files if present. (This applies only for jobs and users, for the old jobListingData,
|
||||
// I cannot migrate the data as the new format is totally different.
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { toJson } from '../../../../utils.js';
|
||||
|
||||
export function up(db) {
|
||||
// 1) Create tables
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS users
|
||||
(
|
||||
id TEXT PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
password TEXT NOT NULL,
|
||||
last_login INTEGER,
|
||||
is_admin INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_users_username ON users (username);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS jobs
|
||||
(
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
name TEXT,
|
||||
blacklist JSONB NOT NULL DEFAULT '[]',
|
||||
provider JSONB NOT NULL DEFAULT '[]',
|
||||
notification_adapter JSONB NOT NULL DEFAULT '[]',
|
||||
FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_user_id ON jobs (user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_enabled ON jobs (enabled);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS listings
|
||||
(
|
||||
id TEXT PRIMARY KEY,
|
||||
created_at INTEGER,
|
||||
hash TEXT,
|
||||
provider TEXT,
|
||||
job_id TEXT,
|
||||
price INTEGER,
|
||||
size INTEGER,
|
||||
title TEXT,
|
||||
image_url TEXT,
|
||||
description TEXT,
|
||||
address TEXT,
|
||||
link TEXT,
|
||||
FOREIGN KEY (job_id) REFERENCES jobs (id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_listings_hash ON listings (hash);
|
||||
`);
|
||||
|
||||
// 2) Optionally import data from JSON files if present for users and jobs
|
||||
const ROOT = path.resolve('.');
|
||||
const usersJsonPath = path.join(ROOT, 'db', 'users.json');
|
||||
const jobsJsonPath = path.join(ROOT, 'db', 'jobs.json');
|
||||
|
||||
// Insert users
|
||||
if (fs.existsSync(usersJsonPath)) {
|
||||
try {
|
||||
const raw = fs.readFileSync(usersJsonPath, 'utf8');
|
||||
const json = JSON.parse(raw);
|
||||
const arr = Array.isArray(json?.user) ? json.user : [];
|
||||
if (arr.length > 0) {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO users (id, username, password, last_login, is_admin)
|
||||
VALUES (@id, @username, @password, @last_login, @is_admin)`,
|
||||
);
|
||||
for (const u of arr) {
|
||||
stmt.run({
|
||||
id: u.id,
|
||||
username: u.username,
|
||||
password: u.password,
|
||||
last_login: u.lastLogin ?? null,
|
||||
is_admin: u.isAdmin ? 1 : 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// If parsing fails, let it throw to rollback the migration
|
||||
throw new Error(`Failed to import users from ${usersJsonPath}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert jobs
|
||||
if (fs.existsSync(jobsJsonPath)) {
|
||||
try {
|
||||
const raw = fs.readFileSync(jobsJsonPath, 'utf8');
|
||||
const json = JSON.parse(raw);
|
||||
const arr = Array.isArray(json?.jobs) ? json.jobs : [];
|
||||
if (arr.length > 0) {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter)
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter)`,
|
||||
);
|
||||
for (const j of arr) {
|
||||
stmt.run({
|
||||
id: j.id,
|
||||
user_id: j.userId,
|
||||
enabled: j.enabled ? 1 : 0,
|
||||
name: j.name ?? null,
|
||||
blacklist: toJson(j.blacklist ?? []),
|
||||
provider: toJson(j.provider ?? []),
|
||||
notification_adapter: toJson(j.notificationAdapter ?? []),
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to import jobs from ${jobsJsonPath}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user