Compare commits

..

4 Commits

Author SHA1 Message Date
orangecoding
6e8a35a836 adding backup/restore ability 2025-12-17 15:48:56 +01:00
orangecoding
87771655a8 adding new dashboard view. Muchas wow 2025-12-14 12:23:59 +01:00
Christian Kellner
87b5673bf0 Update package.json 2025-12-12 22:22:50 +01:00
lorem-ipsum-dolor-sit
9291155cc2 fix: catch error (#246) 2025-12-12 22:21:49 +01:00
39 changed files with 1628 additions and 1218 deletions

Binary file not shown.

Before

Width:  |  Height:  |  Size: 372 KiB

After

Width:  |  Height:  |  Size: 402 KiB

View File

@@ -4,7 +4,6 @@
*/
import fs from 'fs';
import path from 'path';
import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
import * as jobStorage from './lib/services/storage/jobStorage.js';
@@ -18,7 +17,7 @@ import logger from './lib/services/logger.js';
import { bus } from './lib/services/events/event-bus.js';
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
import { getSettings } from './lib/services/storage/settingsStorage.js';
import SqliteConnection from './lib/services/storage/SqliteConnection.js';
import SqliteConnection, { computeDbPath } from './lib/services/storage/SqliteConnection.js';
//in the config, we store the path of the sqlite file, thus we must check if it is available
const isConfigAccessible = await checkIfConfigIsAccessible();
@@ -38,11 +37,9 @@ await runMigrations();
const settings = await getSettings();
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
const rawDir = settings.sqlitepath || '/db';
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
if (!fs.existsSync(absDir)) {
fs.mkdirSync(absDir, { recursive: true });
const { dir: sqliteDir } = await computeDbPath();
if (!fs.existsSync(sqliteDir)) {
fs.mkdirSync(sqliteDir, { recursive: true });
}
// Load provider modules once at startup
@@ -87,15 +84,19 @@ const execute = () => {
job.provider
.filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
.forEach(async (prov) => {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyPipeline(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
try {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyPipeline(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
} catch (error) {
logger.error(error);
}
});
});
} else {

View File

@@ -6,7 +6,6 @@
import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
import { analyticsRouter } from './routes/analyticsRouter.js';
import { providerRouter } from './routes/providerRouter.js';
import { versionRouter } from './routes/versionRouter.js';
import { loginRouter } from './routes/loginRoute.js';
@@ -22,6 +21,8 @@ import logger from '../services/logger.js';
import { listingsRouter } from './routes/listingsRouter.js';
import { getSettings } from '../services/storage/settingsStorage.js';
import { featureRouter } from './routes/featureRouter.js';
import { dashboardRouter } from './routes/dashboardRouter.js';
import { backupRouter } from './routes/backupRouter.js';
const service = restana();
const staticService = files(path.join(getDirName(), '../ui/public'));
const PORT = (await getSettings()).port || 9998;
@@ -33,19 +34,22 @@ service.use('/api/admin', authInterceptor());
service.use('/api/jobs', authInterceptor());
service.use('/api/version', authInterceptor());
service.use('/api/listings', authInterceptor());
service.use('/api/dashboard', authInterceptor());
service.use('/api/features', authInterceptor());
// /admin can only be accessed when user is having admin permissions
service.use('/api/admin', adminInterceptor());
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
service.use('/api/admin/generalSettings', generalSettingsRouter);
service.use('/api/admin/backup', backupRouter);
service.use('/api/jobs/provider', providerRouter);
service.use('/api/jobs/insights', analyticsRouter);
service.use('/api/admin/users', userRouter);
service.use('/api/version', versionRouter);
service.use('/api/jobs', jobRouter);
service.use('/api/login', loginRouter);
service.use('/api/listings', listingsRouter);
service.use('/api/features', featureRouter);
service.use('/api/dashboard', dashboardRouter);
//this route is unsecured intentionally as it is being queried from the login page
service.use('/api/demo', demoRouter);

View File

@@ -1,15 +0,0 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import * as listingStorage from '../../services/storage/listingsStorage.js';
const service = restana();
const analyticsRouter = service.newRouter();
analyticsRouter.get('/:jobId', async (req, res) => {
const { jobId } = req.params;
res.body = listingStorage.getListingProviderDataForAnalytics(jobId) || {};
res.send();
});
export { analyticsRouter };

View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import {
buildBackupFileName,
createBackupZip,
precheckRestore,
restoreFromZip,
} from '../../services/storage/backupRestoreService.js';
/**
* Backup & Restore Admin Router
*
* Endpoints:
* - GET /api/admin/backup
* Returns the current database as a zip download. Content-Type: application/zip
* - POST /api/admin/backup/restore?dryRun=true
* Accepts a zip file (raw body). Returns a compatibility report, does not restore.
* - POST /api/admin/backup/restore?force=true|false
* Accepts a zip file (raw body). Restores the database; when incompatible and force=false, returns 400.
*/
const service = restana();
const backupRouter = service.newRouter();
backupRouter.get('/', async (req, res) => {
const zipBuffer = await createBackupZip();
const fileName = await buildBackupFileName();
res.setHeader('Content-Type', 'application/zip');
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
res.send(zipBuffer);
});
/**
* Read the full request body as a Buffer. Used for raw zip uploads.
* @param {import('http').IncomingMessage} req
* @returns {Promise<Buffer>}
*/
function readBody(req) {
return new Promise((resolve, reject) => {
const chunks = [];
req.on('data', (c) => chunks.push(c));
req.on('end', () => resolve(Buffer.concat(chunks)));
req.on('error', (e) => reject(e));
});
}
// Upload endpoint. Accepts raw zip (Content-Type: application/zip or application/octet-stream)
// Query parameters:
// - dryRun=true => only validate and return compatibility info
// - force=true => proceed even if incompatible
backupRouter.post('/restore', async (req, res) => {
const { dryRun = 'false', force = 'false' } = req.query || {};
const doDryRun = String(dryRun) === 'true';
const doForce = String(force) === 'true';
const body = await readBody(req);
if (doDryRun) {
res.body = await precheckRestore(body);
return res.send();
}
try {
res.body = await restoreFromZip(body, { force: doForce });
return res.send();
} catch (e) {
res.statusCode = 400;
res.body = { message: e?.message || 'Restore failed', details: e?.payload || null };
return res.send();
}
});
export { backupRouter };

View File

@@ -0,0 +1,71 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import restana from 'restana';
import * as jobStorage from '../../services/storage/jobStorage.js';
import * as userStorage from '../../services/storage/userStorage.js';
import { getListingsKpisForJobIds, getProviderDistributionForJobIds } from '../../services/storage/listingsStorage.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
export const dashboardRouter = service.newRouter();
function isAdmin(req) {
const user = req.session?.currentUser ? userStorage.getUser(req.session.currentUser) : null;
return !!user?.isAdmin;
}
function getAccessibleJobs(req) {
const currentUser = req.session.currentUser;
const admin = isAdmin(req);
return jobStorage
.getJobs()
.filter((job) => admin || job.userId === currentUser || job.shared_with_user.includes(currentUser));
}
function cap(val) {
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
}
dashboardRouter.get('/', async (req, res) => {
const jobs = getAccessibleJobs(req);
const settings = await getSettings();
// KPIs
const totalJobs = jobs.length;
const totalListings = jobs.reduce((sum, j) => sum + (j.numberOfFoundListings || 0), 0);
const jobIds = jobs.map((j) => j.id);
const { numberOfActiveListings, avgPriceOfListings } = getListingsKpisForJobIds(jobIds);
// Build Pie data in a simple shape the frontend can consume directly
// Shape: { labels: string[], values: number[] } with values as percentages
const providerPieRaw = getProviderDistributionForJobIds(jobIds);
const providerPie = Array.isArray(providerPieRaw)
? {
labels: providerPieRaw.map((p) => cap(p.type)),
values: providerPieRaw.map((p) => Number(p.value) || 0),
}
: providerPieRaw && typeof providerPieRaw === 'object'
? {
labels: Array.isArray(providerPieRaw.labels) ? providerPieRaw.labels : [],
values: Array.isArray(providerPieRaw.values) ? providerPieRaw.values : [],
}
: { labels: [], values: [] };
res.body = {
general: {
interval: settings.interval,
lastRun: settings.lastRun || null,
nextRun: settings.lastRun == null ? 0 : settings.lastRun + settings.interval * 60000,
},
kpis: {
totalJobs,
totalListings,
numberOfActiveListings,
avgPriceOfListings,
},
pie: providerPie,
};
res.send();
});

View File

@@ -9,7 +9,6 @@ import * as userStorage from '../../services/storage/userStorage.js';
import { isAdmin } from '../security.js';
import logger from '../../services/logger.js';
import { bus } from '../../services/events/event-bus.js';
import { getSettings } from '../../services/storage/settingsStorage.js';
const service = restana();
const jobRouter = service.newRouter();
@@ -48,15 +47,6 @@ jobRouter.get('/', async (req, res) => {
res.send();
});
jobRouter.get('/processingTimes', async (req, res) => {
const settings = await getSettings();
res.body = {
interval: settings.interval,
lastRun: settings.lastRun || null,
};
res.send();
});
jobRouter.post('/startAll', async (req, res) => {
bus.emit('jobs:runAll');
res.send();

View File

@@ -155,3 +155,21 @@ class SqliteConnection {
}
export default SqliteConnection;
// Centralized DB path computation to avoid duplication across modules
// Returns: { dir, dbPath }
/**
* Compute the absolute SQLite database directory and file path based on configuration.
* Ensures the directory exists on disk.
* @returns {Promise<{dir:string, dbPath:string}>} Absolute directory and database file path.
*/
export async function computeDbPath() {
const cfg = await readConfigFromStorage();
const rawDir = cfg?.sqlitepath && cfg.sqlitepath.length > 0 ? cfg.sqlitepath : '/db';
const relDir = rawDir.startsWith('/') ? rawDir.slice(1) : rawDir;
const absDir = path.isAbsolute(relDir) ? relDir : path.join(process.cwd(), relDir);
const dbPath = path.join(absDir, 'listings.db');
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
return { dir: absDir, dbPath };
}

View File

@@ -0,0 +1,315 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import fs from 'fs';
import os from 'os';
import path from 'path';
import SqliteConnection, { computeDbPath } from './SqliteConnection.js';
import logger from '../../services/logger.js';
import { getPackageVersion } from '../../utils.js';
import { runMigrations, listMigrationFiles } from './migrations/migrate.js';
/**
* Lazily resolve and cache the AdmZip constructor via dynamic import.
* This keeps startup costs low and avoids ESM/CJS interop pitfalls.
* @returns {Promise<any>} AdmZip constructor (class)
*/
let _AdmZipSingleton = null;
async function getAdmZip() {
if (_AdmZipSingleton) return _AdmZipSingleton;
const mod = await import('adm-zip');
_AdmZipSingleton = mod.default || mod;
return _AdmZipSingleton;
}
/**
* Extract numeric migration id from a migration file name like "12.add-users.js".
* @param {string} name
* @returns {number} Parsed id or 0 when not parsable
*/
function parseMigrationIdFromName(name) {
if (typeof name !== 'string') return 0;
const m = name.match(/^(\d+)\./);
return m ? parseInt(m[1], 10) : 0;
}
/**
* Read the highest migration id from available migration files.
* @returns {number} Highest migration id from files, or 0 when none.
*/
function getLatestMigrationIdFromFiles() {
try {
const files = listMigrationFiles();
const ids = files.map((f) => f.id);
return ids.length > 0 ? Math.max(...ids) : 0;
} catch (e) {
logger.warn('Failed to scan migrations directory:', e.message);
return 0;
}
}
/**
* Inspect the current database and return the highest applied migration id.
* @returns {number} Max id from schema_migrations, or 0 when table/rows are missing.
*/
function getCurrentDbMigration() {
try {
const exists = SqliteConnection.tableExists('schema_migrations');
if (!exists) return 0;
const rows = SqliteConnection.query('SELECT name FROM schema_migrations');
if (!rows || rows.length === 0) return 0;
return rows.reduce((max, r) => Math.max(max, parseMigrationIdFromName(r.name)), 0);
} catch (e) {
logger.warn('Failed to read current DB migration:', e.message);
return 0;
}
}
/**
* Create a consistent SQLite snapshot using the native backup API into a temp folder.
* @returns {Promise<{tempDir:string, backupPath:string}>}
*/
async function createTempBackupFile() {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-db-'));
const backupPath = path.join(tempDir, 'listings.db');
// Ensure connection is open and create a consistent snapshot
const db = SqliteConnection.getConnection();
await db.backup(backupPath);
return { tempDir, backupPath };
}
/**
* Build a zip buffer that contains the DB snapshot and metadata marker.
* Files:
* - listings.db
* - fredy-backup.json { formatVersion, createdAt, dbMigration, fredyVersion }
* @returns {Promise<Buffer>}
*/
async function buildBackupZipBuffer() {
const { backupPath, tempDir } = await createTempBackupFile();
try {
const AdmZip = await getAdmZip();
const zip = new AdmZip();
const meta = {
formatVersion: 1,
createdAt: new Date().toISOString(),
dbMigration: getCurrentDbMigration(),
fredyVersion: await getPackageVersion(),
};
// add files
zip.addLocalFile(backupPath, '', 'listings.db');
zip.addFile('fredy-backup.json', Buffer.from(JSON.stringify(meta, null, 2), 'utf-8'));
return zip.toBuffer();
} finally {
// cleanup temp
try {
fs.rmSync(tempDir, { recursive: true, force: true });
} catch (e) {
logger.debug('Failed to cleanup temp backup dir:', e.message);
}
}
}
/**
* Read and parse the metadata file from a backup zip buffer.
* @param {Buffer} zipBuffer
* @returns {Promise<any|null>} Parsed JSON or null when missing/invalid.
*/
async function readMetadataFromZip(zipBuffer) {
const AdmZip = await getAdmZip();
const zip = new AdmZip(zipBuffer);
const entry = zip.getEntry('fredy-backup.json');
if (!entry) return null;
try {
const txt = entry.getData().toString('utf-8');
return JSON.parse(txt);
} catch {
return null;
}
}
/**
* Check if a backup zip contains a listings.db entry.
* @param {Buffer} zipBuffer
* @returns {Promise<boolean>}
*/
async function hasListingsDbInZip(zipBuffer) {
const AdmZip = await getAdmZip();
const zip = new AdmZip(zipBuffer);
return zip.getEntry('listings.db') != null || zip.getEntries().some((e) => /listings\.db$/i.test(e.entryName));
}
/**
* Extract the listings.db from a backup zip buffer to a temp directory.
* @param {Buffer} zipBuffer
* @returns {Promise<{tempDir:string, dbPath:string}>}
*/
async function extractListingsDbToTemp(zipBuffer) {
const AdmZip = await getAdmZip();
const zip = new AdmZip(zipBuffer);
const entry = zip.getEntry('listings.db') || zip.getEntries().find((e) => /listings\.db$/i.test(e.entryName));
if (!entry) throw new Error('Backup zip does not contain listings.db');
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fredy-restore-'));
const outPath = path.join(tempDir, 'listings.db');
fs.writeFileSync(outPath, entry.getData());
return { tempDir, dbPath: outPath };
}
/**
* Public: Create a backup zip buffer ready for download.
* @returns {Promise<Buffer>}
*/
export async function createBackupZip() {
return buildBackupZipBuffer();
}
/**
* Analyze a backup zip for compatibility with the current codebase.
* - Missing DB yields danger.
* - Newer backup migration than required yields danger.
* - Older backup yields warning but is considered compatible (auto-migrate).
* - Equal version yields info.
* @param {Buffer} zipBuffer
* @returns {Promise<{compatible:boolean,severity:'danger'|'warning'|'info',message:string,backupMigration:number|null,requiredMigration:number,fredyVersion?:string|null}>>}
*/
export async function precheckRestore(zipBuffer) {
if (!zipBuffer || zipBuffer.length === 0) {
return {
compatible: false,
severity: 'danger',
message: 'Empty upload',
backupMigration: null,
requiredMigration: getLatestMigrationIdFromFiles(),
};
}
if (!(await hasListingsDbInZip(zipBuffer))) {
return {
compatible: false,
severity: 'danger',
message: 'Zip file is missing the database file (listings.db).',
backupMigration: null,
requiredMigration: getLatestMigrationIdFromFiles(),
};
}
const meta = await readMetadataFromZip(zipBuffer);
const requiredMigration = getLatestMigrationIdFromFiles();
const backupMigration = meta?.dbMigration ?? null;
const fredyVersion = meta?.fredyVersion ?? null;
if (backupMigration == null) {
return {
compatible: false,
severity: 'danger',
message:
'Backup metadata is missing the migration marker. Cannot validate compatibility. It is NOT advised to continue!',
backupMigration,
requiredMigration,
fredyVersion,
};
}
if (backupMigration > requiredMigration) {
return {
compatible: false,
severity: 'danger',
message:
'Backup schema is newer than this Fredy version. Please upgrade Fredy to a version that supports this backup or proceed at your own risk.',
backupMigration,
requiredMigration,
fredyVersion,
};
}
if (backupMigration < requiredMigration) {
return {
compatible: true,
severity: 'warning',
message:
'Backup contains an older database schema than this Fredy version requires. We will apply automatic migrations right after the restore to upgrade the database.',
backupMigration,
requiredMigration,
fredyVersion,
};
}
return {
compatible: true,
severity: 'info',
message: 'Backup is compatible with the current Fredy version.',
backupMigration,
requiredMigration,
fredyVersion,
};
}
/**
* Perform a restore from a validated backup zip.
* - Optionally forces restore when incompatible.
* - Replaces the on-disk DB and runs migrations when needed.
* @param {Buffer} zipBuffer
* @param {{force?:boolean}} [opts]
* @returns {Promise<{restored:true,warning:string|null,details:any}>}
* @throws Error with code 'INCOMPATIBLE' when not forced and incompatible
*/
export async function restoreFromZip(zipBuffer, { force = false } = {}) {
const check = await precheckRestore(zipBuffer);
if (!check.compatible && !force) {
const err = new Error(check.message || 'Backup is incompatible');
err.code = 'INCOMPATIBLE';
err.payload = check;
throw err;
}
const { dbPath } = await computeDbPath();
const { tempDir, dbPath: tempDbPath } = await extractListingsDbToTemp(zipBuffer);
try {
// Close existing connection to allow file replacement
SqliteConnection.close();
// Backup existing DB file
try {
if (fs.existsSync(dbPath)) {
const backupName = `${dbPath}.bak-${Date.now()}`;
fs.copyFileSync(dbPath, backupName);
}
} catch (e) {
logger.warn('Failed to create on-disk backup copy of current DB:', e.message);
}
// Replace DB with the one from the zip
fs.copyFileSync(tempDbPath, dbPath);
// Re-run migrations when needed
if (check.backupMigration < check.requiredMigration) {
await runMigrations();
} else {
// Ensure we can re-open the DB
SqliteConnection.getConnection();
}
} finally {
try {
fs.rmSync(tempDir, { recursive: true, force: true });
} catch (e) {
logger.debug('Failed to cleanup temp restore dir:', e.message);
}
}
return { restored: true, warning: check.severity !== 'info' ? check.message : null, details: check };
}
/**
* Build the backup file name with current date and Fredy version.
* Pattern: YYYY-MM-DD-FredyBackup-{version}.zip
* @returns {Promise<string>}
*/
export async function buildBackupFileName() {
const dt = new Date();
const yyyy = dt.getFullYear();
const mm = String(dt.getMonth() + 1).padStart(2, '0');
const dd = String(dt.getDate()).padStart(2, '0');
const version = await getPackageVersion();
return `${yyyy}-${mm}-${dd}-FredyBackup-${version}.zip`.replaceAll(' ', '');
}

View File

@@ -7,40 +7,6 @@ import { nullOrEmpty } from '../../utils.js';
import SqliteConnection from './SqliteConnection.js';
import { nanoid } from 'nanoid';
/**
* Build analytics data for a given job by grouping all listings by provider and
* mapping each listing hash to its creation timestamp.
*
* SQL shape:
* SELECT json_group_object(provider, json_object(hash, created_at)) AS result
* FROM listings WHERE job_id = @jobId;
*
* The resulting object has the shape:
* {
* providerA: { "<hash1>": <created_at_ms>, "<hash2>": <created_at_ms>, ... },
* providerB: { ... }
* }
*
* @param {string} jobId - ID of the job whose listings should be aggregated.
* @returns {Record<string, Record<string, number>>} Object grouped by provider mapping listing-hash -> created_at epoch ms.
*/
export const getListingProviderDataForAnalytics = (jobId) => {
const row = SqliteConnection.query(
`SELECT COALESCE(
json_group_object(provider, json(provider_map)),
json('{}')
) AS result
FROM (SELECT provider,
json_group_object(hash, created_at) AS provider_map
FROM listings
WHERE job_id = @jobId
GROUP BY provider);`,
{ jobId },
);
return row?.length > 0 ? JSON.parse(row[0].result) : {};
};
/**
* Return a list of known listing hashes for a given job and provider.
* Useful to de-duplicate before inserting new listings.
@@ -59,6 +25,89 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => {
).map((r) => r.hash);
};
/**
* Compute KPI aggregates for a given set of job IDs from the listings table.
*
* - numberOfActiveListings: count of listings where is_active = 1
* - avgPriceOfListings: average of numeric price, rounded to nearest integer
*
* When no jobIds are provided, returns zeros.
*
* @param {string[]} jobIds
* @returns {{ numberOfActiveListings: number, avgPriceOfListings: number }}
*/
export const getListingsKpisForJobIds = (jobIds = []) => {
if (!Array.isArray(jobIds) || jobIds.length === 0) {
return { numberOfActiveListings: 0, avgPriceOfListings: 0 };
}
const placeholders = jobIds.map(() => '?').join(',');
const row =
SqliteConnection.query(
`SELECT
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount,
AVG(price) AS avgPrice
FROM listings
WHERE job_id IN (${placeholders})`,
jobIds,
)[0] || {};
return {
numberOfActiveListings: Number(row.activeCount || 0),
avgPriceOfListings: row?.avgPrice == null ? 0 : Math.round(Number(row.avgPrice)),
};
};
/**
* Compute distribution of listings by provider for the given set of job IDs.
* Returns data ready for the pie chart component with fields `type` and `value` (percentage).
*
* Example return:
* [ { type: 'immoscout', value: 62 }, { type: 'immowelt', value: 38 } ]
*
* When no jobIds are provided or no listings exist, returns empty array.
*
* @param {string[]} jobIds
* @returns {{ type: string, value: number }[]}
*/
export const getProviderDistributionForJobIds = (jobIds = []) => {
if (!Array.isArray(jobIds) || jobIds.length === 0) {
return [];
}
const placeholders = jobIds.map(() => '?').join(',');
const rows = SqliteConnection.query(
`SELECT provider, COUNT(*) AS cnt
FROM listings
WHERE job_id IN (${placeholders})
GROUP BY provider
ORDER BY cnt DESC`,
jobIds,
);
const total = rows.reduce((acc, r) => acc + Number(r.cnt || 0), 0);
if (total === 0) return [];
// Map counts to integer percentage values (0-100). Ensure sum is ~100 by rounding.
const percentages = rows.map((r) => ({
type: r.provider,
value: Math.round((Number(r.cnt) / total) * 100),
}));
// Adjust rounding drift to keep sum at 100 (optional minor correction)
const drift = 100 - percentages.reduce((s, p) => s + p.value, 0);
if (drift !== 0 && percentages.length > 0) {
// apply drift to the largest slice to keep UX simple
let maxIdx = 0;
for (let i = 1; i < percentages.length; i++) {
if (percentages[i].value > percentages[maxIdx].value) maxIdx = i;
}
percentages[maxIdx].value = Math.max(0, percentages[maxIdx].value + drift);
}
return percentages;
};
/**
* Return a list of listing that either are active or have an unknown status
* to constantly check if they are still online

View File

@@ -35,7 +35,11 @@ import SqliteConnection from '../SqliteConnection.js';
import logger from '../../logger.js';
const ROOT = path.resolve('.');
const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
/**
* Absolute path to the migrations directory (lib/services/storage/migrations/sql).
* @type {string}
*/
export const MIGRATIONS_DIR = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'sql');
/**
* Ensures that the given directory exists, creating it recursively if needed.
@@ -50,7 +54,7 @@ function ensureDir(p) {
* Migration files must follow the format: <number>.<label>.js
* @returns {Array<{id:number, name:string, label:string, path:string}>}
*/
function listMigrationFiles() {
export function listMigrationFiles() {
ensureDir(MIGRATIONS_DIR);
return fs
.readdirSync(MIGRATIONS_DIR)

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "16.0.0",
"version": "16.2.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",
@@ -59,15 +59,14 @@
"Firefox ESR"
],
"dependencies": {
"adm-zip": "^0.5.16",
"@douyinfe/semi-icons": "^2.89.0",
"@douyinfe/semi-ui": "2.89.0",
"@sendgrid/mail": "8.1.6",
"@visactor/react-vchart": "^2.0.10",
"@visactor/vchart": "^2.0.10",
"@visactor/vchart-semi-theme": "^1.12.2",
"@vitejs/plugin-react": "5.1.2",
"better-sqlite3": "^12.5.0",
"body-parser": "2.2.1",
"chart.js": "^4.5.1",
"cheerio": "^1.1.2",
"cookie-session": "2.1.1",
"handlebars": "4.7.8",
@@ -78,19 +77,20 @@
"node-mailjet": "6.0.11",
"p-throttle": "^8.1.0",
"package-up": "^5.0.0",
"puppeteer": "^24.32.1",
"puppeteer": "^24.33.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"query-string": "9.3.1",
"react": "18.3.1",
"react-chartjs-2": "^5.3.1",
"react-dom": "18.3.1",
"react-router": "7.10.1",
"react-router-dom": "7.10.1",
"restana": "5.1.0",
"semver": "^7.7.3",
"serve-static": "2.2.0",
"serve-static": "2.2.1",
"slack": "11.0.2",
"vite": "7.2.7",
"vite": "7.3.0",
"x-var": "^3.0.1",
"zustand": "^5.0.9"
},
@@ -100,13 +100,13 @@
"@babel/preset-env": "7.28.5",
"@babel/preset-react": "7.28.5",
"chai": "6.2.1",
"eslint": "9.39.1",
"eslint": "9.39.2",
"eslint-config-prettier": "10.1.8",
"eslint-plugin-react": "7.37.5",
"esmock": "2.7.3",
"history": "5.3.0",
"husky": "9.1.7",
"less": "4.4.2",
"less": "4.5.1",
"lint-staged": "16.2.7",
"mocha": "11.7.5",
"nodemon": "^3.1.11",

View File

@@ -0,0 +1,134 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { expect } from 'chai';
import esmock from 'esmock';
describe('services/storage/backupRestoreService.js - precheck & filename', () => {
let svc;
let admZipMock;
let calls;
beforeEach(async () => {
calls = { logger: { info: [], warn: [], error: [] } };
// Mock AdmZip with configurable state
let state = { hasDb: false, meta: null };
class MockAdmZip {
constructor() {}
getEntry(name) {
if (name === 'listings.db') {
if (state.hasDb) return { getData: () => Buffer.from('db') };
return null;
}
if (name === 'fredy-backup.json') {
if (state.meta) return { getData: () => Buffer.from(JSON.stringify(state.meta)) };
return null;
}
return null;
}
getEntries() {
const arr = [];
if (state.hasDb) arr.push({ entryName: 'listings.db', getData: () => Buffer.from('db') });
return arr;
}
}
admZipMock = { default: MockAdmZip, __set: (s) => (state = { ...state, ...s }) };
const path = await import('node:path');
const ROOT = path.resolve('.');
// Mocks for dependencies
const migratePath = path.join(ROOT, 'lib', 'services', 'storage', 'migrations', 'migrate.js');
const sqlitePath = path.join(ROOT, 'lib', 'services', 'storage', 'SqliteConnection.js');
const loggerPath = path.join(ROOT, 'lib', 'services', 'logger.js');
const utilsPath = path.join(ROOT, 'lib', 'utils.js');
const migrateMock = {
listMigrationFiles: () => [{ id: 10 }],
runMigrations: async () => {},
};
const sqliteMock = {
default: {
getConnection: () => ({ backup: async () => {} }),
close: () => {},
tableExists: () => false,
query: () => [],
withTransaction: (cb) => cb({ prepare: () => ({ run: () => {} }) }),
},
computeDbPath: async () => ({ dir: '/tmp', dbPath: '/tmp/listings.db' }),
};
const loggerMock = {
info: (...a) => calls.logger.info.push(a),
warn: (...a) => calls.logger.warn.push(a),
error: (...a) => calls.logger.error.push(a),
};
const utilsMock = { getPackageVersion: async () => '16.2.0' };
const mod = await esmock(
path.join(ROOT, 'lib', 'services', 'storage', 'backupRestoreService.js'),
{},
{
'adm-zip': admZipMock,
[migratePath]: migrateMock,
[sqlitePath]: sqliteMock,
[loggerPath]: loggerMock,
[utilsPath]: utilsMock,
},
);
svc = mod;
});
it('precheck: empty upload yields danger', async () => {
const res = await svc.precheckRestore(Buffer.alloc(0));
expect(res.compatible).to.equal(false);
expect(res.severity).to.equal('danger');
expect(res.message).to.contain('Empty upload');
expect(res.requiredMigration).to.equal(10);
});
it('precheck: missing listings.db yields danger', async () => {
admZipMock.__set({ hasDb: false, meta: { dbMigration: 9 } });
const res = await svc.precheckRestore(Buffer.from('dummy'));
expect(res.compatible).to.equal(false);
expect(res.severity).to.equal('danger');
expect(res.message).to.match(/missing the database file/i);
});
it('precheck: older backup is compatible with warning', async () => {
admZipMock.__set({ hasDb: true, meta: { dbMigration: 5, fredyVersion: '16.0.0' } });
const res = await svc.precheckRestore(Buffer.from('zip'));
expect(res.compatible).to.equal(true);
expect(res.severity).to.equal('warning');
expect(res.message).to.match(/automatic migrations/i);
expect(res.backupMigration).to.equal(5);
expect(res.requiredMigration).to.equal(10);
});
it('precheck: equal backup is compatible with info', async () => {
admZipMock.__set({ hasDb: true, meta: { dbMigration: 10 } });
const res = await svc.precheckRestore(Buffer.from('zip'));
expect(res.compatible).to.equal(true);
expect(res.severity).to.equal('info');
});
it('precheck: newer backup yields danger', async () => {
admZipMock.__set({ hasDb: true, meta: { dbMigration: 11 } });
const res = await svc.precheckRestore(Buffer.from('zip'));
expect(res.compatible).to.equal(false);
expect(res.severity).to.equal('danger');
});
it('buildBackupFileName: matches pattern and includes version', async () => {
const name = await svc.buildBackupFileName();
expect(name).to.match(/^\d{4}-\d{2}-\d{2}-FredyBackup-/);
expect(name).to.include('16.2.0');
expect(name).to.match(/\.zip$/);
});
});

View File

@@ -10,7 +10,6 @@ import PermissionAwareRoute from './components/permission/PermissionAwareRoute';
import GeneralSettings from './views/generalSettings/GeneralSettings';
import JobMutation from './views/jobs/mutation/JobMutation';
import UserMutator from './views/user/mutation/UserMutator';
import JobInsight from './views/jobs/insights/JobInsight.jsx';
import { useActions, useSelector } from './services/state/store';
import { Routes, Route, Navigate } from 'react-router-dom';
import Login from './views/login/Login';
@@ -25,8 +24,8 @@ import Listings from './views/listings/Listings.jsx';
import Navigation from './components/navigation/Navigation.jsx';
import { Layout } from '@douyinfe/semi-ui';
import FredyFooter from './components/footer/FredyFooter.jsx';
import ProcessingTimes from './views/jobs/ProcessingTimes.jsx';
import WatchlistManagement from './views/listings/management/WatchlistManagement.jsx';
import Dashboard from './views/dashboard/Dashboard.jsx';
export default function FredyApp() {
const actions = useActions();
@@ -34,7 +33,6 @@ export default function FredyApp() {
const currentUser = useSelector((state) => state.user.currentUser);
const versionUpdate = useSelector((state) => state.versionUpdate.versionUpdate);
const settings = useSelector((state) => state.generalSettings.settings);
const processingTimes = useSelector((state) => state.jobs.processingTimes);
useEffect(() => {
async function init() {
@@ -43,7 +41,6 @@ export default function FredyApp() {
await actions.features.getFeatures();
await actions.provider.getProvider();
await actions.jobs.getJobs();
await actions.jobs.getProcessingTimes();
await actions.jobs.getSharableUserList();
await actions.notificationAdapter.getAdapter();
await actions.generalSettings.getGeneralSettings();
@@ -88,14 +85,13 @@ export default function FredyApp() {
</>
)}
{settings.analyticsEnabled === null && !settings.demoMode && <TrackingModal />}
{processingTimes != null && <ProcessingTimes processingTimes={processingTimes} />}
<Divider />
<div className="app__content">
<Routes>
<Route path="/403" element={<InsufficientPermission />} />
<Route path="/jobs/new" element={<JobMutation />} />
<Route path="/jobs/edit/:jobId" element={<JobMutation />} />
<Route path="/jobs/insights/:jobId" element={<JobInsight />} />
<Route path="/dashboard" element={<Dashboard />} />
<Route path="/jobs" element={<Jobs />} />
<Route path="/listings" element={<Listings />} />
<Route path="/watchlistManagement" element={<WatchlistManagement />} />
@@ -134,7 +130,7 @@ export default function FredyApp() {
}
/>
<Route path="/" element={<Navigate to="/jobs" replace />} />
<Route path="/" element={<Navigate to="/dashboard" replace />} />
</Routes>
</div>
</Content>

View File

@@ -9,17 +9,12 @@ import { HashRouter } from 'react-router-dom';
import { createRoot } from 'react-dom/client';
import en_US from '@douyinfe/semi-ui/lib/es/locale/source/en_US';
import { LocaleProvider } from '@douyinfe/semi-ui';
import { initVChartSemiTheme } from '@visactor/vchart-semi-theme';
import App from './App';
import './Index.less';
const container = document.getElementById('fredy');
const root = createRoot(container);
initVChartSemiTheme({
defaultMode: 'dark',
});
root.render(
<HashRouter>
<LocaleProvider locale={en_US}>

BIN
ui/src/assets/heart.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 MiB

View File

@@ -0,0 +1,23 @@
.chartCard {
/* Use provided background with slight transparency and a brighter mix */
background: color-mix(in oklab, var(--card-bg, rgb(70 72 78)) 20%, white 80%);
border-radius: .6rem;
border: 1px solid color-mix(in oklab, var(--card-bg, rgb(70 72 78)) 35%, white 65%);
box-shadow: 0 6px 20px rgba(0,0,0,0.08);
/* Ensure base text has strong contrast */
color: var(--semi-color-text-0);
/* Semi Card header/title styling */
.semi-card-header .semi-card-header-title {
/* Derive a tinted title color with stronger contrast towards black */
color: color-mix(in oklab, var(--card-bg, rgb(70 72 78)) 60%, black 40%);
font-weight: 600;
}
&__no__data {
display: grid;
place-items: center;
height: 14rem;
opacity: .7;
}
}

View File

@@ -0,0 +1,92 @@
@import "DashboardCardColors.less";
.color-variant(@bg, @border, @text) {
background-color: @bg;
border: 1px solid @border;
color: @text;
}
.dashboard-card {
box-sizing: border-box;
padding: .8rem;
border-radius: .5rem;
border-width: 1px;
font-weight: 600;
box-shadow: 0 6px 20px rgba(0,0,0,0.08);
/* Make all KPI boxes the same size regardless of content/font */
width: 100%;
max-width: none;
height: 10rem;
display: flex;
flex-direction: column;
&.blue {
.color-variant(@color-blue-bg, @color-blue-border, @color-blue-text);
}
&.orange {
.color-variant(@color-orange-bg, @color-orange-border, @color-orange-text);
}
&.green {
.color-variant(@color-green-bg, @color-green-border, @color-green-text);
}
&.purple {
.color-variant(@color-purple-bg, @color-purple-border, @color-purple-text);
}
&.gray {
.color-variant(@color-gray-bg, @color-gray-border, @color-gray-text);
}
&__header {
display: flex;
align-items: center;
gap: .6rem;
/* Keep header from growing content height */
min-height: 2rem;
overflow: hidden;
}
&__icon {
border-radius: .6rem;
display: grid;
place-items: center;
}
&__title {
font-weight: 600;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
&__content {
margin-top: .4rem;
font-size: .7rem;
flex: 1 1 auto;
display: flex;
flex-direction: column;
justify-content: center;
overflow: hidden;
}
&__value {
margin: 0;
font-size: 1.5rem;
line-height: 1.1;
color: #fff;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
&__desc {
opacity: .8;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}

View File

@@ -0,0 +1,19 @@
@color-blue-bg: rgba(0, 123, 255, 0.24);
@color-blue-border: #1E40AFFF;
@color-blue-text: #60a5fa;
@color-orange-bg: rgba(250, 91, 5, 0.12);
@color-orange-border: #d33601;
@color-orange-text: #FB923CFF;
@color-green-bg: rgba(38, 250, 5, 0.12);
@color-green-border: #00c316;
@color-green-text: #33f308;
@color-purple-bg: rgba(91, 3, 218, 0.38);
@color-purple-border: #7500c3;
@color-purple-text: #b15fff;
@color-gray-bg: rgba(110, 110, 110, 0.38);
@color-gray-border: #807f7f;
@color-gray-text: #bab9b9;

View File

@@ -0,0 +1,40 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import './DashboardCard.less';
export default function KpiCard({
title,
icon,
value,
valueFontSize = '1.5rem',
description,
color = 'gray',
children,
}) {
return (
<div className={`dashboard-card ${color}`}>
<div className="dashboard-card__header">
<div className="dashboard-card__icon">{icon}</div>
<div className="dashboard-card__title">
<span>{title}</span>
</div>
</div>
<div className="dashboard-card__content">
<p className="dashboard-card__value" style={{ fontSize: valueFontSize }}>
{value}
{children}
</p>
{description && <span className="dashboard-card__desc">{description}</span>}
</div>
</div>
);
}

View File

@@ -0,0 +1,97 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { Pie } from 'react-chartjs-2';
import { Chart as ChartJS, ArcElement, Tooltip, Legend, Title as ChartTitle } from 'chart.js';
import './ChartCard.less';
ChartJS.register(ArcElement, Tooltip, Legend, ChartTitle);
export default function PieChartCard({ data = [] }) {
const { labels, values } = React.useMemo(() => {
if (data && typeof data === 'object' && !Array.isArray(data)) {
const lbls = Array.isArray(data.labels) ? data.labels : [];
const vals = Array.isArray(data.values)
? data.values.map((v) => (Number.isFinite(Number(v)) ? Number(v) : 0))
: [];
return { labels: lbls, values: vals };
}
if (Array.isArray(data)) {
const lbls = data.map((d) => d?.type ?? 'Unknown');
const vals = data.map((d) => {
const v = Number(d?.value);
return Number.isFinite(v) ? v : 0;
});
return { labels: lbls, values: vals };
}
return { labels: [], values: [] };
}, [data]);
const palette = React.useMemo(
() => [
'#4e79a7',
'#f28e2b',
'#e15759',
'#76b7b2',
'#59a14f',
'#edc948',
'#b07aa1',
'#ff9da7',
'#9c755f',
'#bab0ab',
],
[],
);
const chartData = React.useMemo(
() => ({
labels,
datasets: [
{
data: values,
backgroundColor: labels.map((_, i) => palette[i % palette.length]),
borderColor: labels.map((_, i) => palette[i % palette.length]),
borderWidth: 1,
},
],
}),
[labels, values, palette],
);
const options = React.useMemo(
() => ({
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: {
display: true,
position: 'right',
labels: {
color: () => '#fff',
},
},
title: { display: false },
tooltip: {
callbacks: {
label: (ctx) => {
const label = ctx.label || '';
const val = ctx.parsed !== undefined ? ctx.parsed : ctx.raw;
return `${label}: ${val}%`;
},
},
},
},
}),
[],
);
const isEmpty = !labels || labels.length === 0 || !values || values.length === 0;
return (
<>{isEmpty ? <div className="chartCard__no__data">No Data</div> : <Pie data={chartData} options={options} />}</>
);
}

View File

@@ -1,9 +1,10 @@
.navigate {
&__logout_Button {
&__footer {
align-items: center;
justify-content: center;
flex-direction: column;
gap: 0.5rem;
width: 100%;
display: flex;
}
}

View File

@@ -3,26 +3,34 @@
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { Nav } from '@douyinfe/semi-ui';
import { IconStar, IconSetting, IconTerminal } from '@douyinfe/semi-icons';
import React, { useEffect, useState } from 'react';
import { Button, Nav } from '@douyinfe/semi-ui';
import { IconStar, IconSetting, IconTerminal, IconHistogram, IconSidebar } from '@douyinfe/semi-icons';
import logoWhite from '../../assets/logo_white.png';
import heart from '../../assets/heart.png';
import Logout from '../logout/Logout.jsx';
import { useLocation, useNavigate } from 'react-router-dom';
import './Navigate.less';
import { useScreenWidth } from '../../hooks/screenWidth.js';
import { useFeature } from '../../hooks/featureHook.js';
import { useScreenWidth } from '../../hooks/screenWidth.js';
export default function Navigation({ isAdmin }) {
const navigate = useNavigate();
const location = useLocation();
const width = useScreenWidth();
const collapsed = width <= 850;
const [collapsed, setCollapsed] = useState(width <= 850);
const watchlistFeature = useFeature('WATCHLIST_MANAGEMENT') || false;
useEffect(() => {
if (width <= 850) {
setCollapsed(true);
}
}, [width]);
const items = [
{ itemKey: '/dashboard', text: 'Dashboard', icon: <IconHistogram /> },
{ itemKey: '/jobs', text: 'Jobs', icon: <IconTerminal /> },
{ itemKey: '/listings', text: 'Listings', icon: <IconStar /> },
];
@@ -51,18 +59,21 @@ export default function Navigation({ isAdmin }) {
return (
<Nav
style={{ height: '100%', width: collapsed ? '' : '13.2rem' }}
style={{ height: '100%' }}
items={items}
isCollapsed={collapsed}
selectedKeys={[parsePathName(location.pathname)]}
onSelect={(key) => {
navigate(key.itemKey);
}}
header={<img src={logoWhite} width="180" alt="Fredy Logo" />}
header={<img src={collapsed ? heart : logoWhite} width={collapsed ? '80' : '160'} alt="Fredy Logo" />}
footer={
<div className="navigate__logout_Button">
<Nav.Footer className="navigate__footer">
<Logout text={!collapsed} />
</div>
<Button icon={<IconSidebar />} onClick={() => setCollapsed(!collapsed)}>
{!collapsed && 'Collapse'}
</Button>
</Nav.Footer>
}
/>
);

View File

@@ -8,14 +8,16 @@ import { Card } from '@douyinfe/semi-ui';
import './SegmentParts.less';
export const SegmentPart = ({ name, Icon = null, children, helpText }) => {
export const SegmentPart = ({ name, Icon = null, children, helpText = null }) => {
const { Meta } = Card;
return (
<Card
className="segmentParts"
title={
<Meta title={name} description={helpText} avatar={Icon == null ? null : <Icon size="extra-extra-small" />} />
(helpText || name) && (
<Meta title={name} description={helpText} avatar={Icon == null ? null : <Icon size="extra-extra-small" />} />
)
}
>
{children}

View File

@@ -1,6 +1,6 @@
.segmentParts {
border: 1px solid #323232 !important;
border-radius: 5px !important;
border-radius: .9rem !important;
color: rgba(var(--semi-grey-8), 1);
background: rgb(53, 54, 60);
margin: 2rem;

View File

@@ -6,7 +6,7 @@
import React from 'react';
import { Button, Empty, Table, Switch, Popover } from '@douyinfe/semi-ui';
import { IconAlertTriangle, IconDelete, IconDescend2, IconEdit, IconHistogram } from '@douyinfe/semi-icons';
import { IconAlertTriangle, IconDelete, IconDescend2, IconEdit } from '@douyinfe/semi-icons';
import { IllustrationNoResult, IllustrationNoResultDark } from '@douyinfe/semi-illustrations';
import './JobTable.less';
@@ -21,14 +21,7 @@ const empty = (
const getPopoverContent = (text) => <article className="jobPopoverContent">{text}</article>;
export default function JobTable({
jobs = {},
onJobRemoval,
onJobStatusChanged,
onJobEdit,
onJobInsight,
onListingRemoval,
} = {}) {
export default function JobTable({ jobs = {}, onJobRemoval, onJobStatusChanged, onJobEdit, onListingRemoval } = {}) {
return (
<Table
pagination={false}
@@ -98,14 +91,6 @@ export default function JobTable({
render: (_, job) => {
return (
<div className="interactions">
<Popover content={getPopoverContent('Job Insights')}>
<Button
type="primary"
icon={<IconHistogram />}
disabled={job.isOnlyShared}
onClick={() => onJobInsight(job.id)}
/>
</Popover>
<Popover content={getPopoverContent('Edit a Job')}>
<Button
type="secondary"

View File

@@ -0,0 +1,85 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
/**
* Lightweight client for Backup & Restore interactions with the backend.
*
* Usage (in React components):
* ```js
* import { downloadBackup, precheckRestore, restore } from '../../services/backupRestoreClient';
* await downloadBackup();
* const info = await precheckRestore(file);
* await restore(file, false);
* ```
*/
function extractFileNameFromDisposition(disposition) {
const dispo = disposition || '';
const match = dispo.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/);
return decodeURIComponent(match?.[1] || match?.[2] || 'FredyBackup.zip');
}
export class BackupRestoreClient {
/**
* Trigger a backup download and save it using the filename provided by the server.
* @returns {Promise<void>}
*/
static async downloadBackup() {
const resp = await fetch('/api/admin/backup', { credentials: 'include' });
if (!resp.ok) throw new Error('Failed to create backup');
const blob = await resp.blob();
const fileName = extractFileNameFromDisposition(resp.headers.get('Content-Disposition'));
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = fileName;
document.body.appendChild(a);
a.click();
a.remove();
window.URL.revokeObjectURL(url);
}
/**
* Upload a backup zip for analysis without restoring.
* @param {Blob|ArrayBuffer|Buffer} file - Backup zip content.
* @returns {Promise<{compatible:boolean,severity:string,message:string,backupMigration:number|null,requiredMigration:number,fredyVersion?:string|null}>>}
*/
static async precheckRestore(file) {
const resp = await fetch('/api/admin/backup/restore?dryRun=true', {
method: 'POST',
credentials: 'include',
headers: { 'Content-Type': 'application/zip' },
body: file,
});
return resp.json();
}
/**
* Perform a database restore from a backup zip.
* @param {Blob|ArrayBuffer|Buffer} file - Backup zip content.
* @param {boolean} force - When true, proceed even if reported incompatible.
* @returns {Promise<{restored:true,warning:string|null,details:any}>}
*/
static async restore(file, force) {
const resp = await fetch(`/api/admin/backup/restore?force=${force ? 'true' : 'false'}`, {
method: 'POST',
credentials: 'include',
headers: { 'Content-Type': 'application/zip' },
body: file,
});
const data = await resp.json();
if (!resp.ok) {
const err = new Error(data?.message || 'Restore failed');
err.payload = data;
throw err;
}
return data;
}
}
// Convenience named exports
export const downloadBackup = (...args) => BackupRestoreClient.downloadBackup(...args);
export const precheckRestore = (...args) => BackupRestoreClient.precheckRestore(...args);
export const restore = (...args) => BackupRestoreClient.restore(...args);

View File

@@ -33,6 +33,16 @@ export const useFredyState = create(
(set) => {
// Async actions that directly set state (no separate reducer concept)
const effects = {
dashboard: {
async getDashboard() {
try {
const response = await xhrGet('/api/dashboard');
set((state) => ({ dashboard: { ...state.dashboard, data: response.json } }));
} catch (Exception) {
console.error('Error while trying to get resource for /api/dashboard. Error:', Exception);
}
},
},
notificationAdapter: {
async getAdapter() {
try {
@@ -90,27 +100,6 @@ export const useFredyState = create(
console.error(`Error while trying to get resource for api/jobs. Error:`, Exception);
}
},
async getProcessingTimes() {
try {
const response = await xhrGet('/api/jobs/processingTimes');
set((state) => ({ jobs: { ...state.jobs, processingTimes: Object.freeze(response.json) } }));
} catch (Exception) {
console.error(`Error while trying to get resource for api/processingTimes. Error:`, Exception);
}
},
async getInsightDataForJob(jobId) {
try {
const response = await xhrGet(`/api/jobs/insights/${jobId}`);
set((state) => ({
jobs: {
...state.jobs,
insights: { ...state.jobs.insights, [jobId]: Object.freeze(response.json) },
},
}));
} catch (Exception) {
console.error(`Error while trying to get resource for api/jobs/insights. Error:`, Exception);
}
},
},
user: {
async getUsers() {
@@ -185,6 +174,7 @@ export const useFredyState = create(
// Initial state
const initial = {
dashboard: { data: null },
notificationAdapter: [],
listingsTable: {
totalNumber: 0,
@@ -196,12 +186,13 @@ export const useFredyState = create(
demoMode: { demoMode: false },
versionUpdate: {},
provider: [],
jobs: { jobs: [], insights: {}, processingTimes: {}, shareableUserList: [] },
jobs: { jobs: [], shareableUserList: [] },
user: { users: [], currentUser: null },
};
// Expose actions by grouping them per slice
const actions = {
dashboard: { ...effects.dashboard },
notificationAdapter: { ...effects.notificationAdapter },
generalSettings: { ...effects.generalSettings },
demoMode: { ...effects.demoMode },

View File

@@ -0,0 +1,156 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { Button, Col, Row, Toast } from '@douyinfe/semi-ui';
import {
IconTerminal,
IconStar,
IconClock,
IconDoubleChevronLeft,
IconDoubleChevronRight,
IconStarStroked,
IconNoteMoney,
IconSearch,
IconPlayCircle,
} from '@douyinfe/semi-icons';
import { useSelector, useActions } from '../../services/state/store';
import KpiCard from '../../components/cards/KpiCard.jsx';
import PieChartCard from '../../components/cards/PieChartCard.jsx';
import Headline from '../../components/headline/Headline.jsx';
import './Dashboard.less';
import { SegmentPart } from '../../components/segment/SegmentPart.jsx';
import { xhrPost } from '../../services/xhr.js';
import { format } from '../../services/time/timeService.js';
export default function Dashboard() {
const actions = useActions();
const dashboard = useSelector((state) => state.dashboard.data);
React.useEffect(() => {
actions.dashboard.getDashboard();
}, []);
const kpis = dashboard?.kpis || { totalJobs: 0, totalListings: 0, providersUsed: 0 };
const pieData = dashboard?.pie || [];
return (
<div className="dashboard">
<Headline text="Dashboard" size={3} />
<Row gutter={16} className="dashboard__row">
<Col span={12} xs={24} sm={24} md={24} lg={24} xl={12}>
<SegmentPart name="General" Icon={IconTerminal}>
<Row gutter={16} className="dashboard__row">
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Search Interval"
value={`${dashboard?.general?.interval} min`}
icon={<IconClock />}
description="Time interval for job execution"
/>
</Col>
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Last Search"
valueFontSize="14px"
value={
dashboard?.general?.lastRun == null || dashboard?.general?.lastRun === 0
? '---'
: format(dashboard?.general?.lastRun)
}
icon={<IconDoubleChevronLeft />}
description="Last execution timestamp"
/>
</Col>
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Next Search"
value={
dashboard?.general?.nextRun == null || dashboard?.general?.nextRun === 0
? '---'
: format(dashboard?.general?.nextRun)
}
valueFontSize="14px"
icon={<IconDoubleChevronRight />}
description="Next execution timestamp"
/>
</Col>
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard title="Search Now" icon={<IconSearch />} description="Run a search now">
<Button
size="small"
style={{ marginTop: '.2rem' }}
icon={<IconPlayCircle />}
aria-label="Start now"
onClick={async () => {
try {
await xhrPost('/api/jobs/startAll', null);
Toast.success('Successfully triggered Fredy search.');
} catch {
Toast.error('Failed to trigger search');
}
}}
>
Search now
</Button>
</KpiCard>
</Col>
</Row>
</SegmentPart>
</Col>
<Col span={12} xs={24} sm={24} md={24} lg={24} xl={12}>
<SegmentPart name="Overview" Icon={IconStar}>
<Row gutter={16} className="dashboard__row">
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Jobs"
color="blue"
value={!kpis.totalJobs ? '---' : kpis.totalJobs}
icon={<IconTerminal />}
description="Total number of jobs"
/>
</Col>
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Listings"
color="orange"
value={!kpis.totalListings ? '---' : kpis.totalListings}
icon={<IconStarStroked />}
description="Total listings found"
/>
</Col>
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Active Listings"
color="green"
value={!kpis.numberOfActiveListings ? '---' : kpis.numberOfActiveListings}
icon={<IconStar />}
description="Total active listings"
/>
</Col>
<Col span={12} xs={24} sm={12} md={12} lg={12} xl={12}>
<KpiCard
title="Avg. Price"
color="purple"
value={`${!kpis.avgPriceOfListings ? '---' : kpis.avgPriceOfListings} EUR`}
icon={<IconNoteMoney />}
description="Avg. Price of listings"
/>
</Col>
</Row>
</SegmentPart>
</Col>
</Row>
<SegmentPart name="Provider Insights" Icon={IconStar} helpText="Percentage of found listings over all providers">
<PieChartCard title="Jobs per Provider" data={pieData} isLoading={false} />
</SegmentPart>
</div>
);
}
Dashboard.displayName = 'Dashboard';

View File

@@ -0,0 +1,11 @@
.dashboard {
&__row {
margin-bottom: 1rem;
/* Ensure grid items wrap to next line on narrow screens */
flex-wrap: wrap;
/* Vertical gap of 1rem between wrapped grid items (no px) */
.semi-col {
margin-bottom: 1rem;
}
}
}

View File

@@ -7,11 +7,16 @@ import React from 'react';
import { useActions, useSelector } from '../../services/state/store';
import { Divider, TimePicker, Button, Checkbox, Input } from '@douyinfe/semi-ui';
import { Divider, TimePicker, Button, Checkbox, Input, Modal } from '@douyinfe/semi-ui';
import { InputNumber } from '@douyinfe/semi-ui';
import { xhrPost } from '../../services/xhr';
import { SegmentPart } from '../../components/segment/SegmentPart';
import { Banner, Toast } from '@douyinfe/semi-ui';
import {
downloadBackup as downloadBackupZip,
precheckRestore as clientPrecheckRestore,
restore as clientRestore,
} from '../../services/backupRestoreClient';
import {
IconSave,
IconCalendar,
@@ -52,6 +57,11 @@ const GeneralSettings = function GeneralSettings() {
const [demoMode, setDemoMode] = React.useState(null);
const [analyticsEnabled, setAnalyticsEnabled] = React.useState(null);
const [sqlitePath, setSqlitePath] = React.useState(null);
const fileInputRef = React.useRef(null);
const [restoreModalVisible, setRestoreModalVisible] = React.useState(false);
const [precheckInfo, setPrecheckInfo] = React.useState(null);
const [restoreBusy, setRestoreBusy] = React.useState(false);
const [selectedRestoreFile, setSelectedRestoreFile] = React.useState(null);
React.useEffect(() => {
async function init() {
@@ -78,7 +88,7 @@ const GeneralSettings = function GeneralSettings() {
const nullOrEmpty = (val) => val == null || val.length === 0;
const onStore = async () => {
const handleStore = async () => {
if (nullOrEmpty(interval)) {
Toast.error('Interval may not be empty.');
return;
@@ -125,6 +135,60 @@ const GeneralSettings = function GeneralSettings() {
}, 3000);
};
const handleDownloadBackup = React.useCallback(async () => {
try {
await downloadBackupZip();
} catch (e) {
console.error(e);
Toast.error('Unexpected error while downloading backup.');
}
}, []);
const precheckRestore = React.useCallback(async (file) => {
try {
const data = await clientPrecheckRestore(file);
setPrecheckInfo(data);
setRestoreModalVisible(true);
} catch (e) {
console.error(e);
Toast.error('Failed to analyze backup.');
}
}, []);
const performRestore = React.useCallback(
async (force) => {
try {
setRestoreBusy(true);
await clientRestore(selectedRestoreFile, force);
Toast.success('Restore completed. Please restart the Fredy backend now!');
} catch (e) {
console.error(e);
Toast.error(e?.message || 'Unexpected error while restoring backup.');
} finally {
setRestoreBusy(false);
}
},
[selectedRestoreFile],
);
const handleSelectRestoreFile = React.useCallback(
async (ev) => {
const file = ev?.target?.files?.[0];
if (!file) return;
setSelectedRestoreFile(file);
await precheckRestore(file);
// reset the input to allow same file re-select
ev.target.value = '';
},
[precheckRestore],
);
const handleOpenFilePicker = React.useCallback(() => {
if (fileInputRef.current) {
fileInputRef.current.click();
}
}, []);
return (
<div>
{!loading && (
@@ -146,6 +210,28 @@ const GeneralSettings = function GeneralSettings() {
/>
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart
name="Backup & Restore"
helpText="Download a zipped backup of your database or restore it from a backup zip."
Icon={IconSave}
>
<div style={{ display: 'flex', gap: '0.5rem', alignItems: 'center', flexWrap: 'wrap' }}>
<Button theme="solid" icon={<IconSave />} onClick={handleDownloadBackup}>
Download backup
</Button>
<input
type="file"
accept=".zip,application/zip"
ref={fileInputRef}
style={{ display: 'none' }}
onChange={handleSelectRestoreFile}
/>
<Button onClick={handleOpenFilePicker} theme="light" icon={<IconFolder />}>
Restore from zip
</Button>
</div>
</SegmentPart>
<Divider margin="1rem" />
<SegmentPart name="Port" helpText="Port on which Fredy is running." Icon={IconSignal}>
<InputNumber
min={0}
@@ -271,12 +357,55 @@ const GeneralSettings = function GeneralSettings() {
</SegmentPart>
<Divider margin="1rem" />
<Button type="primary" theme="solid" onClick={onStore} icon={<IconSave />}>
<Button type="primary" theme="solid" onClick={handleStore} icon={<IconSave />}>
Save
</Button>
</div>
</React.Fragment>
)}
{restoreModalVisible && (
<Modal
title="Restore database"
visible={restoreModalVisible}
onCancel={() => setRestoreModalVisible(false)}
onOk={() => performRestore(!precheckInfo?.compatible)}
okText={precheckInfo?.compatible ? 'Restore now' : 'Restore anyway'}
okType={precheckInfo?.compatible ? 'primary' : 'danger'}
confirmLoading={restoreBusy}
>
{precheckInfo?.severity === 'danger' && (
<Banner
type="danger"
fullMode={false}
closeIcon={null}
title={<div style={{ fontWeight: 600, fontSize: '14px' }}>Problem detected</div>}
description={<div>{precheckInfo?.message}</div>}
/>
)}
{precheckInfo?.severity === 'warning' && (
<Banner
type="warning"
fullMode={false}
closeIcon={null}
title={<div style={{ fontWeight: 600, fontSize: '14px' }}>Automatic migrations will be applied</div>}
description={<div>{precheckInfo?.message}</div>}
/>
)}
{precheckInfo?.severity === 'info' && (
<Banner
type="success"
fullMode={false}
closeIcon={null}
title={<div style={{ fontWeight: 600, fontSize: '14px' }}>Backup is compatible</div>}
description={<div>{precheckInfo?.message}</div>}
/>
)}
<div style={{ marginTop: '0.5rem', fontSize: '12px', color: 'var(--semi-color-text-2)' }}>
Backup migration: {precheckInfo?.backupMigration ?? 'unknown'} | Required migration:{' '}
{precheckInfo?.requiredMigration ?? 'unknown'}
</div>
</Modal>
)}
</div>
);
};

View File

@@ -66,7 +66,6 @@ export default function Jobs() {
onJobRemoval={onJobRemoval}
onListingRemoval={onListingRemoval}
onJobStatusChanged={onJobStatusChanged}
onJobInsight={(jobId) => navigate(`/jobs/insights/${jobId}`)}
onJobEdit={(jobId) => navigate(`/jobs/edit/${jobId}`)}
/>
</div>

View File

@@ -1,102 +0,0 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { format } from '../../services/time/timeService';
import { Button, Card, Col, Row, Toast } from '@douyinfe/semi-ui';
import {
IconClock,
IconDoubleChevronLeft,
IconDoubleChevronRight,
IconPlayCircle,
IconSearch,
} from '@douyinfe/semi-icons';
import { xhrPost } from '../../services/xhr.js';
import './ProsessingTimes.less';
import { useScreenWidth } from '../../hooks/screenWidth.js';
function InfoCard({ title, value, icon }) {
const { Meta } = Card;
return (
<div
style={{
margin: '1rem',
background: 'rgb(53, 54, 60)',
borderRadius: '.3rem',
padding: '1rem',
minHeight: '3rem',
}}
>
<Meta title={title} description={value} avatar={icon} />
</div>
);
}
export default function ProcessingTimes({ processingTimes = {} }) {
if (Object.keys(processingTimes).length === 0) {
return null;
}
const width = useScreenWidth();
const invisible = width <= 1180;
if (invisible) {
return null;
}
return (
<Row>
<Col span={6}>
<InfoCard
title="Search Interval"
value={`${processingTimes.interval} min`}
icon={<IconClock style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
/>
</Col>
{processingTimes.lastRun && (
<>
<Col span={6}>
<InfoCard
title="Last search"
icon={<IconDoubleChevronLeft style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
value={format(processingTimes.lastRun)}
/>
</Col>
<Col span={6}>
<InfoCard
title="Next search"
icon={<IconDoubleChevronRight style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
value={format(processingTimes.lastRun + processingTimes.interval * 60000)}
/>
</Col>
</>
)}
<Col span={6}>
<InfoCard
title="Search Now"
icon={<IconSearch style={{ color: 'rgba(var(--semi-grey-4), 1)' }} />}
value={
<Button
size="small"
style={{ marginTop: '.2rem' }}
icon={<IconPlayCircle />}
aria-label="Start now"
onClick={async () => {
try {
await xhrPost('/api/jobs/startAll', null);
Toast.success('Successfully triggered Fredy search.');
} catch {
Toast.error('Failed to trigger search');
}
}}
>
Search now
</Button>
}
/>
</Col>
</Row>
);
}

View File

@@ -1,5 +0,0 @@
.processingTimes {
display: flex;
gap: 1rem;
justify-content: space-between;
}

View File

@@ -1,91 +0,0 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import { roundToHour } from '../../../services/time/timeService';
import Headline from '../../../components/headline/Headline';
import { useActions, useSelector } from '../../../services/state/store';
import { useParams } from 'react-router-dom';
import Linechart from './Linechart';
const JobInsight = function JobInsight() {
const actions = useActions();
const insights = useSelector((state) => state.jobs.insights);
const jobs = useSelector((state) => state.jobs.jobs);
const params = useParams();
React.useEffect(() => {
actions.jobs.getInsightDataForJob(params.jobId);
actions.jobs.getJobs();
}, []);
const getData = () => {
const data = insights[params.jobId] || {};
const providers = Object.keys(data);
const countsByProvider = {};
const allTimes = new Set();
const cap = (s) => (s ? s[0].toUpperCase() + s.slice(1) : 'Unknown');
providers.forEach((key) => {
const providerName = cap(key);
const tmpTimeObj = {};
Object.values(data[key] || {}).forEach((listingTs) => {
const time = roundToHour(listingTs);
tmpTimeObj[time] = tmpTimeObj[time] == null ? 1 : tmpTimeObj[time] + 1;
allTimes.add(time);
});
countsByProvider[providerName] = tmpTimeObj;
});
const sortedTimes = Array.from(allTimes).sort((a, b) => a - b);
const result = [];
providers.forEach((key) => {
const providerName = cap(key);
const bucket = countsByProvider[providerName] || {};
sortedTimes.forEach((t) => {
result.push({
listings: new Intl.DateTimeFormat('en-US', {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
hour12: false,
}).format(new Date(parseInt(t))),
listingsNumber: bucket[t] || 0, // y value
provider: providerName, // series key
});
});
});
return result;
};
const getJobName = () => {
const job = jobs.find((job) => job.id === params.jobId);
if (job == null) {
return 'unknown';
} else {
return job.name;
}
};
return (
<div>
<Headline text={`Insights into Job: ${getJobName()}`} />
<Linechart isLoading={false} series={getData()} />
</div>
);
};
export default JobInsight;

View File

@@ -1,57 +0,0 @@
/*
* Copyright (c) 2025 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import React from 'react';
import Placeholder from '../../../components/placeholder/Placeholder';
import { VChart } from '@visactor/react-vchart';
import './Linechart.less';
const commonSpec = {
type: 'line',
xField: 'listings',
yField: 'listingsNumber',
seriesField: 'provider',
legends: { visible: true },
line: {
style: {
lineWidth: 2,
},
},
point: {
visible: false,
},
axes: [
{
orient: 'bottom',
field: 'listings',
zero: false,
},
],
};
const Linechart = function Linechart({ title, series, isLoading = false }) {
return (
<Placeholder ready={!isLoading} rows={6}>
{series == null || series.length === 0 ? (
<div className="linechart__no__data">No Data for selected timeframe :-/</div>
) : (
<VChart
spec={{
...commonSpec,
title: {
visible: true,
text: title,
},
data: { values: series },
}}
/>
)}
</Placeholder>
);
};
export default Linechart;

View File

@@ -1,15 +0,0 @@
.linechart {
&__no__data {
width: 100%;
text-align: center;
display: flex;
align-items: center;
justify-content: center;
color: #06dcfff2;
flex-direction: column;
&__height {
height: 30.7rem;
}
}
}

View File

@@ -52,7 +52,7 @@ export default function Login() {
Toast.success('Login successful!');
await actions.user.getCurrentUser();
navigate('/jobs');
navigate('/dashboard');
};
return (

982
yarn.lock

File diff suppressed because it is too large Load Diff