mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d43c5b3f97 | ||
|
|
7fd8be07a2 | ||
|
|
2926ee7e08 | ||
|
|
9506d1a9db | ||
|
|
feaa06c132 | ||
|
|
ad46500d4e | ||
|
|
3c209a8f97 | ||
|
|
398259ff20 | ||
|
|
cf030bfa39 | ||
|
|
5dc976c7e3 | ||
|
|
05f1bc61c9 |
@@ -34,7 +34,8 @@ WORKDIR /fredy
|
||||
# Using Alpine's chromium package which is much smaller
|
||||
RUN apk add --no-cache chromium curl
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
ENV NODE_ENV=production \
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
# Install build dependencies for native modules, then remove them after yarn install
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -210,5 +210,5 @@ different name or branding without the explicit written permission of the
|
||||
original copyright holder.
|
||||
|
||||
|
||||
Copyright (c) 2025 Christian Kellner
|
||||
Copyright (c) 2026 Christian Kellner
|
||||
Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
|
||||
@@ -119,7 +119,7 @@ Should you use [Unraid](https://unraid.net/), you can now install Fredy from the
|
||||
|
||||
## 📸 Screenshots
|
||||
|
||||
| Fredy Main Overview | Job Configuration | Found Listings |
|
||||
| Fredy Maps View | Dashboard | Found Listings |
|
||||
|--------------------------------------------------|-----------------------------------------------------------------------|-----------------------------------------------------------------------------|
|
||||
|  |  |  |
|
||||
|
||||
@@ -206,7 +206,7 @@ flowchart TD
|
||||
F2["Adapter 2"]
|
||||
end
|
||||
|
||||
A1 --> B["FredyPipeline"]
|
||||
A1 --> B["FredyPipelineExecutioner"]
|
||||
A2 --> B
|
||||
A3 --> B
|
||||
B --> C1 & C2 & C3
|
||||
|
||||
12
copyright.js
12
copyright.js
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -30,12 +30,16 @@ async function getAllFiles(dir = '.') {
|
||||
|
||||
/* eslint-disable no-console */
|
||||
async function addCopyright(files) {
|
||||
const oldCopyrightRegex =
|
||||
/^(\/\*\n \* Copyright \(c\) \d{4} by Christian Kellner\.\n \* Licensed under Apache-2.0 with Commons Clause and Attribution\/Naming Clause\n \*\/\n\n)+/;
|
||||
for (let file of files) {
|
||||
try {
|
||||
let content = await fs.readFile(file, 'utf8');
|
||||
if (!content.startsWith(COPYRIGHT)) {
|
||||
await fs.writeFile(file, COPYRIGHT + content);
|
||||
console.log(`Added copyright to ${file}`);
|
||||
const strippedContent = content.replace(oldCopyrightRegex, '');
|
||||
const newContent = COPYRIGHT + strippedContent;
|
||||
if (content !== newContent) {
|
||||
await fs.writeFile(file, newContent);
|
||||
console.log(`Added/Updated copyright in ${file}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error processing ${file}: ${err}`);
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 197 KiB After Width: | Height: | Size: 3.7 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 512 KiB After Width: | Height: | Size: 4.7 MiB |
@@ -5,6 +5,8 @@ services:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: ghcr.io/orangecoding/fredy
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
volumes:
|
||||
- ./conf:/conf
|
||||
- ./db:/db
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
56
index.js
56
index.js
@@ -1,23 +1,21 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import { checkIfConfigIsAccessible, getProviders, refreshConfig } from './lib/utils.js';
|
||||
import * as similarityCache from './lib/services/similarity-check/similarityCache.js';
|
||||
import * as jobStorage from './lib/services/storage/jobStorage.js';
|
||||
import FredyPipeline from './lib/FredyPipeline.js';
|
||||
import { duringWorkingHoursOrNotSet } from './lib/utils.js';
|
||||
import { runMigrations } from './lib/services/storage/migrations/migrate.js';
|
||||
import { ensureDemoUserExists, ensureAdminUserExists } from './lib/services/storage/userStorage.js';
|
||||
import { cleanupDemoAtMidnight } from './lib/services/crons/demoCleanup-cron.js';
|
||||
import { initTrackerCron } from './lib/services/crons/tracker-cron.js';
|
||||
import logger from './lib/services/logger.js';
|
||||
import { bus } from './lib/services/events/event-bus.js';
|
||||
import { initActiveCheckerCron } from './lib/services/crons/listing-alive-cron.js';
|
||||
import { initGeocodingCron } from './lib/services/crons/geocoding-cron.js';
|
||||
import { getSettings } from './lib/services/storage/settingsStorage.js';
|
||||
import SqliteConnection, { computeDbPath } from './lib/services/storage/SqliteConnection.js';
|
||||
import { initJobExecutionService } from './lib/services/jobs/jobExecutionService.js';
|
||||
|
||||
//in the config, we store the path of the sqlite file, thus we must check if it is available
|
||||
const isConfigAccessible = await checkIfConfigIsAccessible();
|
||||
@@ -36,7 +34,7 @@ await runMigrations();
|
||||
|
||||
const settings = await getSettings();
|
||||
|
||||
// Ensure sqlite directory exists before loading anything else (based on config.sqlitepath)
|
||||
// Ensure the sqlite directory exists before loading anything else (based on config.sqlitepath)
|
||||
const { dir: sqliteDir } = await computeDbPath();
|
||||
if (!fs.existsSync(sqliteDir)) {
|
||||
fs.mkdirSync(sqliteDir, { recursive: true });
|
||||
@@ -59,52 +57,14 @@ if (settings.demoMode) {
|
||||
cleanupDemoAtMidnight();
|
||||
}
|
||||
|
||||
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
|
||||
|
||||
ensureAdminUserExists();
|
||||
ensureDemoUserExists();
|
||||
await initTrackerCron();
|
||||
//do not wait for this to finish, let it run in the background
|
||||
initActiveCheckerCron();
|
||||
initGeocodingCron();
|
||||
|
||||
bus.on('jobs:runAll', () => {
|
||||
logger.debug('Running Fredy Job manually');
|
||||
execute();
|
||||
});
|
||||
logger.info(`Started Fredy successfully. Ui can be accessed via http://localhost:${settings.port}`);
|
||||
|
||||
const execute = () => {
|
||||
const isDuringWorkingHoursOrNotSet = duringWorkingHoursOrNotSet(settings, Date.now());
|
||||
if (!settings.demoMode) {
|
||||
if (isDuringWorkingHoursOrNotSet) {
|
||||
settings.lastRun = Date.now();
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.forEach((job) => {
|
||||
job.provider
|
||||
.filter((p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null)
|
||||
.forEach(async (prov) => {
|
||||
try {
|
||||
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
||||
matchedProvider.init(prov, job.blacklist);
|
||||
await new FredyPipeline(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
).execute();
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
setInterval(execute, INTERVAL);
|
||||
//start once at startup
|
||||
execute();
|
||||
// Initialize the lean Job Execution Service (schedules and bus listeners)
|
||||
initJobExecutionService({ providers, settings, intervalMs: INTERVAL });
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -9,6 +9,7 @@ import * as notify from './notification/notify.js';
|
||||
import Extractor from './services/extractor/extractor.js';
|
||||
import urlModifier from './services/queryStringMutator.js';
|
||||
import logger from './services/logger.js';
|
||||
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
||||
|
||||
/**
|
||||
* @typedef {Object} Listing
|
||||
@@ -40,7 +41,7 @@ import logger from './services/logger.js';
|
||||
* 7) Filter out entries similar to already seen ones
|
||||
* 8) Dispatch notifications
|
||||
*/
|
||||
class FredyPipeline {
|
||||
class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Create a new runtime instance for a single provider/job execution.
|
||||
*
|
||||
@@ -79,12 +80,32 @@ class FredyPipeline {
|
||||
.then(this._normalize.bind(this))
|
||||
.then(this._filter.bind(this))
|
||||
.then(this._findNew.bind(this))
|
||||
.then(this._geocode.bind(this))
|
||||
.then(this._save.bind(this))
|
||||
.then(this._filterBySimilarListings.bind(this))
|
||||
.then(this._notify.bind(this))
|
||||
.catch(this._handleError.bind(this));
|
||||
}
|
||||
|
||||
/**
|
||||
* Geocode new listings.
|
||||
*
|
||||
* @param {Listing[]} newListings New listings to geocode.
|
||||
* @returns {Promise<Listing[]>} Resolves with the listings (potentially with added coordinates).
|
||||
*/
|
||||
async _geocode(newListings) {
|
||||
for (const listing of newListings) {
|
||||
if (listing.address) {
|
||||
const coords = await geocodeAddress(listing.address);
|
||||
if (coords) {
|
||||
listing.latitude = coords.lat;
|
||||
listing.longitude = coords.lng;
|
||||
}
|
||||
}
|
||||
}
|
||||
return newListings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch listings from the provider, using the default Extractor flow unless
|
||||
* a provider-specific getListings override is supplied.
|
||||
@@ -218,4 +239,4 @@ class FredyPipeline {
|
||||
}
|
||||
}
|
||||
|
||||
export default FredyPipeline;
|
||||
export default FredyPipelineExecutioner;
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -9,6 +9,8 @@ import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { bus } from '../../services/events/event-bus.js';
|
||||
import { isRunning as isJobRunning } from '../../services/jobs/run-state.js';
|
||||
import { addClient as addSseClient, removeClient } from '../../services/sse/sse-broker.js';
|
||||
|
||||
const service = restana();
|
||||
const jobRouter = service.newRouter();
|
||||
@@ -37,6 +39,7 @@ jobRouter.get('/', async (req, res) => {
|
||||
.map((job) => {
|
||||
return {
|
||||
...job,
|
||||
running: isJobRunning(job.id),
|
||||
isOnlyShared:
|
||||
!isUserAdmin &&
|
||||
job.userId !== req.session.currentUser &&
|
||||
@@ -47,11 +50,115 @@ jobRouter.get('/', async (req, res) => {
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.post('/startAll', async (req, res) => {
|
||||
bus.emit('jobs:runAll');
|
||||
jobRouter.get('/data', async (req, res) => {
|
||||
const { page, pageSize = 50, activityFilter, sortfield = null, sortdir = 'asc', freeTextFilter } = req.query || {};
|
||||
|
||||
// normalize booleans
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
|
||||
const queryResult = jobStorage.queryJobs({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdmin(req),
|
||||
});
|
||||
|
||||
const isUserAdmin = isAdmin(req);
|
||||
|
||||
// Map result to include runtime status
|
||||
queryResult.result = queryResult.result.map((job) => {
|
||||
return {
|
||||
...job,
|
||||
running: isJobRunning(job.id),
|
||||
isOnlyShared:
|
||||
!isUserAdmin &&
|
||||
job.userId !== req.session.currentUser &&
|
||||
job.shared_with_user.includes(req.session.currentUser),
|
||||
};
|
||||
});
|
||||
|
||||
res.body = queryResult;
|
||||
res.send();
|
||||
});
|
||||
|
||||
// Server-Sent Events for job status updates
|
||||
jobRouter.get('/events', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
if (userId == null) {
|
||||
res.send({ message: 'Unauthorized' }, 401);
|
||||
return;
|
||||
}
|
||||
// SSE headers
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
try {
|
||||
// Initial comment to establish stream
|
||||
res.write(': connected\n\n');
|
||||
addSseClient(userId, res);
|
||||
// Cleanup on close/aborted
|
||||
const onClose = () => removeClient(userId, res);
|
||||
// restana exposes original req/res; use both close and finish
|
||||
req.on('close', onClose);
|
||||
req.on('aborted', onClose);
|
||||
res.on('close', onClose);
|
||||
} catch (e) {
|
||||
logger.error('Error establishing SSE connection', e);
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
//noop
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
jobRouter.post('/startAll', async (req, res) => {
|
||||
try {
|
||||
const userId = req.session.currentUser;
|
||||
// Emit only the userId; handler will decide based on admin/ownership
|
||||
bus.emit('jobs:runAll', { userId });
|
||||
res.send({ message: 'Run all accepted' }, 202);
|
||||
} catch (err) {
|
||||
logger.error('Failed to trigger startAll', err);
|
||||
res.send({ message: 'Unexpected error' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
// Trigger a single job run
|
||||
jobRouter.post('/:jobId/run', async (req, res) => {
|
||||
const { jobId } = req.params;
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) {
|
||||
res.send({ message: 'Job not found' }, 404);
|
||||
return;
|
||||
}
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send({ message: 'You are trying to run a job that is not associated to your user' }, 403);
|
||||
return;
|
||||
}
|
||||
if (isJobRunning(jobId)) {
|
||||
res.send({ message: 'Job is already running' }, 409);
|
||||
return;
|
||||
}
|
||||
// fire and forget; actual execution handled by index.js listener
|
||||
bus.emit('jobs:runOne', { jobId });
|
||||
res.send({ message: 'Job run accepted' }, 202);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
res.send({ message: 'Unexpected error triggering job' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
jobRouter.post('/', async (req, res) => {
|
||||
const { provider, notificationAdapter, name, blacklist = [], jobId, enabled, shareWithUsers = [] } = req.body;
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -28,10 +28,14 @@ listingsRouter.get('/table', async (req, res) => {
|
||||
freeTextFilter,
|
||||
} = req.query || {};
|
||||
|
||||
// normalize booleans (accept true, 'true', 1, '1')
|
||||
const toBool = (v) => v === true || v === 'true' || v === 1 || v === '1';
|
||||
const normalizedActivity = toBool(activityFilter) ? true : null;
|
||||
const normalizedWatch = toBool(watchListFilter) ? true : null;
|
||||
// normalize booleans (accept true, 'true', 1, '1' for true; false, 'false', 0, '0' for false)
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
const normalizedWatch = toBool(watchListFilter);
|
||||
|
||||
let jobFilter = null;
|
||||
let jobIdFilter = null;
|
||||
@@ -59,6 +63,19 @@ listingsRouter.get('/table', async (req, res) => {
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.get('/map', async (req, res) => {
|
||||
const { jobId, minPrice, maxPrice } = req.query || {};
|
||||
|
||||
res.body = listingStorage.getListingsForMap({
|
||||
jobId: nullOrEmpty(jobId) ? null : jobId,
|
||||
minPrice: minPrice ? parseInt(minPrice, 10) : null,
|
||||
maxPrice: maxPrice ? parseInt(maxPrice, 10) : null,
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdminFn(req),
|
||||
});
|
||||
res.send();
|
||||
});
|
||||
|
||||
// Toggle watch state for the current user on a listing
|
||||
listingsRouter.post('/watch', async (req, res) => {
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -44,7 +44,7 @@ export const init = (sourceConfig, blacklist) => {
|
||||
|
||||
export const metaInformation = {
|
||||
name: 'OhneMakler',
|
||||
baseUrl: 'https://www.ohne-makler.net/immobilien',
|
||||
baseUrl: 'https://www.ohne-makler.net',
|
||||
id: 'ohneMakler',
|
||||
};
|
||||
export { config };
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
56
lib/provider/wohnungsboerse.js
Normal file
56
lib/provider/wohnungsboerse.js
Normal file
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import * as utils from '../utils.js';
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
function normalize(o) {
|
||||
const id = o.link.split('/').pop();
|
||||
const price = o.price;
|
||||
const size = o.size;
|
||||
const rooms = o.rooms;
|
||||
const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim());
|
||||
const address = `${part}, ${city}`;
|
||||
return Object.assign(o, { id, price, size, rooms, address });
|
||||
}
|
||||
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return o.id != null && o.title != null && titleNotBlacklisted && descNotBlacklisted && o.link.startsWith(o.link);
|
||||
}
|
||||
|
||||
const config = {
|
||||
url: null,
|
||||
sortByDateParam: null,
|
||||
waitForSelector: 'body',
|
||||
crawlContainer: '.search_result_container > a',
|
||||
crawlFields: {
|
||||
id: '*',
|
||||
title: 'h3 | trim',
|
||||
price: 'dl:nth-of-type(1) dd | removeNewline | trim',
|
||||
rooms: 'dl:nth-of-type(2) dd | removeNewline | trim',
|
||||
size: 'dl:nth-of-type(3) dd | removeNewline | trim',
|
||||
description: 'div.before\\:icon-location_marker | trim',
|
||||
link: '@href',
|
||||
imageUrl: 'img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
};
|
||||
|
||||
export const init = (sourceConfig, blacklistTerms) => {
|
||||
config.url = sourceConfig.url;
|
||||
appliedBlackList = blacklistTerms || [];
|
||||
};
|
||||
|
||||
export const metaInformation = {
|
||||
name: 'Wohnungsboerse',
|
||||
baseUrl: 'https://www.wohnungsboerse.net',
|
||||
id: 'wohnungsboerse',
|
||||
};
|
||||
|
||||
export { config };
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
33
lib/services/crons/geocoding-cron.js
Normal file
33
lib/services/crons/geocoding-cron.js
Normal file
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import cron from 'node-cron';
|
||||
import { getListingsToGeocode, updateListingGeocoordinates } from '../storage/listingsStorage.js';
|
||||
import { geocodeAddress, isGeocodingPaused } from '../geocoding/geoCodingService.js';
|
||||
|
||||
async function runTask() {
|
||||
const listings = getListingsToGeocode();
|
||||
if (listings.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const listing of listings) {
|
||||
if (isGeocodingPaused()) {
|
||||
break;
|
||||
}
|
||||
|
||||
const coords = await geocodeAddress(listing.address);
|
||||
if (coords) {
|
||||
updateListingGeocoordinates(listing.id, coords.lat, coords.lng);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function initGeocodingCron() {
|
||||
// run directly on start
|
||||
await runTask();
|
||||
// then every 6 hours
|
||||
cron.schedule('0 */6 * * *', runTask);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -104,7 +104,11 @@ export default async function execute(url, waitForSelector, options) {
|
||||
result = pageSource || (await page.content());
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Error executing with puppeteer executor', error);
|
||||
if (error?.message?.includes('Timeout')) {
|
||||
logger.debug('Error executing with puppeteer executor', error);
|
||||
} else {
|
||||
logger.warn('Error executing with puppeteer executor', error);
|
||||
}
|
||||
result = null;
|
||||
} finally {
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
105
lib/services/geocoding/client/nominatimClient.js
Normal file
105
lib/services/geocoding/client/nominatimClient.js
Normal file
@@ -0,0 +1,105 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import os from 'os';
|
||||
import crypto from 'crypto';
|
||||
import https from 'https';
|
||||
import fetch from 'node-fetch';
|
||||
import pThrottle from 'p-throttle';
|
||||
import logger from '../../logger.js';
|
||||
|
||||
const API_URL = 'https://nominatim.openstreetmap.org/search';
|
||||
|
||||
const agent = new https.Agent({
|
||||
keepAlive: true,
|
||||
keepAliveMsecs: 1000,
|
||||
});
|
||||
|
||||
const throttle = pThrottle({
|
||||
limit: 1,
|
||||
interval: 1000,
|
||||
});
|
||||
|
||||
function computeMachineId() {
|
||||
const hostname = os.hostname() || 'unknown-host';
|
||||
const nets = os.networkInterfaces?.() || {};
|
||||
const macs = [];
|
||||
|
||||
for (const ifname of Object.keys(nets)) {
|
||||
for (const addr of nets[ifname] || []) {
|
||||
if (!addr) continue;
|
||||
if (addr.internal) continue;
|
||||
if (addr.mac && addr.mac !== '00:00:00:00:00:00') macs.push(addr.mac);
|
||||
}
|
||||
}
|
||||
|
||||
macs.sort();
|
||||
|
||||
const raw = [hostname, os.platform(), os.arch(), ...macs].join('|');
|
||||
|
||||
return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 20);
|
||||
}
|
||||
|
||||
/**
|
||||
* Nominatim requires a specific User-Agent.
|
||||
* Since Fredy is self-hosted, we use a unique machine ID to make it specific.
|
||||
*/
|
||||
const userAgent = `Fredy-Self-Hosted (${computeMachineId()}; https://github.com/orangecoding/fredy)`;
|
||||
|
||||
let last429 = 0;
|
||||
const PAUSE_DURATION = 3600000; // 1 hour
|
||||
|
||||
/**
|
||||
* Geocodes an address using Nominatim.
|
||||
*
|
||||
* @param {string} address - The address to geocode.
|
||||
* @returns {Promise<{lat: number, lng: number}|null>} The geocoordinates or null if error. {lat: -1, lng: -1} if not found.
|
||||
*/
|
||||
async function doGeocode(address) {
|
||||
if (Date.now() - last429 < PAUSE_DURATION) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const url = `${API_URL}?q=${encodeURIComponent(address)}&format=json&countrycodes=de`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
agent,
|
||||
headers: {
|
||||
'User-Agent': userAgent,
|
||||
},
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
logger.warn('Nominatim rate limit hit. Pausing for 1 hour.');
|
||||
last429 = Date.now();
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(`Nominatim API error: ${response.status} ${response.statusText}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (Array.isArray(data) && data.length > 0) {
|
||||
const result = data[0];
|
||||
return {
|
||||
lat: parseFloat(result.lat),
|
||||
lng: parseFloat(result.lon),
|
||||
};
|
||||
}
|
||||
|
||||
return { lat: -1, lng: -1 };
|
||||
} catch (error) {
|
||||
logger.error('Error during Nominatim geocoding:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const geocode = throttle(doGeocode);
|
||||
|
||||
export const isPaused = () => Date.now() - last429 < PAUSE_DURATION;
|
||||
43
lib/services/geocoding/geoCodingService.js
Normal file
43
lib/services/geocoding/geoCodingService.js
Normal file
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { getGeocoordinatesByAddress } from '../storage/listingsStorage.js';
|
||||
import { geocode as nominatimGeocode, isPaused as isNominatimPaused } from './client/nominatimClient.js';
|
||||
import logger from '../logger.js';
|
||||
|
||||
/**
|
||||
* Geocodes an address using Nominatim or cached results from the database.
|
||||
*
|
||||
* @param {string} address - The address to geocode.
|
||||
* @returns {Promise<{lat: number, lng: number}|null>} The geocoordinates or null if error. {lat: -1, lng: -1} if not found.
|
||||
*/
|
||||
export async function geocodeAddress(address) {
|
||||
if (!address) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. Check if we already have this address geocoded in our database
|
||||
const cachedCoordinates = getGeocoordinatesByAddress(address);
|
||||
if (cachedCoordinates) {
|
||||
logger.debug(`Found cached geocoordinates for address: ${address}`);
|
||||
return cachedCoordinates;
|
||||
}
|
||||
|
||||
// 2. If not, use Nominatim
|
||||
return await nominatimGeocode(address);
|
||||
} catch (error) {
|
||||
logger.error('Error during geocoding:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if we are currently in a rate limit pause.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isGeocodingPaused() {
|
||||
return isNominatimPaused();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
187
lib/services/jobs/jobExecutionService.js
Normal file
187
lib/services/jobs/jobExecutionService.js
Normal file
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import logger from '../logger.js';
|
||||
import { bus } from '../events/event-bus.js';
|
||||
import * as jobStorage from '../storage/jobStorage.js';
|
||||
import * as userStorage from '../storage/userStorage.js';
|
||||
import { getUser } from '../storage/userStorage.js';
|
||||
import { duringWorkingHoursOrNotSet } from '../../utils.js';
|
||||
import FredyPipelineExecutioner from '../../FredyPipelineExecutioner.js';
|
||||
import * as similarityCache from '../similarity-check/similarityCache.js';
|
||||
import { isRunning, markFinished, markRunning } from './run-state.js';
|
||||
import { sendToUsers } from '../sse/sse-broker.js';
|
||||
|
||||
/**
|
||||
* Initializes the job execution service.
|
||||
* - Registers event-bus listeners for `jobs:runAll`, `jobs:runOne`, and `jobs:status`.
|
||||
* - Starts the periodic scheduler (if `intervalMs` > 0) and performs an initial run respecting working hours.
|
||||
* - Forwards job status updates to affected users via Server-Sent Events (SSE).
|
||||
*
|
||||
* This function is intentionally side-effectful and exposes no external API.
|
||||
*
|
||||
* @param {Object} deps - Dependencies required to initialize the service.
|
||||
* @param {Array<Object>} deps.providers - Loaded provider modules. Each module must expose `metaInformation.id`, `config`, and `init(config, blacklist)`.
|
||||
* @param {Object} deps.settings - Global settings object (read/write). Must include `demoMode`, `interval`, and working-hours attributes used by `duringWorkingHoursOrNotSet`.
|
||||
* @param {number} deps.intervalMs - Scheduler interval in milliseconds. If not finite or <= 0, the scheduler is not started.
|
||||
* @returns {void}
|
||||
*/
|
||||
export function initJobExecutionService({ providers, settings, intervalMs }) {
|
||||
// Forward job status via SSE to relevant recipients
|
||||
bus.on('jobs:status', ({ jobId, running }) => {
|
||||
try {
|
||||
const recipients = resolveRecipients(jobId);
|
||||
if (recipients.length > 0) {
|
||||
sendToUsers(recipients, 'jobStatus', { jobId, running });
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to forward job status', jobId, err);
|
||||
}
|
||||
});
|
||||
|
||||
// Listen for "run all" requests (admin = all, user = own)
|
||||
bus.on('jobs:runAll', (payload) => {
|
||||
const userId = payload?.userId ?? null;
|
||||
const user = userId ? getUser(userId) : null;
|
||||
const isAdmin = !!user?.isAdmin;
|
||||
if (isAdmin) {
|
||||
logger.debug('Running all jobs manually (admin request)');
|
||||
} else if (userId) {
|
||||
logger.debug(`Running all jobs manually for user ${userId}`);
|
||||
} else {
|
||||
logger.debug('Running all jobs manually (no user provided)');
|
||||
}
|
||||
runAll(false, { userId, isAdmin });
|
||||
});
|
||||
|
||||
// Listen for single job run requests
|
||||
bus.on('jobs:runOne', ({ jobId }) => {
|
||||
logger.debug(`Running single job manually: ${jobId}`);
|
||||
// fire and forget, do not block the bus
|
||||
runSingle(jobId);
|
||||
});
|
||||
|
||||
// Start scheduler and initial run
|
||||
if (Number.isFinite(intervalMs) && intervalMs > 0) {
|
||||
setInterval(() => runAll(true), intervalMs);
|
||||
}
|
||||
// start once at startup, respecting working hours
|
||||
runAll(true);
|
||||
|
||||
/**
|
||||
* Resolve all recipients who should receive SSE updates for a job.
|
||||
* Includes job owner, users with whom the job is shared, and all admins.
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @returns {string[]} unique userIds
|
||||
*/
|
||||
function resolveRecipients(jobId) {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) return [];
|
||||
const admins = (userStorage.getUsers && userStorage.getUsers(false)) || [];
|
||||
const adminIds = admins.filter((u) => u.isAdmin).map((u) => u.id);
|
||||
const shared = Array.isArray(job.shared_with_user) ? job.shared_with_user : [];
|
||||
const recipients = [job.userId, ...shared, ...adminIds].filter(Boolean);
|
||||
return Array.from(new Set(recipients));
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute all enabled jobs, optionally filtering by context (admin/owner) and respecting working hours.
|
||||
*
|
||||
* @param {boolean} [respectWorkingHours=true] - If true, skip execution when outside configured working hours.
|
||||
* @param {{userId?: string, isAdmin?: boolean}} [context] - Who requested the run; determines job filtering.
|
||||
* @returns {void}
|
||||
*/
|
||||
function runAll(respectWorkingHours = true, context = undefined) {
|
||||
if (settings.demoMode) return;
|
||||
const now = Date.now();
|
||||
const withinHours = duringWorkingHoursOrNotSet(settings, now);
|
||||
if (respectWorkingHours && !withinHours) {
|
||||
logger.debug('Working hours set. Skipping as outside of working hours.');
|
||||
return;
|
||||
}
|
||||
settings.lastRun = now;
|
||||
jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => job.enabled)
|
||||
.filter((job) => {
|
||||
if (!context) return true; // startup/cron → all
|
||||
if (context.isAdmin) return true; // admin → all
|
||||
return context.userId ? job.userId === context.userId : false; // user → own
|
||||
})
|
||||
.forEach((job) => executeJob(job));
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single job by id.
|
||||
* Manual runs are allowed even if the job is disabled, but never duplicated when already running.
|
||||
*
|
||||
* @param {string} jobId
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function runSingle(jobId) {
|
||||
if (settings.demoMode) return;
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) return;
|
||||
// allow manual run even if disabled; keep guard to avoid duplicates
|
||||
await executeJob(job);
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes one job across all of its configured providers.
|
||||
* Emits SSE start/finish events via the bus and ensures the run-state guard is always cleared.
|
||||
* Provider errors are surfaced via logging but do not abort other providers.
|
||||
*
|
||||
* @param {Object} job
|
||||
* @param {string} job.id
|
||||
* @param {Array<{id:string}>} job.provider
|
||||
* @param {Array<string>} [job.blacklist]
|
||||
* @param {*} job.notificationAdapter
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function executeJob(job) {
|
||||
if (isRunning(job.id)) {
|
||||
logger.debug(`Job ${job.id} is already running. Skipping.`);
|
||||
return;
|
||||
}
|
||||
const acquired = markRunning(job.id);
|
||||
if (!acquired) return;
|
||||
// notify listeners (SSE) that the job started
|
||||
try {
|
||||
bus.emit('jobs:status', { jobId: job.id, running: true });
|
||||
} catch (err) {
|
||||
logger.warn('Failed to emit start status for job', job.id, err);
|
||||
}
|
||||
try {
|
||||
const jobProviders = job.provider.filter(
|
||||
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
|
||||
);
|
||||
const executions = jobProviders.map(async (prov) => {
|
||||
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
|
||||
matchedProvider.init(prov, job.blacklist);
|
||||
await new FredyPipelineExecutioner(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
).execute();
|
||||
});
|
||||
const results = await Promise.allSettled(executions);
|
||||
for (const r of results) {
|
||||
if (r.status === 'rejected') {
|
||||
logger.error(r.reason);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
markFinished(job.id);
|
||||
try {
|
||||
bus.emit('jobs:status', { jobId: job.id, running: false });
|
||||
} catch (err) {
|
||||
logger.warn('Failed to emit finish status for job', job.id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
42
lib/services/jobs/run-state.js
Normal file
42
lib/services/jobs/run-state.js
Normal file
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* Simple in-memory running state registry for jobs.
|
||||
* Prevents concurrent execution of the same job within a single process.
|
||||
* This registry is reset on process restart.
|
||||
* @type {Set<string>}
|
||||
*/
|
||||
const running = new Set();
|
||||
|
||||
/**
|
||||
* Check if a job is currently marked as running.
|
||||
* @param {string} jobId
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isRunning(jobId) {
|
||||
return running.has(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to mark a job as running.
|
||||
* If it was already running, returns false and does not modify the set.
|
||||
* @param {string} jobId
|
||||
* @returns {boolean} true if the job was successfully marked as running
|
||||
*/
|
||||
export function markRunning(jobId) {
|
||||
if (running.has(jobId)) return false;
|
||||
running.add(jobId);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a job as finished (remove from the running registry).
|
||||
* @param {string} jobId
|
||||
* @returns {void}
|
||||
*/
|
||||
export function markFinished(jobId) {
|
||||
running.delete(jobId);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
108
lib/services/sse/sse-broker.js
Normal file
108
lib/services/sse/sse-broker.js
Normal file
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* In-memory SSE client registry.
|
||||
* Maps a userId to a Set of Node.js ServerResponse objects representing open streams.
|
||||
* @type {Map<string, Set<import('http').ServerResponse>>}
|
||||
*/
|
||||
const clients = new Map(); // Map<userId, Set<ServerResponse>>
|
||||
|
||||
/**
|
||||
* Write a single SSE event frame to a response.
|
||||
*
|
||||
* @param {import('http').ServerResponse} res - The open SSE HTTP response.
|
||||
* @param {string} [event] - Optional event name (sent as `event:`). If omitted, a generic message is sent.
|
||||
* @param {any} [data] - Optional payload. Objects are JSON.stringified.
|
||||
* @returns {void}
|
||||
*/
|
||||
function writeEvent(res, event, data) {
|
||||
try {
|
||||
if (event) {
|
||||
res.write(`event: ${event}\n`);
|
||||
}
|
||||
if (data !== undefined) {
|
||||
const payload = typeof data === 'string' ? data : JSON.stringify(data);
|
||||
res.write(`data: ${payload}\n`);
|
||||
}
|
||||
res.write('\n');
|
||||
} catch {
|
||||
// ignore write errors here; cleanup happens on close
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new SSE client for the given user.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {import('http').ServerResponse} res
|
||||
* @returns {void}
|
||||
*/
|
||||
export function addClient(userId, res) {
|
||||
let set = clients.get(userId);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
clients.set(userId, set);
|
||||
}
|
||||
set.add(res);
|
||||
// send a hello event
|
||||
writeEvent(res, 'hello', { ok: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a specific SSE client for a user. Removes the user entry when empty.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {import('http').ServerResponse} res
|
||||
* @returns {void}
|
||||
*/
|
||||
export function removeClient(userId, res) {
|
||||
const set = clients.get(userId);
|
||||
if (!set) return;
|
||||
set.delete(res);
|
||||
if (set.size === 0) clients.delete(userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an SSE event to all open connections of a user.
|
||||
*
|
||||
* @param {string} userId
|
||||
* @param {string} event
|
||||
* @param {any} data
|
||||
* @returns {void}
|
||||
*/
|
||||
export function sendToUser(userId, event, data) {
|
||||
const set = clients.get(userId);
|
||||
if (!set) return;
|
||||
for (const res of set) {
|
||||
writeEvent(res, event, data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Broadcast an SSE event to multiple users (unique by id).
|
||||
*
|
||||
* @param {string[]} userIds
|
||||
* @param {string} event
|
||||
* @param {any} data
|
||||
* @returns {void}
|
||||
*/
|
||||
export function sendToUsers(userIds, event, data) {
|
||||
const unique = Array.from(new Set(userIds));
|
||||
unique.forEach((id) => sendToUser(id, event, data));
|
||||
}
|
||||
|
||||
// Heartbeat to keep connections alive on proxies (every 25s)
|
||||
setInterval(() => {
|
||||
for (const set of clients.values()) {
|
||||
for (const res of set) {
|
||||
try {
|
||||
res.write(`: ping ${Date.now()}\n\n`);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 25000);
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -19,8 +19,13 @@ import { runMigrations, listMigrationFiles } from './migrations/migrate.js';
|
||||
let _AdmZipSingleton = null;
|
||||
async function getAdmZip() {
|
||||
if (_AdmZipSingleton) return _AdmZipSingleton;
|
||||
// Allow tests to provide a mock constructor without ESM loader intricacies
|
||||
if (globalThis && globalThis.__TEST_ADM_ZIP__) {
|
||||
_AdmZipSingleton = globalThis.__TEST_ADM_ZIP__;
|
||||
return _AdmZipSingleton;
|
||||
}
|
||||
const mod = await import('adm-zip');
|
||||
_AdmZipSingleton = mod.default || mod;
|
||||
_AdmZipSingleton = (mod && mod.default) || mod;
|
||||
return _AdmZipSingleton;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -85,6 +85,7 @@ export const getJob = (jobId) => {
|
||||
j.name,
|
||||
j.blacklist,
|
||||
j.provider,
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
@@ -98,6 +99,7 @@ export const getJob = (jobId) => {
|
||||
enabled: !!row.enabled,
|
||||
blacklist: fromJson(row.blacklist, []),
|
||||
provider: fromJson(row.provider, []),
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
};
|
||||
};
|
||||
@@ -161,3 +163,109 @@ export const getJobs = () => {
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
}));
|
||||
};
|
||||
|
||||
/**
|
||||
* Query jobs with pagination, filtering and sorting.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {number} [params.pageSize=50]
|
||||
* @param {number} [params.page=1]
|
||||
* @param {string} [params.freeTextFilter]
|
||||
* @param {object} [params.activityFilter]
|
||||
* @param {string|null} [params.sortField=null]
|
||||
* @param {('asc'|'desc')} [params.sortDir='asc']
|
||||
* @param {string} [params.userId] - Current user id used to scope jobs (ignored for admins).
|
||||
* @param {boolean} [params.isAdmin=false] - When true, returns all jobs.
|
||||
* @returns {{ totalNumber:number, page:number, result:Object[] }}
|
||||
*/
|
||||
export const queryJobs = ({
|
||||
pageSize = 50,
|
||||
page = 1,
|
||||
activityFilter,
|
||||
freeTextFilter,
|
||||
sortField = null,
|
||||
sortDir = 'asc',
|
||||
userId = null,
|
||||
isAdmin = false,
|
||||
} = {}) => {
|
||||
// sanitize inputs
|
||||
const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.min(500, Math.floor(pageSize)) : 50;
|
||||
const safePage = Number.isFinite(page) && page > 0 ? Math.floor(page) : 1;
|
||||
const offset = (safePage - 1) * safePageSize;
|
||||
|
||||
// build WHERE filter
|
||||
const whereParts = [];
|
||||
const params = { limit: safePageSize, offset };
|
||||
params.userId = userId || '__NO_USER__';
|
||||
|
||||
if (!isAdmin) {
|
||||
whereParts.push(
|
||||
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
|
||||
);
|
||||
}
|
||||
|
||||
if (freeTextFilter && String(freeTextFilter).trim().length > 0) {
|
||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||
whereParts.push(`(j.name LIKE @filter)`);
|
||||
}
|
||||
|
||||
if (activityFilter === true) {
|
||||
whereParts.push('(j.enabled = 1)');
|
||||
} else if (activityFilter === false) {
|
||||
whereParts.push('(j.enabled = 0)');
|
||||
}
|
||||
|
||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
|
||||
// whitelist sortable fields
|
||||
const sortable = new Set(['name', 'numberOfFoundListings', 'enabled']);
|
||||
const safeSortField = sortField && sortable.has(sortField) ? sortField : null;
|
||||
const safeSortDir = String(sortDir).toLowerCase() === 'desc' ? 'DESC' : 'ASC';
|
||||
|
||||
let orderSql = 'ORDER BY j.name IS NULL, j.name ASC';
|
||||
if (safeSortField) {
|
||||
if (safeSortField === 'numberOfFoundListings') {
|
||||
orderSql = `ORDER BY numberOfFoundListings ${safeSortDir}`;
|
||||
} else {
|
||||
orderSql = `ORDER BY j.${safeSortField} ${safeSortDir}`;
|
||||
}
|
||||
}
|
||||
|
||||
// count total
|
||||
const countRow = SqliteConnection.query(
|
||||
`SELECT COUNT(1) as cnt
|
||||
FROM jobs j
|
||||
${whereSql}`,
|
||||
params,
|
||||
);
|
||||
const totalNumber = countRow?.[0]?.cnt ?? 0;
|
||||
|
||||
// fetch page
|
||||
const rows = SqliteConnection.query(
|
||||
`SELECT j.id,
|
||||
j.user_id AS userId,
|
||||
j.enabled,
|
||||
j.name,
|
||||
j.blacklist,
|
||||
j.provider,
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
${whereSql}
|
||||
${orderSql}
|
||||
LIMIT @limit OFFSET @offset`,
|
||||
params,
|
||||
);
|
||||
|
||||
const result = rows.map((row) => ({
|
||||
...row,
|
||||
enabled: !!row.enabled,
|
||||
blacklist: fromJson(row.blacklist, []),
|
||||
provider: fromJson(row.provider, []),
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
}));
|
||||
|
||||
return { totalNumber, page: safePage, result };
|
||||
};
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -173,9 +173,9 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
|
||||
link, created_at, is_active)
|
||||
link, created_at, is_active, latitude, longitude)
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
|
||||
@created_at, 1)
|
||||
@created_at, 1, @latitude, @longitude)
|
||||
ON CONFLICT(job_id, hash) DO NOTHING`,
|
||||
);
|
||||
|
||||
@@ -193,6 +193,8 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
address: removeParentheses(item.address),
|
||||
link: item.link,
|
||||
created_at: Date.now(),
|
||||
latitude: item.latitude || null,
|
||||
longitude: item.longitude || null,
|
||||
};
|
||||
stmt.run(params);
|
||||
}
|
||||
@@ -277,9 +279,11 @@ export const queryListings = ({
|
||||
params.filter = `%${String(freeTextFilter).trim()}%`;
|
||||
whereParts.push(`(title LIKE @filter OR address LIKE @filter OR provider LIKE @filter OR link LIKE @filter)`);
|
||||
}
|
||||
// activityFilter: when true -> only active listings (is_active = 1)
|
||||
// activityFilter: when true -> only active listings (is_active = 1), false -> only inactive
|
||||
if (activityFilter === true) {
|
||||
whereParts.push('(is_active = 1)');
|
||||
} else if (activityFilter === false) {
|
||||
whereParts.push('(is_active = 0)');
|
||||
}
|
||||
// Prefer filtering by job id when provided (unambiguous and robust)
|
||||
if (jobIdFilter && String(jobIdFilter).trim().length > 0) {
|
||||
@@ -295,9 +299,11 @@ export const queryListings = ({
|
||||
params.providerName = String(providerFilter).trim();
|
||||
whereParts.push('(provider = @providerName)');
|
||||
}
|
||||
// watchListFilter: when true -> only watched listings
|
||||
// watchListFilter: when true -> only watched listings, false -> only unwatched
|
||||
if (watchListFilter === true) {
|
||||
whereParts.push('(wl.id IS NOT NULL)');
|
||||
} else if (watchListFilter === false) {
|
||||
whereParts.push('(wl.id IS NULL)');
|
||||
}
|
||||
|
||||
const whereSql = whereParts.length ? `WHERE ${whereParts.join(' AND ')}` : '';
|
||||
@@ -388,6 +394,105 @@ export const deleteListingsById = (ids) => {
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all listings that are active, have an address, and do not yet have geocoordinates.
|
||||
*
|
||||
* @returns {Object[]} Array of listing objects {id, address}.
|
||||
*/
|
||||
export const getListingsToGeocode = () => {
|
||||
return SqliteConnection.query(
|
||||
`SELECT id, address
|
||||
FROM listings
|
||||
WHERE is_active = 1
|
||||
AND address IS NOT NULL
|
||||
AND (latitude IS NULL OR longitude IS NULL)`,
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Update the geocoordinates for a listing.
|
||||
*
|
||||
* @param {string} id - The listing ID.
|
||||
* @param {number} latitude
|
||||
* @param {number} longitude
|
||||
* @returns {void}
|
||||
*/
|
||||
export const updateListingGeocoordinates = (id, latitude, longitude) => {
|
||||
SqliteConnection.execute(
|
||||
`UPDATE listings
|
||||
SET latitude = @latitude,
|
||||
longitude = @longitude
|
||||
WHERE id = @id`,
|
||||
{ id, latitude, longitude },
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return listings with geocoordinates for the map view, with optional filtering.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {string} [params.jobId]
|
||||
* @param {boolean} [params.activeOnly=true]
|
||||
* @param {number} [params.minPrice]
|
||||
* @param {number} [params.maxPrice]
|
||||
* @param {string} [params.userId]
|
||||
* @param {boolean} [params.isAdmin=false]
|
||||
* @returns {{listings: Object[], maxPrice: number}} Object containing listings and maxPrice.
|
||||
*/
|
||||
export const getListingsForMap = ({ jobId, minPrice, maxPrice, userId = null, isAdmin = false } = {}) => {
|
||||
const baseWhereParts = [
|
||||
'l.latitude IS NOT NULL',
|
||||
'l.longitude IS NOT NULL',
|
||||
'l.latitude != -1',
|
||||
'l.longitude != -1',
|
||||
'l.is_active = 1',
|
||||
];
|
||||
const params = { userId: userId || '__NO_USER__' };
|
||||
|
||||
if (!isAdmin) {
|
||||
baseWhereParts.push(
|
||||
`(j.user_id = @userId OR EXISTS (SELECT 1 FROM json_each(j.shared_with_user) AS sw WHERE sw.value = @userId))`,
|
||||
);
|
||||
}
|
||||
|
||||
if (jobId) {
|
||||
params.jobId = jobId;
|
||||
baseWhereParts.push('l.job_id = @jobId');
|
||||
}
|
||||
|
||||
const wherePartsForListings = [...baseWhereParts];
|
||||
if (minPrice !== undefined && minPrice !== null) {
|
||||
params.minPrice = minPrice;
|
||||
wherePartsForListings.push('l.price >= @minPrice');
|
||||
}
|
||||
|
||||
if (maxPrice !== undefined && maxPrice !== null) {
|
||||
params.maxPrice = maxPrice;
|
||||
wherePartsForListings.push('l.price <= @maxPrice');
|
||||
}
|
||||
|
||||
const listings = SqliteConnection.query(
|
||||
`SELECT l.*, j.name AS job_name
|
||||
FROM listings l
|
||||
LEFT JOIN jobs j ON j.id = l.job_id
|
||||
WHERE ${wherePartsForListings.join(' AND ')}`,
|
||||
params,
|
||||
);
|
||||
|
||||
const maxPriceRow = SqliteConnection.query(
|
||||
`SELECT MAX(l.price) AS maxPrice
|
||||
FROM listings l
|
||||
LEFT JOIN jobs j ON j.id = l.job_id
|
||||
WHERE ${baseWhereParts.join(' AND ')}`,
|
||||
params,
|
||||
)[0];
|
||||
|
||||
return {
|
||||
listings,
|
||||
maxPrice: maxPriceRow?.maxPrice || 0,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Return all listings with only the fields: title, address, and price.
|
||||
* This is the single helper requested for simple consumers.
|
||||
@@ -397,3 +502,24 @@ export const deleteListingsById = (ids) => {
|
||||
export const getAllEntriesFromListings = () => {
|
||||
return SqliteConnection.query(`SELECT title, address, price FROM listings`);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return geocoordinates for a given address if it has been geocoded before.
|
||||
*
|
||||
* @param {string} address
|
||||
* @returns {{lat: number, lng: number}|null}
|
||||
*/
|
||||
export const getGeocoordinatesByAddress = (address) => {
|
||||
const row = SqliteConnection.query(
|
||||
`SELECT latitude, longitude
|
||||
FROM listings
|
||||
WHERE address = @address
|
||||
AND latitude IS NOT NULL
|
||||
AND longitude IS NOT NULL
|
||||
AND latitude != -1
|
||||
AND longitude != -1
|
||||
LIMIT 1`,
|
||||
{ address },
|
||||
)[0];
|
||||
return row ? { lat: row.latitude, lng: row.longitude } : null;
|
||||
};
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
// Migration: Add geocoordinates to listings for map display
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN latitude REAL;
|
||||
ALTER TABLE listings ADD COLUMN longitude REAL;
|
||||
`);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (c) 2025 by Christian Kellner.
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
@@ -95,7 +95,7 @@ function isOneOf(word, arr) {
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function nullOrEmpty(val) {
|
||||
return val == null || val.length === 0;
|
||||
return val == null || val.length === 0 || val === 'null' || val === 'undefined';
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
13934
package-lock.json
generated
Normal file
13934
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
27
package.json
27
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fredy",
|
||||
"version": "16.2.0",
|
||||
"version": "18.0.0",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"prepare": "husky",
|
||||
@@ -12,7 +12,7 @@
|
||||
"format": "prettier --write \"**/*.js\"",
|
||||
"format:check": "prettier --check \"**/*.js\"",
|
||||
"test": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 test/**/*.test.js",
|
||||
"testGH": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 --exclude test/provider/immonet.test.js --exclude test/provider/immowelt.test.js test/**/*.test.js",
|
||||
"testGH": "node --import ./test/esmock-loader.mjs ./node_modules/mocha/bin/mocha.js --timeout 60000 --exclude test/provider/immonet.test.js --exclude test/provider/immobilienDe.test.js --exclude test/provider/immowelt.test.js test/**/*.test.js",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "yarn lint --fix",
|
||||
"migratedb": "node lib/services/storage/migrations/migrate.js",
|
||||
@@ -59,47 +59,48 @@
|
||||
"Firefox ESR"
|
||||
],
|
||||
"dependencies": {
|
||||
"adm-zip": "^0.5.16",
|
||||
"@douyinfe/semi-icons": "^2.89.0",
|
||||
"@douyinfe/semi-ui": "2.89.0",
|
||||
"@douyinfe/semi-icons": "^2.90.11",
|
||||
"@douyinfe/semi-ui": "2.90.11",
|
||||
"@sendgrid/mail": "8.1.6",
|
||||
"@vitejs/plugin-react": "5.1.2",
|
||||
"better-sqlite3": "^12.5.0",
|
||||
"body-parser": "2.2.1",
|
||||
"adm-zip": "^0.5.16",
|
||||
"better-sqlite3": "^12.6.0",
|
||||
"body-parser": "2.2.2",
|
||||
"chart.js": "^4.5.1",
|
||||
"cheerio": "^1.1.2",
|
||||
"cookie-session": "2.1.1",
|
||||
"handlebars": "4.7.8",
|
||||
"lodash": "4.17.21",
|
||||
"maplibre-gl": "^5.16.0",
|
||||
"nanoid": "5.1.6",
|
||||
"node-cron": "^4.2.1",
|
||||
"node-fetch": "3.3.2",
|
||||
"node-mailjet": "6.0.11",
|
||||
"p-throttle": "^8.1.0",
|
||||
"package-up": "^5.0.0",
|
||||
"puppeteer": "^24.33.0",
|
||||
"puppeteer": "^24.35.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"query-string": "9.3.1",
|
||||
"react": "18.3.1",
|
||||
"react-chartjs-2": "^5.3.1",
|
||||
"react-dom": "18.3.1",
|
||||
"react-router": "7.10.1",
|
||||
"react-router-dom": "7.10.1",
|
||||
"react-router": "7.12.0",
|
||||
"react-router-dom": "7.12.0",
|
||||
"restana": "5.1.0",
|
||||
"semver": "^7.7.3",
|
||||
"serve-static": "2.2.1",
|
||||
"slack": "11.0.2",
|
||||
"vite": "7.3.0",
|
||||
"vite": "7.3.1",
|
||||
"x-var": "^3.0.1",
|
||||
"zustand": "^5.0.9"
|
||||
"zustand": "^5.0.10"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "7.28.5",
|
||||
"@babel/eslint-parser": "7.28.5",
|
||||
"@babel/preset-env": "7.28.5",
|
||||
"@babel/preset-react": "7.28.5",
|
||||
"chai": "6.2.1",
|
||||
"chai": "6.2.2",
|
||||
"eslint": "9.39.2",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-plugin-react": "7.37.5",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user