External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Puppeteer improvements (#270)

Browse Source 
* improve puppeteer handling. Now only 1 puppeteer instance is being used which is WAY more efficient

* removing package-lock

* reduce logging

* removing problematic docker command

* Remove Immonet. They now belong to immowelt
This commit is contained in:
Christian Kellner
2026-02-18 20:05:02 +01:00
committed by GitHub
parent 05218800d2
commit 00d6a12b30
10 changed files with 106 additions and 13729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
lib/services/jobs/jobExecutionService.js
View File

@@ -13,6 +13,7 @@ import FredyPipelineExecutioner from '../../FredyPipelineExecutioner.js';
import * as similarityCache from '../similarity-check/similarityCache.js';
import { isRunning, markFinished, markRunning } from './run-state.js';
import { sendToUsers } from '../sse/sse-broker.js';
import * as puppeteerExtractor from '../extractor/puppeteerExtractor.js';
/**
* Initializes the job execution service.
@@ -94,7 +95,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
* @param {{userId?: string, isAdmin?: boolean}} [context] - Who requested the run; determines job filtering.
* @returns {void}
*/
function runAll(respectWorkingHours = true, context = undefined) {
async function runAll(respectWorkingHours = true, context = undefined) {
if (settings.demoMode) return;
const now = Date.now();
const withinHours = duringWorkingHoursOrNotSet(settings, now);
@@ -103,15 +104,18 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
return;
}
settings.lastRun = now;
jobStorage
const jobs = jobStorage
.getJobs()
.filter((job) => job.enabled)
.filter((job) => {
if (!context) return true; // startup/cron → all
if (context.isAdmin) return true; // admin → all
return context.userId ? job.userId === context.userId : false; // user → own
})
.forEach((job) => executeJob(job));
});
for (const job of jobs) {
await executeJob(job);
}
}
/**
@@ -154,28 +158,36 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
} catch (err) {
logger.warn('Failed to emit start status for job', job.id, err);
}
let browser;
try {
const jobProviders = job.provider.filter(
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
);
const executions = jobProviders.map(async (prov) => {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
await new FredyPipelineExecutioner(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
).execute();
});
const results = await Promise.allSettled(executions);
for (const r of results) {
if (r.status === 'rejected') {
logger.error(r.reason);
for (const prov of jobProviders) {
try {
const matchedProvider = providers.find((loaded) => loaded.metaInformation.id === prov.id);
matchedProvider.init(prov, job.blacklist);
if (!browser && matchedProvider.config.getListings == null) {
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
}
await new FredyPipelineExecutioner(
matchedProvider.config,
job.notificationAdapter,
prov.id,
job.id,
similarityCache,
browser,
).execute();
} catch (err) {
logger.error(err);
}
}
} finally {
if (browser) {
await puppeteerExtractor.closeBrowser(browser);
}
markFinished(job.id);
try {
bus.emit('jobs:status', { jobId: job.id, running: false });