External/fredy
1
0
Fork 0
mirror of https://github.com/orangecoding/fredy.git synced 2026-06-16 12:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding ability to add proxies for cloak

Browse Source
This commit is contained in:
orangecoding
2026-05-24 20:49:27 +02:00
parent b2e294e38c
commit 996b841cfb
7 changed files with 160 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lib/services/jobs/jobExecutionService.js
View File

@@ -14,6 +14,7 @@ import * as similarityCache from '../similarity-check/similarityCache.js';
import { isRunning, markFinished, markRunning } from './run-state.js';
import { sendToUsers } from '../sse/sse-broker.js';
import * as puppeteerExtractor from '../extractor/puppeteerExtractor.js';
import { getSettings } from '../storage/settingsStorage.js';
/**
* Initializes the job execution service.
@@ -160,6 +161,14 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
}
let browser;
try {
// Read the proxy live (not from the startup snapshot) so changing it in the
// UI takes effect on the next run without a backend restart. An empty value
// disables the proxy. Routing the headless browser through a (German
// residential) proxy avoids datacenter-IP based bot detection on the
// Puppeteer-based providers (immowelt, immonet, kleinanzeigen, ...).
const liveSettings = await getSettings();
const proxyUrl = typeof liveSettings?.proxyUrl === 'string' ? liveSettings.proxyUrl.trim() : '';
const jobProviders = job.provider.filter(
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
);
@@ -175,7 +184,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
}
if (!browser && matchedProvider.config.getListings == null) {
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, proxyUrl ? { proxyUrl } : {});
}
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();