adding ability to add proxies for cloak

This commit is contained in:
orangecoding
2026-05-24 20:49:27 +02:00
parent b2e294e38c
commit 996b841cfb
7 changed files with 160 additions and 61 deletions

View File

@@ -167,6 +167,40 @@ For more information on how to set it up and use it, please refer to the [MCP Re
Immoscout has implemented advanced bot detection. In order to work around this, we are using a reversed engineered version of their mobile api. See [Immoscout Reverse Engineering Documentation](https://github.com/orangecoding/fredy/blob/master/reverse-engineered-immoscout.md)
## 🛡️ Bot Detection & Proxies
Most browser-based providers (immowelt, immonet, kleinanzeigen, ...) are scraped through a hardened headless browser ([CloakBrowser](https://www.npmjs.com/package/cloakbrowser)). It makes the **browser fingerprint** indistinguishable from a real Chrome, which is enough when you run Fredy on a normal home connection.
On a **server / VPS the requests usually originate from a datacenter IP**, and providers behind anti-bot systems (e.g. AWS CloudFront/WAF) block those based on **IP reputation alone**, no matter how perfect the fingerprint is. The typical symptom: it works locally but you get `We have been detected as a bot :-/` on the server.
### The fix: a residential proxy
A **residential proxy** routes Fredy's browser through the internet connection of a real household, so the provider sees a "normal user" IP instead of a datacenter. For German portals, use a **German (DE) residential** (or mobile/4G) proxy. Plain VPNs and **datacenter proxies do not help** here, they share the same bad reputation as your server.
**Configure it** under **Settings → Execution → Proxy URL**. Supported formats:
```
http://user:pass@host:port
socks5://user:pass@host:port
```
Leave the field empty to disable. The proxy applies to all headless-browser providers and takes effect on the next job run (no restart needed). Immoscout uses a separate mobile API and is not affected.
### Where to get a residential proxy
Residential proxies are a paid service (usually billed per GB, Fredy's traffic is small). Well-known providers offering German residential IPs include:
| Provider | Notes |
|---|---|
| [IPRoyal](https://iproyal.com) | Pay-as-you-go, no monthly minimum, good for low volume |
| [Webshare](https://www.webshare.io) | Cheap entry tier, has a small free plan to test with |
| [Decodo (formerly Smartproxy)](https://decodo.com) | Easy setup, country/city targeting |
| [SOAX](https://soax.com) | Residential + mobile, fine-grained geo-targeting |
| [Bright Data](https://brightdata.com) | Largest pool, most features, higher complexity/price |
| [Oxylabs](https://oxylabs.io) | Enterprise-grade, larger plans |
This is not an endorsement, pick whatever fits your budget. For low-volume use like Fredy, a pay-as-you-go plan (e.g. IPRoyal) or a cheap entry tier (e.g. Webshare) is usually plenty. Make sure to select **Germany** as the proxy location and keep the search interval reasonable (the higher the interval, the less you look like a bot).
## Analytics
Fredy is completely free (and will always remain free). However, it would be a huge help if youd allow me to collect some analytical data.

View File

@@ -14,6 +14,7 @@ import * as similarityCache from '../similarity-check/similarityCache.js';
import { isRunning, markFinished, markRunning } from './run-state.js';
import { sendToUsers } from '../sse/sse-broker.js';
import * as puppeteerExtractor from '../extractor/puppeteerExtractor.js';
import { getSettings } from '../storage/settingsStorage.js';
/**
* Initializes the job execution service.
@@ -160,6 +161,14 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
}
let browser;
try {
// Read the proxy live (not from the startup snapshot) so changing it in the
// UI takes effect on the next run without a backend restart. An empty value
// disables the proxy. Routing the headless browser through a (German
// residential) proxy avoids datacenter-IP based bot detection on the
// Puppeteer-based providers (immowelt, immonet, kleinanzeigen, ...).
const liveSettings = await getSettings();
const proxyUrl = typeof liveSettings?.proxyUrl === 'string' ? liveSettings.proxyUrl.trim() : '';
const jobProviders = job.provider.filter(
(p) => providers.find((loaded) => loaded.metaInformation.id === p.id) != null,
);
@@ -175,7 +184,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
}
if (!browser && matchedProvider.config.getListings == null) {
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, proxyUrl ? { proxyUrl } : {});
}
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();

View File

@@ -1,6 +1,6 @@
{
"name": "fredy",
"version": "22.0.10",
"version": "22.1.0",
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
"scripts": {
"prepare": "husky",
@@ -62,9 +62,9 @@
"Firefox ESR"
],
"dependencies": {
"@douyinfe/semi-icons": "^2.99.0",
"@douyinfe/semi-ui": "2.99.0",
"@douyinfe/semi-ui-19": "^2.99.0",
"@douyinfe/semi-icons": "^2.99.2",
"@douyinfe/semi-ui": "2.99.2",
"@douyinfe/semi-ui-19": "^2.99.2",
"@fastify/cookie": "^11.0.2",
"@fastify/helmet": "^13.0.2",
"@fastify/session": "^11.1.1",

View File

@@ -0,0 +1,37 @@
/*
* Copyright (c) 2026 by Christian Kellner.
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
*/
import { vi, describe, it, expect, beforeEach } from 'vitest';
// Mock the CloakBrowser launcher so no real Chromium binary is needed and we can
// assert which options get forwarded to it.
const { launchMock } = vi.hoisted(() => ({ launchMock: vi.fn() }));
vi.mock('cloakbrowser/puppeteer', () => ({
launch: launchMock,
}));
const { launchBrowser } = await import('../../../lib/services/extractor/puppeteerExtractor.js');
describe('launchBrowser proxy forwarding', () => {
beforeEach(() => {
launchMock.mockReset();
launchMock.mockResolvedValue({ close: async () => {} });
});
it('forwards proxyUrl to CloakBrowser as the proxy option', async () => {
await launchBrowser('https://www.immowelt.de/', { proxyUrl: 'http://user:pass@host:8080' });
expect(launchMock).toHaveBeenCalledTimes(1);
expect(launchMock.mock.calls[0][0]).toMatchObject({ proxy: 'http://user:pass@host:8080' });
});
it('does not set a proxy when no proxyUrl is given', async () => {
await launchBrowser('https://www.immowelt.de/', {});
expect(launchMock).toHaveBeenCalledTimes(1);
expect(launchMock.mock.calls[0][0].proxy).toBeUndefined();
});
});

View File

@@ -18,6 +18,7 @@ describe('services/jobs/jobExecutionService', () => {
const busPath = root + '/lib/services/events/event-bus.js';
const jobStoragePath = root + '/lib/services/storage/jobStorage.js';
const userStoragePath = root + '/lib/services/storage/userStorage.js';
const settingsStoragePath = root + '/lib/services/storage/settingsStorage.js';
const brokerPath = root + '/lib/services/sse/sse-broker.js';
const utilsPath = root + '/lib/utils.js';
const loggerPath = root + '/lib/services/logger.js';
@@ -33,6 +34,9 @@ describe('services/jobs/jobExecutionService', () => {
getUsers: () => state.users.slice(),
getUser: (id) => state.users.find((u) => u.id === id) || null,
}));
vi.doMock(settingsStoragePath, () => ({
getSettings: async () => ({}),
}));
vi.doMock(brokerPath, () => ({
sendToUsers: (...args) => calls.sent.push(args),
}));

View File

@@ -57,6 +57,7 @@ const GeneralSettings = function GeneralSettings() {
const currentUser = useSelector((state) => state.user.currentUser);
const [interval, setInterval] = React.useState('');
const [proxyUrl, setProxyUrl] = React.useState('');
const [port, setPort] = React.useState('');
const [workingHourFrom, setWorkingHourFrom] = React.useState(null);
const [workingHourTo, setWorkingHourTo] = React.useState(null);
@@ -91,6 +92,7 @@ const GeneralSettings = function GeneralSettings() {
React.useEffect(() => {
async function init() {
setInterval(settings?.interval);
setProxyUrl(settings?.proxyUrl ?? '');
setPort(settings?.port);
setWorkingHourFrom(settings?.workingHours?.from);
setWorkingHourTo(settings?.workingHours?.to);
@@ -133,6 +135,7 @@ const GeneralSettings = function GeneralSettings() {
try {
await xhrPost('/api/admin/generalSettings', {
interval,
proxyUrl: proxyUrl?.trim() ?? '',
port,
workingHours: {
from: workingHourFrom,
@@ -376,6 +379,18 @@ const GeneralSettings = function GeneralSettings() {
</div>
</SegmentPart>
<SegmentPart
name="Proxy URL"
helpText="Optional. Routes the scraping browser through a proxy. Server/datacenter IPs are frequently blocked by providers (e.g. immowelt) regardless of browser fingerprint, a German residential proxy makes requests look like a normal household and is the most effective fix. Format: http://user:pass@host:port or socks5://user:pass@host:port. Leave empty to disable."
>
<Input
type="text"
placeholder="http://user:pass@host:port"
value={proxyUrl}
onChange={(value) => setProxyUrl(value)}
/>
</SegmentPart>
<div className="generalSettings__save-row">
<Button type="primary" theme="solid" onClick={handleStore} icon={<IconSave />}>
Save

112
yarn.lock
View File

@@ -895,34 +895,34 @@
dependencies:
tslib "^2.0.0"
"@douyinfe/semi-animation-react@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-react/-/semi-animation-react-2.99.0.tgz#d44e36eb6bdeb436833c10ddbdff5c53d9c1cd4b"
integrity sha512-AikblPO1X5cJzOONZY5EWuMizSsOsxv6zz0+1FsGRExApY1oDaNyEkFxJZVMHoSoAykEtXATIE+gBlzrVIJ4jw==
"@douyinfe/semi-animation-react@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-react/-/semi-animation-react-2.99.2.tgz#ab33aa9ba9fd1dc6901a657a123e2e0218819f1a"
integrity sha512-7aiU529XoNL6jpV5Tdj0K0o+Pp9dRfypTPcHZY53YMLOQ7b8n4rh+G8C7iRgmgExphu8iifqylwVIOMU1ZirUw==
dependencies:
"@douyinfe/semi-animation" "2.99.0"
"@douyinfe/semi-animation-styled" "2.99.0"
"@douyinfe/semi-animation" "2.99.2"
"@douyinfe/semi-animation-styled" "2.99.2"
classnames "^2.2.6"
"@douyinfe/semi-animation-styled@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-styled/-/semi-animation-styled-2.99.0.tgz#93a97f2b3d3576eceec40e3cb55a78ba12e740f9"
integrity sha512-lF31If2jsflaWfteZJVeTRD2cGTzntl+ElWYpeczEFlp0hehVgqm9KHWVVQKjzJCDkNP7GoYMcCzyWZh/La94A==
"@douyinfe/semi-animation-styled@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation-styled/-/semi-animation-styled-2.99.2.tgz#63521ba2c6a1d747cf80a5719c83ee828d7dd7cb"
integrity sha512-4PvNW02ytNxoyEPpi9emYITrchf9Md/wSDfuvQWJfdFjK0JKvuOITRK10HCVp02GrCtbFK+Hf6pPe9lSRVOlVQ==
"@douyinfe/semi-animation@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation/-/semi-animation-2.99.0.tgz#6f8e98f30f37f1310bba37c58b94f055d7529d3d"
integrity sha512-tR5dUkFEjYa8N2xJ+xv0LaCrjC+L+QjBhmpHFOx8Z/XFt5iKWlgUrLseYIxj1CN+h0JWakqSnOWxY56HwRCyug==
"@douyinfe/semi-animation@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-animation/-/semi-animation-2.99.2.tgz#508f97e9c280dbddfda9154374bf111209970fa0"
integrity sha512-MxAcD2PwHbpUSq7CFltBXST//qcyHzR5yaGbCTVUUQwLPnEctkunC52i/CLPTGqX6JMoivXpeKPHPSKNL2wcyw==
dependencies:
bezier-easing "^2.1.0"
"@douyinfe/semi-foundation@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-foundation/-/semi-foundation-2.99.0.tgz#f281d78f329aa9551f67eb511199269b7fbe8ef1"
integrity sha512-J26qA5UburT3r4g0+6WkYJaAnRAY7y1Ij7csG/xUati7NahLCUgiBpduDOGET94AMNUJFqe4dTinev4kjMJ1wA==
"@douyinfe/semi-foundation@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-foundation/-/semi-foundation-2.99.2.tgz#15e7a2ec74d1c23d7be83157b6f5f5d4de15e902"
integrity sha512-N5ZdrDdMEXSHe5vI+3CIuVwjd4m0OHlLRGfkJjgUctL1W0zOhlg9DRTfJn0tgefIn94ILJOKTOS3zC07Z9ZQ1g==
dependencies:
"@douyinfe/semi-animation" "2.99.0"
"@douyinfe/semi-json-viewer-core" "2.99.0"
"@douyinfe/semi-animation" "2.99.2"
"@douyinfe/semi-json-viewer-core" "2.99.2"
"@mdx-js/mdx" "^3.0.1"
async-validator "^3.5.0"
classnames "^2.2.6"
@@ -936,44 +936,44 @@
remark-gfm "^4.0.0"
scroll-into-view-if-needed "^2.2.24"
"@douyinfe/semi-icons@2.99.0", "@douyinfe/semi-icons@^2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.99.0.tgz#152f6d1f43a8b3b2c0af3cbbe5988638cf5b05dd"
integrity sha512-Fea+6fkdb5ycf6Ktwul9TlJFIsO9RhQOOQbSGNDU++SokI+IfAVBDof0tKQ7MzTAAqTGgxDi4J3BunSi+VFoAw==
"@douyinfe/semi-icons@2.99.2", "@douyinfe/semi-icons@^2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-icons/-/semi-icons-2.99.2.tgz#40d565b58a2511b2085fa35d336407b5766653fd"
integrity sha512-S4TwmY7oPpPj3lg6OI9l2HNbK6KExP8tUO+wrOcPLXSe9XkQr5CCA7T+B3qNb1CPZ6PGsj7v2v51xlvOsD1Hpg==
dependencies:
classnames "^2.2.6"
"@douyinfe/semi-illustrations@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-illustrations/-/semi-illustrations-2.99.0.tgz#7aae087c318beef50f83fc8d22a0605e6e9b7157"
integrity sha512-11ZWVIqsv8qhc7NwwhSTXRlkD1FW+Uw97iT32tM3oIkwPZKdnJzT7ckAIZc1dTYmBrXJJoaOfemGWSB3lERwqA==
"@douyinfe/semi-illustrations@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-illustrations/-/semi-illustrations-2.99.2.tgz#0dbf80e73b9d9a2737d7c7152db6f23a210fb706"
integrity sha512-Ozla5vM9+tS5YQVEwO1IogBcxD+GE+NajZB2DIC3dEfxpPb2pyYg12sECbx2hq1jtLQ61CjZGOwWcohCm86Cig==
"@douyinfe/semi-json-viewer-core@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-json-viewer-core/-/semi-json-viewer-core-2.99.0.tgz#3c494bc2d2adc12c9e109902570aee48eaa54a59"
integrity sha512-eZ+0jb39I6ZsMgI+fHV0Db3UQa+AXqxuE9hbMr8821hLgAAklRXjJETKykt8LasFZCJsGpMsGSoWYUneUCIvBw==
"@douyinfe/semi-json-viewer-core@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-json-viewer-core/-/semi-json-viewer-core-2.99.2.tgz#2e71a05688f0f62d27ce0bece9f03804bea609d0"
integrity sha512-UYv5nfa2kajy7J7du5yDDcuuMkRivS6lYABxbrcHhwwDIg29QIZOFfKrs6JjHwEjZji6pl6BnFIj+tDXeA5R6A==
dependencies:
jsonc-parser "^3.3.1"
"@douyinfe/semi-theme-default@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-theme-default/-/semi-theme-default-2.99.0.tgz#e36a56d723631dca54ba8c853a26a6a4f4495f6a"
integrity sha512-+zpd6fsJBDuR/1P60JkCQINMijl14r8IHZ4YQtv27ip46H7rXQ3x18YeTiSq9paQU97MYWP7ComjFDMarZJFBA==
"@douyinfe/semi-theme-default@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-theme-default/-/semi-theme-default-2.99.2.tgz#7686a0208956d26c9e9d3274876c4381308e3060"
integrity sha512-1vh9IaYgZbIzoP7gknaltmk69vx6udfBYQCKIbHWNtg5gVJbwiUZCocmd2LNihdiZLuevBrWzmYOKbIaPHWh5w==
"@douyinfe/semi-ui-19@^2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-ui-19/-/semi-ui-19-2.99.0.tgz#14345f5736089abc5bc20ca7a2dff042cf73321a"
integrity sha512-uB7v2qd6N/+TIPFhUgptvyzrR8u0uD+66RgBfiFG4LRkdCmyzJNW1uEjeUgy+sjPBhQxIi11uGlVpa4Gw5lfJw==
"@douyinfe/semi-ui-19@^2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-ui-19/-/semi-ui-19-2.99.2.tgz#41860745fb2d2c598b5d48925b6fde7bfc47b858"
integrity sha512-dZidWoEODRFmHwKaSutS4B0ZxyUjlDyP4COcjERwCl8WJEAcJufP6kpgIxJ/WjjuaWL0rZLDJRbDo9Uof73WKw==
dependencies:
"@dnd-kit/core" "^6.0.8"
"@dnd-kit/sortable" "^7.0.2"
"@dnd-kit/utilities" "^3.2.1"
"@douyinfe/semi-animation" "2.99.0"
"@douyinfe/semi-animation-react" "2.99.0"
"@douyinfe/semi-foundation" "2.99.0"
"@douyinfe/semi-icons" "2.99.0"
"@douyinfe/semi-illustrations" "2.99.0"
"@douyinfe/semi-theme-default" "2.99.0"
"@douyinfe/semi-animation" "2.99.2"
"@douyinfe/semi-animation-react" "2.99.2"
"@douyinfe/semi-foundation" "2.99.2"
"@douyinfe/semi-icons" "2.99.2"
"@douyinfe/semi-illustrations" "2.99.2"
"@douyinfe/semi-theme-default" "2.99.2"
"@tiptap/core" "^3.10.7"
"@tiptap/extension-document" "^3.10.7"
"@tiptap/extension-hard-break" "^3.10.7"
@@ -1002,20 +1002,20 @@
scroll-into-view-if-needed "^2.2.24"
utility-types "^3.10.0"
"@douyinfe/semi-ui@2.99.0":
version "2.99.0"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-ui/-/semi-ui-2.99.0.tgz#87b2d7f23d2b5bba66298ed2ce1bb85fb46f6bfc"
integrity sha512-O/g9Y2bMfTHMsbMB4BOQMb8a3r4r8mCKmXOh9E0NsS9e7ogRoZ2Eq+qxvRjptPMBG8IGsXhLjEMiOIuMW5Gnjw==
"@douyinfe/semi-ui@2.99.2":
version "2.99.2"
resolved "https://registry.yarnpkg.com/@douyinfe/semi-ui/-/semi-ui-2.99.2.tgz#a461f0f44e8a75fda64933f1a49ebbae47662429"
integrity sha512-0UKksaHD7HRXyjhEdb/Ak8F70Cz7E2oS9yEu8WInhDN6CFytW5Jx7tD8SB2wb2MkosyxHgl94mzkTg8X2e8zlw==
dependencies:
"@dnd-kit/core" "^6.0.8"
"@dnd-kit/sortable" "^7.0.2"
"@dnd-kit/utilities" "^3.2.1"
"@douyinfe/semi-animation" "2.99.0"
"@douyinfe/semi-animation-react" "2.99.0"
"@douyinfe/semi-foundation" "2.99.0"
"@douyinfe/semi-icons" "2.99.0"
"@douyinfe/semi-illustrations" "2.99.0"
"@douyinfe/semi-theme-default" "2.99.0"
"@douyinfe/semi-animation" "2.99.2"
"@douyinfe/semi-animation-react" "2.99.2"
"@douyinfe/semi-foundation" "2.99.2"
"@douyinfe/semi-icons" "2.99.2"
"@douyinfe/semi-illustrations" "2.99.2"
"@douyinfe/semi-theme-default" "2.99.2"
"@tiptap/core" "^3.10.7"
"@tiptap/extension-document" "^3.10.7"
"@tiptap/extension-hard-break" "^3.10.7"