mirror of
https://github.com/orangecoding/fredy.git
synced 2026-06-16 12:31:07 +00:00
Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0cbfa25062 | ||
|
|
bcd3042026 | ||
|
|
0ce93acaf6 | ||
|
|
cabef973a2 | ||
|
|
3d0fa87d19 | ||
|
|
8b012ef2f1 | ||
|
|
6816b0aded | ||
|
|
ac02817d4e | ||
|
|
fe0a09fe1c | ||
|
|
2f00966f27 | ||
|
|
921057252d | ||
|
|
703c602527 | ||
|
|
0e29c9b9c6 | ||
|
|
f60c5859f9 | ||
|
|
ee54cc495b | ||
|
|
96582ecff4 | ||
|
|
3de82dfa41 | ||
|
|
d7ee4f6909 | ||
|
|
bf4bae9bf5 | ||
|
|
3d10dc6042 | ||
|
|
fef6d06a9d | ||
|
|
951b69a67f | ||
|
|
8a7b14c079 | ||
|
|
f30ec4645c | ||
|
|
c78472bd19 | ||
|
|
8c5607e20b | ||
|
|
64d0515c79 | ||
|
|
cc0164b689 | ||
|
|
522bbc2282 | ||
|
|
c384781137 | ||
|
|
e2d10d179e | ||
|
|
10c94eea0a | ||
|
|
05f74f99ef |
1
.gitattributes
vendored
Normal file
1
.gitattributes
vendored
Normal file
@@ -0,0 +1 @@
|
||||
test/testFixtures/** linguist-vendored
|
||||
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
@@ -19,4 +19,4 @@ jobs:
|
||||
cache: 'yarn'
|
||||
|
||||
- run: yarn install
|
||||
- run: yarn testGH
|
||||
- run: yarn test:offline
|
||||
|
||||
120
CLAUDE.md
Normal file
120
CLAUDE.md
Normal file
@@ -0,0 +1,120 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Fredy is a self-hosted real estate finder for Germany. It scrapes German real estate portals (ImmoScout24, Immowelt, Immonet, Kleinanzeigen, WG-Gesucht, etc.), deduplicates results across providers, and sends notifications via Slack, Telegram, Email, Discord, ntfy, etc. It includes a React web UI and a built-in MCP server for LLM access to listings data.
|
||||
|
||||
- Node.js >= 22, ESM-only (`"type": "module"`)
|
||||
- Default port: 9998, default login: admin / admin
|
||||
- SQLite via `better-sqlite3` (synchronous - all DB ops are sync; only network I/O is async)
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Development
|
||||
yarn run start:backend:dev # nodemon backend
|
||||
yarn run start:frontend:dev # Vite dev server (proxies /api → :9998)
|
||||
|
||||
# Production
|
||||
yarn run start:backend # NODE_ENV=production node index.js
|
||||
yarn run build:frontend # vite build → ui/public/
|
||||
|
||||
# Tests
|
||||
yarn test # Live tests (hits actual providers)
|
||||
yarn test:offline # Offline tests using HTML/JSON fixtures (fast, preferred)
|
||||
yarn test:download-fixtures # Re-download fresh provider HTML fixtures
|
||||
|
||||
# Single test file
|
||||
TEST_MODE=offline npx vitest run test/provider/immoscout.test.js
|
||||
|
||||
# Lint / Format
|
||||
yarn lint && yarn lint:fix
|
||||
yarn format && yarn format:check
|
||||
|
||||
# DB migrations
|
||||
yarn migratedb
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Core data flow
|
||||
|
||||
```
|
||||
index.js (startup)
|
||||
├── runMigrations()
|
||||
├── getProviders() # lazily imports lib/provider/*.js
|
||||
├── similarityCache.init() # preloads hash cache from DB
|
||||
├── api.js # starts restana HTTP server
|
||||
└── initJobExecutionService() # registers event-bus listeners + starts scheduler
|
||||
|
||||
scheduler (every N minutes) or manual trigger via POST /api/jobs/:id/run
|
||||
└── FredyPipelineExecutioner.execute()
|
||||
1. queryStringMutator(url) # inject sort-by-date param
|
||||
2. provider.getListings() # API or Puppeteer+Cheerio
|
||||
3. provider.normalize(listing) # raw → ParsedListing
|
||||
4. provider.filter(listing) # blacklist + required fields
|
||||
5. filter to hashes not yet in DB
|
||||
6. provider.fetchDetails() # optional enrichment
|
||||
7. geocodeAddress() # optional lat/lng
|
||||
8. storeListings()
|
||||
9. similarityCache.checkAndAddEntry() # cross-provider dedup
|
||||
10. _filterBySpecs() + _filterByArea()
|
||||
11. notify.send() # fan-out to all adapters
|
||||
```
|
||||
|
||||
### Plugin systems
|
||||
|
||||
**Providers** (`lib/provider/*.js`) - each module exports:
|
||||
- `metaInformation` - `{ id, name, baseUrl }`
|
||||
- `config` - `ProviderConfig` with `requiredFieldNames`, `crawlContainer`, `crawlFields`, `sortByDateParam`, `normalize()`, `filter()`, optional `getListings()`, `fetchDetails()`, `activeTester()`
|
||||
- `init(sourceConfig, blacklist)` - called before each job run; providers are **stateful modules** holding mutable `url` and `appliedBlackList` at module scope
|
||||
|
||||
**Notification adapters** (`lib/notification/adapter/*.js`) - each exports:
|
||||
- `config` - `{ id, name, description, fields }` (drives the UI form)
|
||||
- `send({ serviceName, newListings, notificationConfig, jobKey, baseUrl })`
|
||||
- Loaded dynamically at startup via `fs.readdirSync`
|
||||
|
||||
### Key services
|
||||
|
||||
| Service | Location | Notes |
|
||||
|---|---|---|
|
||||
| Event bus | `lib/services/events/event-bus.js` | Plain `EventEmitter`; events: `jobs:runAll`, `jobs:runOne`, `jobs:status` |
|
||||
| SSE broker | `lib/services/sse/sse-broker.js` | Per-userId `Set<ServerResponse>`; heartbeat every 25s; pushes job status to UI |
|
||||
| Similarity cache | `lib/services/similarity-check/` | In-memory SHA-256 Set; refreshes hourly; cross-provider dedup by title+price+address |
|
||||
| SqliteConnection | `lib/services/storage/SqliteConnection.js` | Singleton, WAL mode; `execute()`, `query()`, `withTransaction()` |
|
||||
| Migrations | `lib/services/storage/migrations/` | Numbered JS files each exporting `up(db)`; checksum-tracked in `schema_migrations` |
|
||||
| Extractor | `lib/services/extractor/` | Orchestrates Puppeteer + Cheerio; shared browser instance per job |
|
||||
|
||||
### Frontend
|
||||
|
||||
- React 19 SPA, Vite build → `ui/public/` (served as static by backend)
|
||||
- State: Zustand single store with per-domain slices
|
||||
- UI library: `@douyinfe/semi-ui`
|
||||
- Map: MapLibre GL + `@mapbox/mapbox-gl-draw` + `@turf/boolean-point-in-polygon` for GeoJSON polygon filters
|
||||
- In dev: Vite proxies `/api` to `:9998`
|
||||
|
||||
### MCP server
|
||||
|
||||
Two transports:
|
||||
1. **stdio** (`lib/mcp/stdio.js`) - for Claude Desktop/LM Studio; opens its own DB connection (main process need not be running)
|
||||
2. **HTTP** (`/api/mcp`) - authenticated via Bearer token (`mcp_token` column in `users` table)
|
||||
|
||||
Tools: `list_jobs`, `get_job`, `list_listings`, `get_listing`, `get_current_date_time`. Responses are Markdown via `lib/mcp/mcpNormalizer.js`.
|
||||
|
||||
## Key Conventions
|
||||
|
||||
- **ESM only** - `import`/`export` everywhere, no CommonJS
|
||||
- **JSDoc typedefs** (no TypeScript) in `lib/types/` - `listing.js`, `job.js`, `filter.js`, `providerConfig.js`
|
||||
- **Copyright header** required on all `.js` files - enforced by `lint-staged` pre-commit hook via `copyright.js`
|
||||
- **`NoNewListingsWarning`** (`lib/errors.js`) is used as control flow to short-circuit the pipeline (not an error)
|
||||
- **Test fixtures** in `test/testFixtures/` - HTML/JSON snapshots per provider; `TEST_MODE=offline` mocks `puppeteerExtractor` and global `fetch` via `test/offlineFixtures.js`
|
||||
- **`conf/config.json`** is the only runtime config file; created with defaults if missing
|
||||
|
||||
## Coding
|
||||
- After building the task, run the linter
|
||||
- After building the task, run the tests
|
||||
- New features must be tested
|
||||
- New features must be properly documented with JsDoc
|
||||
- You do **not** commit any changes, you do **not** create a new branch unless I told you so
|
||||
13
Dockerfile
13
Dockerfile
@@ -1,16 +1,15 @@
|
||||
FROM node:22-slim
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
# System deps for Chrome for Testing + build tools for native modules (better-sqlite3)
|
||||
# On ARM64 we also install system Chromium (Chrome for Testing has no ARM64 binary)
|
||||
# System deps for CloakBrowser + build tools for native modules (better-sqlite3)
|
||||
# fonts-noto-color-emoji and fonts-freefont-ttf are required so canvas fingerprint
|
||||
# hashes match real browsers; missing emoji fonts cause bot detection on Kasada/Akamai.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl ca-certificates fonts-liberation libasound2 \
|
||||
libatk-bridge2.0-0 libatk1.0-0 libcups2 libdbus-1-3 \
|
||||
libdrm2 libgbm1 libgtk-3-0 libnspr4 libnss3 \
|
||||
libx11-xcb1 libxcomposite1 libxdamage1 libxrandr2 xdg-utils \
|
||||
fonts-noto-color-emoji fonts-freefont-ttf \
|
||||
python3 make g++ \
|
||||
&& if [ "$TARGETARCH" = "arm64" ]; then apt-get install -y --no-install-recommends chromium; fi \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& mkdir -p /db /conf /fredy
|
||||
|
||||
@@ -26,8 +25,8 @@ RUN yarn config set network-timeout 600000 \
|
||||
&& yarn --frozen-lockfile \
|
||||
&& yarn cache clean
|
||||
|
||||
# on arm64 use the system Chromium installed above
|
||||
RUN if [ "$TARGETARCH" != "arm64" ]; then npx puppeteer browsers install chrome; fi
|
||||
# Pre-download the CloakBrowser stealth Chromium binary (supports x86_64 and arm64)
|
||||
RUN node -e "import('cloakbrowser').then(({ensureBinary}) => ensureBinary())"
|
||||
|
||||
# Purge build tools now that native modules are compiled
|
||||
RUN apt-get purge -y python3 make g++ \
|
||||
|
||||
29
README.md
29
README.md
@@ -188,10 +188,25 @@ You should now be able to access _Fredy_ from your browser. Check your Terminal
|
||||
|
||||
### Run Tests
|
||||
|
||||
## "Online" tests
|
||||
These tests are directly executed against the actual providers.
|
||||
``` bash
|
||||
yarn run test
|
||||
```
|
||||
|
||||
## "Offline" tests
|
||||
These tests are using the test fixtures instead of the actual providers. Much faster and "good enough" to test the core functionality.
|
||||
``` bash
|
||||
yarn run test:offline
|
||||
```
|
||||
|
||||
## Download new fixtures
|
||||
If you have to refresh the fixtures (every once in a while needed because the providers change their code), run this command:
|
||||
``` bash
|
||||
yarn run download-fixtures
|
||||
```
|
||||
|
||||
|
||||
------------------------------------------------------------------------
|
||||
|
||||
## 📐 Architecture
|
||||
@@ -225,6 +240,20 @@ flowchart TD
|
||||
F1 --> F2
|
||||
```
|
||||
|
||||
------------------------------------------------------------------------
|
||||
## 🤖 Using AI such as Claude Code
|
||||
When I started building Fredy, LLMs were still basically the wet dream of a few nerdy scientists.
|
||||
|
||||
Nowadays, it’s easier than ever to throw a prompt into the LLM of your choice and let 'the AI' build your stuff. I’m not against that. I use Claude Code myself for smaller tasks, and I do think these tools can be really useful.
|
||||
|
||||
That said, I still believe humans should stay in charge. AI is great-ish at writing code, but it still lacks creativity, context, and the ability to see the full picture.
|
||||
|
||||
So, if you want to contribute to Fredy, using AI tools to get things done is totally fine. Just please don’t stop thinking.
|
||||
|
||||
I’ve had one too many PRs full of hallucinated bullshit.
|
||||
|
||||
**Thanks ;)**
|
||||
|
||||
------------------------------------------------------------------------
|
||||
|
||||
## 👐 Contributing
|
||||
|
||||
@@ -43,13 +43,13 @@ for i in $(seq 1 30); do
|
||||
done
|
||||
|
||||
# Verify the DB is readable/writable via the API.
|
||||
# /api/demo is unauthenticated and reads the settings table — if SQLite is broken this returns an error.
|
||||
# /api/demo is unauthenticated and reads the settings table - if SQLite is broken this returns an error.
|
||||
echo "Testing DB via API (/api/demo)..."
|
||||
DEMO_RESPONSE=$(docker exec fredy curl -sf http://localhost:9998/api/demo 2>&1)
|
||||
if echo "$DEMO_RESPONSE" | grep -q "demoMode"; then
|
||||
echo "DB is readable (got demoMode from /api/demo)"
|
||||
else
|
||||
echo "DB check failed — unexpected response from /api/demo: $DEMO_RESPONSE"
|
||||
echo "DB check failed - unexpected response from /api/demo: $DEMO_RESPONSE"
|
||||
docker logs fredy
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -11,6 +11,9 @@
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
|
||||
|
||||
<title>Fredy || Real Estate Finder</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
||||
</head>
|
||||
<body theme-mode="dark">
|
||||
<div id="fredy" style="position: absolute; top: 0; left: 0; right: 0; bottom: 0"></div>
|
||||
|
||||
9
index.js
9
index.js
@@ -15,6 +15,15 @@ import { initGeocodingCron } from './lib/services/crons/geocoding-cron.js';
|
||||
import { getSettings } from './lib/services/storage/settingsStorage.js';
|
||||
import SqliteConnection, { computeDbPath } from './lib/services/storage/SqliteConnection.js';
|
||||
import { initJobExecutionService } from './lib/services/jobs/jobExecutionService.js';
|
||||
import { ensureValidBinary } from './lib/services/ensureValidBinary.js';
|
||||
|
||||
// Ensure the CloakBrowser stealth Chromium binary is present and complete before
|
||||
// jobs run. ensureValidBinary() also detects and auto-heals partial extractions
|
||||
// (e.g. a newer version that was downloaded but only the chrome executable was
|
||||
// written) so Chrome never crashes with "Invalid file descriptor to ICU data".
|
||||
logger.info('Checking CloakBrowser binary...');
|
||||
await ensureValidBinary();
|
||||
logger.info('CloakBrowser binary ready.');
|
||||
|
||||
//in the config, we store the path of the sqlite file, thus we must check if it is available
|
||||
const isConfigAccessible = await checkIfConfigIsAccessible();
|
||||
|
||||
12
jsconfig.json
Normal file
12
jsconfig.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"target": "ESNext",
|
||||
"checkJs": true,
|
||||
"allowJs": true,
|
||||
"noEmit": true,
|
||||
"strict": false
|
||||
},
|
||||
"exclude": ["node_modules", "ui"]
|
||||
}
|
||||
@@ -16,25 +16,17 @@ import urlModifier from './services/queryStringMutator.js';
|
||||
import logger from './services/logger.js';
|
||||
import { geocodeAddress } from './services/geocoding/geoCodingService.js';
|
||||
import { distanceMeters } from './services/listings/distanceCalculator.js';
|
||||
import { getUserSettings } from './services/storage/settingsStorage.js';
|
||||
import { getUserSettings, getSettings } from './services/storage/settingsStorage.js';
|
||||
import { updateListingDistance } from './services/storage/listingsStorage.js';
|
||||
import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
|
||||
import { formatListing } from './utils/formatListing.js';
|
||||
|
||||
/**
|
||||
* @typedef {Object} Listing
|
||||
* @property {string} id Stable unique identifier (hash) of the listing.
|
||||
* @property {string} title Title or headline of the listing.
|
||||
* @property {string} [address] Optional address/location text.
|
||||
* @property {string} [price] Optional price text/value.
|
||||
* @property {string} [url] Link to the listing detail page.
|
||||
* @property {any} [meta] Provider-specific additional metadata.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SimilarityCache
|
||||
* @property {(title:string, address?:string)=>boolean} hasSimilarEntries Returns true if a similar entry is known.
|
||||
* @property {(title:string, address?:string)=>void} addCacheEntry Adds a new entry to the similarity cache.
|
||||
*/
|
||||
/** @import { ParsedListing } from './types/listing.js' */
|
||||
/** @import { Job } from './types/job.js' */
|
||||
/** @import { ProviderConfig } from './types/providerConfig.js' */
|
||||
/** @import { SpecFilter, SpatialFilter } from './types/filter.js' */
|
||||
/** @import { SimilarityCache } from './types/similarityCache.js' */
|
||||
/** @import { Browser } from './types/browser.js' */
|
||||
|
||||
/**
|
||||
* Runtime orchestrator for fetching, normalizing, filtering, deduplicating, storing,
|
||||
@@ -48,43 +40,43 @@ import booleanPointInPolygon from '@turf/boolean-point-in-polygon';
|
||||
* 5) Identify new listings (vs. previously stored hashes)
|
||||
* 6) Persist new listings
|
||||
* 7) Filter out entries similar to already seen ones
|
||||
* 8) Dispatch notifications
|
||||
* 8) Filter out entries that do not match the job's specFilter
|
||||
* 9) Filter out entries that do not match the job's spatialFilter
|
||||
* 10) Dispatch notifications
|
||||
*/
|
||||
class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Create a new runtime instance for a single provider/job execution.
|
||||
*
|
||||
* @param {Object} providerConfig Provider configuration.
|
||||
* @param {string} providerConfig.url Base URL to crawl.
|
||||
* @param {string} [providerConfig.sortByDateParam] Query parameter used to enforce sorting by date (provider-specific).
|
||||
* @param {string} [providerConfig.waitForSelector] CSS selector to wait for before parsing content.
|
||||
* @param {Object.<string, string>} providerConfig.crawlFields Mapping of field names to selectors/paths to extract.
|
||||
* @param {string} providerConfig.crawlContainer CSS selector for the container holding listing items.
|
||||
* @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape.
|
||||
* @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings.
|
||||
* @param {(url:string, waitForSelector?:string)=>Promise<void>|Promise<Listing[]>} [providerConfig.getListings] Optional override to fetch listings.
|
||||
* @param {(listing:Listing, browser:any)=>Promise<Listing>} [providerConfig.fetchDetails] Optional per-listing detail enrichment. Called in parallel for each new listing after deduplication. Receives the shared browser instance. Must always resolve (never reject).
|
||||
* @param {Object} notificationConfig Notification configuration passed to notification adapters.
|
||||
* @param {Object} spatialFilter Optional spatial filter configuration.
|
||||
* @param {ProviderConfig} providerConfig Provider configuration.
|
||||
* @param {Job} job Job configuration.
|
||||
* @param {string} providerId The ID of the provider currently in use.
|
||||
* @param {string} jobKey Key of the job that is currently running (from within the config).
|
||||
* @param {SimilarityCache} similarityCache Cache instance for checking similar entries.
|
||||
* @param browser
|
||||
* @param {Browser} browser Puppeteer browser instance.
|
||||
*/
|
||||
constructor(providerConfig, notificationConfig, spatialFilter, providerId, jobKey, similarityCache, browser) {
|
||||
constructor(providerConfig, job, providerId, similarityCache, browser) {
|
||||
/** @type {ProviderConfig} */
|
||||
this._providerConfig = providerConfig;
|
||||
this._notificationConfig = notificationConfig;
|
||||
this._spatialFilter = spatialFilter;
|
||||
/** @type {Object} */
|
||||
this._jobNotificationConfig = job.notificationAdapter;
|
||||
/** @type {string} */
|
||||
this._jobKey = job.id;
|
||||
/** @type {SpecFilter | null} */
|
||||
this._jobSpecFilter = job.specFilter;
|
||||
/** @type {SpatialFilter | null} */
|
||||
this._jobSpatialFilter = job.spatialFilter;
|
||||
/** @type {string} */
|
||||
this._providerId = providerId;
|
||||
this._jobKey = jobKey;
|
||||
/** @type {SimilarityCache} */
|
||||
this._similarityCache = similarityCache;
|
||||
/** @type {Browser} */
|
||||
this._browser = browser;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the end-to-end pipeline for a single provider run.
|
||||
*
|
||||
* @returns {Promise<Listing[]|void>} Resolves to the list of new (and similarity-filtered) listings
|
||||
* @returns {Promise<ParsedListing[]|void>} Resolves to the list of new (and similarity-filtered) listings
|
||||
* after notifications have been sent; resolves to void when there are no new listings.
|
||||
*/
|
||||
execute() {
|
||||
@@ -98,6 +90,7 @@ class FredyPipelineExecutioner {
|
||||
.then(this._save.bind(this))
|
||||
.then(this._calculateDistance.bind(this))
|
||||
.then(this._filterBySimilarListings.bind(this))
|
||||
.then(this._filterBySpecs.bind(this))
|
||||
.then(this._filterByArea.bind(this))
|
||||
.then(this._notify.bind(this))
|
||||
.catch(this._handleError.bind(this));
|
||||
@@ -132,8 +125,8 @@ class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Geocode new listings.
|
||||
*
|
||||
* @param {Listing[]} newListings New listings to geocode.
|
||||
* @returns {Promise<Listing[]>} Resolves with the listings (potentially with added coordinates).
|
||||
* @param {ParsedListing[]} newListings New listings to geocode.
|
||||
* @returns {Promise<ParsedListing[]>} Resolves with the listings (potentially with added coordinates).
|
||||
*/
|
||||
async _geocode(newListings) {
|
||||
for (const listing of newListings) {
|
||||
@@ -152,18 +145,18 @@ class FredyPipelineExecutioner {
|
||||
* Filter listings by area using the provider's area filter if available.
|
||||
* Only filters if areaFilter is set on the provider AND the listing has coordinates.
|
||||
*
|
||||
* @param {Listing[]} newListings New listings to filter by area.
|
||||
* @returns {Promise<Listing[]>} Resolves with listings that are within the area (or not filtered if no area is set).
|
||||
* @param {ParsedListing[]} newListings New listings to filter by area.
|
||||
* @returns {ParsedListing[]} Resolves with listings that are within the area (or not filtered if no area is set).
|
||||
*/
|
||||
_filterByArea(newListings) {
|
||||
const polygonFeatures = this._spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon');
|
||||
const polygonFeatures = this._jobSpatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon');
|
||||
|
||||
// If no area filter is set, return all listings
|
||||
if (!polygonFeatures?.length) {
|
||||
return newListings;
|
||||
}
|
||||
|
||||
const filteredIds = [];
|
||||
const toDeleteListingByIds = [];
|
||||
// Filter listings by area - keep only those within the polygon
|
||||
const keptListings = newListings.filter((listing) => {
|
||||
// If listing doesn't have coordinates, keep it (don't filter out)
|
||||
@@ -176,14 +169,48 @@ class FredyPipelineExecutioner {
|
||||
const isInPolygon = polygonFeatures.some((feature) => booleanPointInPolygon(point, feature));
|
||||
|
||||
if (!isInPolygon) {
|
||||
filteredIds.push(listing.id);
|
||||
toDeleteListingByIds.push(listing.id);
|
||||
}
|
||||
|
||||
return isInPolygon;
|
||||
});
|
||||
|
||||
if (filteredIds.length > 0) {
|
||||
deleteListingsById(filteredIds);
|
||||
if (toDeleteListingByIds.length > 0) {
|
||||
deleteListingsById(toDeleteListingByIds);
|
||||
}
|
||||
|
||||
return keptListings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter listings based on its specifications (minRooms, minSize, maxPrice).
|
||||
*
|
||||
* @param {ParsedListing[]} newListings New listings to filter.
|
||||
* @returns {ParsedListing[]} Resolves with listings that pass the specification filters.
|
||||
*/
|
||||
_filterBySpecs(newListings) {
|
||||
const { minRooms, minSize, maxPrice } = this._jobSpecFilter || {};
|
||||
|
||||
// If no specs are set, return all listings
|
||||
if (!minRooms && !minSize && !maxPrice) {
|
||||
return newListings;
|
||||
}
|
||||
|
||||
const toDeleteListingByIds = [];
|
||||
const keptListings = newListings.filter((listing) => {
|
||||
const filterOut =
|
||||
(minRooms && listing.rooms && listing.rooms < minRooms) ||
|
||||
(minSize && listing.size && listing.size < minSize) ||
|
||||
(maxPrice && listing.price && listing.price > maxPrice);
|
||||
|
||||
if (filterOut) {
|
||||
toDeleteListingByIds.push(listing.id);
|
||||
}
|
||||
return !filterOut;
|
||||
});
|
||||
|
||||
if (toDeleteListingByIds.length > 0) {
|
||||
deleteListingsById(toDeleteListingByIds);
|
||||
}
|
||||
|
||||
return keptListings;
|
||||
@@ -194,13 +221,13 @@ class FredyPipelineExecutioner {
|
||||
* a provider-specific getListings override is supplied.
|
||||
*
|
||||
* @param {string} url The provider URL to fetch from.
|
||||
* @returns {Promise<Listing[]>} Resolves with an array of listings (empty when none found).
|
||||
* @returns {Promise<ParsedListing[]>} Resolves with an array of listings (empty when none found).
|
||||
*/
|
||||
_getListings(url) {
|
||||
const extractor = new Extractor({ ...this._providerConfig.puppeteerOptions, browser: this._browser });
|
||||
return new Promise((resolve, reject) => {
|
||||
extractor
|
||||
.execute(url, this._providerConfig.waitForSelector)
|
||||
.execute(url, this._providerConfig.waitForSelector, this._jobKey)
|
||||
.then(() => {
|
||||
const listings = extractor.parseResponseText(
|
||||
this._providerConfig.crawlContainer,
|
||||
@@ -217,33 +244,42 @@ class FredyPipelineExecutioner {
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize raw listings into the provider-specific Listing shape.
|
||||
* Normalize raw listings into the provider-specific ParsedListing shape.
|
||||
*
|
||||
* @param {any[]} listings Raw listing entries from the extractor or override.
|
||||
* @returns {Listing[]} Normalized listings.
|
||||
* @returns {ParsedListing[]} Normalized listings.
|
||||
*/
|
||||
_normalize(listings) {
|
||||
return listings.map(this._providerConfig.normalize);
|
||||
return listings.map((listing) => this._providerConfig.normalize(listing));
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter out listings that are missing required fields and those rejected by the
|
||||
* provider's blacklist/filter function.
|
||||
*
|
||||
* @param {Listing[]} listings Listings to filter.
|
||||
* @returns {Listing[]} Filtered listings that pass validation and provider filter.
|
||||
* @param {ParsedListing[]} listings Listings to filter.
|
||||
* @returns {ParsedListing[]} Filtered listings that pass validation and provider filter.
|
||||
*/
|
||||
_filter(listings) {
|
||||
const keys = Object.keys(this._providerConfig.crawlFields);
|
||||
const filteredListings = listings.filter((item) => keys.every((key) => key in item));
|
||||
return filteredListings.filter(this._providerConfig.filter);
|
||||
const requiredKeys = this._providerConfig.requiredFieldNames;
|
||||
const requireValues = ['id', 'link', 'title'];
|
||||
|
||||
const filteredListings = listings
|
||||
// this should never filter some listings out, because the normalize function should always extract all fields.
|
||||
.filter((item) => requiredKeys.every((key) => key in item))
|
||||
// TODO: move blacklist filter to this file, so it will handle for all providers in same way.
|
||||
.filter(this._providerConfig.filter)
|
||||
// filter out listings that are missing required fields
|
||||
.filter((item) => requireValues.every((key) => item[key] != null));
|
||||
|
||||
return filteredListings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine which listings are new by comparing their IDs against stored hashes.
|
||||
*
|
||||
* @param {Listing[]} listings Listings to evaluate for novelty.
|
||||
* @returns {Listing[]} New listings not seen before.
|
||||
* @param {ParsedListing[]} listings Listings to evaluate for novelty.
|
||||
* @returns {ParsedListing[]} New listings not seen before.
|
||||
* @throws {NoNewListingsWarning} When no new listings are found.
|
||||
*/
|
||||
_findNew(listings) {
|
||||
@@ -260,23 +296,32 @@ class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Send notifications for new listings using the configured notification adapter(s).
|
||||
*
|
||||
* @param {Listing[]} newListings New listings to notify about.
|
||||
* @returns {Promise<Listing[]>} Resolves to the provided listings after notifications complete.
|
||||
* @param {ParsedListing[]} newListings New listings to notify about.
|
||||
* @returns {Promise<ParsedListing[]>} Resolves to the provided listings after notifications complete.
|
||||
* @throws {NoNewListingsWarning} When there are no listings to notify about.
|
||||
*/
|
||||
_notify(newListings) {
|
||||
async _notify(newListings) {
|
||||
if (newListings.length === 0) {
|
||||
throw new NoNewListingsWarning();
|
||||
}
|
||||
const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey);
|
||||
const formattedListings = newListings.map(formatListing);
|
||||
const settings = await getSettings();
|
||||
const baseUrl = settings?.baseUrl ?? '';
|
||||
const sendNotifications = notify.send(
|
||||
this._providerId,
|
||||
formattedListings,
|
||||
this._jobNotificationConfig,
|
||||
this._jobKey,
|
||||
baseUrl,
|
||||
);
|
||||
return Promise.all(sendNotifications).then(() => newListings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist new listings and pass them through.
|
||||
*
|
||||
* @param {Listing[]} newListings Listings to store.
|
||||
* @returns {Listing[]} The same listings, unchanged.
|
||||
* @param {ParsedListing[]} newListings Listings to store.
|
||||
* @returns {ParsedListing[]} The same listings, unchanged.
|
||||
*/
|
||||
_save(newListings) {
|
||||
logger.debug(`Storing ${newListings.length} new listings (Provider: '${this._providerId}')`);
|
||||
@@ -287,8 +332,8 @@ class FredyPipelineExecutioner {
|
||||
/**
|
||||
* Calculate distance for new listings.
|
||||
*
|
||||
* @param {Listing[]} listings
|
||||
* @returns {Listing[]}
|
||||
* @param {ParsedListing[]} listings
|
||||
* @returns {ParsedListing[]}
|
||||
* @private
|
||||
*/
|
||||
_calculateDistance(listings) {
|
||||
@@ -324,8 +369,8 @@ class FredyPipelineExecutioner {
|
||||
* Remove listings that are similar to already known entries according to the similarity cache.
|
||||
* Adds the remaining listings to the cache.
|
||||
*
|
||||
* @param {Listing[]} listings Listings to filter by similarity.
|
||||
* @returns {Listing[]} Listings considered unique enough to keep.
|
||||
* @param {ParsedListing[]} listings Listings to filter by similarity.
|
||||
* @returns {ParsedListing[]} Listings considered unique enough to keep.
|
||||
*/
|
||||
_filterBySimilarListings(listings) {
|
||||
const filteredIds = [];
|
||||
|
||||
@@ -7,4 +7,8 @@ export const TRACKING_POIS = {
|
||||
DISTANCE_ADDRESS_ENTERED: 'DISTANCE_ADDRESS_ENTERED',
|
||||
WELCOME_FINISHED: 'WELCOME_FINISHED',
|
||||
WELCOME_SKIPPED: 'WELCOME_SKIPPED',
|
||||
JOBS_TABLE_VIEW: 'JOBS_TABLE_VIEW',
|
||||
LISTING_TABLE_VIEW: 'LISTING_TABLE_VIEW',
|
||||
BASE_URL_SETTING: 'BASE_URL_SETTING',
|
||||
DETECTED_AS_BOT: 'DETECTED_AS_BOT',
|
||||
};
|
||||
|
||||
142
lib/api/api.js
142
lib/api/api.js
@@ -3,64 +3,100 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { notificationAdapterRouter } from './routes/notificationAdapterRouter.js';
|
||||
import { authInterceptor, cookieSession, adminInterceptor } from './security.js';
|
||||
import { generalSettingsRouter } from './routes/generalSettingsRoute.js';
|
||||
import { providerRouter } from './routes/providerRouter.js';
|
||||
import { versionRouter } from './routes/versionRouter.js';
|
||||
import { loginRouter } from './routes/loginRoute.js';
|
||||
import { userRouter } from './routes/userRoute.js';
|
||||
import { userSettingsRouter } from './routes/userSettingsRoute.js';
|
||||
import { jobRouter } from './routes/jobRouter.js';
|
||||
import bodyParser from 'body-parser';
|
||||
import restana from 'restana';
|
||||
import files from 'serve-static';
|
||||
import Fastify from 'fastify';
|
||||
import fastifyHelmet from '@fastify/helmet';
|
||||
import fastifyCookie from '@fastify/cookie';
|
||||
import fastifySession from '@fastify/session';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import path from 'path';
|
||||
import { getDirName } from '../utils.js';
|
||||
import { demoRouter } from './routes/demoRouter.js';
|
||||
import logger from '../services/logger.js';
|
||||
import { listingsRouter } from './routes/listingsRouter.js';
|
||||
import { getSettings, getOrCreateSessionSecret } from '../services/storage/settingsStorage.js';
|
||||
import { dashboardRouter } from './routes/dashboardRouter.js';
|
||||
import { backupRouter } from './routes/backupRouter.js';
|
||||
import { trackingRouter } from './routes/trackingRoute.js';
|
||||
import logger from '../services/logger.js';
|
||||
import { authHook, adminHook } from './security.js';
|
||||
|
||||
import loginPlugin from './routes/loginRoute.js';
|
||||
import demoPlugin from './routes/demoRouter.js';
|
||||
import jobPlugin from './routes/jobRouter.js';
|
||||
import versionPlugin from './routes/versionRouter.js';
|
||||
import listingsPlugin from './routes/listingsRouter.js';
|
||||
import dashboardPlugin from './routes/dashboardRouter.js';
|
||||
import userSettingsPlugin from './routes/userSettingsRoute.js';
|
||||
import trackingPlugin from './routes/trackingRoute.js';
|
||||
import generalSettingsPlugin from './routes/generalSettingsRoute.js';
|
||||
import backupPlugin from './routes/backupRouter.js';
|
||||
import userPlugin from './routes/userRoute.js';
|
||||
import notificationAdapterPlugin from './routes/notificationAdapterRouter.js';
|
||||
import providerPlugin from './routes/providerRouter.js';
|
||||
import { registerMcpRoutes } from '../mcp/mcpHttpRoute.js';
|
||||
const service = restana();
|
||||
const staticService = files(path.join(getDirName(), '../ui/public'));
|
||||
|
||||
const PORT = (await getSettings()).port || 9998;
|
||||
const sessionSecret = await getOrCreateSessionSecret();
|
||||
const SESSION_MAX_AGE = 2 * 60 * 60 * 1000;
|
||||
|
||||
service.use(bodyParser.json());
|
||||
service.use(cookieSession(sessionSecret));
|
||||
service.use(staticService);
|
||||
service.use('/api/admin', authInterceptor());
|
||||
service.use('/api/jobs', authInterceptor());
|
||||
service.use('/api/version', authInterceptor());
|
||||
service.use('/api/listings', authInterceptor());
|
||||
service.use('/api/dashboard', authInterceptor());
|
||||
service.use('/api/user/settings', authInterceptor());
|
||||
service.use('/api/tracking', authInterceptor());
|
||||
|
||||
// /admin can only be accessed when user is having admin permissions
|
||||
service.use('/api/admin', adminInterceptor());
|
||||
service.use('/api/jobs/notificationAdapter', notificationAdapterRouter);
|
||||
service.use('/api/admin/generalSettings', generalSettingsRouter);
|
||||
service.use('/api/admin/backup', backupRouter);
|
||||
service.use('/api/jobs/provider', providerRouter);
|
||||
service.use('/api/admin/users', userRouter);
|
||||
service.use('/api/user/settings', userSettingsRouter);
|
||||
service.use('/api/version', versionRouter);
|
||||
service.use('/api/jobs', jobRouter);
|
||||
service.use('/api/login', loginRouter);
|
||||
service.use('/api/listings', listingsRouter);
|
||||
service.use('/api/dashboard', dashboardRouter);
|
||||
service.use('/api/tracking', trackingRouter);
|
||||
//this route is unsecured intentionally as it is being queried from the login page
|
||||
service.use('/api/demo', demoRouter);
|
||||
|
||||
// MCP Streamable HTTP endpoint (secured via Bearer token, not cookie-session)
|
||||
registerMcpRoutes(service);
|
||||
|
||||
service.start(PORT).then(() => {
|
||||
logger.debug(`Started API service on port ${PORT}`);
|
||||
const fastify = Fastify({
|
||||
logger: false,
|
||||
bodyLimit: 50 * 1024 * 1024, // 50 MB for backup uploads
|
||||
});
|
||||
|
||||
// Security headers (CSP disabled to avoid breaking the SPA)
|
||||
await fastify.register(fastifyHelmet, { contentSecurityPolicy: false });
|
||||
|
||||
// Cookie + session (in-memory store, signed cookie)
|
||||
await fastify.register(fastifyCookie);
|
||||
await fastify.register(fastifySession, {
|
||||
secret: sessionSecret,
|
||||
cookieName: 'fredy-admin-session',
|
||||
cookie: {
|
||||
maxAge: SESSION_MAX_AGE,
|
||||
httpOnly: true,
|
||||
secure: false,
|
||||
sameSite: 'lax',
|
||||
},
|
||||
saveUninitialized: false,
|
||||
});
|
||||
|
||||
// Serve the React SPA from ui/public/
|
||||
await fastify.register(fastifyStatic, {
|
||||
root: path.join(getDirName(), '../ui/public'),
|
||||
wildcard: false,
|
||||
});
|
||||
|
||||
// Public routes - no auth required
|
||||
fastify.register(loginPlugin, { prefix: '/api/login' });
|
||||
fastify.register(demoPlugin, { prefix: '/api/demo' });
|
||||
|
||||
// User-authenticated routes
|
||||
fastify.register(async (app) => {
|
||||
app.addHook('preHandler', authHook);
|
||||
app.register(jobPlugin, { prefix: '/api/jobs' });
|
||||
app.register(notificationAdapterPlugin, { prefix: '/api/jobs/notificationAdapter' });
|
||||
app.register(providerPlugin, { prefix: '/api/jobs/provider' });
|
||||
app.register(versionPlugin, { prefix: '/api/version' });
|
||||
app.register(listingsPlugin, { prefix: '/api/listings' });
|
||||
app.register(dashboardPlugin, { prefix: '/api/dashboard' });
|
||||
app.register(userSettingsPlugin, { prefix: '/api/user/settings' });
|
||||
app.register(trackingPlugin, { prefix: '/api/tracking' });
|
||||
app.register(generalSettingsPlugin, { prefix: '/api/admin/generalSettings' });
|
||||
});
|
||||
|
||||
// Admin-only routes
|
||||
fastify.register(async (app) => {
|
||||
app.addHook('preHandler', authHook);
|
||||
app.addHook('preHandler', adminHook);
|
||||
app.register(backupPlugin, { prefix: '/api/admin/backup' });
|
||||
app.register(userPlugin, { prefix: '/api/admin/users' });
|
||||
});
|
||||
|
||||
// MCP Streamable HTTP (Bearer token auth - no session)
|
||||
registerMcpRoutes(fastify);
|
||||
|
||||
// SPA fallback - serve index.html for all non-API GET requests
|
||||
fastify.setNotFoundHandler((request, reply) => {
|
||||
if (!request.url.startsWith('/api/')) {
|
||||
return reply.sendFile('index.html');
|
||||
}
|
||||
return reply.code(404).send({ error: 'Not found' });
|
||||
});
|
||||
|
||||
await fastify.listen({ port: PORT, host: '0.0.0.0' });
|
||||
logger.debug(`Started API service on port ${PORT}`);
|
||||
|
||||
@@ -3,73 +3,61 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import {
|
||||
buildBackupFileName,
|
||||
createBackupZip,
|
||||
precheckRestore,
|
||||
restoreFromZip,
|
||||
} from '../../services/storage/backupRestoreService.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
|
||||
const DEMO_MODE_ERROR = 'Backup and restore are not available in demo mode.';
|
||||
|
||||
/**
|
||||
* Backup & Restore Admin Router
|
||||
*
|
||||
* Endpoints:
|
||||
* - GET /api/admin/backup
|
||||
* Returns the current database as a zip download. Content-Type: application/zip
|
||||
* - POST /api/admin/backup/restore?dryRun=true
|
||||
* Accepts a zip file (raw body). Returns a compatibility report, does not restore.
|
||||
* - POST /api/admin/backup/restore?force=true|false
|
||||
* Accepts a zip file (raw body). Restores the database; when incompatible and force=false, returns 400.
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
const service = restana();
|
||||
const backupRouter = service.newRouter();
|
||||
export default async function backupPlugin(fastify) {
|
||||
// Parse raw binary uploads as Buffer
|
||||
fastify.addContentTypeParser(
|
||||
['application/zip', 'application/octet-stream'],
|
||||
{ parseAs: 'buffer' },
|
||||
(req, body, done) => done(null, body),
|
||||
);
|
||||
|
||||
backupRouter.get('/', async (req, res) => {
|
||||
const zipBuffer = await createBackupZip();
|
||||
const fileName = await buildBackupFileName();
|
||||
res.setHeader('Content-Type', 'application/zip');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||
res.send(zipBuffer);
|
||||
});
|
||||
fastify.get('/', async (request, reply) => {
|
||||
const settings = await getSettings();
|
||||
if (settings.demoMode && !isAdmin(request)) {
|
||||
return reply.code(403).send({ error: DEMO_MODE_ERROR });
|
||||
}
|
||||
const zipBuffer = await createBackupZip();
|
||||
const fileName = await buildBackupFileName();
|
||||
reply.header('Content-Type', 'application/zip');
|
||||
reply.header('Content-Disposition', `attachment; filename="${fileName}"`);
|
||||
return reply.send(zipBuffer);
|
||||
});
|
||||
|
||||
/**
|
||||
* Read the full request body as a Buffer. Used for raw zip uploads.
|
||||
* @param {import('http').IncomingMessage} req
|
||||
* @returns {Promise<Buffer>}
|
||||
*/
|
||||
function readBody(req) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
req.on('data', (c) => chunks.push(c));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
req.on('error', (e) => reject(e));
|
||||
fastify.post('/restore', async (request, reply) => {
|
||||
const settings = await getSettings();
|
||||
if (settings.demoMode && !isAdmin(request)) {
|
||||
return reply.code(403).send({ error: DEMO_MODE_ERROR });
|
||||
}
|
||||
const { dryRun = 'false', force = 'false' } = request.query || {};
|
||||
const doDryRun = String(dryRun) === 'true';
|
||||
const doForce = String(force) === 'true';
|
||||
const body = request.body; // Buffer from addContentTypeParser
|
||||
|
||||
if (doDryRun) {
|
||||
return precheckRestore(body);
|
||||
}
|
||||
|
||||
try {
|
||||
return restoreFromZip(body, { force: doForce });
|
||||
} catch (e) {
|
||||
return reply.code(400).send({
|
||||
message: e?.message || 'Restore failed',
|
||||
details: e?.payload || null,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Upload endpoint. Accepts raw zip (Content-Type: application/zip or application/octet-stream)
|
||||
// Query parameters:
|
||||
// - dryRun=true => only validate and return compatibility info
|
||||
// - force=true => proceed even if incompatible
|
||||
backupRouter.post('/restore', async (req, res) => {
|
||||
const { dryRun = 'false', force = 'false' } = req.query || {};
|
||||
const doDryRun = String(dryRun) === 'true';
|
||||
const doForce = String(force) === 'true';
|
||||
const body = await readBody(req);
|
||||
|
||||
if (doDryRun) {
|
||||
res.body = await precheckRestore(body);
|
||||
return res.send();
|
||||
}
|
||||
|
||||
try {
|
||||
res.body = await restoreFromZip(body, { force: doForce });
|
||||
return res.send();
|
||||
} catch (e) {
|
||||
res.statusCode = 400;
|
||||
res.body = { message: e?.message || 'Restore failed', details: e?.payload || null };
|
||||
return res.send();
|
||||
}
|
||||
});
|
||||
|
||||
export { backupRouter };
|
||||
|
||||
@@ -3,23 +3,14 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import { getListingsKpisForJobIds, getProviderDistributionForJobIds } from '../../services/storage/listingsStorage.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
|
||||
const service = restana();
|
||||
export const dashboardRouter = service.newRouter();
|
||||
|
||||
function isAdmin(req) {
|
||||
const user = req.session?.currentUser ? userStorage.getUser(req.session.currentUser) : null;
|
||||
return !!user?.isAdmin;
|
||||
}
|
||||
|
||||
function getAccessibleJobs(req) {
|
||||
const currentUser = req.session.currentUser;
|
||||
const admin = isAdmin(req);
|
||||
function getAccessibleJobs(request) {
|
||||
const currentUser = request.session.currentUser;
|
||||
const admin = isAdmin(request);
|
||||
return jobStorage
|
||||
.getJobs()
|
||||
.filter((job) => admin || job.userId === currentUser || job.shared_with_user.includes(currentUser));
|
||||
@@ -29,43 +20,45 @@ function cap(val) {
|
||||
return String(val).charAt(0).toUpperCase() + String(val).slice(1);
|
||||
}
|
||||
|
||||
dashboardRouter.get('/', async (req, res) => {
|
||||
const jobs = getAccessibleJobs(req);
|
||||
const settings = await getSettings();
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function dashboardPlugin(fastify) {
|
||||
fastify.get('/', async (request) => {
|
||||
const jobs = getAccessibleJobs(request);
|
||||
const settings = await getSettings();
|
||||
|
||||
// KPIs
|
||||
const totalJobs = jobs.length;
|
||||
const totalListings = jobs.reduce((sum, j) => sum + (j.numberOfFoundListings || 0), 0);
|
||||
const jobIds = jobs.map((j) => j.id);
|
||||
const { numberOfActiveListings, avgPriceOfListings } = getListingsKpisForJobIds(jobIds);
|
||||
// Build Pie data in a simple shape the frontend can consume directly
|
||||
// Shape: { labels: string[], values: number[] } with values as percentages
|
||||
const providerPieRaw = getProviderDistributionForJobIds(jobIds);
|
||||
const providerPie = Array.isArray(providerPieRaw)
|
||||
? {
|
||||
labels: providerPieRaw.map((p) => cap(p.type)),
|
||||
values: providerPieRaw.map((p) => Number(p.value) || 0),
|
||||
}
|
||||
: providerPieRaw && typeof providerPieRaw === 'object'
|
||||
const totalJobs = jobs.length;
|
||||
const totalListings = jobs.reduce((sum, j) => sum + (j.numberOfFoundListings || 0), 0);
|
||||
const jobIds = jobs.map((j) => j.id);
|
||||
const { numberOfActiveListings, medianPriceOfListings } = getListingsKpisForJobIds(jobIds);
|
||||
|
||||
const providerPieRaw = getProviderDistributionForJobIds(jobIds);
|
||||
const providerPie = Array.isArray(providerPieRaw)
|
||||
? {
|
||||
labels: Array.isArray(providerPieRaw.labels) ? providerPieRaw.labels : [],
|
||||
values: Array.isArray(providerPieRaw.values) ? providerPieRaw.values : [],
|
||||
labels: providerPieRaw.map((p) => cap(p.type)),
|
||||
values: providerPieRaw.map((p) => Number(p.value) || 0),
|
||||
}
|
||||
: { labels: [], values: [] };
|
||||
: providerPieRaw && typeof providerPieRaw === 'object'
|
||||
? {
|
||||
labels: Array.isArray(providerPieRaw.labels) ? providerPieRaw.labels : [],
|
||||
values: Array.isArray(providerPieRaw.values) ? providerPieRaw.values : [],
|
||||
}
|
||||
: { labels: [], values: [] };
|
||||
|
||||
res.body = {
|
||||
general: {
|
||||
interval: settings.interval,
|
||||
lastRun: settings.lastRun || null,
|
||||
nextRun: settings.lastRun == null ? 0 : settings.lastRun + settings.interval * 60000,
|
||||
},
|
||||
kpis: {
|
||||
totalJobs,
|
||||
totalListings,
|
||||
numberOfActiveListings,
|
||||
avgPriceOfListings,
|
||||
},
|
||||
pie: providerPie,
|
||||
};
|
||||
res.send();
|
||||
});
|
||||
return {
|
||||
general: {
|
||||
interval: settings.interval,
|
||||
lastRun: settings.lastRun || null,
|
||||
nextRun: settings.lastRun == null ? 0 : settings.lastRun + settings.interval * 60000,
|
||||
},
|
||||
kpis: {
|
||||
totalJobs,
|
||||
totalListings,
|
||||
numberOfActiveListings,
|
||||
medianPriceOfListings,
|
||||
},
|
||||
pie: providerPie,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,15 +3,14 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
const service = restana();
|
||||
const demoRouter = service.newRouter();
|
||||
|
||||
demoRouter.get('/', async (req, res) => {
|
||||
const settings = await getSettings();
|
||||
res.body = Object.assign({}, { demoMode: settings.demoMode });
|
||||
res.send();
|
||||
});
|
||||
|
||||
export { demoRouter };
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function demoPlugin(fastify) {
|
||||
fastify.get('/', async () => {
|
||||
const settings = await getSettings();
|
||||
return { demoMode: settings.demoMode };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,39 +3,51 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import { getDirName } from '../../utils.js';
|
||||
import fs from 'fs';
|
||||
import { ensureDemoUserExists } from '../../services/storage/userStorage.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
|
||||
const service = restana();
|
||||
const generalSettingsRouter = service.newRouter();
|
||||
import { isAdmin } from '../security.js';
|
||||
import { trackPoi } from '../../services/tracking/Tracker.js';
|
||||
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||
|
||||
generalSettingsRouter.get('/', async (req, res) => {
|
||||
res.body = Object.assign({}, await getSettings());
|
||||
res.send();
|
||||
});
|
||||
generalSettingsRouter.post('/', async (req, res) => {
|
||||
const { sqlitepath, ...appSettings } = req.body || {};
|
||||
const localSettings = await getSettings();
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function generalSettingsPlugin(fastify) {
|
||||
fastify.get('/', async () => {
|
||||
return Object.assign({}, await getSettings());
|
||||
});
|
||||
|
||||
if (localSettings.demoMode) {
|
||||
res.send(new Error('In demo mode, it is not allowed to change these settings.'));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (typeof sqlitepath !== 'undefined') {
|
||||
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
|
||||
fastify.post('/', async (request, reply) => {
|
||||
const { sqlitepath, ...appSettings } = request.body || {};
|
||||
if (typeof appSettings.baseUrl === 'string') {
|
||||
appSettings.baseUrl = appSettings.baseUrl.trim().replace(/\/$/, '');
|
||||
}
|
||||
upsertSettings(appSettings);
|
||||
ensureDemoUserExists();
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
res.send(new Error('Error while trying to write settings.'));
|
||||
return;
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
export { generalSettingsRouter };
|
||||
const localSettings = await getSettings();
|
||||
|
||||
if (!isAdmin(request)) {
|
||||
const reason = localSettings.demoMode
|
||||
? 'In demo mode, it is not allowed to change these settings.'
|
||||
: 'Only admins can change these settings.';
|
||||
return reply.code(403).send({ error: reason });
|
||||
}
|
||||
|
||||
try {
|
||||
if (typeof sqlitepath !== 'undefined') {
|
||||
fs.writeFileSync(`${getDirName()}/../conf/config.json`, JSON.stringify({ sqlitepath }));
|
||||
}
|
||||
|
||||
upsertSettings(appSettings);
|
||||
ensureDemoUserExists();
|
||||
if (appSettings.baseUrl != null) {
|
||||
await trackPoi(TRACKING_POIS.BASE_URL_SETTING);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
return reply.code(500).send({ error: 'Error while trying to write settings.' });
|
||||
}
|
||||
return reply.send();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
@@ -13,255 +12,234 @@ import { isRunning as isJobRunning } from '../../services/jobs/run-state.js';
|
||||
import { addClient as addSseClient, removeClient } from '../../services/sse/sse-broker.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
|
||||
const service = restana();
|
||||
const jobRouter = service.newRouter();
|
||||
|
||||
const DEMO_JOB_NAME = 'Demo-Job';
|
||||
|
||||
function doesJobBelongsToUser(job, req) {
|
||||
const userId = req.session.currentUser;
|
||||
if (userId == null) {
|
||||
return false;
|
||||
}
|
||||
function doesJobBelongsToUser(job, request) {
|
||||
const userId = request.session.currentUser;
|
||||
if (userId == null) return false;
|
||||
const user = userStorage.getUser(userId);
|
||||
if (user == null) {
|
||||
return false;
|
||||
}
|
||||
if (user == null) return false;
|
||||
return user.isAdmin || job.userId === user.id;
|
||||
}
|
||||
|
||||
jobRouter.get('/', async (req, res) => {
|
||||
const isUserAdmin = isAdmin(req);
|
||||
//show only the jobs which belongs to the user (or all of the user is an admin)
|
||||
res.body = jobStorage
|
||||
.getJobs()
|
||||
.filter(
|
||||
(job) =>
|
||||
isUserAdmin || job.userId === req.session.currentUser || job.shared_with_user.includes(req.session.currentUser),
|
||||
)
|
||||
.map((job) => {
|
||||
return {
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function jobPlugin(fastify) {
|
||||
fastify.get('/', async (request) => {
|
||||
const isUserAdmin = isAdmin(request);
|
||||
return jobStorage
|
||||
.getJobs()
|
||||
.filter(
|
||||
(job) =>
|
||||
isUserAdmin ||
|
||||
job.userId === request.session.currentUser ||
|
||||
job.shared_with_user.includes(request.session.currentUser),
|
||||
)
|
||||
.map((job) => ({
|
||||
...job,
|
||||
running: isJobRunning(job.id),
|
||||
isOnlyShared:
|
||||
!isUserAdmin &&
|
||||
job.userId !== req.session.currentUser &&
|
||||
job.shared_with_user.includes(req.session.currentUser),
|
||||
};
|
||||
});
|
||||
|
||||
res.send();
|
||||
});
|
||||
|
||||
jobRouter.get('/data', async (req, res) => {
|
||||
const { page, pageSize = 50, activityFilter, sortfield = null, sortdir = 'asc', freeTextFilter } = req.query || {};
|
||||
|
||||
// normalize booleans
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
|
||||
const queryResult = jobStorage.queryJobs({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdmin(req),
|
||||
job.userId !== request.session.currentUser &&
|
||||
job.shared_with_user.includes(request.session.currentUser),
|
||||
}));
|
||||
});
|
||||
|
||||
const isUserAdmin = isAdmin(req);
|
||||
fastify.get('/data', async (request) => {
|
||||
const {
|
||||
page,
|
||||
pageSize = 50,
|
||||
activityFilter,
|
||||
sortfield = null,
|
||||
sortdir = 'asc',
|
||||
freeTextFilter,
|
||||
} = request.query || {};
|
||||
|
||||
// Map result to include runtime status
|
||||
queryResult.result = queryResult.result.map((job) => {
|
||||
return {
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
|
||||
const queryResult = jobStorage.queryJobs({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: request.session.currentUser,
|
||||
isAdmin: isAdmin(request),
|
||||
});
|
||||
|
||||
const isUserAdmin = isAdmin(request);
|
||||
queryResult.result = queryResult.result.map((job) => ({
|
||||
...job,
|
||||
running: isJobRunning(job.id),
|
||||
isOnlyShared:
|
||||
!isUserAdmin &&
|
||||
job.userId !== req.session.currentUser &&
|
||||
job.shared_with_user.includes(req.session.currentUser),
|
||||
};
|
||||
job.userId !== request.session.currentUser &&
|
||||
job.shared_with_user.includes(request.session.currentUser),
|
||||
}));
|
||||
|
||||
return queryResult;
|
||||
});
|
||||
|
||||
res.body = queryResult;
|
||||
res.send();
|
||||
});
|
||||
// Server-Sent Events for real-time job status updates
|
||||
fastify.get('/events', async (request, reply) => {
|
||||
const userId = request.session?.currentUser;
|
||||
if (userId == null) {
|
||||
return reply.code(401).send({ message: 'Unauthorized' });
|
||||
}
|
||||
|
||||
reply.hijack();
|
||||
const raw = reply.raw;
|
||||
raw.setHeader('Content-Type', 'text/event-stream');
|
||||
raw.setHeader('Cache-Control', 'no-cache');
|
||||
raw.setHeader('Connection', 'keep-alive');
|
||||
|
||||
// Server-Sent Events for job status updates
|
||||
jobRouter.get('/events', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
if (userId == null) {
|
||||
res.send({ message: 'Unauthorized' }, 401);
|
||||
return;
|
||||
}
|
||||
// SSE headers
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
try {
|
||||
// Initial comment to establish stream
|
||||
res.write(': connected\n\n');
|
||||
addSseClient(userId, res);
|
||||
// Cleanup on close/aborted
|
||||
const onClose = () => removeClient(userId, res);
|
||||
// restana exposes original req/res; use both close and finish
|
||||
req.on('close', onClose);
|
||||
req.on('aborted', onClose);
|
||||
res.on('close', onClose);
|
||||
} catch (e) {
|
||||
logger.error('Error establishing SSE connection', e);
|
||||
try {
|
||||
res.end();
|
||||
} catch {
|
||||
//noop
|
||||
raw.write(': connected\n\n');
|
||||
addSseClient(userId, raw);
|
||||
const onClose = () => removeClient(userId, raw);
|
||||
request.raw.on('close', onClose);
|
||||
} catch (e) {
|
||||
logger.error('Error establishing SSE connection', e);
|
||||
try {
|
||||
raw.end();
|
||||
} catch {
|
||||
/* noop */
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
jobRouter.post('/startAll', async (req, res) => {
|
||||
try {
|
||||
const userId = req.session.currentUser;
|
||||
// Emit only the userId; handler will decide based on admin/ownership
|
||||
bus.emit('jobs:runAll', { userId });
|
||||
res.send({ message: 'Run all accepted' }, 202);
|
||||
} catch (err) {
|
||||
logger.error('Failed to trigger startAll', err);
|
||||
res.send({ message: 'Unexpected error' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
// Trigger a single job run
|
||||
jobRouter.post('/:jobId/run', async (req, res) => {
|
||||
const { jobId } = req.params;
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) {
|
||||
res.send({ message: 'Job not found' }, 404);
|
||||
return;
|
||||
fastify.post('/startAll', async (request, reply) => {
|
||||
try {
|
||||
const userId = request.session.currentUser;
|
||||
bus.emit('jobs:runAll', { userId });
|
||||
return reply.code(202).send({ message: 'Run all accepted' });
|
||||
} catch (err) {
|
||||
logger.error('Failed to trigger startAll', err);
|
||||
return reply.code(500).send({ message: 'Unexpected error' });
|
||||
}
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send({ message: 'You are trying to run a job that is not associated to your user' }, 403);
|
||||
return;
|
||||
}
|
||||
if (isJobRunning(jobId)) {
|
||||
res.send({ message: 'Job is already running' }, 409);
|
||||
return;
|
||||
}
|
||||
// fire and forget; actual execution handled by index.js listener
|
||||
bus.emit('jobs:runOne', { jobId });
|
||||
res.send({ message: 'Job run accepted' }, 202);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
res.send({ message: 'Unexpected error triggering job' }, 500);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
jobRouter.post('/', async (req, res) => {
|
||||
const {
|
||||
provider,
|
||||
notificationAdapter,
|
||||
name,
|
||||
blacklist = [],
|
||||
jobId,
|
||||
enabled,
|
||||
shareWithUsers = [],
|
||||
spatialFilter = null,
|
||||
} = req.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
let jobFromDb = jobStorage.getJob(jobId);
|
||||
|
||||
if (jobFromDb && !doesJobBelongsToUser(jobFromDb, req)) {
|
||||
res.send(new Error('You are trying to change a job that is not associated to your user.'));
|
||||
return;
|
||||
fastify.post('/:jobId/run', async (request, reply) => {
|
||||
const { jobId } = request.params;
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (!job) {
|
||||
return reply.code(404).send({ message: 'Job not found' });
|
||||
}
|
||||
if (!doesJobBelongsToUser(job, request)) {
|
||||
return reply.code(403).send({ message: 'You are trying to run a job that is not associated to your user' });
|
||||
}
|
||||
if (isJobRunning(jobId)) {
|
||||
return reply.code(409).send({ message: 'Job is already running' });
|
||||
}
|
||||
bus.emit('jobs:runOne', { jobId });
|
||||
return reply.code(202).send({ message: 'Job run accepted' });
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ message: 'Unexpected error triggering job' });
|
||||
}
|
||||
});
|
||||
|
||||
if (settings.demoMode && jobFromDb && jobFromDb.name === DEMO_JOB_NAME) {
|
||||
res.send(new Error('Sorry, but you cannot change the Status of our Demo Job ;)'));
|
||||
return;
|
||||
}
|
||||
|
||||
jobStorage.upsertJob({
|
||||
userId: req.session.currentUser,
|
||||
jobId,
|
||||
enabled,
|
||||
name,
|
||||
blacklist,
|
||||
fastify.post('/', async (request, reply) => {
|
||||
const {
|
||||
provider,
|
||||
notificationAdapter,
|
||||
shareWithUsers,
|
||||
spatialFilter,
|
||||
});
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
name,
|
||||
blacklist = [],
|
||||
jobId,
|
||||
enabled,
|
||||
shareWithUsers = [],
|
||||
spatialFilter = null,
|
||||
specFilter = null,
|
||||
} = request.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const jobFromDb = jobStorage.getJob(jobId);
|
||||
|
||||
jobRouter.delete('', async (req, res) => {
|
||||
const { jobId } = req.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (settings.demoMode && job.name === DEMO_JOB_NAME) {
|
||||
res.send(new Error('Sorry, but you cannot remove the Demo Job ;)'));
|
||||
return;
|
||||
}
|
||||
if (jobFromDb && !doesJobBelongsToUser(jobFromDb, request)) {
|
||||
return reply.code(403).send({ error: 'You are trying to change a job that is not associated to your user.' });
|
||||
}
|
||||
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send(new Error('You are trying to remove a job that is not associated to your user'));
|
||||
} else {
|
||||
jobStorage.removeJob(jobId);
|
||||
}
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
jobRouter.put('/:jobId/status', async (req, res) => {
|
||||
const { status } = req.body;
|
||||
const { jobId } = req.params;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (settings.demoMode && !isAdmin(request) && jobFromDb && jobFromDb.name === DEMO_JOB_NAME) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot change the Status of our Demo Job ;)' });
|
||||
}
|
||||
|
||||
if (settings.demoMode && job.name === DEMO_JOB_NAME) {
|
||||
res.send(new Error('Sorry, but you cannot change the Status of our Demo Job ;)'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!doesJobBelongsToUser(job, req)) {
|
||||
res.send(new Error('You are trying change a job that is not associated to your user'));
|
||||
} else {
|
||||
jobStorage.setJobStatus({
|
||||
jobStorage.upsertJob({
|
||||
userId: request.session.currentUser,
|
||||
jobId,
|
||||
status,
|
||||
enabled,
|
||||
name,
|
||||
blacklist,
|
||||
provider,
|
||||
notificationAdapter,
|
||||
shareWithUsers,
|
||||
spatialFilter,
|
||||
specFilter,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
return reply.send();
|
||||
});
|
||||
|
||||
jobRouter.get('/shareableUserList', async (req, res) => {
|
||||
const currentUser = req.session.currentUser;
|
||||
const users = userStorage.getUsers(false);
|
||||
res.body = users
|
||||
.filter((user) => !user.isAdmin && user.id !== currentUser)
|
||||
.map((user) => ({
|
||||
id: user.id,
|
||||
name: user.username,
|
||||
}));
|
||||
res.send();
|
||||
});
|
||||
export { jobRouter };
|
||||
fastify.delete('/', async (request, reply) => {
|
||||
const { jobId } = request.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove the Demo Job ;)' });
|
||||
}
|
||||
|
||||
if (!doesJobBelongsToUser(job, request)) {
|
||||
return reply.code(403).send({ error: 'You are trying to remove a job that is not associated to your user' });
|
||||
}
|
||||
jobStorage.removeJob(jobId);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
return reply.send();
|
||||
});
|
||||
|
||||
fastify.put('/:jobId/status', async (request, reply) => {
|
||||
const { status } = request.body;
|
||||
const { jobId } = request.params;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
const job = jobStorage.getJob(jobId);
|
||||
|
||||
if (settings.demoMode && !isAdmin(request) && job.name === DEMO_JOB_NAME) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot change the Status of our Demo Job ;)' });
|
||||
}
|
||||
|
||||
if (!doesJobBelongsToUser(job, request)) {
|
||||
return reply.code(403).send({ error: 'You are trying change a job that is not associated to your user' });
|
||||
}
|
||||
jobStorage.setJobStatus({ jobId, status });
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
return reply.send();
|
||||
});
|
||||
|
||||
fastify.get('/shareableUserList', async (request) => {
|
||||
const currentUser = request.session.currentUser;
|
||||
const users = userStorage.getUsers(false);
|
||||
return users
|
||||
.filter((user) => !user.isAdmin && user.id !== currentUser)
|
||||
.map((user) => ({
|
||||
id: user.id,
|
||||
name: user.username,
|
||||
}));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as listingStorage from '../../services/storage/listingsStorage.js';
|
||||
import * as watchListStorage from '../../services/storage/watchListStorage.js';
|
||||
import { isAdmin as isAdminFn } from '../security.js';
|
||||
@@ -12,128 +11,114 @@ import { nullOrEmpty } from '../../utils.js';
|
||||
import { getJobs } from '../../services/storage/jobStorage.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
|
||||
const service = restana();
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function listingsPlugin(fastify) {
|
||||
fastify.get('/table', async (request) => {
|
||||
const {
|
||||
page,
|
||||
pageSize = 50,
|
||||
activityFilter,
|
||||
jobNameFilter,
|
||||
providerFilter,
|
||||
watchListFilter,
|
||||
sortfield = null,
|
||||
sortdir = 'asc',
|
||||
freeTextFilter,
|
||||
} = request.query || {};
|
||||
|
||||
const listingsRouter = service.newRouter();
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
const normalizedWatch = toBool(watchListFilter);
|
||||
|
||||
listingsRouter.get('/table', async (req, res) => {
|
||||
const {
|
||||
page,
|
||||
pageSize = 50,
|
||||
activityFilter,
|
||||
jobNameFilter,
|
||||
providerFilter,
|
||||
watchListFilter,
|
||||
sortfield = null,
|
||||
sortdir = 'asc',
|
||||
freeTextFilter,
|
||||
} = req.query || {};
|
||||
let jobFilter = null;
|
||||
let jobIdFilter = null;
|
||||
const jobs = getJobs();
|
||||
if (!nullOrEmpty(jobNameFilter)) {
|
||||
const job = jobs.find((j) => j.id === jobNameFilter);
|
||||
jobFilter = job != null ? job.name : null;
|
||||
jobIdFilter = job != null ? job.id : null;
|
||||
}
|
||||
|
||||
// normalize booleans (accept true, 'true', 1, '1' for true; false, 'false', 0, '0' for false)
|
||||
const toBool = (v) => {
|
||||
if (v === true || v === 'true' || v === 1 || v === '1') return true;
|
||||
if (v === false || v === 'false' || v === 0 || v === '0') return false;
|
||||
return null;
|
||||
};
|
||||
const normalizedActivity = toBool(activityFilter);
|
||||
const normalizedWatch = toBool(watchListFilter);
|
||||
|
||||
let jobFilter = null;
|
||||
let jobIdFilter = null;
|
||||
const jobs = getJobs();
|
||||
if (!nullOrEmpty(jobNameFilter)) {
|
||||
const job = jobs.find((j) => j.id === jobNameFilter);
|
||||
jobFilter = job != null ? job.name : null;
|
||||
jobIdFilter = job != null ? job.id : null;
|
||||
}
|
||||
|
||||
res.body = listingStorage.queryListings({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
jobNameFilter: jobFilter,
|
||||
jobIdFilter: jobIdFilter,
|
||||
providerFilter,
|
||||
watchListFilter: normalizedWatch,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdminFn(req),
|
||||
return listingStorage.queryListings({
|
||||
page: page ? parseInt(page, 10) : 1,
|
||||
pageSize: pageSize ? parseInt(pageSize, 10) : 50,
|
||||
freeTextFilter: freeTextFilter || null,
|
||||
activityFilter: normalizedActivity,
|
||||
jobNameFilter: jobFilter,
|
||||
jobIdFilter: jobIdFilter,
|
||||
providerFilter,
|
||||
watchListFilter: normalizedWatch,
|
||||
sortField: sortfield || null,
|
||||
sortDir: sortdir === 'desc' ? 'desc' : 'asc',
|
||||
userId: request.session.currentUser,
|
||||
isAdmin: isAdminFn(request),
|
||||
});
|
||||
});
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.get('/map', async (req, res) => {
|
||||
const { jobId } = req.query || {};
|
||||
|
||||
res.body = listingStorage.getListingsForMap({
|
||||
jobId: nullOrEmpty(jobId) ? null : jobId,
|
||||
userId: req.session.currentUser,
|
||||
isAdmin: isAdminFn(req),
|
||||
fastify.get('/map', async (request) => {
|
||||
const { jobId } = request.query || {};
|
||||
return listingStorage.getListingsForMap({
|
||||
jobId: nullOrEmpty(jobId) ? null : jobId,
|
||||
userId: request.session.currentUser,
|
||||
isAdmin: isAdminFn(request),
|
||||
});
|
||||
});
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.get('/:listingId', async (req, res) => {
|
||||
const { listingId } = req.params;
|
||||
const listing = listingStorage.getListingById(listingId, req.session.currentUser, isAdminFn(req));
|
||||
if (!listing) {
|
||||
res.statusCode = 404;
|
||||
res.body = { message: 'Listing not found' };
|
||||
return res.send();
|
||||
}
|
||||
res.body = listing;
|
||||
res.send();
|
||||
});
|
||||
|
||||
// Toggle watch state for the current user on a listing
|
||||
listingsRouter.post('/watch', async (req, res) => {
|
||||
try {
|
||||
const { listingId } = req.body || {};
|
||||
const userId = req.session?.currentUser;
|
||||
if (!listingId || !userId) {
|
||||
res.statusCode = 400;
|
||||
res.body = { message: 'listingId or user not provided' };
|
||||
return res.send();
|
||||
fastify.get('/:listingId', async (request, reply) => {
|
||||
const { listingId } = request.params;
|
||||
const listing = listingStorage.getListingById(listingId, request.session.currentUser, isAdminFn(request));
|
||||
if (!listing) {
|
||||
return reply.code(404).send({ message: 'Listing not found' });
|
||||
}
|
||||
watchListStorage.toggleWatch(listingId, userId);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
res.statusCode = 500;
|
||||
res.body = { message: 'Failed to toggle watch' };
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
return listing;
|
||||
});
|
||||
|
||||
listingsRouter.delete('/job', async (req, res) => {
|
||||
const { jobId, hardDelete = false } = req.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
if (settings.demoMode) {
|
||||
res.send(new Error('Sorry, but you cannot remove listings in demo mode ;)'));
|
||||
return;
|
||||
fastify.post('/watch', async (request, reply) => {
|
||||
try {
|
||||
const { listingId } = request.body || {};
|
||||
const userId = request.session?.currentUser;
|
||||
if (!listingId || !userId) {
|
||||
return reply.code(400).send({ message: 'listingId or user not provided' });
|
||||
}
|
||||
watchListStorage.toggleWatch(listingId, userId);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ message: 'Failed to toggle watch' });
|
||||
}
|
||||
return reply.send();
|
||||
});
|
||||
|
||||
listingStorage.deleteListingsByJobId(jobId, hardDelete);
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
|
||||
listingsRouter.delete('/', async (req, res) => {
|
||||
const { ids, hardDelete = false } = req.body;
|
||||
try {
|
||||
if (Array.isArray(ids) && ids.length > 0) {
|
||||
listingStorage.deleteListingsById(ids, hardDelete);
|
||||
fastify.delete('/job', async (request, reply) => {
|
||||
const { jobId, hardDelete = false } = request.body;
|
||||
const settings = await getSettings();
|
||||
try {
|
||||
if (settings.demoMode && !isAdminFn(request)) {
|
||||
return reply.code(403).send({ error: 'Sorry, but you cannot remove listings in demo mode ;)' });
|
||||
}
|
||||
listingStorage.deleteListingsByJobId(jobId, hardDelete);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
} catch (error) {
|
||||
res.send(new Error(error));
|
||||
logger.error(error);
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
return reply.send();
|
||||
});
|
||||
|
||||
export { listingsRouter };
|
||||
fastify.delete('/', async (request, reply) => {
|
||||
const { ids, hardDelete = false } = request.body;
|
||||
try {
|
||||
if (Array.isArray(ids) && ids.length > 0) {
|
||||
listingStorage.deleteListingsById(ids, hardDelete);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
return reply.send();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import * as hasher from '../../services/security/hash.js';
|
||||
import { trackDemoAccessed } from '../../services/tracking/Tracker.js';
|
||||
@@ -11,12 +10,12 @@ import logger from '../../services/logger.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
|
||||
const MAX_LOGIN_ATTEMPTS = 10;
|
||||
const LOGIN_WINDOW_MS = 15 * 60 * 1000; // 15 minutes
|
||||
const loginAttempts = new Map(); // ip -> { count, firstAttempt }
|
||||
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
|
||||
const loginAttempts = new Map();
|
||||
|
||||
function getClientIp(req) {
|
||||
const forwarded = req.headers['x-forwarded-for'];
|
||||
return (forwarded ? forwarded.split(',')[0] : req.socket?.remoteAddress) || 'unknown';
|
||||
function getClientIp(request) {
|
||||
const forwarded = request.headers['x-forwarded-for'];
|
||||
return (forwarded ? forwarded.split(',')[0] : request.socket?.remoteAddress) || 'unknown';
|
||||
}
|
||||
|
||||
function isRateLimited(ip) {
|
||||
@@ -30,53 +29,51 @@ function isRateLimited(ip) {
|
||||
return record.count > MAX_LOGIN_ATTEMPTS;
|
||||
}
|
||||
|
||||
const service = restana();
|
||||
const loginRouter = service.newRouter();
|
||||
loginRouter.get('/user', async (req, res) => {
|
||||
const currentUserId = req.session.currentUser;
|
||||
const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
|
||||
if (currentUser == null) {
|
||||
res.body = {};
|
||||
} else {
|
||||
res.body = {
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function loginPlugin(fastify) {
|
||||
fastify.get('/user', async (request) => {
|
||||
const currentUserId = request.session?.currentUser;
|
||||
const currentUser = currentUserId == null ? null : userStorage.getUser(currentUserId);
|
||||
if (currentUser == null) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
userId: currentUser.id,
|
||||
isAdmin: currentUser.isAdmin,
|
||||
};
|
||||
}
|
||||
res.send();
|
||||
});
|
||||
loginRouter.post('/', async (req, res) => {
|
||||
const ip = getClientIp(req);
|
||||
if (isRateLimited(ip)) {
|
||||
logger.error(`Login rate limit exceeded for IP ${ip}`);
|
||||
res.send(429);
|
||||
return;
|
||||
}
|
||||
const settings = await getSettings();
|
||||
const { username, password } = req.body;
|
||||
const user = userStorage.getUsers(true).find((user) => user.username === username);
|
||||
if (user == null) {
|
||||
res.send(401);
|
||||
return;
|
||||
}
|
||||
if (user.password === hasher.hash(password)) {
|
||||
if (settings.demoMode) {
|
||||
await trackDemoAccessed();
|
||||
}
|
||||
});
|
||||
|
||||
req.session.currentUser = user.id;
|
||||
req.session.createdAt = Date.now();
|
||||
loginAttempts.delete(ip);
|
||||
userStorage.setLastLoginToNow({ userId: user.id });
|
||||
res.send(200);
|
||||
return;
|
||||
} else {
|
||||
logger.error(`User ${username} tried to login, but password was wrong.`);
|
||||
}
|
||||
res.send(401);
|
||||
});
|
||||
loginRouter.post('/logout', async (req, res) => {
|
||||
req.session = null;
|
||||
res.send(200);
|
||||
});
|
||||
export { loginRouter };
|
||||
fastify.post('/', async (request, reply) => {
|
||||
const ip = getClientIp(request);
|
||||
if (isRateLimited(ip)) {
|
||||
logger.error(`Login rate limit exceeded for IP ${ip}`);
|
||||
return reply.code(429).send();
|
||||
}
|
||||
const settings = await getSettings();
|
||||
const { username, password } = request.body;
|
||||
const user = userStorage.getUsers(true).find((u) => u.username === username);
|
||||
if (user == null) {
|
||||
return reply.code(401).send();
|
||||
}
|
||||
if (user.password === hasher.hash(password)) {
|
||||
if (settings.demoMode) {
|
||||
await trackDemoAccessed();
|
||||
}
|
||||
request.session.currentUser = user.id;
|
||||
request.session.createdAt = Date.now();
|
||||
loginAttempts.delete(ip);
|
||||
userStorage.setLastLoginToNow({ userId: user.id });
|
||||
return reply.code(200).send();
|
||||
} else {
|
||||
logger.error(`User ${username} tried to login, but password was wrong.`);
|
||||
}
|
||||
return reply.code(401).send();
|
||||
});
|
||||
|
||||
fastify.post('/logout', async (request, reply) => {
|
||||
await request.session.destroy();
|
||||
return reply.code(200).send();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4,62 +4,64 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import restana from 'restana';
|
||||
import logger from '../../services/logger.js';
|
||||
|
||||
const service = restana();
|
||||
const notificationAdapterRouter = service.newRouter();
|
||||
const notificationAdapterList = fs.readdirSync('./lib//notification/adapter').filter((file) => file.endsWith('.js'));
|
||||
const notificationAdapter = await Promise.all(
|
||||
notificationAdapterList.map(async (pro) => {
|
||||
return await import(`../../notification/adapter/${pro}`);
|
||||
}),
|
||||
);
|
||||
notificationAdapterRouter.post('/try', async (req, res) => {
|
||||
const { id, fields } = req.body;
|
||||
const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
|
||||
if (adapter == null) {
|
||||
res.send(404);
|
||||
}
|
||||
const notificationConfig = [];
|
||||
const notificationObject = {};
|
||||
Object.keys(fields).forEach((key) => {
|
||||
notificationObject[key] = fields[key].value;
|
||||
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function notificationAdapterPlugin(fastify) {
|
||||
fastify.get('/', async () => {
|
||||
return notificationAdapter.map((adapter) => adapter.config);
|
||||
});
|
||||
notificationConfig.push({
|
||||
fields: { ...notificationObject },
|
||||
enabled: true,
|
||||
id,
|
||||
});
|
||||
try {
|
||||
await adapter.send({
|
||||
serviceName: 'TestCall',
|
||||
newListings: [
|
||||
{
|
||||
address: 'Heidestrasse 17, 51147 Köln',
|
||||
description: exampleDescription,
|
||||
id: '1',
|
||||
imageUrl: 'https://placehold.co/600x400/png',
|
||||
price: '1.000 €',
|
||||
size: '76 m²',
|
||||
title: 'Stilvolle gepflegte 3-Raum-Wohnung mit gehobener Innenausstattung',
|
||||
url: 'https://www.orange-coding.net',
|
||||
},
|
||||
],
|
||||
notificationConfig,
|
||||
jobKey: 'TestJob',
|
||||
|
||||
fastify.post('/try', async (request, reply) => {
|
||||
const { id, fields } = request.body;
|
||||
const adapter = notificationAdapter.find((adapter) => adapter.config.id === id);
|
||||
if (adapter == null) {
|
||||
return reply.code(404).send();
|
||||
}
|
||||
const notificationConfig = [];
|
||||
const notificationObject = {};
|
||||
Object.keys(fields).forEach((key) => {
|
||||
notificationObject[key] = fields[key].value;
|
||||
});
|
||||
res.send();
|
||||
} catch (Exception) {
|
||||
logger.error('Error during notification adapter test:', Exception);
|
||||
res.send(new Error(Exception));
|
||||
}
|
||||
});
|
||||
notificationAdapterRouter.get('/', async (req, res) => {
|
||||
res.body = notificationAdapter.map((adapter) => adapter.config);
|
||||
res.send();
|
||||
});
|
||||
export { notificationAdapterRouter };
|
||||
notificationConfig.push({
|
||||
fields: { ...notificationObject },
|
||||
enabled: true,
|
||||
id,
|
||||
});
|
||||
try {
|
||||
await adapter.send({
|
||||
serviceName: 'TestCall',
|
||||
newListings: [
|
||||
{
|
||||
address: 'Heidestrasse 17, 51147 Köln',
|
||||
description: exampleDescription,
|
||||
id: '1',
|
||||
imageUrl: 'https://placehold.co/600x400/png',
|
||||
price: '1.000 €',
|
||||
size: '76 m²',
|
||||
title: 'Stilvolle gepflegte 3-Raum-Wohnung mit gehobener Innenausstattung',
|
||||
url: 'https://www.orange-coding.net',
|
||||
},
|
||||
],
|
||||
notificationConfig,
|
||||
jobKey: 'TestJob',
|
||||
});
|
||||
return reply.send();
|
||||
} catch (Exception) {
|
||||
logger.error('Error during notification adapter test:', Exception);
|
||||
return reply.code(500).send({ error: String(Exception) });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const exampleDescription = `
|
||||
Wohnungstyp: Etagenwohnung
|
||||
@@ -94,7 +96,7 @@ Die Wohnung ist ideal für Paare oder kleine Familien geeignet.
|
||||
Ausstattung:
|
||||
- neuer hochwertiger Bodenbelag (Holzoptik) in allen Räumen außer Bad/Küche
|
||||
- sonniger Balkon (Süd)
|
||||
- Tiefgaragenstellplatz
|
||||
- Tiefgaragenstellplatz
|
||||
- Kellerabteil
|
||||
- gepflegtes Mehrfamilienhaus
|
||||
|
||||
@@ -104,7 +106,7 @@ Vermietung direkt vom Eigentümer - provisionsfrei!
|
||||
|
||||
Lage:
|
||||
• Park: 1 Minute zu Fuß
|
||||
• S-Bahn Station: 2 Minuten zu Fuß
|
||||
• S-Bahn Station: 2 Minuten zu Fuß
|
||||
• Supermärkte, Restaurants, täglicher Bedarf in der Nähe
|
||||
• Gute Anbindung Richtung Großstadt und Flughafen
|
||||
`;
|
||||
|
||||
@@ -4,17 +4,15 @@
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import restana from 'restana';
|
||||
const service = restana();
|
||||
const providerRouter = service.newRouter();
|
||||
|
||||
const providerList = fs.readdirSync('./lib/provider').filter((file) => file.endsWith('.js'));
|
||||
const provider = await Promise.all(
|
||||
providerList.map(async (pro) => {
|
||||
return await import(`../../provider/${pro}`);
|
||||
}),
|
||||
);
|
||||
providerRouter.get('/', async (req, res) => {
|
||||
res.body = provider.map((p) => p.metaInformation);
|
||||
res.send();
|
||||
});
|
||||
export { providerRouter };
|
||||
const providers = await Promise.all(providerList.map(async (pro) => import(`../../provider/${pro}`)));
|
||||
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function providerPlugin(fastify) {
|
||||
fastify.get('/', async () => {
|
||||
return providers.map((p) => p.metaInformation);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,35 +3,29 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import { trackPoi } from '../../services/tracking/Tracker.js';
|
||||
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||
import logger from '../../services/logger.js';
|
||||
|
||||
const service = restana();
|
||||
const trackingRouter = service.newRouter();
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function trackingPlugin(fastify) {
|
||||
fastify.get('/trackingPois', async () => {
|
||||
return TRACKING_POIS;
|
||||
});
|
||||
|
||||
trackingRouter.get('/trackingPois', async (req, res) => {
|
||||
res.body = TRACKING_POIS;
|
||||
res.send();
|
||||
});
|
||||
|
||||
trackingRouter.post('/poi', async (req, res) => {
|
||||
const { poi } = req.body;
|
||||
if (!poi) {
|
||||
res.statusCode = 400;
|
||||
res.send({ error: 'Feature name is required' });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await trackPoi(poi);
|
||||
res.send({ success: true });
|
||||
} catch (error) {
|
||||
logger.error('Error tracking feature', error);
|
||||
res.statusCode = 500;
|
||||
res.send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
export { trackingRouter };
|
||||
fastify.post('/poi', async (request, reply) => {
|
||||
const { poi } = request.body;
|
||||
if (!poi) {
|
||||
return reply.code(400).send({ error: 'Feature name is required' });
|
||||
}
|
||||
try {
|
||||
await trackPoi(poi);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('Error tracking feature', error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,81 +3,73 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import * as userStorage from '../../services/storage/userStorage.js';
|
||||
import * as jobStorage from '../../services/storage/jobStorage.js';
|
||||
import { getSettings } from '../../services/storage/settingsStorage.js';
|
||||
const service = restana();
|
||||
const userRouter = service.newRouter();
|
||||
import { isAdmin as isAdminUser } from '../security.js';
|
||||
|
||||
function checkIfAnyAdminAfterRemovingUser(userIdToBeRemoved, allUser) {
|
||||
return allUser.filter((user) => user.id !== userIdToBeRemoved && user.isAdmin).length > 0;
|
||||
}
|
||||
function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, req) {
|
||||
return req.session.currentUser === userIdToBeRemoved;
|
||||
|
||||
function checkIfUserToBeRemovedIsLoggedIn(userIdToBeRemoved, request) {
|
||||
return request.session.currentUser === userIdToBeRemoved;
|
||||
}
|
||||
|
||||
const nullOrEmpty = (str) => str == null || str.length === 0;
|
||||
|
||||
userRouter.get('/', async (req, res) => {
|
||||
res.body = userStorage.getUsers(false);
|
||||
res.send();
|
||||
});
|
||||
|
||||
userRouter.get('/:userId', async (req, res) => {
|
||||
const { userId } = req.params;
|
||||
res.body = userStorage.getUser(userId);
|
||||
res.send();
|
||||
});
|
||||
userRouter.delete('/', async (req, res) => {
|
||||
const settings = await getSettings();
|
||||
if (settings.demoMode) {
|
||||
res.send(new Error('In demo mode, it is not allowed to remove user.'));
|
||||
return;
|
||||
}
|
||||
|
||||
const { userId } = req.body;
|
||||
const allUser = userStorage.getUsers(false);
|
||||
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||
res.send(new Error('You are trying to remove the last admin user. This is prohibited.'));
|
||||
return;
|
||||
}
|
||||
if (checkIfUserToBeRemovedIsLoggedIn(userId, req)) {
|
||||
res.send(new Error('You are trying to remove yourself. This is prohibited.'));
|
||||
return;
|
||||
}
|
||||
//TODO: Remove also analytics
|
||||
jobStorage.removeJobsByUserId(userId);
|
||||
userStorage.removeUser(userId);
|
||||
res.send();
|
||||
});
|
||||
userRouter.post('/', async (req, res) => {
|
||||
const settings = await getSettings();
|
||||
if (settings.demoMode) {
|
||||
res.send(new Error('In demo mode, it is not allowed to change or add user.'));
|
||||
return;
|
||||
}
|
||||
|
||||
const { username, password, password2, isAdmin, userId } = req.body;
|
||||
if (password !== password2) {
|
||||
res.send(new Error('Passwords does not match'));
|
||||
return;
|
||||
}
|
||||
if (nullOrEmpty(username) || nullOrEmpty(password) || nullOrEmpty(password2)) {
|
||||
res.send(new Error('Username and password are mandatory.'));
|
||||
return;
|
||||
}
|
||||
const allUser = userStorage.getUsers(false);
|
||||
if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||
res.send(
|
||||
new Error('You cannot change the admin flag for this user as otherwise, there is no other user in the system'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
userStorage.upsertUser({
|
||||
userId,
|
||||
username,
|
||||
password,
|
||||
isAdmin,
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function userPlugin(fastify) {
|
||||
fastify.get('/', async () => {
|
||||
return userStorage.getUsers(false);
|
||||
});
|
||||
res.send();
|
||||
});
|
||||
export { userRouter };
|
||||
|
||||
fastify.get('/:userId', async (request) => {
|
||||
const { userId } = request.params;
|
||||
return userStorage.getUser(userId);
|
||||
});
|
||||
|
||||
fastify.delete('/', async (request, reply) => {
|
||||
const settings = await getSettings();
|
||||
if (settings.demoMode && !isAdminUser(request)) {
|
||||
return reply.code(403).send({ error: 'In demo mode, it is not allowed to remove user.' });
|
||||
}
|
||||
|
||||
const { userId } = request.body;
|
||||
const allUser = userStorage.getUsers(false);
|
||||
if (!checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||
return reply.code(400).send({ error: 'You are trying to remove the last admin user. This is prohibited.' });
|
||||
}
|
||||
if (checkIfUserToBeRemovedIsLoggedIn(userId, request)) {
|
||||
return reply.code(400).send({ error: 'You are trying to remove yourself. This is prohibited.' });
|
||||
}
|
||||
jobStorage.removeJobsByUserId(userId);
|
||||
userStorage.removeUser(userId);
|
||||
return reply.send();
|
||||
});
|
||||
|
||||
fastify.post('/', async (request, reply) => {
|
||||
const settings = await getSettings();
|
||||
if (settings.demoMode && !isAdminUser(request)) {
|
||||
return reply.code(403).send({ error: 'In demo mode, it is not allowed to change or add user.' });
|
||||
}
|
||||
|
||||
const { username, password, password2, isAdmin, userId } = request.body;
|
||||
if (password !== password2) {
|
||||
return reply.code(400).send({ error: 'Passwords do not match.' });
|
||||
}
|
||||
if (nullOrEmpty(username) || nullOrEmpty(password) || nullOrEmpty(password2)) {
|
||||
return reply.code(400).send({ error: 'Username and password are mandatory.' });
|
||||
}
|
||||
const allUser = userStorage.getUsers(false);
|
||||
if (!isAdmin && !checkIfAnyAdminAfterRemovingUser(userId, allUser)) {
|
||||
return reply.code(400).send({
|
||||
error: 'You cannot change the admin flag for this user as otherwise, there is no other user in the system',
|
||||
});
|
||||
}
|
||||
userStorage.upsertUser({ userId, username, password, isAdmin });
|
||||
return reply.send();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import SqliteConnection from '../../services/storage/SqliteConnection.js';
|
||||
import { getSettings, upsertSettings } from '../../services/storage/settingsStorage.js';
|
||||
import { isAdmin } from '../security.js';
|
||||
import { resetGeocoordinatesAndDistanceForUser } from '../../services/storage/listingsStorage.js';
|
||||
import { geocodeAddress } from '../../services/geocoding/geoCodingService.js';
|
||||
import { autocompleteAddress } from '../../services/geocoding/autocompleteService.js';
|
||||
@@ -15,113 +15,140 @@ import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||
import logger from '../../services/logger.js';
|
||||
import { runGeoCordTask } from '../../services/crons/geocoding-cron.js';
|
||||
|
||||
const service = restana();
|
||||
const userSettingsRouter = service.newRouter();
|
||||
|
||||
userSettingsRouter.get('/', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
|
||||
const settings = {};
|
||||
for (const r of rows) {
|
||||
settings[r.name] = fromJson(r.value, null);
|
||||
}
|
||||
res.body = settings;
|
||||
res.send();
|
||||
});
|
||||
|
||||
userSettingsRouter.get('/autocomplete', async (req, res) => {
|
||||
const { q } = req.query;
|
||||
try {
|
||||
const results = await autocompleteAddress(q);
|
||||
res.body = results;
|
||||
res.send();
|
||||
} catch (error) {
|
||||
res.statusCode = 500;
|
||||
res.send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
userSettingsRouter.post('/home-address', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
const { home_address } = req.body;
|
||||
const settings = await getSettings();
|
||||
|
||||
if (settings.demoMode) {
|
||||
res.send(new Error('In demo mode, it is not allowed to change the home address.'));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (home_address) {
|
||||
await trackPoi(TRACKING_POIS.DISTANCE_ADDRESS_ENTERED);
|
||||
const coords = await geocodeAddress(home_address);
|
||||
if (coords && coords.lat !== -1) {
|
||||
upsertSettings({ home_address: { address: home_address, coords } }, userId);
|
||||
resetGeocoordinatesAndDistanceForUser(userId);
|
||||
//we do NOT wait for this to finish, as we don't want to block the response
|
||||
runGeoCordTask();
|
||||
res.send({ success: true, coords });
|
||||
} else {
|
||||
res.statusCode = 400;
|
||||
res.send({ error: 'Could not geocode address' });
|
||||
}
|
||||
} else {
|
||||
upsertSettings({ home_address: null }, userId);
|
||||
res.send({ success: true });
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function userSettingsPlugin(fastify) {
|
||||
fastify.get('/', async (request) => {
|
||||
const userId = request.session.currentUser;
|
||||
const rows = SqliteConnection.query('SELECT name, value FROM settings WHERE user_id = @userId', { userId });
|
||||
const settings = {};
|
||||
for (const r of rows) {
|
||||
settings[r.name] = fromJson(r.value, null);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error updating home address settings', error);
|
||||
res.statusCode = 500;
|
||||
res.send({ error: error.message });
|
||||
}
|
||||
});
|
||||
return settings;
|
||||
});
|
||||
|
||||
userSettingsRouter.post('/news-hash', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
const { news_hash } = req.body;
|
||||
fastify.get('/autocomplete', async (request, reply) => {
|
||||
const { q } = request.query;
|
||||
try {
|
||||
const results = await autocompleteAddress(q);
|
||||
return results;
|
||||
} catch (error) {
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
const globalSettings = await getSettings();
|
||||
if (globalSettings.demoMode) {
|
||||
res.statusCode = 403;
|
||||
res.send({ error: 'In demo mode, it is not allowed to change settings.' });
|
||||
return;
|
||||
}
|
||||
fastify.post('/home-address', async (request, reply) => {
|
||||
const userId = request.session.currentUser;
|
||||
const { home_address } = request.body;
|
||||
const settings = await getSettings();
|
||||
|
||||
try {
|
||||
upsertSettings({ news_hash }, userId);
|
||||
res.send({ success: true });
|
||||
} catch (error) {
|
||||
logger.error('Error updating news hash', error);
|
||||
res.statusCode = 500;
|
||||
res.send({ error: error.message });
|
||||
}
|
||||
});
|
||||
if (settings.demoMode && !isAdmin(request)) {
|
||||
return reply.code(403).send({ error: 'In demo mode, it is not allowed to change the home address.' });
|
||||
}
|
||||
|
||||
userSettingsRouter.post('/provider-details', async (req, res) => {
|
||||
const userId = req.session.currentUser;
|
||||
const { provider_details } = req.body;
|
||||
try {
|
||||
if (home_address) {
|
||||
await trackPoi(TRACKING_POIS.DISTANCE_ADDRESS_ENTERED);
|
||||
const coords = await geocodeAddress(home_address);
|
||||
if (coords && coords.lat !== -1) {
|
||||
upsertSettings({ home_address: { address: home_address, coords } }, userId);
|
||||
resetGeocoordinatesAndDistanceForUser(userId);
|
||||
runGeoCordTask();
|
||||
return { success: true, coords };
|
||||
} else {
|
||||
return reply.code(400).send({ error: 'Could not geocode address' });
|
||||
}
|
||||
} else {
|
||||
upsertSettings({ home_address: null }, userId);
|
||||
return { success: true };
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error updating home address settings', error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
const globalSettings = await getSettings();
|
||||
if (globalSettings.demoMode) {
|
||||
res.statusCode = 403;
|
||||
res.send({ error: 'In demo mode, it is not allowed to change settings.' });
|
||||
return;
|
||||
}
|
||||
fastify.post('/news-hash', async (request, reply) => {
|
||||
const userId = request.session.currentUser;
|
||||
const { news_hash } = request.body;
|
||||
|
||||
if (!Array.isArray(provider_details)) {
|
||||
res.statusCode = 400;
|
||||
res.send({ error: 'provider_details must be an array of provider ids.' });
|
||||
return;
|
||||
}
|
||||
const globalSettings = await getSettings();
|
||||
if (globalSettings.demoMode && !isAdmin(request)) {
|
||||
return reply.code(403).send({ error: 'In demo mode, it is not allowed to change settings.' });
|
||||
}
|
||||
|
||||
try {
|
||||
upsertSettings({ provider_details }, userId);
|
||||
res.send({ success: true });
|
||||
} catch (error) {
|
||||
logger.error('Error updating provider details setting', error);
|
||||
res.statusCode = 500;
|
||||
res.send({ error: error.message });
|
||||
}
|
||||
});
|
||||
try {
|
||||
upsertSettings({ news_hash }, userId);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('Error updating news hash', error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
export { userSettingsRouter };
|
||||
fastify.post('/provider-details', async (request, reply) => {
|
||||
const userId = request.session.currentUser;
|
||||
const { provider_details } = request.body;
|
||||
|
||||
const globalSettings = await getSettings();
|
||||
if (globalSettings.demoMode && !isAdmin(request)) {
|
||||
return reply.code(403).send({ error: 'In demo mode, it is not allowed to change settings.' });
|
||||
}
|
||||
|
||||
if (!Array.isArray(provider_details)) {
|
||||
return reply.code(400).send({ error: 'provider_details must be an array of provider ids.' });
|
||||
}
|
||||
|
||||
try {
|
||||
upsertSettings({ provider_details }, userId);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('Error updating provider details setting', error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
fastify.post('/listings-view-mode', async (request, reply) => {
|
||||
const userId = request.session.currentUser;
|
||||
const { listings_view_mode } = request.body;
|
||||
|
||||
if (listings_view_mode !== 'grid' && listings_view_mode !== 'table') {
|
||||
return reply.code(400).send({ error: 'listings_view_mode must be "grid" or "table".' });
|
||||
}
|
||||
|
||||
if (listings_view_mode === 'table') {
|
||||
await trackPoi(TRACKING_POIS.LISTING_TABLE_VIEW);
|
||||
}
|
||||
|
||||
try {
|
||||
upsertSettings({ listings_view_mode }, userId);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('Error updating listings view mode setting', error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
fastify.post('/jobs-view-mode', async (request, reply) => {
|
||||
const userId = request.session.currentUser;
|
||||
const { jobs_view_mode } = request.body;
|
||||
|
||||
if (jobs_view_mode !== 'grid' && jobs_view_mode !== 'table') {
|
||||
return reply.code(400).send({ error: 'jobs_view_mode must be "grid" or "table".' });
|
||||
}
|
||||
|
||||
if (jobs_view_mode === 'table') {
|
||||
await trackPoi(TRACKING_POIS.JOBS_TABLE_VIEW);
|
||||
}
|
||||
|
||||
try {
|
||||
upsertSettings({ jobs_view_mode }, userId);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('Error updating jobs view mode setting', error);
|
||||
return reply.code(500).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,27 +3,10 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import restana from 'restana';
|
||||
import fetch from 'node-fetch';
|
||||
import { getPackageVersion } from '../../utils.js';
|
||||
import semver from 'semver';
|
||||
|
||||
const service = restana();
|
||||
const versionRouter = service.newRouter();
|
||||
|
||||
versionRouter.get('/', async (req, res) => {
|
||||
const versionPayload = await getCurrentVersionFromGithub();
|
||||
const localFredyVersion = await getPackageVersion();
|
||||
res.body =
|
||||
versionPayload == null
|
||||
? {
|
||||
newVersion: false,
|
||||
localFredyVersion,
|
||||
}
|
||||
: versionPayload;
|
||||
res.send();
|
||||
});
|
||||
|
||||
async function getCurrentVersionFromGithub() {
|
||||
const raw = await fetch('https://api.github.com/repos/orangecoding/fredy/releases/latest');
|
||||
const data = await raw.json();
|
||||
@@ -40,4 +23,13 @@ async function getCurrentVersionFromGithub() {
|
||||
};
|
||||
}
|
||||
|
||||
export { versionRouter };
|
||||
/**
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export default async function versionPlugin(fastify) {
|
||||
fastify.get('/', async () => {
|
||||
const versionPayload = await getCurrentVersionFromGithub();
|
||||
const localFredyVersion = await getPackageVersion();
|
||||
return versionPayload ?? { newVersion: false, localFredyVersion };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4,53 +4,50 @@
|
||||
*/
|
||||
|
||||
import * as userStorage from '../services/storage/userStorage.js';
|
||||
import cookieSession from 'cookie-session';
|
||||
|
||||
const SESSION_MAX_AGE = 2 * 60 * 60 * 1000; // 2 hours
|
||||
const unauthorized = (res) => {
|
||||
return res.send(401);
|
||||
};
|
||||
const isUnauthorized = (req) => {
|
||||
if (req.session.currentUser == null) return true;
|
||||
if (Date.now() - req.session.createdAt > SESSION_MAX_AGE) {
|
||||
req.session = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when the request has no valid, non-expired session.
|
||||
* @param {import('fastify').FastifyRequest} request
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isUnauthorized(request) {
|
||||
if (!request.session?.currentUser) return true;
|
||||
if (Date.now() - (request.session.createdAt || 0) > SESSION_MAX_AGE) return true;
|
||||
return false;
|
||||
};
|
||||
const isAdmin = (req) => {
|
||||
if (!isUnauthorized(req)) {
|
||||
const user = userStorage.getUser(req.session.currentUser);
|
||||
return user != null && user.isAdmin;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when the session belongs to an admin user.
|
||||
* @param {import('fastify').FastifyRequest} request
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isAdmin(request) {
|
||||
if (isUnauthorized(request)) return false;
|
||||
const user = userStorage.getUser(request.session.currentUser);
|
||||
return user != null && user.isAdmin;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fastify preHandler hook - rejects unauthenticated requests with 401.
|
||||
* @param {import('fastify').FastifyRequest} request
|
||||
* @param {import('fastify').FastifyReply} reply
|
||||
*/
|
||||
export async function authHook(request, reply) {
|
||||
if (isUnauthorized(request)) {
|
||||
reply.code(401).send();
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const authInterceptor = () => {
|
||||
return (req, res, next) => {
|
||||
if (isUnauthorized(req)) {
|
||||
return unauthorized(res);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
};
|
||||
const adminInterceptor = () => {
|
||||
return (req, res, next) => {
|
||||
if (!isAdmin(req)) {
|
||||
return unauthorized(res);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
};
|
||||
const cookieSession$0 = (secret) => {
|
||||
return cookieSession({
|
||||
name: 'fredy-admin-session',
|
||||
keys: [secret],
|
||||
maxAge: SESSION_MAX_AGE,
|
||||
});
|
||||
};
|
||||
export { cookieSession$0 as cookieSession };
|
||||
export { adminInterceptor };
|
||||
export { authInterceptor };
|
||||
export { isUnauthorized };
|
||||
export { isAdmin };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fastify preHandler hook - rejects non-admin requests with 401.
|
||||
* Apply after authHook.
|
||||
* @param {import('fastify').FastifyRequest} request
|
||||
* @param {import('fastify').FastifyReply} reply
|
||||
*/
|
||||
export async function adminHook(request, reply) {
|
||||
if (!isAdmin(request)) {
|
||||
reply.code(401).send();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ The LLM will automatically call the appropriate Fredy MCP tools and present the
|
||||
#### Setup
|
||||
|
||||
1. Open **Claude Desktop**
|
||||
2. Go to **Settings → Developer → Edit Config** — this opens the `claude_desktop_config.json` file
|
||||
2. Go to **Settings → Developer → Edit Config** - this opens the `claude_desktop_config.json` file
|
||||
3. Add the `fredy` server to the `mcpServers` object:
|
||||
|
||||
```json
|
||||
@@ -158,7 +158,7 @@ The LLM will automatically call the appropriate Fredy MCP tools and present the
|
||||
> - nvm: `/Users/<you>/.nvm/versions/node/<version>/bin/node`
|
||||
|
||||
4. Save the file and **restart Claude Desktop**
|
||||
5. You should see a hammer icon (🔨) in the chat input — click it to confirm the Fredy tools are listed
|
||||
5. You should see a hammer icon (🔨) in the chat input - click it to confirm the Fredy tools are listed
|
||||
|
||||
#### Usage
|
||||
|
||||
@@ -170,7 +170,7 @@ Once connected, simply ask Claude about your real estate data:
|
||||
|
||||
Claude will automatically call the appropriate Fredy MCP tools.
|
||||
|
||||
> **Note:** Fredy's main web process does not need to be running — the stdio transport opens its own database connection directly. But the SQLite database file must exist and migrations must have been applied.
|
||||
> **Note:** Fredy's main web process does not need to be running - the stdio transport opens its own database connection directly. But the SQLite database file must exist and migrations must have been applied.
|
||||
|
||||
---
|
||||
|
||||
@@ -252,7 +252,7 @@ Example list response:
|
||||
```
|
||||
**Tool:** list_listings | **Status:** OK
|
||||
|
||||
Found **85** listing(s). Showing page 1 of 2 (50 on this page). More pages available — use page=2 to continue.
|
||||
Found **85** listing(s). Showing page 1 of 2 (50 on this page). More pages available - use page=2 to continue.
|
||||
|
||||
| ID | Title | Address | Price | Size | Provider | Active | Created | Job |
|
||||
|----|-------|---------|-------|------|----------|--------|---------|-----|
|
||||
|
||||
@@ -49,7 +49,7 @@ export function createMcpServer() {
|
||||
'list_listings to search listings (supports time filters like createdAfter/createdBefore), ' +
|
||||
'and get_listing for full details of a single listing. ' +
|
||||
'Responses are formatted as markdown with a summary, data (tables for lists, key-value for details), and pagination info. ' +
|
||||
'Always present results to the user as soon as you have them — do NOT call the tool again unless you need additional pages or different data.',
|
||||
'Always present results to the user as soon as you have them - do NOT call the tool again unless you need additional pages or different data.',
|
||||
},
|
||||
);
|
||||
|
||||
@@ -220,6 +220,122 @@ export function createMcpServer() {
|
||||
},
|
||||
);
|
||||
|
||||
// ── get_photo_for_listing ─────────────────────────────────────────────────────
|
||||
server.tool(
|
||||
'get_photo_for_listing',
|
||||
'Fetch and return the photo of a listing by its ID as an image for vision analysis.',
|
||||
{
|
||||
listingId: z.string().describe('The listing ID whose photo to fetch'),
|
||||
},
|
||||
async ({ listingId }, extra) => {
|
||||
const { user, error } = authenticateToolCall(extra, 'get_photo_for_listing');
|
||||
if (error) return normalizeError(error, 'get_photo_for_listing');
|
||||
|
||||
const listing = getListingById(listingId, user.id, user.isAdmin);
|
||||
if (!listing) {
|
||||
return normalizeError('Listing not found or access denied.', 'get_photo_for_listing');
|
||||
}
|
||||
|
||||
const imageUrl = listing.image_url;
|
||||
if (!imageUrl) {
|
||||
return normalizeError('No image available for this listing.', 'get_photo_for_listing');
|
||||
}
|
||||
|
||||
const SUPPORTED_MIME_TYPES = new Set(['image/jpeg', 'image/png', 'image/gif', 'image/webp']);
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(imageUrl, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'image/jpeg,image/png,image/webp,image/gif,image/*,*/*',
|
||||
},
|
||||
});
|
||||
} catch (fetchErr) {
|
||||
return normalizeError(`Failed to fetch image: ${fetchErr.message}`, 'get_photo_for_listing');
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return normalizeError(
|
||||
`Image fetch returned HTTP ${response.status}. Image URL: ${imageUrl}`,
|
||||
'get_photo_for_listing',
|
||||
);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') ?? '';
|
||||
const headerMimeType = contentType.split(';')[0].trim().toLowerCase();
|
||||
|
||||
let buffer;
|
||||
try {
|
||||
buffer = await response.arrayBuffer();
|
||||
} catch (readErr) {
|
||||
return normalizeError(`Failed to read image body: ${readErr.message}`, 'get_photo_for_listing');
|
||||
}
|
||||
|
||||
const bytes = new Uint8Array(buffer);
|
||||
|
||||
if (bytes.length < 12) {
|
||||
return normalizeError(
|
||||
`Downloaded file is too small to determine image type. Image URL: ${imageUrl}`,
|
||||
'get_photo_for_listing',
|
||||
);
|
||||
}
|
||||
|
||||
let resolvedMime;
|
||||
|
||||
if (SUPPORTED_MIME_TYPES.has(headerMimeType)) {
|
||||
resolvedMime = headerMimeType;
|
||||
} else {
|
||||
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
||||
resolvedMime = 'image/jpeg';
|
||||
} else if (
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47 &&
|
||||
bytes[4] === 0x0d &&
|
||||
bytes[5] === 0x0a &&
|
||||
bytes[6] === 0x1a &&
|
||||
bytes[7] === 0x0a
|
||||
) {
|
||||
resolvedMime = 'image/png';
|
||||
} else if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x38) {
|
||||
resolvedMime = 'image/gif';
|
||||
} else if (
|
||||
bytes[0] === 0x52 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x46 &&
|
||||
bytes[8] === 0x57 &&
|
||||
bytes[9] === 0x45 &&
|
||||
bytes[10] === 0x42 &&
|
||||
bytes[11] === 0x50
|
||||
) {
|
||||
resolvedMime = 'image/webp';
|
||||
} else {
|
||||
return normalizeError(
|
||||
`Image format not supported by vision models (header: ${headerMimeType || 'unknown'}). Image URL: ${imageUrl}`,
|
||||
'get_photo_for_listing',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const base64 = Buffer.from(buffer).toString('base64');
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'image',
|
||||
data: base64,
|
||||
mimeType: resolvedMime,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── get_current_date_ime ─────────────────────────────────────────────────────
|
||||
server.tool('get_current_date_time', 'Returns the current date and time.', {}, () => {
|
||||
return {
|
||||
|
||||
@@ -3,10 +3,6 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
||||
import { createMcpServer } from './mcpAdapter.js';
|
||||
import { authenticateRequest } from './mcpAuthentication.js';
|
||||
@@ -15,16 +11,13 @@ import crypto from 'crypto';
|
||||
|
||||
/**
|
||||
* Active transports keyed by session id.
|
||||
* Each session gets its own McpServer + StreamableHTTPServerTransport pair.
|
||||
* @type {Map<string, { server: McpServer, transport: StreamableHTTPServerTransport }>}
|
||||
*/
|
||||
const sessions = new Map();
|
||||
|
||||
/**
|
||||
* Get or create a session for the given session id with authentication.
|
||||
* @param {string|undefined} sessionId
|
||||
* @param {{ userId: string }} auth
|
||||
* @returns {{ server: McpServer, transport: StreamableHTTPServerTransport }}
|
||||
*/
|
||||
function getOrCreateSession(sessionId, auth) {
|
||||
if (sessionId && sessions.has(sessionId)) {
|
||||
@@ -54,77 +47,67 @@ function getOrCreateSession(sessionId, auth) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Register MCP Streamable HTTP routes on a restana service.
|
||||
* Register MCP Streamable HTTP routes on a fastify instance.
|
||||
*
|
||||
* Mounts handlers at /api/mcp to handle the MCP Streamable HTTP protocol:
|
||||
* - POST /api/mcp – JSON-RPC messages (initialize, tool calls, etc.)
|
||||
* - GET /api/mcp – SSE stream for server-initiated notifications
|
||||
* - DELETE /api/mcp – session termination
|
||||
* POST /api/mcp – JSON-RPC messages
|
||||
* GET /api/mcp – SSE stream for server-initiated notifications
|
||||
* DELETE /api/mcp – session termination
|
||||
*
|
||||
* All endpoints require a valid Bearer token in the Authorization header.
|
||||
*
|
||||
* @param {import('restana').Service} service - The restana service instance.
|
||||
* @param {import('fastify').FastifyInstance} fastify
|
||||
*/
|
||||
export function registerMcpRoutes(service) {
|
||||
// POST – main JSON-RPC endpoint
|
||||
service.post('/api/mcp', async (req, res) => {
|
||||
const auth = authenticateRequest(req);
|
||||
export function registerMcpRoutes(fastify) {
|
||||
fastify.post('/api/mcp', async (request, reply) => {
|
||||
const auth = authenticateRequest(request.raw);
|
||||
if (!auth) {
|
||||
res.statusCode = 401;
|
||||
return res.send({ error: 'Unauthorized. Provide a valid Bearer token.' });
|
||||
return reply.code(401).send({ error: 'Unauthorized. Provide a valid Bearer token.' });
|
||||
}
|
||||
|
||||
const sessionId = req.headers['mcp-session-id'];
|
||||
const sessionId = request.raw.headers['mcp-session-id'];
|
||||
const { server, transport } = getOrCreateSession(sessionId, auth);
|
||||
|
||||
// Connect server to transport if not already connected
|
||||
if (!transport.onmessage) {
|
||||
await server.connect(transport);
|
||||
}
|
||||
|
||||
// Inject authInfo so tools can access the authenticated user
|
||||
req.auth = { userId: auth.userId };
|
||||
request.raw.auth = { userId: auth.userId };
|
||||
|
||||
await transport.handleRequest(req, res, req.body);
|
||||
reply.hijack();
|
||||
await transport.handleRequest(request.raw, reply.raw, request.body);
|
||||
});
|
||||
|
||||
// GET – SSE stream for server-initiated messages
|
||||
service.get('/api/mcp', async (req, res) => {
|
||||
const auth = authenticateRequest(req);
|
||||
fastify.get('/api/mcp', async (request, reply) => {
|
||||
const auth = authenticateRequest(request.raw);
|
||||
if (!auth) {
|
||||
res.statusCode = 401;
|
||||
return res.send({ error: 'Unauthorized. Provide a valid Bearer token.' });
|
||||
return reply.code(401).send({ error: 'Unauthorized. Provide a valid Bearer token.' });
|
||||
}
|
||||
|
||||
const sessionId = req.headers['mcp-session-id'];
|
||||
const sessionId = request.raw.headers['mcp-session-id'];
|
||||
if (!sessionId || !sessions.has(sessionId)) {
|
||||
res.statusCode = 400;
|
||||
return res.send({ error: 'Invalid or missing session. Send an initialize request first.' });
|
||||
return reply.code(400).send({ error: 'Invalid or missing session. Send an initialize request first.' });
|
||||
}
|
||||
|
||||
const { transport } = sessions.get(sessionId);
|
||||
await transport.handleRequest(req, res);
|
||||
reply.hijack();
|
||||
await transport.handleRequest(request.raw, reply.raw);
|
||||
});
|
||||
|
||||
// DELETE – terminate session
|
||||
service.delete('/api/mcp', async (req, res) => {
|
||||
const auth = authenticateRequest(req);
|
||||
fastify.delete('/api/mcp', async (request, reply) => {
|
||||
const auth = authenticateRequest(request.raw);
|
||||
if (!auth) {
|
||||
res.statusCode = 401;
|
||||
return res.send({ error: 'Unauthorized. Provide a valid Bearer token.' });
|
||||
return reply.code(401).send({ error: 'Unauthorized. Provide a valid Bearer token.' });
|
||||
}
|
||||
|
||||
const sessionId = req.headers['mcp-session-id'];
|
||||
const sessionId = request.raw.headers['mcp-session-id'];
|
||||
if (!sessionId || !sessions.has(sessionId)) {
|
||||
res.statusCode = 404;
|
||||
return res.send({ error: 'Session not found.' });
|
||||
return reply.code(404).send({ error: 'Session not found.' });
|
||||
}
|
||||
|
||||
const { transport } = sessions.get(sessionId);
|
||||
await transport.close();
|
||||
sessions.delete(sessionId);
|
||||
res.statusCode = 200;
|
||||
res.send({ ok: true });
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
logger.debug('MCP Streamable HTTP endpoint registered at /api/mcp');
|
||||
|
||||
@@ -70,7 +70,7 @@ export function normalizeListJobs(queryResult, { page, pageSize }) {
|
||||
|
||||
let md = `**Tool:** list_jobs | **Status:** OK\n\n`;
|
||||
md += `Found **${queryResult.totalNumber}** job(s). Showing page ${page} of ${maxPage} (${jobs.length} on this page).`;
|
||||
if (hasMore) md += ` More pages available — use page=${page + 1} to continue.`;
|
||||
if (hasMore) md += ` More pages available - use page=${page + 1} to continue.`;
|
||||
md += '\n\n';
|
||||
|
||||
if (jobs.length > 0) {
|
||||
@@ -120,7 +120,7 @@ export function normalizeListListings(queryResult, { page, pageSize }) {
|
||||
|
||||
let md = `**Tool:** list_listings | **Status:** OK\n\n`;
|
||||
md += `Found **${queryResult.totalNumber}** listing(s). Showing page ${page} of ${maxPage} (${listings.length} on this page).`;
|
||||
if (hasMore) md += ` More pages available — use page=${page + 1} to continue.`;
|
||||
if (hasMore) md += ` More pages available - use page=${page + 1} to continue.`;
|
||||
md += '\n\n';
|
||||
|
||||
if (listings.length > 0) {
|
||||
|
||||
@@ -7,13 +7,14 @@ import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { server } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
const promises = newListings.map((newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
|
||||
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${newListing.id}` : '';
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}${fredyLine}`;
|
||||
return fetch(server, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
|
||||
@@ -5,9 +5,18 @@
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
|
||||
export const send = ({ serviceName, newListings, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, jobKey, baseUrl }) => {
|
||||
/* eslint-disable no-console */
|
||||
return [Promise.resolve(console.info(`Found entry from service ${serviceName}, Job: ${jobKey}:`, newListings))];
|
||||
const fredyLinks = baseUrl ? newListings.map((l) => `${baseUrl}/listings/listing/${l.id}`).join(', ') : null;
|
||||
return [
|
||||
Promise.resolve(
|
||||
console.info(
|
||||
`Found entry from service ${serviceName}, Job: ${jobKey}:`,
|
||||
newListings,
|
||||
...(fredyLinks ? [`Open in Fredy: ${fredyLinks}`] : []),
|
||||
),
|
||||
),
|
||||
];
|
||||
/* eslint-enable no-console */
|
||||
};
|
||||
export const config = {
|
||||
|
||||
@@ -39,9 +39,10 @@ const generateColorFromString = (str) => {
|
||||
*
|
||||
* @param {string} jobKey - Key of job (used to set embed color)
|
||||
* @param {object} listing - Object holding listing details
|
||||
* @param baseUrl
|
||||
* @returns {object} Discord webhook embed
|
||||
*/
|
||||
const buildEmbed = (jobKey, listing) => {
|
||||
const buildEmbed = (jobKey, listing, baseUrl) => {
|
||||
const maxTitleLength = 252; // Max embed title length is 256 characters
|
||||
let title = String(listing.title ?? 'N/A');
|
||||
if (title.length > maxTitleLength) {
|
||||
@@ -79,10 +80,18 @@ const buildEmbed = (jobKey, listing) => {
|
||||
};
|
||||
}
|
||||
|
||||
if (baseUrl && listing.id) {
|
||||
fields.push({
|
||||
name: 'Open in Fredy',
|
||||
value: `[Open in Fredy](${baseUrl}/listings/listing/${listing.id})`,
|
||||
inline: false,
|
||||
});
|
||||
}
|
||||
|
||||
return embed;
|
||||
};
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const adapter = notificationConfig.find((adapter) => adapter.id === config.id);
|
||||
const webhookUrl = adapter?.fields?.webhookUrl;
|
||||
if (!webhookUrl || newListings.length === 0) return Promise.resolve([]);
|
||||
@@ -90,7 +99,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job?.name || jobKey;
|
||||
|
||||
const embeds = newListings.map((listing) => buildEmbed(jobKey, listing));
|
||||
const embeds = newListings.map((listing) => buildEmbed(jobKey, listing, baseUrl));
|
||||
|
||||
const maxEmbedsPerMessage = 10; // Discord only allows up to 10 embeds
|
||||
const webhookPromises = [];
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
|
||||
const mapListing = (listing) => ({
|
||||
const mapListing = (listing, baseUrl) => ({
|
||||
address: listing.address,
|
||||
description: listing.description,
|
||||
id: listing.id,
|
||||
@@ -14,12 +14,13 @@ const mapListing = (listing) => ({
|
||||
size: listing.size,
|
||||
title: listing.title,
|
||||
url: listing.link,
|
||||
fredyUrl: baseUrl && listing.id ? `${baseUrl}/listings/listing/${listing.id}` : null,
|
||||
});
|
||||
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { authToken, endpointUrl, selfSignedCerts } = notificationConfig.find((a) => a.id === config.id).fields;
|
||||
|
||||
const listings = newListings.map(mapListing);
|
||||
const listings = newListings.map((l) => mapListing(l, baseUrl));
|
||||
const body = {
|
||||
jobId: jobKey,
|
||||
timestamp: new Date().toISOString(),
|
||||
|
||||
@@ -35,7 +35,7 @@ const toBase64 = async (url) => {
|
||||
}
|
||||
};
|
||||
|
||||
const mapListingsWithCid = async (serviceName, jobKey, listings) => {
|
||||
const mapListingsWithCid = async (serviceName, jobKey, listings, baseUrl) => {
|
||||
const out = [];
|
||||
const attachments = [];
|
||||
|
||||
@@ -53,6 +53,7 @@ const mapListingsWithCid = async (serviceName, jobKey, listings) => {
|
||||
jobKey,
|
||||
hasImage: false,
|
||||
imageCid: '',
|
||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
||||
};
|
||||
|
||||
if (imgUrl) {
|
||||
@@ -78,7 +79,7 @@ const mapListingsWithCid = async (serviceName, jobKey, listings) => {
|
||||
return { listings: out, attachments };
|
||||
};
|
||||
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
|
||||
(adapter) => adapter.id === config.id,
|
||||
).fields;
|
||||
@@ -89,7 +90,7 @@ export const send = async ({ serviceName, newListings, notificationConfig, jobKe
|
||||
.map((r) => ({ Email: r.trim() }))
|
||||
.filter((r) => r.Email.length > 0);
|
||||
|
||||
const { listings, attachments } = await mapListingsWithCid(serviceName, jobKey, newListings);
|
||||
const { listings, attachments } = await mapListingsWithCid(serviceName, jobKey, newListings, baseUrl);
|
||||
|
||||
const html = emailTemplate({
|
||||
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
|
||||
|
||||
@@ -6,15 +6,20 @@
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { webhook, channel } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
let message = `### *${jobName}* (${serviceName}) found **${newListings.length}** new listings:\n\n`;
|
||||
message += `| Title | Address | Size | Price |\n|:----|:----|:----|:----|\n`;
|
||||
message += newListings.map(
|
||||
(o) => `| [${o.title}](${o.link}) | ` + [o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') + ' |\n',
|
||||
);
|
||||
message += `| Title | Address | Size | Price |${baseUrl ? ' Open in Fredy |' : ''}\n|:----|:----|:----|:----|${baseUrl ? ':----|\n' : '\n'}`;
|
||||
message += newListings.map((o) => {
|
||||
const fredyCell = baseUrl && o.id ? ` [Open in Fredy](${baseUrl}/listings/listing/${o.id}) |` : '';
|
||||
return (
|
||||
`| [${o.title}](${o.link}) | ` +
|
||||
[o.address, o.size.replace(/2m/g, '$m^2$'), o.price].join(' | ') +
|
||||
` |${fredyCell}\n`
|
||||
);
|
||||
});
|
||||
return fetch(webhook, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
|
||||
@@ -8,17 +8,18 @@ import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
|
||||
const promises = newListings.map((newListing) => {
|
||||
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${newListing.id}` : '';
|
||||
const message = `
|
||||
Address: ${newListing.address}
|
||||
Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
|
||||
Price: ${newListing.price}
|
||||
Link: ${newListing.link}`;
|
||||
Link: ${newListing.link}${fredyLine}`;
|
||||
|
||||
const sanitizeHeaderValue = (value) =>
|
||||
String(value ?? '')
|
||||
|
||||
@@ -7,7 +7,7 @@ import { markdown2Html } from '../../services/markdown.js';
|
||||
import { getJob } from '../../services/storage/jobStorage.js';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
const job = getJob(jobKey);
|
||||
const jobName = job == null ? jobKey : job.name;
|
||||
@@ -15,7 +15,8 @@ export const send = async ({ serviceName, newListings, notificationConfig, jobKe
|
||||
const results = await Promise.all(
|
||||
newListings.map(async (newListing) => {
|
||||
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
|
||||
const fredyLine = baseUrl && newListing.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${newListing.id}` : '';
|
||||
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}${fredyLine}`;
|
||||
|
||||
const form = new FormData();
|
||||
form.append('token', token);
|
||||
|
||||
@@ -14,7 +14,7 @@ const __dirname = getDirName();
|
||||
const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
|
||||
const emailTemplate = Handlebars.compile(template);
|
||||
|
||||
const mapListings = (serviceName, jobKey, listings) =>
|
||||
const mapListings = (serviceName, jobKey, listings, baseUrl) =>
|
||||
listings.map((l) => {
|
||||
const image = normalizeImageUrl(l.image);
|
||||
return {
|
||||
@@ -25,12 +25,13 @@ const mapListings = (serviceName, jobKey, listings) =>
|
||||
price: l.price || '',
|
||||
image,
|
||||
hasImage: Boolean(image),
|
||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
||||
serviceName,
|
||||
jobKey,
|
||||
};
|
||||
});
|
||||
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { apiKey, receiver, from } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
|
||||
const to = receiver
|
||||
@@ -41,7 +42,7 @@ export const send = async ({ serviceName, newListings, notificationConfig, jobKe
|
||||
|
||||
const resend = new Resend(apiKey);
|
||||
|
||||
const listings = mapListings(serviceName, jobKey, newListings);
|
||||
const listings = mapListings(serviceName, jobKey, newListings, baseUrl);
|
||||
|
||||
const html = emailTemplate({
|
||||
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
|
||||
|
||||
@@ -7,7 +7,7 @@ import sgMail from '@sendgrid/mail';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const mapListings = (serviceName, jobKey, listings) =>
|
||||
const mapListings = (serviceName, jobKey, listings, baseUrl) =>
|
||||
listings.map((l) => {
|
||||
const image = normalizeImageUrl(l.image);
|
||||
return {
|
||||
@@ -20,12 +20,13 @@ const mapListings = (serviceName, jobKey, listings) =>
|
||||
hasImage: Boolean(image),
|
||||
// optional plain text snippet
|
||||
snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
|
||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
||||
serviceName,
|
||||
jobKey,
|
||||
};
|
||||
});
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
|
||||
|
||||
sgMail.setApiKey(apiKey);
|
||||
@@ -36,7 +37,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
||||
.map((r) => r.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const listings = mapListings(serviceName, jobKey, newListings);
|
||||
const listings = mapListings(serviceName, jobKey, newListings, baseUrl);
|
||||
|
||||
const msg = {
|
||||
templateId,
|
||||
|
||||
@@ -7,7 +7,7 @@ import Slack from 'slack';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const buildBlocks = (serviceName, jobKey, p) => {
|
||||
const buildBlocks = (serviceName, jobKey, p, baseUrl) => {
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
@@ -36,6 +36,13 @@ const buildBlocks = (serviceName, jobKey, p) => {
|
||||
});
|
||||
}
|
||||
|
||||
if (baseUrl && p.id) {
|
||||
blocks.push({
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `<${baseUrl}/listings/listing/${p.id}|Open in Fredy>` },
|
||||
});
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
type: 'context',
|
||||
elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
|
||||
@@ -44,7 +51,7 @@ const buildBlocks = (serviceName, jobKey, p) => {
|
||||
return blocks;
|
||||
};
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { token, channel } = notificationConfig.find((a) => a.id === config.id).fields;
|
||||
|
||||
return Promise.allSettled(
|
||||
@@ -53,7 +60,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
||||
token,
|
||||
channel,
|
||||
text: `${serviceName} ${jobKey}: ${p.title}`,
|
||||
blocks: buildBlocks(serviceName, jobKey, p),
|
||||
blocks: buildBlocks(serviceName, jobKey, p, baseUrl),
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
}),
|
||||
|
||||
@@ -7,7 +7,7 @@ import fetch from 'node-fetch';
|
||||
import { markdown2Html } from '../../services/markdown.js';
|
||||
import { normalizeImageUrl } from '../../utils.js';
|
||||
|
||||
const buildBlocks = (serviceName, jobKey, p) => {
|
||||
const buildBlocks = (serviceName, jobKey, p, baseUrl) => {
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
@@ -36,6 +36,13 @@ const buildBlocks = (serviceName, jobKey, p) => {
|
||||
});
|
||||
}
|
||||
|
||||
if (baseUrl && p.id) {
|
||||
blocks.push({
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `<${baseUrl}/listings/listing/${p.id}|Open in Fredy>` },
|
||||
});
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
type: 'context',
|
||||
elements: [{ type: 'mrkdwn', text: 'Powered by Fredy' }],
|
||||
@@ -51,7 +58,7 @@ const postJson = (url, body) =>
|
||||
body,
|
||||
});
|
||||
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const adapter = notificationConfig.find((a) => a.id === config.id);
|
||||
const webhookUrl = adapter?.fields?.webhookUrl;
|
||||
if (!webhookUrl) return Promise.resolve([]);
|
||||
@@ -59,7 +66,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) =
|
||||
const promises = newListings.map((p) => {
|
||||
const body = JSON.stringify({
|
||||
text: `${serviceName} ${jobKey}: ${p.title}`,
|
||||
blocks: buildBlocks(serviceName, jobKey, p),
|
||||
blocks: buildBlocks(serviceName, jobKey, p, baseUrl),
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
});
|
||||
|
||||
@@ -14,7 +14,7 @@ const __dirname = getDirName();
|
||||
const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
|
||||
const emailTemplate = Handlebars.compile(template);
|
||||
|
||||
const mapListings = (serviceName, jobKey, listings) =>
|
||||
const mapListings = (serviceName, jobKey, listings, baseUrl) =>
|
||||
listings.map((l) => {
|
||||
const image = normalizeImageUrl(l.image);
|
||||
return {
|
||||
@@ -25,12 +25,13 @@ const mapListings = (serviceName, jobKey, listings) =>
|
||||
price: l.price || '',
|
||||
image,
|
||||
hasImage: Boolean(image),
|
||||
fredyUrl: baseUrl && l.id ? `${baseUrl}/listings/listing/${l.id}` : null,
|
||||
serviceName,
|
||||
jobKey,
|
||||
};
|
||||
});
|
||||
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
|
||||
export const send = async ({ serviceName, newListings, notificationConfig, jobKey, baseUrl }) => {
|
||||
const { host, port, secure, username, password, receiver, from } = notificationConfig.find(
|
||||
(adapter) => adapter.id === config.id,
|
||||
).fields;
|
||||
@@ -51,7 +52,7 @@ export const send = async ({ serviceName, newListings, notificationConfig, jobKe
|
||||
},
|
||||
});
|
||||
|
||||
const listings = mapListings(serviceName, jobKey, newListings);
|
||||
const listings = mapListings(serviceName, jobKey, newListings, baseUrl);
|
||||
|
||||
const html = emailTemplate({
|
||||
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
|
||||
|
||||
@@ -17,6 +17,7 @@ Multiple recipients:
|
||||
|
||||
Common SMTP settings:
|
||||
|
||||
- **Gmail** — `smtp.gmail.com`, port 587, secure: false
|
||||
- **Outlook** — `smtp.office365.com`, port 587, secure: false
|
||||
- **Yahoo** — `smtp.mail.yahoo.com`, port 465, secure: true
|
||||
- **Gmail** - `smtp.gmail.com`, port 587, secure: false
|
||||
- **Outlook** - `smtp.office365.com`, port 587, secure: false
|
||||
- **Yahoo** - `smtp.mail.yahoo.com`, port 465, secure: true
|
||||
- **Gmx** - `mail.gmx.net`, port 587, secure: true
|
||||
|
||||
@@ -80,12 +80,14 @@ function escapeHtml(s = '') {
|
||||
* @param {string} [o.link]
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildCaption(jobName, serviceName, o) {
|
||||
function buildCaption(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLink =
|
||||
baseUrl && o.id ? `\n<a href='${escapeHtml(`${baseUrl}/listings/listing/${o.id}`)}'>Open in Fredy</a>` : '';
|
||||
return `<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n<a href='${escapeHtml(
|
||||
o.link || '',
|
||||
)}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}`.slice(0, 1024);
|
||||
)}'><b>${escapeHtml(title)}</b></a>\n${escapeHtml(meta)}${fredyLink}`.slice(0, 1024);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -95,16 +97,47 @@ function buildCaption(jobName, serviceName, o) {
|
||||
* @param {Object} o - Listing object
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildText(jobName, serviceName, o) {
|
||||
function buildText(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLink =
|
||||
baseUrl && o.id ? `\n<a href='${escapeHtml(`${baseUrl}/listings/listing/${o.id}`)}'>Open in Fredy</a>` : '';
|
||||
return (
|
||||
`<i>${escapeHtml(jobName)}</i> (${escapeHtml(serviceName)})\n` +
|
||||
`<a href='${escapeHtml(o.link || '')}'><b>${escapeHtml(title)}</b></a>\n` +
|
||||
`${escapeHtml(meta)}`
|
||||
`${escapeHtml(meta)}${fredyLink}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a plain text Telegram photo caption (max 4096 characters).
|
||||
* @param {string} jobName
|
||||
* @param {string} serviceName
|
||||
* @param {Object} o - Listing object
|
||||
* @param baseUrl
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildCaptionPlain(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${o.id}` : '';
|
||||
return `${jobName} (${serviceName})\n${title}\n${meta}\n\n${o.link || ''}${fredyLine}`.slice(0, 4096);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a plain text Telegram message.
|
||||
* @param {string} jobName
|
||||
* @param {string} serviceName
|
||||
* @param {Object} o - Listing object
|
||||
* @returns {string}
|
||||
*/
|
||||
function buildTextPlain(jobName, serviceName, o, baseUrl) {
|
||||
const title = shorten((o.title || '').replace(/\*/g, ''), 90);
|
||||
const meta = [o.address, o.price, o.size].filter(Boolean).join(' | ');
|
||||
const fredyLine = baseUrl && o.id ? `\nOpen in Fredy: ${baseUrl}/listings/listing/${o.id}` : '';
|
||||
return `${jobName} (${serviceName})\n${title}\n${o.link || ''}\n${meta}${fredyLine}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send new listings to Telegram.
|
||||
* - Respects per-chat Telegram rate limits using a lightweight throttle cache.
|
||||
@@ -117,12 +150,12 @@ function buildText(jobName, serviceName, o) {
|
||||
* @param {string} params.jobKey - Storage job key to resolve the human readable job name.
|
||||
* @returns {Promise<Array<Response>>} Promise resolving when all send operations complete.
|
||||
*/
|
||||
export const send = ({ serviceName, newListings = [], notificationConfig, jobKey }) => {
|
||||
export const send = ({ serviceName, newListings = [], notificationConfig, jobKey, baseUrl }) => {
|
||||
const adapterCfg = notificationConfig.find((adapter) => adapter.id === config.id);
|
||||
if (!adapterCfg || !adapterCfg.fields) {
|
||||
throw new Error(`Telegram adapter configuration missing for job '${jobKey || ''}'`);
|
||||
}
|
||||
const { token, chatId, messageThreadId } = adapterCfg.fields;
|
||||
const { token, chatId, messageThreadId, plainText } = adapterCfg.fields;
|
||||
if (!token || !chatId) {
|
||||
throw new Error("Telegram 'token' and 'chatId' must be provided in notification config");
|
||||
}
|
||||
@@ -163,8 +196,8 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
|
||||
const img = normalizeImageUrl(o.image);
|
||||
const textPayload = {
|
||||
chat_id: chatId,
|
||||
text: buildText(jobName, serviceName, o),
|
||||
parse_mode: 'HTML',
|
||||
text: plainText ? buildTextPlain(jobName, serviceName, o, baseUrl) : buildText(jobName, serviceName, o, baseUrl),
|
||||
...(plainText ? {} : { parse_mode: 'HTML' }),
|
||||
disable_web_page_preview: true,
|
||||
...(message_thread_id ? { message_thread_id } : {}),
|
||||
};
|
||||
@@ -178,8 +211,10 @@ export const send = ({ serviceName, newListings = [], notificationConfig, jobKey
|
||||
return await throttledCall('sendPhoto', {
|
||||
chat_id: chatId,
|
||||
photo: img,
|
||||
caption: buildCaption(jobName, serviceName, o),
|
||||
parse_mode: 'HTML',
|
||||
caption: plainText
|
||||
? buildCaptionPlain(jobName, serviceName, o, baseUrl)
|
||||
: buildCaption(jobName, serviceName, o, baseUrl),
|
||||
...(plainText ? {} : { parse_mode: 'HTML' }),
|
||||
...(message_thread_id ? { message_thread_id } : {}),
|
||||
}).catch(async (e) => {
|
||||
logger.error(`Error sending photo to Telegram and use a fallback: ${e.message}`);
|
||||
@@ -220,5 +255,11 @@ export const config = {
|
||||
description:
|
||||
'Optional: The topic/thread id within a supergroup to post into (Telegram message_thread_id). Provide a positive integer.',
|
||||
},
|
||||
plainText: {
|
||||
type: 'boolean',
|
||||
optional: true,
|
||||
label: 'Send as plain text',
|
||||
description: 'Send messages as plain text instead of HTML formatted.',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -106,6 +106,9 @@
|
||||
<![endif]-->
|
||||
<!--[if !mso]><!-- -->
|
||||
<a href="{{this.link}}" class="btn" target="_blank">View Listing</a>
|
||||
{{#if this.fredyUrl}}
|
||||
<a href="{{this.fredyUrl}}" class="btn" style="background:#1a6fff;color:#ffffff;margin-left:8px;" target="_blank">Open in Fredy</a>
|
||||
{{/if}}
|
||||
<!--<![endif]-->
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
@@ -20,10 +20,10 @@ if (adapter.length === 0) {
|
||||
const findAdapter = (notificationAdapter) => {
|
||||
return adapter.find((a) => a.config.id === notificationAdapter.id);
|
||||
};
|
||||
export const send = (serviceName, newListings, notificationConfig, jobKey) => {
|
||||
export const send = (serviceName, newListings, notificationConfig, jobKey, baseUrl) => {
|
||||
//this is not being used in tests, therefore adapter are always set
|
||||
return notificationConfig
|
||||
.filter((notificationAdapter) => findAdapter(notificationAdapter) != null)
|
||||
.map((notificationAdapter) => findAdapter(notificationAdapter))
|
||||
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey }));
|
||||
.map((a) => a.send({ serviceName, newListings, notificationConfig, jobKey, baseUrl }));
|
||||
};
|
||||
|
||||
@@ -5,8 +5,16 @@
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const baseUrl = 'https://www.1a-immobilienmarkt.de';
|
||||
const link = `${baseUrl}/expose/${o.id}.html`;
|
||||
@@ -14,7 +22,17 @@ function normalize(o) {
|
||||
const id = buildHash(o.id, price);
|
||||
const image = baseUrl + o.image;
|
||||
const address = o.address == null ? null : o.address.trim().replaceAll('/', ',');
|
||||
return Object.assign(o, { id, price, link, image, address });
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address,
|
||||
image,
|
||||
description: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -34,13 +52,19 @@ function normalizePrice(price) {
|
||||
}
|
||||
return result[0];
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: '.tabelle',
|
||||
sortByDateParam: 'sort_type=newest',
|
||||
@@ -48,7 +72,8 @@ const config = {
|
||||
crawlFields: {
|
||||
id: '.inner_object_data input[name="marker_objekt_id"]@value | int',
|
||||
price: '.inner_object_data .single_data_price | removeNewline | trim',
|
||||
size: '.tabelle .tabelle_inhalt_infos .single_data_box | removeNewline | trim',
|
||||
size: '.tabelle .tabelle_inhalt_infos .single_data_box:nth-of-type(1) | removeNewline | trim',
|
||||
rooms: '.tabelle .tabelle_inhalt_infos .single_data_box:nth-of-type(2) | removeNewline | trim',
|
||||
title: '.inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim',
|
||||
image: '.inner_object_pic img@src',
|
||||
address: '.tabelle .tabelle_inhalt_infos .left_information > div:nth-child(2) | removeNewline | trim',
|
||||
|
||||
@@ -5,9 +5,12 @@
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import logger from '../services/logger.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -23,7 +26,7 @@ function parseId(shortenedLink) {
|
||||
|
||||
async function fetchDetails(listing, browser) {
|
||||
try {
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser });
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser, name: 'immobilienDe_details' });
|
||||
if (!html) return listing;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
@@ -65,27 +68,44 @@ async function fetchDetails(listing, browser) {
|
||||
return listing;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const baseUrl = 'https://www.immobilien.de';
|
||||
const size = o.size || null;
|
||||
const price = o.price || null;
|
||||
const title = o.title || 'No title available';
|
||||
const title = o.title || '';
|
||||
const address = o.address || null;
|
||||
const shortLink = shortenLink(o.link);
|
||||
const link = shortLink ? (shortLink.startsWith('http') ? shortLink : baseUrl + shortLink) : baseUrl;
|
||||
const image = o.image ? (o.image.startsWith('http') ? o.image : baseUrl + o.image) : null;
|
||||
const id = buildHash(parseId(shortLink), o.price);
|
||||
return Object.assign(o, { id, price, size, title, address, link, image });
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title,
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address,
|
||||
image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: 'a.lr-card',
|
||||
sortByDateParam: 'sort_col=*created_ts&sort_dir=desc',
|
||||
@@ -94,6 +114,7 @@ const config = {
|
||||
id: '@href', //will be transformed later
|
||||
price: '.lr-card__price-amount | trim',
|
||||
size: '.lr-card__fact:has(.lr-card__fact-label:contains("Fläche")) .lr-card__fact-value | trim',
|
||||
rooms: '.zimmer .label_info',
|
||||
title: '.lr-card__title | trim',
|
||||
description: '.description | trim',
|
||||
link: '@href',
|
||||
|
||||
@@ -46,6 +46,10 @@ import {
|
||||
convertWebToMobile,
|
||||
} from '../services/immoscout/immoscout-web-translator.js';
|
||||
import logger from '../services/logger.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
async function getListings(url) {
|
||||
@@ -168,22 +172,44 @@ async function isListingActive(link) {
|
||||
function nullOrEmpty(val) {
|
||||
return val == null || val.length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
|
||||
const title = (o.title || '').replace('NEU', '').trim();
|
||||
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
|
||||
const id = buildHash(o.id, o.price);
|
||||
return Object.assign(o, { id, title, address });
|
||||
return {
|
||||
id,
|
||||
link: o.link,
|
||||
title,
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
return !isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlFields: {
|
||||
id: 'id',
|
||||
title: 'title',
|
||||
price: 'price',
|
||||
size: 'size',
|
||||
rooms: 'rooms',
|
||||
link: 'link',
|
||||
address: 'address',
|
||||
},
|
||||
|
||||
@@ -5,27 +5,46 @@
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const size = o.size || 'N/A m²';
|
||||
const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
|
||||
const title = o.title || 'No title available';
|
||||
const immoId = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
|
||||
const link = `https://immo.swp.de/immobilien/${immoId}`;
|
||||
const description = o.description;
|
||||
const id = buildHash(immoId, price);
|
||||
return Object.assign(o, { id, price, size, title, link, description });
|
||||
const id = buildHash(immoId, o.price);
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address: o.address,
|
||||
image: o.image,
|
||||
description: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: '.js-serp-item',
|
||||
sortByDateParam: 's=most_recently_updated_first',
|
||||
@@ -34,9 +53,10 @@ const config = {
|
||||
id: '.js-bookmark-btn@data-id',
|
||||
price: 'div.align-items-start div:first-child | trim',
|
||||
size: 'div.align-items-start div:nth-child(3) | trim',
|
||||
rooms: 'div.align-items-start div:nth-child(2) | trim',
|
||||
address: '.js-bookmark-btn@data-address',
|
||||
title: '.js-item-title-link@title | trim',
|
||||
link: '.ci-search-result__link@href',
|
||||
description: '.js-show-more-item-sm | removeNewline | trim',
|
||||
image: 'img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
|
||||
@@ -5,15 +5,18 @@
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import logger from '../services/logger.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
async function fetchDetails(listing, browser) {
|
||||
try {
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser });
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser, name: 'immowelt_details' });
|
||||
if (!html) return listing;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
@@ -48,27 +51,60 @@ async function fetchDetails(listing, browser) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const id = buildHash(o.id, o.price);
|
||||
return Object.assign(o, { id });
|
||||
return {
|
||||
id,
|
||||
link: o.link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address: o.address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer:
|
||||
'div[data-testid="serp-core-scrollablelistview-testid"]:not(div[data-testid="serp-enlargementlist-testid"] div[data-testid="serp-card-testid"]) div[data-testid="serp-core-classified-card-testid"]',
|
||||
sortByDateParam: 'order=DateDesc',
|
||||
waitForSelector: 'div[data-testid="serp-gridcontainer-testid"]',
|
||||
// waitForSelector is null: extract the full page via page.content() so the
|
||||
// Cheerio crawler can search anywhere in the rendered document.
|
||||
// preNavigateUrl visits the homepage first to establish a trusted session
|
||||
// before hitting the search URL; this prevents CDN-level bot challenges that
|
||||
// fire on cold sessions. waitForNetworkIdle (phase 2) then catches React's
|
||||
// listing API round-trip that fires well after domcontentloaded.
|
||||
waitForSelector: null,
|
||||
puppeteerOptions: {
|
||||
puppeteerTimeout: 60_000,
|
||||
preNavigateUrl: 'https://www.immowelt.de/',
|
||||
waitForNetworkIdle: true,
|
||||
waitForNetworkIdleTimeout: 60_000,
|
||||
},
|
||||
crawlFields: {
|
||||
id: 'a@href',
|
||||
price: 'div[data-testid="cardmfe-price-testid"] | removeNewline | trim',
|
||||
size: 'div[data-testid="cardmfe-keyfacts-testid"] | removeNewline | trim',
|
||||
size: 'div[data-testid="cardmfe-keyfacts-testid"] div:nth-of-type(3) | removeNewline | trim',
|
||||
rooms: 'div[data-testid="cardmfe-keyfacts-testid"] div:nth-of-type(1) | removeNewline | trim',
|
||||
title: 'div[data-testid="cardmfe-description-box-text-test-id"] > div:nth-of-type(2)',
|
||||
link: 'a@href',
|
||||
description: 'div[data-testid="cardmfe-description-text-test-id"] > div:nth-of-type(2) | removeNewline | trim',
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
|
||||
import { buildHash, isOneOf } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
|
||||
import logger from '../services/logger.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
@@ -125,7 +128,7 @@ async function enrichListingFromDetails(listing, browser) {
|
||||
if (!absoluteLink) return listing;
|
||||
|
||||
try {
|
||||
const html = await puppeteerExtractor(absoluteLink, null, { browser });
|
||||
const html = await puppeteerExtractor(absoluteLink, null, { browser, name: 'kleinanzeigen_details' });
|
||||
if (!html) return { ...listing, link: absoluteLink };
|
||||
|
||||
const { detailAddress, detailDescription } = extractDetailFromHtml(html);
|
||||
@@ -146,13 +149,33 @@ async function fetchDetails(listing, browser) {
|
||||
return enrichListingFromDetails(listing, browser);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const size = o.size || '--- m²';
|
||||
const parts = (o.tags || '').split('·').map((p) => p.trim());
|
||||
const size = parts.find((p) => p.includes('m²'));
|
||||
const rooms = parts.find((p) => p.includes('Zi.'));
|
||||
const id = buildHash(o.id, o.price);
|
||||
const link = toAbsoluteLink(o.link) || o.link;
|
||||
return Object.assign(o, { id, size, link });
|
||||
|
||||
return {
|
||||
id,
|
||||
title: o.title,
|
||||
link: toAbsoluteLink(o.link) || o.link,
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(size),
|
||||
rooms: extractNumber(rooms),
|
||||
address: o.address,
|
||||
description: o.description,
|
||||
image: o.image,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
@@ -161,16 +184,18 @@ function applyBlacklist(o) {
|
||||
return o.title != null && !isBlacklistedDistrict && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: '#srchrslt-adtable .ad-listitem ',
|
||||
//sort by date is standard oO
|
||||
sortByDateParam: null,
|
||||
waitForSelector: 'body',
|
||||
crawlFields: {
|
||||
id: '.aditem@data-adid | int',
|
||||
id: '.aditem@data-adid',
|
||||
price: '.aditem-main--middle--price-shipping--price | removeNewline | trim',
|
||||
size: '.aditem-main .text-module-end | removeNewline | trim',
|
||||
tags: '.aditem-main--middle--tags | removeNewline | trim',
|
||||
title: '.aditem-main .text-module-begin a | removeNewline | trim',
|
||||
link: '.aditem-main .text-module-begin a@href | removeNewline | trim',
|
||||
description: '.aditem-main .aditem-main--middle--description | removeNewline | trim',
|
||||
|
||||
@@ -5,23 +5,46 @@
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const originalId = o.id.split('/').pop();
|
||||
const id = buildHash(originalId, o.price);
|
||||
const size = o.size ?? 'N/A m²';
|
||||
const title = o.title || 'No title available';
|
||||
const link = o.link != null ? `https://www.mcmakler.de${o.link}` : o.link;
|
||||
const [rooms, size] = o.tags.split(' | ');
|
||||
const address = o.address?.replace(' / ', ' ') || null;
|
||||
const link = o.link != null ? `https://www.mcmakler.de${o.link}` : config.url;
|
||||
return Object.assign(o, { id, size, title, link, address });
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(size),
|
||||
rooms: extractNumber(rooms),
|
||||
address,
|
||||
image: o.image,
|
||||
description: undefined,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: 'article[data-testid="propertyCard"]',
|
||||
sortByDateParam: 'sortBy=DATE&sortOn=DESC',
|
||||
@@ -30,7 +53,7 @@ const config = {
|
||||
id: 'h2 a@href',
|
||||
title: 'h2 a | removeNewline | trim',
|
||||
price: 'footer > p:first-of-type | trim',
|
||||
size: 'footer > p:nth-of-type(2) | trim',
|
||||
tags: 'footer > p:nth-of-type(2) | trim',
|
||||
address: 'div > h2 + p | removeNewline | trim',
|
||||
image: 'img@src',
|
||||
link: 'h2 a@href',
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
@@ -12,19 +15,39 @@ function nullOrEmpty(val) {
|
||||
return val == null || val.length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const link = nullOrEmpty(o.link)
|
||||
? 'NO LINK'
|
||||
: `https://www.neubaukompass.de${o.link.substring(o.link.indexOf('/neubau'))}`;
|
||||
const id = buildHash(o.link, o.price);
|
||||
return Object.assign(o, { id, link });
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address: o.address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
return !isOneOf(o.title, appliedBlackList);
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: '.col-12.mb-4',
|
||||
sortByDateParam: 'Sortierung=Id&Richtung=DESC',
|
||||
@@ -34,7 +57,9 @@ const config = {
|
||||
title: 'a@title | removeNewline | trim',
|
||||
link: 'a@href',
|
||||
address: '.nbk-project-card__description | removeNewline | trim',
|
||||
price: '.nbk-project-card__spec-item .nbk-project-card__spec-value | removeNewline | trim',
|
||||
price: '.nbk-project-card__spec-item:nth-child(1) .nbk-project-card__spec-value | removeNewline | trim',
|
||||
size: '.nbk-project-card__spec-item:nth-child(2) .nbk-project-card__spec-value | removeNewline | trim',
|
||||
rooms: '.nbk-project-card__spec-item:nth-child(3) .nbk-project-card__spec-value | removeNewline | trim',
|
||||
image: '.nbk-project-card__image@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
|
||||
@@ -5,19 +5,43 @@
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const link = metaInformation.baseUrl + o.link;
|
||||
const id = buildHash(o.title, o.link, o.price);
|
||||
return Object.assign(o, { link, id });
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address: o.address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: 'div[data-livecomponent-id*="search/property_list"] .grid > div',
|
||||
sortByDateParam: null,
|
||||
@@ -27,6 +51,7 @@ const config = {
|
||||
title: 'h4 | removeNewline | trim',
|
||||
price: '.text-xl | trim',
|
||||
size: 'div[title="Wohnfläche"] | trim',
|
||||
rooms: 'div[title="Zimmer"] | trim',
|
||||
address: '.text-slate-800 | removeNewline | trim',
|
||||
image: 'img@src',
|
||||
link: 'a@href',
|
||||
|
||||
@@ -5,24 +5,47 @@
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const id = buildHash(o.id, o.price);
|
||||
const address = o.address?.replace(/^adresse /i, '') ?? null;
|
||||
const title = o.title || 'No title available';
|
||||
const link = o.link != null ? decodeURIComponent(o.link) : config.url;
|
||||
|
||||
const urlReg = new RegExp(/url\((.*?)\)/gim);
|
||||
const image = o.image != null ? urlReg.exec(o.image)[1] : null;
|
||||
return Object.assign(o, { id, address, title, link, image });
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address,
|
||||
image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: '.listentry-content',
|
||||
sortByDateParam: null, // sort by date is standard
|
||||
@@ -32,6 +55,7 @@ const config = {
|
||||
title: 'h2 | trim',
|
||||
price: '.listentry-details-price .listentry-details-v | trim',
|
||||
size: '.listentry-details-size .listentry-details-v | trim',
|
||||
rooms: '.listentry-details-rooms .listentry-details-v | trim',
|
||||
address: '.listentry-adress | trim',
|
||||
image: '.listentry-img@style',
|
||||
link: '.shariff@data-url',
|
||||
|
||||
@@ -8,12 +8,15 @@ import checkIfListingIsActive from '../services/listings/listingActiveTester.js'
|
||||
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import logger from '../services/logger.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
async function fetchDetails(listing, browser) {
|
||||
try {
|
||||
const html = await puppeteerExtractor(listing.link, 'body', { browser });
|
||||
const html = await puppeteerExtractor(listing.link, 'body', { browser, name: 'sparkasse_details' });
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
const nextDataRaw = $('#__NEXT_DATA__').text;
|
||||
@@ -55,20 +58,39 @@ async function fetchDetails(listing, browser) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const originalId = o.id.split('/').pop().replace('.html', '');
|
||||
const id = buildHash(originalId, o.price);
|
||||
const size = o.size?.replace(' Wohnfläche', '').replace(' m²', 'm²') ?? null;
|
||||
const title = o.title || 'No title available';
|
||||
const link = o.link != null ? `https://immobilien.sparkasse.de${o.link}` : config.url;
|
||||
return Object.assign(o, { id, size, title, link });
|
||||
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address: o.address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
crawlContainer: 'div[data-testid="estate-link"]',
|
||||
sortByDateParam: 'sortBy=date_desc',
|
||||
@@ -77,7 +99,8 @@ const config = {
|
||||
id: 'a@href',
|
||||
title: 'h3 | trim',
|
||||
price: '.estate-list-price | trim',
|
||||
size: '.estate-mainfact span | trim',
|
||||
size: '.estate-mainfact:nth-child(1) span | trim',
|
||||
rooms: '.estate-mainfact:nth-child(2) span | trim',
|
||||
address: 'h6 | trim',
|
||||
image: 'img@src',
|
||||
link: 'a@href',
|
||||
|
||||
@@ -5,15 +5,18 @@
|
||||
|
||||
import { isOneOf, buildHash } from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
|
||||
import * as cheerio from 'cheerio';
|
||||
import logger from '../services/logger.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
async function fetchDetails(listing, browser) {
|
||||
try {
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser });
|
||||
const html = await puppeteerExtractor(listing.link, null, { browser, name: 'wgGesucht_details' });
|
||||
if (!html) return listing;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
@@ -32,20 +35,39 @@ async function fetchDetails(listing, browser) {
|
||||
return listing;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const id = buildHash(o.id, o.price);
|
||||
const link = `https://www.wg-gesucht.de${o.link}`;
|
||||
const image = o.image != null ? o.image.replace('small', 'large') : null;
|
||||
return Object.assign(o, { id, link, image });
|
||||
const [rooms, city, road] = o.details?.split(' | ') || [];
|
||||
return {
|
||||
id,
|
||||
link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(rooms),
|
||||
address: `${city}, ${road}`,
|
||||
image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
|
||||
return o.id != null && titleNotBlacklisted && descNotBlacklisted;
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
url: null,
|
||||
crawlContainer: '#main_column .wgg_card',
|
||||
@@ -56,10 +78,13 @@ const config = {
|
||||
details: '.row .noprint .col-xs-11 |removeNewline |trim',
|
||||
price: '.middle .col-xs-3 |removeNewline |trim',
|
||||
size: '.middle .text-right |removeNewline |trim',
|
||||
rooms: '.middle .text-right |removeNewline |trim',
|
||||
title: '.truncate_title a |removeNewline |trim',
|
||||
link: '.truncate_title a@href',
|
||||
image: '.img-responsive@src',
|
||||
description: '.row .noprint .col-xs-11 |removeNewline |trim',
|
||||
},
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
fetchDetails,
|
||||
|
||||
@@ -5,26 +5,45 @@
|
||||
|
||||
import * as utils from '../utils.js';
|
||||
import checkIfListingIsActive from '../services/listings/listingActiveTester.js';
|
||||
import { extractNumber } from '../utils/extract-number.js';
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
/** @import { ProviderConfig } from '../types/providerConfig.js' */
|
||||
|
||||
let appliedBlackList = [];
|
||||
|
||||
/**
|
||||
* @param {any} o
|
||||
* @returns {ParsedListing}
|
||||
*/
|
||||
function normalize(o) {
|
||||
const id = o.link.split('/').pop();
|
||||
const price = o.price;
|
||||
const size = o.size;
|
||||
const rooms = o.rooms;
|
||||
const [city = '', part = ''] = (o.description || '').split('-').map((v) => v.trim());
|
||||
const address = `${part}, ${city}`;
|
||||
return Object.assign(o, { id, price, size, rooms, address });
|
||||
return {
|
||||
id: o.link.split('/').pop(),
|
||||
link: o.link,
|
||||
title: o.title || '',
|
||||
price: extractNumber(o.price),
|
||||
size: extractNumber(o.size),
|
||||
rooms: extractNumber(o.rooms),
|
||||
address,
|
||||
image: o.image,
|
||||
description: o.description,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ParsedListing} o
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function applyBlacklist(o) {
|
||||
const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
|
||||
const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
|
||||
return o.id != null && o.title != null && titleNotBlacklisted && descNotBlacklisted && o.link.startsWith(o.link);
|
||||
}
|
||||
|
||||
/** @type {ProviderConfig} */
|
||||
const config = {
|
||||
requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
|
||||
url: null,
|
||||
sortByDateParam: null,
|
||||
waitForSelector: 'body',
|
||||
@@ -37,7 +56,7 @@ const config = {
|
||||
size: 'dl:nth-of-type(3) dd | removeNewline | trim',
|
||||
description: 'div.before\\:icon-location_marker | trim',
|
||||
link: '@href',
|
||||
imageUrl: 'img@src',
|
||||
image: 'img@src',
|
||||
},
|
||||
normalize: normalize,
|
||||
filter: applyBlacklist,
|
||||
|
||||
147
lib/services/ensureValidBinary.js
Normal file
147
lib/services/ensureValidBinary.js
Normal file
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { ensureBinary } from 'cloakbrowser';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
|
||||
/**
|
||||
* Resource files required on Linux/Windows — they must live next to the chrome binary.
|
||||
* macOS packages these inside the .app bundle's Frameworks directory so a different
|
||||
* check is used there (see isBinaryComplete).
|
||||
*/
|
||||
const LINUX_WIN_REQUIRED_FILES = ['icudtl.dat', 'resources.pak'];
|
||||
|
||||
/**
|
||||
* Return the top-level versioned installation directory for any platform.
|
||||
*
|
||||
* - Linux/Windows: binaryPath is ~/.cloakbrowser/chromium-X.Y.Z/chrome
|
||||
* → dirname ~/.cloakbrowser/chromium-X.Y.Z/
|
||||
* - macOS: binaryPath is ~/.cloakbrowser/chromium-X.Y.Z/Chromium.app/Contents/MacOS/Chromium
|
||||
* → 4 levels up ~/.cloakbrowser/chromium-X.Y.Z/
|
||||
*
|
||||
* @param {string} binaryPath
|
||||
* @returns {string}
|
||||
*/
|
||||
function getVersionedDir(binaryPath) {
|
||||
if (process.platform === 'darwin') {
|
||||
return path.resolve(path.dirname(binaryPath), '../../..');
|
||||
}
|
||||
return path.dirname(binaryPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true when the binary at binaryPath belongs to a complete installation.
|
||||
*
|
||||
* On macOS the binary lives inside an .app bundle:
|
||||
* Chromium.app/Contents/MacOS/Chromium
|
||||
* Resource files (icudtl.dat etc.) are deep inside
|
||||
* Chromium.app/Contents/Frameworks/…
|
||||
* so checking for them next to the binary is wrong. Instead we verify the two
|
||||
* structural markers that are only present after a full extraction: Info.plist
|
||||
* and the Frameworks directory inside Contents/.
|
||||
*
|
||||
* On Linux/Windows the binary and all resource files are siblings in the same
|
||||
* directory.
|
||||
*
|
||||
* @param {string} binaryPath
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isBinaryComplete(binaryPath) {
|
||||
if (process.platform === 'darwin') {
|
||||
const contentsDir = path.resolve(path.dirname(binaryPath), '..');
|
||||
return fs.existsSync(path.join(contentsDir, 'Info.plist')) && fs.existsSync(path.join(contentsDir, 'Frameworks'));
|
||||
}
|
||||
const dir = path.dirname(binaryPath);
|
||||
return LINUX_WIN_REQUIRED_FILES.every((f) => fs.existsSync(path.join(dir, f)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a human-readable description of which required files/dirs are missing.
|
||||
*
|
||||
* @param {string} binaryPath
|
||||
* @returns {string}
|
||||
*/
|
||||
function missingDescription(binaryPath) {
|
||||
if (process.platform === 'darwin') {
|
||||
const contentsDir = path.resolve(path.dirname(binaryPath), '..');
|
||||
return ['Info.plist', 'Frameworks'].filter((f) => !fs.existsSync(path.join(contentsDir, f))).join(', ');
|
||||
}
|
||||
const dir = path.dirname(binaryPath);
|
||||
return LINUX_WIN_REQUIRED_FILES.filter((f) => !fs.existsSync(path.join(dir, f))).join(', ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a corrupt binary installation and all `latest_version*` markers from
|
||||
* the CloakBrowser cache so the next `ensureBinary()` call falls back to the
|
||||
* package-bundled version.
|
||||
*
|
||||
* Removes the full versioned directory (e.g. chromium-X.Y.Z/) on all platforms,
|
||||
* not just the subdirectory that contains the binary.
|
||||
*
|
||||
* @param {string} binaryPath - Path to the (corrupt) chrome/Chromium binary.
|
||||
*/
|
||||
function removeCorruptInstallation(binaryPath) {
|
||||
const versionedDir = getVersionedDir(binaryPath);
|
||||
const cacheDir = process.env.CLOAKBROWSER_CACHE_DIR || path.join(os.homedir(), '.cloakbrowser');
|
||||
|
||||
fs.rmSync(versionedDir, { recursive: true, force: true });
|
||||
|
||||
try {
|
||||
for (const entry of fs.readdirSync(cacheDir)) {
|
||||
if (entry.startsWith('latest_version')) {
|
||||
fs.rmSync(path.join(cacheDir, entry), { force: true });
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Cache dir may not exist if versionedDir was the only entry — ignore.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the CloakBrowser stealth Chromium binary is present **and** complete.
|
||||
*
|
||||
* `cloakbrowser`'s own `ensureBinary()` only checks that the chrome/Chromium
|
||||
* file exists. An incomplete extraction (e.g. interrupted download, disk full)
|
||||
* can leave a directory that contains the executable but is missing essential
|
||||
* resource files. Chrome then crashes immediately on launch.
|
||||
*
|
||||
* This wrapper validates the path returned by `ensureBinary()`. If the
|
||||
* installation is incomplete it removes the corrupt directory, clears the
|
||||
* version marker files, and calls `ensureBinary()` again so it falls back to
|
||||
* (or re-downloads) a complete build.
|
||||
*
|
||||
* The validated path is also pinned via `CLOAKBROWSER_BINARY_PATH` so that
|
||||
* CloakBrowser's own internal `ensureBinary()` call inside `launch()` always
|
||||
* picks up the same, verified binary.
|
||||
*
|
||||
* @returns {Promise<string>} Absolute path to the validated binary.
|
||||
* @throws {Error} When even the fallback binary is incomplete.
|
||||
*/
|
||||
export async function ensureValidBinary() {
|
||||
const binaryPath = await ensureBinary();
|
||||
|
||||
if (isBinaryComplete(binaryPath)) {
|
||||
process.env.CLOAKBROWSER_BINARY_PATH = binaryPath;
|
||||
return binaryPath;
|
||||
}
|
||||
|
||||
console.warn(
|
||||
`[fredy] CloakBrowser installation at ${getVersionedDir(binaryPath)} is missing: ${missingDescription(binaryPath)}. Removing and retrying.`,
|
||||
);
|
||||
|
||||
removeCorruptInstallation(binaryPath);
|
||||
|
||||
const fallbackPath = await ensureBinary();
|
||||
if (!isBinaryComplete(fallbackPath)) {
|
||||
throw new Error(
|
||||
`CloakBrowser binary at ${getVersionedDir(fallbackPath)} is still missing required files after re-download: ${missingDescription(fallbackPath)}`,
|
||||
);
|
||||
}
|
||||
|
||||
process.env.CLOAKBROWSER_BINARY_PATH = fallbackPath;
|
||||
return fallbackPath;
|
||||
}
|
||||
@@ -94,7 +94,7 @@ export async function applyBotPreventionToPage(page, cfg) {
|
||||
// webdriver
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
|
||||
// chrome runtime — expose loadTimes, csi and app like real Chrome
|
||||
// chrome runtime - expose loadTimes, csi and app like real Chrome
|
||||
// @ts-ignore
|
||||
window.chrome = {
|
||||
runtime: {},
|
||||
@@ -129,7 +129,7 @@ export async function applyBotPreventionToPage(page, cfg) {
|
||||
get: () => (window.localStorage.getItem('__LANGS__') || 'de-DE,de').split(','),
|
||||
});
|
||||
|
||||
// plugins — mimic real Chrome's built-in PDF plugins
|
||||
// plugins - mimic real Chrome's built-in PDF plugins
|
||||
const makePlugin = (name, filename, description, mimeType, mimeTypeSuffix) => {
|
||||
const mimeObj = { type: mimeType, suffixes: mimeTypeSuffix, description, enabledPlugin: null };
|
||||
const plugin = { name, filename, description, length: 1, 0: mimeObj };
|
||||
@@ -274,14 +274,14 @@ export async function applyBotPreventionToPage(page, cfg) {
|
||||
//noop
|
||||
}
|
||||
|
||||
// document.hasFocus — headless returns false; real active tabs return true
|
||||
// document.hasFocus - headless returns false; real active tabs return true
|
||||
try {
|
||||
document.hasFocus = () => true;
|
||||
} catch {
|
||||
//noop
|
||||
}
|
||||
|
||||
// screen color depth — normalise in case headless reports 0
|
||||
// screen color depth - normalise in case headless reports 0
|
||||
try {
|
||||
Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
|
||||
Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
|
||||
|
||||
@@ -29,11 +29,12 @@ export default class Extractor {
|
||||
* your response will never contain what you are really looking for
|
||||
* @param url
|
||||
* @param waitForSelector
|
||||
* @param jobKey
|
||||
*/
|
||||
execute = async (url, waitForSelector = null) => {
|
||||
execute = async (url, waitForSelector = null, jobKey = null) => {
|
||||
this.responseText = null;
|
||||
try {
|
||||
this.responseText = await puppeteerExtractor(url, waitForSelector, this.options);
|
||||
this.responseText = await puppeteerExtractor(url, waitForSelector, { ...this.options, name: jobKey });
|
||||
if (this.responseText != null) {
|
||||
loadParser(this.responseText);
|
||||
}
|
||||
|
||||
@@ -3,121 +3,135 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
|
||||
import { launch } from 'cloakbrowser/puppeteer';
|
||||
import { debug, botDetected } from './utils.js';
|
||||
import {
|
||||
getPreLaunchConfig,
|
||||
applyBotPreventionToPage,
|
||||
applyLanguagePersistence,
|
||||
applyPostNavigationHumanSignals,
|
||||
} from './botPrevention.js';
|
||||
import { getPreLaunchConfig } from './botPrevention.js';
|
||||
import logger from '../logger.js';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
puppeteer.use(StealthPlugin());
|
||||
import { trackPoi } from '../tracking/Tracker.js';
|
||||
import { TRACKING_POIS } from '../../TRACKING_POIS.js';
|
||||
|
||||
/**
|
||||
* Launch a CloakBrowser/Puppeteer browser instance with stealth and humanizer enabled.
|
||||
*
|
||||
* CloakBrowser applies 49 C++ source-level patches (canvas, WebGL, audio, WebRTC,
|
||||
* navigator.*, automation signals) that are indistinguishable from a real browser.
|
||||
* All fingerprinting and human-behaviour simulation is handled natively; no CDP
|
||||
* overrides (setUserAgent, setExtraHTTPHeaders, evaluateOnNewDocument) are applied
|
||||
* here because they would create detectable inconsistencies on top of the C++ patches.
|
||||
*
|
||||
* @param {string} url - Initial URL (used to derive locale/timezone hints).
|
||||
* @param {object} [options]
|
||||
* @param {boolean} [options.puppeteerHeadless]
|
||||
* @param {number} [options.puppeteerTimeout]
|
||||
* @param {string} [options.proxyUrl]
|
||||
* @param {string} [options.timezone]
|
||||
* @param {string} [options.acceptLanguage]
|
||||
* @param {object} [options.viewport]
|
||||
* @returns {Promise<import('puppeteer-core').Browser>}
|
||||
*/
|
||||
export async function launchBrowser(url, options) {
|
||||
const preCfg = getPreLaunchConfig(url, options || {});
|
||||
const launchArgs = [
|
||||
|
||||
// Docker requires --no-sandbox; CloakBrowser handles all stealth args internally.
|
||||
// --ignore-certificate-errors is needed because CloakBrowser ships its own Chromium
|
||||
// binary with an independent CA bundle that may not trust proxies or interceptors
|
||||
// present in the host environment.
|
||||
const args = [
|
||||
'--no-sandbox',
|
||||
'--disable-gpu',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-crash-reporter',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
preCfg.langArg,
|
||||
'--ignore-certificate-errors',
|
||||
// Disables the zygote process model. Required in some container environments
|
||||
// (e.g. limited kernel namespaces) where the zygote cannot acquire the
|
||||
// locks it needs and exits with "Invalid file descriptor to ICU data received".
|
||||
'--no-zygote',
|
||||
preCfg.windowSizeArg,
|
||||
...preCfg.extraArgs,
|
||||
];
|
||||
if (options?.proxyUrl) {
|
||||
launchArgs.push(`--proxy-server=${options.proxyUrl}`);
|
||||
}
|
||||
|
||||
let userDataDir;
|
||||
let removeUserDataDir = false;
|
||||
if (options && options.userDataDir) {
|
||||
userDataDir = options.userDataDir;
|
||||
} else {
|
||||
const prefix = path.join(os.tmpdir(), 'puppeteer-fredy-');
|
||||
userDataDir = fs.mkdtempSync(prefix);
|
||||
removeUserDataDir = true;
|
||||
}
|
||||
|
||||
// On ARM64 Docker, Chrome for Testing has no native binary — use system Chromium instead.
|
||||
const executablePath =
|
||||
options?.executablePath ||
|
||||
(process.arch === 'arm64' && process.env.IS_DOCKER === 'true' ? '/usr/bin/chromium' : undefined);
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
const browser = await launch({
|
||||
headless: options?.puppeteerHeadless ?? true,
|
||||
args: launchArgs,
|
||||
timeout: options?.puppeteerTimeout || 45_000,
|
||||
userDataDir,
|
||||
executablePath,
|
||||
humanize: true,
|
||||
args,
|
||||
// locale sets Accept-Language headers and JS navigator.language consistently
|
||||
locale: preCfg.langForFlag,
|
||||
...(options?.proxyUrl ? { proxy: options.proxyUrl } : {}),
|
||||
...(preCfg.timezone ? { timezone: preCfg.timezone } : {}),
|
||||
});
|
||||
|
||||
browser.__fredy_userDataDir = userDataDir;
|
||||
browser.__fredy_removeUserDataDir = removeUserDataDir;
|
||||
|
||||
return browser;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close a browser instance returned by {@link launchBrowser}.
|
||||
*
|
||||
* @param {import('puppeteer-core').Browser | null} browser
|
||||
*/
|
||||
export async function closeBrowser(browser) {
|
||||
if (!browser) return;
|
||||
const userDataDir = browser.__fredy_userDataDir;
|
||||
const removeUserDataDir = browser.__fredy_removeUserDataDir;
|
||||
try {
|
||||
await browser.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (removeUserDataDir && userDataDir) {
|
||||
try {
|
||||
await fs.promises.rm(userDataDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a page in a (possibly reused) browser, navigate to `url`, and return the HTML source.
|
||||
* Returns `null` when a bot-detection page is encountered or on timeout.
|
||||
*
|
||||
* @param {string} url
|
||||
* @param {string | null} waitForSelector
|
||||
* @param {object} [options]
|
||||
* @returns {Promise<string | null>}
|
||||
*/
|
||||
export default async function execute(url, waitForSelector, options) {
|
||||
let browser = options?.browser;
|
||||
let isExternalBrowser = !!browser;
|
||||
let page;
|
||||
let result;
|
||||
try {
|
||||
debug(`Sending request to ${url} using Puppeteer.`);
|
||||
debug(`Sending request to ${url} using CloakBrowser.`);
|
||||
|
||||
if (!isExternalBrowser) {
|
||||
browser = await launchBrowser(url, options);
|
||||
}
|
||||
|
||||
page = await browser.newPage();
|
||||
const preCfg = getPreLaunchConfig(url, options || {});
|
||||
await applyBotPreventionToPage(page, preCfg);
|
||||
// Provide languages value before navigation
|
||||
await applyLanguagePersistence(page, preCfg);
|
||||
|
||||
// Optional cookies
|
||||
if (Array.isArray(options?.cookies) && options.cookies.length > 0) {
|
||||
await page.setCookie(...options.cookies);
|
||||
}
|
||||
|
||||
// Navigation
|
||||
// Warm-up navigation: visit a trusted page first so the site sees an
|
||||
// established session before the actual target URL. Silently ignored on
|
||||
// failure so it never blocks the main request.
|
||||
if (options?.preNavigateUrl) {
|
||||
try {
|
||||
await page.goto(options.preNavigateUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await new Promise((r) => setTimeout(r, 1500 + Math.random() * 2000));
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
const response = await page.goto(url, {
|
||||
waitUntil: options?.waitUntil || 'domcontentloaded',
|
||||
timeout: options?.puppeteerTimeout || 60000,
|
||||
});
|
||||
|
||||
// Optionally wait and add subtle human-like interactions
|
||||
await applyPostNavigationHumanSignals(page, preCfg);
|
||||
// Optional second idle wait: useful for React SPAs that trigger API calls
|
||||
// after domcontentloaded. Times out silently so we use whatever is rendered.
|
||||
if (options?.waitForNetworkIdle) {
|
||||
try {
|
||||
await page.waitForNetworkIdle({ timeout: options?.waitForNetworkIdleTimeout ?? 60_000 });
|
||||
} catch {
|
||||
// ignore — we proceed with whatever the DOM contains at this point
|
||||
}
|
||||
}
|
||||
|
||||
let pageSource;
|
||||
// if we're extracting data from a SPA, we must wait for the selector
|
||||
if (waitForSelector != null) {
|
||||
const selectorTimeout = options?.puppeteerSelectorTimeout ?? options?.puppeteerTimeout ?? 30_000;
|
||||
await page.waitForSelector(waitForSelector, { timeout: selectorTimeout });
|
||||
@@ -133,15 +147,22 @@ export default async function execute(url, waitForSelector, options) {
|
||||
|
||||
if (botDetected(pageSource, statusCode)) {
|
||||
logger.warn('We have been detected as a bot :-/ Tried url: => ', url);
|
||||
|
||||
if (options != null && options.name != null) {
|
||||
await trackPoi(TRACKING_POIS.DETECTED_AS_BOT + '_' + options.name);
|
||||
} else {
|
||||
await trackPoi(TRACKING_POIS.DETECTED_AS_BOT);
|
||||
}
|
||||
|
||||
result = null;
|
||||
} else {
|
||||
result = pageSource || (await page.content());
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.name?.includes('Timeout')) {
|
||||
logger.debug('Error executing with puppeteer executor', error);
|
||||
logger.debug('Error executing with CloakBrowser executor', error);
|
||||
} else {
|
||||
logger.warn('Error executing with puppeteer executor', error);
|
||||
logger.warn('Error executing with CloakBrowser executor', error);
|
||||
}
|
||||
result = null;
|
||||
} finally {
|
||||
|
||||
@@ -168,10 +168,6 @@ export function convertWebToMobile(webUrl) {
|
||||
}
|
||||
}
|
||||
|
||||
if (segments.includes('shape')) {
|
||||
throw new Error('Shape is currently not supported using Immoscout');
|
||||
}
|
||||
|
||||
const { query: rawParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
|
||||
const webParams = Object.fromEntries(
|
||||
Object.entries(rawParams).filter(([key]) => key !== 'enteredFrom' && PARAM_NAME_MAP[key]),
|
||||
@@ -179,18 +175,31 @@ export function convertWebToMobile(webUrl) {
|
||||
|
||||
const geocodes = `/${segments.slice(2, segments.length - 1).join('/')}`;
|
||||
const isRadius = segments.includes('radius');
|
||||
const isShape = segments.includes('shape');
|
||||
const mobileParams = {
|
||||
searchType: isRadius ? 'radius' : 'region',
|
||||
searchType: isRadius ? 'radius' : isShape ? 'shape' : 'region',
|
||||
realestatetype: realType,
|
||||
...(isRadius ? {} : { geocodes }),
|
||||
...(isRadius || isShape ? {} : { geocodes }),
|
||||
...additionalParamsFromWebPath,
|
||||
};
|
||||
|
||||
if (isShape && !webParams.shape) {
|
||||
throw new Error('Shape search URL is missing the required "shape" query parameter');
|
||||
}
|
||||
|
||||
if (isShape && webParams.shape) {
|
||||
const browserShape = webParams.shape;
|
||||
const normalized = browserShape.replace(/\.\./g, '==').replace(/\./g, '=');
|
||||
const polyline = Buffer.from(normalized, 'base64').toString('utf-8');
|
||||
mobileParams.shape = polyline;
|
||||
}
|
||||
|
||||
if (webParams.geocoordinates) {
|
||||
mobileParams.geocoordinates = webParams.geocoordinates;
|
||||
}
|
||||
|
||||
for (const [key, val] of Object.entries(webParams)) {
|
||||
if (key === 'shape') continue;
|
||||
if (key === 'equipment') {
|
||||
const items = [].concat(val).flatMap((v) => `${v}`.split(','));
|
||||
const currentEquipmentParams = mobileParams[PARAM_NAME_MAP[key]];
|
||||
|
||||
@@ -178,15 +178,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) {
|
||||
browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {});
|
||||
}
|
||||
|
||||
await new FredyPipelineExecutioner(
|
||||
matchedProvider.config,
|
||||
job.notificationAdapter,
|
||||
job.spatialFilter,
|
||||
prov.id,
|
||||
job.id,
|
||||
similarityCache,
|
||||
browser,
|
||||
).execute();
|
||||
await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute();
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ export const upsertJob = ({
|
||||
userId,
|
||||
shareWithUsers = [],
|
||||
spatialFilter = null,
|
||||
specFilter = null,
|
||||
}) => {
|
||||
const id = jobId || nanoid();
|
||||
const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0];
|
||||
@@ -44,7 +45,8 @@ export const upsertJob = ({
|
||||
provider = @provider,
|
||||
notification_adapter = @notification_adapter,
|
||||
shared_with_user = @shareWithUsers,
|
||||
spatial_filter = @spatialFilter
|
||||
spatial_filter = @spatialFilter,
|
||||
spec_filter = @specFilter
|
||||
WHERE id = @id`,
|
||||
{
|
||||
id,
|
||||
@@ -55,12 +57,13 @@ export const upsertJob = ({
|
||||
provider: toJson(provider ?? []),
|
||||
notification_adapter: toJson(notificationAdapter ?? []),
|
||||
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
|
||||
specFilter: specFilter ? toJson(specFilter) : null,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
SqliteConnection.execute(
|
||||
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter)
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`,
|
||||
`INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter, spec_filter)
|
||||
VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter, @specFilter)`,
|
||||
{
|
||||
id,
|
||||
user_id: ownerId,
|
||||
@@ -71,6 +74,7 @@ export const upsertJob = ({
|
||||
shareWithUsers: toJson(shareWithUsers ?? []),
|
||||
notification_adapter: toJson(notificationAdapter ?? []),
|
||||
spatialFilter: spatialFilter ? toJson(spatialFilter) : null,
|
||||
specFilter: specFilter ? toJson(specFilter) : null,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -92,6 +96,7 @@ export const getJob = (jobId) => {
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
j.spatial_filter AS spatialFilter,
|
||||
j.spec_filter AS specFilter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
WHERE j.id = @id
|
||||
@@ -107,6 +112,7 @@ export const getJob = (jobId) => {
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
spatialFilter: fromJson(row.spatialFilter, null),
|
||||
specFilter: fromJson(row.specFilter, null),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -157,6 +163,7 @@ export const getJobs = () => {
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
j.spatial_filter AS spatialFilter,
|
||||
j.spec_filter AS specFilter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
WHERE j.enabled = 1
|
||||
@@ -170,6 +177,7 @@ export const getJobs = () => {
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
spatialFilter: fromJson(row.spatialFilter, null),
|
||||
specFilter: fromJson(row.specFilter, null),
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -260,6 +268,7 @@ export const queryJobs = ({
|
||||
j.shared_with_user,
|
||||
j.notification_adapter AS notificationAdapter,
|
||||
j.spatial_filter AS spatialFilter,
|
||||
j.spec_filter AS specFilter,
|
||||
(SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings
|
||||
FROM jobs j
|
||||
${whereSql}
|
||||
@@ -276,6 +285,7 @@ export const queryJobs = ({
|
||||
shared_with_user: fromJson(row.shared_with_user, []),
|
||||
notificationAdapter: fromJson(row.notificationAdapter, []),
|
||||
spatialFilter: fromJson(row.spatialFilter, null),
|
||||
specFilter: fromJson(row.specFilter, null),
|
||||
}));
|
||||
|
||||
return { totalNumber, page: safePage, result };
|
||||
|
||||
@@ -29,33 +29,47 @@ export const getKnownListingHashesForJobAndProvider = (jobId, providerId) => {
|
||||
* Compute KPI aggregates for a given set of job IDs from the listings table.
|
||||
*
|
||||
* - numberOfActiveListings: count of listings where is_active = 1
|
||||
* - avgPriceOfListings: average of numeric price, rounded to nearest integer
|
||||
* - medianPriceOfListings: median of numeric price, rounded to nearest integer
|
||||
*
|
||||
* When no jobIds are provided, returns zeros.
|
||||
*
|
||||
* @param {string[]} jobIds
|
||||
* @returns {{ numberOfActiveListings: number, avgPriceOfListings: number }}
|
||||
* @returns {{ numberOfActiveListings: number, medianPriceOfListings: number }}
|
||||
*/
|
||||
export const getListingsKpisForJobIds = (jobIds = []) => {
|
||||
if (!Array.isArray(jobIds) || jobIds.length === 0) {
|
||||
return { numberOfActiveListings: 0, avgPriceOfListings: 0 };
|
||||
return { numberOfActiveListings: 0, medianPriceOfListings: 0 };
|
||||
}
|
||||
|
||||
const placeholders = jobIds.map(() => '?').join(',');
|
||||
const row =
|
||||
SqliteConnection.query(
|
||||
`SELECT
|
||||
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) AS activeCount,
|
||||
AVG(price) AS avgPrice
|
||||
FROM listings
|
||||
WHERE job_id IN (${placeholders})
|
||||
AND manually_deleted = 0`,
|
||||
jobIds,
|
||||
)[0] || {};
|
||||
const rows = SqliteConnection.query(
|
||||
`SELECT
|
||||
SUM(CASE WHEN is_active = 1 THEN 1 ELSE 0 END) OVER() AS active_count,
|
||||
price
|
||||
FROM listings
|
||||
WHERE job_id IN (${placeholders})
|
||||
AND manually_deleted = 0
|
||||
GROUP BY
|
||||
id`,
|
||||
jobIds,
|
||||
);
|
||||
|
||||
const activeCount = rows[0]?.active_count ?? 0;
|
||||
|
||||
const prices = rows
|
||||
.map((r) => r.price)
|
||||
.filter((p) => p !== null)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
let medianPrice = 0;
|
||||
if (prices.length > 0) {
|
||||
const mid = Math.floor(prices.length / 2);
|
||||
medianPrice = prices.length % 2 !== 0 ? prices[mid] : (prices[mid - 1] + prices[mid]) / 2;
|
||||
}
|
||||
|
||||
return {
|
||||
numberOfActiveListings: Number(row.activeCount || 0),
|
||||
avgPriceOfListings: row?.avgPrice == null ? 0 : Math.round(Number(row.avgPrice)),
|
||||
numberOfActiveListings: activeCount,
|
||||
medianPriceOfListings: medianPrice,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -174,9 +188,9 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
|
||||
SqliteConnection.withTransaction((db) => {
|
||||
const stmt = db.prepare(
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address,
|
||||
`INSERT INTO listings (id, hash, provider, job_id, price, size, rooms, title, image_url, description, address,
|
||||
link, created_at, is_active, latitude, longitude)
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link,
|
||||
VALUES (@id, @hash, @provider, @job_id, @price, @size, @rooms, @title, @image_url, @description, @address, @link,
|
||||
@created_at, 1, @latitude, @longitude)
|
||||
ON CONFLICT(job_id, hash) DO NOTHING`,
|
||||
);
|
||||
@@ -187,8 +201,9 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
hash: item.id,
|
||||
provider: providerId,
|
||||
job_id: jobId,
|
||||
price: extractNumber(item.price),
|
||||
size: extractNumber(item.size),
|
||||
price: item.price,
|
||||
size: item.size,
|
||||
rooms: item.rooms,
|
||||
title: item.title,
|
||||
image_url: item.image,
|
||||
description: item.description,
|
||||
@@ -202,19 +217,6 @@ export const storeListings = (jobId, providerId, listings) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Extract the first number from a string like "1.234 €" or "70 m²".
|
||||
* Removes dots/commas before parsing. Returns null on invalid input.
|
||||
* @param {string|undefined|null} str
|
||||
* @returns {number|null}
|
||||
*/
|
||||
function extractNumber(str) {
|
||||
if (!str) return null;
|
||||
const cleaned = str.replace(/\./g, '').replace(',', '.');
|
||||
const num = parseFloat(cleaned);
|
||||
return isNaN(num) ? null : num;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove any parentheses segments (including surrounding whitespace) from a string.
|
||||
* Returns null for empty input.
|
||||
|
||||
10
lib/services/storage/migrations/sql/15.add-listing-specs.js
Normal file
10
lib/services/storage/migrations/sql/15.add-listing-specs.js
Normal file
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE jobs ADD COLUMN spec_filter JSONB DEFAULT NULL;
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
export function up(db) {
|
||||
db.exec(`
|
||||
ALTER TABLE listings ADD COLUMN rooms INTEGER;
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { nanoid } from 'nanoid';
|
||||
import { guessBaseUrl } from '../../../../utils/detectBaseUrl.js';
|
||||
|
||||
export function up(db) {
|
||||
const exists = db.prepare(`SELECT 1 FROM settings WHERE name = 'baseUrl' AND user_id IS NULL LIMIT 1`).get();
|
||||
if (exists) return;
|
||||
|
||||
const portRow = db.prepare(`SELECT value FROM settings WHERE name = 'port' AND user_id IS NULL LIMIT 1`).get();
|
||||
let port = 9998;
|
||||
try {
|
||||
port = JSON.parse(portRow?.value ?? '9998');
|
||||
} catch {
|
||||
/* keep default */
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO settings (id, create_date, name, value, user_id)
|
||||
VALUES (@id, @create_date, 'baseUrl', @value, NULL)`,
|
||||
).run({ id: nanoid(), create_date: Date.now(), value: JSON.stringify(guessBaseUrl(port)) });
|
||||
}
|
||||
10
lib/types/browser.js
Normal file
10
lib/types/browser.js
Normal file
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import('puppeteer').Browser} Browser
|
||||
*/
|
||||
|
||||
export {};
|
||||
19
lib/types/filter.js
Normal file
19
lib/types/filter.js
Normal file
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SpecFilter
|
||||
* @property {number} [minRooms] Minimum number of rooms.
|
||||
* @property {number} [minSize] Minimum size in m².
|
||||
* @property {number} [maxPrice] Maximum price.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SpatialFilter GeoJSON FeatureCollection.
|
||||
* @property {Array<Object>} [features] GeoJSON features for spatial filtering (typically Polygons).
|
||||
* @property {string} [type] Type 'FeatureCollection'.
|
||||
*/
|
||||
|
||||
export {};
|
||||
23
lib/types/job.js
Normal file
23
lib/types/job.js
Normal file
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/** @import { SpecFilter, SpatialFilter } from './filter.js' */
|
||||
|
||||
/**
|
||||
* @typedef {Object} Job
|
||||
* @property {string} id Job ID.
|
||||
* @property {string} [userId] Owner user id.
|
||||
* @property {string} [name] Job display name.
|
||||
* @property {boolean} [enabled] Whether the job is enabled.
|
||||
* @property {Array<any>} [blacklist] Blacklist entries.
|
||||
* @property {Array<any>} [provider] Provider configuration list.
|
||||
* @property {Object} [notificationAdapter] Notification configuration.
|
||||
* @property {Array<string>} [shared_with_user] Users this job is shared with.
|
||||
* @property {SpatialFilter | null} [spatialFilter] Optional spatial filter configuration as GeoJSON FeatureCollection.
|
||||
* @property {SpecFilter | null} [specFilter] Optional listing specifications.
|
||||
* @property {number} [numberOfFoundListings] Count of active listings for this job.
|
||||
*/
|
||||
|
||||
export {};
|
||||
22
lib/types/listing.js
Normal file
22
lib/types/listing.js
Normal file
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} ParsedListing
|
||||
* @property {string} id Stable unique identifier (hash) of the listing.
|
||||
* @property {string} link Link to the listing detail page.
|
||||
* @property {string} image Link to the listing image.
|
||||
* @property {string} title Title or headline of the listing.
|
||||
* @property {string} [description] Description of the listing.
|
||||
* @property {string} [address] Optional address/location text.
|
||||
* @property {number} [price] Optional price of the listing.
|
||||
* @property {number} [size] Optional size of the listing.
|
||||
* @property {number} [rooms] Optional number of rooms.
|
||||
* @property {number} [latitude] Optional latitude.
|
||||
* @property {number} [longitude] Optional longitude.
|
||||
* @property {number} [distance_to_destination] Optional distance to destination.
|
||||
*/
|
||||
|
||||
export {};
|
||||
25
lib/types/providerConfig.js
Normal file
25
lib/types/providerConfig.js
Normal file
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/** @import { ParsedListing } from './listing.js' */
|
||||
|
||||
/**
|
||||
* @typedef {Object} ProviderConfig
|
||||
* @property {string} [url] Base URL to crawl.
|
||||
* @property {string} [sortByDateParam] Query parameter used to enforce sorting by date.
|
||||
* @property {string} [waitForSelector] CSS selector to wait for before parsing content.
|
||||
* @property {Object.<string, string>} crawlFields Mapping of field names to selectors/paths.
|
||||
* @property {string[]} requiredFieldNames List of field names that this provider supports.
|
||||
* @property {string} [crawlContainer] CSS selector for the container holding listing items.
|
||||
* @property {(raw: any) => ParsedListing} normalize Function to convert raw scraped data into a ParsedListing shape.
|
||||
* @property {(listing: ParsedListing) => boolean} filter Function to filter out unwanted listings.
|
||||
* @property {(url: string, waitForSelector?: string) => Promise<any[]>} [getListings] Optional override to fetch listings.
|
||||
* @property {(listing:ParsedListing, browser:any)=>Promise<ParsedListing>} [providerConfig.fetchDetails] Optional per-listing detail enrichment. Called in parallel for each new listing after deduplication. Receives the shared browser instance. Must always resolve (never reject).
|
||||
* @property {Object} [puppeteerOptions] Puppeteer specific options.
|
||||
* @property {boolean} [enabled] Whether the provider is enabled.
|
||||
* @property {(url: string) => Promise<number> | number} [activeTester] Function to check if a listing is still active.
|
||||
*/
|
||||
|
||||
export {};
|
||||
11
lib/types/similarityCache.js
Normal file
11
lib/types/similarityCache.js
Normal file
@@ -0,0 +1,11 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SimilarityCache
|
||||
* @property {(params: { title?: string, address?: string, price?: number|string }) => boolean} checkAndAddEntry Checks if a listing is similar and adds it if not.
|
||||
*/
|
||||
|
||||
export {};
|
||||
55
lib/utils/detectBaseUrl.js
Normal file
55
lib/utils/detectBaseUrl.js
Normal file
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import os from 'os';
|
||||
import fs from 'fs';
|
||||
|
||||
const DOCKER_BRIDGE_PREFIXES = ['172.17.', '172.18.', '172.19.', '172.20.'];
|
||||
|
||||
export function isRunningInDocker() {
|
||||
if (process.env.FREDY_DOCKER === '1') return true;
|
||||
try {
|
||||
fs.accessSync('/.dockerenv');
|
||||
return true;
|
||||
} catch {
|
||||
/* not docker */
|
||||
}
|
||||
try {
|
||||
const cgroup = fs.readFileSync('/proc/self/cgroup', 'utf8');
|
||||
return /docker|containerd|kubepods/.test(cgroup);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isDockerBridgeIp(addr) {
|
||||
return DOCKER_BRIDGE_PREFIXES.some((prefix) => addr.startsWith(prefix));
|
||||
}
|
||||
|
||||
export function detectLocalIp() {
|
||||
if (isRunningInDocker()) {
|
||||
return process.env.FREDY_HOST_IP ?? '172.17.0.1';
|
||||
}
|
||||
const ifaces = os.networkInterfaces();
|
||||
for (const preferred of ['en0', 'eth0', 'wlan0', 'ens3', 'ens18']) {
|
||||
for (const entry of ifaces[preferred] ?? []) {
|
||||
if (entry.family === 'IPv4' && !entry.internal && !isDockerBridgeIp(entry.address)) {
|
||||
return entry.address;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const iface of Object.values(ifaces)) {
|
||||
for (const entry of iface ?? []) {
|
||||
if (entry.family === 'IPv4' && !entry.internal && !isDockerBridgeIp(entry.address)) {
|
||||
return entry.address;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 'localhost';
|
||||
}
|
||||
|
||||
export function guessBaseUrl(port) {
|
||||
return `http://${detectLocalIp()}:${port}`;
|
||||
}
|
||||
18
lib/utils/extract-number.js
Normal file
18
lib/utils/extract-number.js
Normal file
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extract the first number from a string like "1.234 €" or "70 m²".
|
||||
* Removes dots/commas before parsing. Returns null on invalid input.
|
||||
* @param {string|undefined|null} str
|
||||
* @returns {number|null}
|
||||
*/
|
||||
export const extractNumber = (str) => {
|
||||
if (str == null) return 0;
|
||||
if (typeof str === 'number') return str;
|
||||
const cleaned = str.replace(/\./g, '').replace(',', '.');
|
||||
const num = parseFloat(cleaned);
|
||||
return isNaN(num) ? null : num;
|
||||
};
|
||||
29
lib/utils/formatListing.js
Normal file
29
lib/utils/formatListing.js
Normal file
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
/** @import { ParsedListing } from '../types/listing.js' */
|
||||
|
||||
/**
|
||||
* @typedef {Omit<import('../types/listing.js').ParsedListing, 'price' | 'size' | 'rooms'> & {
|
||||
* price: string | null,
|
||||
* size: string | null,
|
||||
* rooms: string | null,
|
||||
* }} FormattedListing
|
||||
*/
|
||||
|
||||
/**
|
||||
* Formats a listing's numerical fields (price, size, rooms) into strings with their respective units.
|
||||
*
|
||||
* @param {import('../types/listing.js').ParsedListing} listing The original listing object.
|
||||
* @returns {FormattedListing} A copy of the listing with formatted strings for price, size, and rooms.
|
||||
*/
|
||||
export const formatListing = (listing) => {
|
||||
return {
|
||||
...listing,
|
||||
price: listing.price != null ? `${listing.price} €` : null,
|
||||
size: listing.size != null ? `${listing.size} m²` : null,
|
||||
rooms: listing.rooms != null ? `${listing.rooms} Zimmer` : null,
|
||||
};
|
||||
};
|
||||
65
package.json
65
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fredy",
|
||||
"version": "20.2.1",
|
||||
"version": "22.0.6",
|
||||
"description": "[F]ind [R]eal [E]states [d]amn eas[y].",
|
||||
"scripts": {
|
||||
"prepare": "husky",
|
||||
@@ -11,8 +11,9 @@
|
||||
"build:frontend": "vite build",
|
||||
"format": "prettier --write \"**/*.js\"",
|
||||
"format:check": "prettier --check \"**/*.js\"",
|
||||
"test": "vitest run",
|
||||
"testGH": "vitest run --config vitest.gh.config.js",
|
||||
"test": "x-var TEST_MODE=live vitest run",
|
||||
"test:offline": "x-var TEST_MODE=offline vitest run",
|
||||
"test:download-fixtures": "node tools/testFixtures/downloadFixtures.js",
|
||||
"lint": "eslint .",
|
||||
"mcp:stdio": "node lib/mcp/stdio.js",
|
||||
"lint:fix": "yarn lint --fix",
|
||||
@@ -61,65 +62,65 @@
|
||||
"Firefox ESR"
|
||||
],
|
||||
"dependencies": {
|
||||
"@douyinfe/semi-icons": "^2.93.0",
|
||||
"@douyinfe/semi-ui": "2.93.0",
|
||||
"@douyinfe/semi-ui-19": "^2.93.0",
|
||||
"@douyinfe/semi-icons": "^2.97.0",
|
||||
"@douyinfe/semi-ui": "2.97.0",
|
||||
"@douyinfe/semi-ui-19": "^2.97.0",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/helmet": "^13.0.2",
|
||||
"@fastify/session": "^11.1.1",
|
||||
"@fastify/static": "^9.1.3",
|
||||
"@mapbox/mapbox-gl-draw": "^1.5.1",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@sendgrid/mail": "8.1.6",
|
||||
"@turf/boolean-point-in-polygon": "^7.3.4",
|
||||
"@turf/boolean-point-in-polygon": "^7.3.5",
|
||||
"@vitejs/plugin-react": "6.0.1",
|
||||
"adm-zip": "^0.5.17",
|
||||
"better-sqlite3": "^12.8.0",
|
||||
"body-parser": "2.2.2",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"chart.js": "^4.5.1",
|
||||
"cheerio": "^1.2.0",
|
||||
"cookie-session": "2.1.1",
|
||||
"cloakbrowser": "^0.3.28",
|
||||
"fastify": "^5.8.5",
|
||||
"handlebars": "4.7.9",
|
||||
"maplibre-gl": "^5.22.0",
|
||||
"nanoid": "5.1.7",
|
||||
"maplibre-gl": "^5.24.0",
|
||||
"nanoid": "5.1.11",
|
||||
"node-cron": "^4.2.1",
|
||||
"node-fetch": "3.3.2",
|
||||
"node-mailjet": "6.0.11",
|
||||
"nodemailer": "^8.0.5",
|
||||
"nodemailer": "^8.0.7",
|
||||
"p-throttle": "^8.1.0",
|
||||
"package-up": "^5.0.0",
|
||||
"puppeteer": "^24.40.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
||||
"puppeteer-core": "^24.43.1",
|
||||
"query-string": "9.3.1",
|
||||
"react": "19.2.4",
|
||||
"react": "19.2.6",
|
||||
"react-chartjs-2": "^5.3.1",
|
||||
"react-dom": "19.2.4",
|
||||
"react-dom": "19.2.6",
|
||||
"react-range-slider-input": "^3.3.5",
|
||||
"react-router": "7.14.0",
|
||||
"react-router-dom": "7.14.0",
|
||||
"resend": "^6.10.0",
|
||||
"restana": "5.1.0",
|
||||
"semver": "^7.7.4",
|
||||
"serve-static": "2.2.1",
|
||||
"react-router": "7.15.0",
|
||||
"react-router-dom": "7.15.0",
|
||||
"resend": "^6.12.3",
|
||||
"semver": "^7.8.0",
|
||||
"slack": "11.0.2",
|
||||
"vite": "8.0.7",
|
||||
"vite": "8.0.12",
|
||||
"x-var": "^3.0.1",
|
||||
"zustand": "^5.0.12"
|
||||
"zustand": "^5.0.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "7.29.0",
|
||||
"@babel/eslint-parser": "7.28.6",
|
||||
"@babel/preset-env": "7.29.2",
|
||||
"@babel/preset-env": "7.29.5",
|
||||
"@babel/preset-react": "7.28.5",
|
||||
"@eslint/js": "^10.0.1",
|
||||
"chalk": "^5.6.2",
|
||||
"eslint": "10.2.0",
|
||||
"eslint": "10.3.0",
|
||||
"eslint-config-prettier": "10.1.8",
|
||||
"eslint-plugin-react": "7.37.5",
|
||||
"globals": "^17.4.0",
|
||||
"globals": "^17.6.0",
|
||||
"history": "5.3.0",
|
||||
"husky": "9.1.7",
|
||||
"less": "4.6.4",
|
||||
"lint-staged": "16.4.0",
|
||||
"lint-staged": "17.0.4",
|
||||
"nodemon": "^3.1.14",
|
||||
"prettier": "3.8.1",
|
||||
"vitest": "^4.1.3"
|
||||
"prettier": "3.8.3",
|
||||
"vitest": "^4.1.6"
|
||||
}
|
||||
}
|
||||
|
||||
18
test/globalSetup.js
Normal file
18
test/globalSetup.js
Normal file
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { ensureValidBinary } from '../lib/services/ensureValidBinary.js';
|
||||
|
||||
/**
|
||||
* Vitest global setup — runs once in the main process before any workers start.
|
||||
* Downloads and validates the CloakBrowser stealth Chromium binary.
|
||||
* ensureValidBinary() also removes and re-downloads partial/corrupt installations
|
||||
* so tests never fail with "Invalid file descriptor to ICU data received".
|
||||
* Skipped in offline mode because the browser is fully mocked there.
|
||||
*/
|
||||
export async function setup() {
|
||||
if (process.env.TEST_MODE === 'offline') return;
|
||||
await ensureValidBinary();
|
||||
}
|
||||
@@ -21,6 +21,10 @@ export function getUserSettings(userId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function getSettings() {
|
||||
return { baseUrl: '' };
|
||||
}
|
||||
|
||||
export const updateListingDistance = (id, distance) => {
|
||||
// noop
|
||||
};
|
||||
|
||||
99
test/offlineFixtures.js
Normal file
99
test/offlineFixtures.js
Normal file
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright (c) 2026 by Christian Kellner.
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { readFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURES_DIR = path.join(__dirname, 'testFixtures');
|
||||
|
||||
const testProviderConfig = JSON.parse(
|
||||
await readFile(new URL('./provider/testProvider.json', import.meta.url), 'utf-8'),
|
||||
);
|
||||
|
||||
// hostname → providerName, built from testProvider.json
|
||||
const hostnameToProvider = {};
|
||||
// providerName → list page pathname (for distinguishing list vs detail URLs)
|
||||
const providerListPath = {};
|
||||
|
||||
for (const [name, cfg] of Object.entries(testProviderConfig)) {
|
||||
if (!cfg.url) continue;
|
||||
try {
|
||||
const parsed = new URL(cfg.url);
|
||||
hostnameToProvider[parsed.hostname] = name;
|
||||
providerListPath[name] = parsed.pathname;
|
||||
} catch {
|
||||
// skip malformed URLs
|
||||
}
|
||||
}
|
||||
|
||||
async function tryReadFile(filepath) {
|
||||
try {
|
||||
return await readFile(filepath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns fixture HTML for the given URL by mapping hostname → provider name,
|
||||
* then distinguishing list vs detail pages by comparing the URL path against
|
||||
* the configured list URL path from testProvider.json.
|
||||
*/
|
||||
export async function readFixture(url) {
|
||||
let hostname, pathname;
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
hostname = parsed.hostname;
|
||||
pathname = parsed.pathname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const providerName = hostnameToProvider[hostname];
|
||||
if (!providerName) return null;
|
||||
|
||||
if (providerListPath[providerName] === pathname) {
|
||||
return tryReadFile(path.join(FIXTURES_DIR, `${providerName}.html`));
|
||||
}
|
||||
|
||||
// Detail page: prefer dedicated detail fixture, fall back to list fixture
|
||||
const detailHtml = await tryReadFile(path.join(FIXTURES_DIR, `${providerName}_detail.html`));
|
||||
if (detailHtml) return detailHtml;
|
||||
return tryReadFile(path.join(FIXTURES_DIR, `${providerName}.html`));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a fetch replacement that intercepts immoscout mobile API calls and
|
||||
* serves pre-downloaded JSON fixtures. Throws for any other URL to prevent
|
||||
* accidental live network traffic in offline mode.
|
||||
*/
|
||||
export function buildFetchMock() {
|
||||
let listData = null;
|
||||
let detailData = null;
|
||||
|
||||
return async (url) => {
|
||||
const urlStr = String(url);
|
||||
|
||||
if (urlStr.includes('api.mobile.immobilienscout24.de/search/list')) {
|
||||
if (!listData) {
|
||||
const raw = await tryReadFile(path.join(FIXTURES_DIR, 'immoscout_list.json'));
|
||||
listData = raw ? JSON.parse(raw) : { resultListItems: [] };
|
||||
}
|
||||
return { ok: true, status: 200, json: () => Promise.resolve(listData) };
|
||||
}
|
||||
|
||||
if (urlStr.includes('api.mobile.immobilienscout24.de/expose/')) {
|
||||
if (!detailData) {
|
||||
const raw = await tryReadFile(path.join(FIXTURES_DIR, 'immoscout_detail.json'));
|
||||
detailData = raw ? JSON.parse(raw) : { sections: [], contact: {} };
|
||||
}
|
||||
return { ok: true, status: 200, json: () => Promise.resolve(detailData) };
|
||||
}
|
||||
|
||||
throw new Error(`Network request blocked in offline mode: ${urlStr}`);
|
||||
};
|
||||
}
|
||||
@@ -17,13 +17,22 @@ describe('Issue reproduction: listings filtered by similarity or area should be
|
||||
|
||||
const providerConfig = {
|
||||
url: 'http://example.com',
|
||||
getListings: () => Promise.resolve([{ id: '1', title: 'test', address: 'addr', price: '100' }]),
|
||||
getListings: () =>
|
||||
Promise.resolve([{ id: '1', title: 'test', address: 'addr', price: '100', link: 'http://example.com/1' }]),
|
||||
normalize: (l) => l,
|
||||
filter: () => true,
|
||||
crawlFields: { id: 'id', title: 'title', address: 'address', price: 'price' },
|
||||
requiredFieldNames: ['id', 'title', 'address', 'price'],
|
||||
};
|
||||
|
||||
const fredy = new Fredy(providerConfig, null, null, 'test-provider', 'test-job', mockSimilarityCache);
|
||||
const mockedJob = {
|
||||
id: 'test-job',
|
||||
notificationAdapter: null,
|
||||
specFilter: null,
|
||||
spatialFilter: null,
|
||||
};
|
||||
|
||||
const fredy = new Fredy(providerConfig, mockedJob, 'test-provider', mockSimilarityCache, undefined);
|
||||
|
||||
// Clear deletedIds before test
|
||||
mockStore.deletedIds.length = 0;
|
||||
@@ -64,18 +73,35 @@ describe('Issue reproduction: listings filtered by similarity or area should be
|
||||
],
|
||||
};
|
||||
|
||||
const mockedJob = {
|
||||
id: 'test-job',
|
||||
notificationAdapter: null,
|
||||
specFilter: null,
|
||||
spatialFilter: spatialFilter,
|
||||
};
|
||||
|
||||
const providerConfig = {
|
||||
url: 'http://example.com',
|
||||
getListings: () =>
|
||||
Promise.resolve([{ id: '2', title: 'test', address: 'addr', price: '100', latitude: 2, longitude: 2 }]), // outside polygon
|
||||
Promise.resolve([
|
||||
{
|
||||
id: '2',
|
||||
title: 'test',
|
||||
address: 'addr',
|
||||
price: '100',
|
||||
latitude: 2,
|
||||
longitude: 2,
|
||||
link: 'http://example.com/2',
|
||||
},
|
||||
]), // outside polygon
|
||||
normalize: (l) => l,
|
||||
filter: () => true,
|
||||
crawlFields: { id: 'id', title: 'title', address: 'address', price: 'price' },
|
||||
requiredFieldNames: ['id', 'title', 'address', 'price'],
|
||||
};
|
||||
|
||||
const fredy = new Fredy(providerConfig, null, spatialFilter, 'test-provider', 'test-job', mockSimilarityCache);
|
||||
const fredy = new Fredy(providerConfig, mockedJob, 'test-provider', mockSimilarityCache, undefined);
|
||||
|
||||
// Clear deletedIds before test
|
||||
mockStore.deletedIds.length = 0;
|
||||
|
||||
try {
|
||||
|
||||
@@ -10,18 +10,17 @@ import { expect } from 'vitest';
|
||||
import * as provider from '../../lib/provider/einsAImmobilien.js';
|
||||
|
||||
describe('#einsAImmobilien testsuite()', () => {
|
||||
provider.init(providerConfig.einsAImmobilien, [], []);
|
||||
provider.init(providerConfig.einsAImmobilien, []);
|
||||
it('should test einsAImmobilien provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'einsAImmobilien',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(
|
||||
provider.config,
|
||||
null,
|
||||
null,
|
||||
provider.metaInformation.id,
|
||||
'einsAImmobilien',
|
||||
similarityCache,
|
||||
);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
fredy.execute().then((listings) => {
|
||||
if (listings == null || listings.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
@@ -35,12 +34,14 @@ describe('#einsAImmobilien testsuite()', () => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).not.toBe('');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.1a-immobilienmarkt.de');
|
||||
});
|
||||
|
||||
@@ -6,70 +6,89 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { providerConfig, mockFredy } from '../utils.js';
|
||||
import { expect, vi } from 'vitest';
|
||||
import { expect } from 'vitest';
|
||||
import * as provider from '../../lib/provider/immobilienDe.js';
|
||||
import * as mockStore from '../mocks/mockStore.js';
|
||||
import { launchBrowser, closeBrowser } from '../../lib/services/extractor/puppeteerExtractor.js';
|
||||
|
||||
// One browser shared across the whole suite so both requests (search + detail)
|
||||
// come from the same warm session, avoiding double cold-start bot detection.
|
||||
const TEST_TIMEOUT = 120_000;
|
||||
|
||||
describe('#immobilien.de testsuite()', () => {
|
||||
provider.init(providerConfig.immobilienDe, [], []);
|
||||
it('should test immobilien.de provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'test1', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
|
||||
if (listing == null || listing.length === 0) {
|
||||
throw new Error('Listings is empty!');
|
||||
}
|
||||
let browser;
|
||||
let liveListings;
|
||||
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('immobilienDe');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.immobilien.de');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
beforeAll(async () => {
|
||||
browser = await launchBrowser(providerConfig.immobilienDe.url);
|
||||
}, TEST_TIMEOUT);
|
||||
|
||||
afterAll(async () => {
|
||||
await closeBrowser(browser);
|
||||
});
|
||||
|
||||
it(
|
||||
'should test immobilien.de provider',
|
||||
async () => {
|
||||
const mockedJob = {
|
||||
id: 'test1',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
|
||||
const Fredy = await mockFredy();
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, browser);
|
||||
liveListings = await fredy.execute();
|
||||
|
||||
if (liveListings == null || liveListings.length === 0) {
|
||||
throw new Error('Listings is empty!');
|
||||
}
|
||||
|
||||
expect(liveListings).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('immobilienDe');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.immobilien.de');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
|
||||
describe('with provider_details enabled', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(mockStore, 'getUserSettings').mockReturnValue({ provider_details: [provider.metaInformation.id] });
|
||||
vi.spyOn(mockStore, 'getKnownListingHashesForJobAndProvider').mockReturnValue([]);
|
||||
});
|
||||
it(
|
||||
'should enrich listings with details',
|
||||
async () => {
|
||||
if (!liveListings?.length) throw new Error('No listings from first test to enrich');
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
// Call fetchDetails directly on the first live listing — no need to
|
||||
// re-scrape the search page. The shared browser keeps the session warm.
|
||||
const enriched = await provider.config.fetchDetails(liveListings[0], browser);
|
||||
|
||||
it('should enrich listings with details', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.immobilienDe, [], []);
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'test1', {
|
||||
checkAndAddEntry: () => false,
|
||||
});
|
||||
const listings = await fredy.execute();
|
||||
if (listings == null) return;
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
listings.forEach((listing) => {
|
||||
expect(listing.link).toContain('https://www.immobilien.de');
|
||||
expect(listing.address).toBeTypeOf('string');
|
||||
expect(listing.address).not.toBe('');
|
||||
if (enriched == null) return;
|
||||
expect(enriched.link).toContain('https://www.immobilien.de');
|
||||
expect(enriched.address).toBeTypeOf('string');
|
||||
expect(enriched.address).not.toBe('');
|
||||
// description may be null if selectors don't match yet — falls back gracefully
|
||||
if (listing.description != null) {
|
||||
expect(listing.description).toBeTypeOf('string');
|
||||
if (enriched.description != null) {
|
||||
expect(enriched.description).toBeTypeOf('string');
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,69 +3,85 @@
|
||||
* Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
|
||||
*/
|
||||
|
||||
import { expect, vi } from 'vitest';
|
||||
import { expect } from 'vitest';
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import * as provider from '../../lib/provider/immoscout.js';
|
||||
import * as mockStore from '../mocks/mockStore.js';
|
||||
|
||||
// immoscout uses the mobile REST API (fetch-based, no browser). Both tests share
|
||||
// the same module-level listings so the API is only queried once, avoiding
|
||||
// duplicate requests that could trigger rate-limiting.
|
||||
const TEST_TIMEOUT = 120_000;
|
||||
|
||||
describe('#immoscout provider testsuite()', () => {
|
||||
provider.init(providerConfig.immoscout, [], []);
|
||||
it('should test immoscout provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, '', similarityCache);
|
||||
fredy.execute().then((listings) => {
|
||||
if (listings == null || listings.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
return;
|
||||
}
|
||||
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('immoscout');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).not.toBe('');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.immobilienscout24.de/');
|
||||
let liveListings;
|
||||
|
||||
it(
|
||||
'should test immoscout provider',
|
||||
async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: '',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
fredy.execute().then((listings) => {
|
||||
if (listings == null || listings.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
return;
|
||||
}
|
||||
|
||||
liveListings = listings;
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
|
||||
// check if there is at least one valid notification
|
||||
const hasValidNotification = notificationObj.payload.some((notify) => {
|
||||
return (
|
||||
typeof notify.id === 'string' &&
|
||||
typeof notify.price === 'string' &&
|
||||
notify.price.includes('€') &&
|
||||
typeof notify.size === 'string' &&
|
||||
notify.size.includes('m²') &&
|
||||
typeof notify.title === 'string' &&
|
||||
notify.title !== '' &&
|
||||
typeof notify.link === 'string' &&
|
||||
notify.link.includes('https://www.immobilienscout24.de/') &&
|
||||
typeof notify.address === 'string'
|
||||
);
|
||||
});
|
||||
|
||||
expect(hasValidNotification).toBe(true);
|
||||
resolve();
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
|
||||
describe('with provider_details enabled', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(mockStore, 'getUserSettings').mockReturnValue({ provider_details: [provider.metaInformation.id] });
|
||||
vi.spyOn(mockStore, 'getKnownListingHashesForJobAndProvider').mockReturnValue([]);
|
||||
});
|
||||
it(
|
||||
'should enrich listings with details',
|
||||
async () => {
|
||||
if (!liveListings?.length) throw new Error('No listings from first test to enrich');
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
// Call fetchDetails directly on the first live listing — no need to
|
||||
// re-query the search API. immoscout uses fetch (no browser).
|
||||
const enriched = await provider.config.fetchDetails(liveListings[0]);
|
||||
|
||||
it('should enrich listings with details', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.immoscout, [], []);
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, '', {
|
||||
checkAndAddEntry: () => false,
|
||||
});
|
||||
const listings = await fredy.execute();
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
listings.forEach((listing) => {
|
||||
expect(listing.description).toBeTypeOf('string');
|
||||
expect(listing.description).not.toBe('');
|
||||
});
|
||||
});
|
||||
expect(enriched).toBeTruthy();
|
||||
expect(enriched.description).toBeTypeOf('string');
|
||||
expect(enriched.description).not.toBe('');
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,8 +13,16 @@ describe('#immoswp testsuite()', () => {
|
||||
provider.init(providerConfig.immoswp, [], []);
|
||||
it('should test immoswp provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'immoswp',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immoswp', similarityCache);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
|
||||
fredy.execute().then((listing) => {
|
||||
if (listing == null || listing.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
@@ -29,11 +37,13 @@ describe('#immoswp testsuite()', () => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://immo.swp.de');
|
||||
});
|
||||
|
||||
@@ -6,70 +6,95 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect, vi } from 'vitest';
|
||||
import { expect } from 'vitest';
|
||||
import * as provider from '../../lib/provider/immowelt.js';
|
||||
import * as mockStore from '../mocks/mockStore.js';
|
||||
import { launchBrowser, closeBrowser } from '../../lib/services/extractor/puppeteerExtractor.js';
|
||||
|
||||
// One browser shared across the whole suite so both requests (search + detail)
|
||||
// come from the same warm session. Immowelt's CDN challenges cold sessions
|
||||
// aggressively; a shared warm browser prevents the second request from being
|
||||
// blocked as a bot hit.
|
||||
const TEST_TIMEOUT = 180_000;
|
||||
|
||||
describe('#immowelt testsuite()', () => {
|
||||
it('should test immowelt provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.immowelt, [], []);
|
||||
let browser;
|
||||
let liveListings;
|
||||
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immowelt', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
beforeAll(async () => {
|
||||
browser = await launchBrowser(providerConfig.immowelt.url);
|
||||
}, TEST_TIMEOUT);
|
||||
|
||||
if (listing == null || listing.length === 0) {
|
||||
throw new Error('Listings is empty!');
|
||||
}
|
||||
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('immowelt');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).toContain('m²');
|
||||
}
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.immowelt.de');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
afterAll(async () => {
|
||||
await closeBrowser(browser);
|
||||
});
|
||||
|
||||
it(
|
||||
'should test immowelt provider',
|
||||
async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'immowelt',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
provider.init(providerConfig.immowelt, [], []);
|
||||
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, browser);
|
||||
|
||||
liveListings = await fredy.execute();
|
||||
|
||||
if (liveListings == null || liveListings.length === 0) {
|
||||
throw new Error('Listings is empty!');
|
||||
}
|
||||
|
||||
expect(liveListings).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('immowelt');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
if (notify.price != null) {
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
}
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
if (notify.size != null && notify.size.trim().toLowerCase() !== 'k.a.') {
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
}
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.immowelt.de');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
|
||||
describe('with provider_details enabled', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(mockStore, 'getUserSettings').mockReturnValue({ provider_details: [provider.metaInformation.id] });
|
||||
vi.spyOn(mockStore, 'getKnownListingHashesForJobAndProvider').mockReturnValue([]);
|
||||
});
|
||||
it(
|
||||
'should enrich listings with details',
|
||||
async () => {
|
||||
if (!liveListings?.length) throw new Error('No listings from first test to enrich');
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
// Call fetchDetails directly on the first live listing — no need to
|
||||
// re-scrape the search page. The shared browser keeps the session warm.
|
||||
const enriched = await provider.config.fetchDetails(liveListings[0], browser);
|
||||
|
||||
it('should enrich listings with details', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.immowelt, [], []);
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immowelt', {
|
||||
checkAndAddEntry: () => false,
|
||||
});
|
||||
const listings = await fredy.execute();
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
listings.forEach((listing) => {
|
||||
expect(listing.link).toContain('https://www.immowelt.de');
|
||||
expect(listing.address).toBeTypeOf('string');
|
||||
expect(listing.address).not.toBe('');
|
||||
expect(enriched).toBeTruthy();
|
||||
expect(enriched.link).toContain('https://www.immowelt.de');
|
||||
expect(enriched.address).toBeTypeOf('string');
|
||||
expect(enriched.address).not.toBe('');
|
||||
// description is enriched from the detail page; falls back gracefully if blocked
|
||||
if (listing.description != null) {
|
||||
expect(listing.description).toBeTypeOf('string');
|
||||
if (enriched.description != null) {
|
||||
expect(enriched.description).toBeTypeOf('string');
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,74 +6,88 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect, vi } from 'vitest';
|
||||
import { expect } from 'vitest';
|
||||
import * as provider from '../../lib/provider/kleinanzeigen.js';
|
||||
import * as mockStore from '../mocks/mockStore.js';
|
||||
import { launchBrowser, closeBrowser } from '../../lib/services/extractor/puppeteerExtractor.js';
|
||||
|
||||
// One browser shared across the whole suite so both requests (search + detail)
|
||||
// come from the same warm session. Kleinanzeigen rate-limits cold browser
|
||||
// sessions; a shared warm browser prevents the second request from being blocked.
|
||||
const TEST_TIMEOUT = 180_000;
|
||||
|
||||
describe('#kleinanzeigen testsuite()', () => {
|
||||
it('should test kleinanzeigen provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.kleinanzeigen, [], []);
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(
|
||||
provider.config,
|
||||
null,
|
||||
null,
|
||||
provider.metaInformation.id,
|
||||
'kleinanzeigen',
|
||||
similarityCache,
|
||||
);
|
||||
fredy.execute().then((listing) => {
|
||||
if (listing == null || listing.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
return;
|
||||
}
|
||||
let browser;
|
||||
let liveListings;
|
||||
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('kleinanzeigen');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.kleinanzeigen.de');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
beforeAll(async () => {
|
||||
browser = await launchBrowser(providerConfig.kleinanzeigen.url);
|
||||
}, TEST_TIMEOUT);
|
||||
|
||||
afterAll(async () => {
|
||||
await closeBrowser(browser);
|
||||
});
|
||||
|
||||
it(
|
||||
'should test kleinanzeigen provider',
|
||||
async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'kleinanzeigen',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
provider.init(providerConfig.kleinanzeigen, [], []);
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, browser);
|
||||
|
||||
fredy.execute().then((listing) => {
|
||||
if (listing == null || listing.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
return;
|
||||
}
|
||||
|
||||
liveListings = listing;
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('kleinanzeigen');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.kleinanzeigen.de');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
|
||||
describe('with provider_details enabled', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(mockStore, 'getUserSettings').mockReturnValue({ provider_details: [provider.metaInformation.id] });
|
||||
vi.spyOn(mockStore, 'getKnownListingHashesForJobAndProvider').mockReturnValue([]);
|
||||
});
|
||||
it(
|
||||
'should enrich listings with details',
|
||||
async () => {
|
||||
if (!liveListings?.length) throw new Error('No listings from first test to enrich');
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
// Call fetchDetails directly on the first live listing — no need to
|
||||
// re-scrape the search page. The shared browser keeps the session warm.
|
||||
const enriched = await provider.config.fetchDetails(liveListings[0], browser);
|
||||
|
||||
it('should enrich listings with details', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.kleinanzeigen, [], []);
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'kleinanzeigen', {
|
||||
checkAndAddEntry: () => false,
|
||||
});
|
||||
const listings = await fredy.execute();
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
listings.forEach((listing) => {
|
||||
expect(listing.link).toContain('https://www.kleinanzeigen.de');
|
||||
expect(listing.address).toBeTypeOf('string');
|
||||
expect(listing.address).not.toBe('');
|
||||
expect(listing.description).toBeTypeOf('string');
|
||||
expect(listing.description).not.toBe('');
|
||||
});
|
||||
});
|
||||
expect(enriched).toBeTruthy();
|
||||
expect(enriched.link).toContain('https://www.kleinanzeigen.de');
|
||||
expect(enriched.address).toBeTypeOf('string');
|
||||
expect(enriched.address).not.toBe('');
|
||||
expect(enriched.description).toBeTypeOf('string');
|
||||
expect(enriched.description).not.toBe('');
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/mcMakler.js';
|
||||
describe('#mcMakler testsuite()', () => {
|
||||
it('should test mcMakler provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'mcMakler',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
provider.init(providerConfig.mcMakler, []);
|
||||
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'mcMakler', similarityCache);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
|
||||
const listing = await fredy.execute();
|
||||
|
||||
if (listing == null || listing.length === 0) {
|
||||
@@ -29,12 +36,14 @@ describe('#mcMakler testsuite()', () => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
|
||||
@@ -13,15 +13,16 @@ describe('#neubauKompass testsuite()', () => {
|
||||
provider.init(providerConfig.neubauKompass, [], []);
|
||||
it('should test neubauKompass provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'neubauKompass',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(
|
||||
provider.config,
|
||||
null,
|
||||
null,
|
||||
provider.metaInformation.id,
|
||||
'neubauKompass',
|
||||
similarityCache,
|
||||
);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
|
||||
fredy.execute().then((listing) => {
|
||||
if (listing == null || listing.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
|
||||
@@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/ohneMakler.js';
|
||||
describe('#ohneMakler testsuite()', () => {
|
||||
it('should test ohneMakler provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'ohneMakler',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
provider.init(providerConfig.ohneMakler, []);
|
||||
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'ohneMakler', similarityCache);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
|
||||
const listing = await fredy.execute();
|
||||
|
||||
if (listing == null || listing.length === 0) {
|
||||
@@ -29,12 +36,14 @@ describe('#ohneMakler testsuite()', () => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
|
||||
@@ -12,16 +12,16 @@ import * as provider from '../../lib/provider/regionalimmobilien24.js';
|
||||
describe('#regionalimmobilien24 testsuite()', () => {
|
||||
it('should test regionalimmobilien24 provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'regionalimmobilien24',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
provider.init(providerConfig.regionalimmobilien24, []);
|
||||
|
||||
const fredy = new Fredy(
|
||||
provider.config,
|
||||
null,
|
||||
null,
|
||||
provider.metaInformation.id,
|
||||
'regionalimmobilien24',
|
||||
similarityCache,
|
||||
);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
|
||||
const listing = await fredy.execute();
|
||||
|
||||
if (listing == null || listing.length === 0) {
|
||||
@@ -36,12 +36,14 @@ describe('#regionalimmobilien24 testsuite()', () => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
|
||||
@@ -9,65 +9,97 @@ import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect, vi } from 'vitest';
|
||||
import * as provider from '../../lib/provider/sparkasse.js';
|
||||
import * as mockStore from '../mocks/mockStore.js';
|
||||
import { launchBrowser, closeBrowser } from '../../lib/services/extractor/puppeteerExtractor.js';
|
||||
|
||||
// One browser shared across the whole suite so both requests (search + detail)
|
||||
// come from the same warm session. This prevents the second request from being
|
||||
// flagged as a cold-start bot hit.
|
||||
const TEST_TIMEOUT = 120_000;
|
||||
|
||||
describe('#sparkasse testsuite()', () => {
|
||||
it('should test sparkasse provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.sparkasse, []);
|
||||
let browser;
|
||||
let liveListings;
|
||||
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'sparkasse', similarityCache);
|
||||
const listing = await fredy.execute();
|
||||
beforeAll(async () => {
|
||||
browser = await launchBrowser(providerConfig.sparkasse.url);
|
||||
}, TEST_TIMEOUT);
|
||||
|
||||
if (listing == null || listing.length === 0) {
|
||||
throw new Error('Listings is empty!');
|
||||
}
|
||||
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('sparkasse');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
afterAll(async () => {
|
||||
await closeBrowser(browser);
|
||||
});
|
||||
|
||||
it(
|
||||
'should test sparkasse provider',
|
||||
async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'sparkasse',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
provider.init(providerConfig.sparkasse, []);
|
||||
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, browser);
|
||||
|
||||
liveListings = await fredy.execute();
|
||||
|
||||
if (liveListings == null || liveListings.length === 0) {
|
||||
throw new Error('Listings is empty!');
|
||||
}
|
||||
|
||||
expect(liveListings).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj).toBeTypeOf('object');
|
||||
expect(notificationObj.serviceName).toBe('sparkasse');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.address).not.toBe('');
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
|
||||
describe('with provider_details enabled', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(mockStore, 'getUserSettings').mockReturnValue({ provider_details: [provider.metaInformation.id] });
|
||||
vi.spyOn(mockStore, 'getKnownListingHashesForJobAndProvider').mockReturnValue([]);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('should enrich listings with details', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.sparkasse, []);
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'sparkasse', {
|
||||
checkAndAddEntry: () => false,
|
||||
});
|
||||
const listings = await fredy.execute();
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
listings.forEach((listing) => {
|
||||
expect(listing.link).toContain('https://immobilien.sparkasse.de');
|
||||
expect(listing.address).toBeTypeOf('string');
|
||||
expect(listing.address).not.toBe('');
|
||||
// description is enriched from the detail page; falls back gracefully if bot-detected
|
||||
if (listing.description != null) {
|
||||
expect(listing.description).toBeTypeOf('string');
|
||||
expect(listing.description).not.toBe('');
|
||||
it(
|
||||
'should enrich listings with details',
|
||||
async () => {
|
||||
if (!liveListings?.length) throw new Error('No listings from first test to enrich');
|
||||
|
||||
// Call fetchDetails directly on the first live listing — no need to
|
||||
// re-scrape the search page. The shared browser keeps the session warm.
|
||||
const enriched = await provider.config.fetchDetails(liveListings[0], browser);
|
||||
|
||||
expect(enriched).toBeTruthy();
|
||||
expect(enriched.link).toContain('https://immobilien.sparkasse.de');
|
||||
expect(enriched.address).toBeTypeOf('string');
|
||||
expect(enriched.address).not.toBe('');
|
||||
// description is enriched from the detail page; falls back gracefully if blocked
|
||||
if (enriched.description != null) {
|
||||
expect(enriched.description).toBeTypeOf('string');
|
||||
expect(enriched.description).not.toBe('');
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,62 +6,85 @@
|
||||
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
|
||||
import { get } from '../mocks/mockNotification.js';
|
||||
import { mockFredy, providerConfig } from '../utils.js';
|
||||
import { expect, vi } from 'vitest';
|
||||
import { expect } from 'vitest';
|
||||
import * as provider from '../../lib/provider/wgGesucht.js';
|
||||
import * as mockStore from '../mocks/mockStore.js';
|
||||
import { launchBrowser, closeBrowser } from '../../lib/services/extractor/puppeteerExtractor.js';
|
||||
|
||||
// One browser shared across the whole suite so both requests (search + detail)
|
||||
// come from the same warm session, avoiding double cold-start bot detection.
|
||||
const TEST_TIMEOUT = 120_000;
|
||||
|
||||
describe('#wgGesucht testsuite()', () => {
|
||||
provider.init(providerConfig.wgGesucht, [], []);
|
||||
it('should test wgGesucht provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'wgGesucht', similarityCache);
|
||||
fredy.execute().then((listing) => {
|
||||
if (listing == null || listing.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
return;
|
||||
}
|
||||
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj.serviceName).toBe('wgGesucht');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
expect(notify).toBeTypeOf('object');
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.details).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
let browser;
|
||||
let liveListings;
|
||||
|
||||
beforeAll(async () => {
|
||||
browser = await launchBrowser(providerConfig.wgGesucht.url);
|
||||
}, TEST_TIMEOUT);
|
||||
|
||||
afterAll(async () => {
|
||||
await closeBrowser(browser);
|
||||
});
|
||||
|
||||
it(
|
||||
'should test wgGesucht provider',
|
||||
async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'wgGesucht',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, browser);
|
||||
|
||||
fredy.execute().then((listing) => {
|
||||
if (listing == null || listing.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
return;
|
||||
}
|
||||
|
||||
liveListings = listing;
|
||||
expect(listing).toBeInstanceOf(Array);
|
||||
const notificationObj = get();
|
||||
expect(notificationObj.serviceName).toBe('wgGesucht');
|
||||
notificationObj.payload.forEach((notify) => {
|
||||
expect(notify).toBeTypeOf('object');
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
// expect(notify.details).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
});
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
|
||||
describe('with provider_details enabled', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(mockStore, 'getUserSettings').mockReturnValue({ provider_details: [provider.metaInformation.id] });
|
||||
vi.spyOn(mockStore, 'getKnownListingHashesForJobAndProvider').mockReturnValue([]);
|
||||
});
|
||||
it(
|
||||
'should enrich listings with details',
|
||||
async () => {
|
||||
if (!liveListings?.length) throw new Error('No listings from first test to enrich');
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
// Call fetchDetails directly on the first live listing — no need to
|
||||
// re-scrape the search page. The shared browser keeps the session warm.
|
||||
const enriched = await provider.config.fetchDetails(liveListings[0], browser);
|
||||
|
||||
it('should enrich listings with details', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
provider.init(providerConfig.wgGesucht, [], []);
|
||||
const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'wgGesucht', {
|
||||
checkAndAddEntry: () => false,
|
||||
});
|
||||
const listings = await fredy.execute();
|
||||
expect(listings).toBeInstanceOf(Array);
|
||||
listings.forEach((listing) => {
|
||||
expect(listing.link).toContain('https://www.wg-gesucht.de');
|
||||
expect(listing.description).toBeTypeOf('string');
|
||||
expect(listing.description).not.toBe('');
|
||||
});
|
||||
});
|
||||
expect(enriched).toBeTruthy();
|
||||
expect(enriched.link).toContain('https://www.wg-gesucht.de');
|
||||
expect(enriched.description).toBeTypeOf('string');
|
||||
expect(enriched.description).not.toBe('');
|
||||
},
|
||||
TEST_TIMEOUT,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,15 +13,16 @@ describe('#wohnungsboerse testsuite()', () => {
|
||||
provider.init(providerConfig.wohnungsboerse, [], []);
|
||||
it('should test wohnungsboerse provider', async () => {
|
||||
const Fredy = await mockFredy();
|
||||
const mockedJob = {
|
||||
id: 'wohnungsboerse',
|
||||
notificationAdapter: null,
|
||||
spatialFilter: null,
|
||||
specFilter: null,
|
||||
};
|
||||
|
||||
return await new Promise((resolve, reject) => {
|
||||
const fredy = new Fredy(
|
||||
provider.config,
|
||||
null,
|
||||
null,
|
||||
provider.metaInformation.id,
|
||||
'wohnungsboerse',
|
||||
similarityCache,
|
||||
);
|
||||
const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
|
||||
|
||||
fredy.execute().then((listings) => {
|
||||
if (listings == null || listings.length === 0) {
|
||||
reject('Listings is empty!');
|
||||
@@ -36,12 +37,14 @@ describe('#wohnungsboerse testsuite()', () => {
|
||||
/** check the actual structure **/
|
||||
expect(notify.id).toBeTypeOf('string');
|
||||
expect(notify.price).toBeTypeOf('string');
|
||||
expect(notify.price).toContain('€');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.size).toContain('m²');
|
||||
expect(notify.title).toBeTypeOf('string');
|
||||
expect(notify.link).toBeTypeOf('string');
|
||||
expect(notify.address).toBeTypeOf('string');
|
||||
/** check the values if possible **/
|
||||
expect(notify.size).not.toBe('');
|
||||
expect(notify.size).toBeTypeOf('string');
|
||||
expect(notify.title).not.toBe('');
|
||||
expect(notify.link).toContain('https://www.wohnungsboerse.net');
|
||||
});
|
||||
|
||||
@@ -10,6 +10,17 @@ import { readFile } from 'fs/promises';
|
||||
export const testData = JSON.parse(await readFile(new URL('./testdata.json', import.meta.url)));
|
||||
|
||||
describe('#immoscout-mobile URL conversion', () => {
|
||||
// Test shape URL conversion
|
||||
it('should convert a full web URL with shape to mobile URL', () => {
|
||||
const webUrl =
|
||||
'https://www.immobilienscout24.de/Suche/shape/haus-kaufen?shape=aW9yfkhfa3htQXJgUGlnYEBmekhte3BAcXNAfWBsQGNyQ2lkUHVvbEB3eX5Ab25WYn5Fa2BLaGRQY29FaGtTfEhme3xBdHBEdHFMamlHbmdRfHhMcmxPeHlWYnpS&price=-600000.0&ground=240.0-&enteredFrom=result_list';
|
||||
const expectedMobileUrl =
|
||||
'https://api.mobile.immobilienscout24.de/search/list?ground=240.0-&price=-600000.0&realestatetype=housebuy&searchType=shape&shape=ior~H_kxmAr%60Pig%60%40fzHm%7Bp%40qs%40%7D%60l%40crCidPuol%40wy~%40onVb~Ek%60KhdPcoEhkS%7CHf%7B%7CAtpDtqLjiGngQ%7CxLrlOxyVbzR';
|
||||
|
||||
const actualMobileUrl = convertWebToMobile(webUrl);
|
||||
expect(actualMobileUrl).toBe(expectedMobileUrl);
|
||||
});
|
||||
|
||||
// Test URL conversion
|
||||
it('should convert a full web URL to mobile URL', () => {
|
||||
const webUrl =
|
||||
|
||||
@@ -18,5 +18,9 @@
|
||||
"rentHouse": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/de/nordrhein-westfalen/duesseldorf/haus-mieten?enteredFrom=one_step_search",
|
||||
"type": "houserent"
|
||||
},
|
||||
"buyHouseWithShape": {
|
||||
"url": "https://www.immobilienscout24.de/Suche/shape/haus-kaufen?shape=aW9yfkhfa3htQXJgUGlnYEBmekhte3BAcXNAfWBsQGNyQ2lkUHVvbEB3eX5Ab25WYn5Fa2BLaGRQY29FaGtTfEhme3xBdHBEdHFMamlHbmdRfHhMcmxPeHlWYnpS&price=-600000.0&ground=240.0-&enteredFrom=result_list",
|
||||
"type": "housebuy"
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user